hydrus/hydrus/client/caches/ClientCaches.py

import collections
import json
import os
import threading
import time

from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusFileHandling
from hydrus.core import HydrusThreading
from hydrus.core import HydrusData
from hydrus.core import HydrusGlobals as HG
from hydrus.core import HydrusTime
from hydrus.core.images import HydrusBlurhash
from hydrus.core.images import HydrusImageHandling

from hydrus.client import ClientConstants as CC
from hydrus.client import ClientFiles
from hydrus.client import ClientImageHandling
from hydrus.client import ClientParsing
from hydrus.client import ClientRendering
from hydrus.client import ClientThreading
from hydrus.client.caches import ClientCachesBase

class LocalBooruCache( object ):
    
    def __init__( self, controller ):
        
        self._controller = controller
        
        self._lock = threading.Lock()
        
        self._RefreshShares()
        
        self._controller.sub( self, 'RefreshShares', 'refresh_local_booru_shares' )
        self._controller.sub( self, 'RefreshShares', 'restart_client_server_service' )
        
    
    def _CheckDataUsage( self ):
        
        if not self._local_booru_service.BandwidthOK():
            
            raise HydrusExceptions.InsufficientCredentialsException( 'This booru has used all its monthly data. Please try again next month.' )
            
        
    
    def _CheckFileAuthorised( self, share_key, hash ):
        
        self._CheckShareAuthorised( share_key )
        
        info = self._GetInfo( share_key )
        
        if hash not in info[ 'hashes_set' ]:
            
            raise HydrusExceptions.NotFoundException( 'That file was not found in that share.' )
            
        
    
    def _CheckShareAuthorised( self, share_key ):
        
        self._CheckDataUsage()
        
        info = self._GetInfo( share_key )
        
        timeout = info[ 'timeout' ]
        
        if timeout is not None and HydrusTime.TimeHasPassed( timeout ):
            
            raise HydrusExceptions.NotFoundException( 'This share has expired.' )
            
        
    
    def _GetInfo( self, share_key ):
        
        try: info = self._keys_to_infos[ share_key ]
        except: raise HydrusExceptions.NotFoundException( 'Did not find that share on this booru.' )
        
        if info is None:
            
            info = self._controller.Read( 'local_booru_share', share_key )
            
            hashes = info[ 'hashes' ]
            
            info[ 'hashes_set' ] = set( hashes )
            
            media_results = self._controller.Read( 'media_results', hashes )
            
            info[ 'media_results' ] = media_results
            
            hashes_to_media_results = { media_result.GetHash() : media_result for media_result in media_results }
            
            info[ 'hashes_to_media_results' ] = hashes_to_media_results
            
            self._keys_to_infos[ share_key ] = info
            
        
        return info
        
    
    def _RefreshShares( self ):
        
        self._local_booru_service = self._controller.services_manager.GetService( CC.LOCAL_BOORU_SERVICE_KEY )
        
        self._keys_to_infos = {}
        
        share_keys = self._controller.Read( 'local_booru_share_keys' )
        
        for share_key in share_keys:
            
            self._keys_to_infos[ share_key ] = None
            
        
    
    def CheckShareAuthorised( self, share_key ):
        
        with self._lock: self._CheckShareAuthorised( share_key )
        
    
    def CheckFileAuthorised( self, share_key, hash ):
        
        with self._lock: self._CheckFileAuthorised( share_key, hash )
        
    
    def GetGalleryInfo( self, share_key ):
        
        with self._lock:
            
            self._CheckShareAuthorised( share_key )
            
            info = self._GetInfo( share_key )
            
            name = info[ 'name' ]
            text = info[ 'text' ]
            timeout = info[ 'timeout' ]
            media_results = info[ 'media_results' ]
            
            return ( name, text, timeout, media_results )
            
        
    
    def GetMediaResult( self, share_key, hash ):
        
        with self._lock:
            
            info = self._GetInfo( share_key )
            
            media_result = info[ 'hashes_to_media_results' ][ hash ]
            
            return media_result
            
        
    
    def GetPageInfo( self, share_key, hash ):
        
        with self._lock:
            
            self._CheckFileAuthorised( share_key, hash )
            
            info = self._GetInfo( share_key )
            
            name = info[ 'name' ]
            text = info[ 'text' ]
            timeout = info[ 'timeout' ]
            media_result = info[ 'hashes_to_media_results' ][ hash ]
            
            return ( name, text, timeout, media_result )
            
        
    
    def RefreshShares( self, *args, **kwargs ):
        
        with self._lock:
            
            self._RefreshShares()
            
        
    
class ParsingCache( object ):
    
    def __init__( self ):
        
        self._next_clean_cache_time = HydrusTime.GetNow()
        
        self._html_to_soups = {}
        self._json_to_jsons = {}
        
        self._lock = threading.Lock()
        
    
    def _CleanCache( self ):
        
        if HydrusTime.TimeHasPassed( self._next_clean_cache_time ):
            
            for cache in ( self._html_to_soups, self._json_to_jsons ):
                
                dead_datas = set()
                
                for ( data, ( last_accessed, parsed_object ) ) in cache.items():
                    
                    if HydrusTime.TimeHasPassed( last_accessed + 10 ):
                        
                        dead_datas.add( data )
                        
                    
                
                for dead_data in dead_datas:
                    
                    del cache[ dead_data ]
                    
                
            
            self._next_clean_cache_time = HydrusTime.GetNow() + 5
            
        
    
    def CleanCache( self ):
        
        with self._lock:
            
            self._CleanCache()
            
        
    
    def GetJSON( self, json_text ):
        
        with self._lock:
            
            now = HydrusTime.GetNow()
            
            if json_text not in self._json_to_jsons:
                
                json_object = json.loads( json_text )
                
                self._json_to_jsons[ json_text ] = ( now, json_object )
                
            
            ( last_accessed, json_object ) = self._json_to_jsons[ json_text ]
            
            if last_accessed != now:
                
                self._json_to_jsons[ json_text ] = ( now, json_object )
                
            
            if len( self._json_to_jsons ) > 10:
                
                self._CleanCache()
                
            
            return json_object
            
        
    
    def GetSoup( self, html ):
        
        with self._lock:
            
            now = HydrusTime.GetNow()
            
            if html not in self._html_to_soups:
                
                soup = ClientParsing.GetSoup( html )
                
                self._html_to_soups[ html ] = ( now, soup )
                
            
            ( last_accessed, soup ) = self._html_to_soups[ html ]
            
            if last_accessed != now:
                
                self._html_to_soups[ html ] = ( now, soup )
                
            
            if len( self._html_to_soups ) > 10:
                
                self._CleanCache()
                
            
            return soup
            
        
    
class ImageRendererCache( object ):
    
    def __init__( self, controller ):
        
        self._controller = controller
        
        cache_size = self._controller.new_options.GetInteger( 'image_cache_size' )
        cache_timeout = self._controller.new_options.GetInteger( 'image_cache_timeout' )
        
        self._data_cache = ClientCachesBase.DataCache( self._controller, 'image cache', cache_size, timeout = cache_timeout )
        
        self._controller.sub( self, 'NotifyNewOptions', 'notify_new_options' )
        
    
    def Clear( self ):
        
        self._data_cache.Clear()
        
    
    def GetImageRenderer( self, media ):
        
        hash = media.GetHash()
        
        key = hash
        
        result = self._data_cache.GetIfHasData( key )
        
        if result is None:
            
            image_renderer = ClientRendering.ImageRenderer( media )
            
            # we are no longer going to let big lads flush the whole cache. they can render on demand
            
            image_cache_storage_limit_percentage = self._controller.new_options.GetInteger( 'image_cache_storage_limit_percentage' )
            
            if image_renderer.GetEstimatedMemoryFootprint() < self._data_cache.GetSizeLimit() * ( image_cache_storage_limit_percentage / 100 ):
                
                self._data_cache.AddData( key, image_renderer )
                
            
        else:
            
            image_renderer = result
            
        
        return image_renderer
        
    
    def HasImageRenderer( self, hash ):
        
        key = hash
        
        return self._data_cache.HasData( key )
        
    
    def NotifyNewOptions( self ):
        
        cache_size = self._controller.new_options.GetInteger( 'image_cache_size' )
        cache_timeout = self._controller.new_options.GetInteger( 'image_cache_timeout' )
        
        self._data_cache.SetCacheSizeAndTimeout( cache_size, cache_timeout )
        
    
    def PrefetchImageRenderer( self, media ):
        
        ( width, height ) = media.GetResolution()
        
        # essentially, we are not going to prefetch giganto images any more. they can render on demand and not mess our queue
        
        image_cache_prefetch_limit_percentage = self._controller.new_options.GetInteger( 'image_cache_prefetch_limit_percentage' )
        
        if width * height * 3 < self._data_cache.GetSizeLimit() * ( image_cache_prefetch_limit_percentage / 100 ):
            
            self.GetImageRenderer( media )
            
        
    
class ImageTileCache( object ):
    
    def __init__( self, controller ):
        
        self._controller = controller
        
        cache_size = self._controller.new_options.GetInteger( 'image_tile_cache_size' )
        cache_timeout = self._controller.new_options.GetInteger( 'image_tile_cache_timeout' )
        
        self._data_cache = ClientCachesBase.DataCache( self._controller, 'image tile cache', cache_size, timeout = cache_timeout )
        
        self._controller.sub( self, 'NotifyNewOptions', 'notify_new_options' )
        self._controller.sub( self, 'Clear', 'clear_image_tile_cache' )
        
    
    def Clear( self ):
        
        self._data_cache.Clear()
        
    
    def GetTile( self, image_renderer: ClientRendering.ImageRenderer, media, clip_rect, target_resolution ):
        
        hash = media.GetHash()
        
        key = (
            hash,
            clip_rect.left(),
            clip_rect.top(),
            clip_rect.right(),
            clip_rect.bottom(),
            target_resolution.width(),
            target_resolution.height()
        )
        
        result = self._data_cache.GetIfHasData( key )
        
        if result is None:
            
            qt_pixmap = image_renderer.GetQtPixmap( clip_rect = clip_rect, target_resolution = target_resolution )
            
            tile = ClientRendering.ImageTile( hash, clip_rect, qt_pixmap )
            
            self._data_cache.AddData( key, tile )
            
        else:
            
            tile = result
            
        
        return tile
        
    
    def NotifyNewOptions( self ):
        
        cache_size = self._controller.new_options.GetInteger( 'image_tile_cache_size' )
        cache_timeout = self._controller.new_options.GetInteger( 'image_tile_cache_timeout' )
        
        self._data_cache.SetCacheSizeAndTimeout( cache_size, cache_timeout )
        
    
class ThumbnailCache( object ):
    
    def __init__( self, controller ):
        
        self._controller = controller
        
        cache_size = self._controller.new_options.GetInteger( 'thumbnail_cache_size' )
        cache_timeout = self._controller.new_options.GetInteger( 'thumbnail_cache_timeout' )
        
        self._data_cache = ClientCachesBase.DataCache( self._controller, 'thumbnail cache', cache_size, timeout = cache_timeout )
        
        self._magic_mime_thumbnail_ease_score_lookup = {}
        
        self._InitialiseMagicMimeScores()
        
        self._lock = threading.Lock()
        
        self._thumbnail_error_occurred = False
        
        self._waterfall_queue_quick = set()
        self._waterfall_queue = []
        
        self._waterfall_queue_empty_event = threading.Event()
        
        self._delayed_regeneration_queue_quick = set()
        self._delayed_regeneration_queue = []
        
        self._allow_blurhash_fallback = self._controller.new_options.GetBoolean( 'allow_blurhash_fallback' )
        
        self._waterfall_event = threading.Event()
        
        self._special_thumbs = {}
        
        self.Clear()
        
        self._controller.CallToThreadLongRunning( self.MainLoop )
        
        self._controller.sub( self, 'Clear', 'reset_thumbnail_cache' )
        self._controller.sub( self, 'ClearThumbnails', 'clear_thumbnails' )
        self._controller.sub( self, 'NotifyNewOptions', 'notify_new_options' )
        
    
    def _GetBestRecoveryThumbnailHydrusBitmap( self, display_media ):
        
        if self._allow_blurhash_fallback:
            
            blurhash = display_media.GetFileInfoManager().blurhash
            
            if blurhash is not None:
                
                try:
                    
                    ( media_width, media_height ) = display_media.GetResolution()
                    
                    bounding_dimensions = self._controller.options[ 'thumbnail_dimensions' ]
                    thumbnail_scale_type = self._controller.new_options.GetInteger( 'thumbnail_scale_type' )
                    thumbnail_dpr_percent = HG.client_controller.new_options.GetInteger( 'thumbnail_dpr_percent' )
                    
                    ( expected_width, expected_height ) = HydrusImageHandling.GetThumbnailResolution( ( media_width, media_height ), bounding_dimensions, thumbnail_scale_type, thumbnail_dpr_percent )
                    
                    numpy_image = HydrusBlurhash.GetNumpyFromBlurhash( blurhash, expected_width, expected_height )
                    
                    hydrus_bitmap = ClientRendering.GenerateHydrusBitmapFromNumPyImage( numpy_image )
                    
                    return hydrus_bitmap
                    
                except:
                    
                    pass
                    
                
            
        
        return self._special_thumbs[ 'hydrus' ]
        
    
    def _GetThumbnailHydrusBitmap( self, display_media ):
        
        if HG.blurhash_mode:
            
            return self._GetBestRecoveryThumbnailHydrusBitmap( display_media )
            
        
        hash = display_media.GetHash()
        
        locations_manager = display_media.GetLocationsManager()
        
        try:
            
            thumbnail_path = self._controller.client_files_manager.GetThumbnailPath( display_media )
            
        except HydrusExceptions.FileMissingException as e:
            
            if locations_manager.IsLocal():
                
                summary = 'Unable to get thumbnail for file {}.'.format( hash.hex() )
                
                self._HandleThumbnailException( hash, e, summary )
                
            
            return self._GetBestRecoveryThumbnailHydrusBitmap( display_media )
            
        
        thumbnail_mime = HC.IMAGE_JPEG
        
        try:
            
            thumbnail_mime = HydrusFileHandling.GetThumbnailMime( thumbnail_path )
            
            numpy_image = ClientImageHandling.GenerateNumPyImage( thumbnail_path, thumbnail_mime )
            
        except Exception as e:
            
            try:
                
                # file is malformed, let's force a regen
                self._controller.files_maintenance_manager.RunJobImmediately( [ display_media ], ClientFiles.REGENERATE_FILE_DATA_JOB_FORCE_THUMBNAIL, pub_job_key = False )
                
            except Exception as e:
                
                summary = 'The thumbnail for file {} was not loadable. An attempt to regenerate it failed.'.format( hash.hex() )
                
                self._HandleThumbnailException( hash, e, summary )
                
                return self._GetBestRecoveryThumbnailHydrusBitmap( display_media )
                
            
            try:
                
                numpy_image = ClientImageHandling.GenerateNumPyImage( thumbnail_path, thumbnail_mime )
                
            except Exception as e:
                
                summary = 'The thumbnail for file {} was not loadable. It was regenerated, but that file would not render either. Your image libraries or hard drive connection are unreliable. Please inform the hydrus developer what has happened.'.format( hash.hex() )
                
                self._HandleThumbnailException( hash, e, summary )
                
                return self._GetBestRecoveryThumbnailHydrusBitmap( display_media )
                
            
        
        ( current_width, current_height ) = HydrusImageHandling.GetResolutionNumPy( numpy_image )
        
        ( media_width, media_height ) = display_media.GetResolution()
        
        bounding_dimensions = self._controller.options[ 'thumbnail_dimensions' ]
        thumbnail_scale_type = self._controller.new_options.GetInteger( 'thumbnail_scale_type' )
        thumbnail_dpr_percent = HG.client_controller.new_options.GetInteger( 'thumbnail_dpr_percent' )
        
        ( expected_width, expected_height ) = HydrusImageHandling.GetThumbnailResolution( ( media_width, media_height ), bounding_dimensions, thumbnail_scale_type, thumbnail_dpr_percent )
        
        exactly_as_expected = current_width == expected_width and current_height == expected_height
        
        rotation_exception = current_width == expected_height and current_height == expected_width
        
        correct_size = exactly_as_expected or rotation_exception
        
        if not correct_size:
            
            numpy_image = HydrusImageHandling.ResizeNumPyImage( numpy_image, ( expected_width, expected_height ) )
            
            if locations_manager.IsLocal():
                
                # we have the master file, so we should regen the thumb from source
                
                if HG.file_report_mode:
                    
                    HydrusData.ShowText( 'Thumbnail {} wrong size ({}x{} instead of {}x{}), scheduling regeneration from source.'.format( hash.hex(), current_width, current_height, expected_width, expected_height ) )
                    
                
                delayed_item = display_media.GetMediaResult()
                
                with self._lock:
                    
                    if delayed_item not in self._delayed_regeneration_queue_quick:
                        
                        self._delayed_regeneration_queue_quick.add( delayed_item )
                        
                        self._delayed_regeneration_queue.append( delayed_item )
                        
                    
                
            else:
                
                # we do not have the master file, so we have to scale up from what we have
                
                if HG.file_report_mode:
                    
                    HydrusData.ShowText( 'Thumbnail {} wrong size ({}x{} instead of {}x{}), only scaling due to no local source.'.format( hash.hex(), current_width, current_height, expected_width, expected_height ) )
                    
                
            
        
        hydrus_bitmap = ClientRendering.GenerateHydrusBitmapFromNumPyImage( numpy_image )
        
        return hydrus_bitmap
        
    
    def _HandleThumbnailException( self, hash, e, summary ):
        
        if self._thumbnail_error_occurred:
            
            HydrusData.Print( summary )
            
        else:
            
            self._thumbnail_error_occurred = True
            
            message = 'A thumbnail error has occurred. The problem thumbnail will appear with the default \'hydrus\' symbol. You may need to take hard drive recovery actions, and if the error is not obviously fixable, you can contact hydrus dev for additional help. Specific information for this first error follows. Subsequent thumbnail errors in this session will be silently printed to the log.'
            message += os.linesep * 2
            message += str( e )
            message += os.linesep * 2
            message += summary
            
            job_key = ClientThreading.JobKey()
            
            job_key.SetStatusText( message )
            job_key.SetFiles( { hash }, 'broken thumbnail' )
            
            HG.client_controller.pub( 'message', job_key )
            
        
    
    def _InitialiseMagicMimeScores( self ):
        
        # let's render our thumbs in order of ease of regeneration, so we rush what we can to screen as fast as possible and leave big vids until the end
        
        for mime in HC.ALLOWED_MIMES:
            
            self._magic_mime_thumbnail_ease_score_lookup[ mime ] = 5
            
        
        # default filetype thumbs are easiest
        
        self._magic_mime_thumbnail_ease_score_lookup[ None ] = 0
        self._magic_mime_thumbnail_ease_score_lookup[ HC.APPLICATION_UNKNOWN ] = 0
        
        for mime in HC.APPLICATIONS:
            
            self._magic_mime_thumbnail_ease_score_lookup[ mime ] = 0
            
        
        for mime in HC.AUDIO:
            
            self._magic_mime_thumbnail_ease_score_lookup[ mime ] = 0
            
        
        # images a little trickier
        
        for mime in HC.IMAGES:
            
            self._magic_mime_thumbnail_ease_score_lookup[ mime ] = 1
            
        
        for mime in HC.ANIMATIONS:
            
            self._magic_mime_thumbnail_ease_score_lookup[ mime ] = 2
            

        # could get more specific here because some applications will probably be even worse than videos
        for mime in HC.APPLICATIONS_WITH_THUMBNAILS:
            
            self._magic_mime_thumbnail_ease_score_lookup[ mime ] = 3
            
        
        # ffmpeg hellzone
        
        for mime in HC.VIDEO:
            
            self._magic_mime_thumbnail_ease_score_lookup[ mime ] = 3
            
        
    
    def _RecalcQueues( self ):
        
        # here we sort by the hash since this is both breddy random and more likely to access faster on a well defragged hard drive!
        # and now with the magic mime order
        
        def sort_waterfall( item ):
            
            ( page_key, media ) = item
            
            display_media = media.GetDisplayMedia()
            
            if display_media is None:
                
                magic_score = self._magic_mime_thumbnail_ease_score_lookup[ None ]
                hash = ''
                
            else:
                
                magic_score = self._magic_mime_thumbnail_ease_score_lookup[ display_media.GetMime() ]
                hash = display_media.GetHash()
                
            
            return ( magic_score, hash )
            
        
        self._waterfall_queue = list( self._waterfall_queue_quick )
        
        # we pop off the end, so reverse
        self._waterfall_queue.sort( key = sort_waterfall, reverse = True )
        
        if len( self._waterfall_queue ) == 0:
            
            self._waterfall_queue_empty_event.set()
            
        else:
            
            self._waterfall_queue_empty_event.clear()
            
        
        def sort_regen( item ):
            
            media_result = item
            
            hash = media_result.GetHash()
            mime = media_result.GetMime()
            
            magic_score = self._magic_mime_thumbnail_ease_score_lookup[ mime ]
            
            return ( magic_score, hash )
            
        
        self._delayed_regeneration_queue = list( self._delayed_regeneration_queue_quick )
        
        # we pop off the end, so reverse
        self._delayed_regeneration_queue.sort( key = sort_regen, reverse = True )
        
    
    def _ShouldBeAbleToProvideThumb( self, media ):
        
        locations_manager = media.GetLocationsManager()
        
        we_have_file = locations_manager.IsLocal()
        we_should_have_thumb = not locations_manager.GetCurrent().isdisjoint( HG.client_controller.services_manager.GetServiceKeys( ( HC.FILE_REPOSITORY, ) ) )
        we_have_blurhash = media.GetFileInfoManager().blurhash is not None
        
        return we_have_file or we_should_have_thumb or we_have_blurhash
        
    
    def CancelWaterfall( self, page_key: bytes, medias: list ):
        
        with self._lock:
            
            self._waterfall_queue_quick.difference_update( ( ( page_key, media ) for media in medias ) )
            
            cancelled_display_medias = { media.GetDisplayMedia() for media in medias }
            
            cancelled_display_medias.discard( None )
            
            cancelled_media_results = { media.GetMediaResult() for media in cancelled_display_medias }
            
            outstanding_delayed_hashes = { media_result.GetHash() for media_result in cancelled_media_results if media_result in self._delayed_regeneration_queue_quick }
            
            if len( outstanding_delayed_hashes ) > 0:
                
                self._controller.files_maintenance_manager.ScheduleJob( outstanding_delayed_hashes, ClientFiles.REGENERATE_FILE_DATA_JOB_FORCE_THUMBNAIL )
                
            
            self._delayed_regeneration_queue_quick.difference_update( cancelled_media_results )
            
            self._RecalcQueues()
            
        
    
    def Clear( self ):
        
        with self._lock:
            
            self._data_cache.Clear()
            
            self._special_thumbs = {}
            
            names = [ 'hydrus', 'pdf', 'psd', 'clip', 'sai', 'krita', 'xcf', 'svg', 'audio', 'video', 'zip', 'epub', 'djvu' ]
            
            bounding_dimensions = self._controller.options[ 'thumbnail_dimensions' ]
            thumbnail_scale_type = self._controller.new_options.GetInteger( 'thumbnail_scale_type' )
            thumbnail_dpr_percent = HG.client_controller.new_options.GetInteger( 'thumbnail_dpr_percent' )
            
            # it would be ideal to replace this with mimes_to_default_thumbnail_paths at a convenient point
            
            for name in names:
                
                path = os.path.join( HC.STATIC_DIR, '{}.png'.format( name ) )
                
                numpy_image = ClientImageHandling.GenerateNumPyImage( path, HC.IMAGE_PNG )
                
                numpy_image_resolution = HydrusImageHandling.GetResolutionNumPy( numpy_image )
                
                target_resolution = HydrusImageHandling.GetThumbnailResolution( numpy_image_resolution, bounding_dimensions, thumbnail_scale_type, thumbnail_dpr_percent )
                
                numpy_image = HydrusImageHandling.ResizeNumPyImage( numpy_image, target_resolution )
                
                hydrus_bitmap = ClientRendering.GenerateHydrusBitmapFromNumPyImage( numpy_image )
                
                self._special_thumbs[ name ] = hydrus_bitmap
                
            
            self._controller.pub( 'notify_complete_thumbnail_reset' )
            
            self._waterfall_queue_quick = set()
            self._delayed_regeneration_queue_quick = set()
            
            self._RecalcQueues()
            
        
    
    def ClearThumbnails( self, hashes ):
        
        with self._lock:
            
            for hash in hashes:
                
                self._data_cache.DeleteData( hash )
                
            
        
    
    def WaitUntilFree( self ):
        
        while True:
            
            if HG.started_shutdown:
                
                raise HydrusExceptions.ShutdownException( 'Application shutting down!' )
                
            
            queue_is_empty = self._waterfall_queue_empty_event.wait( 1 )
            
            if queue_is_empty:
                
                return
                
            
        
    
    def GetThumbnail( self, media ):
        
        display_media = media.GetDisplayMedia()
        
        if display_media is None:
            
            # sometimes media can get switched around during a collect event, and if this happens during waterfall, we have a problem here
            # just return for now, we'll see how it goes
            
            return self._special_thumbs[ 'hydrus' ]
            
        
        can_provide = self._ShouldBeAbleToProvideThumb( display_media )
        
        if can_provide:
            
            mime = display_media.GetMime()
            
            if mime in HC.MIMES_WITH_THUMBNAILS:
                
                hash = display_media.GetHash()
                
                result = self._data_cache.GetIfHasData( hash )
                
                if result is None:
                    
                    try:
                        
                        hydrus_bitmap = self._GetThumbnailHydrusBitmap( display_media )
                        
                    except:
                        
                        hydrus_bitmap = self._special_thumbs[ 'hydrus' ]
                        
                    
                    self._data_cache.AddData( hash, hydrus_bitmap )
                    
                else:
                    
                    hydrus_bitmap = result
                    
                
                return hydrus_bitmap
                
            elif mime in HC.AUDIO: return self._special_thumbs[ 'audio' ]
            elif mime in HC.VIDEO: return self._special_thumbs[ 'video' ]
            elif mime == HC.APPLICATION_PDF: return self._special_thumbs[ 'pdf' ]
            elif mime == HC.APPLICATION_EPUB: return self._special_thumbs[ 'epub' ]
            elif mime == HC.APPLICATION_DJVU: return self._special_thumbs[ 'djvu' ]
            elif mime == HC.APPLICATION_PSD: return self._special_thumbs[ 'psd' ]
            elif mime == HC.APPLICATION_SAI2: return self._special_thumbs[ 'sai' ]
            elif mime == HC.APPLICATION_KRITA: return self._special_thumbs[ 'krita' ]
            elif mime == HC.APPLICATION_XCF: return self._special_thumbs[ 'xcf' ]
            elif mime == HC.IMAGE_SVG: return self._special_thumbs[ 'svg' ]
            elif mime in HC.ARCHIVES: return self._special_thumbs[ 'zip' ]
            else: return self._special_thumbs[ 'hydrus' ]
            
        else:
            
            return self._special_thumbs[ 'hydrus' ]
            
        
    
    def HasThumbnailCached( self, media ):
        
        display_media = media.GetDisplayMedia()
        
        if display_media is None:
            
            return True
            
        
        mime = display_media.GetMime()
        
        if mime in HC.MIMES_WITH_THUMBNAILS:
            
            if self._ShouldBeAbleToProvideThumb( display_media ):
                
                hash = display_media.GetHash()
                
                return self._data_cache.HasData( hash )
                
            else:
                
                # yes because we provide the hydrus icon instantly
                return True
                
            
        else:
            
            return True
            
        
    
    def NotifyNewOptions( self ):
        
        cache_size = self._controller.new_options.GetInteger( 'thumbnail_cache_size' )
        cache_timeout = self._controller.new_options.GetInteger( 'thumbnail_cache_timeout' )
        
        self._data_cache.SetCacheSizeAndTimeout( cache_size, cache_timeout )
        
        allow_blurhash_fallback = self._controller.new_options.GetBoolean( 'allow_blurhash_fallback' )
        
        if allow_blurhash_fallback != self._allow_blurhash_fallback:
            
            self._allow_blurhash_fallback = allow_blurhash_fallback
            
            self.Clear()
            
        
    
    def Waterfall( self, page_key, medias ):
        
        with self._lock:
            
            self._waterfall_queue_quick.update( ( ( page_key, media ) for media in medias ) )
            
            self._RecalcQueues()
            
        
        self._waterfall_event.set()
        
    
    def MainLoop( self ):
        
        while not HydrusThreading.IsThreadShuttingDown():
            
            time.sleep( 0.00001 )
            
            with self._lock:
                
                do_wait = len( self._waterfall_queue ) == 0 and len( self._delayed_regeneration_queue ) == 0
                
            
            if do_wait:
                
                self._waterfall_event.wait( 1 )
                
                self._waterfall_event.clear()
                
            
            start_time = HydrusTime.GetNowPrecise()
            stop_time = start_time + 0.005 # a bit of a typical frame
            
            page_keys_to_rendered_medias = collections.defaultdict( list )
            
            num_done = 0
            max_at_once = 16
            
            while not HydrusTime.TimeHasPassedPrecise( stop_time ) and num_done <= max_at_once:
                
                with self._lock:
                    
                    if len( self._waterfall_queue ) == 0:
                        
                        break
                        
                    
                    result = self._waterfall_queue.pop()
                    
                    if len( self._waterfall_queue ) == 0:
                        
                        self._waterfall_queue_empty_event.set()
                        
                    
                    self._waterfall_queue_quick.discard( result )
                    
                
                ( page_key, media ) = result
                
                if media.GetDisplayMedia() is not None:
                    
                    self.GetThumbnail( media )
                    
                    page_keys_to_rendered_medias[ page_key ].append( media )
                    
                
                num_done += 1
                
            
            if len( page_keys_to_rendered_medias ) > 0:
                
                for ( page_key, rendered_medias ) in page_keys_to_rendered_medias.items():
                    
                    self._controller.pub( 'waterfall_thumbnails', page_key, rendered_medias )
                    
                
                time.sleep( 0.00001 )
                
            
            # now we will do regen if appropriate
            
            with self._lock:
                
                # got more important work or no work to do
                if len( self._waterfall_queue ) > 0 or len( self._delayed_regeneration_queue ) == 0 or HG.client_controller.CurrentlyPubSubbing():
                    
                    continue
                    
                
                media_result = self._delayed_regeneration_queue.pop()
                
                self._delayed_regeneration_queue_quick.discard( media_result )
                
            
            if HG.file_report_mode:
                
                hash = media_result.GetHash()
                
                HydrusData.ShowText( 'Thumbnail {} now regenerating from source.'.format( hash.hex() ) )
                
            
            try:
                
                self._controller.files_maintenance_manager.RunJobImmediately( [ media_result ], ClientFiles.REGENERATE_FILE_DATA_JOB_FORCE_THUMBNAIL, pub_job_key = False )
                
            except HydrusExceptions.FileMissingException:
                
                pass
                
            except Exception as e:
                
                hash = media_result.GetHash()
                
                summary = 'The thumbnail for file {} was incorrect, but a later attempt to regenerate it or load the new file back failed.'.format( hash.hex() )
                
                self._HandleThumbnailException( hash, e, summary )