hydrus/hydrus/client/importing/ClientImportLocal.py

import collections
import os
import threading
import time
import typing

from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusFileHandling
from hydrus.core import HydrusGlobals as HG
from hydrus.core import HydrusPaths
from hydrus.core import HydrusSerialisable
from hydrus.core import HydrusThreading

from hydrus.client import ClientConstants as CC
from hydrus.client import ClientData
from hydrus.client import ClientFiles
from hydrus.client import ClientPaths
from hydrus.client import ClientSearch
from hydrus.client import ClientThreading
from hydrus.client.importing import ClientImportControl
from hydrus.client.importing import ClientImporting
from hydrus.client.importing import ClientImportFileSeeds
from hydrus.client.importing.options import FileImportOptions
from hydrus.client.importing.options import TagImportOptions
from hydrus.client.metadata import ClientMetadataMigration
from hydrus.client.metadata import ClientMetadataMigrationExporters
from hydrus.client.metadata import ClientMetadataMigrationImporters
from hydrus.client.metadata import ClientTags

class HDDImport( HydrusSerialisable.SerialisableBase ):
    
    SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_HDD_IMPORT
    SERIALISABLE_NAME = 'Local File Import'
    SERIALISABLE_VERSION = 3
    
    def __init__( self, paths = None, file_import_options = None, metadata_routers = None, paths_to_additional_service_keys_to_tags = None, delete_after_success = None ):
        
        HydrusSerialisable.SerialisableBase.__init__( self )
        
        if metadata_routers is None:
            
            metadata_routers = []
            
        
        if paths_to_additional_service_keys_to_tags is None:
            
            paths_to_additional_service_keys_to_tags = collections.defaultdict( ClientTags.ServiceKeysToTags )
            
        
        if delete_after_success is None:
            
            delete_after_success = False
            
        
        if paths is None:
            
            self._file_seed_cache = None
            
        else:
            
            self._file_seed_cache = ClientImportFileSeeds.FileSeedCache()
            
            file_seeds = []
            
            for path in paths:
                
                file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_HDD, path )
                
                try:
                    
                    file_modified_time = HydrusFileHandling.GetFileModifiedTimestamp( path )
                    
                    file_seed.source_time = file_modified_time
                    
                except:
                    
                    pass
                    
                
                if path in paths_to_additional_service_keys_to_tags:
                    
                    file_seed.SetExternalAdditionalServiceKeysToTags( paths_to_additional_service_keys_to_tags[ path ] )
                    
                
                file_seeds.append( file_seed )
                
            
            self._file_seed_cache.AddFileSeeds( file_seeds )
            
        
        self._metadata_routers = HydrusSerialisable.SerialisableList( metadata_routers )
        
        self._file_import_options = file_import_options
        self._delete_after_success = delete_after_success
        
        self._page_key = b'initialising page key'
        
        self._files_status = ''
        self._paused = False
        
        self._lock = threading.Lock()
        
        self._files_repeating_job = None
        
        self._last_serialisable_change_timestamp = 0
        
        HG.client_controller.sub( self, 'NotifyFileSeedsUpdated', 'file_seed_cache_file_seeds_updated' )
        
    
    def _GetSerialisableInfo( self ):
        
        serialisable_file_seed_cache = self._file_seed_cache.GetSerialisableTuple()
        serialisable_options = self._file_import_options.GetSerialisableTuple()
        serialisable_metadata_routers = self._metadata_routers.GetSerialisableTuple()
        
        return ( serialisable_file_seed_cache, serialisable_options, serialisable_metadata_routers, self._delete_after_success, self._paused )
        
    
    def _InitialiseFromSerialisableInfo( self, serialisable_info ):
        
        ( serialisable_file_seed_cache, serialisable_options, serialisable_metadata_routers, self._delete_after_success, self._paused ) = serialisable_info
        
        self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
        self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_options )
        self._metadata_routers = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_metadata_routers )
        
    
    def _SerialisableChangeMade( self ):
        
        self._last_serialisable_change_timestamp = HydrusData.GetNow()
        
    
    def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
        
        if version == 1:
            
            ( serialisable_file_seed_cache, serialisable_options, serialisable_paths_to_tags, delete_after_success, paused ) = old_serialisable_info
            
            file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
            
            paths_to_additional_service_keys_to_tags = { path : { bytes.fromhex( service_key ) : tags for ( service_key, tags ) in service_keys_to_tags.items() } for ( path, service_keys_to_tags ) in serialisable_paths_to_tags.items() }
            
            for file_seed in file_seed_cache.GetFileSeeds():
                
                path = file_seed.file_seed_data
                
                if path in paths_to_additional_service_keys_to_tags:
                    
                    file_seed.SetExternalAdditionalServiceKeysToTags( paths_to_additional_service_keys_to_tags[ path ] )
                    
                
            
            serialisable_file_seed_cache = file_seed_cache.GetSerialisableTuple()
            
            new_serialisable_info = ( serialisable_file_seed_cache, serialisable_options, delete_after_success, paused )
            
            return ( 2, new_serialisable_info )
            
        
        if version == 2:
            
            ( serialisable_file_seed_cache, serialisable_options, delete_after_success, paused ) = old_serialisable_info
            
            metadata_routers = HydrusSerialisable.SerialisableList()
            
            serialisable_metadata_routers = metadata_routers.GetSerialisableTuple()
            
            new_serialisable_info = ( serialisable_file_seed_cache, serialisable_options, serialisable_metadata_routers, delete_after_success, paused )
            
            return ( 3, new_serialisable_info )
            
        
    
    def _WorkOnFiles( self ):
        
        file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
        
        if file_seed is None:
            
            return
            
        
        path = file_seed.file_seed_data
        
        with self._lock:
            
            self._files_status = 'importing'
            
        
        def status_hook( text ):
            
            with self._lock:
                
                self._files_status = ClientImportControl.NeatenStatusText( text )
                
            
        
        file_seed.ImportPath( self._file_seed_cache, self._file_import_options, FileImportOptions.IMPORT_TYPE_LOUD, status_hook = status_hook )
        
        if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:
            
            if len( self._metadata_routers ) > 0:
                
                hash = file_seed.GetHash()
                
                media_result = HG.client_controller.Read( 'media_result', hash )
                
                for router in self._metadata_routers:
                    
                    try:
                        
                        router.Work( media_result, file_seed.file_seed_data )
                        
                    except Exception as e:
                        
                        HydrusData.ShowText( 'Trying to run metadata routing on the file "{}" threw an error!'.format( file_seed.file_seed_data ) )
                        HydrusData.ShowException( e )
                        
                    
                
            
            real_presentation_import_options = FileImportOptions.GetRealPresentationImportOptions( self._file_import_options, FileImportOptions.IMPORT_TYPE_LOUD )
            
            if file_seed.ShouldPresent( real_presentation_import_options ):
                
                file_seed.PresentToPage( self._page_key )
                
            
            if self._delete_after_success:
                
                try:
                    
                    ClientPaths.DeletePath( path )
                    
                except Exception as e:
                    
                    HydrusData.ShowText( 'While attempting to delete {}, the following error occurred:'.format( path ) )
                    HydrusData.ShowException( e )
                    
                
                possible_sidecar_paths = set()
                
                for router in self._metadata_routers:
                    
                    possible_sidecar_paths.update( router.GetPossibleImporterSidecarPaths( path ) )
                    
                
                for possible_sidecar_path in possible_sidecar_paths:
                    
                    if os.path.exists( possible_sidecar_path ):
                        
                        try:
                            
                            ClientPaths.DeletePath( possible_sidecar_path )
                            
                        except Exception as e:
                            
                            HydrusData.ShowText( 'While attempting to delete {}, the following error occurred:'.format( possible_sidecar_path ) )
                            HydrusData.ShowException( e )
                            
                        
                    
            
        
        with self._lock:
            
            self._files_status = ''
            
        
        time.sleep( ClientImporting.DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )
        
    
    def CurrentlyWorking( self ):
        
        with self._lock:
            
            work_to_do = self._file_seed_cache.WorkToDo()
            
            return work_to_do and not self._paused
            
        
    
    def GetAPIInfoDict( self, simple ):
        
        with self._lock:
            
            d = {}
            
            d[ 'imports' ] = self._file_seed_cache.GetAPIInfoDict( simple )
            
            d[ 'files_paused' ] = self._paused
            
            return d
            
        
    
    def GetFileImportOptions( self ):
        
        with self._lock:
            
            return self._file_import_options
            
        
    
    def GetFileSeedCache( self ):
        
        return self._file_seed_cache
        
    
    def GetNumSeeds( self ):
        
        with self._lock:
            
            return len( self._file_seed_cache )
            
        
    
    def GetStatus( self ):
        
        with self._lock:
            
            text = ClientImportControl.GenerateLiveStatusText( self._files_status, self._paused, 0, '' )
            
            return ( text, self._paused )
            
        
    
    def GetValueRange( self ):
        
        with self._lock:
            
            return self._file_seed_cache.GetValueRange()
            
        
    
    def HasSerialisableChangesSince( self, since_timestamp ):
        
        with self._lock:
            
            return self._last_serialisable_change_timestamp > since_timestamp
            
        
    
    def NotifyFileSeedsUpdated( self, file_seed_cache_key, file_seeds ):
        
        if file_seed_cache_key == self._file_seed_cache.GetFileSeedCacheKey():
            
            ClientImporting.WakeRepeatingJob( self._files_repeating_job )
            
            self._SerialisableChangeMade()
            
        
    
    def PausePlay( self ):
        
        with self._lock:
            
            self._paused = not self._paused
            
            ClientImporting.WakeRepeatingJob( self._files_repeating_job )
            
            self._SerialisableChangeMade()
            
        
    
    def SetFileImportOptions( self, file_import_options: FileImportOptions.FileImportOptions ):
        
        with self._lock:
            
            if file_import_options.DumpToString() != self._file_import_options.DumpToString():
                
                self._file_import_options = file_import_options
                
                self._SerialisableChangeMade()
                
            
        
    
    def Start( self, page_key ):
        
        self._page_key = page_key
        
        self._files_repeating_job = HG.client_controller.CallRepeating( ClientImporting.GetRepeatingJobInitialDelay(), ClientImporting.REPEATING_JOB_TYPICAL_PERIOD, self.REPEATINGWorkOnFiles )
        
        self._files_repeating_job.SetThreadSlotType( 'misc' )
        
    
    def CheckCanDoFileWork( self ):
        
        with self._lock:
            
            try:
                
                ClientImportControl.CheckImporterCanDoWorkBecauseStopped( self._page_key )
                
            except HydrusExceptions.VetoException:
                
                self._files_repeating_job.Cancel()
                
                raise
                
            
            ClientImportControl.CheckImporterCanDoFileWorkBecausePaused( self._paused, self._file_seed_cache, self._page_key )
            
        
        return True
        
    
    def REPEATINGWorkOnFiles( self ):
        
        while True:
            
            try:
                
                try:
                    
                    self.CheckCanDoFileWork()
                    
                except HydrusExceptions.VetoException as e:
                    
                    with self._lock:
                        
                        self._files_status = str( e )
                        
                    
                    break
                    
                
                self._WorkOnFiles()
                
                HG.client_controller.WaitUntilViewFree()
                
                self._SerialisableChangeMade()
                
            except Exception as e:
                
                with self._lock:
                    
                    self._files_status = 'stopping work: {}'.format( str( e ) )
                    
                
                HydrusData.ShowException( e )
                
                return
                
            
        
    
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_HDD_IMPORT ] = HDDImport

class ImportFolder( HydrusSerialisable.SerialisableBaseNamed ):
    
    SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_IMPORT_FOLDER
    SERIALISABLE_NAME = 'Import Folder'
    SERIALISABLE_VERSION = 8
    
    def __init__(
        self,
        name,
        path = '',
        file_import_options = None,
        tag_import_options = None,
        metadata_routers: typing.Optional[ typing.Collection[ ClientMetadataMigration.SingleFileMetadataRouter ] ] = None,
        tag_service_keys_to_filename_tagging_options = None,
        actions = None,
        action_locations = None,
        period = 3600,
        check_regularly = True,
        show_working_popup = True,
        publish_files_to_popup_button = True,
        publish_files_to_page = False
    ):
        
        if file_import_options is None:
            
            file_import_options = FileImportOptions.FileImportOptions()
            file_import_options.SetIsDefault( True )
            
        
        if tag_import_options is None:
            
            tag_import_options = TagImportOptions.TagImportOptions()
            
        
        if metadata_routers is None:
            
            metadata_routers = []
            
        
        metadata_routers = HydrusSerialisable.SerialisableList( metadata_routers )
        
        if tag_service_keys_to_filename_tagging_options is None:
            
            tag_service_keys_to_filename_tagging_options = {}
            
        
        if actions is None:
            
            actions = {}
            
            actions[ CC.STATUS_SUCCESSFUL_AND_NEW ] = CC.IMPORT_FOLDER_IGNORE
            actions[ CC.STATUS_SUCCESSFUL_BUT_REDUNDANT ] = CC.IMPORT_FOLDER_IGNORE
            actions[ CC.STATUS_DELETED ] = CC.IMPORT_FOLDER_IGNORE
            actions[ CC.STATUS_ERROR ] = CC.IMPORT_FOLDER_IGNORE
            
        
        if action_locations is None:
            
            action_locations = {}
            
        
        HydrusSerialisable.SerialisableBaseNamed.__init__( self, name )
        
        self._path = path
        self._file_import_options = file_import_options
        self._tag_import_options = tag_import_options
        self._metadata_routers = metadata_routers
        self._tag_service_keys_to_filename_tagging_options = tag_service_keys_to_filename_tagging_options
        self._actions = actions
        self._action_locations = action_locations
        self._period = period
        self._check_regularly = check_regularly
        
        self._file_seed_cache = ClientImportFileSeeds.FileSeedCache()
        self._last_checked = 0
        self._paused = False
        self._check_now = False
        
        self._show_working_popup = show_working_popup
        self._publish_files_to_popup_button = publish_files_to_popup_button
        self._publish_files_to_page = publish_files_to_page
        
    
    def _ActionPaths( self ):
        
        for status in ( CC.STATUS_SUCCESSFUL_AND_NEW, CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, CC.STATUS_DELETED, CC.STATUS_ERROR ):
            
            action = self._actions[ status ]
            
            if action == CC.IMPORT_FOLDER_DELETE:
                
                while True:
                    
                    file_seed = self._file_seed_cache.GetNextFileSeed( status )
                    
                    if file_seed is None or HG.started_shutdown:
                        
                        break
                        
                    
                    path = file_seed.file_seed_data
                    
                    try:
                        
                        if os.path.exists( path ) and not os.path.isdir( path ):
                            
                            ClientPaths.DeletePath( path )
                            
                        
                        possible_sidecar_paths = set()
                        
                        for router in self._metadata_routers:
                            
                            possible_sidecar_paths.update( router.GetPossibleImporterSidecarPaths( path ) )
                            
                        
                        for possible_sidecar_path in possible_sidecar_paths:
                            
                            if os.path.exists( possible_sidecar_path ):
                                
                                ClientPaths.DeletePath( possible_sidecar_path )
                                
                            
                        
                        self._file_seed_cache.RemoveFileSeeds( ( file_seed, ) )
                        
                    except Exception as e:
                        
                        raise Exception( 'Tried to delete "{}", but could not.'.format( path ) )
                        
                    
                
            elif action == CC.IMPORT_FOLDER_MOVE:
                
                while True:
                    
                    file_seed = self._file_seed_cache.GetNextFileSeed( status )
                    
                    if file_seed is None or HG.started_shutdown:
                        
                        break
                        
                    
                    path = file_seed.file_seed_data
                    
                    try:
                        
                        dest_dir = self._action_locations[ status ]
                        
                        if not os.path.exists( dest_dir ):
                            
                            raise Exception( 'Tried to move "{}" to "{}", but the destination directory did not exist.'.format( path, dest_dir ) )
                            
                        
                        if os.path.exists( path ) and not os.path.isdir( path ):
                            
                            filename = os.path.basename( path )
                            
                            dest_path = os.path.join( dest_dir, filename )
                            
                            dest_path = HydrusPaths.AppendPathUntilNoConflicts( dest_path )
                            
                            HydrusPaths.MergeFile( path, dest_path )
                            
                        
                        txt_path = path + '.txt'
                        
                        if os.path.exists( txt_path ):
                            
                            txt_filename = os.path.basename( txt_path )
                            
                            txt_dest_path = os.path.join( dest_dir, txt_filename )
                            
                            txt_dest_path = HydrusPaths.AppendPathUntilNoConflicts( txt_dest_path )
                            
                            HydrusPaths.MergeFile( txt_path, txt_dest_path )
                            
                        
                        self._file_seed_cache.RemoveFileSeeds( ( file_seed, ) )
                        
                    except Exception as e:
                        
                        HydrusData.ShowText( 'Import folder tried to move ' + path + ', but could not:' )
                        
                        HydrusData.ShowException( e )
                        
                        HydrusData.ShowText( 'Import folder has been paused.' )
                        
                        self._paused = True
                        
                        return
                        
                    
                
            elif status == CC.IMPORT_FOLDER_IGNORE:
                
                file_seeds = self._file_seed_cache.GetFileSeeds( status )
                
                for file_seed in file_seeds:
                    
                    path = file_seed.file_seed_data
                    
                    try:
                        
                        if not os.path.exists( path ):
                            
                            self._file_seed_cache.RemoveFileSeeds( ( file_seed, ) )
                            
                        
                    except Exception as e:
                        
                        raise Exception( 'Tried to check existence of "{}", but could not.'.format( path ) )
                        
                    
                
            
        
    
    def _CheckFolder( self, job_key ):
    
        ( all_paths, num_sidecars ) = ClientFiles.GetAllFilePaths( [ self._path ] )
        
        all_paths = HydrusPaths.FilterFreePaths( all_paths )
        
        file_seeds = []
        
        for path in all_paths:
            
            if job_key.IsCancelled():
                
                break
                
            
            file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_HDD, path )
            
            if not self._file_seed_cache.HasFileSeed( file_seed ):
                
                file_seeds.append( file_seed )
                
            
            job_key.SetVariable( 'popup_text_1', 'checking: found ' + HydrusData.ToHumanInt( len( file_seeds ) ) + ' new files' )
            
        
        self._file_seed_cache.AddFileSeeds( file_seeds )
        
        self._last_checked = HydrusData.GetNow()
        self._check_now = False
        
    
    def _GetSerialisableInfo( self ):
        
        serialisable_file_import_options = self._file_import_options.GetSerialisableTuple()
        serialisable_tag_import_options = self._tag_import_options.GetSerialisableTuple()
        serialisable_metadata_routers = self._metadata_routers.GetSerialisableTuple()
        serialisable_tag_service_keys_to_filename_tagging_options = [ ( service_key.hex(), filename_tagging_options.GetSerialisableTuple() ) for ( service_key, filename_tagging_options ) in list(self._tag_service_keys_to_filename_tagging_options.items()) ]
        serialisable_file_seed_cache = self._file_seed_cache.GetSerialisableTuple()
        
        # json turns int dict keys to strings
        action_pairs = list(self._actions.items())
        action_location_pairs = list(self._action_locations.items())
        
        return ( self._path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_metadata_routers, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, self._period, self._check_regularly, serialisable_file_seed_cache, self._last_checked, self._paused, self._check_now, self._show_working_popup, self._publish_files_to_popup_button, self._publish_files_to_page )
        
    
    def _ImportFiles( self, job_key ):
        
        did_work = False
        
        time_to_save = HydrusData.GetNow() + 600
        
        num_files_imported = 0
        presentation_hashes = []
        presentation_hashes_fast = set()
        
        i = 0
        
        # don't want to start at 23/100 because of carrying over failed results or whatever
        # num_to_do is num currently unknown
        num_total = self._file_seed_cache.GetFileSeedCount( CC.STATUS_UNKNOWN )
        
        while True:
            
            file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
            
            p1 = HG.client_controller.new_options.GetBoolean( 'pause_import_folders_sync' ) or self._paused
            p2 = HydrusThreading.IsThreadShuttingDown()
            p3 = job_key.IsCancelled()
            
            if file_seed is None or p1 or p2 or p3:
                
                break
                
            
            did_work = True
            
            if HydrusData.TimeHasPassed( time_to_save ):
                
                HG.client_controller.WriteSynchronous( 'serialisable', self )
                
                time_to_save = HydrusData.GetNow() + 600
                
            
            gauge_num_done = num_files_imported + 1
            
            job_key.SetVariable( 'popup_text_1', 'importing file ' + HydrusData.ConvertValueRangeToPrettyString( gauge_num_done, num_total ) )
            job_key.SetVariable( 'popup_gauge_1', ( gauge_num_done, num_total ) )
            
            path = file_seed.file_seed_data
            
            file_seed.ImportPath( self._file_seed_cache, self._file_import_options, FileImportOptions.IMPORT_TYPE_QUIET )
            
            if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:
                
                hash = None
                
                if file_seed.HasHash():
                    
                    hash = file_seed.GetHash()
                    
                    if self._tag_import_options.HasAdditionalTags() or len( self._metadata_routers ) > 0:
                        
                        media_result = HG.client_controller.Read( 'media_result', hash )
                        
                        if self._tag_import_options.HasAdditionalTags():
                            
                            downloaded_tags = []
                            
                            service_keys_to_content_updates = self._tag_import_options.GetServiceKeysToContentUpdates( file_seed.status, media_result, downloaded_tags ) # additional tags
                            
                            if len( service_keys_to_content_updates ) > 0:
                                
                                HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
                                
                            
                        
                        for metadata_router in self._metadata_routers:
                            
                            try:
                                
                                metadata_router.Work( media_result, path )
                                
                            except Exception as e:
                                
                                HydrusData.ShowText( 'Trying to run metadata routing in the import folder "' + self._name + '" threw an error!' )
                                
                                HydrusData.ShowException( e )
                                
                            
                        
                    
                    service_keys_to_tags = ClientTags.ServiceKeysToTags()
                    
                    for ( tag_service_key, filename_tagging_options ) in self._tag_service_keys_to_filename_tagging_options.items():
                        
                        if not HG.client_controller.services_manager.ServiceExists( tag_service_key ):
                            
                            continue
                            
                        
                        try:
                            
                            tags = filename_tagging_options.GetTags( tag_service_key, path )
                            
                            if len( tags ) > 0:
                                
                                service_keys_to_tags[ tag_service_key ] = tags
                                
                            
                        except Exception as e:
                            
                            HydrusData.ShowText( 'Trying to parse filename tags in the import folder "' + self._name + '" threw an error!' )
                            
                            HydrusData.ShowException( e )
                            
                        
                    
                    if len( service_keys_to_tags ) > 0:
                        
                        service_keys_to_content_updates = ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates( { hash }, service_keys_to_tags )
                        
                        HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
                        
                    
                
                num_files_imported += 1
                
                if hash not in presentation_hashes_fast:
                    
                    real_presentation_import_options = FileImportOptions.GetRealPresentationImportOptions( self._file_import_options, FileImportOptions.IMPORT_TYPE_LOUD )
                    
                    if file_seed.ShouldPresent( real_presentation_import_options ):
                        
                        presentation_hashes.append( hash )
                        
                        presentation_hashes_fast.add( hash )
                        
                    
                
            elif file_seed.status == CC.STATUS_ERROR:
                
                HydrusData.Print( 'A file failed to import from import folder ' + self._name + ':' + path )
                
            
            i += 1
            
            if i % 10 == 0:
                
                self._ActionPaths()
                
            
        
        if num_files_imported > 0:
            
            HydrusData.Print( 'Import folder ' + self._name + ' imported ' + HydrusData.ToHumanInt( num_files_imported ) + ' files.' )
            
            if len( presentation_hashes ) > 0:
                
                ClientImporting.PublishPresentationHashes( self._name, presentation_hashes, self._publish_files_to_popup_button, self._publish_files_to_page )
                
            
        
        self._ActionPaths()
        
        return did_work
        
    
    def _InitialiseFromSerialisableInfo( self, serialisable_info ):
        
        ( self._path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_metadata_routers, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, self._period, self._check_regularly, serialisable_file_seed_cache, self._last_checked, self._paused, self._check_now, self._show_working_popup, self._publish_files_to_popup_button, self._publish_files_to_page ) = serialisable_info
        
        self._actions = dict( action_pairs )
        self._action_locations = dict( action_location_pairs )
        
        self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_import_options )
        self._tag_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_import_options )
        self._metadata_routers = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_metadata_routers )
        self._tag_service_keys_to_filename_tagging_options = dict( [ ( bytes.fromhex( encoded_service_key ), HydrusSerialisable.CreateFromSerialisableTuple( serialisable_filename_tagging_options ) ) for ( encoded_service_key, serialisable_filename_tagging_options ) in serialisable_tag_service_keys_to_filename_tagging_options ] )
        self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
        
    
    def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
        
        if version == 1:
            
            ( path, mimes, serialisable_file_import_options, action_pairs, action_location_pairs, period, open_popup, tag, serialisable_file_seed_cache, last_checked, paused ) = old_serialisable_info
            
            # edited out tag carry-over to tio due to bit rot
            
            tag_import_options = TagImportOptions.TagImportOptions()
            
            serialisable_tag_import_options = tag_import_options.GetSerialisableTuple()
            
            new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused )
            
            return ( 2, new_serialisable_info )
            
        
        if version == 2:
            
            ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused ) = old_serialisable_info
            
            serialisable_txt_parse_tag_service_keys = []
            
            new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_txt_parse_tag_service_keys, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused )
            
            return ( 3, new_serialisable_info )
            
        
        if version == 3:
            
            ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_txt_parse_tag_service_keys, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused ) = old_serialisable_info
            
            check_now = False
            
            new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_txt_parse_tag_service_keys, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused, check_now )
            
            return ( 4, new_serialisable_info )
            
        
        if version == 4:
            
            ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_txt_parse_tag_service_keys, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused, check_now ) = old_serialisable_info
            
            txt_parse_tag_service_keys = [ bytes.fromhex( service_key ) for service_key in serialisable_txt_parse_tag_service_keys ]
            
            tag_service_keys_to_filename_tagging_options = {}
            
            for service_key in txt_parse_tag_service_keys:
                
                filename_tagging_options = TagImportOptions.FilenameTaggingOptions()
                
                filename_tagging_options._load_from_neighbouring_txt_files = True
                
                tag_service_keys_to_filename_tagging_options[ service_key ] = filename_tagging_options
                
            
            serialisable_tag_service_keys_to_filename_tagging_options = [ ( service_key.hex(), filename_tagging_options.GetSerialisableTuple() ) for ( service_key, filename_tagging_options ) in list(tag_service_keys_to_filename_tagging_options.items()) ]
            
            new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused, check_now )
            
            return ( 5, new_serialisable_info )
            
        
        if version == 5:
            
            ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused, check_now ) = old_serialisable_info
            
            check_regularly = not paused
            show_working_popup = True
            publish_files_to_page = False
            publish_files_to_popup_button = open_popup
            
            new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, check_regularly, serialisable_file_seed_cache, last_checked, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page )
            
            return ( 6, new_serialisable_info )
            
        
        if version == 6:
            
            ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, check_regularly, serialisable_file_seed_cache, last_checked, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page ) = old_serialisable_info
            
            file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_import_options )
            
            file_import_options.SetAllowedSpecificFiletypes( mimes )
            
            serialisable_file_import_options = file_import_options.GetSerialisableTuple()
            
            new_serialisable_info = ( path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, check_regularly, serialisable_file_seed_cache, last_checked, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page )
            
            return ( 7, new_serialisable_info )
            
        
        if version == 7:
            
            ( path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, check_regularly, serialisable_file_seed_cache, last_checked, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page ) = old_serialisable_info
            
            tag_service_keys_to_filename_tagging_options = dict( [ ( bytes.fromhex( encoded_service_key ), HydrusSerialisable.CreateFromSerialisableTuple( serialisable_filename_tagging_options ) ) for ( encoded_service_key, serialisable_filename_tagging_options ) in serialisable_tag_service_keys_to_filename_tagging_options ] )
            
            metadata_routers = HydrusSerialisable.SerialisableList()
            
            try:
                
                for ( service_key, filename_tagging_options ) in tag_service_keys_to_filename_tagging_options.items():
                    
                    # beardy access here, but this is once off
                    if hasattr( filename_tagging_options, '_load_from_neighbouring_txt_files' ) and filename_tagging_options._load_from_neighbouring_txt_files:
                        
                        importers = [ ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT() ]
                        exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags( service_key = service_key )
                        
                        metadata_router = ClientMetadataMigration.SingleFileMetadataRouter( importers = importers, exporter = exporter )
                        
                        metadata_routers.append( metadata_router )
                        
                    
                
            except Exception as e:
                
                HydrusData.Print( 'Failed to update import folder with new metadata routers.' )
                
                HydrusData.PrintException( e )
                
            
            serialisable_metadata_routers = metadata_routers.GetSerialisableTuple()
            
            new_serialisable_info = ( path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_metadata_routers, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, check_regularly, serialisable_file_seed_cache, last_checked, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page )
            
            return ( 8, new_serialisable_info )
            
        
    
    def CheckNow( self ):
        
        self._paused = False
        self._check_now = True
        
    
    def DoWork( self ):
        
        if HG.started_shutdown:
            
            return
            
        
        if HG.client_controller.new_options.GetBoolean( 'pause_import_folders_sync' ) or self._paused:
            
            return
            
        
        checked_folder = False
        
        did_import_file_work = False
        
        error_occured = False
        
        stop_time = HydrusData.GetNow() + 3600
        
        job_key = ClientThreading.JobKey( pausable = False, cancellable = True, stop_time = stop_time )
        
        try:
            
            real_file_import_options = FileImportOptions.GetRealFileImportOptions( self._file_import_options, FileImportOptions.IMPORT_TYPE_QUIET )
            
            real_file_import_options.CheckReadyToImport()
            
            if not os.path.exists( self._path ) or not os.path.isdir( self._path ):
                
                raise Exception( 'Path "' + self._path + '" does not seem to exist, or is not a directory.' )
                
            
            pubbed_job_key = False
            
            job_key.SetStatusTitle( 'import folder - ' + self._name )
            
            due_by_check_now = self._check_now
            due_by_period = self._check_regularly and HydrusData.TimeHasPassed( self._last_checked + self._period )
            
            if due_by_check_now or due_by_period:
                
                if not pubbed_job_key and self._show_working_popup:
                    
                    HG.client_controller.pub( 'message', job_key )
                    
                    pubbed_job_key = True
                    
                
                self._CheckFolder( job_key )
                
                checked_folder = True
                
            
            file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
            
            if file_seed is not None:
                
                if not pubbed_job_key and self._show_working_popup:
                    
                    HG.client_controller.pub( 'message', job_key )
                    
                    pubbed_job_key = True
                    
                
                did_import_file_work = self._ImportFiles( job_key )
                
            
        except Exception as e:
            
            error_occured = True
            self._paused = True
            
            HydrusData.ShowText( 'The import folder "' + self._name + '" encountered an exception! It has been paused!' )
            HydrusData.ShowException( e )
            
        
        if checked_folder or did_import_file_work or error_occured:
            
            HG.client_controller.WriteSynchronous( 'serialisable', self )
            
        
        job_key.Delete()
        
    
    def GetFileSeedCache( self ):
        
        return self._file_seed_cache
        
    
    def GetMetadataRouters( self ):
        
        return list( self._metadata_routers )
        
    
    def Paused( self ):
        
        return self._paused
        
    
    def PausePlay( self ):
        
        self._paused = not self._paused
        
    
    def ToListBoxTuple( self ):
        
        return ( self._name, self._path, self._paused, self._check_regularly, self._period )
        
    
    def ToTuple( self ):
        
        return ( self._name, self._path, self._file_import_options, self._tag_import_options, self._tag_service_keys_to_filename_tagging_options, self._actions, self._action_locations, self._period, self._check_regularly, self._paused, self._check_now, self._show_working_popup, self._publish_files_to_popup_button, self._publish_files_to_page )
        
    
    def SetFileSeedCache( self, file_seed_cache ):
        
        self._file_seed_cache = file_seed_cache
        
    
    def SetMetadataRouters( self, metadata_routers: typing.Collection[ ClientMetadataMigration.SingleFileMetadataRouter ] ):
        
        self._metadata_routers = HydrusSerialisable.SerialisableList( metadata_routers )
        
    
    def SetTuple( self, name, path, file_import_options, tag_import_options, tag_service_keys_to_filename_tagging_options, actions, action_locations, period, check_regularly, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page ):
        
        if path != self._path:
            
            self._file_seed_cache = ClientImportFileSeeds.FileSeedCache()
            
        
        if not file_import_options.IsDefault() and not self._file_import_options.IsDefault():
            
            mimes = set( file_import_options.GetAllowedSpecificFiletypes() )
            
            if mimes != set( self._file_import_options.GetAllowedSpecificFiletypes() ):
                
                self._file_seed_cache.RemoveFileSeedsByStatus( ( CC.STATUS_VETOED, ) )
                
            
        
        self._name = name
        self._path = path
        self._file_import_options = file_import_options
        self._tag_import_options = tag_import_options
        self._tag_service_keys_to_filename_tagging_options = tag_service_keys_to_filename_tagging_options
        self._actions = actions
        self._action_locations = action_locations
        self._period = period
        self._check_regularly = check_regularly
        self._paused = paused
        self._check_now = check_now
        self._show_working_popup = show_working_popup
        self._publish_files_to_popup_button = publish_files_to_popup_button
        self._publish_files_to_page = publish_files_to_page
        
    
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_IMPORT_FOLDER ] = ImportFolder