1189 lines
48 KiB
Python
1189 lines
48 KiB
Python
import collections
|
|
import os
|
|
import threading
|
|
import time
|
|
import typing
|
|
|
|
from hydrus.core import HydrusConstants as HC
|
|
from hydrus.core import HydrusData
|
|
from hydrus.core import HydrusExceptions
|
|
from hydrus.core import HydrusFileHandling
|
|
from hydrus.core import HydrusGlobals as HG
|
|
from hydrus.core import HydrusPaths
|
|
from hydrus.core import HydrusSerialisable
|
|
from hydrus.core import HydrusThreading
|
|
|
|
from hydrus.client import ClientConstants as CC
|
|
from hydrus.client import ClientData
|
|
from hydrus.client import ClientFiles
|
|
from hydrus.client import ClientPaths
|
|
from hydrus.client import ClientSearch
|
|
from hydrus.client import ClientThreading
|
|
from hydrus.client.importing import ClientImportControl
|
|
from hydrus.client.importing import ClientImporting
|
|
from hydrus.client.importing import ClientImportFileSeeds
|
|
from hydrus.client.importing.options import FileImportOptions
|
|
from hydrus.client.importing.options import TagImportOptions
|
|
from hydrus.client.metadata import ClientMetadataMigration
|
|
from hydrus.client.metadata import ClientMetadataMigrationExporters
|
|
from hydrus.client.metadata import ClientMetadataMigrationImporters
|
|
from hydrus.client.metadata import ClientTags
|
|
|
|
class HDDImport( HydrusSerialisable.SerialisableBase ):
|
|
|
|
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_HDD_IMPORT
|
|
SERIALISABLE_NAME = 'Local File Import'
|
|
SERIALISABLE_VERSION = 3
|
|
|
|
def __init__( self, paths = None, file_import_options = None, metadata_routers = None, paths_to_additional_service_keys_to_tags = None, delete_after_success = None ):
|
|
|
|
HydrusSerialisable.SerialisableBase.__init__( self )
|
|
|
|
if metadata_routers is None:
|
|
|
|
metadata_routers = []
|
|
|
|
|
|
if paths_to_additional_service_keys_to_tags is None:
|
|
|
|
paths_to_additional_service_keys_to_tags = collections.defaultdict( ClientTags.ServiceKeysToTags )
|
|
|
|
|
|
if delete_after_success is None:
|
|
|
|
delete_after_success = False
|
|
|
|
|
|
if paths is None:
|
|
|
|
self._file_seed_cache = None
|
|
|
|
else:
|
|
|
|
self._file_seed_cache = ClientImportFileSeeds.FileSeedCache()
|
|
|
|
file_seeds = []
|
|
|
|
for path in paths:
|
|
|
|
file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_HDD, path )
|
|
|
|
try:
|
|
|
|
file_modified_time = HydrusFileHandling.GetFileModifiedTimestamp( path )
|
|
|
|
file_seed.source_time = file_modified_time
|
|
|
|
except:
|
|
|
|
pass
|
|
|
|
|
|
if path in paths_to_additional_service_keys_to_tags:
|
|
|
|
file_seed.SetExternalAdditionalServiceKeysToTags( paths_to_additional_service_keys_to_tags[ path ] )
|
|
|
|
|
|
file_seeds.append( file_seed )
|
|
|
|
|
|
self._file_seed_cache.AddFileSeeds( file_seeds )
|
|
|
|
|
|
self._metadata_routers = HydrusSerialisable.SerialisableList( metadata_routers )
|
|
|
|
self._file_import_options = file_import_options
|
|
self._delete_after_success = delete_after_success
|
|
|
|
self._page_key = b'initialising page key'
|
|
|
|
self._files_status = ''
|
|
self._paused = False
|
|
|
|
self._lock = threading.Lock()
|
|
|
|
self._files_repeating_job = None
|
|
|
|
self._last_serialisable_change_timestamp = 0
|
|
|
|
HG.client_controller.sub( self, 'NotifyFileSeedsUpdated', 'file_seed_cache_file_seeds_updated' )
|
|
|
|
|
|
def _GetSerialisableInfo( self ):
|
|
|
|
serialisable_file_seed_cache = self._file_seed_cache.GetSerialisableTuple()
|
|
serialisable_options = self._file_import_options.GetSerialisableTuple()
|
|
serialisable_metadata_routers = self._metadata_routers.GetSerialisableTuple()
|
|
|
|
return ( serialisable_file_seed_cache, serialisable_options, serialisable_metadata_routers, self._delete_after_success, self._paused )
|
|
|
|
|
|
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
|
|
|
|
( serialisable_file_seed_cache, serialisable_options, serialisable_metadata_routers, self._delete_after_success, self._paused ) = serialisable_info
|
|
|
|
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
|
|
self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_options )
|
|
self._metadata_routers = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_metadata_routers )
|
|
|
|
|
|
def _SerialisableChangeMade( self ):
|
|
|
|
self._last_serialisable_change_timestamp = HydrusData.GetNow()
|
|
|
|
|
|
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
|
|
|
|
if version == 1:
|
|
|
|
( serialisable_file_seed_cache, serialisable_options, serialisable_paths_to_tags, delete_after_success, paused ) = old_serialisable_info
|
|
|
|
file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
|
|
|
|
paths_to_additional_service_keys_to_tags = { path : { bytes.fromhex( service_key ) : tags for ( service_key, tags ) in service_keys_to_tags.items() } for ( path, service_keys_to_tags ) in serialisable_paths_to_tags.items() }
|
|
|
|
for file_seed in file_seed_cache.GetFileSeeds():
|
|
|
|
path = file_seed.file_seed_data
|
|
|
|
if path in paths_to_additional_service_keys_to_tags:
|
|
|
|
file_seed.SetExternalAdditionalServiceKeysToTags( paths_to_additional_service_keys_to_tags[ path ] )
|
|
|
|
|
|
|
|
serialisable_file_seed_cache = file_seed_cache.GetSerialisableTuple()
|
|
|
|
new_serialisable_info = ( serialisable_file_seed_cache, serialisable_options, delete_after_success, paused )
|
|
|
|
return ( 2, new_serialisable_info )
|
|
|
|
|
|
if version == 2:
|
|
|
|
( serialisable_file_seed_cache, serialisable_options, delete_after_success, paused ) = old_serialisable_info
|
|
|
|
metadata_routers = HydrusSerialisable.SerialisableList()
|
|
|
|
serialisable_metadata_routers = metadata_routers.GetSerialisableTuple()
|
|
|
|
new_serialisable_info = ( serialisable_file_seed_cache, serialisable_options, serialisable_metadata_routers, delete_after_success, paused )
|
|
|
|
return ( 3, new_serialisable_info )
|
|
|
|
|
|
|
|
def _WorkOnFiles( self ):
|
|
|
|
file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
|
|
|
|
if file_seed is None:
|
|
|
|
return
|
|
|
|
|
|
path = file_seed.file_seed_data
|
|
|
|
with self._lock:
|
|
|
|
self._files_status = 'importing'
|
|
|
|
|
|
def status_hook( text ):
|
|
|
|
with self._lock:
|
|
|
|
self._files_status = ClientImportControl.NeatenStatusText( text )
|
|
|
|
|
|
|
|
file_seed.ImportPath( self._file_seed_cache, self._file_import_options, FileImportOptions.IMPORT_TYPE_LOUD, status_hook = status_hook )
|
|
|
|
if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:
|
|
|
|
if len( self._metadata_routers ) > 0:
|
|
|
|
hash = file_seed.GetHash()
|
|
|
|
media_result = HG.client_controller.Read( 'media_result', hash )
|
|
|
|
for router in self._metadata_routers:
|
|
|
|
try:
|
|
|
|
router.Work( media_result, file_seed.file_seed_data )
|
|
|
|
except Exception as e:
|
|
|
|
HydrusData.ShowText( 'Trying to run metadata routing on the file "{}" threw an error!'.format( file_seed.file_seed_data ) )
|
|
HydrusData.ShowException( e )
|
|
|
|
|
|
|
|
|
|
real_presentation_import_options = FileImportOptions.GetRealPresentationImportOptions( self._file_import_options, FileImportOptions.IMPORT_TYPE_LOUD )
|
|
|
|
if file_seed.ShouldPresent( real_presentation_import_options ):
|
|
|
|
file_seed.PresentToPage( self._page_key )
|
|
|
|
|
|
if self._delete_after_success:
|
|
|
|
try:
|
|
|
|
ClientPaths.DeletePath( path )
|
|
|
|
except Exception as e:
|
|
|
|
HydrusData.ShowText( 'While attempting to delete {}, the following error occurred:'.format( path ) )
|
|
HydrusData.ShowException( e )
|
|
|
|
|
|
possible_sidecar_paths = set()
|
|
|
|
for router in self._metadata_routers:
|
|
|
|
possible_sidecar_paths.update( router.GetPossibleImporterSidecarPaths( path ) )
|
|
|
|
|
|
for possible_sidecar_path in possible_sidecar_paths:
|
|
|
|
if os.path.exists( possible_sidecar_path ):
|
|
|
|
try:
|
|
|
|
ClientPaths.DeletePath( possible_sidecar_path )
|
|
|
|
except Exception as e:
|
|
|
|
HydrusData.ShowText( 'While attempting to delete {}, the following error occurred:'.format( possible_sidecar_path ) )
|
|
HydrusData.ShowException( e )
|
|
|
|
|
|
|
|
|
|
|
|
with self._lock:
|
|
|
|
self._files_status = ''
|
|
|
|
|
|
time.sleep( ClientImporting.DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )
|
|
|
|
|
|
def CurrentlyWorking( self ):
|
|
|
|
with self._lock:
|
|
|
|
work_to_do = self._file_seed_cache.WorkToDo()
|
|
|
|
return work_to_do and not self._paused
|
|
|
|
|
|
|
|
def GetAPIInfoDict( self, simple ):
|
|
|
|
with self._lock:
|
|
|
|
d = {}
|
|
|
|
d[ 'imports' ] = self._file_seed_cache.GetAPIInfoDict( simple )
|
|
|
|
d[ 'files_paused' ] = self._paused
|
|
|
|
return d
|
|
|
|
|
|
|
|
def GetFileImportOptions( self ):
|
|
|
|
with self._lock:
|
|
|
|
return self._file_import_options
|
|
|
|
|
|
|
|
def GetFileSeedCache( self ):
|
|
|
|
return self._file_seed_cache
|
|
|
|
|
|
def GetNumSeeds( self ):
|
|
|
|
with self._lock:
|
|
|
|
return len( self._file_seed_cache )
|
|
|
|
|
|
|
|
def GetStatus( self ):
|
|
|
|
with self._lock:
|
|
|
|
text = ClientImportControl.GenerateLiveStatusText( self._files_status, self._paused, 0, '' )
|
|
|
|
return ( text, self._paused )
|
|
|
|
|
|
|
|
def GetValueRange( self ):
|
|
|
|
with self._lock:
|
|
|
|
return self._file_seed_cache.GetValueRange()
|
|
|
|
|
|
|
|
def HasSerialisableChangesSince( self, since_timestamp ):
|
|
|
|
with self._lock:
|
|
|
|
return self._last_serialisable_change_timestamp > since_timestamp
|
|
|
|
|
|
|
|
def NotifyFileSeedsUpdated( self, file_seed_cache_key, file_seeds ):
|
|
|
|
if file_seed_cache_key == self._file_seed_cache.GetFileSeedCacheKey():
|
|
|
|
ClientImporting.WakeRepeatingJob( self._files_repeating_job )
|
|
|
|
self._SerialisableChangeMade()
|
|
|
|
|
|
|
|
def PausePlay( self ):
|
|
|
|
with self._lock:
|
|
|
|
self._paused = not self._paused
|
|
|
|
ClientImporting.WakeRepeatingJob( self._files_repeating_job )
|
|
|
|
self._SerialisableChangeMade()
|
|
|
|
|
|
|
|
def SetFileImportOptions( self, file_import_options: FileImportOptions.FileImportOptions ):
|
|
|
|
with self._lock:
|
|
|
|
if file_import_options.DumpToString() != self._file_import_options.DumpToString():
|
|
|
|
self._file_import_options = file_import_options
|
|
|
|
self._SerialisableChangeMade()
|
|
|
|
|
|
|
|
|
|
def Start( self, page_key ):
|
|
|
|
self._page_key = page_key
|
|
|
|
self._files_repeating_job = HG.client_controller.CallRepeating( ClientImporting.GetRepeatingJobInitialDelay(), ClientImporting.REPEATING_JOB_TYPICAL_PERIOD, self.REPEATINGWorkOnFiles )
|
|
|
|
self._files_repeating_job.SetThreadSlotType( 'misc' )
|
|
|
|
|
|
def CheckCanDoFileWork( self ):
|
|
|
|
with self._lock:
|
|
|
|
try:
|
|
|
|
ClientImportControl.CheckImporterCanDoWorkBecauseStopped( self._page_key )
|
|
|
|
except HydrusExceptions.VetoException:
|
|
|
|
self._files_repeating_job.Cancel()
|
|
|
|
raise
|
|
|
|
|
|
ClientImportControl.CheckImporterCanDoFileWorkBecausePaused( self._paused, self._file_seed_cache, self._page_key )
|
|
|
|
|
|
return True
|
|
|
|
|
|
def REPEATINGWorkOnFiles( self ):
|
|
|
|
while True:
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
self.CheckCanDoFileWork()
|
|
|
|
except HydrusExceptions.VetoException as e:
|
|
|
|
with self._lock:
|
|
|
|
self._files_status = str( e )
|
|
|
|
|
|
break
|
|
|
|
|
|
self._WorkOnFiles()
|
|
|
|
HG.client_controller.WaitUntilViewFree()
|
|
|
|
self._SerialisableChangeMade()
|
|
|
|
except Exception as e:
|
|
|
|
with self._lock:
|
|
|
|
self._files_status = 'stopping work: {}'.format( str( e ) )
|
|
|
|
|
|
HydrusData.ShowException( e )
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_HDD_IMPORT ] = HDDImport
|
|
|
|
class ImportFolder( HydrusSerialisable.SerialisableBaseNamed ):
|
|
|
|
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_IMPORT_FOLDER
|
|
SERIALISABLE_NAME = 'Import Folder'
|
|
SERIALISABLE_VERSION = 8
|
|
|
|
def __init__(
|
|
self,
|
|
name,
|
|
path = '',
|
|
file_import_options = None,
|
|
tag_import_options = None,
|
|
metadata_routers: typing.Optional[ typing.Collection[ ClientMetadataMigration.SingleFileMetadataRouter ] ] = None,
|
|
tag_service_keys_to_filename_tagging_options = None,
|
|
actions = None,
|
|
action_locations = None,
|
|
period = 3600,
|
|
check_regularly = True,
|
|
show_working_popup = True,
|
|
publish_files_to_popup_button = True,
|
|
publish_files_to_page = False
|
|
):
|
|
|
|
if file_import_options is None:
|
|
|
|
file_import_options = FileImportOptions.FileImportOptions()
|
|
file_import_options.SetIsDefault( True )
|
|
|
|
|
|
if tag_import_options is None:
|
|
|
|
tag_import_options = TagImportOptions.TagImportOptions()
|
|
|
|
|
|
if metadata_routers is None:
|
|
|
|
metadata_routers = []
|
|
|
|
|
|
metadata_routers = HydrusSerialisable.SerialisableList( metadata_routers )
|
|
|
|
if tag_service_keys_to_filename_tagging_options is None:
|
|
|
|
tag_service_keys_to_filename_tagging_options = {}
|
|
|
|
|
|
if actions is None:
|
|
|
|
actions = {}
|
|
|
|
actions[ CC.STATUS_SUCCESSFUL_AND_NEW ] = CC.IMPORT_FOLDER_IGNORE
|
|
actions[ CC.STATUS_SUCCESSFUL_BUT_REDUNDANT ] = CC.IMPORT_FOLDER_IGNORE
|
|
actions[ CC.STATUS_DELETED ] = CC.IMPORT_FOLDER_IGNORE
|
|
actions[ CC.STATUS_ERROR ] = CC.IMPORT_FOLDER_IGNORE
|
|
|
|
|
|
if action_locations is None:
|
|
|
|
action_locations = {}
|
|
|
|
|
|
HydrusSerialisable.SerialisableBaseNamed.__init__( self, name )
|
|
|
|
self._path = path
|
|
self._file_import_options = file_import_options
|
|
self._tag_import_options = tag_import_options
|
|
self._metadata_routers = metadata_routers
|
|
self._tag_service_keys_to_filename_tagging_options = tag_service_keys_to_filename_tagging_options
|
|
self._actions = actions
|
|
self._action_locations = action_locations
|
|
self._period = period
|
|
self._check_regularly = check_regularly
|
|
|
|
self._file_seed_cache = ClientImportFileSeeds.FileSeedCache()
|
|
self._last_checked = 0
|
|
self._paused = False
|
|
self._check_now = False
|
|
|
|
self._show_working_popup = show_working_popup
|
|
self._publish_files_to_popup_button = publish_files_to_popup_button
|
|
self._publish_files_to_page = publish_files_to_page
|
|
|
|
|
|
def _ActionPaths( self ):
|
|
|
|
for status in ( CC.STATUS_SUCCESSFUL_AND_NEW, CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, CC.STATUS_DELETED, CC.STATUS_ERROR ):
|
|
|
|
action = self._actions[ status ]
|
|
|
|
if action == CC.IMPORT_FOLDER_DELETE:
|
|
|
|
while True:
|
|
|
|
file_seed = self._file_seed_cache.GetNextFileSeed( status )
|
|
|
|
if file_seed is None or HG.started_shutdown:
|
|
|
|
break
|
|
|
|
|
|
path = file_seed.file_seed_data
|
|
|
|
try:
|
|
|
|
if os.path.exists( path ) and not os.path.isdir( path ):
|
|
|
|
ClientPaths.DeletePath( path )
|
|
|
|
|
|
possible_sidecar_paths = set()
|
|
|
|
for router in self._metadata_routers:
|
|
|
|
possible_sidecar_paths.update( router.GetPossibleImporterSidecarPaths( path ) )
|
|
|
|
|
|
for possible_sidecar_path in possible_sidecar_paths:
|
|
|
|
if os.path.exists( possible_sidecar_path ):
|
|
|
|
ClientPaths.DeletePath( possible_sidecar_path )
|
|
|
|
|
|
|
|
self._file_seed_cache.RemoveFileSeeds( ( file_seed, ) )
|
|
|
|
except Exception as e:
|
|
|
|
raise Exception( 'Tried to delete "{}", but could not.'.format( path ) )
|
|
|
|
|
|
|
|
elif action == CC.IMPORT_FOLDER_MOVE:
|
|
|
|
while True:
|
|
|
|
file_seed = self._file_seed_cache.GetNextFileSeed( status )
|
|
|
|
if file_seed is None or HG.started_shutdown:
|
|
|
|
break
|
|
|
|
|
|
path = file_seed.file_seed_data
|
|
|
|
try:
|
|
|
|
dest_dir = self._action_locations[ status ]
|
|
|
|
if not os.path.exists( dest_dir ):
|
|
|
|
raise Exception( 'Tried to move "{}" to "{}", but the destination directory did not exist.'.format( path, dest_dir ) )
|
|
|
|
|
|
if os.path.exists( path ) and not os.path.isdir( path ):
|
|
|
|
filename = os.path.basename( path )
|
|
|
|
dest_path = os.path.join( dest_dir, filename )
|
|
|
|
dest_path = HydrusPaths.AppendPathUntilNoConflicts( dest_path )
|
|
|
|
HydrusPaths.MergeFile( path, dest_path )
|
|
|
|
|
|
txt_path = path + '.txt'
|
|
|
|
if os.path.exists( txt_path ):
|
|
|
|
txt_filename = os.path.basename( txt_path )
|
|
|
|
txt_dest_path = os.path.join( dest_dir, txt_filename )
|
|
|
|
txt_dest_path = HydrusPaths.AppendPathUntilNoConflicts( txt_dest_path )
|
|
|
|
HydrusPaths.MergeFile( txt_path, txt_dest_path )
|
|
|
|
|
|
self._file_seed_cache.RemoveFileSeeds( ( file_seed, ) )
|
|
|
|
except Exception as e:
|
|
|
|
HydrusData.ShowText( 'Import folder tried to move ' + path + ', but could not:' )
|
|
|
|
HydrusData.ShowException( e )
|
|
|
|
HydrusData.ShowText( 'Import folder has been paused.' )
|
|
|
|
self._paused = True
|
|
|
|
return
|
|
|
|
|
|
|
|
elif status == CC.IMPORT_FOLDER_IGNORE:
|
|
|
|
file_seeds = self._file_seed_cache.GetFileSeeds( status )
|
|
|
|
for file_seed in file_seeds:
|
|
|
|
path = file_seed.file_seed_data
|
|
|
|
try:
|
|
|
|
if not os.path.exists( path ):
|
|
|
|
self._file_seed_cache.RemoveFileSeeds( ( file_seed, ) )
|
|
|
|
|
|
except Exception as e:
|
|
|
|
raise Exception( 'Tried to check existence of "{}", but could not.'.format( path ) )
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _CheckFolder( self, job_key ):
|
|
|
|
( all_paths, num_sidecars ) = ClientFiles.GetAllFilePaths( [ self._path ] )
|
|
|
|
all_paths = HydrusPaths.FilterFreePaths( all_paths )
|
|
|
|
file_seeds = []
|
|
|
|
for path in all_paths:
|
|
|
|
if job_key.IsCancelled():
|
|
|
|
break
|
|
|
|
|
|
file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_HDD, path )
|
|
|
|
if not self._file_seed_cache.HasFileSeed( file_seed ):
|
|
|
|
file_seeds.append( file_seed )
|
|
|
|
|
|
job_key.SetVariable( 'popup_text_1', 'checking: found ' + HydrusData.ToHumanInt( len( file_seeds ) ) + ' new files' )
|
|
|
|
|
|
self._file_seed_cache.AddFileSeeds( file_seeds )
|
|
|
|
self._last_checked = HydrusData.GetNow()
|
|
self._check_now = False
|
|
|
|
|
|
def _GetSerialisableInfo( self ):
|
|
|
|
serialisable_file_import_options = self._file_import_options.GetSerialisableTuple()
|
|
serialisable_tag_import_options = self._tag_import_options.GetSerialisableTuple()
|
|
serialisable_metadata_routers = self._metadata_routers.GetSerialisableTuple()
|
|
serialisable_tag_service_keys_to_filename_tagging_options = [ ( service_key.hex(), filename_tagging_options.GetSerialisableTuple() ) for ( service_key, filename_tagging_options ) in list(self._tag_service_keys_to_filename_tagging_options.items()) ]
|
|
serialisable_file_seed_cache = self._file_seed_cache.GetSerialisableTuple()
|
|
|
|
# json turns int dict keys to strings
|
|
action_pairs = list(self._actions.items())
|
|
action_location_pairs = list(self._action_locations.items())
|
|
|
|
return ( self._path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_metadata_routers, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, self._period, self._check_regularly, serialisable_file_seed_cache, self._last_checked, self._paused, self._check_now, self._show_working_popup, self._publish_files_to_popup_button, self._publish_files_to_page )
|
|
|
|
|
|
def _ImportFiles( self, job_key ):
|
|
|
|
did_work = False
|
|
|
|
time_to_save = HydrusData.GetNow() + 600
|
|
|
|
num_files_imported = 0
|
|
presentation_hashes = []
|
|
presentation_hashes_fast = set()
|
|
|
|
i = 0
|
|
|
|
# don't want to start at 23/100 because of carrying over failed results or whatever
|
|
# num_to_do is num currently unknown
|
|
num_total = self._file_seed_cache.GetFileSeedCount( CC.STATUS_UNKNOWN )
|
|
|
|
while True:
|
|
|
|
file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
|
|
|
|
p1 = HG.client_controller.new_options.GetBoolean( 'pause_import_folders_sync' ) or self._paused
|
|
p2 = HydrusThreading.IsThreadShuttingDown()
|
|
p3 = job_key.IsCancelled()
|
|
|
|
if file_seed is None or p1 or p2 or p3:
|
|
|
|
break
|
|
|
|
|
|
did_work = True
|
|
|
|
if HydrusData.TimeHasPassed( time_to_save ):
|
|
|
|
HG.client_controller.WriteSynchronous( 'serialisable', self )
|
|
|
|
time_to_save = HydrusData.GetNow() + 600
|
|
|
|
|
|
gauge_num_done = num_files_imported + 1
|
|
|
|
job_key.SetVariable( 'popup_text_1', 'importing file ' + HydrusData.ConvertValueRangeToPrettyString( gauge_num_done, num_total ) )
|
|
job_key.SetVariable( 'popup_gauge_1', ( gauge_num_done, num_total ) )
|
|
|
|
path = file_seed.file_seed_data
|
|
|
|
file_seed.ImportPath( self._file_seed_cache, self._file_import_options, FileImportOptions.IMPORT_TYPE_QUIET )
|
|
|
|
if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:
|
|
|
|
hash = None
|
|
|
|
if file_seed.HasHash():
|
|
|
|
hash = file_seed.GetHash()
|
|
|
|
if self._tag_import_options.HasAdditionalTags() or len( self._metadata_routers ) > 0:
|
|
|
|
media_result = HG.client_controller.Read( 'media_result', hash )
|
|
|
|
if self._tag_import_options.HasAdditionalTags():
|
|
|
|
downloaded_tags = []
|
|
|
|
service_keys_to_content_updates = self._tag_import_options.GetServiceKeysToContentUpdates( file_seed.status, media_result, downloaded_tags ) # additional tags
|
|
|
|
if len( service_keys_to_content_updates ) > 0:
|
|
|
|
HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
|
|
|
|
|
|
|
|
for metadata_router in self._metadata_routers:
|
|
|
|
try:
|
|
|
|
metadata_router.Work( media_result, path )
|
|
|
|
except Exception as e:
|
|
|
|
HydrusData.ShowText( 'Trying to run metadata routing in the import folder "' + self._name + '" threw an error!' )
|
|
|
|
HydrusData.ShowException( e )
|
|
|
|
|
|
|
|
|
|
service_keys_to_tags = ClientTags.ServiceKeysToTags()
|
|
|
|
for ( tag_service_key, filename_tagging_options ) in self._tag_service_keys_to_filename_tagging_options.items():
|
|
|
|
if not HG.client_controller.services_manager.ServiceExists( tag_service_key ):
|
|
|
|
continue
|
|
|
|
|
|
try:
|
|
|
|
tags = filename_tagging_options.GetTags( tag_service_key, path )
|
|
|
|
if len( tags ) > 0:
|
|
|
|
service_keys_to_tags[ tag_service_key ] = tags
|
|
|
|
|
|
except Exception as e:
|
|
|
|
HydrusData.ShowText( 'Trying to parse filename tags in the import folder "' + self._name + '" threw an error!' )
|
|
|
|
HydrusData.ShowException( e )
|
|
|
|
|
|
|
|
if len( service_keys_to_tags ) > 0:
|
|
|
|
service_keys_to_content_updates = ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates( { hash }, service_keys_to_tags )
|
|
|
|
HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
|
|
|
|
|
|
|
|
num_files_imported += 1
|
|
|
|
if hash not in presentation_hashes_fast:
|
|
|
|
real_presentation_import_options = FileImportOptions.GetRealPresentationImportOptions( self._file_import_options, FileImportOptions.IMPORT_TYPE_LOUD )
|
|
|
|
if file_seed.ShouldPresent( real_presentation_import_options ):
|
|
|
|
presentation_hashes.append( hash )
|
|
|
|
presentation_hashes_fast.add( hash )
|
|
|
|
|
|
|
|
elif file_seed.status == CC.STATUS_ERROR:
|
|
|
|
HydrusData.Print( 'A file failed to import from import folder ' + self._name + ':' + path )
|
|
|
|
|
|
i += 1
|
|
|
|
if i % 10 == 0:
|
|
|
|
self._ActionPaths()
|
|
|
|
|
|
|
|
if num_files_imported > 0:
|
|
|
|
HydrusData.Print( 'Import folder ' + self._name + ' imported ' + HydrusData.ToHumanInt( num_files_imported ) + ' files.' )
|
|
|
|
if len( presentation_hashes ) > 0:
|
|
|
|
ClientImporting.PublishPresentationHashes( self._name, presentation_hashes, self._publish_files_to_popup_button, self._publish_files_to_page )
|
|
|
|
|
|
|
|
self._ActionPaths()
|
|
|
|
return did_work
|
|
|
|
|
|
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
|
|
|
|
( self._path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_metadata_routers, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, self._period, self._check_regularly, serialisable_file_seed_cache, self._last_checked, self._paused, self._check_now, self._show_working_popup, self._publish_files_to_popup_button, self._publish_files_to_page ) = serialisable_info
|
|
|
|
self._actions = dict( action_pairs )
|
|
self._action_locations = dict( action_location_pairs )
|
|
|
|
self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_import_options )
|
|
self._tag_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_import_options )
|
|
self._metadata_routers = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_metadata_routers )
|
|
self._tag_service_keys_to_filename_tagging_options = dict( [ ( bytes.fromhex( encoded_service_key ), HydrusSerialisable.CreateFromSerialisableTuple( serialisable_filename_tagging_options ) ) for ( encoded_service_key, serialisable_filename_tagging_options ) in serialisable_tag_service_keys_to_filename_tagging_options ] )
|
|
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
|
|
|
|
|
|
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
|
|
|
|
if version == 1:
|
|
|
|
( path, mimes, serialisable_file_import_options, action_pairs, action_location_pairs, period, open_popup, tag, serialisable_file_seed_cache, last_checked, paused ) = old_serialisable_info
|
|
|
|
# edited out tag carry-over to tio due to bit rot
|
|
|
|
tag_import_options = TagImportOptions.TagImportOptions()
|
|
|
|
serialisable_tag_import_options = tag_import_options.GetSerialisableTuple()
|
|
|
|
new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused )
|
|
|
|
return ( 2, new_serialisable_info )
|
|
|
|
|
|
if version == 2:
|
|
|
|
( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused ) = old_serialisable_info
|
|
|
|
serialisable_txt_parse_tag_service_keys = []
|
|
|
|
new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_txt_parse_tag_service_keys, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused )
|
|
|
|
return ( 3, new_serialisable_info )
|
|
|
|
|
|
if version == 3:
|
|
|
|
( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_txt_parse_tag_service_keys, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused ) = old_serialisable_info
|
|
|
|
check_now = False
|
|
|
|
new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_txt_parse_tag_service_keys, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused, check_now )
|
|
|
|
return ( 4, new_serialisable_info )
|
|
|
|
|
|
if version == 4:
|
|
|
|
( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_txt_parse_tag_service_keys, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused, check_now ) = old_serialisable_info
|
|
|
|
txt_parse_tag_service_keys = [ bytes.fromhex( service_key ) for service_key in serialisable_txt_parse_tag_service_keys ]
|
|
|
|
tag_service_keys_to_filename_tagging_options = {}
|
|
|
|
for service_key in txt_parse_tag_service_keys:
|
|
|
|
filename_tagging_options = TagImportOptions.FilenameTaggingOptions()
|
|
|
|
filename_tagging_options._load_from_neighbouring_txt_files = True
|
|
|
|
tag_service_keys_to_filename_tagging_options[ service_key ] = filename_tagging_options
|
|
|
|
|
|
serialisable_tag_service_keys_to_filename_tagging_options = [ ( service_key.hex(), filename_tagging_options.GetSerialisableTuple() ) for ( service_key, filename_tagging_options ) in list(tag_service_keys_to_filename_tagging_options.items()) ]
|
|
|
|
new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused, check_now )
|
|
|
|
return ( 5, new_serialisable_info )
|
|
|
|
|
|
if version == 5:
|
|
|
|
( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused, check_now ) = old_serialisable_info
|
|
|
|
check_regularly = not paused
|
|
show_working_popup = True
|
|
publish_files_to_page = False
|
|
publish_files_to_popup_button = open_popup
|
|
|
|
new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, check_regularly, serialisable_file_seed_cache, last_checked, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page )
|
|
|
|
return ( 6, new_serialisable_info )
|
|
|
|
|
|
if version == 6:
|
|
|
|
( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, check_regularly, serialisable_file_seed_cache, last_checked, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page ) = old_serialisable_info
|
|
|
|
file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_import_options )
|
|
|
|
file_import_options.SetAllowedSpecificFiletypes( mimes )
|
|
|
|
serialisable_file_import_options = file_import_options.GetSerialisableTuple()
|
|
|
|
new_serialisable_info = ( path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, check_regularly, serialisable_file_seed_cache, last_checked, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page )
|
|
|
|
return ( 7, new_serialisable_info )
|
|
|
|
|
|
if version == 7:
|
|
|
|
( path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, check_regularly, serialisable_file_seed_cache, last_checked, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page ) = old_serialisable_info
|
|
|
|
tag_service_keys_to_filename_tagging_options = dict( [ ( bytes.fromhex( encoded_service_key ), HydrusSerialisable.CreateFromSerialisableTuple( serialisable_filename_tagging_options ) ) for ( encoded_service_key, serialisable_filename_tagging_options ) in serialisable_tag_service_keys_to_filename_tagging_options ] )
|
|
|
|
metadata_routers = HydrusSerialisable.SerialisableList()
|
|
|
|
try:
|
|
|
|
for ( service_key, filename_tagging_options ) in tag_service_keys_to_filename_tagging_options.items():
|
|
|
|
# beardy access here, but this is once off
|
|
if hasattr( filename_tagging_options, '_load_from_neighbouring_txt_files' ) and filename_tagging_options._load_from_neighbouring_txt_files:
|
|
|
|
importers = [ ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT() ]
|
|
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags( service_key = service_key )
|
|
|
|
metadata_router = ClientMetadataMigration.SingleFileMetadataRouter( importers = importers, exporter = exporter )
|
|
|
|
metadata_routers.append( metadata_router )
|
|
|
|
|
|
|
|
except Exception as e:
|
|
|
|
HydrusData.Print( 'Failed to update import folder with new metadata routers.' )
|
|
|
|
HydrusData.PrintException( e )
|
|
|
|
|
|
serialisable_metadata_routers = metadata_routers.GetSerialisableTuple()
|
|
|
|
new_serialisable_info = ( path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_metadata_routers, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, check_regularly, serialisable_file_seed_cache, last_checked, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page )
|
|
|
|
return ( 8, new_serialisable_info )
|
|
|
|
|
|
|
|
def CheckNow( self ):
|
|
|
|
self._paused = False
|
|
self._check_now = True
|
|
|
|
|
|
def DoWork( self ):
|
|
|
|
if HG.started_shutdown:
|
|
|
|
return
|
|
|
|
|
|
if HG.client_controller.new_options.GetBoolean( 'pause_import_folders_sync' ) or self._paused:
|
|
|
|
return
|
|
|
|
|
|
checked_folder = False
|
|
|
|
did_import_file_work = False
|
|
|
|
error_occured = False
|
|
|
|
stop_time = HydrusData.GetNow() + 3600
|
|
|
|
job_key = ClientThreading.JobKey( pausable = False, cancellable = True, stop_time = stop_time )
|
|
|
|
try:
|
|
|
|
real_file_import_options = FileImportOptions.GetRealFileImportOptions( self._file_import_options, FileImportOptions.IMPORT_TYPE_QUIET )
|
|
|
|
real_file_import_options.CheckReadyToImport()
|
|
|
|
if not os.path.exists( self._path ) or not os.path.isdir( self._path ):
|
|
|
|
raise Exception( 'Path "' + self._path + '" does not seem to exist, or is not a directory.' )
|
|
|
|
|
|
pubbed_job_key = False
|
|
|
|
job_key.SetStatusTitle( 'import folder - ' + self._name )
|
|
|
|
due_by_check_now = self._check_now
|
|
due_by_period = self._check_regularly and HydrusData.TimeHasPassed( self._last_checked + self._period )
|
|
|
|
if due_by_check_now or due_by_period:
|
|
|
|
if not pubbed_job_key and self._show_working_popup:
|
|
|
|
HG.client_controller.pub( 'message', job_key )
|
|
|
|
pubbed_job_key = True
|
|
|
|
|
|
self._CheckFolder( job_key )
|
|
|
|
checked_folder = True
|
|
|
|
|
|
file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
|
|
|
|
if file_seed is not None:
|
|
|
|
if not pubbed_job_key and self._show_working_popup:
|
|
|
|
HG.client_controller.pub( 'message', job_key )
|
|
|
|
pubbed_job_key = True
|
|
|
|
|
|
did_import_file_work = self._ImportFiles( job_key )
|
|
|
|
|
|
except Exception as e:
|
|
|
|
error_occured = True
|
|
self._paused = True
|
|
|
|
HydrusData.ShowText( 'The import folder "' + self._name + '" encountered an exception! It has been paused!' )
|
|
HydrusData.ShowException( e )
|
|
|
|
|
|
if checked_folder or did_import_file_work or error_occured:
|
|
|
|
HG.client_controller.WriteSynchronous( 'serialisable', self )
|
|
|
|
|
|
job_key.Delete()
|
|
|
|
|
|
def GetFileSeedCache( self ):
|
|
|
|
return self._file_seed_cache
|
|
|
|
|
|
def GetMetadataRouters( self ):
|
|
|
|
return list( self._metadata_routers )
|
|
|
|
|
|
def Paused( self ):
|
|
|
|
return self._paused
|
|
|
|
|
|
def PausePlay( self ):
|
|
|
|
self._paused = not self._paused
|
|
|
|
|
|
def ToListBoxTuple( self ):
|
|
|
|
return ( self._name, self._path, self._paused, self._check_regularly, self._period )
|
|
|
|
|
|
def ToTuple( self ):
|
|
|
|
return ( self._name, self._path, self._file_import_options, self._tag_import_options, self._tag_service_keys_to_filename_tagging_options, self._actions, self._action_locations, self._period, self._check_regularly, self._paused, self._check_now, self._show_working_popup, self._publish_files_to_popup_button, self._publish_files_to_page )
|
|
|
|
|
|
def SetFileSeedCache( self, file_seed_cache ):
|
|
|
|
self._file_seed_cache = file_seed_cache
|
|
|
|
|
|
def SetMetadataRouters( self, metadata_routers: typing.Collection[ ClientMetadataMigration.SingleFileMetadataRouter ] ):
|
|
|
|
self._metadata_routers = HydrusSerialisable.SerialisableList( metadata_routers )
|
|
|
|
|
|
def SetTuple( self, name, path, file_import_options, tag_import_options, tag_service_keys_to_filename_tagging_options, actions, action_locations, period, check_regularly, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page ):
|
|
|
|
if path != self._path:
|
|
|
|
self._file_seed_cache = ClientImportFileSeeds.FileSeedCache()
|
|
|
|
|
|
if not file_import_options.IsDefault() and not self._file_import_options.IsDefault():
|
|
|
|
mimes = set( file_import_options.GetAllowedSpecificFiletypes() )
|
|
|
|
if mimes != set( self._file_import_options.GetAllowedSpecificFiletypes() ):
|
|
|
|
self._file_seed_cache.RemoveFileSeedsByStatus( ( CC.STATUS_VETOED, ) )
|
|
|
|
|
|
|
|
self._name = name
|
|
self._path = path
|
|
self._file_import_options = file_import_options
|
|
self._tag_import_options = tag_import_options
|
|
self._tag_service_keys_to_filename_tagging_options = tag_service_keys_to_filename_tagging_options
|
|
self._actions = actions
|
|
self._action_locations = action_locations
|
|
self._period = period
|
|
self._check_regularly = check_regularly
|
|
self._paused = paused
|
|
self._check_now = check_now
|
|
self._show_working_popup = show_working_popup
|
|
self._publish_files_to_popup_button = publish_files_to_popup_button
|
|
self._publish_files_to_page = publish_files_to_page
|
|
|
|
|
|
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_IMPORT_FOLDER ] = ImportFolder
|