454 lines
16 KiB
Python
454 lines
16 KiB
Python
import ClientConstants as CC
|
|
import ClientPaths
|
|
import ClientSearch
|
|
import HydrusConstants as HC
|
|
import HydrusData
|
|
import HydrusGlobals as HG
|
|
import HydrusPaths
|
|
import HydrusSerialisable
|
|
import HydrusTags
|
|
import HydrusThreading
|
|
import os
|
|
import re
|
|
import stat
|
|
|
|
MAX_PATH_LENGTH = 245 # bit of padding from 255 for .txt neigbouring and other surprises
|
|
|
|
def GenerateExportFilename( destination_directory, media, terms ):
|
|
|
|
def clean_tag_text( t ):
|
|
|
|
if HC.PLATFORM_WINDOWS:
|
|
|
|
t = re.sub( r'\\', '_', t, flags = re.UNICODE )
|
|
|
|
else:
|
|
|
|
t = re.sub( '/', '_', t, flags = re.UNICODE )
|
|
|
|
|
|
return t
|
|
|
|
|
|
if len( destination_directory ) > ( MAX_PATH_LENGTH - 10 ):
|
|
|
|
raise Exception( 'The destination directory is too long!' )
|
|
|
|
|
|
filename = ''
|
|
|
|
for ( term_type, term ) in terms:
|
|
|
|
tags_manager = media.GetTagsManager()
|
|
|
|
if term_type == 'string':
|
|
|
|
filename += term
|
|
|
|
elif term_type == 'namespace':
|
|
|
|
tags = tags_manager.GetNamespaceSlice( ( term, ) )
|
|
|
|
subtags = [ HydrusTags.SplitTag( tag )[1] for tag in tags ]
|
|
|
|
subtags.sort()
|
|
|
|
filename += clean_tag_text( ', '.join( subtags ) )
|
|
|
|
elif term_type == 'predicate':
|
|
|
|
if term in ( 'tags', 'nn tags' ):
|
|
|
|
current = tags_manager.GetCurrent()
|
|
pending = tags_manager.GetPending()
|
|
|
|
tags = list( current.union( pending ) )
|
|
|
|
if term == 'nn tags':
|
|
|
|
tags = [ tag for tag in tags if ':' not in tag ]
|
|
|
|
else:
|
|
|
|
tags = [ HydrusTags.SplitTag( tag )[1] for tag in tags ]
|
|
|
|
|
|
tags.sort()
|
|
|
|
filename += clean_tag_text( ', '.join( tags ) )
|
|
|
|
elif term == 'hash':
|
|
|
|
hash = media.GetHash()
|
|
|
|
filename += hash.encode( 'hex' )
|
|
|
|
|
|
elif term_type == 'tag':
|
|
|
|
tag = term
|
|
|
|
( namespace, subtag ) = HydrusTags.SplitTag( tag )
|
|
|
|
if tags_manager.HasTag( subtag ):
|
|
|
|
filename += clean_tag_text( subtag )
|
|
|
|
|
|
|
|
|
|
if HC.PLATFORM_WINDOWS:
|
|
|
|
# replace many consecutive backspace with single backspace
|
|
filename = re.sub( r'\\+', r'\\', filename, flags = re.UNICODE )
|
|
|
|
# /, :, *, ?, ", <, >, |
|
|
filename = re.sub( r'/|:|\*|\?|"|<|>|\|', '_', filename, flags = re.UNICODE )
|
|
|
|
else:
|
|
|
|
filename = re.sub( '/', '_', filename, flags = re.UNICODE )
|
|
|
|
|
|
#
|
|
|
|
mime = media.GetMime()
|
|
|
|
ext = HC.mime_ext_lookup[ mime ]
|
|
|
|
if filename.endswith( ext ):
|
|
|
|
filename = filename[ : - len( ext ) ]
|
|
|
|
|
|
example_dest_path = os.path.join( destination_directory, filename + ext )
|
|
|
|
excess_chars = len( example_dest_path ) - MAX_PATH_LENGTH
|
|
|
|
if excess_chars > 0:
|
|
|
|
filename = filename[ : - excess_chars ]
|
|
|
|
|
|
filename = filename + ext
|
|
|
|
return filename
|
|
|
|
def GetExportPath():
|
|
|
|
portable_path = HG.client_controller.options[ 'export_path' ]
|
|
|
|
if portable_path is None:
|
|
|
|
path = os.path.join( os.path.expanduser( '~' ), 'hydrus_export' )
|
|
|
|
HydrusPaths.MakeSureDirectoryExists( path )
|
|
|
|
else:
|
|
|
|
path = HydrusPaths.ConvertPortablePathToAbsPath( portable_path )
|
|
|
|
|
|
return path
|
|
|
|
def ParseExportPhrase( phrase ):
|
|
|
|
try:
|
|
|
|
terms = [ ( 'string', phrase ) ]
|
|
|
|
new_terms = []
|
|
|
|
for ( term_type, term ) in terms:
|
|
|
|
if term_type == 'string':
|
|
|
|
while '[' in term:
|
|
|
|
( pre, term ) = term.split( '[', 1 )
|
|
|
|
( namespace, term ) = term.split( ']', 1 )
|
|
|
|
new_terms.append( ( 'string', pre ) )
|
|
new_terms.append( ( 'namespace', namespace ) )
|
|
|
|
|
|
|
|
new_terms.append( ( term_type, term ) )
|
|
|
|
|
|
terms = new_terms
|
|
|
|
new_terms = []
|
|
|
|
for ( term_type, term ) in terms:
|
|
|
|
if term_type == 'string':
|
|
|
|
while '{' in term:
|
|
|
|
( pre, term ) = term.split( '{', 1 )
|
|
|
|
( predicate, term ) = term.split( '}', 1 )
|
|
|
|
new_terms.append( ( 'string', pre ) )
|
|
new_terms.append( ( 'predicate', predicate ) )
|
|
|
|
|
|
|
|
new_terms.append( ( term_type, term ) )
|
|
|
|
|
|
terms = new_terms
|
|
|
|
new_terms = []
|
|
|
|
for ( term_type, term ) in terms:
|
|
|
|
if term_type == 'string':
|
|
|
|
while '(' in term:
|
|
|
|
( pre, term ) = term.split( '(', 1 )
|
|
|
|
( tag, term ) = term.split( ')', 1 )
|
|
|
|
new_terms.append( ( 'string', pre ) )
|
|
new_terms.append( ( 'tag', tag ) )
|
|
|
|
|
|
|
|
new_terms.append( ( term_type, term ) )
|
|
|
|
|
|
terms = new_terms
|
|
|
|
except Exception as e:
|
|
|
|
raise Exception( 'Could not parse that phrase: ' + HydrusData.ToUnicode( e ) )
|
|
|
|
|
|
return terms
|
|
|
|
|
|
class ExportFolder( HydrusSerialisable.SerialisableBaseNamed ):
|
|
|
|
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_EXPORT_FOLDER
|
|
SERIALISABLE_NAME = 'Export Folder'
|
|
SERIALISABLE_VERSION = 3
|
|
|
|
def __init__( self, name, path = '', export_type = HC.EXPORT_FOLDER_TYPE_REGULAR, delete_from_client_after_export = False, file_search_context = None, period = 3600, phrase = None ):
|
|
|
|
HydrusSerialisable.SerialisableBaseNamed.__init__( self, name )
|
|
|
|
if export_type == HC.EXPORT_FOLDER_TYPE_SYNCHRONISE:
|
|
|
|
delete_from_client_after_export = False
|
|
|
|
|
|
if file_search_context is None:
|
|
|
|
file_search_context = ClientSearch.FileSearchContext( file_service_key = CC.LOCAL_FILE_SERVICE_KEY )
|
|
|
|
|
|
if phrase is None:
|
|
|
|
phrase = HG.client_controller.new_options.GetString( 'export_phrase' )
|
|
|
|
|
|
self._path = path
|
|
self._export_type = export_type
|
|
self._delete_from_client_after_export = delete_from_client_after_export
|
|
self._file_search_context = file_search_context
|
|
self._period = period
|
|
self._phrase = phrase
|
|
self._last_checked = 0
|
|
|
|
|
|
def _GetSerialisableInfo( self ):
|
|
|
|
serialisable_file_search_context = self._file_search_context.GetSerialisableTuple()
|
|
|
|
return ( self._path, self._export_type, self._delete_from_client_after_export, serialisable_file_search_context, self._period, self._phrase, self._last_checked )
|
|
|
|
|
|
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
|
|
|
|
( self._path, self._export_type, self._delete_from_client_after_export, serialisable_file_search_context, self._period, self._phrase, self._last_checked ) = serialisable_info
|
|
|
|
if self._export_type == HC.EXPORT_FOLDER_TYPE_SYNCHRONISE:
|
|
|
|
self._delete_from_client_after_export = False
|
|
|
|
|
|
self._file_search_context = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_search_context )
|
|
|
|
|
|
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
|
|
|
|
if version == 1:
|
|
|
|
( export_type, serialisable_file_search_context, period, phrase, last_checked ) = old_serialisable_info
|
|
|
|
path = self._name
|
|
|
|
new_serialisable_info = ( path, export_type, serialisable_file_search_context, period, phrase, last_checked )
|
|
|
|
return ( 2, new_serialisable_info )
|
|
|
|
|
|
if version == 2:
|
|
|
|
( path, export_type, serialisable_file_search_context, period, phrase, last_checked ) = old_serialisable_info
|
|
|
|
delete_from_client_after_export = False
|
|
|
|
new_serialisable_info = ( path, export_type, delete_from_client_after_export, serialisable_file_search_context, period, phrase, last_checked )
|
|
|
|
return ( 3, new_serialisable_info )
|
|
|
|
|
|
|
|
def DoWork( self ):
|
|
|
|
try:
|
|
|
|
if HydrusData.TimeHasPassed( self._last_checked + self._period ):
|
|
|
|
folder_path = HydrusData.ToUnicode( self._path )
|
|
|
|
if folder_path != '' and os.path.exists( folder_path ) and os.path.isdir( folder_path ):
|
|
|
|
query_hash_ids = HG.client_controller.Read( 'file_query_ids', self._file_search_context )
|
|
|
|
media_results = []
|
|
|
|
i = 0
|
|
|
|
base = 256
|
|
|
|
while i < len( query_hash_ids ):
|
|
|
|
if HC.options[ 'pause_export_folders_sync' ] or HydrusThreading.IsThreadShuttingDown():
|
|
|
|
return
|
|
|
|
|
|
if i == 0: ( last_i, i ) = ( 0, base )
|
|
else: ( last_i, i ) = ( i, i + base )
|
|
|
|
sub_query_hash_ids = query_hash_ids[ last_i : i ]
|
|
|
|
more_media_results = HG.client_controller.Read( 'media_results_from_ids', sub_query_hash_ids )
|
|
|
|
media_results.extend( more_media_results )
|
|
|
|
|
|
#
|
|
|
|
terms = ParseExportPhrase( self._phrase )
|
|
|
|
previous_filenames = set( os.listdir( folder_path ) )
|
|
|
|
sync_filenames = set()
|
|
|
|
client_files_manager = HG.client_controller.client_files_manager
|
|
|
|
num_copied = 0
|
|
|
|
for media_result in media_results:
|
|
|
|
if HC.options[ 'pause_export_folders_sync' ] or HydrusThreading.IsThreadShuttingDown():
|
|
|
|
return
|
|
|
|
|
|
hash = media_result.GetHash()
|
|
mime = media_result.GetMime()
|
|
size = media_result.GetSize()
|
|
|
|
source_path = client_files_manager.GetFilePath( hash, mime )
|
|
|
|
filename = GenerateExportFilename( folder_path, media_result, terms )
|
|
|
|
dest_path = os.path.join( folder_path, filename )
|
|
|
|
dest_path_dir = os.path.dirname( dest_path )
|
|
|
|
HydrusPaths.MakeSureDirectoryExists( dest_path_dir )
|
|
|
|
if filename not in sync_filenames:
|
|
|
|
copied = HydrusPaths.MirrorFile( source_path, dest_path )
|
|
|
|
if copied:
|
|
|
|
num_copied += 1
|
|
|
|
try: os.chmod( dest_path, stat.S_IWRITE | stat.S_IREAD )
|
|
except: pass
|
|
|
|
|
|
|
|
sync_filenames.add( filename )
|
|
|
|
|
|
if num_copied > 0:
|
|
|
|
HydrusData.Print( 'Export folder ' + self._name + ' exported ' + HydrusData.ToHumanInt( num_copied ) + ' files.' )
|
|
|
|
|
|
if self._export_type == HC.EXPORT_FOLDER_TYPE_SYNCHRONISE:
|
|
|
|
deletee_filenames = previous_filenames.difference( sync_filenames )
|
|
|
|
for deletee_filename in deletee_filenames:
|
|
|
|
deletee_path = os.path.join( folder_path, deletee_filename )
|
|
|
|
ClientPaths.DeletePath( deletee_path )
|
|
|
|
|
|
if len( deletee_filenames ) > 0:
|
|
|
|
HydrusData.Print( 'Export folder ' + self._name + ' deleted ' + HydrusData.ToHumanInt( len( deletee_filenames ) ) + ' files.' )
|
|
|
|
|
|
|
|
if self._delete_from_client_after_export:
|
|
|
|
deletee_hashes = { media_result.GetHash() for media_result in media_results }
|
|
|
|
chunks_of_hashes = HydrusData.SplitListIntoChunks( deletee_hashes, 64 )
|
|
|
|
content_updates = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, chunk_of_hashes ) for chunk_of_hashes in chunks_of_hashes ]
|
|
|
|
for content_update in content_updates:
|
|
|
|
HG.client_controller.WriteSynchronous( 'content_updates', { CC.LOCAL_FILE_SERVICE_KEY : [ content_update ] } )
|
|
|
|
|
|
|
|
|
|
|
|
except Exception as e:
|
|
|
|
HG.client_controller.options[ 'pause_export_folders_sync' ] = True
|
|
|
|
HydrusData.ShowText( 'The export folder "' + self._name + '" encountered an error! The error will follow! All export folders have now been paused. Please check the folder\'s settings and maybe report to hydrus dev if the error is complicated!' )
|
|
|
|
HydrusData.ShowException( e )
|
|
|
|
|
|
self._last_checked = HydrusData.GetNow()
|
|
|
|
HG.client_controller.WriteSynchronous( 'serialisable', self )
|
|
|
|
|
|
def ToTuple( self ):
|
|
|
|
return ( self._name, self._path, self._export_type, self._delete_from_client_after_export, self._file_search_context, self._period, self._phrase )
|
|
|
|
|
|
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_EXPORT_FOLDER ] = ExportFolder
|