import ClientConstants as CC import ClientData import gc import HydrusConstants as HC import HydrusData import HydrusExceptions import HydrusFileHandling import HydrusGlobals import HydrusPaths import HydrusSerialisable import itertools import os import random import re import shutil import stat import wx def GenerateExportFilename( media, terms ): mime = media.GetMime() filename = '' for ( term_type, term ) in terms: tags_manager = media.GetTagsManager() if term_type == 'string': filename += term elif term_type == 'namespace': tags = tags_manager.GetNamespaceSlice( ( term, ) ) filename += ', '.join( [ tag.split( ':' )[1] for tag in tags ] ) elif term_type == 'predicate': if term in ( 'tags', 'nn tags' ): current = tags_manager.GetCurrent() pending = tags_manager.GetPending() tags = list( current.union( pending ) ) if term == 'nn tags': tags = [ tag for tag in tags if ':' not in tag ] else: tags = [ tag if ':' not in tag else tag.split( ':' )[1] for tag in tags ] tags.sort() filename += ', '.join( tags ) elif term == 'hash': hash = media.GetHash() filename += hash.encode( 'hex' ) elif term_type == 'tag': if ':' in term: term = term.split( ':' )[1] if tags_manager.HasTag( term ): filename += term if HC.PLATFORM_WINDOWS: filename = re.sub( '\\\\|/|:|\\*|\\?|"|<|>|\\|', '_', filename, flags = re.UNICODE ) else: filename = re.sub( '/', '_', filename, flags = re.UNICODE ) ext = HC.mime_ext_lookup[ mime ] if not filename.endswith( ext ): filename += ext return filename def GetAllPaths( raw_paths ): file_paths = [] paths_to_process = raw_paths while len( paths_to_process ) > 0: next_paths_to_process = [] for path in paths_to_process: if os.path.isdir( path ): subpaths = [ os.path.join( path, filename ) for filename in os.listdir( path ) ] next_paths_to_process.extend( subpaths ) else: file_paths.append( path ) paths_to_process = next_paths_to_process gc.collect() return file_paths def GetExportPath(): options = HydrusGlobals.client_controller.GetOptions() portable_path = options[ 'export_path' ] if portable_path is None: path = os.path.join( os.path.expanduser( '~' ), 'hydrus_export' ) HydrusPaths.MakeSureDirectoryExists( path ) else: path = HydrusPaths.ConvertPortablePathToAbsPath( portable_path ) return path def GetExpectedContentUpdatePackagePath( service_key, begin, subindex ): return os.path.join( GetExpectedUpdateDir( service_key ), str( begin ) + '_' + str( subindex ) + '.json' ) def GetExpectedServiceUpdatePackagePath( service_key, begin ): return os.path.join( GetExpectedUpdateDir( service_key ), str( begin ) + '_metadata.json' ) def GetExpectedUpdateDir( service_key ): updates_dir = HydrusGlobals.client_controller.GetUpdatesDir() return os.path.join( updates_dir, service_key.encode( 'hex' ) ) def ParseExportPhrase( phrase ): try: terms = [ ( 'string', phrase ) ] new_terms = [] for ( term_type, term ) in terms: if term_type == 'string': while '[' in term: ( pre, term ) = term.split( '[', 1 ) ( namespace, term ) = term.split( ']', 1 ) new_terms.append( ( 'string', pre ) ) new_terms.append( ( 'namespace', namespace ) ) new_terms.append( ( term_type, term ) ) terms = new_terms new_terms = [] for ( term_type, term ) in terms: if term_type == 'string': while '{' in term: ( pre, term ) = term.split( '{', 1 ) ( predicate, term ) = term.split( '}', 1 ) new_terms.append( ( 'string', pre ) ) new_terms.append( ( 'predicate', predicate ) ) new_terms.append( ( term_type, term ) ) terms = new_terms new_terms = [] for ( term_type, term ) in terms: if term_type == 'string': while '(' in term: ( pre, term ) = term.split( '(', 1 ) ( tag, term ) = term.split( ')', 1 ) new_terms.append( ( 'string', pre ) ) new_terms.append( ( 'tag', tag ) ) new_terms.append( ( term_type, term ) ) terms = new_terms except: raise Exception( 'Could not parse that phrase!' ) return terms class ExportFolder( HydrusSerialisable.SerialisableBaseNamed ): SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_EXPORT_FOLDER SERIALISABLE_VERSION = 1 def __init__( self, name, export_type = HC.EXPORT_FOLDER_TYPE_REGULAR, file_search_context = None, period = 3600, phrase = '{hash}' ): HydrusSerialisable.SerialisableBaseNamed.__init__( self, name ) self._export_type = export_type self._file_search_context = file_search_context self._period = period self._phrase = phrase self._last_checked = 0 def _GetSerialisableInfo( self ): serialisable_file_search_context = self._file_search_context.GetSerialisableTuple() return ( self._export_type, serialisable_file_search_context, self._period, self._phrase, self._last_checked ) def _InitialiseFromSerialisableInfo( self, serialisable_info ): ( self._export_type, serialisable_file_search_context, self._period, self._phrase, self._last_checked ) = serialisable_info self._file_search_context = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_search_context ) def DoWork( self ): if HydrusData.TimeHasPassed( self._last_checked + self._period ): folder_path = HydrusData.ToUnicode( self._name ) if os.path.exists( folder_path ) and os.path.isdir( folder_path ): query_hash_ids = HydrusGlobals.client_controller.Read( 'file_query_ids', self._file_search_context ) media_results = [] i = 0 base = 256 while i < len( query_hash_ids ): if HC.options[ 'pause_export_folders_sync' ]: return if i == 0: ( last_i, i ) = ( 0, base ) else: ( last_i, i ) = ( i, i + base ) sub_query_hash_ids = query_hash_ids[ last_i : i ] more_media_results = HydrusGlobals.client_controller.Read( 'media_results_from_ids', sub_query_hash_ids ) media_results.extend( more_media_results ) # terms = ParseExportPhrase( self._phrase ) previous_filenames = set( os.listdir( folder_path ) ) sync_filenames = set() client_files_manager = HydrusGlobals.client_controller.GetClientFilesManager() num_copied = 0 for media_result in media_results: hash = media_result.GetHash() mime = media_result.GetMime() size = media_result.GetSize() source_path = client_files_manager.GetFilePath( hash, mime ) filename = GenerateExportFilename( media_result, terms ) dest_path = os.path.join( folder_path, filename ) if filename not in sync_filenames: copied = HydrusPaths.MirrorFile( source_path, dest_path ) if copied: num_copied += 1 try: os.chmod( dest_path, stat.S_IWRITE | stat.S_IREAD ) except: pass sync_filenames.add( filename ) if num_copied > 0: HydrusData.Print( 'Export folder ' + self._name + ' exported ' + HydrusData.ConvertIntToPrettyString( num_copied ) + ' files.' ) if self._export_type == HC.EXPORT_FOLDER_TYPE_SYNCHRONISE: deletee_filenames = previous_filenames.difference( sync_filenames ) for deletee_filename in deletee_filenames: deletee_path = os.path.join( folder_path, deletee_filename ) ClientData.DeletePath( deletee_path ) if len( deletee_filenames ) > 0: HydrusData.Print( 'Export folder ' + self._name + ' deleted ' + HydrusData.ConvertIntToPrettyString( len( deletee_filenames ) ) + ' files.' ) self._last_checked = HydrusData.GetNow() HydrusGlobals.client_controller.WriteSynchronous( 'serialisable', self ) def ToTuple( self ): return ( self._name, self._export_type, self._file_search_context, self._period, self._phrase ) def SetTuple( self, folder_path, export_type, file_search_context, period, phrase ): self._name = folder_path self._export_type = export_type self._file_search_context = file_search_context self._period = period self._phrase = phrase HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_EXPORT_FOLDER ] = ExportFolder