import collections import threading import time import typing from hydrus.core import HydrusConstants as HC from hydrus.core import HydrusData from hydrus.core import HydrusExceptions from hydrus.core import HydrusGlobals as HG from hydrus.core import HydrusImageHandling from hydrus.core import HydrusSerialisable from hydrus.core import HydrusTags from hydrus.client import ClientConstants as CC from hydrus.client import ClientData from hydrus.client import ClientThreading from hydrus.client.importing.options import NoteImportOptions from hydrus.client.media import ClientMedia from hydrus.client.metadata import ClientTags hashes_to_jpeg_quality = {} hashes_to_pixel_hashes = {} def GetDuplicateComparisonScore( shown_media, comparison_media ): statements_and_scores = GetDuplicateComparisonStatements( shown_media, comparison_media ) total_score = sum( ( score for ( statement, score ) in statements_and_scores.values() ) ) return total_score # TODO: ok, let's make an enum here at some point and a DuplicateComparisonSetting serialisable object # Then we can attach 'show/hide' boolean and allow editable scores and whatnot in a nice class that will one day evolve the enum to an editable MetadataConditional/MetadataComparison object # also have banding so we can have 'at this filesize difference, score 10, at this, score 15' # show it in a listctrl or whatever in the options, ditch the hardcoding # metadatacomparison needs to handle 'if one is a png and one is a jpeg', and then orient to A/B and give it a score def GetDuplicateComparisonStatements( shown_media, comparison_media ): new_options = HG.client_controller.new_options duplicate_comparison_score_higher_jpeg_quality = new_options.GetInteger( 'duplicate_comparison_score_higher_jpeg_quality' ) duplicate_comparison_score_much_higher_jpeg_quality = new_options.GetInteger( 'duplicate_comparison_score_much_higher_jpeg_quality' ) duplicate_comparison_score_higher_filesize = new_options.GetInteger( 'duplicate_comparison_score_higher_filesize' ) duplicate_comparison_score_much_higher_filesize = new_options.GetInteger( 'duplicate_comparison_score_much_higher_filesize' ) duplicate_comparison_score_higher_resolution = new_options.GetInteger( 'duplicate_comparison_score_higher_resolution' ) duplicate_comparison_score_much_higher_resolution = new_options.GetInteger( 'duplicate_comparison_score_much_higher_resolution' ) duplicate_comparison_score_more_tags = new_options.GetInteger( 'duplicate_comparison_score_more_tags' ) duplicate_comparison_score_older = new_options.GetInteger( 'duplicate_comparison_score_older' ) duplicate_comparison_score_nicer_ratio = new_options.GetInteger( 'duplicate_comparison_score_nicer_ratio' ) # statements_and_scores = {} s_hash = shown_media.GetHash() c_hash = comparison_media.GetHash() s_mime = shown_media.GetMime() c_mime = comparison_media.GetMime() # size s_size = shown_media.GetSize() c_size = comparison_media.GetSize() is_a_pixel_dupe = False if shown_media.IsStaticImage() and comparison_media.IsStaticImage() and shown_media.GetResolution() == comparison_media.GetResolution(): global hashes_to_pixel_hashes if s_hash not in hashes_to_pixel_hashes: path = HG.client_controller.client_files_manager.GetFilePath( s_hash, s_mime ) hashes_to_pixel_hashes[ s_hash ] = HydrusImageHandling.GetImagePixelHash( path, s_mime ) if c_hash not in hashes_to_pixel_hashes: path = HG.client_controller.client_files_manager.GetFilePath( c_hash, c_mime ) hashes_to_pixel_hashes[ c_hash ] = HydrusImageHandling.GetImagePixelHash( path, c_mime ) s_pixel_hash = hashes_to_pixel_hashes[ s_hash ] c_pixel_hash = hashes_to_pixel_hashes[ c_hash ] if s_pixel_hash == c_pixel_hash: is_a_pixel_dupe = True if s_mime == HC.IMAGE_PNG and c_mime != HC.IMAGE_PNG: statement = 'this is a pixel-for-pixel duplicate png!' score = -100 elif s_mime != HC.IMAGE_PNG and c_mime == HC.IMAGE_PNG: statement = 'other file is a pixel-for-pixel duplicate png!' score = 100 else: statement = 'images are pixel-for-pixel duplicates!' score = 0 statements_and_scores[ 'pixel_duplicates' ] = ( statement, score ) if s_size != c_size: absolute_size_ratio = max( s_size, c_size ) / min( s_size, c_size ) if absolute_size_ratio > 2.0: if s_size > c_size: operator = '>>' score = duplicate_comparison_score_much_higher_filesize else: operator = '<<' score = -duplicate_comparison_score_much_higher_filesize elif absolute_size_ratio > 1.05: if s_size > c_size: operator = '>' score = duplicate_comparison_score_higher_filesize else: operator = '<' score = -duplicate_comparison_score_higher_filesize else: operator = CC.UNICODE_ALMOST_EQUAL_TO score = 0 if s_size > c_size: sign = '+' percentage_difference = ( s_size / c_size ) - 1.0 else: sign = '' percentage_difference = ( s_size / c_size ) - 1.0 percentage_different_string = ' ({}{})'.format( sign, HydrusData.ConvertFloatToPercentage( percentage_difference ) ) if is_a_pixel_dupe: score = 0 statement = '{} {} {}{}'.format( HydrusData.ToHumanBytes( s_size ), operator, HydrusData.ToHumanBytes( c_size ), percentage_different_string ) statements_and_scores[ 'filesize' ] = ( statement, score ) # higher/same res s_resolution = shown_media.GetResolution() c_resolution = comparison_media.GetResolution() if s_resolution != c_resolution: ( s_w, s_h ) = s_resolution ( c_w, c_h ) = c_resolution all_measurements_are_good = None not in ( s_w, s_h, c_w, c_h ) and True not in ( d <= 0 for d in ( s_w, s_h, c_w, c_h ) ) if all_measurements_are_good: resolution_ratio = ( s_w * s_h ) / ( c_w * c_h ) if resolution_ratio == 1.0: operator = '!=' score = 0 elif resolution_ratio > 2.0: operator = '>>' score = duplicate_comparison_score_much_higher_resolution elif resolution_ratio > 1.00: operator = '>' score = duplicate_comparison_score_higher_resolution elif resolution_ratio < 0.5: operator = '<<' score = -duplicate_comparison_score_much_higher_resolution else: operator = '<' score = -duplicate_comparison_score_higher_resolution if s_resolution in HC.NICE_RESOLUTIONS: s_string = HC.NICE_RESOLUTIONS[ s_resolution ] else: s_string = HydrusData.ConvertResolutionToPrettyString( s_resolution ) if s_w % 2 == 1 or s_h % 2 == 1: s_string += ' (unusual)' if c_resolution in HC.NICE_RESOLUTIONS: c_string = HC.NICE_RESOLUTIONS[ c_resolution ] else: c_string = HydrusData.ConvertResolutionToPrettyString( c_resolution ) if c_w % 2 == 1 or c_h % 2 == 1: c_string += ' (unusual)' statement = '{} {} {}'.format( s_string, operator, c_string ) statements_and_scores[ 'resolution' ] = ( statement, score ) # s_ratio = s_w / s_h c_ratio = c_w / c_h s_nice = s_ratio in HC.NICE_RATIOS c_nice = c_ratio in HC.NICE_RATIOS if s_nice or c_nice: if s_nice: s_string = HC.NICE_RATIOS[ s_ratio ] else: s_string = 'unusual' if c_nice: c_string = HC.NICE_RATIOS[ c_ratio ] else: c_string = 'unusual' if s_nice and c_nice: operator = '-' score = 0 elif s_nice: operator = '>' score = duplicate_comparison_score_nicer_ratio elif c_nice: operator = '<' score = -duplicate_comparison_score_nicer_ratio if s_string == c_string: statement = 'both {}'.format( s_string ) else: statement = '{} {} {}'.format( s_string, operator, c_string ) statements_and_scores[ 'ratio' ] = ( statement, score ) # same/diff mime if s_mime != c_mime: statement = '{} vs {}'.format( HC.mime_string_lookup[ s_mime ], HC.mime_string_lookup[ c_mime ] ) score = 0 statements_and_scores[ 'mime' ] = ( statement, score ) # more tags s_num_tags = len( shown_media.GetTagsManager().GetCurrentAndPending( CC.COMBINED_TAG_SERVICE_KEY, ClientTags.TAG_DISPLAY_ACTUAL ) ) c_num_tags = len( comparison_media.GetTagsManager().GetCurrentAndPending( CC.COMBINED_TAG_SERVICE_KEY, ClientTags.TAG_DISPLAY_ACTUAL ) ) if s_num_tags != c_num_tags: if s_num_tags > 0 and c_num_tags > 0: if s_num_tags > c_num_tags: operator = '>' score = duplicate_comparison_score_more_tags else: operator = '<' score = -duplicate_comparison_score_more_tags elif s_num_tags > 0: operator = '>>' score = duplicate_comparison_score_more_tags elif c_num_tags > 0: operator = '<<' score = -duplicate_comparison_score_more_tags statement = '{} tags {} {} tags'.format( HydrusData.ToHumanInt( s_num_tags ), operator, HydrusData.ToHumanInt( c_num_tags ) ) statements_and_scores[ 'num_tags' ] = ( statement, score ) # older s_ts = shown_media.GetLocationsManager().GetCurrentTimestamp( CC.COMBINED_LOCAL_FILE_SERVICE_KEY ) c_ts = comparison_media.GetLocationsManager().GetCurrentTimestamp( CC.COMBINED_LOCAL_FILE_SERVICE_KEY ) one_month = 86400 * 30 if s_ts is not None and c_ts is not None and abs( s_ts - c_ts ) > one_month: if s_ts < c_ts: operator = 'older than' score = duplicate_comparison_score_older else: operator = 'newer than' score = -duplicate_comparison_score_older if is_a_pixel_dupe: score = 0 statement = '{}, {} {}'.format( ClientData.TimestampToPrettyTimeDelta( s_ts, history_suffix = ' old' ), operator, ClientData.TimestampToPrettyTimeDelta( c_ts, history_suffix = ' old' ) ) statements_and_scores[ 'time_imported' ] = ( statement, score ) if s_mime == HC.IMAGE_JPEG and c_mime == HC.IMAGE_JPEG: global hashes_to_jpeg_quality if s_hash not in hashes_to_jpeg_quality: path = HG.client_controller.client_files_manager.GetFilePath( s_hash, s_mime ) hashes_to_jpeg_quality[ s_hash ] = HydrusImageHandling.GetJPEGQuantizationQualityEstimate( path ) if c_hash not in hashes_to_jpeg_quality: path = HG.client_controller.client_files_manager.GetFilePath( c_hash, c_mime ) hashes_to_jpeg_quality[ c_hash ] = HydrusImageHandling.GetJPEGQuantizationQualityEstimate( path ) ( s_label, s_jpeg_quality ) = hashes_to_jpeg_quality[ s_hash ] ( c_label, c_jpeg_quality ) = hashes_to_jpeg_quality[ c_hash ] score = 0 if s_label != c_label: if c_jpeg_quality is None or s_jpeg_quality is None: score = 0 else: # other way around, low score is good here quality_ratio = c_jpeg_quality / s_jpeg_quality if quality_ratio > 2.0: score = duplicate_comparison_score_much_higher_jpeg_quality elif quality_ratio > 1.0: score = duplicate_comparison_score_higher_jpeg_quality elif quality_ratio < 0.5: score = -duplicate_comparison_score_much_higher_jpeg_quality else: score = -duplicate_comparison_score_higher_jpeg_quality statement = '{} vs {} jpeg quality'.format( s_label, c_label ) statements_and_scores[ 'jpeg_quality' ] = ( statement, score ) def has_exif( m ): try: hash = m.GetHash() mime = m.GetMime() if mime not in ( HC.IMAGE_JPEG, HC.IMAGE_TIFF ): return False path = HG.client_controller.client_files_manager.GetFilePath( hash, mime ) pil_image = HydrusImageHandling.RawOpenPILImage( path ) exif_dict = HydrusImageHandling.GetEXIFDict( pil_image ) if exif_dict is None: return False return len( exif_dict ) > 0 except: return False s_has_exif = has_exif( shown_media ) c_has_exif = has_exif( comparison_media ) if s_has_exif ^ c_has_exif: if s_has_exif: exif_statement = 'has exif data, the other does not' else: exif_statement = 'the other has exif data, this does not' statements_and_scores[ 'exif_data' ] = ( exif_statement, 0 ) s_has_human_readable_embedded_metadata = shown_media.GetMediaResult().GetFileInfoManager().has_human_readable_embedded_metadata c_has_human_readable_embedded_metadata = comparison_media.GetMediaResult().GetFileInfoManager().has_human_readable_embedded_metadata if s_has_human_readable_embedded_metadata ^ c_has_human_readable_embedded_metadata: if s_has_human_readable_embedded_metadata: embedded_metadata_statement = 'has embedded metadata, the other does not' else: embedded_metadata_statement = 'the other has embedded metadata, this does not' statements_and_scores[ 'embedded_metadata' ] = ( embedded_metadata_statement, 0 ) s_has_icc = shown_media.GetMediaResult().GetFileInfoManager().has_icc_profile c_has_icc = comparison_media.GetMediaResult().GetFileInfoManager().has_icc_profile if s_has_icc ^ c_has_icc: if s_has_icc: icc_statement = 'has icc profile, the other does not' else: icc_statement = 'the other has icc profile, this does not' statements_and_scores[ 'icc_profile' ] = ( icc_statement, 0 ) return statements_and_scores class DuplicatesManager( object ): my_instance = None def __init__( self ): DuplicatesManager.my_instance = self self._similar_files_maintenance_status = None self._currently_refreshing_maintenance_numbers = False self._refresh_maintenance_numbers = True self._currently_doing_potentials_search = False self._lock = threading.Lock() @staticmethod def instance() -> 'DuplicatesManager': if DuplicatesManager.my_instance is None: DuplicatesManager() return DuplicatesManager.my_instance def GetMaintenanceNumbers( self ): with self._lock: if self._refresh_maintenance_numbers and not self._currently_refreshing_maintenance_numbers: self._refresh_maintenance_numbers = False self._currently_refreshing_maintenance_numbers = True HG.client_controller.pub( 'new_similar_files_maintenance_numbers' ) HG.client_controller.CallToThread( self.THREADRefreshMaintenanceNumbers ) return ( self._similar_files_maintenance_status, self._currently_refreshing_maintenance_numbers, self._currently_doing_potentials_search ) def RefreshMaintenanceNumbers( self ): with self._lock: self._refresh_maintenance_numbers = True HG.client_controller.pub( 'new_similar_files_maintenance_numbers' ) def NotifyNewPotentialsSearchNumbers( self ): HG.client_controller.pub( 'new_similar_files_potentials_search_numbers' ) def StartPotentialsSearch( self ): with self._lock: if self._currently_doing_potentials_search or self._similar_files_maintenance_status is None: return self._currently_doing_potentials_search = True HG.client_controller.CallToThreadLongRunning( self.THREADSearchPotentials ) def THREADRefreshMaintenanceNumbers( self ): try: similar_files_maintenance_status = HG.client_controller.Read( 'similar_files_maintenance_status' ) with self._lock: self._similar_files_maintenance_status = similar_files_maintenance_status if self._refresh_maintenance_numbers: self._refresh_maintenance_numbers = False HG.client_controller.CallToThread( self.THREADRefreshMaintenanceNumbers ) else: self._currently_refreshing_maintenance_numbers = False self._refresh_maintenance_numbers = False HG.client_controller.pub( 'new_similar_files_maintenance_numbers' ) except: self._currently_refreshing_maintenance_numbers = False HG.client_controller.pub( 'new_similar_files_maintenance_numbers' ) raise def THREADSearchPotentials( self ): try: search_distance = HG.client_controller.new_options.GetInteger( 'similar_files_duplicate_pairs_search_distance' ) with self._lock: if self._similar_files_maintenance_status is None: return searched_distances_to_count = self._similar_files_maintenance_status total_num_files = sum( searched_distances_to_count.values() ) num_searched = sum( ( count for ( value, count ) in searched_distances_to_count.items() if value is not None and value >= search_distance ) ) all_files_searched = num_searched >= total_num_files if all_files_searched: return # no work to do num_searched_estimate = num_searched HG.client_controller.pub( 'new_similar_files_maintenance_numbers' ) job_key = ClientThreading.JobKey( cancellable = True ) job_key.SetStatusTitle( 'searching for potential duplicates' ) HG.client_controller.pub( 'message', job_key ) still_work_to_do = True while still_work_to_do: search_distance = HG.client_controller.new_options.GetInteger( 'similar_files_duplicate_pairs_search_distance' ) start_time = HydrusData.GetNowPrecise() ( still_work_to_do, num_done ) = HG.client_controller.WriteSynchronous( 'maintain_similar_files_search_for_potential_duplicates', search_distance, maintenance_mode = HC.MAINTENANCE_FORCED, job_key = job_key, work_time_float = 0.5 ) time_it_took = HydrusData.GetNowPrecise() - start_time num_searched_estimate += num_done if num_searched_estimate > total_num_files: similar_files_maintenance_status = HG.client_controller.Read( 'similar_files_maintenance_status' ) if similar_files_maintenance_status is None: break with self._lock: self._similar_files_maintenance_status = similar_files_maintenance_status searched_distances_to_count = self._similar_files_maintenance_status total_num_files = max( num_searched_estimate, sum( searched_distances_to_count.values() ) ) text = 'searching: {}'.format( HydrusData.ConvertValueRangeToPrettyString( num_searched_estimate, total_num_files ) ) job_key.SetVariable( 'popup_text_1', text ) job_key.SetVariable( 'popup_gauge_1', ( num_searched_estimate, total_num_files ) ) if job_key.IsCancelled() or HG.model_shutdown: break time.sleep( min( 5, time_it_took ) ) # ideally 0.5s, but potentially longer job_key.Delete() finally: with self._lock: self._currently_doing_potentials_search = False self.RefreshMaintenanceNumbers() self.NotifyNewPotentialsSearchNumbers() SYNC_ARCHIVE_NONE = 0 SYNC_ARCHIVE_IF_ONE_DO_BOTH = 1 SYNC_ARCHIVE_DO_BOTH_REGARDLESS = 2 class DuplicateContentMergeOptions( HydrusSerialisable.SerialisableBase ): SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_DUPLICATE_CONTENT_MERGE_OPTIONS SERIALISABLE_NAME = 'Duplicate Content Merge Options' SERIALISABLE_VERSION = 6 def __init__( self ): HydrusSerialisable.SerialisableBase.__init__( self ) self._tag_service_actions = [] self._rating_service_actions = [] self._sync_notes_action = HC.CONTENT_MERGE_ACTION_NONE self._sync_note_import_options = NoteImportOptions.NoteImportOptions() self._sync_archive_action = SYNC_ARCHIVE_NONE self._sync_urls_action = HC.CONTENT_MERGE_ACTION_NONE def _GetSerialisableInfo( self ): if HG.client_controller.IsBooted(): services_manager = HG.client_controller.services_manager self._tag_service_actions = [ ( service_key, action, tag_filter ) for ( service_key, action, tag_filter ) in self._tag_service_actions if services_manager.ServiceExists( service_key ) and services_manager.GetServiceType( service_key ) in HC.REAL_TAG_SERVICES ] self._rating_service_actions = [ ( service_key, action ) for ( service_key, action ) in self._rating_service_actions if services_manager.ServiceExists( service_key ) and services_manager.GetServiceType( service_key ) in ( HC.LOCAL_RATING_LIKE, HC.LOCAL_RATING_NUMERICAL ) ] serialisable_tag_service_actions = [ ( service_key.hex(), action, tag_filter.GetSerialisableTuple() ) for ( service_key, action, tag_filter ) in self._tag_service_actions ] serialisable_rating_service_actions = [ ( service_key.hex(), action ) for ( service_key, action ) in self._rating_service_actions ] serialisable_sync_note_import_options = self._sync_note_import_options.GetSerialisableTuple() return ( serialisable_tag_service_actions, serialisable_rating_service_actions, self._sync_notes_action, serialisable_sync_note_import_options, self._sync_archive_action, self._sync_urls_action ) def _InitialiseFromSerialisableInfo( self, serialisable_info ): ( serialisable_tag_service_actions, serialisable_rating_service_actions, self._sync_notes_action, serialisable_sync_note_import_options, self._sync_archive_action, self._sync_urls_action ) = serialisable_info self._tag_service_actions = [ ( bytes.fromhex( serialisable_service_key ), action, HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_filter ) ) for ( serialisable_service_key, action, serialisable_tag_filter ) in serialisable_tag_service_actions ] self._rating_service_actions = [ ( bytes.fromhex( serialisable_service_key ), action ) for ( serialisable_service_key, action ) in serialisable_rating_service_actions ] self._sync_note_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_sync_note_import_options ) def _UpdateSerialisableInfo( self, version, old_serialisable_info ): if version == 1: ( serialisable_service_actions, delete_second_file ) = old_serialisable_info tag_service_actions = [] rating_service_actions = [] # As the client isn't booted when this is loaded in options, there isn't a good way to figure out tag from rating # So, let's just dupe and purge later on, in serialisation for ( service_key_encoded, action ) in serialisable_service_actions: service_key = bytes.fromhex( service_key_encoded ) tag_filter = HydrusTags.TagFilter() tag_service_actions.append( ( service_key, action, tag_filter ) ) rating_service_actions.append( ( service_key, action ) ) serialisable_tag_service_actions = [ ( service_key.hex(), action, tag_filter.GetSerialisableTuple() ) for ( service_key, action, tag_filter ) in tag_service_actions ] serialisable_rating_service_actions = [ ( service_key.hex(), action ) for ( service_key, action ) in rating_service_actions ] sync_archive = delete_second_file delete_both_files = False new_serialisable_info = ( serialisable_tag_service_actions, serialisable_rating_service_actions, delete_second_file, sync_archive, delete_both_files ) return ( 2, new_serialisable_info ) if version == 2: ( serialisable_tag_service_actions, serialisable_rating_service_actions, delete_second_file, sync_archive, delete_both_files ) = old_serialisable_info sync_urls_action = None new_serialisable_info = ( serialisable_tag_service_actions, serialisable_rating_service_actions, delete_second_file, sync_archive, delete_both_files, sync_urls_action ) return ( 3, new_serialisable_info ) if version == 3: ( serialisable_tag_service_actions, serialisable_rating_service_actions, delete_second_file, sync_archive, delete_both_files, sync_urls_action ) = old_serialisable_info new_serialisable_info = ( serialisable_tag_service_actions, serialisable_rating_service_actions, sync_archive, sync_urls_action ) return ( 4, new_serialisable_info ) if version == 4: ( serialisable_tag_service_actions, serialisable_rating_service_actions, sync_archive, sync_urls_action ) = old_serialisable_info if sync_archive: sync_archive_action = SYNC_ARCHIVE_IF_ONE_DO_BOTH else: sync_archive_action = SYNC_ARCHIVE_NONE new_serialisable_info = ( serialisable_tag_service_actions, serialisable_rating_service_actions, sync_archive_action, sync_urls_action ) return ( 5, new_serialisable_info ) if version == 5: ( serialisable_tag_service_actions, serialisable_rating_service_actions, sync_archive_action, sync_urls_action ) = old_serialisable_info if sync_urls_action is None: sync_urls_action = HC.CONTENT_MERGE_ACTION_NONE sync_notes_action = HC.CONTENT_MERGE_ACTION_NONE sync_note_import_options = NoteImportOptions.NoteImportOptions() serialisable_sync_note_import_options = sync_note_import_options.GetSerialisableTuple() new_serialisable_info = ( serialisable_tag_service_actions, serialisable_rating_service_actions, sync_notes_action, serialisable_sync_note_import_options, sync_archive_action, sync_urls_action ) return ( 6, new_serialisable_info ) def GetRatingServiceActions( self ) -> typing.Collection[ tuple ]: return self._rating_service_actions def GetTagServiceActions( self ) -> typing.Collection[ tuple ]: return self._tag_service_actions def GetSyncArchiveAction( self ) -> int: return self._sync_archive_action def GetSyncNotesAction( self ) -> int: return self._sync_notes_action def GetSyncNoteImportOptions( self ) -> NoteImportOptions.NoteImportOptions: return self._sync_note_import_options def GetSyncURLsAction( self ) -> int: return self._sync_urls_action def SetRatingServiceActions( self, rating_service_actions: typing.Collection[ tuple ] ): self._rating_service_actions = rating_service_actions def SetTagServiceActions( self, tag_service_actions: typing.Collection[ tuple ] ): self._tag_service_actions = tag_service_actions def SetSyncArchiveAction( self, sync_archive_action: int ): self._sync_archive_action = sync_archive_action def SetSyncNotesAction( self, sync_notes_action: int ): self._sync_notes_action = sync_notes_action def SetSyncNoteImportOptions( self, sync_note_import_options: NoteImportOptions.NoteImportOptions ): self._sync_note_import_options = sync_note_import_options def SetSyncURLsAction( self, sync_urls_action: int ): self._sync_urls_action = sync_urls_action def ProcessPairIntoContentUpdates( self, first_media: ClientMedia.MediaSingleton, second_media: ClientMedia.MediaSingleton, delete_first = False, delete_second = False, file_deletion_reason = None, do_not_do_deletes = False ): if file_deletion_reason is None: file_deletion_reason = 'unknown reason' service_keys_to_content_updates = collections.defaultdict( list ) first_hash = first_media.GetHash() second_hash = second_media.GetHash() first_hashes = { first_hash } second_hashes = { second_hash } first_media_result = first_media.GetMediaResult() second_media_result = second_media.GetMediaResult() # services_manager = HG.client_controller.services_manager for ( service_key, action, tag_filter ) in self._tag_service_actions: content_updates = [] try: service = services_manager.GetService( service_key ) except HydrusExceptions.DataMissing: continue service_type = service.GetServiceType() if service_type == HC.LOCAL_TAG: add_content_action = HC.CONTENT_UPDATE_ADD elif service_type == HC.TAG_REPOSITORY: add_content_action = HC.CONTENT_UPDATE_PEND else: continue first_tags = first_media.GetTagsManager().GetCurrentAndPending( service_key, ClientTags.TAG_DISPLAY_STORAGE ) second_tags = second_media.GetTagsManager().GetCurrentAndPending( service_key, ClientTags.TAG_DISPLAY_STORAGE ) first_tags = tag_filter.Filter( first_tags ) second_tags = tag_filter.Filter( second_tags ) if action == HC.CONTENT_MERGE_ACTION_TWO_WAY_MERGE: first_needs = second_tags.difference( first_tags ) second_needs = first_tags.difference( second_tags ) content_updates.extend( ( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, add_content_action, ( tag, first_hashes ) ) for tag in first_needs ) ) content_updates.extend( ( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, add_content_action, ( tag, second_hashes ) ) for tag in second_needs ) ) elif action == HC.CONTENT_MERGE_ACTION_COPY: first_needs = second_tags.difference( first_tags ) content_updates.extend( ( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, add_content_action, ( tag, first_hashes ) ) for tag in first_needs ) ) elif service_type == HC.LOCAL_TAG and action == HC.CONTENT_MERGE_ACTION_MOVE: first_needs = second_tags.difference( first_tags ) content_updates.extend( ( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, add_content_action, ( tag, first_hashes ) ) for tag in first_needs ) ) content_updates.extend( ( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_DELETE, ( tag, second_hashes ) ) for tag in second_tags ) ) if len( content_updates ) > 0: service_keys_to_content_updates[ service_key ].extend( content_updates ) def worth_updating_rating( source_rating, dest_rating ): if source_rating is not None: if dest_rating is None or source_rating > dest_rating: return True return False for ( service_key, action ) in self._rating_service_actions: content_updates = [] try: services_manager.GetService( service_key ) except HydrusExceptions.DataMissing: continue first_current_value = first_media.GetRatingsManager().GetRating( service_key ) second_current_value = second_media.GetRatingsManager().GetRating( service_key ) if action == HC.CONTENT_MERGE_ACTION_TWO_WAY_MERGE: if worth_updating_rating( first_current_value, second_current_value ): content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_RATINGS, HC.CONTENT_UPDATE_ADD, ( first_current_value, second_hashes ) ) ) elif worth_updating_rating( second_current_value, first_current_value ): content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_RATINGS, HC.CONTENT_UPDATE_ADD, ( second_current_value, first_hashes ) ) ) elif action == HC.CONTENT_MERGE_ACTION_COPY: if worth_updating_rating( second_current_value, first_current_value ): content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_RATINGS, HC.CONTENT_UPDATE_ADD, ( second_current_value, first_hashes ) ) ) elif action == HC.CONTENT_MERGE_ACTION_MOVE: if second_current_value is not None: if worth_updating_rating( second_current_value, first_current_value ): content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_RATINGS, HC.CONTENT_UPDATE_ADD, ( second_current_value, first_hashes ) ) ) content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_RATINGS, HC.CONTENT_UPDATE_ADD, ( None, second_hashes ) ) ) if len( content_updates ) > 0: service_keys_to_content_updates[ service_key ].extend( content_updates ) # if self._sync_notes_action != HC.CONTENT_MERGE_ACTION_NONE: first_names_and_notes = list( first_media.GetNotesManager().GetNamesToNotes().items() ) second_names_and_notes = list( second_media.GetNotesManager().GetNamesToNotes().items() ) content_updates = [] # TODO: rework this to UpdateeNamesToNotes if self._sync_notes_action == HC.CONTENT_MERGE_ACTION_TWO_WAY_MERGE: first_service_keys_to_content_updates = self._sync_note_import_options.GetServiceKeysToContentUpdates( first_media_result, second_names_and_notes ) second_service_keys_to_content_updates = self._sync_note_import_options.GetServiceKeysToContentUpdates( second_media_result, first_names_and_notes ) content_updates.extend( first_service_keys_to_content_updates[ CC.LOCAL_NOTES_SERVICE_KEY ] ) content_updates.extend( second_service_keys_to_content_updates[ CC.LOCAL_NOTES_SERVICE_KEY ] ) elif self._sync_notes_action == HC.CONTENT_MERGE_ACTION_COPY: first_service_keys_to_content_updates = self._sync_note_import_options.GetServiceKeysToContentUpdates( first_media_result, second_names_and_notes ) content_updates.extend( first_service_keys_to_content_updates[ CC.LOCAL_NOTES_SERVICE_KEY ] ) elif self._sync_notes_action == HC.CONTENT_MERGE_ACTION_MOVE: first_service_keys_to_content_updates = self._sync_note_import_options.GetServiceKeysToContentUpdates( first_media_result, second_names_and_notes ) content_updates.extend( first_service_keys_to_content_updates[ CC.LOCAL_NOTES_SERVICE_KEY ] ) content_updates.extend( [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_NOTES, HC.CONTENT_UPDATE_DELETE, ( second_hash, name ) ) for ( name, note ) in second_names_and_notes ] ) if len( content_updates ) > 0: service_keys_to_content_updates[ CC.LOCAL_NOTES_SERVICE_KEY ].extend( content_updates ) # content_update_archive_first = HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ARCHIVE, first_hashes ) content_update_archive_second = HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ARCHIVE, second_hashes ) # and not delete_first gubbins here to help out the delete lock lmao. don't want to archive and then try to delete # TODO: this is obviously a bad solution, so better to refactor this function to return a list of service_keys_to_content_updates and stick the delete command right up top, tested for locks on current info if self._sync_archive_action == SYNC_ARCHIVE_IF_ONE_DO_BOTH: if first_media.HasInbox() and second_media.HasArchive() and not delete_first: service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ].append( content_update_archive_first ) elif first_media.HasArchive() and second_media.HasInbox() and not delete_second: service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ].append( content_update_archive_second ) elif self._sync_archive_action == SYNC_ARCHIVE_DO_BOTH_REGARDLESS: if first_media.HasInbox() and not delete_first: service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ].append( content_update_archive_first ) if second_media.HasInbox() and not delete_second: service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ].append( content_update_archive_second ) # if self._sync_urls_action != HC.CONTENT_MERGE_ACTION_NONE: first_urls = set( first_media.GetLocationsManager().GetURLs() ) second_urls = set( second_media.GetLocationsManager().GetURLs() ) content_updates = [] if self._sync_urls_action == HC.CONTENT_MERGE_ACTION_TWO_WAY_MERGE: first_needs = second_urls.difference( first_urls ) second_needs = first_urls.difference( second_urls ) if len( first_needs ) > 0: content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( first_needs, first_hashes ) ) ) if len( second_needs ) > 0: content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( second_needs, second_hashes ) ) ) elif self._sync_urls_action == HC.CONTENT_MERGE_ACTION_COPY: first_needs = second_urls.difference( first_urls ) if len( first_needs ) > 0: content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( first_needs, first_hashes ) ) ) if len( content_updates ) > 0: service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ].extend( content_updates ) # deletee_media = [] if delete_first: deletee_media.append( first_media ) if delete_second: deletee_media.append( second_media ) for media in deletee_media: if do_not_do_deletes: continue if media.HasDeleteLocked(): ClientMedia.ReportDeleteLockFailures( [ media ] ) continue if media.GetLocationsManager().IsTrashed(): deletee_service_keys = ( CC.COMBINED_LOCAL_FILE_SERVICE_KEY, ) else: local_file_service_keys = HG.client_controller.services_manager.GetServiceKeys( ( HC.LOCAL_FILE_DOMAIN, ) ) deletee_service_keys = media.GetLocationsManager().GetCurrent().intersection( local_file_service_keys ) for deletee_service_key in deletee_service_keys: content_update = HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, media.GetHashes(), reason = file_deletion_reason ) service_keys_to_content_updates[ deletee_service_key ].append( content_update ) # return service_keys_to_content_updates HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_DUPLICATE_CONTENT_MERGE_OPTIONS ] = DuplicateContentMergeOptions