RepoMirrors
/
hydrus
mirror of https://github.com/hydrusnetwork/hydrus.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
hydrus/hydrus/client/ClientDuplicates.py

1252 lines
47 KiB
Python
Raw Blame History

import collections
import threading
import time
import typing
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusGlobals as HG
from hydrus.core import HydrusImageHandling
from hydrus.core import HydrusSerialisable
from hydrus.core import HydrusTags
from hydrus.client import ClientConstants as CC
from hydrus.client import ClientData
from hydrus.client import ClientThreading
from hydrus.client.importing.options import NoteImportOptions
from hydrus.client.media import ClientMedia
from hydrus.client.metadata import ClientTags
hashes_to_jpeg_quality = {}
hashes_to_pixel_hashes = {}
def GetDuplicateComparisonScore( shown_media, comparison_media ):
statements_and_scores = GetDuplicateComparisonStatements( shown_media, comparison_media )
total_score = sum( ( score for ( statement, score ) in statements_and_scores.values() ) )
return total_score
# TODO: ok, let's make an enum here at some point and a DuplicateComparisonSetting serialisable object
# Then we can attach 'show/hide' boolean and allow editable scores and whatnot in a nice class that will one day evolve the enum to an editable MetadataConditional/MetadataComparison object
# also have banding so we can have 'at this filesize difference, score 10, at this, score 15'
# show it in a listctrl or whatever in the options, ditch the hardcoding
# metadatacomparison needs to handle 'if one is a png and one is a jpeg', and then orient to A/B and give it a score
def GetDuplicateComparisonStatements( shown_media, comparison_media ):
new_options = HG.client_controller.new_options
duplicate_comparison_score_higher_jpeg_quality = new_options.GetInteger( 'duplicate_comparison_score_higher_jpeg_quality' )
duplicate_comparison_score_much_higher_jpeg_quality = new_options.GetInteger( 'duplicate_comparison_score_much_higher_jpeg_quality' )
duplicate_comparison_score_higher_filesize = new_options.GetInteger( 'duplicate_comparison_score_higher_filesize' )
duplicate_comparison_score_much_higher_filesize = new_options.GetInteger( 'duplicate_comparison_score_much_higher_filesize' )
duplicate_comparison_score_higher_resolution = new_options.GetInteger( 'duplicate_comparison_score_higher_resolution' )
duplicate_comparison_score_much_higher_resolution = new_options.GetInteger( 'duplicate_comparison_score_much_higher_resolution' )
duplicate_comparison_score_more_tags = new_options.GetInteger( 'duplicate_comparison_score_more_tags' )
duplicate_comparison_score_older = new_options.GetInteger( 'duplicate_comparison_score_older' )
duplicate_comparison_score_nicer_ratio = new_options.GetInteger( 'duplicate_comparison_score_nicer_ratio' )
#
statements_and_scores = {}
s_hash = shown_media.GetHash()
c_hash = comparison_media.GetHash()
s_mime = shown_media.GetMime()
c_mime = comparison_media.GetMime()
# size
s_size = shown_media.GetSize()
c_size = comparison_media.GetSize()
is_a_pixel_dupe = False
if shown_media.IsStaticImage() and comparison_media.IsStaticImage() and shown_media.GetResolution() == comparison_media.GetResolution():
global hashes_to_pixel_hashes
if s_hash not in hashes_to_pixel_hashes:
path = HG.client_controller.client_files_manager.GetFilePath( s_hash, s_mime )
hashes_to_pixel_hashes[ s_hash ] = HydrusImageHandling.GetImagePixelHash( path, s_mime )
if c_hash not in hashes_to_pixel_hashes:
path = HG.client_controller.client_files_manager.GetFilePath( c_hash, c_mime )
hashes_to_pixel_hashes[ c_hash ] = HydrusImageHandling.GetImagePixelHash( path, c_mime )
s_pixel_hash = hashes_to_pixel_hashes[ s_hash ]
c_pixel_hash = hashes_to_pixel_hashes[ c_hash ]
if s_pixel_hash == c_pixel_hash:
is_a_pixel_dupe = True
if s_mime == HC.IMAGE_PNG and c_mime != HC.IMAGE_PNG:
statement = 'this is a pixel-for-pixel duplicate png!'
score = -100
elif s_mime != HC.IMAGE_PNG and c_mime == HC.IMAGE_PNG:
statement = 'other file is a pixel-for-pixel duplicate png!'
score = 100
else:
statement = 'images are pixel-for-pixel duplicates!'
score = 0
statements_and_scores[ 'pixel_duplicates' ] = ( statement, score )
if s_size != c_size:
absolute_size_ratio = max( s_size, c_size ) / min( s_size, c_size )
if absolute_size_ratio > 2.0:
if s_size > c_size:
operator = '>>'
score = duplicate_comparison_score_much_higher_filesize
else:
operator = '<<'
score = -duplicate_comparison_score_much_higher_filesize
elif absolute_size_ratio > 1.05:
if s_size > c_size:
operator = '>'
score = duplicate_comparison_score_higher_filesize
else:
operator = '<'
score = -duplicate_comparison_score_higher_filesize
else:
operator = CC.UNICODE_ALMOST_EQUAL_TO
score = 0
if s_size > c_size:
sign = '+'
percentage_difference = ( s_size / c_size ) - 1.0
else:
sign = ''
percentage_difference = ( s_size / c_size ) - 1.0
percentage_different_string = ' ({}{})'.format( sign, HydrusData.ConvertFloatToPercentage( percentage_difference ) )
if is_a_pixel_dupe:
score = 0
statement = '{} {} {}{}'.format( HydrusData.ToHumanBytes( s_size ), operator, HydrusData.ToHumanBytes( c_size ), percentage_different_string )
statements_and_scores[ 'filesize' ] = ( statement, score )
# higher/same res
s_resolution = shown_media.GetResolution()
c_resolution = comparison_media.GetResolution()
if s_resolution != c_resolution:
( s_w, s_h ) = s_resolution
( c_w, c_h ) = c_resolution
all_measurements_are_good = None not in ( s_w, s_h, c_w, c_h ) and True not in ( d <= 0 for d in ( s_w, s_h, c_w, c_h ) )
if all_measurements_are_good:
resolution_ratio = ( s_w * s_h ) / ( c_w * c_h )
if resolution_ratio == 1.0:
operator = '!='
score = 0
elif resolution_ratio > 2.0:
operator = '>>'
score = duplicate_comparison_score_much_higher_resolution
elif resolution_ratio > 1.00:
operator = '>'
score = duplicate_comparison_score_higher_resolution
elif resolution_ratio < 0.5:
operator = '<<'
score = -duplicate_comparison_score_much_higher_resolution
else:
operator = '<'
score = -duplicate_comparison_score_higher_resolution
if s_resolution in HC.NICE_RESOLUTIONS:
s_string = HC.NICE_RESOLUTIONS[ s_resolution ]
else:
s_string = HydrusData.ConvertResolutionToPrettyString( s_resolution )
if s_w % 2 == 1 or s_h % 2 == 1:
s_string += ' (unusual)'
if c_resolution in HC.NICE_RESOLUTIONS:
c_string = HC.NICE_RESOLUTIONS[ c_resolution ]
else:
c_string = HydrusData.ConvertResolutionToPrettyString( c_resolution )
if c_w % 2 == 1 or c_h % 2 == 1:
c_string += ' (unusual)'
statement = '{} {} {}'.format( s_string, operator, c_string )
statements_and_scores[ 'resolution' ] = ( statement, score )
#
s_ratio = s_w / s_h
c_ratio = c_w / c_h
s_nice = s_ratio in HC.NICE_RATIOS
c_nice = c_ratio in HC.NICE_RATIOS
if s_nice or c_nice:
if s_nice:
s_string = HC.NICE_RATIOS[ s_ratio ]
else:
s_string = 'unusual'
if c_nice:
c_string = HC.NICE_RATIOS[ c_ratio ]
else:
c_string = 'unusual'
if s_nice and c_nice:
operator = '-'
score = 0
elif s_nice:
operator = '>'
score = duplicate_comparison_score_nicer_ratio
elif c_nice:
operator = '<'
score = -duplicate_comparison_score_nicer_ratio
if s_string == c_string:
statement = 'both {}'.format( s_string )
else:
statement = '{} {} {}'.format( s_string, operator, c_string )
statements_and_scores[ 'ratio' ] = ( statement, score )
# same/diff mime
if s_mime != c_mime:
statement = '{} vs {}'.format( HC.mime_string_lookup[ s_mime ], HC.mime_string_lookup[ c_mime ] )
score = 0
statements_and_scores[ 'mime' ] = ( statement, score )
# more tags
s_num_tags = len( shown_media.GetTagsManager().GetCurrentAndPending( CC.COMBINED_TAG_SERVICE_KEY, ClientTags.TAG_DISPLAY_ACTUAL ) )
c_num_tags = len( comparison_media.GetTagsManager().GetCurrentAndPending( CC.COMBINED_TAG_SERVICE_KEY, ClientTags.TAG_DISPLAY_ACTUAL ) )
if s_num_tags != c_num_tags:
if s_num_tags > 0 and c_num_tags > 0:
if s_num_tags > c_num_tags:
operator = '>'
score = duplicate_comparison_score_more_tags
else:
operator = '<'
score = -duplicate_comparison_score_more_tags
elif s_num_tags > 0:
operator = '>>'
score = duplicate_comparison_score_more_tags
elif c_num_tags > 0:
operator = '<<'
score = -duplicate_comparison_score_more_tags
statement = '{} tags {} {} tags'.format( HydrusData.ToHumanInt( s_num_tags ), operator, HydrusData.ToHumanInt( c_num_tags ) )
statements_and_scores[ 'num_tags' ] = ( statement, score )
# older
s_ts = shown_media.GetLocationsManager().GetCurrentTimestamp( CC.COMBINED_LOCAL_FILE_SERVICE_KEY )
c_ts = comparison_media.GetLocationsManager().GetCurrentTimestamp( CC.COMBINED_LOCAL_FILE_SERVICE_KEY )
one_month = 86400 * 30
if s_ts is not None and c_ts is not None and abs( s_ts - c_ts ) > one_month:
if s_ts < c_ts:
operator = 'older than'
score = duplicate_comparison_score_older
else:
operator = 'newer than'
score = -duplicate_comparison_score_older
if is_a_pixel_dupe:
score = 0
statement = '{}, {} {}'.format( ClientData.TimestampToPrettyTimeDelta( s_ts, history_suffix = ' old' ), operator, ClientData.TimestampToPrettyTimeDelta( c_ts, history_suffix = ' old' ) )
statements_and_scores[ 'time_imported' ] = ( statement, score )
if s_mime == HC.IMAGE_JPEG and c_mime == HC.IMAGE_JPEG:
global hashes_to_jpeg_quality
if s_hash not in hashes_to_jpeg_quality:
path = HG.client_controller.client_files_manager.GetFilePath( s_hash, s_mime )
hashes_to_jpeg_quality[ s_hash ] = HydrusImageHandling.GetJPEGQuantizationQualityEstimate( path )
if c_hash not in hashes_to_jpeg_quality:
path = HG.client_controller.client_files_manager.GetFilePath( c_hash, c_mime )
hashes_to_jpeg_quality[ c_hash ] = HydrusImageHandling.GetJPEGQuantizationQualityEstimate( path )
( s_label, s_jpeg_quality ) = hashes_to_jpeg_quality[ s_hash ]
( c_label, c_jpeg_quality ) = hashes_to_jpeg_quality[ c_hash ]
score = 0
if s_label != c_label:
if c_jpeg_quality is None or s_jpeg_quality is None:
score = 0
else:
# other way around, low score is good here
quality_ratio = c_jpeg_quality / s_jpeg_quality
if quality_ratio > 2.0:
score = duplicate_comparison_score_much_higher_jpeg_quality
elif quality_ratio > 1.0:
score = duplicate_comparison_score_higher_jpeg_quality
elif quality_ratio < 0.5:
score = -duplicate_comparison_score_much_higher_jpeg_quality
else:
score = -duplicate_comparison_score_higher_jpeg_quality
statement = '{} vs {} jpeg quality'.format( s_label, c_label )
statements_and_scores[ 'jpeg_quality' ] = ( statement, score )
def has_exif( m ):
try:
hash = m.GetHash()
mime = m.GetMime()
if mime not in ( HC.IMAGE_JPEG, HC.IMAGE_TIFF ):
return False
path = HG.client_controller.client_files_manager.GetFilePath( hash, mime )
pil_image = HydrusImageHandling.RawOpenPILImage( path )
exif_dict = HydrusImageHandling.GetEXIFDict( pil_image )
if exif_dict is None:
return False
return len( exif_dict ) > 0
except:
return False
s_has_exif = has_exif( shown_media )
c_has_exif = has_exif( comparison_media )
if s_has_exif ^ c_has_exif:
if s_has_exif:
exif_statement = 'has exif data, the other does not'
else:
exif_statement = 'the other has exif data, this does not'
statements_and_scores[ 'exif_data' ] = ( exif_statement, 0 )
s_has_human_readable_embedded_metadata = shown_media.GetMediaResult().GetFileInfoManager().has_human_readable_embedded_metadata
c_has_human_readable_embedded_metadata = comparison_media.GetMediaResult().GetFileInfoManager().has_human_readable_embedded_metadata
if s_has_human_readable_embedded_metadata ^ c_has_human_readable_embedded_metadata:
if s_has_human_readable_embedded_metadata:
embedded_metadata_statement = 'has embedded metadata, the other does not'
else:
embedded_metadata_statement = 'the other has embedded metadata, this does not'
statements_and_scores[ 'embedded_metadata' ] = ( embedded_metadata_statement, 0 )
s_has_icc = shown_media.GetMediaResult().GetFileInfoManager().has_icc_profile
c_has_icc = comparison_media.GetMediaResult().GetFileInfoManager().has_icc_profile
if s_has_icc ^ c_has_icc:
if s_has_icc:
icc_statement = 'has icc profile, the other does not'
else:
icc_statement = 'the other has icc profile, this does not'
statements_and_scores[ 'icc_profile' ] = ( icc_statement, 0 )
return statements_and_scores
class DuplicatesManager( object ):
my_instance = None
def __init__( self ):
DuplicatesManager.my_instance = self
self._similar_files_maintenance_status = None
self._currently_refreshing_maintenance_numbers = False
self._refresh_maintenance_numbers = True
self._currently_doing_potentials_search = False
self._lock = threading.Lock()
@staticmethod
def instance() -> 'DuplicatesManager':
if DuplicatesManager.my_instance is None:
DuplicatesManager()
return DuplicatesManager.my_instance
def GetMaintenanceNumbers( self ):
with self._lock:
if self._refresh_maintenance_numbers and not self._currently_refreshing_maintenance_numbers:
self._refresh_maintenance_numbers = False
self._currently_refreshing_maintenance_numbers = True
HG.client_controller.pub( 'new_similar_files_maintenance_numbers' )
HG.client_controller.CallToThread( self.THREADRefreshMaintenanceNumbers )
return ( self._similar_files_maintenance_status, self._currently_refreshing_maintenance_numbers, self._currently_doing_potentials_search )
def RefreshMaintenanceNumbers( self ):
with self._lock:
self._refresh_maintenance_numbers = True
HG.client_controller.pub( 'new_similar_files_maintenance_numbers' )
def NotifyNewPotentialsSearchNumbers( self ):
HG.client_controller.pub( 'new_similar_files_potentials_search_numbers' )
def StartPotentialsSearch( self ):
with self._lock:
if self._currently_doing_potentials_search or self._similar_files_maintenance_status is None:
return
self._currently_doing_potentials_search = True
HG.client_controller.CallToThreadLongRunning( self.THREADSearchPotentials )
def THREADRefreshMaintenanceNumbers( self ):
try:
similar_files_maintenance_status = HG.client_controller.Read( 'similar_files_maintenance_status' )
with self._lock:
self._similar_files_maintenance_status = similar_files_maintenance_status
if self._refresh_maintenance_numbers:
self._refresh_maintenance_numbers = False
HG.client_controller.CallToThread( self.THREADRefreshMaintenanceNumbers )
else:
self._currently_refreshing_maintenance_numbers = False
self._refresh_maintenance_numbers = False
HG.client_controller.pub( 'new_similar_files_maintenance_numbers' )
except:
self._currently_refreshing_maintenance_numbers = False
HG.client_controller.pub( 'new_similar_files_maintenance_numbers' )
raise
def THREADSearchPotentials( self ):
try:
search_distance = HG.client_controller.new_options.GetInteger( 'similar_files_duplicate_pairs_search_distance' )
with self._lock:
if self._similar_files_maintenance_status is None:
return
searched_distances_to_count = self._similar_files_maintenance_status
total_num_files = sum( searched_distances_to_count.values() )
num_searched = sum( ( count for ( value, count ) in searched_distances_to_count.items() if value is not None and value >= search_distance ) )
all_files_searched = num_searched >= total_num_files
if all_files_searched:
return # no work to do
num_searched_estimate = num_searched
HG.client_controller.pub( 'new_similar_files_maintenance_numbers' )
job_key = ClientThreading.JobKey( cancellable = True )
job_key.SetStatusTitle( 'searching for potential duplicates' )
HG.client_controller.pub( 'message', job_key )
still_work_to_do = True
while still_work_to_do:
search_distance = HG.client_controller.new_options.GetInteger( 'similar_files_duplicate_pairs_search_distance' )
start_time = HydrusData.GetNowPrecise()
( still_work_to_do, num_done ) = HG.client_controller.WriteSynchronous( 'maintain_similar_files_search_for_potential_duplicates', search_distance, maintenance_mode = HC.MAINTENANCE_FORCED, job_key = job_key, work_time_float = 0.5 )
time_it_took = HydrusData.GetNowPrecise() - start_time
num_searched_estimate += num_done
if num_searched_estimate > total_num_files:
similar_files_maintenance_status = HG.client_controller.Read( 'similar_files_maintenance_status' )
if similar_files_maintenance_status is None:
break
with self._lock:
self._similar_files_maintenance_status = similar_files_maintenance_status
searched_distances_to_count = self._similar_files_maintenance_status
total_num_files = max( num_searched_estimate, sum( searched_distances_to_count.values() ) )
text = 'searching: {}'.format( HydrusData.ConvertValueRangeToPrettyString( num_searched_estimate, total_num_files ) )
job_key.SetVariable( 'popup_text_1', text )
job_key.SetVariable( 'popup_gauge_1', ( num_searched_estimate, total_num_files ) )
if job_key.IsCancelled() or HG.model_shutdown:
break
time.sleep( min( 5, time_it_took ) ) # ideally 0.5s, but potentially longer
job_key.Delete()
finally:
with self._lock:
self._currently_doing_potentials_search = False
self.RefreshMaintenanceNumbers()
self.NotifyNewPotentialsSearchNumbers()
SYNC_ARCHIVE_NONE = 0
SYNC_ARCHIVE_IF_ONE_DO_BOTH = 1
SYNC_ARCHIVE_DO_BOTH_REGARDLESS = 2
class DuplicateContentMergeOptions( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_DUPLICATE_CONTENT_MERGE_OPTIONS
SERIALISABLE_NAME = 'Duplicate Content Merge Options'
SERIALISABLE_VERSION = 6
def __init__( self ):
HydrusSerialisable.SerialisableBase.__init__( self )
self._tag_service_actions = []
self._rating_service_actions = []
self._sync_notes_action = HC.CONTENT_MERGE_ACTION_NONE
self._sync_note_import_options = NoteImportOptions.NoteImportOptions()
self._sync_archive_action = SYNC_ARCHIVE_NONE
self._sync_urls_action = HC.CONTENT_MERGE_ACTION_NONE
def _GetSerialisableInfo( self ):
if HG.client_controller.IsBooted():
services_manager = HG.client_controller.services_manager
self._tag_service_actions = [ ( service_key, action, tag_filter ) for ( service_key, action, tag_filter ) in self._tag_service_actions if services_manager.ServiceExists( service_key ) and services_manager.GetServiceType( service_key ) in HC.REAL_TAG_SERVICES ]
self._rating_service_actions = [ ( service_key, action ) for ( service_key, action ) in self._rating_service_actions if services_manager.ServiceExists( service_key ) and services_manager.GetServiceType( service_key ) in ( HC.LOCAL_RATING_LIKE, HC.LOCAL_RATING_NUMERICAL ) ]
serialisable_tag_service_actions = [ ( service_key.hex(), action, tag_filter.GetSerialisableTuple() ) for ( service_key, action, tag_filter ) in self._tag_service_actions ]
serialisable_rating_service_actions = [ ( service_key.hex(), action ) for ( service_key, action ) in self._rating_service_actions ]
serialisable_sync_note_import_options = self._sync_note_import_options.GetSerialisableTuple()
return ( serialisable_tag_service_actions, serialisable_rating_service_actions, self._sync_notes_action, serialisable_sync_note_import_options, self._sync_archive_action, self._sync_urls_action )
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_tag_service_actions, serialisable_rating_service_actions, self._sync_notes_action, serialisable_sync_note_import_options, self._sync_archive_action, self._sync_urls_action ) = serialisable_info
self._tag_service_actions = [ ( bytes.fromhex( serialisable_service_key ), action, HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_filter ) ) for ( serialisable_service_key, action, serialisable_tag_filter ) in serialisable_tag_service_actions ]
self._rating_service_actions = [ ( bytes.fromhex( serialisable_service_key ), action ) for ( serialisable_service_key, action ) in serialisable_rating_service_actions ]
self._sync_note_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_sync_note_import_options )
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
if version == 1:
( serialisable_service_actions, delete_second_file ) = old_serialisable_info
tag_service_actions = []
rating_service_actions = []
# As the client isn't booted when this is loaded in options, there isn't a good way to figure out tag from rating
# So, let's just dupe and purge later on, in serialisation
for ( service_key_encoded, action ) in serialisable_service_actions:
service_key = bytes.fromhex( service_key_encoded )
tag_filter = HydrusTags.TagFilter()
tag_service_actions.append( ( service_key, action, tag_filter ) )
rating_service_actions.append( ( service_key, action ) )
serialisable_tag_service_actions = [ ( service_key.hex(), action, tag_filter.GetSerialisableTuple() ) for ( service_key, action, tag_filter ) in tag_service_actions ]
serialisable_rating_service_actions = [ ( service_key.hex(), action ) for ( service_key, action ) in rating_service_actions ]
sync_archive = delete_second_file
delete_both_files = False
new_serialisable_info = ( serialisable_tag_service_actions, serialisable_rating_service_actions, delete_second_file, sync_archive, delete_both_files )
return ( 2, new_serialisable_info )
if version == 2:
( serialisable_tag_service_actions, serialisable_rating_service_actions, delete_second_file, sync_archive, delete_both_files ) = old_serialisable_info
sync_urls_action = None
new_serialisable_info = ( serialisable_tag_service_actions, serialisable_rating_service_actions, delete_second_file, sync_archive, delete_both_files, sync_urls_action )
return ( 3, new_serialisable_info )
if version == 3:
( serialisable_tag_service_actions, serialisable_rating_service_actions, delete_second_file, sync_archive, delete_both_files, sync_urls_action ) = old_serialisable_info
new_serialisable_info = ( serialisable_tag_service_actions, serialisable_rating_service_actions, sync_archive, sync_urls_action )
return ( 4, new_serialisable_info )
if version == 4:
( serialisable_tag_service_actions, serialisable_rating_service_actions, sync_archive, sync_urls_action ) = old_serialisable_info
if sync_archive:
sync_archive_action = SYNC_ARCHIVE_IF_ONE_DO_BOTH
else:
sync_archive_action = SYNC_ARCHIVE_NONE
new_serialisable_info = ( serialisable_tag_service_actions, serialisable_rating_service_actions, sync_archive_action, sync_urls_action )
return ( 5, new_serialisable_info )
if version == 5:
( serialisable_tag_service_actions, serialisable_rating_service_actions, sync_archive_action, sync_urls_action ) = old_serialisable_info
if sync_urls_action is None:
sync_urls_action = HC.CONTENT_MERGE_ACTION_NONE
sync_notes_action = HC.CONTENT_MERGE_ACTION_NONE
sync_note_import_options = NoteImportOptions.NoteImportOptions()
serialisable_sync_note_import_options = sync_note_import_options.GetSerialisableTuple()
new_serialisable_info = ( serialisable_tag_service_actions, serialisable_rating_service_actions, sync_notes_action, serialisable_sync_note_import_options, sync_archive_action, sync_urls_action )
return ( 6, new_serialisable_info )
def GetRatingServiceActions( self ) -> typing.Collection[ tuple ]:
return self._rating_service_actions
def GetTagServiceActions( self ) -> typing.Collection[ tuple ]:
return self._tag_service_actions
def GetSyncArchiveAction( self ) -> int:
return self._sync_archive_action
def GetSyncNotesAction( self ) -> int:
return self._sync_notes_action
def GetSyncNoteImportOptions( self ) -> NoteImportOptions.NoteImportOptions:
return self._sync_note_import_options
def GetSyncURLsAction( self ) -> int:
return self._sync_urls_action
def SetRatingServiceActions( self, rating_service_actions: typing.Collection[ tuple ] ):
self._rating_service_actions = rating_service_actions
def SetTagServiceActions( self, tag_service_actions: typing.Collection[ tuple ] ):
self._tag_service_actions = tag_service_actions
def SetSyncArchiveAction( self, sync_archive_action: int ):
self._sync_archive_action = sync_archive_action
def SetSyncNotesAction( self, sync_notes_action: int ):
self._sync_notes_action = sync_notes_action
def SetSyncNoteImportOptions( self, sync_note_import_options: NoteImportOptions.NoteImportOptions ):
self._sync_note_import_options = sync_note_import_options
def SetSyncURLsAction( self, sync_urls_action: int ):
self._sync_urls_action = sync_urls_action
def ProcessPairIntoContentUpdates( self, first_media: ClientMedia.MediaSingleton, second_media: ClientMedia.MediaSingleton, delete_first = False, delete_second = False, file_deletion_reason = None, do_not_do_deletes = False ):
if file_deletion_reason is None:
file_deletion_reason = 'unknown reason'
service_keys_to_content_updates = collections.defaultdict( list )
first_hash = first_media.GetHash()
second_hash = second_media.GetHash()
first_hashes = { first_hash }
second_hashes = { second_hash }
first_media_result = first_media.GetMediaResult()
second_media_result = second_media.GetMediaResult()
#
services_manager = HG.client_controller.services_manager
for ( service_key, action, tag_filter ) in self._tag_service_actions:
content_updates = []
try:
service = services_manager.GetService( service_key )
except HydrusExceptions.DataMissing:
continue
service_type = service.GetServiceType()
if service_type == HC.LOCAL_TAG:
add_content_action = HC.CONTENT_UPDATE_ADD
elif service_type == HC.TAG_REPOSITORY:
add_content_action = HC.CONTENT_UPDATE_PEND
else:
continue
first_tags = first_media.GetTagsManager().GetCurrentAndPending( service_key, ClientTags.TAG_DISPLAY_STORAGE )
second_tags = second_media.GetTagsManager().GetCurrentAndPending( service_key, ClientTags.TAG_DISPLAY_STORAGE )
first_tags = tag_filter.Filter( first_tags )
second_tags = tag_filter.Filter( second_tags )
if action == HC.CONTENT_MERGE_ACTION_TWO_WAY_MERGE:
first_needs = second_tags.difference( first_tags )
second_needs = first_tags.difference( second_tags )
content_updates.extend( ( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, add_content_action, ( tag, first_hashes ) ) for tag in first_needs ) )
content_updates.extend( ( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, add_content_action, ( tag, second_hashes ) ) for tag in second_needs ) )
elif action == HC.CONTENT_MERGE_ACTION_COPY:
first_needs = second_tags.difference( first_tags )
content_updates.extend( ( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, add_content_action, ( tag, first_hashes ) ) for tag in first_needs ) )
elif service_type == HC.LOCAL_TAG and action == HC.CONTENT_MERGE_ACTION_MOVE:
first_needs = second_tags.difference( first_tags )
content_updates.extend( ( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, add_content_action, ( tag, first_hashes ) ) for tag in first_needs ) )
content_updates.extend( ( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_DELETE, ( tag, second_hashes ) ) for tag in second_tags ) )
if len( content_updates ) > 0:
service_keys_to_content_updates[ service_key ].extend( content_updates )
def worth_updating_rating( source_rating, dest_rating ):
if source_rating is not None:
if dest_rating is None or source_rating > dest_rating:
return True
return False
for ( service_key, action ) in self._rating_service_actions:
content_updates = []
try:
services_manager.GetService( service_key )
except HydrusExceptions.DataMissing:
continue
first_current_value = first_media.GetRatingsManager().GetRating( service_key )
second_current_value = second_media.GetRatingsManager().GetRating( service_key )
if action == HC.CONTENT_MERGE_ACTION_TWO_WAY_MERGE:
if worth_updating_rating( first_current_value, second_current_value ):
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_RATINGS, HC.CONTENT_UPDATE_ADD, ( first_current_value, second_hashes ) ) )
elif worth_updating_rating( second_current_value, first_current_value ):
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_RATINGS, HC.CONTENT_UPDATE_ADD, ( second_current_value, first_hashes ) ) )
elif action == HC.CONTENT_MERGE_ACTION_COPY:
if worth_updating_rating( second_current_value, first_current_value ):
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_RATINGS, HC.CONTENT_UPDATE_ADD, ( second_current_value, first_hashes ) ) )
elif action == HC.CONTENT_MERGE_ACTION_MOVE:
if second_current_value is not None:
if worth_updating_rating( second_current_value, first_current_value ):
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_RATINGS, HC.CONTENT_UPDATE_ADD, ( second_current_value, first_hashes ) ) )
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_RATINGS, HC.CONTENT_UPDATE_ADD, ( None, second_hashes ) ) )
if len( content_updates ) > 0:
service_keys_to_content_updates[ service_key ].extend( content_updates )
#
if self._sync_notes_action != HC.CONTENT_MERGE_ACTION_NONE:
first_names_and_notes = list( first_media.GetNotesManager().GetNamesToNotes().items() )
second_names_and_notes = list( second_media.GetNotesManager().GetNamesToNotes().items() )
content_updates = []
# TODO: rework this to UpdateeNamesToNotes
if self._sync_notes_action == HC.CONTENT_MERGE_ACTION_TWO_WAY_MERGE:
first_service_keys_to_content_updates = self._sync_note_import_options.GetServiceKeysToContentUpdates( first_media_result, second_names_and_notes )
second_service_keys_to_content_updates = self._sync_note_import_options.GetServiceKeysToContentUpdates( second_media_result, first_names_and_notes )
content_updates.extend( first_service_keys_to_content_updates[ CC.LOCAL_NOTES_SERVICE_KEY ] )
content_updates.extend( second_service_keys_to_content_updates[ CC.LOCAL_NOTES_SERVICE_KEY ] )
elif self._sync_notes_action == HC.CONTENT_MERGE_ACTION_COPY:
first_service_keys_to_content_updates = self._sync_note_import_options.GetServiceKeysToContentUpdates( first_media_result, second_names_and_notes )
content_updates.extend( first_service_keys_to_content_updates[ CC.LOCAL_NOTES_SERVICE_KEY ] )
elif self._sync_notes_action == HC.CONTENT_MERGE_ACTION_MOVE:
first_service_keys_to_content_updates = self._sync_note_import_options.GetServiceKeysToContentUpdates( first_media_result, second_names_and_notes )
content_updates.extend( first_service_keys_to_content_updates[ CC.LOCAL_NOTES_SERVICE_KEY ] )
content_updates.extend(
[ HydrusData.ContentUpdate( HC.CONTENT_TYPE_NOTES, HC.CONTENT_UPDATE_DELETE, ( second_hash, name ) ) for ( name, note ) in second_names_and_notes ]
)
if len( content_updates ) > 0:
service_keys_to_content_updates[ CC.LOCAL_NOTES_SERVICE_KEY ].extend( content_updates )
#
content_update_archive_first = HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ARCHIVE, first_hashes )
content_update_archive_second = HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ARCHIVE, second_hashes )
# and not delete_first gubbins here to help out the delete lock lmao. don't want to archive and then try to delete
# TODO: this is obviously a bad solution, so better to refactor this function to return a list of service_keys_to_content_updates and stick the delete command right up top, tested for locks on current info
if self._sync_archive_action == SYNC_ARCHIVE_IF_ONE_DO_BOTH:
if first_media.HasInbox() and second_media.HasArchive() and not delete_first:
service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ].append( content_update_archive_first )
elif first_media.HasArchive() and second_media.HasInbox() and not delete_second:
service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ].append( content_update_archive_second )
elif self._sync_archive_action == SYNC_ARCHIVE_DO_BOTH_REGARDLESS:
if first_media.HasInbox() and not delete_first:
service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ].append( content_update_archive_first )
if second_media.HasInbox() and not delete_second:
service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ].append( content_update_archive_second )
#
if self._sync_urls_action != HC.CONTENT_MERGE_ACTION_NONE:
first_urls = set( first_media.GetLocationsManager().GetURLs() )
second_urls = set( second_media.GetLocationsManager().GetURLs() )
content_updates = []
if self._sync_urls_action == HC.CONTENT_MERGE_ACTION_TWO_WAY_MERGE:
first_needs = second_urls.difference( first_urls )
second_needs = first_urls.difference( second_urls )
if len( first_needs ) > 0:
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( first_needs, first_hashes ) ) )
if len( second_needs ) > 0:
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( second_needs, second_hashes ) ) )
elif self._sync_urls_action == HC.CONTENT_MERGE_ACTION_COPY:
first_needs = second_urls.difference( first_urls )
if len( first_needs ) > 0:
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( first_needs, first_hashes ) ) )
if len( content_updates ) > 0:
service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ].extend( content_updates )
#
deletee_media = []
if delete_first:
deletee_media.append( first_media )
if delete_second:
deletee_media.append( second_media )
for media in deletee_media:
if do_not_do_deletes:
continue
if media.HasDeleteLocked():
ClientMedia.ReportDeleteLockFailures( [ media ] )
continue
if media.GetLocationsManager().IsTrashed():
deletee_service_keys = ( CC.COMBINED_LOCAL_FILE_SERVICE_KEY, )
else:
local_file_service_keys = HG.client_controller.services_manager.GetServiceKeys( ( HC.LOCAL_FILE_DOMAIN, ) )
deletee_service_keys = media.GetLocationsManager().GetCurrent().intersection( local_file_service_keys )
for deletee_service_key in deletee_service_keys:
content_update = HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, media.GetHashes(), reason = file_deletion_reason )
service_keys_to_content_updates[ deletee_service_key ].append( content_update )
#
return service_keys_to_content_updates
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_DUPLICATE_CONTENT_MERGE_OPTIONS ] = DuplicateContentMergeOptions
Reference in New Issue View Git Blame Copy Permalink