Version 204

This commit is contained in:
Hydrus Network Developer 2016-05-04 16:50:55 -05:00
parent 4ec034b181
commit 79ad271710
18 changed files with 876 additions and 285 deletions

View File

@ -8,10 +8,26 @@
<div class="content">
<h3>changelog</h3>
<ul>
<li><h3>version 204</h3></li>
<ul>
<li>current, deleted, pending, and petitioned mappings are now stored on service-separated dynamic tables</li>
<li>any tag service deletion or reset should now only take a few seconds</li>
<li>fixed a physical file deletion bug that seems to have affected (some?) versions of linux and os x</li>
<li>reintroduced a revamped 'clear orphans' database->maintenance routine for the client, which will delete or move orphaned files and delete orphaned thumbnails. run this if you are on os x or linux</li>
<li>improved misc file deletion code</li>
<li>fixed some multiplatform path conversion, and generally improved how paths are stored and retrieved</li>
<li>the network application of a parent tag to all files with the child tag now happens serverside, and only on petitioner-approval of a tag parent relationship. it still occurs locally to the uploader, but is now wrapped up in the parent commit</li>
<li>fixed some numtags search logic</li>
<li>fixed an issue when mixing system:age with any tag-based predicate</li>
<li>media_results db calls no longer require a file domain</li>
<li>'hide inbox and archive preds if either has no files' now defaults to off</li>
<li>added an option to disable OpenCV for gif rendering</li>
<li>the selection tags panel now initialises with the correct tag domain (it was previously always starting as 'all known tags', which is not always true for session pages)</li>
</ul>
<li><h3>version 203</h3></li>
<ul>
<li>thumbnail resize now happens on the fly--feel free to change it as often as you like, it takes no time at all</li>
<li>added 'lexicopgrahic (grouped by namespace)' tag sorting to the selection tags box</li>
<li>added 'lexicographic (grouped by namespace)' tag sorting to the selection tags box</li>
<li>added an option to disable OpenCV for static images under the media options page</li>
<li>panning with the shortcut keys now pans by a twelfth of the media size or canvas size, whichever is smaller</li>
<li>cleared the analyze timestamp cache for both client and server, which will force a reanalyze of the new db files on the next db maintenance run</li>

View File

@ -695,7 +695,7 @@ class LocalBooruCache( object ):
info[ 'hashes_set' ] = set( hashes )
media_results = self._controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, hashes )
media_results = self._controller.Read( 'media_results', hashes )
info[ 'media_results' ] = media_results

View File

@ -1009,15 +1009,13 @@ class Controller( HydrusController.HydrusController ):
query_hash_ids = self.Read( 'file_query_ids', search_context )
service_key = search_context.GetFileServiceKey()
media_results = []
for sub_query_hash_ids in HydrusData.SplitListIntoChunks( query_hash_ids, 256 ):
if query_key.IsCancelled(): return
more_media_results = self.Read( 'media_results_from_ids', service_key, sub_query_hash_ids )
more_media_results = self.Read( 'media_results_from_ids', sub_query_hash_ids )
media_results.extend( more_media_results )

File diff suppressed because it is too large Load Diff

View File

@ -92,7 +92,7 @@ def DAEMONDownloadFiles( controller ):
job_key.SetVariable( 'popup_text_1', 'downloading ' + HydrusData.ConvertIntToPrettyString( num_downloads - len( successful_hashes ) ) + ' files from repositories' )
( media_result, ) = controller.Read( 'media_results', CC.COMBINED_FILE_SERVICE_KEY, ( hash, ) )
( media_result, ) = controller.Read( 'media_results', ( hash, ) )
service_keys = list( media_result.GetLocationsManager().GetCurrent() )

View File

@ -347,7 +347,16 @@ def SortTagsList( tags, sort_type ):
if ':' in tag:
return tag.split( ':', 1 )
( namespace, subtag ) = tag.split( ':', 1 )
if namespace == '':
return ( '{', subtag )
else:
return ( namespace, subtag )
else:
@ -436,13 +445,14 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
self._dictionary[ 'booleans' ][ 'apply_all_parents_to_all_services' ] = False
self._dictionary[ 'booleans' ][ 'apply_all_siblings_to_all_services' ] = False
self._dictionary[ 'booleans' ][ 'filter_inbox_and_archive_predicates' ] = True
self._dictionary[ 'booleans' ][ 'filter_inbox_and_archive_predicates' ] = False
self._dictionary[ 'booleans' ][ 'waiting_politely_text' ] = False
self._dictionary[ 'booleans' ][ 'show_thumbnail_title_banner' ] = True
self._dictionary[ 'booleans' ][ 'show_thumbnail_page' ] = True
self._dictionary[ 'booleans' ][ 'disable_cv_for_static_images' ] = False
self._dictionary[ 'booleans' ][ 'disable_cv_for_gifs' ] = False
self._dictionary[ 'noneable_integers' ] = {}

View File

@ -124,9 +124,9 @@ def GetExportPath():
options = HydrusGlobals.client_controller.GetOptions()
path = options[ 'export_path' ]
portable_path = options[ 'export_path' ]
if path is None:
if portable_path is None:
path = os.path.join( os.path.expanduser( '~' ), 'hydrus_export' )
@ -135,10 +135,10 @@ def GetExportPath():
os.makedirs( path )
path = os.path.normpath( path ) # converts slashes to backslashes for windows
path = HydrusPaths.ConvertPortablePathToAbsPath( path )
else:
path = HydrusPaths.ConvertPortablePathToAbsPath( portable_path )
return path
@ -466,7 +466,7 @@ class ExportFolder( HydrusSerialisable.SerialisableBaseNamed ):
sub_query_hash_ids = query_hash_ids[ last_i : i ]
more_media_results = HydrusGlobals.client_controller.Read( 'media_results_from_ids', CC.LOCAL_FILE_SERVICE_KEY, sub_query_hash_ids )
more_media_results = HydrusGlobals.client_controller.Read( 'media_results_from_ids', sub_query_hash_ids )
media_results.extend( more_media_results )

View File

@ -515,6 +515,41 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
def _ClearOrphans( self ):
text = 'This will iterate through every file in your database\'s file storage, removing any it does not expect to be there. It may take some time.'
with ClientGUIDialogs.DialogYesNo( self, text, yes_label = 'do it', no_label = 'forget it' ) as dlg:
if dlg.ShowModal() == wx.ID_YES:
text = 'What would you like to do with the orphaned files? Note that all orphaned thumbnails will be deleted.'
with ClientGUIDialogs.DialogYesNo( self, text, title = 'Choose what do to with the orphans.', yes_label = 'move them somewhere', no_label = 'delete them' ) as dlg_2:
result = dlg_2.ShowModal()
if result == wx.ID_YES:
with wx.DirDialog( self, 'Select location.' ) as dlg_3:
if dlg_3.ShowModal() == wx.ID_OK:
path = HydrusData.ToUnicode( dlg_3.GetPath() )
self._controller.Write( 'clear_orphans', path )
elif result == wx.ID_NO:
self._controller.Write( 'clear_orphans' )
def _CloseCurrentPage( self, polite = True ):
selection = self._notebook.GetSelection()
@ -890,6 +925,7 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
submenu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'rebalance_client_files' ), p( '&Rebalance File Storage' ), p( 'Move your files around your chosen storage directories until they satisfy the weights you have set in the options.' ) )
submenu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'regenerate_thumbnails' ), p( '&Regenerate All Thumbnails' ), p( 'Delete all thumbnails and regenerate from original files.' ) )
submenu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'file_integrity' ), p( '&Check File Integrity' ), p( 'Review and fix all local file records.' ) )
submenu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'clear_orphans' ), p( '&Clear Orphans' ), p( 'Clear out surplus files that have found their way into the database.' ) )
submenu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'check_db_integrity' ), p( 'Check Database Integrity' ) )
menu.AppendMenu( CC.ID_NULL, p( '&Maintenance' ), submenu )
@ -1241,13 +1277,11 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
if len( initial_hashes ) > 0:
file_service_key = management_controller.GetKey( 'file_service' )
initial_media_results = []
for group_of_inital_hashes in HydrusData.SplitListIntoChunks( initial_hashes, 256 ):
more_media_results = self._controller.Read( 'media_results', file_service_key, group_of_inital_hashes )
more_media_results = self._controller.Read( 'media_results', group_of_inital_hashes )
initial_media_results.extend( more_media_results )
@ -2304,6 +2338,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
elif command == 'backup_service': self._BackupService( data )
elif command == 'check_db_integrity': self._CheckDBIntegrity()
elif command == 'clear_caches': self._controller.ClearCaches()
elif command == 'clear_orphans': self._ClearOrphans()
elif command == 'close_page': self._CloseCurrentPage()
elif command == 'db_profile_mode':
@ -3473,7 +3508,7 @@ class FrameReviewServices( ClientGUICommon.Frame ):
for ( name, text, timeout, ( num_hashes, hashes, share_key ) ) in self._booru_shares.GetSelectedClientData():
media_results = self._controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, hashes )
media_results = self._controller.Read( 'media_results', hashes )
self._controller.pub( 'new_page_query', CC.LOCAL_FILE_SERVICE_KEY, initial_media_results = media_results )

View File

@ -4527,7 +4527,7 @@ class PopupMessage( PopupWindow ):
hashes = self._job_key.GetVariable( 'popup_files' )
media_results = HydrusGlobals.client_controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, hashes )
media_results = HydrusGlobals.client_controller.Read( 'media_results', hashes )
HydrusGlobals.client_controller.pub( 'new_page_query', CC.LOCAL_FILE_SERVICE_KEY, initial_media_results = media_results )

View File

@ -4363,6 +4363,7 @@ class DialogManageOptions( ClientGUIDialogs.Dialog ):
self._animation_start_position = wx.SpinCtrl( self, min = 0, max = 100 )
self._disable_cv_for_static_images = wx.CheckBox( self, label = '' )
self._disable_cv_for_gifs = wx.CheckBox( self, label = '' )
self._mime_media_viewer_panel = ClientGUICommon.StaticBox( self, 'media viewer mime handling' )
@ -4391,6 +4392,7 @@ class DialogManageOptions( ClientGUIDialogs.Dialog ):
self._fit_to_canvas.SetValue( HC.options[ 'fit_to_canvas' ] )
self._animation_start_position.SetValue( int( HC.options[ 'animation_start_position' ] * 100.0 ) )
self._disable_cv_for_static_images.SetValue( self._new_options.GetBoolean( 'disable_cv_for_static_images' ) )
self._disable_cv_for_gifs.SetValue( self._new_options.GetBoolean( 'disable_cv_for_gifs' ) )
gridbox = wx.FlexGridSizer( 0, 2 )
@ -4421,6 +4423,9 @@ class DialogManageOptions( ClientGUIDialogs.Dialog ):
gridbox.AddF( wx.StaticText( self, label = 'Disable OpenCV for static images: ' ), CC.FLAGS_MIXED )
gridbox.AddF( self._disable_cv_for_static_images, CC.FLAGS_MIXED )
gridbox.AddF( wx.StaticText( self, label = 'Disable OpenCV for gifs: ' ), CC.FLAGS_MIXED )
gridbox.AddF( self._disable_cv_for_gifs, CC.FLAGS_MIXED )
vbox.AddF( gridbox, CC.FLAGS_EXPAND_PERPENDICULAR )
vbox.AddF( self._mime_media_viewer_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
@ -4442,6 +4447,7 @@ class DialogManageOptions( ClientGUIDialogs.Dialog ):
HC.options[ 'mime_media_viewer_actions' ] = mime_media_viewer_actions
self._new_options.SetBoolean( 'disable_cv_for_static_images', self._disable_cv_for_static_images.GetValue() )
self._new_options.SetBoolean( 'disable_cv_for_gifs', self._disable_cv_for_gifs.GetValue() )
@ -9797,7 +9803,7 @@ class DialogManageTags( ClientGUIDialogs.Dialog ):
hashes = { hash for hash in itertools.chain.from_iterable( ( m.GetHashes() for m in media ) ) }
if len( hashes ) > 0: media_results = HydrusGlobals.client_controller.Read( 'media_results', self._file_service_key, hashes )
if len( hashes ) > 0: media_results = HydrusGlobals.client_controller.Read( 'media_results', hashes )
else: media_results = []
# this should now be a nice clean copy of the original media

View File

@ -2485,7 +2485,7 @@ class ManagementPanelPetitions( ManagementPanel ):
file_service_key = self._management_controller.GetKey( 'file_service' )
with wx.BusyCursor(): media_results = self._controller.Read( 'media_results', file_service_key, hashes )
with wx.BusyCursor(): media_results = self._controller.Read( 'media_results', hashes )
panel = ClientGUIMedia.MediaPanelThumbnails( self._page, self._page_key, file_service_key, media_results )
@ -2724,6 +2724,12 @@ class ManagementPanelQuery( ManagementPanel ):
t = ClientGUICommon.ListBoxTagsSelectionManagementPanel( tags_box, self._page_key, predicates_callable = self._current_predicates_box.GetPredicates )
file_search_context = self._management_controller.GetVariable( 'file_search_context' )
tag_service_key = file_search_context.GetTagServiceKey()
t.ChangeTagService( tag_service_key )
else:
t = ClientGUICommon.ListBoxTagsSelectionManagementPanel( tags_box, self._page_key )

View File

@ -1017,7 +1017,7 @@ class MediaPanel( ClientMedia.ListeningMediaList, wx.ScrolledWindow ):
if hashes is not None and len( hashes ) > 0:
media_results = HydrusGlobals.client_controller.Read( 'media_results', self._file_service_key, hashes )
media_results = HydrusGlobals.client_controller.Read( 'media_results', hashes )
hashes_to_media_results = { media_result.GetHash() : media_result for media_result in media_results }

View File

@ -235,7 +235,7 @@ class GalleryImport( HydrusSerialisable.SerialisableBase ):
HydrusGlobals.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, ( hash, ) )
( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', ( hash, ) )
HydrusGlobals.client_controller.pub( 'add_media_results', page_key, ( media_result, ) )
@ -700,7 +700,7 @@ class HDDImport( HydrusSerialisable.SerialisableBase ):
HydrusGlobals.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, ( hash, ) )
( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', ( hash, ) )
HydrusGlobals.client_controller.pub( 'add_media_results', page_key, ( media_result, ) )
@ -1338,7 +1338,7 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
if status in ( CC.STATUS_SUCCESSFUL, CC.STATUS_REDUNDANT ):
( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, ( hash, ) )
( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', ( hash, ) )
HydrusGlobals.client_controller.pub( 'add_media_results', page_key, ( media_result, ) )
@ -2630,7 +2630,7 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
HydrusGlobals.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, ( hash, ) )
( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', ( hash, ) )
HydrusGlobals.client_controller.pub( 'add_media_results', page_key, ( media_result, ) )

View File

@ -2,6 +2,7 @@ import numpy.core.multiarray # important this comes before cv!
import cv2
import ClientImageHandling
import HydrusExceptions
import HydrusGlobals
import HydrusImageHandling
if cv2.__version__.startswith( '2' ):
@ -61,7 +62,9 @@ class GIFRenderer( object ):
self._num_frames = num_frames
self._target_resolution = target_resolution
if cv2.__version__.startswith( '2' ):
new_options = HydrusGlobals.client_controller.GetNewOptions()
if new_options.GetBoolean( 'disable_cv_for_gifs' ) or cv2.__version__.startswith( '2' ):
self._InitialisePIL()

View File

@ -53,7 +53,7 @@ options = {}
# Misc
NETWORK_VERSION = 17
SOFTWARE_VERSION = 203
SOFTWARE_VERSION = 204
UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )

View File

@ -5,6 +5,7 @@ import os
import psutil
import send2trash
import shutil
import stat
import subprocess
import sys
import tempfile
@ -70,13 +71,39 @@ def CleanUpTempPath( os_file_handle, temp_path ):
def ConvertAbsPathToPortablePath( abs_path ):
try: return os.path.relpath( abs_path, HC.BASE_DIR )
except: return abs_path
try:
portable_path = os.path.relpath( abs_path, HC.BASE_DIR )
except:
portable_path = abs_path
if HC.PLATFORM_WINDOWS:
portable_path = portable_path.replace( '\\', '/' ) # store seps as /, to maintain multiplatform uniformity
return portable_path
def ConvertPortablePathToAbsPath( portable_path ):
if os.path.isabs( portable_path ): abs_path = portable_path
else: abs_path = os.path.normpath( os.path.join( HC.BASE_DIR, portable_path ) )
portable_path = os.path.normpath( portable_path ) # collapses .. stuff and converts / to \\ for windows only
if os.path.isabs( portable_path ):
abs_path = portable_path
else:
abs_path = os.path.join( HC.BASE_DIR, portable_path )
if not HC.PLATFORM_WINDOWS and not os.path.exists( abs_path ):
abs_path = abs_path.replace( '\\', '/' )
return abs_path
@ -127,6 +154,8 @@ def DeletePath( path ):
if os.path.exists( path ):
MakeFileWritable( path )
if os.path.isdir( path ):
shutil.rmtree( path )
@ -222,6 +251,11 @@ def LaunchFile( path ):
thread.start()
def MakeFileWritable( path ):
try: os.chmod( dest_path, stat.S_IWRITE | stat.S_IREAD )
except: pass
def MirrorTree( source, dest ):
pauser = HydrusData.BigJobPauser()
@ -330,6 +364,8 @@ def RecyclePath( path ):
if os.path.exists( path ):
MakeFileWritable( path )
try:
send2trash.send2trash( path )

View File

@ -420,6 +420,13 @@ class DB( HydrusDB.HydrusDB ):
self._c.execute( 'INSERT OR IGNORE INTO tag_parents ( service_id, account_id, old_tag_id, new_tag_id, reason_id, status, timestamp ) VALUES ( ?, ?, ?, ?, ?, ?, ? );', ( service_id, account_id, old_tag_id, new_tag_id, reason_id, new_status, now ) )
if new_status == HC.CURRENT:
child_hash_ids = [ hash_id for ( hash_id, ) in self._c.execute( 'SELECT hash_id FROM mappings WHERE service_id = ? AND tag_id = ?;', ( service_id, old_tag_id ) ) ]
self._AddMappings( service_id, account_id, new_tag_id, child_hash_ids, True )
def _ApproveTagSiblingPetition( self, service_id, account_id, old_tag_id, new_tag_id, reason_id, status ):

View File

@ -51,10 +51,12 @@ class TestClientDB( unittest.TestCase ):
c = db.cursor()
c.execute( 'DELETE FROM current_mappings;' )
c.execute( 'DELETE FROM pending_mappings;' )
c.execute( 'DELETE FROM deleted_mappings;' )
c.execute( 'DELETE FROM mapping_petitions;' )
table_names = [ name for ( name, ) in c.execute( 'SELECT name FROM sqlite_master where type = "table";' ).fetchall() ]
for name in table_names:
c.execute( 'DELETE FROM ' + name + ';' )
def _read( self, action, *args, **kwargs ): return self._db.Read( action, HC.HIGH_PRIORITY, *args, **kwargs )
@ -540,6 +542,8 @@ class TestClientDB( unittest.TestCase ):
predicates = []
predicates.append( ClientSearch.Predicate( HC.PREDICATE_TYPE_SYSTEM_EVERYTHING, None, counts = { HC.CURRENT : 1 } ) )
predicates.append( ClientSearch.Predicate( HC.PREDICATE_TYPE_SYSTEM_INBOX, None, counts = { HC.CURRENT : 1 } ) )
predicates.append( ClientSearch.Predicate( HC.PREDICATE_TYPE_SYSTEM_ARCHIVE, None, counts = { HC.CURRENT : 0 } ) )
predicates.extend( [ ClientSearch.Predicate( predicate_type, None ) for predicate_type in [ HC.PREDICATE_TYPE_SYSTEM_UNTAGGED, HC.PREDICATE_TYPE_SYSTEM_NUM_TAGS, HC.PREDICATE_TYPE_SYSTEM_LIMIT, HC.PREDICATE_TYPE_SYSTEM_SIZE, HC.PREDICATE_TYPE_SYSTEM_AGE, HC.PREDICATE_TYPE_SYSTEM_HASH, HC.PREDICATE_TYPE_SYSTEM_DIMENSIONS, HC.PREDICATE_TYPE_SYSTEM_DURATION, HC.PREDICATE_TYPE_SYSTEM_NUM_WORDS, HC.PREDICATE_TYPE_SYSTEM_MIME, HC.PREDICATE_TYPE_SYSTEM_SIMILAR_TO, HC.PREDICATE_TYPE_SYSTEM_FILE_SERVICE ] ] )
self.assertEqual( result, predicates )
@ -662,7 +666,7 @@ class TestClientDB( unittest.TestCase ):
self.assertEqual( written_result, CC.STATUS_REDUNDANT )
self.assertEqual( written_hash, hash )
( media_result, ) = self._read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, ( written_hash, ) )
( media_result, ) = self._read( 'media_results', ( written_hash, ) )
( mr_hash, mr_inbox, mr_size, mr_mime, mr_width, mr_height, mr_duration, mr_num_frames, mr_num_words, mr_tags_manager, mr_locations_manager, mr_local_ratings, mr_remote_ratings ) = media_result.ToTuple()
@ -791,7 +795,7 @@ class TestClientDB( unittest.TestCase ):
#
( media_result, ) = self._read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, ( hash, ) )
( media_result, ) = self._read( 'media_results', ( hash, ) )
( mr_hash, mr_inbox, mr_size, mr_mime, mr_width, mr_height, mr_duration, mr_num_frames, mr_num_words, mr_tags_manager, mr_locations_manager, mr_local_ratings, mr_remote_ratings ) = media_result.ToTuple()
@ -808,7 +812,7 @@ class TestClientDB( unittest.TestCase ):
self.assertEqual( mr_num_frames, None )
self.assertEqual( mr_num_words, None )
( media_result, ) = self._read( 'media_results_from_ids', CC.LOCAL_FILE_SERVICE_KEY, ( 1, ) )
( media_result, ) = self._read( 'media_results_from_ids', ( 1, ) )
( mr_hash, mr_inbox, mr_size, mr_mime, mr_width, mr_height, mr_duration, mr_num_frames, mr_num_words, mr_tags_manager, mr_locations_manager, mr_local_ratings, mr_remote_ratings ) = media_result.ToTuple()