RepoMirrors
/
hydrus
mirror of https://github.com/hydrusnetwork/hydrus.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Version 371

This commit is contained in:
Hydrus Network Developer 2019-10-09 17:03:03 -05:00
parent 8d75db00f5
commit 2aee0dc94c
30 changed files with 1181 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
help/changelog.html
View File

@ -8,6 +8,35 @@
<div class="content">
<h3>changelog</h3>
<ul>
<li><h3>version 371</h3></li>
<ul>
<li>the edit tag filter panel now has load/save/delete buttons at the top to manage tag filter favourites. it starts with a handful of examples</li>
<li>sorting thumbnails by num tags or namespaces now uses the 'single' tag display context</li>
<li>the 'sort by media views/viewtime' sorts now do not put the other (viewtime/views) as an implicit secondary sort, so as to better let the user's secondary sort be used</li>
<li>highlighting a downloader should now not be able to create a page with duplicate thumbnails</li>
<li>all thumbnail pages now do an addition de-dupe check when they are created with media</li>
<li>when a gallery page parser now adds new urls to a file import list, urls that are invalid will now be skipped (previously, they threw an error and failed the parse</li>
<li>fixed a bug where if a default collect is set, pages without a collect (e.g. download pages) would nonetheless initialise with collected+sorted initial media on session load</li>
<li>file imports now publish the same 'refresh existing media metadata' call as the file maintenance system, meaning if the import already exists in the gui session as an 'unknown thumb', it should now refresh itself correctly</li>
<li>if the media canvas is called to display an invalid media (due to mime mixup or a faulty parse that slips through), it should now better recognise that and skip/dump out</li>
<li>fixed import of videos that have 'Duration:' in their title metadata</li>
<li>improved the error reporting when the old options object fails to save</li>
<li>removed some old ratings dialog position options storage that was causing errors on certain ratings dialog ok events</li>
<li>url classes now support options regarding the 'referer' http header they send (their referral url). you can set an optional converter to generate a referral url based on the url class's url and choose to always use the given referrer if available, never use a referrer, use the converter if no referrer is available, or always use the converter</li>
<li>the network report mode now reports on referral urls used in requests</li>
<li>the 'quoted' referral url (a unicode workaround) is now only applied if the referral url cannot be encoded to latin-1</li>
<li>the janitorial petitions processing page now lets you copy tags and left/right tags of pairs with a right-click on selected checkbox rows</li>
<li>cleaned a little server code</li>
<li>improved how the server sets and releases its 'currently busy' mode</li>
<li>the server no longer does <5min vacuums in a backup command</li>
<li>added a specific 'vacuum' server POST command that forces a full vacuum</li>
<li>added 'lock_on' and 'lock_off' server POST commands to lock the server and shut down the db, and restart</li>
<li>the new vacuum, lock_on, lock_off, and a 'is server busy?' check commands are added to the services->admin menu</li>
<li>added 'pause and disconnect' ability to the database mainloop</li>
<li>added some unit tests for url classes and the new referral url conversions and server commands</li>
<li>cleaned some of the thumbnail banner/icon drawing code</li>
<li>some misc label fixes</li>
</ul>
<li><h3>version 370</h3></li>
<ul>
<li>tag display updates:</li>

125
include/ClientDB.py
View File

@ -1515,6 +1515,51 @@ class DB( HydrusDB.HydrusDB ):
new_options.SetSimpleDownloaderFormulae( ClientDefaults.GetDefaultSimpleDownloaderFormulae() )
names_to_tag_filters = {}
tag_filter = ClientTags.TagFilter()
tag_filter.SetRule( 'diaper', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'gore', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'guro', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'scat', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'vore', CC.FILTER_BLACKLIST )
names_to_tag_filters[ 'example blacklist' ] = tag_filter
tag_filter = ClientTags.TagFilter()
tag_filter.SetRule( '', CC.FILTER_BLACKLIST )
tag_filter.SetRule( ':', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'series:', CC.FILTER_WHITELIST )
tag_filter.SetRule( 'creator:', CC.FILTER_WHITELIST )
tag_filter.SetRule( 'studio:', CC.FILTER_WHITELIST )
tag_filter.SetRule( 'character:', CC.FILTER_WHITELIST )
names_to_tag_filters[ 'basic namespaces only' ] = tag_filter
tag_filter = ClientTags.TagFilter()
tag_filter.SetRule( ':', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'series:', CC.FILTER_WHITELIST )
tag_filter.SetRule( 'creator:', CC.FILTER_WHITELIST )
tag_filter.SetRule( 'studio:', CC.FILTER_WHITELIST )
tag_filter.SetRule( 'character:', CC.FILTER_WHITELIST )
names_to_tag_filters[ 'basic booru tags only' ] = tag_filter
tag_filter = ClientTags.TagFilter()
tag_filter.SetRule( 'title:', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'filename:', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'source:', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'booru:', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'url:', CC.FILTER_BLACKLIST )
names_to_tag_filters[ 'exclude long/spammy namespaces' ] = tag_filter
new_options.SetFavouriteTagFilters( names_to_tag_filters )
self._SetJSONDump( new_options )
list_of_shortcuts = ClientDefaults.GetDefaultShortcuts()
@ -8027,6 +8072,10 @@ class DB( HydrusDB.HydrusDB ):
status = CC.STATUS_SUCCESSFUL_AND_NEW
self._weakref_media_result_cache.DropMediaResult( hash_id, hash )
self._controller.pub( 'new_file_info', set( ( hash, ) ) )
if HG.file_import_report_mode:
@ -10996,7 +11045,17 @@ class DB( HydrusDB.HydrusDB ):
def _SaveOptions( self, options ):
self._c.execute( 'UPDATE options SET options = ?;', ( options, ) )
try:
self._c.execute( 'UPDATE options SET options = ?;', ( options, ) )
except:
HydrusData.Print( 'Failed options save dump:' )
HydrusData.Print( options )
raise
self.pub_after_job( 'reset_thumbnail_cache' )
self.pub_after_job( 'notify_new_options' )
@ -13040,6 +13099,70 @@ class DB( HydrusDB.HydrusDB ):
if version == 370:
try:
new_options = self._GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_CLIENT_OPTIONS )
names_to_tag_filters = {}
tag_filter = ClientTags.TagFilter()
tag_filter.SetRule( 'diaper', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'gore', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'guro', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'scat', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'vore', CC.FILTER_BLACKLIST )
names_to_tag_filters[ 'example blacklist' ] = tag_filter
tag_filter = ClientTags.TagFilter()
tag_filter.SetRule( '', CC.FILTER_BLACKLIST )
tag_filter.SetRule( ':', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'series:', CC.FILTER_WHITELIST )
tag_filter.SetRule( 'creator:', CC.FILTER_WHITELIST )
tag_filter.SetRule( 'studio:', CC.FILTER_WHITELIST )
tag_filter.SetRule( 'character:', CC.FILTER_WHITELIST )
names_to_tag_filters[ 'basic namespaces only' ] = tag_filter
tag_filter = ClientTags.TagFilter()
tag_filter.SetRule( ':', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'series:', CC.FILTER_WHITELIST )
tag_filter.SetRule( 'creator:', CC.FILTER_WHITELIST )
tag_filter.SetRule( 'studio:', CC.FILTER_WHITELIST )
tag_filter.SetRule( 'character:', CC.FILTER_WHITELIST )
tag_filter.SetRule( '', CC.FILTER_WHITELIST )
names_to_tag_filters[ 'basic booru tags only' ] = tag_filter
tag_filter = ClientTags.TagFilter()
tag_filter.SetRule( 'title:', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'filename:', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'source:', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'booru:', CC.FILTER_BLACKLIST )
tag_filter.SetRule( 'url:', CC.FILTER_BLACKLIST )
names_to_tag_filters[ 'exclude long/spammy namespaces' ] = tag_filter
new_options.SetFavouriteTagFilters( names_to_tag_filters )
self._SetJSONDump( new_options )
except Exception as e:
HydrusData.PrintException( e )
message = 'Trying to save new default favourite tag filters failed! Please let hydrus dev know!'
self.pub_initial_message( message )
self._controller.pub( 'splash_set_title_text', 'updated db to v' + str( version + 1 ) )
self._c.execute( 'UPDATE version SET version = ?;', ( version + 1, ) )

2
include/ClientDefaults.py
View File

@ -99,8 +99,6 @@ def GetClientDefaultOptions():
options[ 'pause_repo_sync' ] = False
options[ 'pause_subs_sync' ] = False
options[ 'rating_dialog_position' ] = ( False, None )
return options
def GetDefaultCheckerOptions( name ):

124
include/ClientGUI.py
View File

@ -742,14 +742,12 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
def _BackupService( self, service_key ):
def _BackupServer( self, service_key ):
def do_it():
def do_it( service ):
started = HydrusData.GetNow()
service = self._controller.services_manager.GetService( service_key )
service.Request( HC.POST, 'backup' )
HydrusData.ShowText( 'Server backup started!' )
@ -781,7 +779,9 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
if result == wx.ID_YES:
self._controller.CallToThread( do_it )
service = self._controller.services_manager.GetService( service_key )
self._controller.CallToThread( do_it, service )
@ -2144,7 +2144,14 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
ClientGUIMenus.AppendSeparator( submenu )
ClientGUIMenus.AppendMenuItem( self, submenu, 'manage services', 'Add, edit, and delete this server\'s services.', self._ManageServer, service_key )
ClientGUIMenus.AppendMenuItem( self, submenu, 'make a backup', 'Command the server to temporarily pause and back up its database.', self._BackupService, service_key )
ClientGUIMenus.AppendSeparator( submenu )
ClientGUIMenus.AppendMenuItem( self, submenu, 'backup server', 'Command the server to temporarily pause and back up its database.', self._BackupServer, service_key )
ClientGUIMenus.AppendSeparator( submenu )
ClientGUIMenus.AppendMenuItem( self, submenu, 'vacuum server', 'Command the server to temporarily pause and vacuum its database.', self._VacuumServer, service_key )
ClientGUIMenus.AppendSeparator( submenu )
ClientGUIMenus.AppendMenuItem( self, submenu, 'server/db lock: on', 'Command the server to lock itself and disconnect its db.', self._LockServer, service_key, True )
ClientGUIMenus.AppendMenuItem( self, submenu, 'server/db lock: test', 'See if the server is currently busy.', self._TestServerBusy, service_key )
ClientGUIMenus.AppendMenuItem( self, submenu, 'server/db lock: off', 'Command the server to unlock itself and resume its db.', self._LockServer, service_key, False )
ClientGUIMenus.AppendMenu( admin_menu, submenu, service.GetName() )
@ -2641,6 +2648,43 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
self._clipboard_watcher_repeating_job = self._controller.CallRepeatingWXSafe( self, 1.0, 1.0, self.REPEATINGClipboardWatcher )
def _LockServer( self, service_key, lock ):
def do_it( service, lock ):
if lock:
command = 'lock_on'
done_message = 'Server locked!'
else:
command = 'lock_off'
done_message = 'Server unlocked!'
service.Request( HC.POST, command )
HydrusData.ShowText( done_message )
if lock:
message = 'This will tell the server to lock and disconnect its database, in case you wish to make a db backup using an external program. It will not be able to serve any requests as long as it is locked. It may get funky if it is locked for hours and hours--if you need it paused for that long, I recommend just shutting it down instead.'
result = ClientGUIDialogsQuick.GetYesNo( self, message, yes_label = 'do it', no_label = 'forget it' )
if result != wx.ID_YES:
return
service = self._controller.services_manager.GetService( service_key )
self._controller.CallToThread( do_it, service, lock )
def _ManageAccountTypes( self, service_key ):
title = 'manage account types'
@ -4119,6 +4163,31 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
def _TestServerBusy( self, service_key ):
def do_it( service ):
result_bytes = service.Request( HC.GET, 'busy' )
if result_bytes == b'1':
HydrusData.ShowText( 'server is busy' )
elif result_bytes == b'0':
HydrusData.ShowText( 'server is not busy' )
else:
HydrusData.ShowText( 'server responded in a way I do not understand' )
service = self._controller.services_manager.GetService( service_key )
self._controller.CallToThread( do_it, service )
def _UnclosePage( self, closed_page_index = None ):
if closed_page_index is None:
@ -4197,6 +4266,49 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
def _VacuumServer( self, service_key ):
def do_it( service ):
started = HydrusData.GetNow()
service.Request( HC.POST, 'vacuum' )
HydrusData.ShowText( 'Server vacuum started!' )
time.sleep( 10 )
result_bytes = service.Request( HC.GET, 'busy' )
while result_bytes == b'1':
if HG.view_shutdown:
return
time.sleep( 10 )
result_bytes = service.Request( HC.GET, 'busy' )
it_took = HydrusData.GetNow() - started
HydrusData.ShowText( 'Server vacuum done in ' + HydrusData.TimeDeltaToPrettyTimeDelta( it_took ) + '!' )
message = 'This will tell the server to lock and vacuum its database files. It may take some time to complete, during which time it will not be able to serve any requests.'
result = ClientGUIDialogsQuick.GetYesNo( self, message, yes_label = 'do it', no_label = 'forget it' )
if result == wx.ID_YES:
service = self._controller.services_manager.GetService( service_key )
self._controller.CallToThread( do_it, service )
def AddModalMessage( self, job_key ):
if job_key.IsCancelled() or job_key.IsDeleted():

2
include/ClientGUICanvas.py
View File

@ -1446,7 +1446,7 @@ class Canvas( wx.Window ):
mime = media.GetMime()
if mime == HC.APPLICATION_HYDRUS_CLIENT_COLLECTION:
if mime not in HC.ALLOWED_MIMES: # stopgap to catch a collection or application_unknown due to unusual import order/media moving
return CC.MEDIA_VIEWER_ACTION_DO_NOT_SHOW

8
include/ClientGUICommon.py
View File

@ -705,10 +705,12 @@ class BufferedWindowIcon( BufferedWindow ):
class CheckboxCollect( wx.Panel ):
def __init__( self, parent, management_controller = None ):
def __init__( self, parent, management_controller = None, silent = False ):
wx.Panel.__init__( self, parent )
# this is trash, rewrite it to deal with the media_collect object, not the management controller
self._management_controller = management_controller
if self._management_controller is not None and self._management_controller.HasVariable( 'media_collect' ):
@ -720,6 +722,8 @@ class CheckboxCollect( wx.Panel ):
self._media_collect = HG.client_controller.new_options.GetDefaultCollect()
self._silent = silent
self._collect_comboctrl = wx.ComboCtrl( self, style = wx.CB_READONLY )
self._collect_combopopup = self._Popup( self._media_collect, self )
@ -777,7 +781,7 @@ class CheckboxCollect( wx.Panel ):
self._collect_comboctrl.SetValue( description )
if self._management_controller is not None:
if not self._silent and self._management_controller is not None:
self._management_controller.SetVariable( 'media_collect', self._media_collect )

27
include/ClientGUIDialogsManage.py
View File

@ -61,22 +61,12 @@ class DialogManageRatings( ClientGUIDialogs.Dialog ):
self._hashes = set()
for m in media: self._hashes.update( m.GetHashes() )
( remember, position ) = HC.options[ 'rating_dialog_position' ]
if remember and position is not None:
for m in media:
my_position = 'custom'
wx.CallAfter( self.SetPosition, position )
else:
my_position = 'topleft'
self._hashes.update( m.GetHashes() )
ClientGUIDialogs.Dialog.__init__( self, parent, 'manage ratings for ' + HydrusData.ToHumanInt( len( self._hashes ) ) + ' files', position = my_position )
ClientGUIDialogs.Dialog.__init__( self, parent, 'manage ratings for ' + HydrusData.ToHumanInt( len( self._hashes ) ) + ' files', position = 'topleft' )
#
@ -147,17 +137,6 @@ class DialogManageRatings( ClientGUIDialogs.Dialog ):
HG.client_controller.Write( 'content_updates', service_keys_to_content_updates )
( remember, position ) = HC.options[ 'rating_dialog_position' ]
current_position = self.GetPosition()
if remember and position != current_position:
HC.options[ 'rating_dialog_position' ] = ( remember, current_position )
HG.client_controller.Write( 'save_options', HC.options )
finally:
self.EndModal( wx.ID_OK )

95
include/ClientGUIManagement.py
View File

@ -938,6 +938,8 @@ HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIAL
class ManagementPanel( wx.lib.scrolledpanel.ScrolledPanel ):
SHOW_COLLECT = True
def __init__( self, parent, page, controller, management_controller ):
wx.lib.scrolledpanel.ScrolledPanel.__init__( self, parent, style = wx.BORDER_NONE | wx.VSCROLL )
@ -954,12 +956,19 @@ class ManagementPanel( wx.lib.scrolledpanel.ScrolledPanel ):
self._media_sort = ClientGUICommon.ChoiceSort( self, management_controller = self._management_controller )
self._media_collect = ClientGUICommon.CheckboxCollect( self, management_controller = self._management_controller )
silent_collect = not self.SHOW_COLLECT
self._media_collect = ClientGUICommon.CheckboxCollect( self, management_controller = self._management_controller, silent = silent_collect )
if not self.SHOW_COLLECT:
self._media_collect.Hide()
def GetMediaCollect( self ):
if self._media_collect.IsShown():
if self.SHOW_COLLECT:
return self._media_collect.GetValue()
@ -1046,6 +1055,8 @@ def WaitOnDupeFilterJob( job_key ):
class ManagementPanelDuplicateFilter( ManagementPanel ):
SHOW_COLLECT = False
def __init__( self, parent, page, controller, management_controller ):
ManagementPanel.__init__( self, parent, page, controller, management_controller )
@ -1178,7 +1189,6 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
#
self._media_sort.Hide()
self._media_collect.Hide()
distance_hbox = wx.BoxSizer( wx.HORIZONTAL )
@ -1642,6 +1652,8 @@ management_panel_types_to_classes[ MANAGEMENT_TYPE_DUPLICATE_FILTER ] = Manageme
class ManagementPanelImporter( ManagementPanel ):
SHOW_COLLECT = False
def __init__( self, parent, page, controller, management_controller ):
ManagementPanel.__init__( self, parent, page, controller, management_controller )
@ -1705,8 +1717,6 @@ class ManagementPanelImporterHDD( ManagementPanelImporter ):
vbox.Add( self._media_sort, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
self._media_collect.Hide()
self._import_queue_panel.Add( self._current_action, CC.FLAGS_EXPAND_PERPENDICULAR )
self._import_queue_panel.Add( self._file_seed_cache_control, CC.FLAGS_EXPAND_PERPENDICULAR )
self._import_queue_panel.Add( self._pause_button, CC.FLAGS_LONE_BUTTON )
@ -1876,8 +1886,6 @@ class ManagementPanelImporterMultipleGallery( ManagementPanelImporter ):
vbox.Add( self._media_sort, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
self._media_collect.Hide()
vbox.Add( self._gallery_downloader_panel, CC.FLAGS_EXPAND_BOTH_WAYS )
vbox.Add( self._highlighted_gallery_import_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
@ -2528,8 +2536,6 @@ class ManagementPanelImporterMultipleWatcher( ManagementPanelImporter ):
vbox.Add( self._media_sort, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
self._media_collect.Hide()
vbox.Add( self._watchers_panel, CC.FLAGS_EXPAND_BOTH_WAYS )
vbox.Add( self._highlighted_watcher_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
@ -3251,8 +3257,6 @@ class ManagementPanelImporterSimpleDownloader( ManagementPanelImporter ):
vbox.Add( self._media_sort, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
self._media_collect.Hide()
vbox.Add( self._simple_downloader_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
self._MakeCurrentSelectionTagsBox( vbox )
@ -3619,8 +3623,6 @@ class ManagementPanelImporterURLs( ManagementPanelImporter ):
vbox.Add( self._media_sort, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
self._media_collect.Hide()
vbox.Add( self._url_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
self._MakeCurrentSelectionTagsBox( vbox )
@ -3846,6 +3848,8 @@ class ManagementPanelPetitions( ManagementPanel ):
self.SetSizer( vbox )
self._contents.Bind( wx.EVT_RIGHT_DOWN, self.EventRowRightClick )
self._controller.sub( self, 'RefreshQuery', 'refresh_query' )
@ -4371,6 +4375,71 @@ class ManagementPanelPetitions( ManagementPanel ):
def EventRowRightClick( self, event ):
selected_indices = self._contents.GetSelections()
selected_contents = []
for i in selected_indices:
content = self._contents.GetClientData( i )
selected_contents.append( content )
copyable_items_a = []
copyable_items_b = []
for content in selected_contents:
content_type = content.GetContentType()
if content_type == HC.CONTENT_TYPE_MAPPINGS:
( tag, hashes ) = content.GetContentData()
copyable_items_a.append( tag )
elif content_type in ( HC.CONTENT_TYPE_TAG_SIBLINGS, HC.CONTENT_TYPE_TAG_PARENTS ):
( tag_a, tag_b ) = content.GetContentData()
copyable_items_a.append( tag_a )
copyable_items_b.append( tag_b )
copyable_items_a = HydrusData.DedupeList( copyable_items_a )
copyable_items_b = HydrusData.DedupeList( copyable_items_b )
if len( copyable_items_a ) + len( copyable_items_b ) > 0:
menu = wx.Menu()
for copyable_items in [ copyable_items_a, copyable_items_b ]:
if len( copyable_items ) > 0:
if len( copyable_items ) == 1:
tag = copyable_items[0]
ClientGUIMenus.AppendMenuItem( self, menu, 'copy {}'.format( tag ), 'Copy this tag.', HG.client_controller.pub, 'clipboard', 'text', tag )
else:
text = os.linesep.join( copyable_items )
ClientGUIMenus.AppendMenuItem( self, menu, 'copy {} tags'.format( HydrusData.ToHumanInt( len( copyable_items ) ) ), 'Copy this tag.', HG.client_controller.pub, 'clipboard', 'text', text )
HG.client_controller.PopupMenu( self, menu )
def RefreshQuery( self, page_key ):
if page_key == self._page_key: self._DrawCurrentPetition()

37
include/ClientGUIMedia.py
View File

@ -5008,6 +5008,8 @@ class Thumbnail( Selectable ):
HG.client_controller.bitmap_manager.ReleaseBitmap( wx_bmp )
TEXT_BORDER = 1
new_options = HG.client_controller.new_options
tags = self.GetTagsManager().GetCurrentAndPending( CC.COMBINED_TAG_SERVICE_KEY, ClientTags.TAG_DISPLAY_SINGLE_MEDIA )
@ -5043,12 +5045,17 @@ class Thumbnail( Selectable ):
( text_width, text_height ) = gc.GetTextExtent( upper_summary )
top_left_x = int( ( width - text_width ) // 2 )
top_left_y = thumbnail_border
box_x = thumbnail_border
box_y = thumbnail_border
box_width = width - ( thumbnail_border * 2 )
box_height = text_height + 2
gc.DrawRectangle( thumbnail_border, top_left_y, width - ( thumbnail_border * 2 ), text_height + 1 )
gc.DrawRectangle( box_x, box_y, box_width, box_height )
gc.DrawText( upper_summary, top_left_x, top_left_y )
text_x = ( width - text_width ) // 2
text_y = box_y + TEXT_BORDER
gc.DrawText( upper_summary, text_x, text_y )
if len( lower_summary ) > 0:
@ -5065,12 +5072,17 @@ class Thumbnail( Selectable ):
( text_width, text_height ) = gc.GetTextExtent( lower_summary )
top_left_x = width - text_width - thumbnail_border
top_left_y = height - text_height - thumbnail_border
box_width = text_width + ( TEXT_BORDER * 2 )
box_height = text_height + ( TEXT_BORDER * 2 )
box_x = width - box_width - thumbnail_border
box_y = height - text_height - thumbnail_border
gc.DrawRectangle( top_left_x - 1, top_left_y - 1, text_width + 1, text_height + 1 )
gc.DrawRectangle( box_x, box_y, box_width, box_height )
gc.DrawText( lower_summary, top_left_x, top_left_y )
text_x = box_x + TEXT_BORDER
text_y = box_y + TEXT_BORDER
gc.DrawText( lower_summary, text_x, text_y )
del gc
@ -5211,12 +5223,15 @@ class Thumbnail( Selectable ):
box_x = thumbnail_border + top_left_x
box_y = thumbnail_border
box_width = text_width + 2
box_height = text_height + 2
box_width = text_width + ( TEXT_BORDER * 2 )
box_height = text_height + ( TEXT_BORDER * 2 )
dc.DrawRectangle( box_x, box_y, box_width, box_height )
dc.DrawText( label, box_x + 1, box_y + 1 )
text_x = box_x + TEXT_BORDER
text_y = box_y + TEXT_BORDER
dc.DrawText( label, text_x, text_y )
top_left_x += box_width + 2

95
include/ClientGUIScrolledPanelsEdit.py
View File

@ -4473,15 +4473,12 @@ class EditSubscriptionsPanel( ClientGUIScrolledPanels.EditPanel ):
vbox.Add( help_hbox, CC.FLAGS_BUTTON_SIZER )
if subs_are_globally_paused:
message = 'Subscriptions do not work well if they get too large! If any sub has >200,000 items, separate it into smaller pieces immediately!'
st = ClientGUICommon.BetterStaticText( self, message )
st.SetForegroundColour( ( 127, 0, 0 ) )
vbox.Add( st, CC.FLAGS_EXPAND_PERPENDICULAR )
message = 'Subscriptions do not work well if they get too large! If any sub has >200,000 items, separate it into smaller pieces immediately!'
st = ClientGUICommon.BetterStaticText( self, message )
st.SetForegroundColour( ( 127, 0, 0 ) )
vbox.Add( st, CC.FLAGS_EXPAND_PERPENDICULAR )
if subs_are_globally_paused:
@ -6336,10 +6333,33 @@ class EditURLClassPanel( ClientGUIScrolledPanels.EditPanel ):
self._normalised_url.SetToolTip( tt )
( url_type, preferred_scheme, netloc, match_subdomains, keep_matched_subdomains, path_components, parameters, api_lookup_converter, can_produce_multiple_files, should_be_associated_with_files, example_url ) = url_class.ToTuple()
( url_type, preferred_scheme, netloc, match_subdomains, keep_matched_subdomains, path_components, parameters, api_lookup_converter, send_referral_url, referral_url_converter, can_produce_multiple_files, should_be_associated_with_files, example_url ) = url_class.ToTuple()
self._send_referral_url = ClientGUICommon.BetterChoice( self )
for send_referral_url_type in ClientNetworkingDomain.SEND_REFERRAL_URL_TYPES:
self._send_referral_url.Append( ClientNetworkingDomain.send_referral_url_string_lookup[ send_referral_url_type ], send_referral_url_type )
tt = 'Do not change this unless you know you need to. It fixes complicated problems.'
self._send_referral_url.SetToolTip( tt )
self._referral_url_converter = ClientGUIControls.StringConverterButton( self, referral_url_converter )
tt = 'This will generate a referral URL from the original URL. If the URL needs a referral URL, and you can infer what that would be from just this URL, this will let hydrus download this URL without having to previously visit the referral URL (e.g. letting the user drag-and-drop import). It also lets you set up alternate referral URLs for perculiar situations.'
self._referral_url_converter.SetToolTip( tt )
self._referral_url = wx.TextCtrl( self, style = wx.TE_READONLY )
self._api_lookup_converter = ClientGUIControls.StringConverterButton( self, api_lookup_converter )
tt = 'This will let you generate an alternate URL for the client to use for the actual download whenever it encounters a URL in this class. You must have a separate URL class to match the API type (which will link to parsers).'
self._api_lookup_converter.SetToolTip( tt )
self._api_url = wx.TextCtrl( self, style = wx.TE_READONLY )
self._next_gallery_page_url = wx.TextCtrl( self, style = wx.TE_READONLY )
@ -6420,6 +6440,9 @@ class EditURLClassPanel( ClientGUIScrolledPanels.EditPanel ):
rows.append( ( 'example url: ', self._example_url ) )
rows.append( ( 'normalised url: ', self._normalised_url ) )
rows.append( ( 'send referral url?: ', self._send_referral_url ) )
rows.append( ( 'optional referral url converter: ', self._referral_url_converter ) )
rows.append( ( 'referral url: ', self._referral_url ) )
rows.append( ( 'optional api url converter: ', self._api_lookup_converter ) )
rows.append( ( 'api url: ', self._api_url ) )
rows.append( ( 'next gallery page url: ', self._next_gallery_page_url ) )
@ -6447,7 +6470,10 @@ class EditURLClassPanel( ClientGUIScrolledPanels.EditPanel ):
self._example_url.Bind( wx.EVT_TEXT, self.EventUpdate )
self.Bind( ClientGUIListBoxes.EVT_LIST_BOX, self.EventUpdate )
self._url_type.Bind( wx.EVT_CHOICE, self.EventURLTypeUpdate )
self._send_referral_url.Bind( wx.EVT_CHOICE, self.EventUpdate )
self._referral_url_converter.Bind( ClientGUIControls.EVT_STRING_CONVERTER, self.EventUpdate )
self._api_lookup_converter.Bind( ClientGUIControls.EVT_STRING_CONVERTER, self.EventUpdate )
self._should_be_associated_with_files.Bind( wx.EVT_CHECKBOX, self.EventAssociationUpdate )
@ -6734,13 +6760,15 @@ class EditURLClassPanel( ClientGUIScrolledPanels.EditPanel ):
path_components = self._path_components.GetData()
parameters = dict( self._parameters.GetData() )
api_lookup_converter = self._api_lookup_converter.GetValue()
send_referral_url = self._send_referral_url.GetValue()
referral_url_converter = self._referral_url_converter.GetValue()
( gallery_index_type, gallery_index_identifier ) = self._next_gallery_page_choice.GetValue()
gallery_index_delta = self._next_gallery_page_delta.GetValue()
example_url = self._example_url.GetValue()
url_class = ClientNetworkingDomain.URLClass( name, url_class_key = url_class_key, url_type = url_type, preferred_scheme = preferred_scheme, netloc = netloc, match_subdomains = match_subdomains, keep_matched_subdomains = keep_matched_subdomains, path_components = path_components, parameters = parameters, api_lookup_converter = api_lookup_converter, can_produce_multiple_files = can_produce_multiple_files, should_be_associated_with_files = should_be_associated_with_files, gallery_index_type = gallery_index_type, gallery_index_identifier = gallery_index_identifier, gallery_index_delta = gallery_index_delta, example_url = example_url )
url_class = ClientNetworkingDomain.URLClass( name, url_class_key = url_class_key, url_type = url_type, preferred_scheme = preferred_scheme, netloc = netloc, match_subdomains = match_subdomains, keep_matched_subdomains = keep_matched_subdomains, path_components = path_components, parameters = parameters, api_lookup_converter = api_lookup_converter, send_referral_url = send_referral_url, referral_url_converter = referral_url_converter, can_produce_multiple_files = can_produce_multiple_files, should_be_associated_with_files = should_be_associated_with_files, gallery_index_type = gallery_index_type, gallery_index_identifier = gallery_index_identifier, gallery_index_delta = gallery_index_delta, example_url = example_url )
return url_class
@ -6834,6 +6862,7 @@ class EditURLClassPanel( ClientGUIScrolledPanels.EditPanel ):
example_url = self._example_url.GetValue()
self._referral_url_converter.SetExampleString( example_url )
self._api_lookup_converter.SetExampleString( example_url )
url_class.Test( example_url )
@ -6845,6 +6874,50 @@ class EditURLClassPanel( ClientGUIScrolledPanels.EditPanel ):
self._normalised_url.SetValue( normalised )
if url_class.UsesAPIURL():
self._send_referral_url.Disable()
self._referral_url_converter.Disable()
self._referral_url.SetValue( 'Not used, as API converter will redirect.' )
else:
self._send_referral_url.Enable()
self._referral_url_converter.Enable()
send_referral_url = self._send_referral_url.GetValue()
if send_referral_url in ( ClientNetworkingDomain.SEND_REFERRAL_URL_ONLY_IF_PROVIDED, ClientNetworkingDomain.SEND_REFERRAL_URL_NEVER ):
self._referral_url_converter.Disable()
else:
self._referral_url_converter.Enable()
if send_referral_url == ClientNetworkingDomain.SEND_REFERRAL_URL_CONVERTER_IF_NONE_PROVIDED:
referral_url = url_class.GetReferralURL( normalised, None )
referral_url = 'normal referral url -or- {}'.format( referral_url )
else:
referral_url = url_class.GetReferralURL( normalised, 'normal referral url' )
if referral_url is None:
self._referral_url.SetValue( 'None' )
else:
self._referral_url.SetValue( referral_url )
try:
if url_class.UsesAPIURL():

4
include/ClientGUIScrolledPanelsManagement.py
View File

@ -3436,7 +3436,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._save_page_sort_on_change = wx.CheckBox( self )
self._default_media_collect = ClientGUICommon.CheckboxCollect( self )
self._default_media_collect = ClientGUICommon.CheckboxCollect( self, silent = True )
self._sort_by = wx.ListBox( self )
self._sort_by.Bind( wx.EVT_LEFT_DCLICK, self.EventRemoveSortBy )
@ -3932,7 +3932,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
rows.append( ( 'Default tag service in manage tag dialogs: ', self._default_tag_repository ) )
rows.append( ( 'Default tag service in search pages: ', self._default_tag_service_search_page ) )
rows.append( ( 'Default tag sort: ', self._default_tag_sort ) )
rows.append( ( 'By default, search non-local tags in write-autocomplete: ', self._show_all_tags_in_autocomplete ) )
rows.append( ( 'By default, search \'all known files\' in \'write\' tag autocomplete inputs: ', self._show_all_tags_in_autocomplete ) )
rows.append( ( 'By default, select the first tag result with actual count in write-autocomplete: ', self._ac_select_first_with_count ) )
rows.append( ( 'Suggest all parents for all services: ', self._apply_all_parents_to_all_services ) )
rows.append( ( 'Apply all siblings to all services (local siblings have precedence): ', self._apply_all_siblings_to_all_services ) )

183
include/ClientGUITags.py
View File

@ -9,6 +9,7 @@ from . import ClientGUIDialogsQuick
from . import ClientGUIFunctions
from . import ClientGUIListBoxes
from . import ClientGUIListCtrl
from . import ClientGUIMenus
from . import ClientGUITopLevelWindows
from . import ClientGUIScrolledPanels
from . import ClientGUIScrolledPanelsEdit
@ -39,6 +40,7 @@ class EditTagFilterPanel( ClientGUIScrolledPanels.EditPanel ):
ClientGUIScrolledPanels.EditPanel.__init__( self, parent )
self._prefer_blacklist = prefer_blacklist
self._namespaces = namespaces
#
@ -49,6 +51,12 @@ class EditTagFilterPanel( ClientGUIScrolledPanels.EditPanel ):
#
self._load_favourite = ClientGUICommon.BetterButton( self, 'load', self._LoadFavourite )
self._save_favourite = ClientGUICommon.BetterButton( self, 'save', self._SaveFavourite )
self._delete_favourite = ClientGUICommon.BetterButton( self, 'delete', self._DeleteFavourite )
#
self._notebook = ClientGUICommon.BetterNotebook( self )
#
@ -60,7 +68,7 @@ class EditTagFilterPanel( ClientGUIScrolledPanels.EditPanel ):
#
if prefer_blacklist:
if self._prefer_blacklist:
self._notebook.AddPage( self._blacklist_panel, 'blacklist' )
self._notebook.AddPage( self._whitelist_panel, 'whitelist' )
@ -73,39 +81,6 @@ class EditTagFilterPanel( ClientGUIScrolledPanels.EditPanel ):
self._notebook.AddPage( self._advanced_panel, 'advanced' )
blacklist_tag_slices = [ tag_slice for ( tag_slice, rule ) in list(tag_filter.GetTagSlicesToRules().items()) if rule == CC.FILTER_BLACKLIST ]
whitelist_tag_slices = [ tag_slice for ( tag_slice, rule ) in list(tag_filter.GetTagSlicesToRules().items()) if rule == CC.FILTER_WHITELIST ]
self._advanced_blacklist.AddTags( blacklist_tag_slices )
self._advanced_whitelist.AddTags( whitelist_tag_slices )
( whitelist_possible, blacklist_possible ) = self._GetWhiteBlacklistsPossible()
selection_tests = []
if prefer_blacklist:
selection_tests.append( ( blacklist_possible, self._blacklist_panel ) )
selection_tests.append( ( whitelist_possible, self._whitelist_panel ) )
selection_tests.append( ( True, self._advanced_panel ) )
else:
selection_tests.append( ( whitelist_possible, self._whitelist_panel ) )
selection_tests.append( ( blacklist_possible, self._blacklist_panel ) )
selection_tests.append( ( True, self._advanced_panel ) )
for ( test, page ) in selection_tests:
if test:
self._notebook.SelectPage( page )
break
#
self._redundant_st = ClientGUICommon.BetterStaticText( self, '', style = wx.ST_ELLIPSIZE_END )
@ -127,6 +102,13 @@ class EditTagFilterPanel( ClientGUIScrolledPanels.EditPanel ):
vbox.Add( ClientGUICommon.BetterStaticText( self, message ), CC.FLAGS_EXPAND_PERPENDICULAR )
hbox = wx.BoxSizer( wx.HORIZONTAL )
hbox.Add( self._load_favourite, CC.FLAGS_SMALL_INDENT )
hbox.Add( self._save_favourite, CC.FLAGS_SMALL_INDENT )
hbox.Add( self._delete_favourite, CC.FLAGS_SMALL_INDENT )
vbox.Add( hbox, CC.FLAGS_BUTTON_SIZER )
vbox.Add( self._notebook, CC.FLAGS_EXPAND_BOTH_WAYS )
vbox.Add( self._redundant_st, CC.FLAGS_EXPAND_PERPENDICULAR )
vbox.Add( self._current_filter_st, CC.FLAGS_EXPAND_PERPENDICULAR )
@ -150,7 +132,7 @@ class EditTagFilterPanel( ClientGUIScrolledPanels.EditPanel ):
self._test_input.Bind( wx.EVT_TEXT, self.EventTestText )
self._UpdateStatus()
self.SetValue( tag_filter )
def _AdvancedAddBlacklist( self, tag_slice ):
@ -302,6 +284,48 @@ class EditTagFilterPanel( ClientGUIScrolledPanels.EditPanel ):
return not blacklist.isdisjoint( test_slices )
def _DeleteFavourite( self ):
def do_it( name ):
names_to_tag_filters = HG.client_controller.new_options.GetFavouriteTagFilters()
if name in names_to_tag_filters:
message = 'Delete "{}"?'.format( name )
result = ClientGUIDialogsQuick.GetYesNo( self, message )
if result != wx.ID_YES:
return
del names_to_tag_filters[ name ]
HG.client_controller.new_options.SetFavouriteTagFilters( names_to_tag_filters )
names_to_tag_filters = HG.client_controller.new_options.GetFavouriteTagFilters()
menu = wx.Menu()
if len( names_to_tag_filters ) == 0:
ClientGUIMenus.AppendMenuLabel( menu, 'no favourites set!' )
else:
for ( name, tag_filter ) in names_to_tag_filters.items():
ClientGUIMenus.AppendMenuItem( self, menu, name, 'delete {}'.format( name ), do_it, name )
HG.client_controller.PopupMenu( self, menu )
def _GetWhiteBlacklistsPossible( self ):
blacklist_tag_slices = self._advanced_blacklist.GetClientData()
@ -493,6 +517,57 @@ class EditTagFilterPanel( ClientGUIScrolledPanels.EditPanel ):
return whitelist_panel
def _LoadFavourite( self ):
names_to_tag_filters = HG.client_controller.new_options.GetFavouriteTagFilters()
menu = wx.Menu()
if len( names_to_tag_filters ) == 0:
ClientGUIMenus.AppendMenuLabel( menu, 'no favourites set!' )
else:
for ( name, tag_filter ) in names_to_tag_filters.items():
ClientGUIMenus.AppendMenuItem( self, menu, name, 'load {}'.format( name ), self.SetValue, tag_filter )
HG.client_controller.PopupMenu( self, menu )
def _SaveFavourite( self ):
with ClientGUIDialogs.DialogTextEntry( self, 'Enter a name for the favourite.' ) as dlg:
if dlg.ShowModal() == wx.ID_OK:
names_to_tag_filters = HG.client_controller.new_options.GetFavouriteTagFilters()
name = dlg.GetValue()
tag_filter = self.GetValue()
if name in names_to_tag_filters:
message = '"{}" already exists! Overwrite?'.format( name )
result = ClientGUIDialogsQuick.GetYesNo( self, message )
if result != wx.ID_YES:
return
names_to_tag_filters[ name ] = tag_filter
HG.client_controller.new_options.SetFavouriteTagFilters( names_to_tag_filters )
def _ShowHelp( self ):
help = 'Here you can set rules to filter tags for one purpose or another. The default is typically to permit all tags. Check the current filter summary text at the bottom-left of the panel to ensure you have your logic correct.'
@ -852,6 +927,44 @@ class EditTagFilterPanel( ClientGUIScrolledPanels.EditPanel ):
return tag_filter
def SetValue( self, tag_filter ):
blacklist_tag_slices = [ tag_slice for ( tag_slice, rule ) in tag_filter.GetTagSlicesToRules().items() if rule == CC.FILTER_BLACKLIST ]
whitelist_tag_slices = [ tag_slice for ( tag_slice, rule ) in tag_filter.GetTagSlicesToRules().items() if rule == CC.FILTER_WHITELIST ]
self._advanced_blacklist.SetTags( blacklist_tag_slices )
self._advanced_whitelist.SetTags( whitelist_tag_slices )
( whitelist_possible, blacklist_possible ) = self._GetWhiteBlacklistsPossible()
selection_tests = []
if self._prefer_blacklist:
selection_tests.append( ( blacklist_possible, self._blacklist_panel ) )
selection_tests.append( ( whitelist_possible, self._whitelist_panel ) )
selection_tests.append( ( True, self._advanced_panel ) )
else:
selection_tests.append( ( whitelist_possible, self._whitelist_panel ) )
selection_tests.append( ( blacklist_possible, self._blacklist_panel ) )
selection_tests.append( ( True, self._advanced_panel ) )
for ( test, page ) in selection_tests:
if test:
self._notebook.SelectPage( page )
break
self._UpdateStatus()
class ManageTagsPanel( ClientGUIScrolledPanels.ManagePanel ):
def __init__( self, parent, file_service_key, media, immediate_commit = False, canvas_key = None ):

18
include/ClientImportFileSeeds.py
View File

@ -1826,7 +1826,16 @@ class FileSeedCache( HydrusSerialisable.SerialisableBase ):
continue
file_seed.Normalise()
try:
file_seed.Normalise()
except HydrusExceptions.URLClassException:
# this is some borked 'https://' url that makes no sense
continue
new_file_seeds.append( file_seed )
@ -2119,16 +2128,23 @@ class FileSeedCache( HydrusSerialisable.SerialisableBase ):
inbox_hashes = HG.client_controller.Read( 'in_inbox', file_seed_hashes )
hashes = []
hashes_seen = set()
for file_seed in eligible_file_seeds:
hash = file_seed.GetHash()
if hash in hashes_seen:
continue
in_inbox = hash in inbox_hashes
if file_seed.ShouldPresent( file_import_options, in_inbox = in_inbox ):
hashes.append( hash )
hashes_seen.add( hash )

31
include/ClientMedia.py
View File

@ -934,6 +934,25 @@ class MediaList( object ):
def __init__( self, file_service_key, media_results ):
hashes_seen = set()
media_results_dedupe = []
for media_result in media_results:
hash = media_result.GetHash()
if hash in hashes_seen:
continue
media_results_dedupe.append( media_result )
hashes_seen.add( hash )
media_results = media_results_dedupe
self._file_service_key = file_service_key
self._hashes = set()
@ -2638,7 +2657,7 @@ class MediaSort( HydrusSerialisable.SerialisableBase ):
tags_manager = x.GetTagsManager()
return len( tags_manager.GetCurrentAndPending( CC.COMBINED_TAG_SERVICE_KEY, ClientTags.TAG_DISPLAY_SIBLINGS_AND_PARENTS ) )
return len( tags_manager.GetCurrentAndPending( CC.COMBINED_TAG_SERVICE_KEY, ClientTags.TAG_DISPLAY_SINGLE_MEDIA ) )
elif sort_data == CC.SORT_FILES_BY_MIME:
@ -2654,7 +2673,9 @@ class MediaSort( HydrusSerialisable.SerialisableBase ):
fvsm = x.GetFileViewingStatsManager()
return ( fvsm.media_views, fvsm.media_viewtime )
# do not do viewtime as a secondary sort here, to allow for user secondary sort to help out
return fvsm.media_views
elif sort_data == CC.SORT_FILES_BY_MEDIA_VIEWTIME:
@ -2663,7 +2684,9 @@ class MediaSort( HydrusSerialisable.SerialisableBase ):
fvsm = x.GetFileViewingStatsManager()
return ( fvsm.media_viewtime, fvsm.media_views )
# do not do views as a secondary sort here, to allow for user secondary sort to help out
return fvsm.media_viewtime
@ -2675,7 +2698,7 @@ class MediaSort( HydrusSerialisable.SerialisableBase ):
x_tags_manager = x.GetTagsManager()
return [ x_tags_manager.GetComparableNamespaceSlice( ( namespace, ), ClientTags.TAG_DISPLAY_SIBLINGS_AND_PARENTS ) for namespace in namespaces ]
return [ x_tags_manager.GetComparableNamespaceSlice( ( namespace, ), ClientTags.TAG_DISPLAY_SINGLE_MEDIA ) for namespace in namespaces ]
elif sort_metadata == 'rating':

103
include/ClientNetworkingDomain.py
View File

@ -1320,6 +1320,23 @@ class NetworkDomainManager( HydrusSerialisable.SerialisableBase ):
def GetReferralURL( self, url, referral_url ):
with self._lock:
url_class = self._GetURLClass( url )
if url_class is None:
return referral_url
else:
return url_class.GetReferralURL( url, referral_url )
def GetShareableCustomHeaders( self, network_context ):
with self._lock:
@ -2541,13 +2558,27 @@ class NestedGalleryURLGenerator( HydrusSerialisable.SerialisableBaseNamed ):
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_NESTED_GALLERY_URL_GENERATOR ] = NestedGalleryURLGenerator
SEND_REFERRAL_URL_ONLY_IF_PROVIDED = 0
SEND_REFERRAL_URL_NEVER = 1
SEND_REFERRAL_URL_CONVERTER_IF_NONE_PROVIDED = 2
SEND_REFERRAL_URL_ONLY_CONVERTER = 3
SEND_REFERRAL_URL_TYPES = [ SEND_REFERRAL_URL_ONLY_IF_PROVIDED, SEND_REFERRAL_URL_NEVER, SEND_REFERRAL_URL_CONVERTER_IF_NONE_PROVIDED, SEND_REFERRAL_URL_ONLY_CONVERTER ]
send_referral_url_string_lookup = {}
send_referral_url_string_lookup[ SEND_REFERRAL_URL_ONLY_IF_PROVIDED ] = 'send a referral url if available'
send_referral_url_string_lookup[ SEND_REFERRAL_URL_NEVER ] = 'never send a referral url'
send_referral_url_string_lookup[ SEND_REFERRAL_URL_CONVERTER_IF_NONE_PROVIDED ] = 'use the converter if no referral is available'
send_referral_url_string_lookup[ SEND_REFERRAL_URL_ONLY_CONVERTER ] = 'always use the converter referral url'
class URLClass( HydrusSerialisable.SerialisableBaseNamed ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_URL_CLASS
SERIALISABLE_NAME = 'URL Class'
SERIALISABLE_VERSION = 6
SERIALISABLE_VERSION = 7
def __init__( self, name, url_class_key = None, url_type = None, preferred_scheme = 'https', netloc = 'hostname.com', match_subdomains = False, keep_matched_subdomains = False, path_components = None, parameters = None, api_lookup_converter = None, can_produce_multiple_files = False, should_be_associated_with_files = True, gallery_index_type = None, gallery_index_identifier = None, gallery_index_delta = 1, example_url = 'https://hostname.com/post/page.php?id=123456&s=view' ):
def __init__( self, name, url_class_key = None, url_type = None, preferred_scheme = 'https', netloc = 'hostname.com', match_subdomains = False, keep_matched_subdomains = False, path_components = None, parameters = None, api_lookup_converter = None, send_referral_url = SEND_REFERRAL_URL_ONLY_IF_PROVIDED, referral_url_converter = None, can_produce_multiple_files = False, should_be_associated_with_files = True, gallery_index_type = None, gallery_index_identifier = None, gallery_index_delta = 1, example_url = 'https://hostname.com/post/page.php?id=123456&s=view' ):
if url_class_key is None:
@ -2573,7 +2604,6 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ):
parameters[ 's' ] = ( ClientParsing.StringMatch( match_type = ClientParsing.STRING_MATCH_FIXED, match_value = 'view', example_string = 'view' ), None )
parameters[ 'id' ] = ( ClientParsing.StringMatch( match_type = ClientParsing.STRING_MATCH_FLEXIBLE, match_value = ClientParsing.NUMERIC, example_string = '123456' ), None )
parameters[ 'page' ] = ( ClientParsing.StringMatch( match_type = ClientParsing.STRING_MATCH_FLEXIBLE, match_value = ClientParsing.NUMERIC, example_string = '1' ), '1' )
if api_lookup_converter is None:
@ -2581,6 +2611,11 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ):
api_lookup_converter = ClientParsing.StringConverter( example_string = 'https://hostname.com/post/page.php?id=123456&s=view' )
if referral_url_converter is None:
referral_url_converter = ClientParsing.StringConverter( example_string = 'https://hostname.com/post/page.php?id=123456&s=view' )
# if the args are not serialisable stuff, lets overwrite here
path_components = HydrusSerialisable.SerialisableList( path_components )
@ -2602,6 +2637,9 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ):
self._parameters = parameters
self._api_lookup_converter = api_lookup_converter
self._send_referral_url = send_referral_url
self._referral_url_converter = referral_url_converter
self._gallery_index_type = gallery_index_type
self._gallery_index_identifier = gallery_index_identifier
self._gallery_index_delta = gallery_index_delta
@ -2712,18 +2750,20 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ):
serialisable_path_components = [ ( string_match.GetSerialisableTuple(), default ) for ( string_match, default ) in self._path_components ]
serialisable_parameters = [ ( key, ( string_match.GetSerialisableTuple(), default ) ) for ( key, ( string_match, default ) ) in list(self._parameters.items()) ]
serialisable_api_lookup_converter = self._api_lookup_converter.GetSerialisableTuple()
serialisable_referral_url_converter = self._referral_url_converter.GetSerialisableTuple()
return ( serialisable_url_class_key, self._url_type, self._preferred_scheme, self._netloc, self._match_subdomains, self._keep_matched_subdomains, serialisable_path_components, serialisable_parameters, serialisable_api_lookup_converter, self._can_produce_multiple_files, self._should_be_associated_with_files, self._gallery_index_type, self._gallery_index_identifier, self._gallery_index_delta, self._example_url )
return ( serialisable_url_class_key, self._url_type, self._preferred_scheme, self._netloc, self._match_subdomains, self._keep_matched_subdomains, serialisable_path_components, serialisable_parameters, serialisable_api_lookup_converter, self._send_referral_url, serialisable_referral_url_converter, self._can_produce_multiple_files, self._should_be_associated_with_files, self._gallery_index_type, self._gallery_index_identifier, self._gallery_index_delta, self._example_url )
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_url_class_key, self._url_type, self._preferred_scheme, self._netloc, self._match_subdomains, self._keep_matched_subdomains, serialisable_path_components, serialisable_parameters, serialisable_api_lookup_converter, self._can_produce_multiple_files, self._should_be_associated_with_files, self._gallery_index_type, self._gallery_index_identifier, self._gallery_index_delta, self._example_url ) = serialisable_info
( serialisable_url_class_key, self._url_type, self._preferred_scheme, self._netloc, self._match_subdomains, self._keep_matched_subdomains, serialisable_path_components, serialisable_parameters, serialisable_api_lookup_converter, self._send_referral_url, serialisable_referral_url_converter, self._can_produce_multiple_files, self._should_be_associated_with_files, self._gallery_index_type, self._gallery_index_identifier, self._gallery_index_delta, self._example_url ) = serialisable_info
self._url_class_key = bytes.fromhex( serialisable_url_class_key )
self._path_components = [ ( HydrusSerialisable.CreateFromSerialisableTuple( serialisable_string_match ), default ) for ( serialisable_string_match, default ) in serialisable_path_components ]
self._parameters = { key : ( HydrusSerialisable.CreateFromSerialisableTuple( serialisable_string_match ), default ) for ( key, ( serialisable_string_match, default ) ) in serialisable_parameters }
self._api_lookup_converter = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_api_lookup_converter )
self._referral_url_converter = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_referral_url_converter )
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
@ -2805,6 +2845,20 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ):
return ( 6, new_serialisable_info )
if version == 6:
( serialisable_url_class_key, url_type, preferred_scheme, netloc, match_subdomains, keep_matched_subdomains, serialisable_path_components, serialisable_parameters, serialisable_api_lookup_converter, can_produce_multiple_files, should_be_associated_with_files, gallery_index_type, gallery_index_identifier, gallery_index_delta, example_url ) = old_serialisable_info
send_referral_url = SEND_REFERRAL_URL_ONLY_IF_PROVIDED
referral_url_converter = ClientParsing.StringConverter( example_string = 'https://hostname.com/post/page.php?id=123456&s=view' )
serialisable_referrel_url_converter = referral_url_converter.GetSerialisableTuple()
new_serialisable_info = ( serialisable_url_class_key, url_type, preferred_scheme, netloc, match_subdomains, keep_matched_subdomains, serialisable_path_components, serialisable_parameters, serialisable_api_lookup_converter, send_referral_url, serialisable_referrel_url_converter, can_produce_multiple_files, should_be_associated_with_files, gallery_index_type, gallery_index_identifier, gallery_index_delta, example_url )
return ( 7, new_serialisable_info )
def CanGenerateNextGalleryPage( self ):
@ -2947,6 +3001,43 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ):
return r.geturl()
def GetReferralURL( self, url, referral_url ):
if self._send_referral_url == SEND_REFERRAL_URL_ONLY_IF_PROVIDED:
return referral_url
elif self._send_referral_url == SEND_REFERRAL_URL_NEVER:
return None
elif self._send_referral_url in ( SEND_REFERRAL_URL_CONVERTER_IF_NONE_PROVIDED, SEND_REFERRAL_URL_ONLY_CONVERTER ):
try:
converted_referral_url = self._referral_url_converter.Convert( url )
except HydrusExceptions.StringConvertException:
return referral_url
p1 = self._send_referral_url == SEND_REFERRAL_URL_ONLY_CONVERTER
p2 = self._send_referral_url == SEND_REFERRAL_URL_CONVERTER_IF_NONE_PROVIDED and referral_url is None
if p1 or p2:
return converted_referral_url
else:
return referral_url
return referral_url
def GetSafeSummary( self ):
return 'URL Class "' + self._name + '" - ' + ConvertURLIntoDomain( self.GetExampleURL() )
@ -3126,7 +3217,7 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ):
def ToTuple( self ):
return ( self._url_type, self._preferred_scheme, self._netloc, self._match_subdomains, self._keep_matched_subdomains, self._path_components, self._parameters, self._api_lookup_converter, self._can_produce_multiple_files, self._should_be_associated_with_files, self._example_url )
return ( self._url_type, self._preferred_scheme, self._netloc, self._match_subdomains, self._keep_matched_subdomains, self._path_components, self._parameters, self._api_lookup_converter, self._send_referral_url, self._referral_url_converter, self._can_produce_multiple_files, self._should_be_associated_with_files, self._example_url )
def UsesAPIURL( self ):

28
include/ClientNetworkingJobs.py
View File

@ -238,12 +238,34 @@ class NetworkJob( object ):
headers[ 'User-Agent' ] = 'hydrus client/' + str( HC.NETWORK_VERSION )
if self._referral_url is not None:
referral_url = self.engine.domain_manager.GetReferralURL( self._url, self._referral_url )
if HG.network_report_mode:
headers[ 'referer' ] = urllib.parse.quote( self._referral_url, "!#$%&'()*+,/:;=?@[]~" ) # quick and dirty way to quote this url when it comes here with full unicode chars. not perfect, but does the job
HydrusData.ShowText( 'Network Jobs Referral URLs for {}:{}Given: {}{}Used: {}'.format( self._url, os.linesep, self._referral_url, os.linesep, referral_url ) )
for ( key, value ) in list(self._additional_headers.items()):
if referral_url is not None:
try:
referral_url.encode( 'latin-1' )
except UnicodeEncodeError:
# quick and dirty way to quote this url when it comes here with full unicode chars. not perfect, but does the job
referral_url = urllib.parse.quote( referral_url, "!#$%&'()*+,/:;=?@[]~" )
if HG.network_report_mode:
HydrusData.ShowText( 'Network Jobs Quoted Referral URL for {}:{}{}'.format( self._url, os.linesep, referral_url ) )
headers[ 'referer' ] = referral_url
for ( key, value ) in self._additional_headers.items():
headers[ key ] = value

24
include/ClientOptions.py
View File

@ -326,6 +326,10 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
#
self._dictionary[ 'favourite_tag_filters' ] = HydrusSerialisable.SerialisableDictionary()
#
self._dictionary[ 'tag_summary_generators' ] = HydrusSerialisable.SerialisableDictionary()
namespace_info = []
@ -752,6 +756,14 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
def GetFavouriteTagFilters( self ):
with self._lock:
return dict( self._dictionary[ 'favourite_tag_filters' ] )
def GetFrameLocation( self, frame_key ):
with self._lock:
@ -979,6 +991,14 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
def SetDefaultFileImportOptions( self, options_type, file_import_options ):
with self._lock:
self._dictionary[ 'default_file_import_options' ][ options_type ] = file_import_options
def SetDefaultSort( self, media_sort ):
with self._lock:
@ -1019,11 +1039,11 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
def SetDefaultFileImportOptions( self, options_type, file_import_options ):
def SetFavouriteTagFilters( self, names_to_tag_filters ):
with self._lock:
self._dictionary[ 'default_file_import_options' ][ options_type ] = file_import_options
self._dictionary[ 'favourite_tag_filters' ] = HydrusSerialisable.SerialisableDictionary( names_to_tag_filters )

2
include/HydrusConstants.py
View File

@ -67,7 +67,7 @@ options = {}
# Misc
NETWORK_VERSION = 18
SOFTWARE_VERSION = 370
SOFTWARE_VERSION = 371
CLIENT_API_VERSION = 11
SERVER_THUMBNAIL_DIMENSIONS = ( 200, 200 )

23
include/HydrusDB.py
View File

@ -176,6 +176,7 @@ class HydrusDB( object ):
self._is_first_start = False
self._is_db_updated = False
self._local_shutdown = False
self._pause_and_disconnect = False
self._loop_finished = False
self._ready_to_serve_requests = False
self._could_not_initialise = False
@ -942,6 +943,23 @@ class HydrusDB( object ):
self._InitDBCursor()
if self._pause_and_disconnect:
self._CloseDBCursor()
while self._pause_and_disconnect:
if self._local_shutdown or HG.model_shutdown:
break
time.sleep( 1 )
self._InitDBCursor()
self._CleanUpCaches()
@ -954,6 +972,11 @@ class HydrusDB( object ):
self._loop_finished = True
def PauseAndDisconnect( self, pause_and_disconnect ):
self._pause_and_disconnect = pause_and_disconnect
def Read( self, action, *args, **kwargs ):
if action in self.READ_WRITE_ACTIONS:

2
include/HydrusGlobals.py
View File

@ -38,7 +38,6 @@ daemon_report_mode = False
force_idle_mode = False
no_page_limit_mode = False
thumbnail_debug_mode = False
server_busy = False
currently_uploading_pending = False
shutting_down_due_to_already_running = False
@ -54,3 +53,4 @@ twisted_is_broke = False
do_not_catch_char_hook = False
dirty_object_lock = threading.Lock()
server_busy = threading.Lock()

54
include/HydrusNetwork.py
View File

@ -2324,32 +2324,42 @@ class ServerServiceRepository( ServerServiceRestricted ):
if update_due:
HG.server_busy = True
locked = HG.server_busy.acquire( False )
while update_due:
if not locked:
with self._lock:
service_key = self._service_key
begin = self._metadata.GetNextUpdateBegin()
end = begin + HC.UPDATE_DURATION
update_hashes = HG.server_controller.WriteSynchronous( 'create_update', service_key, begin, end )
next_update_due = end + HC.UPDATE_DURATION + 1
with self._lock:
self._metadata.AppendUpdate( update_hashes, begin, end, next_update_due )
update_due = self._metadata.UpdateDue()
return
HG.server_busy = False
try:
while update_due:
with self._lock:
service_key = self._service_key
begin = self._metadata.GetNextUpdateBegin()
end = begin + HC.UPDATE_DURATION
update_hashes = HG.server_controller.WriteSynchronous( 'create_update', service_key, begin, end )
next_update_due = end + HC.UPDATE_DURATION + 1
with self._lock:
self._metadata.AppendUpdate( update_hashes, begin, end, next_update_due )
update_due = self._metadata.UpdateDue()
finally:
HG.server_busy.release()
with self._lock:

5
include/HydrusServerResources.py
View File

@ -344,11 +344,6 @@ class HydrusResource( Resource ):
def _checkService( self, request ):
if HG.server_busy:
raise HydrusExceptions.ServerBusyException( 'This server is busy, please try again later.' )
return request

7
include/HydrusVideoHandling.py
View File

@ -59,13 +59,13 @@ def GetFFMPEGVersion():
except FileNotFoundError:
return 'no ffmpeg found'
return 'no ffmpeg found at path "{}"'.format( FFMPEG_PATH )
except Exception as e:
HydrusData.ShowException( e )
return 'unable to execute ffmpeg'
return 'unable to execute ffmpeg at path "{}"'.format( FFMPEG_PATH )
( stdout, stderr ) = process.communicate()
@ -388,7 +388,8 @@ def ParseFFMPEGDuration( lines ):
# Duration: 00:00:02.46, start: 0.033000, bitrate: 1069 kb/s
try:
line = [ l for l in lines if 'Duration:' in l ][0]
# had a vid with 'Duration:' in title, ha ha, so now a regex
line = [ l for l in lines if re.search( r'^\s*Duration:', l ) is not None ][0]
if 'Duration: N/A' in line:

5
include/ServerController.py
View File

@ -465,11 +465,6 @@ class Controller( HydrusController.HydrusController ):
def SyncRepositories( self ):
if HG.server_busy:
return
repositories = [ service for service in self._services if service.GetServiceType() in HC.REPOSITORIES ]
for service in repositories:

165
include/ServerDB.py
View File

@ -244,61 +244,61 @@ class DB( HydrusDB.HydrusDB ):
if len( names_to_analyze ) > 0:
HG.server_busy = True
locked = HG.server_busy.acquire( False )
for name in names_to_analyze:
started = HydrusData.GetNowPrecise()
self._c.execute( 'ANALYZE ' + name + ';' )
self._c.execute( 'DELETE FROM analyze_timestamps WHERE name = ?;', ( name, ) )
self._c.execute( 'INSERT OR IGNORE INTO analyze_timestamps ( name, timestamp ) VALUES ( ?, ? );', ( name, HydrusData.GetNow() ) )
time_took = HydrusData.GetNowPrecise() - started
if time_took > 1:
if not locked:
HydrusData.Print( 'Analyzed ' + name + ' in ' + HydrusData.TimeDeltaToPrettyTimeDelta( time_took ) )
return
if HG.server_controller.ShouldStopThisWork( maintenance_mode, stop_time = stop_time ):
try:
break
for name in names_to_analyze:
started = HydrusData.GetNowPrecise()
self._c.execute( 'ANALYZE ' + name + ';' )
self._c.execute( 'DELETE FROM analyze_timestamps WHERE name = ?;', ( name, ) )
self._c.execute( 'INSERT OR IGNORE INTO analyze_timestamps ( name, timestamp ) VALUES ( ?, ? );', ( name, HydrusData.GetNow() ) )
time_took = HydrusData.GetNowPrecise() - started
if time_took > 1:
HydrusData.Print( 'Analyzed ' + name + ' in ' + HydrusData.TimeDeltaToPrettyTimeDelta( time_took ) )
if HG.server_controller.ShouldStopThisWork( maintenance_mode, stop_time = stop_time ):
break
self._c.execute( 'ANALYZE sqlite_master;' ) # this reloads the current stats into the query planner
finally:
HG.server_busy.release()
self._c.execute( 'ANALYZE sqlite_master;' ) # this reloads the current stats into the query planner
HG.server_busy = False
def _Backup( self, skip_vacuum = False ):
def _Backup( self ):
self._CloseDBCursor()
locked = HG.server_busy.acquire( False )
HG.server_busy = True
if not locked:
HydrusData.Print( 'Could not backup because the server was locked.' )
return
try:
stop_time = HydrusData.GetNow() + 300
if not skip_vacuum:
for filename in self._db_filenames.values():
db_path = os.path.join( self._db_dir, filename )
if HydrusDB.CanVacuum( db_path, stop_time ):
HydrusData.Print( 'backing up: vacuuming ' + filename )
HydrusDB.VacuumDB( db_path )
self._CloseDBCursor()
backup_path = os.path.join( self._db_dir, 'server_backup' )
@ -327,14 +327,14 @@ class DB( HydrusDB.HydrusDB ):
HydrusData.Print( 'backing up: copying files' )
HydrusPaths.MirrorTree( self._files_dir, os.path.join( backup_path, 'server_files' ) )
finally:
HG.server_busy = False
self._InitDBCursor()
HydrusData.Print( 'backing up: done!' )
HydrusData.Print( 'backing up: done!' )
finally:
HG.server_busy.release()
def _CreateDB( self ):
@ -3277,6 +3277,76 @@ class DB( HydrusDB.HydrusDB ):
self._c.execute( 'UPDATE version SET version = ?;', ( version + 1, ) )
def _Vacuum( self ):
locked = HG.server_busy.acquire( False )
if not locked:
HydrusData.Print( 'Could not vacuum because the server was locked!' )
return
try:
db_names = [ name for ( index, name, path ) in self._c.execute( 'PRAGMA database_list;' ) if name not in ( 'mem', 'temp', 'durable_temp' ) ]
self._CloseDBCursor()
try:
names_done = []
for name in db_names:
if name not in self._db_filenames:
continue
try:
db_path = os.path.join( self._db_dir, self._db_filenames[ name ] )
if HydrusDB.CanVacuum( db_path ):
started = HydrusData.GetNowPrecise()
HydrusDB.VacuumDB( db_path )
time_took = HydrusData.GetNowPrecise() - started
HydrusData.Print( 'Vacuumed ' + db_path + ' in ' + HydrusData.TimeDeltaToPrettyTimeDelta( time_took ) )
else:
HydrusData.Print( 'Could not vacuum ' + db_path + ' (probably due to limited disk space on db or system drive).' )
names_done.append( name )
except Exception as e:
HydrusData.Print( 'vacuum failed:' )
HydrusData.ShowException( e )
return
finally:
self._InitDBCursor()
finally:
HG.server_busy.release()
def _VerifyAccessKey( self, service_key, access_key ):
service_id = self._GetServiceId( service_key )
@ -3312,6 +3382,7 @@ class DB( HydrusDB.HydrusDB ):
elif action == 'services': result = self._ModifyServices( *args, **kwargs )
elif action == 'session': self._AddSession( *args, **kwargs )
elif action == 'update': self._RepositoryProcessClientToServerUpdate( *args, **kwargs )
elif action == 'vacuum': self._Vacuum( *args, **kwargs )
else: raise Exception( 'db received an unknown write command: ' + action )
return result

3
include/ServerServer.py
View File

@ -28,8 +28,11 @@ class HydrusServiceAdmin( HydrusServiceRestricted ):
root.putChild( b'busy', ServerServerResources.HydrusResourceBusyCheck() )
root.putChild( b'backup', ServerServerResources.HydrusResourceRestrictedBackup( self._service, HydrusServer.REMOTE_DOMAIN ) )
root.putChild( b'lock_on', ServerServerResources.HydrusResourceRestrictedLockOn( self._service, HydrusServer.REMOTE_DOMAIN ) )
root.putChild( b'lock_off', ServerServerResources.HydrusResourceRestrictedLockOff( self._service, HydrusServer.REMOTE_DOMAIN ) )
root.putChild( b'services', ServerServerResources.HydrusResourceRestrictedServices( self._service, HydrusServer.REMOTE_DOMAIN ) )
root.putChild( b'shutdown', ServerServerResources.HydrusResourceShutdown( self._service, HydrusServer.LOCAL_DOMAIN ) )
root.putChild( b'vacuum', ServerServerResources.HydrusResourceRestrictedVacuum( self._service, HydrusServer.REMOTE_DOMAIN ) )
return root

81
include/ServerServerResources.py
View File

@ -9,6 +9,7 @@ from . import HydrusPaths
from . import HydrusSerialisable
from . import HydrusServerResources
from . import ServerFiles
import threading
class HydrusResourceBusyCheck( HydrusServerResources.Resource ):
@ -25,7 +26,7 @@ class HydrusResourceBusyCheck( HydrusServerResources.Resource ):
request.setHeader( 'Server', self._server_version_string )
if HG.server_busy:
if HG.server_busy.locked():
return b'1'
@ -37,6 +38,8 @@ class HydrusResourceBusyCheck( HydrusServerResources.Resource ):
class HydrusResourceHydrusNetwork( HydrusServerResources.HydrusResource ):
BLOCKED_WHEN_BUSY = True
def _callbackParseGETArgs( self, request ):
parsed_request_args = HydrusNetwork.ParseHydrusNetworkGETArgs( request.args )
@ -108,6 +111,16 @@ class HydrusResourceHydrusNetwork( HydrusServerResources.HydrusResource ):
return request
def _checkService( self, request ):
if self.BLOCKED_WHEN_BUSY and HG.server_busy.locked():
raise HydrusExceptions.ServerBusyException( 'This server is busy, please try again later.' )
return request
class HydrusResourceAccessKey( HydrusResourceHydrusNetwork ):
def _threadDoGETJob( self, request ):
@ -378,9 +391,7 @@ class HydrusResourceRestrictedBackup( HydrusResourceRestricted ):
# check permission here since this is an asynchronous job
request.hydrus_account.CheckPermission( HC.CONTENT_TYPE_SERVICES, HC.PERMISSION_ACTION_OVERRULE )
skip_vacuum = request.parsed_request_args.GetValue( 'skip_vacuum', bool, False )
HG.server_controller.Write( 'backup', skip_vacuum )
HG.server_controller.Write( 'backup' )
response_context = HydrusServerResources.ResponseContext( 200 )
@ -402,6 +413,54 @@ class HydrusResourceRestrictedIP( HydrusResourceRestricted ):
return response_context
class HydrusResourceRestrictedLockOn( HydrusResourceRestricted ):
def _threadDoPOSTJob( self, request ):
# check permission here since no db work
request.hydrus_account.CheckPermission( HC.CONTENT_TYPE_SERVICES, HC.PERMISSION_ACTION_OVERRULE )
locked = HG.server_busy.acquire( False )
if not locked:
raise HydrusExceptions.BadRequestException( 'The server was already locked!' )
HG.server_controller.db.PauseAndDisconnect( True )
# shut down db, wait until it is done?
response_context = HydrusServerResources.ResponseContext( 200 )
return response_context
class HydrusResourceRestrictedLockOff( HydrusResourceRestricted ):
BLOCKED_WHEN_BUSY = False
def _threadDoPOSTJob( self, request ):
# check permission here since no db work
request.hydrus_account.CheckPermission( HC.CONTENT_TYPE_SERVICES, HC.PERMISSION_ACTION_OVERRULE )
try:
HG.server_busy.release()
except threading.ThreadError:
raise HydrusExceptions.BadRequestException( 'The server is not busy!' )
HG.server_controller.db.PauseAndDisconnect( False )
response_context = HydrusServerResources.ResponseContext( 200 )
return response_context
class HydrusResourceRestrictedNumPetitions( HydrusResourceRestricted ):
def _threadDoGETJob( self, request ):
@ -644,3 +703,17 @@ class HydrusResourceRestrictedMetadataUpdate( HydrusResourceRestricted ):
return response_context
class HydrusResourceRestrictedVacuum( HydrusResourceRestricted ):
def _threadDoPOSTJob( self, request ):
# check permission here since this is an asynchronous job
request.hydrus_account.CheckPermission( HC.CONTENT_TYPE_SERVICES, HC.PERMISSION_ACTION_OVERRULE )
HG.server_controller.Write( 'vacuum' )
response_context = HydrusServerResources.ResponseContext( 200 )
return response_context

86
include/TestClientNetworking.py
View File

@ -6,6 +6,7 @@ from . import ClientNetworkingDomain
from . import ClientNetworkingJobs
from . import ClientNetworkingLogin
from . import ClientNetworkingSessions
from . import ClientParsing
from . import ClientServices
import collections
from . import HydrusConstants as HC
@ -220,6 +221,91 @@ class TestBandwidthManager( unittest.TestCase ):
pass
class TestNetworkingDomain( unittest.TestCase ):
def test_url_classes( self ):
name = 'test'
url_type = HC.URL_TYPE_POST
preferred_scheme = 'https'
netloc = 'testbooru.cx'
match_subdomains = False
keep_matched_subdomains = False
path_components = []
path_components.append( ( ClientParsing.StringMatch( match_type = ClientParsing.STRING_MATCH_FIXED, match_value = 'post', example_string = 'post' ), None ) )
path_components.append( ( ClientParsing.StringMatch( match_type = ClientParsing.STRING_MATCH_FIXED, match_value = 'page.php', example_string = 'page.php' ), None ) )
parameters = {}
parameters[ 's' ] = ( ClientParsing.StringMatch( match_type = ClientParsing.STRING_MATCH_FIXED, match_value = 'view', example_string = 'view' ), None )
parameters[ 'id' ] = ( ClientParsing.StringMatch( match_type = ClientParsing.STRING_MATCH_FLEXIBLE, match_value = ClientParsing.NUMERIC, example_string = '123456' ), None )
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_ONLY_IF_PROVIDED
referral_url_converter = None
can_produce_multiple_files = False
should_be_associated_with_files = True
gallery_index_type = None
gallery_index_identifier = None
gallery_index_delta = 1
example_url = 'https://testbooru.cx/post/page.php?id=123456&s=view'
#
referral_url = 'https://testbooru.cx/gallery/tags=samus_aran'
good_url = 'https://testbooru.cx/post/page.php?id=123456&s=view'
unnormalised_good_url_1 = 'https://testbooru.cx/post/page.php?id=123456&s=view&additional_gumpf=stuff'
unnormalised_good_url_2 = 'https://testbooru.cx/post/page.php?s=view&id=123456'
bad_url = 'https://wew.lad/123456'
url_class = ClientNetworkingDomain.URLClass( name, url_type = url_type, preferred_scheme = preferred_scheme, netloc = netloc, match_subdomains = match_subdomains, keep_matched_subdomains = keep_matched_subdomains, path_components = path_components, parameters = parameters, send_referral_url = send_referral_url, referral_url_converter = referral_url_converter, can_produce_multiple_files = can_produce_multiple_files, should_be_associated_with_files = should_be_associated_with_files, gallery_index_type = gallery_index_type, gallery_index_identifier = gallery_index_identifier, gallery_index_delta = gallery_index_delta, example_url = example_url )
self.assertEqual( url_class.Matches( example_url ), True )
self.assertEqual( url_class.Matches( bad_url ), False )
self.assertEqual( url_class.Normalise( unnormalised_good_url_1 ), good_url )
self.assertEqual( url_class.Normalise( unnormalised_good_url_2 ), good_url )
self.assertEqual( url_class.GetReferralURL( good_url, referral_url ), referral_url )
self.assertEqual( url_class.GetReferralURL( good_url, None ), None )
#
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_NEVER
url_class = ClientNetworkingDomain.URLClass( name, url_type = url_type, preferred_scheme = preferred_scheme, netloc = netloc, match_subdomains = match_subdomains, keep_matched_subdomains = keep_matched_subdomains, path_components = path_components, parameters = parameters, send_referral_url = send_referral_url, referral_url_converter = referral_url_converter, can_produce_multiple_files = can_produce_multiple_files, should_be_associated_with_files = should_be_associated_with_files, gallery_index_type = gallery_index_type, gallery_index_identifier = gallery_index_identifier, gallery_index_delta = gallery_index_delta, example_url = example_url )
self.assertEqual( url_class.GetReferralURL( good_url, referral_url ), None )
self.assertEqual( url_class.GetReferralURL( good_url, None ), None )
#
converted_referral_url = good_url.replace( 'testbooru.cx', 'replace.com' )
transformations = []
transformations.append( ( ClientParsing.STRING_TRANSFORMATION_REGEX_SUB, ( 'testbooru.cx', 'replace.com' ) ) )
referral_url_converter = ClientParsing.StringConverter( transformations, good_url )
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_CONVERTER_IF_NONE_PROVIDED
url_class = ClientNetworkingDomain.URLClass( name, url_type = url_type, preferred_scheme = preferred_scheme, netloc = netloc, match_subdomains = match_subdomains, keep_matched_subdomains = keep_matched_subdomains, path_components = path_components, parameters = parameters, send_referral_url = send_referral_url, referral_url_converter = referral_url_converter, can_produce_multiple_files = can_produce_multiple_files, should_be_associated_with_files = should_be_associated_with_files, gallery_index_type = gallery_index_type, gallery_index_identifier = gallery_index_identifier, gallery_index_delta = gallery_index_delta, example_url = example_url )
self.assertEqual( url_class.GetReferralURL( good_url, referral_url ), referral_url )
self.assertEqual( url_class.GetReferralURL( good_url, None ), converted_referral_url )
#
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_ONLY_CONVERTER
url_class = ClientNetworkingDomain.URLClass( name, url_type = url_type, preferred_scheme = preferred_scheme, netloc = netloc, match_subdomains = match_subdomains, keep_matched_subdomains = keep_matched_subdomains, path_components = path_components, parameters = parameters, send_referral_url = send_referral_url, referral_url_converter = referral_url_converter, can_produce_multiple_files = can_produce_multiple_files, should_be_associated_with_files = should_be_associated_with_files, gallery_index_type = gallery_index_type, gallery_index_identifier = gallery_index_identifier, gallery_index_delta = gallery_index_delta, example_url = example_url )
self.assertEqual( url_class.GetReferralURL( good_url, referral_url ), converted_referral_url )
self.assertEqual( url_class.GetReferralURL( good_url, None ), converted_referral_url )
class TestNetworkingEngine( unittest.TestCase ):
def test_engine_shutdown_app( self ):

7
include/TestController.py
View File

@ -24,6 +24,7 @@ from . import ClientNetworkingSessions
from . import ClientServices
from . import ClientTags
from . import ClientThreading
from . import HydrusDB
from . import HydrusExceptions
from . import HydrusPubSub
from . import HydrusSessions
@ -187,6 +188,7 @@ class Controller( object ):
HG.server_controller = self
HG.test_controller = self
self.db = self
self.gui = self
self._call_to_threads = []
@ -589,6 +591,11 @@ class Controller( object ):
return False
def PauseAndDisconnect( self, pause_and_disconnect ):
pass
def Read( self, name, *args, **kwargs ):
return self._reads[ name ]

20
include/TestHydrusServer.py
View File

@ -563,10 +563,26 @@ class TestServer( unittest.TestCase ):
#
## backup
response = service.Request( HC.GET, 'busy' )
self.assertEqual( response, b'0' )
response = service.Request( HC.POST, 'lock_on' )
response = service.Request( HC.GET, 'busy' )
self.assertEqual( response, b'1' )
response = service.Request( HC.POST, 'lock_off' )
response = service.Request( HC.GET, 'busy' )
self.assertEqual( response, b'0' )
#
response = service.Request( HC.POST, 'backup' )
response = service.Request( HC.POST, 'backup', { 'skip_vacuum' : True } )
response = service.Request( HC.POST, 'vacuum' )
#