RepoMirrors
/
hydrus
mirror of https://github.com/hydrusnetwork/hydrus.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Version 280

This commit is contained in:
Hydrus Network Developer 2017-11-01 15:37:39 -05:00
parent 62fa9959fd
commit ab823c072c
23 changed files with 621 additions and 448 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
help/changelog.html
View File

@ -8,6 +8,28 @@
<div class="content">
<h3>changelog</h3>
<ul>
<li><h3>version 280</h3></li>
<ul>
<li>the client should now recognise EXIF file rotation and flipping in resolution discovery, thumbnail generation, and image rendering. existing files will be borked, so will have to either be manually deleted&reimported, or wait for a maintenance routine that will retroactively fix all these</li>
<li>videos with a non 1:1 sample aspect ratio (wew lad) should now import with correct resolutions</li>
<li>all 'tag import options' are now managed with the new button</li>
<li>deleted all old tag option management gui, including deleting ClientGUICollapsible entirely</li>
<li>with the simpler tag import options, increased max page count to 200</li>
<li>page of images downloader now has two pause separate buttons for queue and files processing</li>
<li>the page of images downloader should no longer too-quickly blank out its parser status at the end of queue processing</li>
<li>the default hydrus bandwidth rules now no longer has the choking 'requests per day' limitation that was messing up large mappings uploads. (it is now just 64MB/day)</li>
<li>your hydrus default bandwidth rules will reset to this on update</li>
<li>POST requests (which in for our purposes are always user-driven) no longer obey bandwidth rules</li>
<li>fixed a bug when sometimes hitting the file limit or hitting a siteside 404 in the gallery downloader</li>
<li>a rare bug due to empty string menu labels is now no longer possible. empty string labels, if they slip through, will be replaced with 'invalid label'</li>
<li>you can no longer choose the empty string as a gui session name</li>
<li>improved how network jobs keep track of all applicable network contexts vs the preferred session and login contexts</li>
<li>cleaned up a bunch of login code, improved verification testing</li>
<li>fleshed out domain login system</li>
<li>in the networking engine, the read timeout is now six times the connect timeout--we'll see if this reduces some hydrus and other read timeout problems we've seen</li>
<li>increased the server transaction period from 10s to 120s--we'll see if this helps reduce some spiky server POST lag or what</li>
<li>the default tag import options panel now edits on double-click, rather than deletes. it also yes/no dialogs on a delete action. this control is still trash, but it now works more like the rest of the program</li>
</ul>
<li><h3>version 279</h3></li>
<ul>
<li>moved the hydrus service object over to the new networking engine</li>

19
include/ClientDB.py
View File

@ -9941,6 +9941,25 @@ class DB( HydrusDB.HydrusDB ):
self.pub_initial_message( message )
if version == 279:
bandwidth_manager = self._GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER )
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_DATA, 86400, 64 * 1048576 ) # don't sync a giant db in one day
bandwidth_manager.SetRules( ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_HYDRUS ), rules )
self._SetJSONDump( bandwidth_manager )
message = 'Your default hydrus service bandwidth rules have been reset to a new default that works better now the hydrus network runs on the new networking engine.'
message += os.linesep * 2
message += 'If you don\'t care about bandwidth rules, you do not have to do anything!'
self.pub_initial_message( message )
self._controller.pub( 'splash_set_title_text', 'updated db to v' + str( version + 1 ) )
self._c.execute( 'UPDATE version SET version = ?;', ( version + 1, ) )

2
include/ClientDefaults.py
View File

@ -40,8 +40,6 @@ def SetDefaultBandwidthManagerRules( bandwidth_manager ):
rules = HydrusNetworking.BandwidthRules()
rules.AddRule( HC.BANDWIDTH_TYPE_REQUESTS, 86400, 50 ) # don't sync a giant db in one day
rules.AddRule( HC.BANDWIDTH_TYPE_DATA, 86400, 64 * MB ) # don't sync a giant db in one day
bandwidth_manager.SetRules( ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_HYDRUS ), rules )

2
include/ClientDownloading.py
View File

@ -922,7 +922,7 @@ class GalleryBooru( Gallery ):
if link.string is not None:
if link.string.startswith( image_data ):
if link.string.startswith( image_data ) or link.string.endswith( image_data ):
ok_link = link[ 'href' ]

140
include/ClientGUICollapsible.py
View File

@ -1,140 +0,0 @@
import ClientConstants as CC
import ClientGUICommon
import ClientGUIOptionsPanels
import ClientGUITopLevelWindows
import wx
class CollapsibleOptions( ClientGUICommon.StaticBox ):
options_panel_class = ClientGUIOptionsPanels.OptionsPanel
staticbox_title = 'not implemented'
def __init__( self, parent ):
ClientGUICommon.StaticBox.__init__( self, parent, self.staticbox_title )
self._collapsible_panel = CollapsiblePanel( self )
self._options_panel = self.options_panel_class( self._collapsible_panel )
self._collapsible_panel.SetPanel( self._options_panel )
self.AddF( self._collapsible_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
def ExpandCollapse( self ):
self._collapsible_panel.ExpandCollapse()
def GetOptions( self ):
return self._options_panel.GetOptions()
def SetOptions( self, options ):
self._options_panel.SetOptions( options )
class CollapsibleOptionsTags( CollapsibleOptions ):
options_panel_class = ClientGUIOptionsPanels.OptionsPanelTags
staticbox_title = 'import options - tags'
def __init__( self, parent, namespaces = None ):
CollapsibleOptions.__init__( self, parent )
if namespaces is None: namespaces = []
self.SetNamespaces( namespaces )
def SetNamespaces( self, namespaces ):
self._options_panel.SetNamespaces( namespaces )
if self._collapsible_panel.IsExpanded():
self._collapsible_panel.ExpandCollapse()
class CollapsiblePanel( wx.Panel ):
def __init__( self, parent ):
wx.Panel.__init__( self, parent )
self._expanded = False
self._panel = None
self._vbox = wx.BoxSizer( wx.VERTICAL )
hbox = wx.BoxSizer( wx.HORIZONTAL )
self._button = ClientGUICommon.BetterButton( self, 'expand', self.ExpandCollapse )
line = wx.StaticLine( self, style = wx.LI_HORIZONTAL )
hbox.AddF( self._button, CC.FLAGS_VCENTER )
hbox.AddF( line, CC.FLAGS_EXPAND_DEPTH_ONLY )
self._vbox.AddF( hbox, CC.FLAGS_EXPAND_PERPENDICULAR )
self.SetSizer( self._vbox )
def ExpandCollapse( self ):
if self._expanded:
self._button.SetLabelText( 'expand' )
self._panel.Hide()
self._expanded = False
else:
self._button.SetLabelText( 'collapse' )
self._panel.Show()
self._expanded = True
parent = self
while not isinstance( parent, wx.ScrolledWindow ) and not isinstance( parent, wx.TopLevelWindow ):
parent = parent.GetParent()
if isinstance( parent, wx.ScrolledWindow ):
parent.FitInside()
else:
parent.Layout()
ClientGUITopLevelWindows.PostSizeChangedEvent( self )
def IsExpanded( self ):
return self._expanded
def SetPanel( self, panel ):
self._panel = panel
self._vbox.AddF( self._panel, CC.FLAGS_EXPAND_BOTH_WAYS )
self._panel.Hide()

60
include/ClientGUIDialogs.py
View File

@ -10,7 +10,6 @@ import ClientFiles
import ClientGUIACDropdown
import ClientGUIFrames
import ClientGUICommon
import ClientGUICollapsible
import ClientGUIImport
import ClientGUIListBoxes
import ClientGUIListCtrl
@ -459,65 +458,6 @@ class DialogGenerateNewAccounts( Dialog ):
class DialogInputTagImportOptions( Dialog ):
def __init__( self, parent, pretty_name, gallery_identifier, tag_import_options = None ):
Dialog.__init__( self, parent, 'configure default import tag options for ' + pretty_name )
#
( namespaces, search_value ) = ClientDefaults.GetDefaultNamespacesAndSearchValue( gallery_identifier )
self._tag_import_options = ClientGUICollapsible.CollapsibleOptionsTags( self, namespaces = namespaces )
self._ok = wx.Button( self, id = wx.ID_OK, label = 'ok' )
self._ok.SetForegroundColour( ( 0, 128, 0 ) )
self._cancel = wx.Button( self, id = wx.ID_CANCEL, label = 'cancel' )
self._cancel.SetForegroundColour( ( 128, 0, 0 ) )
#
if tag_import_options is None:
new_options = HG.client_controller.GetNewOptions()
tag_import_options = new_options.GetDefaultTagImportOptions( gallery_identifier )
self._tag_import_options.SetOptions( tag_import_options )
#
b_box = wx.BoxSizer( wx.HORIZONTAL )
b_box.AddF( self._ok, CC.FLAGS_VCENTER )
b_box.AddF( self._cancel, CC.FLAGS_VCENTER )
vbox = wx.BoxSizer( wx.VERTICAL )
vbox.AddF( self._tag_import_options, CC.FLAGS_EXPAND_BOTH_WAYS )
vbox.AddF( b_box, CC.FLAGS_BUTTON_SIZER )
self.SetSizer( vbox )
( x, y ) = self.GetEffectiveMinSize()
x = max( 300, x )
y = max( 300, y )
self.SetInitialSize( ( x, y ) )
wx.CallAfter( self._tag_import_options.ExpandCollapse )
def GetTagImportOptions( self ):
tag_import_options = self._tag_import_options.GetOptions()
return tag_import_options
class DialogInputFileSystemPredicates( Dialog ):
def __init__( self, parent, predicate_type ):

1
include/ClientGUIDialogsManage.py
View File

@ -7,7 +7,6 @@ import ClientDragDrop
import ClientExporting
import ClientFiles
import ClientGUIACDropdown
import ClientGUICollapsible
import ClientGUICommon
import ClientGUIListBoxes
import ClientGUIListCtrl

5
include/ClientGUIImport.py
View File

@ -113,6 +113,11 @@ class TagImportOptionsButton( ClientGUICommon.BetterButton ):
return self._tag_import_options
def SetNamespaces( self, namespaces ):
self._namespaces = namespaces
def SetValue( self, tag_import_options ):
self._SetValue( tag_import_options )

128
include/ClientGUIManagement.py
View File

@ -12,7 +12,6 @@ import ClientCaches
import ClientFiles
import ClientGUIACDropdown
import ClientGUICanvas
import ClientGUICollapsible
import ClientGUICommon
import ClientGUIControls
import ClientGUIDialogs
@ -1357,8 +1356,12 @@ class ManagementPanelImporterGallery( ManagementPanelImporter ):
( file_import_options, tag_import_options, file_limit ) = self._gallery_import.GetOptions()
gallery_identifier = self._gallery_import.GetGalleryIdentifier()
( namespaces, search_value ) = ClientDefaults.GetDefaultNamespacesAndSearchValue( gallery_identifier )
self._file_import_options = ClientGUIImport.FileImportOptionsButton( self._gallery_downloader_panel, file_import_options, self._gallery_import.SetFileImportOptions )
self._tag_import_options = ClientGUICollapsible.CollapsibleOptionsTags( self._gallery_downloader_panel )
self._tag_import_options = ClientGUIImport.TagImportOptionsButton( self._gallery_downloader_panel, namespaces, tag_import_options, self._gallery_import.SetTagImportOptions )
#
@ -1423,21 +1426,12 @@ class ManagementPanelImporterGallery( ManagementPanelImporter ):
#
self.Bind( wx.EVT_MENU, self.EventMenu )
gallery_identifier = self._gallery_import.GetGalleryIdentifier()
( namespaces, search_value ) = ClientDefaults.GetDefaultNamespacesAndSearchValue( gallery_identifier )
seed_cache = self._gallery_import.GetSeedCache()
self._seed_cache_control.SetSeedCache( seed_cache )
self._tag_import_options.SetNamespaces( namespaces )
self._query_input.SetValue( search_value )
self._tag_import_options.SetOptions( tag_import_options )
self._file_limit.SetValue( file_limit )
self._UpdateStatus()
@ -1636,27 +1630,6 @@ class ManagementPanelImporterGallery( ManagementPanelImporter ):
def EventMenu( self, event ):
action = ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetAction( event.GetId() )
if action is not None:
( command, data ) = action
if command == 'tag_import_options_changed':
tag_import_options = self._tag_import_options.GetOptions()
self._gallery_import.SetTagImportOptions( tag_import_options )
else:
event.Skip()
def EventPaste( self, event ):
if wx.TheClipboard.Open():
@ -1820,19 +1793,24 @@ class ManagementPanelImporterPageOfImages( ManagementPanelImporter ):
self._page_of_images_panel = ClientGUICommon.StaticBox( self, 'page of images downloader' )
self._pause_button = wx.BitmapButton( self._page_of_images_panel, bitmap = CC.GlobalBMPs.pause )
self._pause_button.Bind( wx.EVT_BUTTON, self.EventPause )
#
self._import_queue_panel = ClientGUICommon.StaticBox( self._page_of_images_panel, 'imports' )
self._pause_files_button = wx.BitmapButton( self._import_queue_panel, bitmap = CC.GlobalBMPs.pause )
self._pause_files_button.Bind( wx.EVT_BUTTON, self.EventPauseFiles )
self._current_action = ClientGUICommon.BetterStaticText( self._import_queue_panel )
self._seed_cache_control = ClientGUISeedCache.SeedCacheStatusControl( self._import_queue_panel, self._controller )
self._file_download_control = ClientGUIControls.NetworkJobControl( self._import_queue_panel )
#
self._pending_page_urls_panel = ClientGUICommon.StaticBox( self._page_of_images_panel, 'pending page urls' )
self._pause_queue_button = wx.BitmapButton( self._pending_page_urls_panel, bitmap = CC.GlobalBMPs.pause )
self._pause_queue_button.Bind( wx.EVT_BUTTON, self.EventPauseQueue )
self._parser_status = ClientGUICommon.BetterStaticText( self._pending_page_urls_panel )
self._page_download_control = ClientGUIControls.NetworkJobControl( self._pending_page_urls_panel )
@ -1870,6 +1848,11 @@ class ManagementPanelImporterPageOfImages( ManagementPanelImporter ):
#
self._import_queue_panel.AddF( self._current_action, CC.FLAGS_EXPAND_PERPENDICULAR )
self._import_queue_panel.AddF( self._seed_cache_control, CC.FLAGS_EXPAND_PERPENDICULAR )
self._import_queue_panel.AddF( self._file_download_control, CC.FLAGS_EXPAND_PERPENDICULAR )
self._import_queue_panel.AddF( self._pause_files_button, CC.FLAGS_LONE_BUTTON )
queue_buttons_vbox = wx.BoxSizer( wx.VERTICAL )
queue_buttons_vbox.AddF( self._advance_button, CC.FLAGS_VCENTER )
@ -1890,14 +1873,10 @@ class ManagementPanelImporterPageOfImages( ManagementPanelImporter ):
self._pending_page_urls_panel.AddF( self._page_download_control, CC.FLAGS_EXPAND_PERPENDICULAR )
self._pending_page_urls_panel.AddF( queue_hbox, CC.FLAGS_EXPAND_SIZER_BOTH_WAYS )
self._pending_page_urls_panel.AddF( input_hbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
self._pending_page_urls_panel.AddF( self._pause_queue_button, CC.FLAGS_LONE_BUTTON )
#
self._import_queue_panel.AddF( self._current_action, CC.FLAGS_EXPAND_PERPENDICULAR )
self._import_queue_panel.AddF( self._seed_cache_control, CC.FLAGS_EXPAND_PERPENDICULAR )
self._import_queue_panel.AddF( self._file_download_control, CC.FLAGS_EXPAND_PERPENDICULAR )
self._page_of_images_panel.AddF( self._pause_button, CC.FLAGS_LONE_BUTTON )
self._page_of_images_panel.AddF( self._import_queue_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
self._page_of_images_panel.AddF( self._pending_page_urls_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
self._page_of_images_panel.AddF( self._download_image_links, CC.FLAGS_EXPAND_PERPENDICULAR )
@ -1949,7 +1928,7 @@ class ManagementPanelImporterPageOfImages( ManagementPanelImporter ):
def _UpdateStatus( self ):
( pending_page_urls, parser_status, current_action, paused ) = self._page_of_images_import.GetStatus()
( pending_page_urls, parser_status, current_action, queue_paused, files_paused ) = self._page_of_images_import.GetStatus()
if self._pending_page_urls_listbox.GetStrings() != pending_page_urls:
@ -1965,32 +1944,47 @@ class ManagementPanelImporterPageOfImages( ManagementPanelImporter ):
if paused:
if queue_paused:
parser_status = 'paused'
self._parser_status.SetLabelText( parser_status )
if current_action == '' and paused:
if current_action == '' and files_paused:
current_action = 'paused'
self._current_action.SetLabelText( current_action )
if paused:
if queue_paused:
if self._pause_button.GetBitmap() != CC.GlobalBMPs.play:
if self._pause_queue_button.GetBitmap() != CC.GlobalBMPs.play:
self._pause_button.SetBitmap( CC.GlobalBMPs.play )
self._pause_queue_button.SetBitmap( CC.GlobalBMPs.play )
else:
if self._pause_button.GetBitmap() != CC.GlobalBMPs.pause:
if self._pause_queue_button.GetBitmap() != CC.GlobalBMPs.pause:
self._pause_button.SetBitmap( CC.GlobalBMPs.pause )
self._pause_queue_button.SetBitmap( CC.GlobalBMPs.pause )
if files_paused:
if self._pause_files_button.GetBitmap() != CC.GlobalBMPs.play:
self._pause_files_button.SetBitmap( CC.GlobalBMPs.play )
else:
if self._pause_files_button.GetBitmap() != CC.GlobalBMPs.pause:
self._pause_files_button.SetBitmap( CC.GlobalBMPs.pause )
@ -2105,9 +2099,16 @@ class ManagementPanelImporterPageOfImages( ManagementPanelImporter ):
def EventPause( self, event ):
def EventPauseQueue( self, event ):
self._page_of_images_import.PausePlay()
self._page_of_images_import.PausePlayQueue()
self._UpdateStatus()
def EventPauseFiles( self, event ):
self._page_of_images_import.PausePlayFiles()
self._UpdateStatus()
@ -2189,7 +2190,7 @@ class ManagementPanelImporterThreadWatcher( ManagementPanelImporter ):
( thread_url, file_import_options, tag_import_options ) = self._thread_watcher_import.GetOptions()
self._file_import_options = ClientGUIImport.FileImportOptionsButton( self._thread_watcher_panel, file_import_options, self._thread_watcher_import.SetFileImportOptions )
self._tag_import_options = ClientGUICollapsible.CollapsibleOptionsTags( self._thread_watcher_panel, namespaces = [ 'filename' ] )
self._tag_import_options = ClientGUIImport.TagImportOptionsButton( self._thread_watcher_panel, [ 'filename' ], tag_import_options, self._thread_watcher_import.SetTagImportOptions )
#
@ -2247,8 +2248,6 @@ class ManagementPanelImporterThreadWatcher( ManagementPanelImporter ):
#
self.Bind( wx.EVT_MENU, self.EventMenu )
seed_cache = self._thread_watcher_import.GetSeedCache()
self._seed_cache_control.SetSeedCache( seed_cache )
@ -2258,8 +2257,6 @@ class ManagementPanelImporterThreadWatcher( ManagementPanelImporter ):
self._thread_input.SetValue( thread_url )
self._tag_import_options.SetOptions( tag_import_options )
self._UpdateStatus()
@ -2455,27 +2452,6 @@ class ManagementPanelImporterThreadWatcher( ManagementPanelImporter ):
def EventMenu( self, event ):
action = ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetAction( event.GetId() )
if action is not None:
( command, data ) = action
if command == 'tag_import_options_changed':
tag_import_options = self._tag_import_options.GetOptions()
self._thread_watcher_import.SetTagImportOptions( tag_import_options )
else:
event.Skip()
def SetSearchFocus( self, page_key ):
if page_key == self._page_key and self._thread_input.IsEditable():

5
include/ClientGUIMenus.py
View File

@ -126,5 +126,10 @@ def GetEventCallable( callable, *args, **kwargs ):
def SanitiseLabel( label ):
if label == '':
label = '-invalid label-'
return label.replace( '&', '&&' )

1
include/ClientGUIOptionsPanels.py
View File

@ -1,4 +1,3 @@
import ClientGUICollapsible
import ClientConstants as CC
import ClientGUICommon
import ClientCaches

4
include/ClientGUIPages.py
View File

@ -1738,7 +1738,7 @@ class PagesNotebook( wx.Notebook ):
if not HG.no_page_limit_mode:
MAX_TOTAL_PAGES = 150
MAX_TOTAL_PAGES = 200
( total_active_page_count, total_closed_page_count ) = self._controller.gui.GetTotalPageCounts()
@ -2202,7 +2202,7 @@ class PagesNotebook( wx.Notebook ):
name = dlg.GetValue()
if name in ( 'just a blank page', 'last session' ):
if name in ( '', 'just a blank page', 'last session' ):
wx.MessageBox( 'Sorry, you cannot have that name! Try another.' )

12
include/ClientGUIScrolledPanelsEdit.py
View File

@ -4,7 +4,6 @@ import ClientData
import ClientDefaults
import ClientDownloading
import ClientImporting
import ClientGUICollapsible
import ClientGUICommon
import ClientGUIControls
import ClientGUIDialogs
@ -1871,7 +1870,9 @@ class EditSubscriptionPanel( ClientGUIScrolledPanels.EditPanel ):
self._file_import_options = ClientGUIImport.FileImportOptionsButton( self, file_import_options )
self._tag_import_options = ClientGUICollapsible.CollapsibleOptionsTags( self )
( namespaces, search_value ) = ClientDefaults.GetDefaultNamespacesAndSearchValue( gallery_identifier )
self._tag_import_options = ClientGUIImport.TagImportOptionsButton( self, namespaces, tag_import_options )
#
@ -1906,8 +1907,6 @@ class EditSubscriptionPanel( ClientGUIScrolledPanels.EditPanel ):
self._paused.SetValue( paused )
self._tag_import_options.SetOptions( tag_import_options )
if self._last_checked == 0:
self._reset_cache_button.Disable()
@ -2000,8 +1999,9 @@ class EditSubscriptionPanel( ClientGUIScrolledPanels.EditPanel ):
self._query.SetValue( search_value )
self._tag_import_options.SetNamespaces( namespaces )
self._tag_import_options.SetOptions( tag_import_options )
self._tag_import_options.SetValue( tag_import_options )
def _GetGalleryIdentifier( self ):
@ -2176,7 +2176,7 @@ class EditSubscriptionPanel( ClientGUIScrolledPanels.EditPanel ):
file_import_options = self._file_import_options.GetValue()
tag_import_options = self._tag_import_options.GetOptions()
tag_import_options = self._tag_import_options.GetValue()
subscription.SetTuple( gallery_identifier, gallery_stream_identifiers, query, period, self._get_tags_if_url_known_and_file_redundant, initial_file_limit, periodic_file_limit, paused, file_import_options, tag_import_options, self._last_checked, self._last_error, self._check_now, self._seed_cache )

42
include/ClientGUIScrolledPanelsManagement.py
View File

@ -1,6 +1,7 @@
import ClientCaches
import ClientConstants as CC
import ClientData
import ClientDefaults
import ClientDownloading
import ClientGUIACDropdown
import ClientGUICommon
@ -2221,7 +2222,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._new_options = new_options
self._tag_import_options = wx.ListBox( self )
self._tag_import_options.Bind( wx.EVT_LEFT_DCLICK, self.EventDelete )
self._tag_import_options.Bind( wx.EVT_LEFT_DCLICK, self.EventEdit )
self._add = wx.Button( self, label = 'add' )
self._add.Bind( wx.EVT_BUTTON, self.EventAdd )
@ -2296,11 +2297,21 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
with ClientGUIDialogs.DialogInputTagImportOptions( self, name, gallery_identifier ) as ito_dlg:
new_options = HG.client_controller.GetNewOptions()
tag_import_options = new_options.GetDefaultTagImportOptions( gallery_identifier )
( namespaces, search_value ) = ClientDefaults.GetDefaultNamespacesAndSearchValue( gallery_identifier )
with ClientGUITopLevelWindows.DialogEdit( self, 'edit tag import options' ) as dlg:
if ito_dlg.ShowModal() == wx.ID_OK:
panel = ClientGUIScrolledPanelsEdit.EditTagImportOptions( dlg, namespaces, tag_import_options )
dlg.SetPanel( panel )
if dlg.ShowModal() == wx.ID_OK:
tag_import_options = ito_dlg.GetTagImportOptions()
tag_import_options = panel.GetValue()
self._tag_import_options.Append( name, ( gallery_identifier, tag_import_options ) )
@ -2313,7 +2324,18 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
selection = self._tag_import_options.GetSelection()
if selection != wx.NOT_FOUND: self._tag_import_options.Delete( selection )
if selection != wx.NOT_FOUND:
name = self._tag_import_options.GetString( selection )
with ClientGUIDialogs.DialogYesNo( self, 'Delete \'' + name + '\' entry?' ) as dlg:
if dlg.ShowModal() == wx.ID_YES:
self._tag_import_options.Delete( selection )
def EventEdit( self, event ):
@ -2326,11 +2348,17 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
( gallery_identifier, tag_import_options ) = self._tag_import_options.GetClientData( selection )
with ClientGUIDialogs.DialogInputTagImportOptions( self, name, gallery_identifier, tag_import_options ) as dlg:
( namespaces, search_value ) = ClientDefaults.GetDefaultNamespacesAndSearchValue( gallery_identifier )
with ClientGUITopLevelWindows.DialogEdit( self, 'edit tag import options' ) as dlg:
panel = ClientGUIScrolledPanelsEdit.EditTagImportOptions( dlg, namespaces, tag_import_options )
dlg.SetPanel( panel )
if dlg.ShowModal() == wx.ID_OK:
tag_import_options = dlg.GetTagImportOptions()
tag_import_options = panel.GetValue()
self._tag_import_options.SetClientData( selection, ( gallery_identifier, tag_import_options ) )

63
include/ClientImporting.py
View File

@ -723,6 +723,9 @@ class GalleryImport( HydrusSerialisable.SerialisableBase ):
error_occured = False
num_already_in_seed_cache = 0
new_urls = []
try:
( page_of_urls, definitely_no_more_pages ) = gallery.GetPage( query, page_index )
@ -743,9 +746,6 @@ class GalleryImport( HydrusSerialisable.SerialisableBase ):
num_already_in_seed_cache = 0
new_urls = []
for url in page_of_urls:
if self._seed_cache.HasSeed( url ):
@ -1928,7 +1928,7 @@ HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIAL
class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_PAGE_OF_IMAGES_IMPORT
SERIALISABLE_VERSION = 1
SERIALISABLE_VERSION = 2
def __init__( self ):
@ -1941,7 +1941,8 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
self._file_import_options = file_import_options
self._download_image_links = True
self._download_unlinked_images = False
self._paused = False
self._queue_paused = False
self._files_paused = False
self._parser_status = ''
self._current_action = ''
@ -1962,17 +1963,31 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
serialisable_url_cache = self._urls_cache.GetSerialisableTuple()
serialisable_file_options = self._file_import_options.GetSerialisableTuple()
return ( self._pending_page_urls, serialisable_url_cache, serialisable_file_options, self._download_image_links, self._download_unlinked_images, self._paused )
return ( self._pending_page_urls, serialisable_url_cache, serialisable_file_options, self._download_image_links, self._download_unlinked_images, self._queue_paused, self._files_paused )
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( self._pending_page_urls, serialisable_url_cache, serialisable_file_options, self._download_image_links, self._download_unlinked_images, self._paused ) = serialisable_info
( self._pending_page_urls, serialisable_url_cache, serialisable_file_options, self._download_image_links, self._download_unlinked_images, self._queue_paused, self._files_paused ) = serialisable_info
self._urls_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_url_cache )
self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_options )
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
if version == 1:
( pending_page_urls, serialisable_url_cache, serialisable_file_options, download_image_links, download_unlinked_images, paused ) = old_serialisable_info
queue_paused = paused
files_paused = paused
new_serialisable_info = ( pending_page_urls, serialisable_url_cache, serialisable_file_options, download_image_links, download_unlinked_images, queue_paused, files_paused )
return ( 2, new_serialisable_info )
def _WorkOnFiles( self, page_key ):
file_url = self._urls_cache.GetNextSeed( CC.STATUS_UNKNOWN )
@ -2260,18 +2275,15 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
time.sleep( 5 )
with self._lock:
if len( self._pending_page_urls ) == 0:
self._parser_status = ''
return True
else:
with self._lock:
self._parser_status = ''
return False
@ -2280,7 +2292,7 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
while not ( HG.view_shutdown or HG.client_controller.PageCompletelyDestroyed( page_key ) ):
if self._paused or HG.client_controller.PageClosedButNotDestroyed( page_key ):
if self._files_paused or HG.client_controller.PageClosedButNotDestroyed( page_key ):
self._new_files_event.wait( 5 )
@ -2317,7 +2329,7 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
while not ( HG.view_shutdown or HG.client_controller.PageCompletelyDestroyed( page_key ) ):
if self._paused or HG.client_controller.PageClosedButNotDestroyed( page_key ):
if self._queue_paused or HG.client_controller.PageClosedButNotDestroyed( page_key ):
self._new_page_event.wait( 5 )
@ -2374,7 +2386,7 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
finished = not self._urls_cache.WorkToDo() or len( self._pending_page_urls ) > 0
return not finished and not self._paused
return not finished and not self._files_paused
@ -2424,17 +2436,26 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
with self._lock:
return ( list( self._pending_page_urls ), self._parser_status, self._current_action, self._paused )
return ( list( self._pending_page_urls ), self._parser_status, self._current_action, self._queue_paused, self._files_paused )
def PausePlay( self ):
def PausePlayFiles( self ):
with self._lock:
self._paused = not self._paused
self._files_paused = not self._files_paused
self._new_files_event.set()
def PausePlayQueue( self ):
with self._lock:
self._queue_paused = not self._queue_paused
self._new_page_event.set()

93
include/ClientNetworking.py
View File

@ -1002,6 +1002,8 @@ class NetworkJob( object ):
self._network_contexts = self._GenerateNetworkContexts()
( self._session_network_context, self._login_network_context ) = self._GenerateSpecificNetworkContexts()
def _CanReattemptRequest( self ):
@ -1031,6 +1033,20 @@ class NetworkJob( object ):
return network_contexts
def _GenerateSpecificNetworkContexts( self ):
# we always store cookies in the larger session
# but we can login to a specific subdomain
domain = ClientNetworkingDomain.ConvertURLIntoDomain( self._url )
domains = ClientNetworkingDomain.ConvertDomainIntoAllApplicableDomains( domain )
session_network_context = NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, domains[-1] )
login_network_context = NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, domain )
return ( session_network_context, login_network_context )
def _SendRequestAndGetResponse( self ):
with self._lock:
@ -1062,23 +1078,18 @@ class NetworkJob( object ):
self._status_text = u'sending request\u2026'
timeout = HG.client_controller.GetNewOptions().GetInteger( 'network_timeout' )
connect_timeout = HG.client_controller.GetNewOptions().GetInteger( 'network_timeout' )
response = session.request( method, url, data = data, files = files, headers = headers, stream = True, timeout = timeout )
read_timeout = connect_timeout * 6
response = session.request( method, url, data = data, files = files, headers = headers, stream = True, timeout = ( connect_timeout, read_timeout ) )
return response
def _GetSession( self ):
session_network_context = self._GetSessionNetworkContext()
return self.engine.session_manager.GetSession( session_network_context )
def _GetSessionNetworkContext( self ):
return self._network_contexts[-1]
return self.engine.session_manager.GetSession( self._session_network_context )
def _IsCancelled( self ):
@ -1113,7 +1124,7 @@ class NetworkJob( object ):
def _ObeysBandwidth( self ):
return not ( self._bandwidth_manual_override or self._for_login )
return not ( self._method == 'POST' or self._bandwidth_manual_override or self._for_login )
def _OngoingBandwidthOK( self ):
@ -1303,9 +1314,7 @@ class NetworkJob( object ):
else:
session_network_context = self._GetSessionNetworkContext()
return self.engine.login_manager.CanLogin( session_network_context )
return self.engine.login_manager.CanLogin( self._login_network_context )
@ -1328,9 +1337,7 @@ class NetworkJob( object ):
else:
session_network_context = self._GetSessionNetworkContext()
return self.engine.login_manager.GenerateLoginProcess( session_network_context )
return self.engine.login_manager.GenerateLoginProcess( self._login_network_context )
@ -1459,9 +1466,7 @@ class NetworkJob( object ):
else:
session_network_context = self._GetSessionNetworkContext()
return self.engine.login_manager.NeedsLogin( session_network_context )
return self.engine.login_manager.NeedsLogin( self._login_network_context )
@ -1717,11 +1722,6 @@ class NetworkJobDownloader( NetworkJob ):
return network_contexts
def _GetSessionNetworkContext( self ):
return self._network_contexts[-2] # the domain one
class NetworkJobDownloaderQuery( NetworkJobDownloader ):
def __init__( self, downloader_page_key, downloader_key, method, url, body = None, referral_url = None, temp_path = None ):
@ -1740,11 +1740,6 @@ class NetworkJobDownloaderQuery( NetworkJobDownloader ):
return network_contexts
def _GetSessionNetworkContext( self ):
return self._network_contexts[-3] # the domain one
class NetworkJobDownloaderQueryTemporary( NetworkJob ):
def __init__( self, downloader_page_key, method, url, body = None, referral_url = None, temp_path = None ):
@ -1763,11 +1758,6 @@ class NetworkJobDownloaderQueryTemporary( NetworkJob ):
return network_contexts
def _GetSessionNetworkContext( self ):
return self._network_contexts[-2] # the domain one
class NetworkJobSubscription( NetworkJobDownloader ):
def __init__( self, subscription_key, downloader_key, method, url, body = None, referral_url = None, temp_path = None ):
@ -1786,11 +1776,6 @@ class NetworkJobSubscription( NetworkJobDownloader ):
return network_contexts
def _GetSessionNetworkContext( self ):
return self._network_contexts[-3] # the domain one
class NetworkJobSubscriptionTemporary( NetworkJob ):
def __init__( self, subscription_key, method, url, body = None, referral_url = None, temp_path = None ):
@ -1809,11 +1794,6 @@ class NetworkJobSubscriptionTemporary( NetworkJob ):
return network_contexts
def _GetSessionNetworkContext( self ):
return self._network_contexts[-2] # the domain one
class NetworkJobHydrus( NetworkJob ):
IS_HYDRUS_SERVICE = True
@ -1866,6 +1846,16 @@ class NetworkJobHydrus( NetworkJob ):
return network_contexts
def _GenerateSpecificNetworkContexts( self ):
# we store cookies on and login to the same hydrus-specific context
session_network_context = NetworkContext( CC.NETWORK_CONTEXT_HYDRUS, self._service_key )
login_network_context = session_network_context
return ( session_network_context, login_network_context )
def _ReportDataUsed( self, num_bytes ):
service = self.engine.controller.services_manager.GetService( self._service_key )
@ -1923,11 +1913,6 @@ class NetworkJobThreadWatcher( NetworkJob ):
return network_contexts
def _GetSessionNetworkContext( self ):
return self._network_contexts[-2] # the domain one
class NetworkSessionManager( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER
@ -2006,6 +1991,14 @@ class NetworkSessionManager( HydrusSerialisable.SerialisableBase ):
with self._lock:
# just in case one of these slips through somehow
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
second_level_domain = ClientNetworkingDomain.ConvertDomainIntoSecondLevelDomain( network_context.context_data )
network_context = NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, second_level_domain )
if network_context not in self._network_contexts_to_sessions:
self._network_contexts_to_sessions[ network_context ] = self._GenerateSession( network_context )

34
include/ClientNetworkingDomain.py
View File

@ -25,7 +25,7 @@ def ConvertDomainIntoAllApplicableDomains( domain ):
while domain.count( '.' ) > 0:
# let's discard www.blah.com so we don't end up tracking it separately to blah.com--there's not much point!
# let's discard www.blah.com and www2.blah.com so we don't end up tracking it separately to blah.com--there's not much point!
startswith_www = domain.count( '.' ) > 1 and domain.startswith( 'www' )
if not startswith_www:
@ -38,6 +38,10 @@ def ConvertDomainIntoAllApplicableDomains( domain ):
return domains
def ConvertDomainIntoSecondLevelDomain( domain ):
return ConvertDomainIntoAllApplicableDomains( domain )[-1]
def ConvertURLIntoDomain( url ):
parser_result = urlparse.urlparse( url )
@ -46,6 +50,34 @@ def ConvertURLIntoDomain( url ):
return domain
def GetCookie( cookies, search_domain, name ):
existing_domains = cookies.list_domains()
for existing_domain in existing_domains:
# blah.com is viewable by blah.com
matches_exactly = existing_domain == search_domain
# .blah.com is viewable by blah.com
matches_dot = existing_domain == '.' + search_domain
# .blah.com applies to subdomain.blah.com, blah.com does not
valid_subdomain = existing_domain.startwith( '.' ) and search_domain.endswith( existing_domain )
if matches_exactly or matches_dot or valid_subdomain:
cookie_dict = cookies.get_dict( existing_domain )
if name in cookie_dict:
return cookie_dict[ name ]
raise HydrusExceptions.DataMissing( 'Cookie ' + name + ' not found for domain ' + search_domain + '!' )
VALID_DENIED = 0
VALID_APPROVED = 1
VALID_UNKNOWN = 2

336
include/ClientNetworkingLogin.py
View File

@ -8,6 +8,7 @@ import HydrusGlobals as HG
import HydrusData
import HydrusExceptions
import HydrusSerialisable
import os
import json
import requests
@ -26,11 +27,14 @@ class LoginCredentials( object ):
self._credentials = {} # user-facing name (unique) : string
self._validity = VALIDITY_UNTESTED
# get current values
def GetCredential( self, name ):
return self._credentials[ name ]
# test current values, including fail for not having enough/having too many
class NetworkLoginManager( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER
@ -42,11 +46,13 @@ class NetworkLoginManager( HydrusSerialisable.SerialisableBase ):
HydrusSerialisable.SerialisableBase.__init__( self )
# needs _dirty and setdirty and be on that serialisation check and so on
self.engine = None
self._lock = threading.Lock()
self._domains_to_login_scripts = {}
self._domains_to_login_scripts_and_credentials = {}
self._hydrus_login_script = LoginScriptHydrus()
@ -66,28 +72,19 @@ class NetworkLoginManager( HydrusSerialisable.SerialisableBase ):
self._network_contexts_to_session_timeouts = {}
def _GetLoginScript( self, network_context ):
def _GetLoginNetworkContext( self, network_context ):
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
nc_domain = network_context.context_data
domains = ClientNetworkingDomain.ConvertDomainIntoAllApplicableDomains( nc_domain )
for domain in domains:
nc_domain = network_context.context_data
possible_domains = ClientNetworkingDomain.ConvertDomainIntoAllApplicableDomains( nc_domain )
for domain in possible_domains:
if domain in self._domains_to_login_scripts_and_credentials:
if domain in self._domains_to_login_scripts:
login_script = self._domains_to_login_scripts[ domain ]
return login_script
return domain
elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
return self._hydrus_login_script
return None
@ -108,12 +105,18 @@ class NetworkLoginManager( HydrusSerialisable.SerialisableBase ):
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
pass
login_network_context = self._GetLoginNetworkContext( network_context )
# look them up in our structure
# if they have a login, is it valid?
# valid means we have tested credentials and it hasn't been invalidated by a parsing error or similar
# I think this just means saying Login.CanLogin( credentials )
if login_network_context is None:
return False
( login_script, credentials ) = self._domains_to_login_scripts_and_credentials[ login_network_context.context_data ]
( result, reason ) = login_script.CanLogin( credentials )
return result
elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
@ -139,16 +142,25 @@ class NetworkLoginManager( HydrusSerialisable.SerialisableBase ):
with self._lock:
login_script = self._GetLoginScript( network_context )
if login_script is None:
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
login_script = LoginScript()
login_network_context = self._GetLoginNetworkContext( network_context )
if login_network_context is None:
raise HydrusExceptions.DataMissing()
( login_script, credentials ) = self._domains_to_login_scripts_and_credentials[ login_network_context.context_data ]
login_process = LoginProcessDomain( self.engine, login_network_context, login_script, credentials )
elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
login_process = LoginProcessHydrus( self.engine, network_context, self._hydrus_login_script )
return login_process
login_process = LoginProcess( self.engine, network_context, login_script )
return login_process
@ -156,16 +168,23 @@ class NetworkLoginManager( HydrusSerialisable.SerialisableBase ):
with self._lock:
login_script = self._GetLoginScript( network_context )
if login_script is None:
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
return False
login_network_context = self._GetLoginNetworkContext( network_context )
if login_network_context is None:
return False
( login_script, credentials ) = self._domains_to_login_scripts_and_credentials[ login_network_context.context_data ]
return login_script.IsLoggedIn( self.engine, login_network_context )
elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
return not self._hydrus_login_script.IsLoggedIn( self.engine, network_context )
session = self.engine.session_manager.GetSession( network_context )
return not login_script.IsLoggedIn( network_context, session )
@ -435,6 +454,11 @@ class LoginProcess( object ):
self._done = False
def _Start( self ):
raise NotImplementedError()
def IsDone( self ):
return self._done
@ -444,7 +468,7 @@ class LoginProcess( object ):
try:
self.login_script.Start( self.engine, self.network_context )
self._Start()
finally:
@ -452,23 +476,43 @@ class LoginProcess( object ):
class LoginProcessDomain( LoginProcess ):
def __init__( self, engine, network_context, login_script, credentials ):
LoginProcess.__init__( self, engine, network_context, login_script )
self.credentials = credentials
def _Start( self ):
self.login_script.Start( self.engine, self.network_context, self.credentials )
class LoginProcessHydrus( LoginProcess ):
def _Start( self ):
self.login_script.Start( self.engine, self.network_context )
class LoginScriptHydrus( object ):
def _IsLoggedIn( self, network_context, session ):
def _IsLoggedIn( self, session ):
cookies = session.cookies
cookies.clear_expired_cookies()
# I would normally do cookies_dict = cookies.get_dict( domain ) and then inspect that sub-dict, but domain for hydrus is trickier
# the session is cleared on credentials change, so this is no big deal anyway
return 'session_key' in cookies
def IsLoggedIn( self, network_context, session ):
def IsLoggedIn( self, engine, network_context ):
return self._IsLoggedIn( network_context, session )
session = engine.session_manager.GetSession( network_context )
return self._IsLoggedIn( session )
def Start( self, engine, network_context ):
@ -497,7 +541,7 @@ class LoginScriptHydrus( object ):
session = engine.session_manager.GetSession( network_context )
if self._IsLoggedIn( network_context, session ):
if self._IsLoggedIn( session ):
HydrusData.Print( 'Successfully logged into ' + service.GetName() + '.' )
@ -517,20 +561,85 @@ class LoginScriptHydrus( object ):
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER ] = NetworkLoginManager
# make this serialisable
class LoginScript( object ):
class LoginScriptDomain( object ):
def __init__( self ):
# cookie stuff to say 'this is a logged in session'
self._name = 'gelbooru v2.0 login script'
self._login_steps = []
self._validity = VALIDITY_UNTESTED
self._error_reason = ''
self._expected_cookies_for_login = [] # [ name, stringmatch ]
def _IsLoggedIn( self, network_context, session ):
# check session against required cookies
# this is more complicated for sadpanda, right?
# I may need some way to have an override of some kind that is like 'this login script specifically logs in to one domain, although it applies to others'
# need to research sadpanda exact mechanism--is it IP based?
# this should also return ( result, reason ) for testing and other purposes
cookies = session.cookies
cookies.clear_expired_cookies()
search_domain = network_context.context_data
for ( name, string_match ) in self._expected_cookies_for_login:
try:
cookie_text = ClientNetworkingDomain.GetCookie( cookies, search_domain, name )
except HydrusExceptions.DataMissing as e:
return False
( result, reason ) = string_match.Test( cookie_text )
if not result:
return False
return True
def CanLogin( self, credentials ):
if self._validity == VALIDITY_INVALID:
return ( False, 'Script is not valid: ' + self._error_reason )
for step in self._login_steps:
try:
step.TestCredentials( credentials )
except HydrusExceptions.ValidationException as e:
return ( False, str( e ) )
return True
def GetExpectedCredentialDestinations( self, domain ):
# for step in steps, say where each named credential is going
# return a dict like:
# login.pixiv.net : username, password
# evilsite.bg.cx : username, password
# This'll be presented on the cred entering form so it can't be missed
pass
@ -541,18 +650,22 @@ class LoginScript( object ):
for step in self._login_steps:
required_creds.extend( step.GetRequiredCredentials() ) # user facing [ ( name, string match ) ] with an order
required_creds.extend( step.GetRequiredCredentials() ) # [ ( credential_type, name, arg_name, string_match ) ] with an order
return required_creds
def IsLoggedIn( self, network_context, session ):
def IsLoggedIn( self, engine, network_context ):
session = engine.session_manager.GetSession( network_context )
return self._IsLoggedIn( network_context, session )
def Start( self, engine, credentials ):
def Start( self, engine, domain, network_context, credentials ):
# don't mess with the domain--assume that we are given precisely the right domain
# this maybe takes some job_key or something so it can present to the user login process status
# this will be needed in the dialog where we test this. we need good feedback on how it is going
@ -566,15 +679,13 @@ class LoginScript( object ):
step.Start( engine, credentials, temp_variables )
except HydrusExceptions.VetoException: # or something--invalidscript exception?
except HydrusExceptions.ValidationException as e:
# set error info
self._error_reason = str( e )
self._validity = VALIDITY_INVALID
# inform login manager that I'm dirty and need to be saved
return False
engine.login_manager.SetDirty()
except Exception as e:
@ -602,33 +713,108 @@ class LoginStep( object ):
def __init__( self ):
self._name = 'hit home page to establish session'
self._method = None # get/post
self._url = 'blah' # maybe this should be split up more?
self._domain_string_converter = None
self._query = 'login.php'
self._statics = [] # type | arg name | string
self._statics = [] # arg name | string
self._credentials = [] # type | user-facing name (unique) | arg name | string match
self._required_credentials = [] # type | user-facing name (unique) | arg name | string match
self._temps = [] # type | arg name
self._required_temps = [] # arg name
self._expected_cookies = [] # name | string match
self._veto_scripts = [] # list of scripts that can veto
self._temp_variable_scripts = [] # name | script that produces a single bit of text or vetoes
self._content_parsing_nodes = []
def Start( self, engine, credentials, temp_variables ):
def _TestCredentials( self, credentials ):
for ( credential_type, pretty_name, arg_name, string_match ) in self._required_credentials:
if arg_name not in credentials:
raise HydrusExceptions.ValidationException( 'The credential \'' + pretty_name + '\' was missing!' )
arg_value = credentials.GetCredential( arg_name )
( result, reason ) = string_match.Test( arg_name )
if not result:
raise HydrusExceptions.ValidationException( 'The credential \'' + pretty_name + '\' did not match requirements:' + os.linesep + reason )
def GetRequiredCredentials( self ):
return list( self._required_credentials )
def Start( self, engine, domain, credentials, temp_variables ):
# e.g. converting 'website.com' to 'login.website.com'
url_base = self._domain_string_converter.Convert( domain )
arguments = {}
arguments.update( self._statics )
self._TestCredentials( credentials )
for ( credential_type, pretty_name, arg_name, string_match ) in self._required_credentials:
arguments[ arg_name ] = credentials.GetCredential( arg_name )
for name in self._required_temps:
if name not in temp_variables:
raise HydrusExceptions.ValidationException( 'The temporary variable \'' + name + '\' was not found!' )
arguments[ name ] = temp_variables[ name ]
if self._method == 'POST':
pass # make it into body
elif self._method == 'GET':
pass # make it into query
# construct the url, failing if creds or temps missing
# hit the url, failing on connection fault or whatever
# throw the response at veto parsing gubbins, failing appropriately
# throw the response at variable parsing gubbins, failing appropriately
pass
for parsing_node in self._content_parsing_nodes:
try:
parsing_node.Vetoes()
except HydrusExceptions.VetoException as e:
raise HydrusExceptions.ValidationException( str( e ) )
# if content type is a temp variable:
# get it and add to temp_variables
pass
def TestCredentials( self, credentials ):
self._TestCredentials( credentials )

3
include/HydrusConstants.py
View File

@ -49,7 +49,7 @@ options = {}
# Misc
NETWORK_VERSION = 18
SOFTWARE_VERSION = 279
SOFTWARE_VERSION = 280
UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )
@ -106,6 +106,7 @@ CONTENT_TYPE_OPTIONS = 10
CONTENT_TYPE_SERVICES = 11
CONTENT_TYPE_UNKNOWN = 12
CONTENT_TYPE_ACCOUNT_TYPES = 13
CONTENT_TYPE_VARIABLE = 14
content_type_string_lookup = {}

7
include/HydrusDB.py
View File

@ -18,7 +18,6 @@ import traceback
import time
CONNECTION_REFRESH_TIME = 60 * 30
TRANSACTION_COMMIT_TIME = 10
def CanVacuum( db_path, stop_time = None ):
@ -109,6 +108,8 @@ class HydrusDB( object ):
READ_WRITE_ACTIONS = []
UPDATE_WAIT = 2
TRANSACTION_COMMIT_TIME = 10
def __init__( self, controller, db_dir, db_name, no_wal = False ):
self._controller = controller
@ -526,7 +527,7 @@ class HydrusDB( object ):
result = self._Write( action, *args, **kwargs )
if self._transaction_contains_writes and HydrusData.TimeHasPassed( self._transaction_started + TRANSACTION_COMMIT_TIME ):
if self._transaction_contains_writes and HydrusData.TimeHasPassed( self._transaction_started + self.TRANSACTION_COMMIT_TIME ):
self._current_status = 'db committing'
@ -805,7 +806,7 @@ class HydrusDB( object ):
except Queue.Empty:
if self._transaction_contains_writes and HydrusData.TimeHasPassed( self._transaction_started + TRANSACTION_COMMIT_TIME ):
if self._transaction_contains_writes and HydrusData.TimeHasPassed( self._transaction_started + self.TRANSACTION_COMMIT_TIME ):
self._Commit()

64
include/HydrusImageHandling.py
View File

@ -120,6 +120,70 @@ def GeneratePILImage( path ):
raise
if pil_image.format == 'JPEG' and hasattr( pil_image, '_getexif' ):
exif_dict = pil_image._getexif()
if exif_dict is not None:
EXIF_ORIENTATION = 274
if EXIF_ORIENTATION in exif_dict:
orientation = exif_dict[ EXIF_ORIENTATION ]
if orientation == 1:
pass # normal
elif orientation == 2:
# mirrored horizontal
pil_image = pil_image.transpose( PILImage.FLIP_LEFT_RIGHT )
elif orientation == 3:
# 180
pil_image = pil_image.transpose( PILImage.ROTATE_180 )
elif orientation == 4:
# mirrored vertical
pil_image = pil_image.transpose( PILImage.FLIP_TOP_BOTTOM )
elif orientation == 5:
# seems like these 90 degree rotations are wrong, but fliping them works for my posh example images, so I guess the PIL constants are odd
# mirrored horizontal, then 90 CCW
pil_image = pil_image.transpose( PILImage.FLIP_LEFT_RIGHT ).transpose( PILImage.ROTATE_90 )
elif orientation == 6:
# 90 CW
pil_image = pil_image.transpose( PILImage.ROTATE_270 )
elif orientation == 7:
# mirrored horizontal, then 90 CCW
pil_image = pil_image.transpose( PILImage.FLIP_LEFT_RIGHT ).transpose( PILImage.ROTATE_270 )
elif orientation == 8:
# 90 CCW
pil_image = pil_image.transpose( PILImage.ROTATE_90 )
if pil_image is None:
raise Exception( 'The file at ' + path + ' could not be rendered!' )

24
include/HydrusVideoHandling.py
View File

@ -560,11 +560,33 @@ def ParseFFMPEGVideoResolution( lines ):
resolution = list(map(int, line[match.start():match.end()-1].split('x')))
sar_match = re.search( " SAR [0-9]*:[0-9]* ", line )
if sar_match is not None:
# ' SAR 2:3 '
sar_string = line[ sar_match.start() : sar_match.end() ]
# '2:3'
sar_string = sar_string[5:-1]
( sar_w, sar_h ) = sar_string.split( ':' )
( sar_w, sar_h ) = ( int( sar_w ), int( sar_h ) )
( x, y ) = resolution
x *= sar_w
x //= sar_h
resolution = ( x, y )
return resolution
except:
raise HydrusExceptions.MimeException( 'Error counting number of frames!' )
raise HydrusExceptions.MimeException( 'Error parsing resolution!' )
# This was built from moviepy's FFMPEG_VideoReader

2
include/ServerDB.py
View File

@ -89,6 +89,8 @@ class DB( HydrusDB.HydrusDB ):
READ_WRITE_ACTIONS = [ 'access_key', 'immediate_content_update', 'registration_keys' ]
TRANSACTION_COMMIT_TIME = 120
def __init__( self, controller, db_dir, db_name, no_wal = False ):
self._files_dir = os.path.join( db_dir, 'server_files' )