changelog
-
+
version 228
+ - added support for RIFF .avi files +
- fleshed out parsing ui, made progress on node editing, moved edit/test panels to notebook pages to reduce clutter +
- the video renderer now initialises off the gui thread, which should reduce some video browsing chunkiness +
- trying to save a session that would overwrite another (from the same name) now throws up a yes/no warning +
- reintroduced '--no_daemons' and '--no_wal' to the new command line argument parser +
- you can now set the default tag service for new search pages under the file->options->tags page +
- since it can now be anywhere, added 'db directory' to the file->open menu +
- fixed a build capitalisation issue that was making the windows frozen exe release crash when trying to run from source +
- the 'too many redirects' error message is expanded to print the url called and its redirect request +
- errors from trying to delete files in use by another process will no longer make popups--they'll just write a note in the log +
- improved how some errors are written to the log +
- misc fixes +
version 227
- both the client and server now use standard command line input and will produce a proper help with a -h switch (although the 'windowed' frozen client executable can't print back to console) diff --git a/include/ClientCaches.py b/include/ClientCaches.py index 6f43d7dc..9abf17cb 100644 --- a/include/ClientCaches.py +++ b/include/ClientCaches.py @@ -2008,6 +2008,14 @@ class ServicesManager( object ): + def ServiceExists( self, service_key ): + + with self._lock: + + return service_key in self._keys_to_services + + + class TagCensorshipManager( object ): def __init__( self, controller ): diff --git a/include/ClientConstants.py b/include/ClientConstants.py index 31caaa6c..40ffd883 100755 --- a/include/ClientConstants.py +++ b/include/ClientConstants.py @@ -203,6 +203,7 @@ else: media_viewer_capabilities[ HC.APPLICATION_PDF ] = no_support +media_viewer_capabilities[ HC.VIDEO_AVI ] = animated_full_support media_viewer_capabilities[ HC.VIDEO_FLV ] = animated_full_support media_viewer_capabilities[ HC.VIDEO_MOV ] = animated_full_support media_viewer_capabilities[ HC.VIDEO_MP4 ] = animated_full_support diff --git a/include/ClientController.py b/include/ClientController.py index a12b25ba..095e9079 100755 --- a/include/ClientController.py +++ b/include/ClientController.py @@ -37,11 +37,11 @@ class Controller( HydrusController.HydrusController ): pubsub_binding_errors_to_ignore = [ wx.PyDeadObjectError ] - def __init__( self, db_dir ): + def __init__( self, db_dir, no_daemons, no_wal ): self._last_shutdown_was_bad = False - HydrusController.HydrusController.__init__( self, db_dir ) + HydrusController.HydrusController.__init__( self, db_dir, no_daemons, no_wal ) HydrusGlobals.client_controller = self @@ -627,7 +627,7 @@ class Controller( HydrusController.HydrusController ): message += os.linesep * 2 message += 'Don\'t forget to check out the help if you haven\'t already.' message += os.linesep * 2 - message += 'You can right-click popup messages like this to dismiss them.' + message += 'To dismiss popup messages like this, right-click them.' HydrusData.ShowText( message ) @@ -1107,7 +1107,7 @@ class Controller( HydrusController.HydrusController ): HydrusData.DebugPrint( text ) - traceback.print_exc() + HydrusData.DebugPrint( traceback.format_exc() ) wx.CallAfter( wx.MessageBox, traceback.format_exc() ) wx.CallAfter( wx.MessageBox, text ) @@ -1144,7 +1144,7 @@ class Controller( HydrusController.HydrusController ): HydrusData.DebugPrint( text ) - traceback.print_exc() + HydrusData.DebugPrint( traceback.format_exc() ) wx.CallAfter( wx.MessageBox, traceback.format_exc() ) wx.CallAfter( wx.MessageBox, text ) diff --git a/include/ClientDB.py b/include/ClientDB.py index becdad14..e5cf833b 100755 --- a/include/ClientDB.py +++ b/include/ClientDB.py @@ -1254,7 +1254,7 @@ class DB( HydrusDB.HydrusDB ): def _Analyze( self, stop_time = None, only_when_idle = False, force_reanalyze = False ): - stale_time_delta = 14 * 86400 + stale_time_delta = 30 * 86400 existing_names_to_timestamps = dict( self._c.execute( 'SELECT name, timestamp FROM analyze_timestamps;' ).fetchall() ) @@ -5462,7 +5462,7 @@ class DB( HydrusDB.HydrusDB ): # analyze - stale_time_delta = 14 * 86400 + stale_time_delta = 30 * 86400 existing_names_to_timestamps = dict( self._c.execute( 'SELECT name, timestamp FROM analyze_timestamps;' ).fetchall() ) @@ -6968,7 +6968,7 @@ class DB( HydrusDB.HydrusDB ): except: - traceback.print_exc() + HydrusData.DebugPrint( traceback.format_exc() ) self._controller.pub( 'splash_set_status_text', 'error updating subscription ' + name ) @@ -7023,7 +7023,7 @@ class DB( HydrusDB.HydrusDB ): except: - traceback.print_exc() + HydrusData.DebugPrint( traceback.format_exc() ) continue @@ -7085,7 +7085,7 @@ class DB( HydrusDB.HydrusDB ): except: - traceback.print_exc() + HydrusData.DebugPrint( traceback.format_exc() ) continue diff --git a/include/ClientData.py b/include/ClientData.py index 737a7d16..32fbcd12 100644 --- a/include/ClientData.py +++ b/include/ClientData.py @@ -554,6 +554,12 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ): # + self._dictionary[ 'keys' ] = {} + + self._dictionary[ 'keys' ][ 'default_tag_service_search_page' ] = CC.COMBINED_TAG_SERVICE_KEY.encode( 'hex' ) + + # + self._dictionary[ 'noneable_integers' ] = {} self._dictionary[ 'noneable_integers' ][ 'forced_search_limit' ] = None @@ -620,6 +626,7 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ): self._dictionary[ 'media_view' ][ HC.APPLICATION_PDF ] = ( CC.MEDIA_VIEWER_ACTION_SHOW_OPEN_EXTERNALLY_BUTTON, CC.MEDIA_VIEWER_ACTION_SHOW_OPEN_EXTERNALLY_BUTTON, null_zoom_info ) + self._dictionary[ 'media_view' ][ HC.VIDEO_AVI ] = ( CC.MEDIA_VIEWER_ACTION_SHOW_AS_NORMAL, CC.MEDIA_VIEWER_ACTION_SHOW_AS_NORMAL, video_zoom_info ) self._dictionary[ 'media_view' ][ HC.VIDEO_FLV ] = ( CC.MEDIA_VIEWER_ACTION_SHOW_AS_NORMAL, CC.MEDIA_VIEWER_ACTION_SHOW_AS_NORMAL, video_zoom_info ) self._dictionary[ 'media_view' ][ HC.VIDEO_MOV ] = ( CC.MEDIA_VIEWER_ACTION_SHOW_AS_NORMAL, CC.MEDIA_VIEWER_ACTION_SHOW_AS_NORMAL, video_zoom_info ) self._dictionary[ 'media_view' ][ HC.VIDEO_MP4 ] = ( CC.MEDIA_VIEWER_ACTION_SHOW_AS_NORMAL, CC.MEDIA_VIEWER_ACTION_SHOW_AS_NORMAL, video_zoom_info ) @@ -819,12 +826,18 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ): def GetFrameLocation( self, frame_key ): - return self._dictionary[ 'frame_locations' ][ frame_key ] + with self._lock: + + return self._dictionary[ 'frame_locations' ][ frame_key ] + def GetFrameLocations( self ): - return self._dictionary[ 'frame_locations' ].items() + with self._lock: + + return self._dictionary[ 'frame_locations' ].items() + def GetInteger( self, name ): @@ -835,6 +848,14 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ): + def GetKey( self, name ): + + with self._lock: + + return self._dictionary[ 'keys' ][ name ].decode( 'hex' ) + + + def GetMediaShowAction( self, mime ): with self._lock: @@ -954,7 +975,10 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ): def SetFrameLocation( self, frame_key, remember_size, remember_position, last_size, last_position, default_gravity, default_position, maximised, fullscreen ): - self._dictionary[ 'frame_locations' ][ frame_key ] = ( remember_size, remember_position, last_size, last_position, default_gravity, default_position, maximised, fullscreen ) + with self._lock: + + self._dictionary[ 'frame_locations' ][ frame_key ] = ( remember_size, remember_position, last_size, last_position, default_gravity, default_position, maximised, fullscreen ) + def SetInteger( self, name, value ): @@ -965,6 +989,14 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ): + def SetKey( self, name, value ): + + with self._lock: + + self._dictionary[ 'keys' ][ name ] = value.encode( 'hex' ) + + + def SetMediaViewOptions( self, mime, value_tuple ): with self._lock: diff --git a/include/ClientGUI.py b/include/ClientGUI.py index 84515483..fe4f8a85 100755 --- a/include/ClientGUI.py +++ b/include/ClientGUI.py @@ -10,6 +10,7 @@ import ClientGUIDialogs import ClientGUIDialogsManage import ClientGUIManagement import ClientGUIPages +import ClientGUIParsing import ClientGUIScrolledPanels import ClientGUITopLevelWindows import ClientDownloading @@ -728,6 +729,7 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ): open = wx.Menu() open.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'open_install_folder' ), p( 'Installation Directory' ), p( 'Open the installation directory for this client.' ) ) + open.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'open_db_folder' ), p( 'Database Directory' ), p( 'Open the database directory for this instance of the client.' ) ) open.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'open_export_folder' ), p( 'Quick Export Directory' ), p( 'Open the export directory so you can easily access the files you have exported.' ) ) menu.AppendMenu( CC.ID_NULL, p( 'Open' ), open ) @@ -1384,7 +1386,7 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ): with ClientGUITopLevelWindows.DialogManage( self, title, frame_key ) as dlg: - panel = ClientGUIScrolledPanels.ManageParsingScriptsPanel( dlg ) + panel = ClientGUIParsing.ManageParsingScriptsPanel( dlg ) dlg.SetPanel( panel ) @@ -1557,7 +1559,16 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ): search_enabled = len( initial_media_results ) == 0 - file_search_context = ClientSearch.FileSearchContext( file_service_key = file_service_key, predicates = initial_predicates ) + new_options = self._controller.GetNewOptions() + + tag_service_key = new_options.GetKey( 'default_tag_service_search_page' ) + + if not self._controller.GetServicesManager().ServiceExists( tag_service_key ): + + tag_service_key = CC.COMBINED_TAG_SERVICE_KEY + + + file_search_context = ClientSearch.FileSearchContext( file_service_key = file_service_key, tag_service_key = tag_service_key, predicates = initial_predicates ) management_controller = ClientGUIManagement.CreateManagementControllerQuery( file_service_key, file_search_context, search_enabled ) @@ -1569,6 +1580,11 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ): with ClientGUIDialogs.DialogNews( self, service_key ) as dlg: dlg.ShowModal() + def _OpenDBFolder( self ): + + HydrusPaths.LaunchDirectory( self._controller.GetDBDir() ) + + def _OpenExportFolder( self ): export_path = ClientFiles.GetExportPath() @@ -1792,6 +1808,21 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ): else: + existing_session_names = self._controller.Read( 'serialisable_names', HydrusSerialisable.SERIALISABLE_TYPE_GUI_SESSION ) + + if name in existing_session_names: + + message = 'Session \'' + name + '\' already exists! Do you want to overwrite it?' + + with ClientGUIDialogs.DialogYesNo( self , message, title = 'Overwrite existing session?', yes_label = 'yes, overwrite', no_label = 'no, choose another name' ) as yn_dlg: + + if yn_dlg.ShowModal() != wx.ID_YES: + + continue + + + + break @@ -2465,6 +2496,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p HydrusGlobals.no_focus_changed = not HydrusGlobals.no_focus_changed + elif command == 'open_db_folder': self._OpenDBFolder() elif command == 'open_export_folder': self._OpenExportFolder() elif command == 'open_install_folder': self._OpenInstallFolder() elif command == 'options': self._ManageOptions() diff --git a/include/ClientGUICanvas.py b/include/ClientGUICanvas.py index 9d8fa8ca..0dd531bb 100755 --- a/include/ClientGUICanvas.py +++ b/include/ClientGUICanvas.py @@ -479,6 +479,11 @@ class Animation( wx.Window ): def GotoFrame( self, frame_index ): + if not self._video_container.IsInitialised(): + + return + + if frame_index != self._current_frame_index: self._current_frame_index = frame_index diff --git a/include/ClientGUICommon.py b/include/ClientGUICommon.py index 7fd548ee..3879e87b 100755 --- a/include/ClientGUICommon.py +++ b/include/ClientGUICommon.py @@ -184,6 +184,45 @@ class AnimatedStaticTextTimestamp( wx.StaticText ): +class BetterButton( wx.Button ): + + def __init__( self, parent, label, callable ): + + wx.Button.__init__( self, parent, label = label ) + + self.Bind( wx.EVT_BUTTON, self.EventButton ) + + + def EventButton( self, event ): + + callable() + + +class BetterChoice( wx.Choice ): + + def GetChoice( self ): + + selection = self.GetSelection() + + if selection != wx.NOT_FOUND: return self.GetClientData( selection ) + else: return self.GetClientData( 0 ) + + + def SelectClientData( self, client_data ): + + for i in range( self.GetCount() ): + + if client_data == self.GetClientData( i ): + + self.Select( i ) + + return + + + + self.Select( 0 ) + + class BufferedWindow( wx.Window ): def __init__( self, *args, **kwargs ): @@ -263,31 +302,6 @@ class BufferedWindowIcon( BufferedWindow ): self._dirty = False -class BetterChoice( wx.Choice ): - - def GetChoice( self ): - - selection = self.GetSelection() - - if selection != wx.NOT_FOUND: return self.GetClientData( selection ) - else: return self.GetClientData( 0 ) - - - def SelectClientData( self, client_data ): - - for i in range( self.GetCount() ): - - if client_data == self.GetClientData( i ): - - self.Select( i ) - - return - - - - self.Select( 0 ) - - class CheckboxCollect( wx.combo.ComboCtrl ): def __init__( self, parent, page_key = None ): diff --git a/include/ClientGUIManagement.py b/include/ClientGUIManagement.py index abeca5a1..48287356 100755 --- a/include/ClientGUIManagement.py +++ b/include/ClientGUIManagement.py @@ -139,8 +139,6 @@ def CreateManagementControllerQuery( file_service_key, file_search_context, sear management_controller = CreateManagementController( MANAGEMENT_TYPE_QUERY, file_service_key = file_service_key ) - management_controller.SetKey( 'tag_service', CC.COMBINED_TAG_SERVICE_KEY ) - management_controller.SetVariable( 'file_search_context', file_search_context ) management_controller.SetVariable( 'search_enabled', search_enabled ) management_controller.SetVariable( 'synchronised', True ) diff --git a/include/ClientGUIParsing.py b/include/ClientGUIParsing.py new file mode 100644 index 00000000..c8fbeb2c --- /dev/null +++ b/include/ClientGUIParsing.py @@ -0,0 +1,1262 @@ +import ClientConstants as CC +import ClientGUICommon +import ClientGUIDialogs +import ClientGUIScrolledPanels +import ClientGUITopLevelWindows +import HydrusConstants as HC +import HydrusData +import HydrusGlobals +import HydrusParsing +import HydrusSerialisable +import os +import wx + +class EditHTMLFormulaPanel( ClientGUIScrolledPanels.EditPanel ): + + def __init__( self, parent, info ): + + ClientGUIScrolledPanels.EditPanel.__init__( self, parent ) + + self._original_info = info + + self._do_testing_automatically = False + + formula_panel = ClientGUICommon.StaticBox( self, 'formula' ) + + self._tag_rules = wx.ListBox( formula_panel, style = wx.LB_SINGLE ) + self._tag_rules.Bind( wx.EVT_LEFT_DCLICK, self.EventEdit ) + + self._add_rule = wx.Button( formula_panel, label = 'add' ) + self._add_rule.Bind( wx.EVT_BUTTON, self.EventAdd ) + + self._edit_rule = wx.Button( formula_panel, label = 'edit' ) + self._edit_rule.Bind( wx.EVT_BUTTON, self.EventEdit ) + + self._move_rule_up = wx.Button( formula_panel, label = u'\u2191' ) + self._move_rule_up.Bind( wx.EVT_BUTTON, self.EventMoveUp ) + + self._delete_rule = wx.Button( formula_panel, label = 'X' ) + self._delete_rule.Bind( wx.EVT_BUTTON, self.EventDelete ) + + self._move_rule_down = wx.Button( formula_panel, label = u'\u2193' ) + self._move_rule_down.Bind( wx.EVT_BUTTON, self.EventMoveDown ) + + self._content_rule = wx.TextCtrl( formula_panel ) + + testing_panel = ClientGUICommon.StaticBox( self, 'testing' ) + + self._test_html = wx.TextCtrl( testing_panel, style = wx.TE_MULTILINE ) + + self._test_html.SetMinSize( ( -1, 200 ) ) + + self._fetch_from_url = wx.Button( testing_panel, label = 'fetch result from url' ) + self._fetch_from_url.Bind( wx.EVT_BUTTON, self.EventFetchFromURL ) + + self._run_test = wx.Button( testing_panel, label = 'run test' ) + self._run_test.Bind( wx.EVT_BUTTON, self.EventRunTest ) + + self._results = wx.TextCtrl( testing_panel, style = wx.TE_MULTILINE ) + + self._results.SetMinSize( ( -1, 200 ) ) + + # + + ( tag_rules, content_rule ) = self._original_info.ToTuple() + + for rule in tag_rules: + + pretty_rule = HydrusParsing.RenderTagRule( rule ) + + self._tag_rules.Append( pretty_rule, rule ) + + + self._content_rule.SetValue( content_rule ) + + self._test_html.SetValue( 'Enter html here to test it against the above formula.' ) + self._results.SetValue( 'Successfully parsed results will be printed here.' ) + + # + + udd_button_vbox = wx.BoxSizer( wx.VERTICAL ) + + udd_button_vbox.AddF( self._move_rule_up, CC.FLAGS_VCENTER ) + udd_button_vbox.AddF( self._delete_rule, CC.FLAGS_VCENTER ) + udd_button_vbox.AddF( self._move_rule_down, CC.FLAGS_VCENTER ) + + tag_rules_hbox = wx.BoxSizer( wx.HORIZONTAL ) + + tag_rules_hbox.AddF( self._tag_rules, CC.FLAGS_EXPAND_BOTH_WAYS ) + tag_rules_hbox.AddF( udd_button_vbox, CC.FLAGS_VCENTER ) + + ae_button_hbox = wx.BoxSizer( wx.HORIZONTAL ) + + ae_button_hbox.AddF( self._add_rule, CC.FLAGS_VCENTER ) + ae_button_hbox.AddF( self._edit_rule, CC.FLAGS_VCENTER ) + + formula_panel.AddF( tag_rules_hbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) + formula_panel.AddF( ae_button_hbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) + formula_panel.AddF( ClientGUICommon.WrapInText( self._content_rule, formula_panel, 'attribute: ' ), CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) + + testing_panel.AddF( self._test_html, CC.FLAGS_EXPAND_PERPENDICULAR ) + testing_panel.AddF( self._fetch_from_url, CC.FLAGS_EXPAND_PERPENDICULAR ) + testing_panel.AddF( self._run_test, CC.FLAGS_EXPAND_PERPENDICULAR ) + testing_panel.AddF( self._results, CC.FLAGS_EXPAND_PERPENDICULAR ) + + vbox = wx.BoxSizer( wx.VERTICAL ) + + message = 'The html will be searched recursively by each rule in turn and then the attribute of the final tags will be returned.' + message += os.linesep * 2 + message += 'So, to find the \'src\' of the first tag beneath all tags with the class \'content\', use:' + message += os.linesep * 2 + message += 'all span tags with class=content' + message += '1st img tag' + message += 'attribute: src' + message += os.linesep * 2 + message += 'Leave the attribute blank to represent the string of the tag (i.e.
-
+
This part
).' + + vbox.AddF( wx.StaticText( self, label = message ), CC.FLAGS_EXPAND_PERPENDICULAR ) + vbox.AddF( formula_panel, CC.FLAGS_EXPAND_PERPENDICULAR ) + vbox.AddF( testing_panel, CC.FLAGS_EXPAND_PERPENDICULAR ) + + self.SetSizer( vbox ) + + + def _RunTest( self ): + + formula = self.GetValue() + + html = self._test_html.GetValue() + + try: + + results = formula.Parse( html ) + + # do the begin/end to better display '' results and any other whitespace weirdness + results = [ '*** RESULTS BEGIN ***' ] + results + [ '*** RESULTS END ***' ] + + results_text = os.linesep.join( results ) + + self._results.SetValue( results_text ) + + self._do_testing_automatically = True + + except Exception as e: + + message = 'Could not parse! Full error written to log!' + message += os.linesep * 2 + message += HydrusData.ToUnicode( e ) + + wx.MessageBox( message ) + + self._do_testing_automatically = False + + + + def EventAdd( self, event ): + + # spawn dialog, add it and run test + + if self._do_testing_automatically: + + self._RunTest() + + + + def EventDelete( self, event ): + + selection = self._tag_rules.GetSelection() + + if selection != wx.NOT_FOUND: + + if self._tag_rules.GetCount() == 1: + + wx.MessageBox( 'A parsing formula needs at least one tag rule!' ) + + else: + + self._tag_rules.Delete( selection ) + + if self._do_testing_automatically: + + self._RunTest() + + + + + + def EventEdit( self, event ): + + selection = self._tag_rules.GetSelection() + + if selection != wx.NOT_FOUND: + + ( name, attrs, index ) = self._tag_rules.GetClientData( selection ) + + # spawn dialog, then if ok, set it and run test + + if self._do_testing_automatically: + + self._RunTest() + + + + def EventFetchFromURL( self, event ): + + # ask user for url with textdlg + # get it with requests + # handle errors with a messagebox + # try to parse it with bs4 to check it is good html and then splat it to the textctrl, otherwise just messagebox the error + + if self._do_testing_automatically: + + self._RunTest() + + + + def EventMoveDown( self, event ): + + selection = self._tag_rules.GetSelection() + + if selection != wx.NOT_FOUND and selection + 1 < self._tag_rules.GetCount(): + + pretty_rule = self._tag_rules.GetString( selection ) + rule = self._tag_rules.GetClientData( selection ) + + self._tag_rules.Delete( selection ) + + self._tag_rules.Insert( selection + 1, pretty_rule, rule ) + + if self._do_testing_automatically: + + self._RunTest() + + + + + def EventMoveUp( self, event ): + + selection = self._tag_rules.GetSelection() + + if selection != wx.NOT_FOUND and selection > 0: + + pretty_rule = self._tag_rules.GetString( selection ) + rule = self._tag_rules.GetClientData( selection ) + + self._tag_rules.Delete( selection ) + + self._tag_rules.Insert( selection - 1, pretty_rule, rule ) + + if self._do_testing_automatically: + + self._RunTest() + + + + + def EventRunTest( self, event ): + + self._RunTest() + + + def GetValue( self ): + + tags_rules = [ self._tag_rules.GetClientData( i ) for i in range( self._tag_rules.GetCount() ) ] + content_rule = self._content_rule.GetValue() + + if content_rule == '': + + content_rule = None + + + formula = HydrusParsing.ParseFormulaHTML( tags_rules, content_rule ) + + return formula + + +class EditNodes( wx.Panel ): + + def __init__( self, parent, nodes, example_data_callable ): + + wx.Panel.__init__( self, parent ) + + self._example_data_callable = example_data_callable + + self._nodes = ClientGUICommon.SaneListCtrl( self, 200, [ ( 'name', -1 ), ( 'node type', 80 ), ( 'produces', 240 ) ], delete_key_callback = self.Delete, activation_callback = self.Edit, use_display_tuple_for_sort = True ) + + self._add_button = wx.Button( self, label = 'add' ) + self._add_button.Bind( wx.EVT_BUTTON, self.EventAdd ) + + self._copy_button = wx.Button( self, label = 'copy' ) + self._copy_button.Bind( wx.EVT_BUTTON, self.EventCopy ) + + self._paste_button = wx.Button( self, label = 'paste' ) + self._paste_button.Bind( wx.EVT_BUTTON, self.EventPaste ) + + self._duplicate_button = wx.Button( self, label = 'duplicate' ) + self._duplicate_button.Bind( wx.EVT_BUTTON, self.EventDuplicate ) + + self._edit_button = wx.Button( self, label = 'edit' ) + self._edit_button.Bind( wx.EVT_BUTTON, self.EventEdit ) + + self._delete_button = wx.Button( self, label = 'delete' ) + self._delete_button.Bind( wx.EVT_BUTTON, self.EventDelete ) + + # + + for node in nodes: + + ( display_tuple, data_tuple ) = self._ConvertNodeToTuples( node ) + + self._nodes.Append( display_tuple, data_tuple ) + + + # + + vbox = wx.BoxSizer( wx.VERTICAL ) + + button_hbox = wx.BoxSizer( wx.HORIZONTAL ) + + button_hbox.AddF( self._add_button, CC.FLAGS_VCENTER ) + button_hbox.AddF( self._copy_button, CC.FLAGS_VCENTER ) + button_hbox.AddF( self._paste_button, CC.FLAGS_VCENTER ) + button_hbox.AddF( self._duplicate_button, CC.FLAGS_VCENTER ) + button_hbox.AddF( self._edit_button, CC.FLAGS_VCENTER ) + button_hbox.AddF( self._delete_button, CC.FLAGS_VCENTER ) + + vbox.AddF( self._nodes, CC.FLAGS_EXPAND_BOTH_WAYS ) + vbox.AddF( button_hbox, CC.FLAGS_BUTTON_SIZER ) + + self.SetSizer( vbox ) + + + def _ConvertNodeToTuples( self, node ): + + ( name, node_type, produces ) = node.ToPrettyStrings() + + return ( ( name, node_type, produces ), ( node, node_type, produces ) ) + + + def Add( self ): + + with ClientGUIDialogs.DialogSelectFromListOfStrings( self, 'select the node type', [ 'content', 'link' ] ) as dlg_type: + + if dlg_type.ShowModal() == wx.ID_OK: + + node_type_string = dlg_type.GetString() + + if node_type_string == 'content': + + empty_node = HydrusParsing.ParseNodeContent() + + panel_class = EditParseNodeContentPanel + + elif node_type_string == 'link': + + empty_node = HydrusParsing.ParseNodeContentLink() + + panel_class = EditParseNodeContentLinkPanel + + + with ClientGUITopLevelWindows.DialogEdit( self, 'edit node' ) as dlg_edit: + + example_data = self._example_data_callable() + + panel = panel_class( dlg_edit, empty_node, example_data ) + + dlg_edit.SetPanel( panel ) + + if dlg_edit.ShowModal() == wx.ID_OK: + + new_node = panel.GetValue() + + ( display_tuple, data_tuple ) = self._ConvertNodeToTuples( new_node ) + + self._nodes.Append( display_tuple, data_tuple ) + + + + + + + def Copy( self ): + + for i in self._nodes.GetAllSelected(): + + ( node, node_type, produces ) = self._nodes.GetClientData( i ) + + node_json = node.DumpToString() + + HydrusGlobals.client_controller.pub( 'clipboard', 'text', node_json ) + + + + def Delete( self ): + + self._nodes.RemoveAllSelected() + + + def Duplicate( self ): + + nodes_to_dupe = [] + + for i in self._nodes.GetAllSelected(): + + ( node, node_type, produces ) = self._nodes.GetClientData( i ) + + nodes_to_dupe.append( node ) + + + for node in nodes_to_dupe: + + dupe_node = node.Duplicate() + + ( display_tuple, data_tuple ) = self._ConvertNodeToTuples( dupe_node ) + + self._nodes.Append( display_tuple, data_tuple ) + + + + def Edit( self ): + + for i in self._nodes.GetAllSelected(): + + ( node, node_type, produces ) = self._nodes.GetClientData( i ) + + with ClientGUITopLevelWindows.DialogEdit( self, 'edit node' ) as dlg: + + if isinstance( node, HydrusParsing.ParseNodeContent): + + panel_class = EditParseNodeContentPanel + + elif isinstance( node, HydrusParsing.ParseNodeContentLink ): + + panel_class = EditParseNodeContentLinkPanel + + + panel = panel_class( dlg, node ) + + dlg.SetPanel( panel ) + + if dlg.ShowModal() == wx.ID_OK: + + edited_node = panel.GetValue() + + ( display_tuple, data_tuple ) = self._ConvertNodeToTuples( edited_node ) + + self._nodes.UpdateRow( i, display_tuple, data_tuple ) + + + + + + + def GetValue( self ): + + nodes = [ node for ( node, node_type, produces ) in self._nodes.GetClientData() ] + + return nodes + + + def Paste( self ): + + if wx.TheClipboard.Open(): + + data = wx.TextDataObject() + + wx.TheClipboard.GetData( data ) + + wx.TheClipboard.Close() + + raw_text = data.GetText() + + try: + + obj = HydrusSerialisable.CreateFromString( raw_text ) + + if isinstance( obj, ( HydrusParsing.ParseNodeContent, HydrusParsing.ParseNodeContentLink ) ): + + node = obj + + ( display_tuple, data_tuple ) = self._ConvertNodeToTuples( node ) + + self._nodes.Append( display_tuple, data_tuple ) + + + except: + + wx.MessageBox( 'I could not understand what was in the clipboard' ) + + + else: + + wx.MessageBox( 'I could not get permission to access the clipboard.' ) + + + + def EventAdd( self, event ): + + self.Add() + + + def EventCopy( self, event ): + + self.Copy() + + + def EventDelete( self, event ): + + self.Delete() + + + def EventDuplicate( self, event ): + + self.Duplicate() + + + def EventEdit( self, event ): + + self.Edit() + + + def EventPaste( self, event ): + + self.Paste() + + +class EditParseNodeContentPanel( ClientGUIScrolledPanels.EditPanel ): + + def __init__( self, parent, node, example_data = None ): + + ClientGUIScrolledPanels.EditPanel.__init__( self, parent ) + + if example_data is None: + + example_data = '' + + + notebook = wx.Notebook( self ) + + # + + edit_panel = wx.Panel( notebook ) + + edit_panel.SetBackgroundColour( wx.SystemSettings.GetColour( wx.SYS_COLOUR_FRAMEBK ) ) + + self._name = wx.TextCtrl( edit_panel ) + + content_panel = ClientGUICommon.StaticBox( edit_panel, 'content type' ) + + self._content_type = ClientGUICommon.BetterChoice( content_panel ) + + self._content_type.Append( 'tags', HC.CONTENT_TYPE_MAPPINGS ) + + # bind an event here when I add new content types that will dynamically hide/show the namespace/rating stuff and relayout as needed + # it should have a forced name or something. whatever we'll use to discriminate between rating services on 'import options - ratings' + # (this probably means sending and EditPanel size changed event or whatever) + + self._namespace = wx.TextCtrl( content_panel ) + + formula_panel = ClientGUICommon.StaticBox( edit_panel, 'formula' ) + + self._formula_description = wx.TextCtrl( formula_panel, style = wx.TE_MULTILINE ) + + self._formula_description.SetMinSize( ( -1, 200 ) ) + + self._formula_description.Disable() + + self._edit_formula = wx.Button( formula_panel, label = 'edit formula' ) + self._edit_formula.Bind( wx.EVT_BUTTON, self.EventEditFormula ) + + # + + test_panel = wx.Panel( notebook ) + + test_panel.SetBackgroundColour( wx.SystemSettings.GetColour( wx.SYS_COLOUR_FRAMEBK ) ) + + self._example_data = wx.TextCtrl( test_panel, style = wx.TE_MULTILINE ) + + self._example_data.SetMinSize( ( -1, 200 ) ) + + self._test_parse = wx.Button( test_panel, label = 'test parse' ) + self._test_parse.Bind( wx.EVT_BUTTON, self.EventTestParse ) + + self._results = wx.TextCtrl( test_panel, style = wx.TE_MULTILINE ) + + self._results.SetMinSize( ( -1, 200 ) ) + + # + + ( name, content_type, self._current_formula, additional_info ) = node.ToTuple() + + self._name.SetValue( name ) + + self._content_type.SelectClientData( content_type ) + + if content_type == HC.CONTENT_TYPE_MAPPINGS: + + self._namespace.SetValue( additional_info ) + + + self._formula_description.SetValue( self._current_formula.ToPrettyMultilineString() ) + + self._example_data.SetValue( example_data ) + + # + + # hide namespace, ratings additional info, as needed + # since I do it as a gridbox right now, this needs to be rewritten + + # + + rows = [] + + rows.append( ( 'content type: ', self._content_type ) ) + rows.append( ( 'namespace: ', self._namespace ) ) + + gridbox = ClientGUICommon.WrapInGrid( content_panel, rows ) + + content_panel.AddF( gridbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) + + # + + formula_panel.AddF( self._formula_description, CC.FLAGS_EXPAND_PERPENDICULAR ) + formula_panel.AddF( self._edit_formula, CC.FLAGS_EXPAND_PERPENDICULAR ) + + # + + vbox = wx.BoxSizer( wx.VERTICAL ) + + rows = [] + + rows.append( ( 'name or description (optional): ', self._name ) ) + + gridbox = ClientGUICommon.WrapInGrid( edit_panel, rows ) + + vbox.AddF( gridbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) + vbox.AddF( content_panel, CC.FLAGS_EXPAND_PERPENDICULAR ) + vbox.AddF( formula_panel, CC.FLAGS_EXPAND_PERPENDICULAR ) + + edit_panel.SetSizer( vbox ) + + # + + vbox = wx.BoxSizer( wx.VERTICAL ) + + vbox.AddF( self._example_data, CC.FLAGS_EXPAND_PERPENDICULAR ) + vbox.AddF( self._test_parse, CC.FLAGS_EXPAND_PERPENDICULAR ) + vbox.AddF( self._results, CC.FLAGS_EXPAND_PERPENDICULAR ) + + test_panel.SetSizer( vbox ) + + # + + notebook.AddPage( edit_panel, 'edit', select = True ) + notebook.AddPage( test_panel, 'test', select = False ) + + vbox = wx.BoxSizer( wx.VERTICAL ) + + vbox.AddF( notebook, CC.FLAGS_EXPAND_SIZER_BOTH_WAYS ) + + self.SetSizer( vbox ) + + + def EventEditFormula( self, event ): + + # edit it using the class I already wrote above + # update self._current_formula + # update the formula_description + + pass + + + def EventTestParse( self, event ): + + node = self.GetValue() + + try: + + data = self._example_data.GetValue() + url = 'test-url.com/test_query' + desired_content = 'all' + + results = node.Parse( data, url, desired_content ) + + self._results.SetValue( 'put nicely formatted results here' ) + + except Exception as e: + + HydrusData.ShowException( e ) + + message = 'Could not parse!' + message += os.linesep * 2 + message += HydrusData.ToUnicode( e ) + + wx.MessageBox( message ) + + + + def GetValue( self ): + + name = self._name.GetValue() + + content_type = self._content_type.GetChoice() + + if content_type == HC.CONTENT_TYPE_MAPPINGS: + + additional_info = self._namespace.GetValue() + + + formula = self._current_formula + + node = HydrusParsing.ParseNodeContent( name = name, content_type = content_type, formula = formula, additional_info = additional_info ) + + return node + + +class EditParseNodeContentLinkPanel( ClientGUIScrolledPanels.EditPanel ): + + def __init__( self, parent, node, example_data = None ): + + ClientGUIScrolledPanels.EditPanel.__init__( self, parent ) + + if example_data is None: + + example_data = '' + + + +class EditParsingScriptFileLookupPanel( ClientGUIScrolledPanels.EditPanel ): + + def __init__( self, parent, script ): + + ClientGUIScrolledPanels.EditPanel.__init__( self, parent ) + + ( name, example_url, query_type, file_identifier_type, file_identifier_encoding, file_identifier_arg_name, static_args, children ) = script.ToTuple() + + # + + notebook = wx.Notebook( self ) + + # + + edit_panel = wx.Panel( notebook ) + + edit_panel.SetBackgroundColour( wx.SystemSettings.GetColour( wx.SYS_COLOUR_FRAMEBK ) ) + + self._name = wx.TextCtrl( edit_panel ) + + query_panel = ClientGUICommon.StaticBox( edit_panel, 'query' ) + + self._query_type = ClientGUICommon.BetterChoice( query_panel ) + + self._query_type.Append( 'GET', HC.GET ) + self._query_type.Append( 'POST', HC.POST ) + + self._file_identifier_type = ClientGUICommon.BetterChoice( query_panel ) + + for t in [ HydrusParsing.FILE_IDENTIFIER_TYPE_FILE, HydrusParsing.FILE_IDENTIFIER_TYPE_MD5, HydrusParsing.FILE_IDENTIFIER_TYPE_SHA1, HydrusParsing.FILE_IDENTIFIER_TYPE_SHA256, HydrusParsing.FILE_IDENTIFIER_TYPE_SHA512, HydrusParsing.FILE_IDENTIFIER_TYPE_USER_INPUT ]: + + self._file_identifier_type.Append( HydrusParsing.file_identifier_string_lookup[ t ], t ) + + + self._file_identifier_encoding = ClientGUICommon.BetterChoice( query_panel ) + + for e in [ HC.ENCODING_RAW, HC.ENCODING_HEX, HC.ENCODING_BASE64 ]: + + self._file_identifier_encoding.Append( HC.encoding_string_lookup[ e ], e ) + + + self._file_identifier_arg_name = wx.TextCtrl( query_panel ) + + static_args_panel = ClientGUICommon.StaticBox( query_panel, 'static arguments' ) + + self._static_args = ClientGUICommon.EditStringToStringDict( static_args_panel, static_args ) + + children_panel = ClientGUICommon.StaticBox( edit_panel, 'content parsing children' ) + + self._children = EditNodes( children_panel, children, self.GetExampleData ) + + # + + testing_panel = wx.Panel( notebook ) + + testing_panel.SetBackgroundColour( wx.SystemSettings.GetColour( wx.SYS_COLOUR_FRAMEBK ) ) + + self._example_data = '' + + self._test_url = wx.TextCtrl( testing_panel ) + + self._test_url.SetValue( example_url ) + + self._test_arg = wx.TextCtrl( testing_panel ) + + self._test_arg.SetValue( 'enter example file path, hex hash, or raw user input here' ) + + self._fetch_data = wx.Button( testing_panel, label = 'fetch response' ) + self._fetch_data.Bind( wx.EVT_BUTTON, self.EventFetchData ) + + self._example_data = wx.TextCtrl( testing_panel, style = wx.TE_MULTILINE ) + + self._example_data.SetMinSize( ( -1, 200 ) ) + + self._test_parsing = wx.Button( testing_panel, label = 'test parsing' ) + self._test_parsing.Bind( wx.EVT_BUTTON, self.EventTestParsing ) + + self._results = wx.TextCtrl( testing_panel, style = wx.TE_MULTILINE ) + + self._results.SetMinSize( ( -1, 200 ) ) + + # + + self._name.SetValue( name ) + + self._query_type.SelectClientData( query_type ) + self._file_identifier_type.SelectClientData( file_identifier_type ) + self._file_identifier_encoding.SelectClientData( file_identifier_encoding ) + self._file_identifier_arg_name.SetValue( file_identifier_arg_name ) + + # + + rows = [] + + rows.append( ( 'query type: ', self._query_type ) ) + rows.append( ( 'file identifier type: ', self._file_identifier_type ) ) + rows.append( ( 'file identifier encoding: ', self._file_identifier_encoding ) ) + rows.append( ( 'file identifier GET/POST argument name: ', self._file_identifier_arg_name ) ) + + gridbox = ClientGUICommon.WrapInGrid( query_panel, rows ) + + query_panel.AddF( gridbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) + query_panel.AddF( static_args_panel, CC.FLAGS_EXPAND_PERPENDICULAR ) + + static_args_panel.AddF( self._static_args, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) + + children_message = 'The data returned by the query will be passed to each of these children for content parsing.' + + children_panel.AddF( wx.StaticText( children_panel, label = children_message ), CC.FLAGS_EXPAND_PERPENDICULAR ) + children_panel.AddF( self._children, CC.FLAGS_EXPAND_PERPENDICULAR ) + + vbox = wx.BoxSizer( wx.VERTICAL ) + + rows = [] + + rows.append( ( 'script name: ', self._name ) ) + + gridbox = ClientGUICommon.WrapInGrid( edit_panel, rows ) + + vbox.AddF( gridbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) + vbox.AddF( query_panel, CC.FLAGS_EXPAND_PERPENDICULAR ) + vbox.AddF( children_panel, CC.FLAGS_EXPAND_PERPENDICULAR ) + + edit_panel.SetSizer( vbox ) + + # + + vbox = wx.BoxSizer( wx.VERTICAL ) + + vbox.AddF( self._test_url, CC.FLAGS_EXPAND_PERPENDICULAR ) + vbox.AddF( self._test_arg, CC.FLAGS_EXPAND_PERPENDICULAR ) + vbox.AddF( self._fetch_data, CC.FLAGS_EXPAND_PERPENDICULAR ) + vbox.AddF( self._example_data, CC.FLAGS_EXPAND_PERPENDICULAR ) + vbox.AddF( self._test_parsing, CC.FLAGS_EXPAND_PERPENDICULAR ) + vbox.AddF( self._results, CC.FLAGS_EXPAND_PERPENDICULAR ) + + testing_panel.SetSizer( vbox ) + + # + + notebook.AddPage( edit_panel, 'edit', select = True ) + notebook.AddPage( testing_panel, 'test', select = False ) + + vbox = wx.BoxSizer( wx.VERTICAL ) + + vbox.AddF( notebook, CC.FLAGS_EXPAND_SIZER_BOTH_WAYS ) + + self.SetSizer( vbox ) + + + def EventFetchData( self, event ): + + script = self.GetValue() + + test_url = self._test_url.GetValue() + test_arg = self._test_arg.GetValue() + + file_identifier_type = self._file_identifier_type.GetChoice() + + if file_identifier_type == HydrusParsing.FILE_IDENTIFIER_TYPE_FILE: + + if not os.path.exists( test_arg ): + + wx.MessageBox( 'That file does not exist!' ) + + return + + + with open( test_arg, 'rb' ) as f: + + file_identifier = f.read() + + + elif file_identifier_type == HydrusParsing.FILE_IDENTIFIER_TYPE_USER_INPUT: + + file_identifier = test_arg + + else: + + file_identifier = test_arg.decode( 'hex' ) + + + try: + + example_data = script.FetchData( test_url, file_identifier ) + + self._example_data.SetValue( example_data ) + + except Exception as e: + + HydrusData.ShowException( e ) + + message = 'Could not fetch data!' + message += os.linesep * 2 + message += HydrusData.ToUnicode( e ) + + wx.MessageBox( message ) + + + + def EventTestParsing( self, event ): + + script = self.GetValue() + + try: + + data = self._example_data.GetValue() + url = self._test_url.GetValue() + desired_content = 'all' + + results = script.Parse( data, url, desired_content ) + + self._results.SetValue( 'put nicely formatted results here' ) + + except Exception as e: + + HydrusData.ShowException( e ) + + message = 'Could not parse!' + message += os.linesep * 2 + message += HydrusData.ToUnicode( e ) + + wx.MessageBox( message ) + + + + def GetExampleData( self ): + + return self._example_data.GetValue() + + + def GetValue( self ): + + name = self._name.GetValue() + example_url = self._test_url.GetValue() + query_type = self._query_type.GetChoice() + file_identifier_type = self._file_identifier_type.GetChoice() + file_identifier_encoding = self._file_identifier_encoding.GetChoice() + file_identifier_arg_name = self._file_identifier_arg_name.GetValue() + static_args = self._static_args.GetValue() + children = self._children.GetValue() + + script = HydrusParsing.ParseRootFileLookup( name, example_url = example_url, query_type = query_type, file_identifier_type = file_identifier_type, file_identifier_encoding = file_identifier_encoding, file_identifier_arg_name = file_identifier_arg_name, static_args = static_args, children = children ) + + return script + + +class ManageParsingScriptsPanel( ClientGUIScrolledPanels.ManagePanel ): + + def __init__( self, parent ): + + ClientGUIScrolledPanels.ManagePanel.__init__( self, parent ) + + self._scripts = ClientGUICommon.SaneListCtrl( self, 200, [ ( 'name', 140 ), ( 'query type', 80 ), ( 'script type', 80 ), ( 'produces', -1 ) ], delete_key_callback = self.Delete, activation_callback = self.Edit, use_display_tuple_for_sort = True ) + + self._add_button = wx.Button( self, label = 'add' ) + self._add_button.Bind( wx.EVT_BUTTON, self.EventAdd ) + + self._copy_button = wx.Button( self, label = 'copy' ) + self._copy_button.Bind( wx.EVT_BUTTON, self.EventCopy ) + + self._paste_button = wx.Button( self, label = 'paste' ) + self._paste_button.Bind( wx.EVT_BUTTON, self.EventPaste ) + + self._duplicate_button = wx.Button( self, label = 'duplicate' ) + self._duplicate_button.Bind( wx.EVT_BUTTON, self.EventDuplicate ) + + self._edit_button = wx.Button( self, label = 'edit' ) + self._edit_button.Bind( wx.EVT_BUTTON, self.EventEdit ) + + self._delete_button = wx.Button( self, label = 'delete' ) + self._delete_button.Bind( wx.EVT_BUTTON, self.EventDelete ) + + # + + scripts = [] # fetch all scripts from the db, populate listctrl using name column's data to store the script itself or w/e + + for script in scripts: + + ( display_tuple, data_tuple ) = self._ConvertScriptToTuples( script ) + + self._scripts.Append( display_tuple, data_tuple ) + + + # + + vbox = wx.BoxSizer( wx.VERTICAL ) + + button_hbox = wx.BoxSizer( wx.HORIZONTAL ) + + button_hbox.AddF( self._add_button, CC.FLAGS_VCENTER ) + button_hbox.AddF( self._copy_button, CC.FLAGS_VCENTER ) + button_hbox.AddF( self._paste_button, CC.FLAGS_VCENTER ) + button_hbox.AddF( self._duplicate_button, CC.FLAGS_VCENTER ) + button_hbox.AddF( self._edit_button, CC.FLAGS_VCENTER ) + button_hbox.AddF( self._delete_button, CC.FLAGS_VCENTER ) + + vbox.AddF( self._scripts, CC.FLAGS_EXPAND_BOTH_WAYS ) + vbox.AddF( button_hbox, CC.FLAGS_BUTTON_SIZER ) + + self.SetSizer( vbox ) + + + def _ConvertScriptToTuples( self, script ): + + ( name, query_type, script_type, produces ) = script.ToPrettyStrings() + + return ( ( name, query_type, script_type, produces ), ( script, query_type, script_type, produces ) ) + + + def _SetNonDupeName( self, script ): + + name = script.GetName() + + current_names = { script.GetName() for ( script, query_type, script_type, produces ) in self._scripts.GetClientData() } + + if name in current_names: + + i = 1 + + original_name = name + + while name in current_names: + + name = original_name + ' (' + str( i ) + ')' + + i += 1 + + + script.SetName( name ) + + + + def Add( self ): + + with ClientGUIDialogs.DialogSelectFromListOfStrings( self, 'select the script type', [ 'file lookup' ] ) as dlg_type: + + if dlg_type.ShowModal() == wx.ID_OK: + + script_type_string = dlg_type.GetString() + + if script_type_string == 'file lookup': + + name = 'new script' + example_url = 'enter example url here' + query_type = HC.GET + file_identifier_type = HydrusParsing.FILE_IDENTIFIER_TYPE_MD5 + file_identifier_encoding = HC.ENCODING_BASE64 + file_identifier_arg_name = 'md5' + static_args = {} + children = [] + + empty_script = HydrusParsing.ParseRootFileLookup( name, example_url = example_url, query_type = query_type, file_identifier_type = file_identifier_type, file_identifier_encoding = file_identifier_encoding, file_identifier_arg_name = file_identifier_arg_name, static_args = static_args, children = children) + + panel_class = EditParsingScriptFileLookupPanel + + + with ClientGUITopLevelWindows.DialogEdit( self, 'edit script' ) as dlg_edit: + + panel = panel_class( dlg_edit, empty_script ) + + dlg_edit.SetPanel( panel ) + + if dlg_edit.ShowModal() == wx.ID_OK: + + new_script = panel.GetValue() + + self._SetNonDupeName( new_script ) + + ( display_tuple, data_tuple ) = self._ConvertScriptToTuples( new_script ) + + self._scripts.Append( display_tuple, data_tuple ) + + + + + + + def CommitChanges( self ): + + scripts = [ script for ( script, query_type, script_type, produces ) in self._scripts.GetClientData() ] + + # save them to db + # this should completely delete and replace the old stuff in the db to allow for renames + + + def Copy( self ): + + for i in self._scripts.GetAllSelected(): + + ( script, query_type, script_type, produces ) = self._scripts.GetClientData( i ) + + script_json = script.DumpToString() + + HydrusGlobals.client_controller.pub( 'clipboard', 'text', script_json ) + + + + def Delete( self ): + + self._scripts.RemoveAllSelected() + + + def Duplicate( self ): + + scripts_to_dupe = [] + + for i in self._scripts.GetAllSelected(): + + ( script, query_type, script_type, produces ) = self._scripts.GetClientData( i ) + + scripts_to_dupe.append( script ) + + + for script in scripts_to_dupe: + + dupe_script = script.Duplicate() + + self._SetNonDupeName( dupe_script ) + + ( display_tuple, data_tuple ) = self._ConvertScriptToTuples( dupe_script ) + + self._scripts.Append( display_tuple, data_tuple ) + + + + def Edit( self ): + + for i in self._scripts.GetAllSelected(): + + ( script, query_type, script_type, produces ) = self._scripts.GetClientData( i ) + + if isinstance( script, HydrusParsing.ParseRootFileLookup ): + + panel_class = EditParsingScriptFileLookupPanel + + dlg_title = 'edit file lookup script' + + + with ClientGUITopLevelWindows.DialogEdit( self, dlg_title ) as dlg: + + original_name = script.GetName() + + panel = panel_class( dlg, script ) + + dlg.SetPanel( panel ) + + if dlg.ShowModal() == wx.ID_OK: + + edited_script = panel.GetValue() + + name = edited_script.GetName() + + if name != original_name: + + self._SetNonDupeName( edited_script ) + + + ( display_tuple, data_tuple ) = self._ConvertScriptToTuples( edited_script ) + + self._scripts.UpdateRow( i, display_tuple, data_tuple ) + + + + + + + def Paste( self ): + + if wx.TheClipboard.Open(): + + data = wx.TextDataObject() + + wx.TheClipboard.GetData( data ) + + wx.TheClipboard.Close() + + raw_text = data.GetText() + + try: + + obj = HydrusSerialisable.CreateFromString( raw_text ) + + if isinstance( obj, HydrusParsing.ParseRootFileLookup ): + + script = obj + + self._SetNonDupeName( script ) + + ( display_tuple, data_tuple ) = self._ConvertScriptToTuples( script ) + + self._scripts.Append( display_tuple, data_tuple ) + + + except: + + wx.MessageBox( 'I could not understand what was in the clipboard' ) + + + else: + + wx.MessageBox( 'I could not get permission to access the clipboard.' ) + + + + def EventAdd( self, event ): + + self.Add() + + + def EventCopy( self, event ): + + self.Copy() + + + def EventDelete( self, event ): + + self.Delete() + + + def EventDuplicate( self, event ): + + self.Duplicate() + + + def EventEdit( self, event ): + + self.Edit() + + + def EventPaste( self, event ): + + self.Paste() + + \ No newline at end of file diff --git a/include/ClientGUIScrolledPanels.py b/include/ClientGUIScrolledPanels.py index caca225f..29e806ee 100644 --- a/include/ClientGUIScrolledPanels.py +++ b/include/ClientGUIScrolledPanels.py @@ -136,264 +136,6 @@ class EditFrameLocationPanel( EditPanel ): return ( name, remember_size, remember_position, last_size, last_position, default_gravity, default_position, maximised, fullscreen ) -class EditHTMLFormulaPanel( EditPanel ): - - def __init__( self, parent, info ): - - EditPanel.__init__( self, parent ) - - self._original_info = info - - self._do_testing_automatically = False - - formula_panel = ClientGUICommon.StaticBox( self, 'formula' ) - - self._tag_rules = wx.ListBox( formula_panel, style = wx.LB_SINGLE ) - self._tag_rules.Bind( wx.EVT_LEFT_DCLICK, self.EventEdit ) - - self._add_rule = wx.Button( formula_panel, label = 'add' ) - self._add_rule.Bind( wx.EVT_BUTTON, self.EventAdd ) - - self._edit_rule = wx.Button( formula_panel, label = 'edit' ) - self._edit_rule.Bind( wx.EVT_BUTTON, self.EventEdit ) - - self._move_rule_up = wx.Button( formula_panel, label = u'\u2191' ) - self._move_rule_up.Bind( wx.EVT_BUTTON, self.EventMoveUp ) - - self._delete_rule = wx.Button( formula_panel, label = 'X' ) - self._delete_rule.Bind( wx.EVT_BUTTON, self.EventDelete ) - - self._move_rule_down = wx.Button( formula_panel, label = u'\u2193' ) - self._move_rule_down.Bind( wx.EVT_BUTTON, self.EventMoveDown ) - - self._content_rule = wx.TextCtrl( formula_panel ) - - testing_panel = ClientGUICommon.StaticBox( self, 'testing' ) - - self._test_html = wx.TextCtrl( testing_panel, style = wx.TE_MULTILINE ) - - self._fetch_from_url = wx.Button( testing_panel, label = 'fetch result from url' ) - self._fetch_from_url.Bind( wx.EVT_BUTTON, self.EventFetchFromURL ) - - self._run_test = wx.Button( testing_panel, label = 'run test' ) - self._run_test.Bind( wx.EVT_BUTTON, self.EventRunTest ) - - self._results = wx.TextCtrl( testing_panel, style = wx.TE_MULTILINE ) - - # - - ( tag_rules, content_rule ) = self._original_info.ToTuple() - - for rule in tag_rules: - - pretty_rule = HydrusParsing.RenderTagRule( rule ) - - self._tag_rules.Append( pretty_rule, rule ) - - - self._content_rule.SetValue( content_rule ) - - self._test_html.SetValue( 'Enter html here to test it against the above formula.' ) - self._results.SetValue( 'Successfully parsed results will be printed here.' ) - - # - - udd_button_vbox = wx.BoxSizer( wx.VERTICAL ) - - udd_button_vbox.AddF( self._move_rule_up, CC.FLAGS_VCENTER ) - udd_button_vbox.AddF( self._delete_rule, CC.FLAGS_VCENTER ) - udd_button_vbox.AddF( self._move_rule_down, CC.FLAGS_VCENTER ) - - tag_rules_hbox = wx.BoxSizer( wx.HORIZONTAL ) - - tag_rules_hbox.AddF( self._tag_rules, CC.FLAGS_EXPAND_BOTH_WAYS ) - tag_rules_hbox.AddF( udd_button_vbox, CC.FLAGS_VCENTER ) - - ae_button_hbox = wx.BoxSizer( wx.HORIZONTAL ) - - ae_button_hbox.AddF( self._add_rule, CC.FLAGS_VCENTER ) - ae_button_hbox.AddF( self._edit_rule, CC.FLAGS_VCENTER ) - - formula_panel.AddF( tag_rules_hbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) - formula_panel.AddF( ae_button_hbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) - formula_panel.AddF( ClientGUICommon.WrapInText( self._content_rule, formula_panel, 'attribute: ' ), CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) - - testing_panel.AddF( self._test_html, CC.FLAGS_EXPAND_PERPENDICULAR ) - testing_panel.AddF( self._fetch_from_url, CC.FLAGS_EXPAND_PERPENDICULAR ) - testing_panel.AddF( self._run_test, CC.FLAGS_EXPAND_PERPENDICULAR ) - testing_panel.AddF( self._results, CC.FLAGS_EXPAND_PERPENDICULAR ) - - vbox = wx.BoxSizer( wx.VERTICAL ) - - message = 'The html will be searched recursively by each rule in turn and then the attribute of the final tags will be returned.' - message += os.linesep * 2 - message += 'So, to find the \'src\' of the first tag beneath all tags with the class \'content\', use:' - message += os.linesep * 2 - message += 'all span tags with class=content' - message += '1st img tag' - message += 'attribute: src' - message += os.linesep * 2 - message += 'Leave the attribute blank to represent the string of the tag (i.e.This part
).' - - vbox.AddF( wx.StaticText( self, label = message ), CC.FLAGS_EXPAND_PERPENDICULAR ) - vbox.AddF( formula_panel, CC.FLAGS_EXPAND_PERPENDICULAR ) - vbox.AddF( testing_panel, CC.FLAGS_EXPAND_PERPENDICULAR ) - - self.SetSizer( vbox ) - - - def _RunTest( self ): - - formula = self.GetValue() - - html = self._test_html.GetValue() - - try: - - results = formula.Parse( html ) - - # do the begin/end to better display '' results and any other whitespace weirdness - results = [ '*** RESULTS BEGIN ***' ] + results + [ '*** RESULTS END ***' ] - - results_text = os.linesep.join( results ) - - self._results.SetValue( results_text ) - - self._do_testing_automatically = True - - except Exception as e: - - message = 'Could not parse! Full error written to log!' - message += os.linesep * 2 - message += HydrusData.ToUnicode( e ) - - wx.MessageBox( message ) - - self._do_testing_automatically = False - - - - def EventAdd( self, event ): - - # spawn dialog, add it and run test - - if self._do_testing_automatically: - - self._RunTest() - - - - def EventDelete( self, event ): - - selection = self._tag_rules.GetSelection() - - if selection != wx.NOT_FOUND: - - if self._tag_rules.GetCount() == 1: - - wx.MessageBox( 'A parsing formula needs at least one tag rule!' ) - - else: - - self._tag_rules.Delete( selection ) - - if self._do_testing_automatically: - - self._RunTest() - - - - - - def EventEdit( self, event ): - - selection = self._tag_rules.GetSelection() - - if selection != wx.NOT_FOUND: - - ( name, attrs, index ) = self._tag_rules.GetClientData( selection ) - - # spawn dialog, then if ok, set it and run test - - if self._do_testing_automatically: - - self._RunTest() - - - - def EventFetchFromURL( self, event ): - - # ask user for url with textdlg - # get it with requests - # handle errors with a messagebox - # try to parse it with bs4 to check it is good html and then splat it to the textctrl, otherwise just messagebox the error - - if self._do_testing_automatically: - - self._RunTest() - - - - def EventMoveDown( self, event ): - - selection = self._tag_rules.GetSelection() - - if selection != wx.NOT_FOUND and selection + 1 < self._tag_rules.GetCount(): - - pretty_rule = self._tag_rules.GetString( selection ) - rule = self._tag_rules.GetClientData( selection ) - - self._tag_rules.Delete( selection ) - - self._tag_rules.Insert( selection + 1, pretty_rule, rule ) - - if self._do_testing_automatically: - - self._RunTest() - - - - - def EventMoveUp( self, event ): - - selection = self._tag_rules.GetSelection() - - if selection != wx.NOT_FOUND and selection > 0: - - pretty_rule = self._tag_rules.GetString( selection ) - rule = self._tag_rules.GetClientData( selection ) - - self._tag_rules.Delete( selection ) - - self._tag_rules.Insert( selection - 1, pretty_rule, rule ) - - if self._do_testing_automatically: - - self._RunTest() - - - - - def EventRunTest( self, event ): - - self._RunTest() - - - def GetValue( self ): - - tags_rules = [ self._tag_rules.GetClientData( i ) for i in range( self._tag_rules.GetCount() ) ] - content_rule = self._content_rule.GetValue() - - if content_rule == '': - - content_rule = None - - - formula = HydrusParsing.ParseFormulaHTML( tags_rules, content_rule ) - - return formula - - class EditMediaViewOptionsPanel( EditPanel ): def __init__( self, parent, info ): @@ -577,199 +319,6 @@ class EditMediaViewOptionsPanel( EditPanel ): return ( self._mime, media_show_action, preview_show_action, ( media_scale_up, media_scale_down, preview_scale_up, preview_scale_down, exact_zooms_only, scale_up_quality, scale_down_quality ) ) -class EditParsingScriptFileLookupPanel( EditPanel ): - - def __init__( self, parent, script ): - - EditPanel.__init__( self, parent ) - - ( name, example_url, query_type, file_identifier_type, file_identifier_encoding, file_identifier_arg_name, static_args, children ) = script.ToTuple() - - # - - self._name = wx.TextCtrl( self ) - - query_panel = ClientGUICommon.StaticBox( self, 'query' ) - - self._query_type = ClientGUICommon.BetterChoice( query_panel ) - - self._query_type.Append( 'GET', HC.GET ) - self._query_type.Append( 'POST', HC.POST ) - - self._file_identifier_type = ClientGUICommon.BetterChoice( query_panel ) - - for t in [ HydrusParsing.FILE_IDENTIFIER_TYPE_FILE, HydrusParsing.FILE_IDENTIFIER_TYPE_MD5, HydrusParsing.FILE_IDENTIFIER_TYPE_SHA1, HydrusParsing.FILE_IDENTIFIER_TYPE_SHA256, HydrusParsing.FILE_IDENTIFIER_TYPE_SHA512, HydrusParsing.FILE_IDENTIFIER_TYPE_USER_INPUT ]: - - self._file_identifier_type.Append( HydrusParsing.file_identifier_string_lookup[ t ], t ) - - - self._file_identifier_encoding = ClientGUICommon.BetterChoice( query_panel ) - - for e in [ HC.ENCODING_RAW, HC.ENCODING_HEX, HC.ENCODING_BASE64 ]: - - self._file_identifier_encoding.Append( HC.encoding_string_lookup[ e ], e ) - - - self._file_identifier_arg_name = wx.TextCtrl( query_panel ) - - static_args_panel = ClientGUICommon.StaticBox( query_panel, 'static arguments' ) - - self._static_args = ClientGUICommon.EditStringToStringDict( static_args_panel, static_args ) - - # - - children_panel = ClientGUICommon.StaticBox( self, 'content parsing children' ) - - self._children = wx.Panel( children_panel ) - - wx.StaticText( self._children, label = 'flexible edit children panel goes here' ) - - # - - testing_panel = ClientGUICommon.StaticBox( self, 'testing' ) - - # need a url here for the url param - - self._test_url = wx.TextCtrl( testing_panel ) - - self._test_url.SetValue( example_url ) - - self._test_arg = wx.TextCtrl( testing_panel ) - - self._test_arg.SetValue( 'enter example file path, hex hash, or raw user input here' ) - - self._run_test = wx.Button( testing_panel, label = 'run test' ) - self._run_test.Bind( wx.EVT_BUTTON, self.EventRunTest ) - - self._results = wx.TextCtrl( testing_panel, style = wx.TE_MULTILINE ) - - self._results.SetMinSize( ( -1, 200 ) ) - - # - - self._name.SetValue( name ) - - self._query_type.SelectClientData( query_type ) - self._file_identifier_type.SelectClientData( file_identifier_type ) - self._file_identifier_encoding.SelectClientData( file_identifier_encoding ) - self._file_identifier_arg_name.SetValue( file_identifier_arg_name ) - - # set children - - # - - static_args_panel.AddF( self._static_args, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) - - rows = [] - - rows.append( ( 'query type: ', self._query_type ) ) - rows.append( ( 'file identifier type: ', self._file_identifier_type ) ) - rows.append( ( 'file identifier encoding: ', self._file_identifier_encoding ) ) - rows.append( ( 'file identifier GET/POST argument name: ', self._file_identifier_arg_name ) ) - - gridbox = ClientGUICommon.WrapInGrid( query_panel, rows ) - - query_panel.AddF( gridbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) - query_panel.AddF( static_args_panel, CC.FLAGS_EXPAND_PERPENDICULAR ) - - children_message = 'The data returned by the query will be passed to each of these children for content parsing.' - - children_panel.AddF( wx.StaticText( children_panel, label = children_message ), CC.FLAGS_EXPAND_PERPENDICULAR ) - children_panel.AddF( self._children, CC.FLAGS_EXPAND_PERPENDICULAR ) - - testing_panel.AddF( self._test_url, CC.FLAGS_EXPAND_PERPENDICULAR ) - testing_panel.AddF( self._test_arg, CC.FLAGS_EXPAND_PERPENDICULAR ) - testing_panel.AddF( self._run_test, CC.FLAGS_EXPAND_PERPENDICULAR ) - testing_panel.AddF( self._results, CC.FLAGS_EXPAND_PERPENDICULAR ) - - vbox = wx.BoxSizer( wx.VERTICAL ) - - rows = [] - - rows.append( ( 'script name: ', self._name ) ) - - gridbox = ClientGUICommon.WrapInGrid( self, rows ) - - vbox.AddF( gridbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) - vbox.AddF( query_panel, CC.FLAGS_EXPAND_PERPENDICULAR ) - vbox.AddF( children_panel, CC.FLAGS_EXPAND_PERPENDICULAR ) - vbox.AddF( testing_panel, CC.FLAGS_EXPAND_PERPENDICULAR ) - - self.SetSizer( vbox ) - - - def _RunTest( self ): - - script = self.GetValue() - - test_url = self._test_url.GetValue() - test_arg = self._test_arg.GetValue() - - file_identifier_type = self._file_identifier_type.GetChoice() - - if file_identifier_type == HydrusParsing.FILE_IDENTIFIER_TYPE_FILE: - - if not os.path.exists( test_arg ): - - wx.MessageBox( 'That file does not exist!' ) - - return - - - with open( test_arg, 'rb' ) as f: - - file_identifier = f.read() - - - elif file_identifier_type == HydrusParsing.FILE_IDENTIFIER_TYPE_USER_INPUT: - - file_identifier = test_arg - - else: - - file_identifier = test_arg.decode( 'hex' ) - - - try: - - data = script.FetchData( test_url, file_identifier ) - - self._results.SetValue( data ) - - except Exception as e: - - message = 'Could not parse! Full error written to log!' - message += os.linesep * 2 - message += HydrusData.ToUnicode( e ) - - wx.MessageBox( message ) - - - - def EventRunTest( self, event ): - - self._RunTest() - - - def GetValue( self ): - - name = self._name.GetValue() - example_url = self._test_url.GetValue() - query_type = self._query_type.GetChoice() - file_identifier_type = self._file_identifier_type.GetChoice() - file_identifier_encoding = self._file_identifier_encoding.GetChoice() - file_identifier_arg_name = self._file_identifier_arg_name.GetValue() - - # static args - static_args = {} - # children - children = [] - - script = HydrusParsing.ParseRootFileLookup( name, example_url = example_url, query_type = query_type, file_identifier_type = file_identifier_type, file_identifier_encoding = file_identifier_encoding, file_identifier_arg_name = file_identifier_arg_name, static_args = static_args, children = children ) - - return script - - class EditSeedCachePanel( EditPanel ): def __init__( self, parent, controller, seed_cache ): @@ -2083,7 +1632,7 @@ class ManageOptionsPanel( ManagePanel ): self._media_zooms.SetValue( ','.join( ( str( media_zoom ) for media_zoom in media_zooms ) ) ) - mimes_in_correct_order = ( HC.IMAGE_JPEG, HC.IMAGE_PNG, HC.IMAGE_GIF, HC.APPLICATION_FLASH, HC.APPLICATION_PDF, HC.VIDEO_FLV, HC.VIDEO_MOV, HC.VIDEO_MP4, HC.VIDEO_MKV, HC.VIDEO_MPEG, HC.VIDEO_WEBM, HC.VIDEO_WMV, HC.AUDIO_MP3, HC.AUDIO_OGG, HC.AUDIO_FLAC, HC.AUDIO_WMA ) + mimes_in_correct_order = ( HC.IMAGE_JPEG, HC.IMAGE_PNG, HC.IMAGE_GIF, HC.APPLICATION_FLASH, HC.APPLICATION_PDF, HC.VIDEO_AVI, HC.VIDEO_FLV, HC.VIDEO_MOV, HC.VIDEO_MP4, HC.VIDEO_MKV, HC.VIDEO_MPEG, HC.VIDEO_WEBM, HC.VIDEO_WMV, HC.AUDIO_MP3, HC.AUDIO_OGG, HC.AUDIO_FLAC, HC.AUDIO_WMA ) for mime in mimes_in_correct_order: @@ -2822,6 +2371,8 @@ class ManageOptionsPanel( ManagePanel ): self._default_tag_repository = ClientGUICommon.BetterChoice( general_panel ) + self._default_tag_service_search_page = ClientGUICommon.BetterChoice( general_panel ) + self._show_all_tags_in_autocomplete = wx.CheckBox( general_panel ) self._apply_all_parents_to_all_services = wx.CheckBox( general_panel ) @@ -2880,14 +2431,23 @@ class ManageOptionsPanel( ManagePanel ): elif HC.options[ 'default_tag_sort' ] == CC.SORT_BY_INCIDENCE_NAMESPACE_DESC: self._default_tag_sort.Select( 6 ) elif HC.options[ 'default_tag_sort' ] == CC.SORT_BY_INCIDENCE_NAMESPACE_ASC: self._default_tag_sort.Select( 7 ) + self._default_tag_service_search_page.Append( 'all known tags', CC.COMBINED_TAG_SERVICE_KEY ) + services = HydrusGlobals.client_controller.GetServicesManager().GetServices( HC.TAG_SERVICES ) - for service in services: self._default_tag_repository.Append( service.GetName(), service.GetServiceKey() ) + for service in services: + + self._default_tag_repository.Append( service.GetName(), service.GetServiceKey() ) + + self._default_tag_service_search_page.Append( service.GetName(), service.GetServiceKey() ) + default_tag_repository_key = HC.options[ 'default_tag_repository' ] self._default_tag_repository.SelectClientData( default_tag_repository_key ) + self._default_tag_service_search_page.SelectClientData( new_options.GetKey( 'default_tag_service_search_page' ) ) + self._show_all_tags_in_autocomplete.SetValue( HC.options[ 'show_all_tags_in_autocomplete' ] ) self._apply_all_parents_to_all_services.SetValue( self._new_options.GetBoolean( 'apply_all_parents_to_all_services' ) ) @@ -2913,6 +2473,7 @@ class ManageOptionsPanel( ManagePanel ): rows = [] rows.append( ( 'Default tag service in manage tag dialogs: ', self._default_tag_repository ) ) + rows.append( ( 'Default tag service in search pages: ', self._default_tag_service_search_page ) ) rows.append( ( 'Default tag sort: ', self._default_tag_sort ) ) rows.append( ( 'By default, search non-local tags in write-autocomplete: ', self._show_all_tags_in_autocomplete ) ) rows.append( ( 'Suggest all parents for all services: ', self._apply_all_parents_to_all_services ) ) @@ -2995,6 +2556,8 @@ class ManageOptionsPanel( ManagePanel ): HC.options[ 'default_tag_sort' ] = self._default_tag_sort.GetClientData( self._default_tag_sort.GetSelection() ) HC.options[ 'show_all_tags_in_autocomplete' ] = self._show_all_tags_in_autocomplete.GetValue() + self._new_options.SetKey( 'default_tag_service_search_page', self._default_tag_service_search_page.GetChoice() ) + self._new_options.SetNoneableInteger( 'suggested_tags_width', self._suggested_tags_width.GetValue() ) self._new_options.SetBoolean( 'apply_all_parents_to_all_services', self._apply_all_parents_to_all_services.GetValue() ) @@ -3036,293 +2599,6 @@ class ManageOptionsPanel( ManagePanel ): wx.MessageBox( traceback.format_exc() ) - -class ManageParsingScriptsPanel( ManagePanel ): - - def __init__( self, parent ): - - ManagePanel.__init__( self, parent ) - - self._scripts = ClientGUICommon.SaneListCtrl( self, 200, [ ( 'name', -1 ), ( 'query type', 80 ), ( 'script type', 240 ) ], delete_key_callback = self.Delete, activation_callback = self.Edit, use_display_tuple_for_sort = True ) - - self._add_button = wx.Button( self, label = 'add' ) - self._add_button.Bind( wx.EVT_BUTTON, self.EventAdd ) - - self._copy_button = wx.Button( self, label = 'copy' ) - self._copy_button.Bind( wx.EVT_BUTTON, self.EventCopy ) - - self._paste_button = wx.Button( self, label = 'paste' ) - self._paste_button.Bind( wx.EVT_BUTTON, self.EventPaste ) - - self._duplicate_button = wx.Button( self, label = 'duplicate' ) - self._duplicate_button.Bind( wx.EVT_BUTTON, self.EventDuplicate ) - - self._edit_button = wx.Button( self, label = 'edit' ) - self._edit_button.Bind( wx.EVT_BUTTON, self.EventEdit ) - - self._delete_button = wx.Button( self, label = 'delete' ) - self._delete_button.Bind( wx.EVT_BUTTON, self.EventDelete ) - - # - - scripts = [] # fetch all scripts from the db, populate listctrl using name column's data to store the script itself or w/e - - for script in scripts: - - ( display_tuple, data_tuple ) = self._ConvertScriptToTuples( script ) - - self._scripts.Append( display_tuple, data_tuple ) - - - # - - vbox = wx.BoxSizer( wx.VERTICAL ) - - button_hbox = wx.BoxSizer( wx.HORIZONTAL ) - - button_hbox.AddF( self._add_button, CC.FLAGS_VCENTER ) - button_hbox.AddF( self._copy_button, CC.FLAGS_VCENTER ) - button_hbox.AddF( self._paste_button, CC.FLAGS_VCENTER ) - button_hbox.AddF( self._duplicate_button, CC.FLAGS_VCENTER ) - button_hbox.AddF( self._edit_button, CC.FLAGS_VCENTER ) - button_hbox.AddF( self._delete_button, CC.FLAGS_VCENTER ) - - vbox.AddF( self._scripts, CC.FLAGS_EXPAND_BOTH_WAYS ) - vbox.AddF( button_hbox, CC.FLAGS_BUTTON_SIZER ) - - self.SetSizer( vbox ) - - - def _ConvertScriptToTuples( self, script ): - - # fetch these vars from the script, return display/data tuples for the listctrl - ( name, query_type, script_type ) = script.ToPrettyStrings() - - return ( ( name, query_type, script_type ), ( script, query_type, script_type ) ) - - - def _SetNonDupeName( self, script ): - - name = script.GetName() - - current_names = { script.GetName() for ( script, query_type, script_type ) in self._scripts.GetClientData() } - - if name in current_names: - - i = 1 - - original_name = name - - while name in current_names: - - name = original_name + ' (' + str( i ) + ')' - - i += 1 - - - script.SetName( name ) - - - - def Add( self ): - - with ClientGUIDialogs.DialogSelectFromListOfStrings( self, 'select the script type', [ 'file lookup' ] ) as dlg_type: - - if dlg_type.ShowModal() == wx.ID_OK: - - script_type_string = dlg_type.GetString() - - if script_type_string == 'file lookup': - - name = 'new script' - example_url = 'enter example url here' - query_type = HC.GET - file_identifier_type = HydrusParsing.FILE_IDENTIFIER_TYPE_MD5 - file_identifier_encoding = HC.ENCODING_BASE64 - file_identifier_arg_name = 'md5' - static_args = {} - children = [] - - empty_script = HydrusParsing.ParseRootFileLookup( name, example_url = example_url, query_type = query_type, file_identifier_type = file_identifier_type, file_identifier_encoding = file_identifier_encoding, file_identifier_arg_name = file_identifier_arg_name, static_args = static_args, children = children) - - panel_class = EditParsingScriptFileLookupPanel - - - with ClientGUITopLevelWindows.DialogEdit( self, 'edit script' ) as dlg_edit: - - panel = panel_class( dlg_edit, empty_script ) - - dlg_edit.SetPanel( panel ) - - if dlg_edit.ShowModal() == wx.ID_OK: - - new_script = panel.GetValue() - - self._SetNonDupeName( new_script ) - - ( display_tuple, data_tuple ) = self._ConvertScriptToTuples( new_script ) - - self._scripts.Append( display_tuple, data_tuple ) - - - - - - - def CommitChanges( self ): - - scripts = [ script for ( script, query_type, script_type ) in self._scripts.GetClientData() ] - - # save them to db - # this should completely delete and replace the old stuff in the db to allow for renames - - - def Copy( self ): - - for i in self._scripts.GetAllSelected(): - - ( script, query_type, script_type ) = self._scripts.GetClientData( i ) - - script_json = script.DumpToString() - - HydrusGlobals.client_controller.pub( 'clipboard', 'text', script_json ) - - - - def Delete( self ): - - self._scripts.RemoveAllSelected() - - - def Duplicate( self ): - - scripts_to_dupe = [] - - for i in self._scripts.GetAllSelected(): - - ( script, query_type, script_type ) = self._scripts.GetClientData( i ) - - scripts_to_dupe.append( script ) - - - for script in scripts_to_dupe: - - dupe_script = script.Duplicate() - - self._SetNonDupeName( dupe_script ) - - ( display_tuple, data_tuple ) = self._ConvertScriptToTuples( dupe_script ) - - self._scripts.Append( display_tuple, data_tuple ) - - - - def Edit( self ): - - for i in self._scripts.GetAllSelected(): - - ( script, query_type, script_type ) = self._scripts.GetClientData( i ) - - original_name = script.GetName() - - with ClientGUITopLevelWindows.DialogEdit( self, 'edit script' ) as dlg: - - if isinstance( script, HydrusParsing.ParseRootFileLookup ): - - panel_class = EditParsingScriptFileLookupPanel - - - panel = panel_class( dlg, script ) - - dlg.SetPanel( panel ) - - if dlg.ShowModal() == wx.ID_OK: - - edited_script = panel.GetValue() - - name = edited_script.GetName() - - if name != original_name: - - self._SetNonDupeName( edited_script ) - - - ( display_tuple, data_tuple ) = self._ConvertScriptToTuples( edited_script ) - - self._scripts.UpdateRow( i, display_tuple, data_tuple ) - - - - - - - def Paste( self ): - - if wx.TheClipboard.Open(): - - data = wx.TextDataObject() - - wx.TheClipboard.GetData( data ) - - wx.TheClipboard.Close() - - raw_text = data.GetText() - - try: - - obj = HydrusSerialisable.CreateFromString( raw_text ) - - if isinstance( obj, HydrusParsing.ParseRootFileLookup ): - - script = obj - - self._SetNonDupeName( script ) - - ( display_tuple, data_tuple ) = self._ConvertScriptToTuples( script ) - - self._scripts.Append( display_tuple, data_tuple ) - - - except: - - wx.MessageBox( 'I could not understand what was in the clipboard' ) - - - else: - - wx.MessageBox( 'I could not get permission to access the clipboard.' ) - - - - def EventAdd( self, event ): - - self.Add() - - - def EventCopy( self, event ): - - self.Copy() - - - def EventDelete( self, event ): - - self.Delete() - - - def EventDuplicate( self, event ): - - self.Duplicate() - - - def EventEdit( self, event ): - - self.Edit() - - - def EventPaste( self, event ): - - self.Paste() - class ManageTagsPanel( ManagePanel ): diff --git a/include/ClientImporting.py b/include/ClientImporting.py index de4b7cb8..e58ea4fb 100644 --- a/include/ClientImporting.py +++ b/include/ClientImporting.py @@ -374,7 +374,7 @@ class GalleryImport( HydrusSerialisable.SerialisableBase ): text = str( e ) - traceback.print_exc() + HydrusData.DebugPrint( traceback.format_exc() ) with self._lock: diff --git a/include/ClientNetworking.py b/include/ClientNetworking.py index 03a30c5c..30ab7179 100644 --- a/include/ClientNetworking.py +++ b/include/ClientNetworking.py @@ -256,7 +256,16 @@ class HTTPConnectionManager( object ): else: - if num_redirects_permitted == 0: raise Exception( 'Too many redirects!' ) + if num_redirects_permitted == 0: + + message = 'Too many redirects!' + message += os.linesep + message += 'Location was: ' + HydrusData.ToUnicode( location ) + ' and path and query was ' + path_and_query + '.' + message += os.linesep + message += 'Redirect info was: ' + HydrusData.ToUnicode( redirect_info ) + + raise Exception( message ) + ( new_method, new_url ) = redirect_info diff --git a/include/ClientRendering.py b/include/ClientRendering.py index 1d4cbd5e..d195ca10 100644 --- a/include/ClientRendering.py +++ b/include/ClientRendering.py @@ -175,6 +175,10 @@ class RasterContainerVideo( RasterContainer ): RasterContainer.__init__( self, media, target_resolution ) + self._init_position = init_position + + self._initialised = False + self._frames = {} self._buffer_start_index = -1 self._buffer_end_index = -1 @@ -208,24 +212,6 @@ class RasterContainerVideo( RasterContainer ): self._num_frames_backwards = frame_buffer_length * 2 / 3 self._num_frames_forwards = frame_buffer_length / 3 - hash = self._media.GetHash() - mime = self._media.GetMime() - - client_files_manager = HydrusGlobals.client_controller.GetClientFilesManager() - - path = client_files_manager.GetFilePath( hash, mime ) - - if self._media.GetMime() == HC.IMAGE_GIF: - - self._durations = HydrusImageHandling.GetGIFFrameDurations( self._path ) - - self._renderer = ClientVideoHandling.GIFRenderer( path, num_frames, self._target_resolution ) - - else: - - self._renderer = HydrusVideoHandling.VideoRendererFFMPEG( path, mime, duration, num_frames, self._target_resolution ) - - self._render_lock = threading.Lock() self._buffer_lock = threading.Lock() @@ -235,8 +221,6 @@ class RasterContainerVideo( RasterContainer ): self._rendered_first_frame = False self._rush_to_index = None - self.GetReadyForFrame( init_position ) - HydrusGlobals.client_controller.CallToThread( self.THREADRender ) @@ -286,6 +270,8 @@ class RasterContainerVideo( RasterContainer ): self._render_event.set() + self._initialised = True + @@ -307,6 +293,8 @@ class RasterContainerVideo( RasterContainer ): self._render_event.set() + self._initialised = True + @@ -318,12 +306,32 @@ class RasterContainerVideo( RasterContainer ): self._render_event.set() + self._initialised = True + def THREADRender( self ): + hash = self._media.GetHash() + mime = self._media.GetMime() + duration = self._media.GetDuration() num_frames = self._media.GetNumFrames() + client_files_manager = HydrusGlobals.client_controller.GetClientFilesManager() + + if self._media.GetMime() == HC.IMAGE_GIF: + + self._durations = HydrusImageHandling.GetGIFFrameDurations( self._path ) + + self._renderer = ClientVideoHandling.GIFRenderer( self._path, num_frames, self._target_resolution ) + + else: + + self._renderer = HydrusVideoHandling.VideoRendererFFMPEG( self._path, mime, duration, num_frames, self._target_resolution ) + + + self.GetReadyForFrame( self._init_position ) + while True: if self._stop or HydrusGlobals.view_shutdown: @@ -331,7 +339,10 @@ class RasterContainerVideo( RasterContainer ): return - if not self._rendered_first_frame or self._next_render_index != ( self._render_to_index + 1 ) % num_frames: + ready_to_render = self._initialised + frames_needed = not self._rendered_first_frame or self._next_render_index != ( self._render_to_index + 1 ) % num_frames + + if ready_to_render and frames_needed: with self._render_lock: @@ -517,6 +528,11 @@ class RasterContainerVideo( RasterContainer ): + def IsInitialised( self ): + + return self._initialised + + def IsScaled( self ): return self._zoom != 1.0 def Stop( self ): diff --git a/include/HydrusConstants.py b/include/HydrusConstants.py index f7eed65f..fb7e55f1 100755 --- a/include/HydrusConstants.py +++ b/include/HydrusConstants.py @@ -44,7 +44,7 @@ options = {} # Misc NETWORK_VERSION = 17 -SOFTWARE_VERSION = 227 +SOFTWARE_VERSION = 228 UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 ) @@ -289,19 +289,20 @@ VIDEO_APNG = 23 UNDETERMINED_PNG = 24 VIDEO_MPEG = 25 VIDEO_MOV = 26 +VIDEO_AVI = 27 APPLICATION_OCTET_STREAM = 100 APPLICATION_UNKNOWN = 101 -ALLOWED_MIMES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP, APPLICATION_FLASH, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG, APPLICATION_PDF, AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA, VIDEO_WMV ) -SEARCHABLE_MIMES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, APPLICATION_FLASH, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG, APPLICATION_PDF, AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA, VIDEO_WMV ) +ALLOWED_MIMES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP, APPLICATION_FLASH, VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG, APPLICATION_PDF, AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA, VIDEO_WMV ) +SEARCHABLE_MIMES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, APPLICATION_FLASH, VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG, APPLICATION_PDF, AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA, VIDEO_WMV ) IMAGES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP ) AUDIO = ( AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA ) -VIDEO = ( VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_WMV, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG ) +VIDEO = ( VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_WMV, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG ) -NATIVE_VIDEO = ( VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_WMV, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG ) +NATIVE_VIDEO = ( VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_WMV, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG ) APPLICATIONS = ( APPLICATION_FLASH, APPLICATION_PDF, APPLICATION_ZIP ) @@ -309,7 +310,7 @@ NOISY_MIMES = tuple( [ APPLICATION_FLASH ] + list( AUDIO ) + list( VIDEO ) ) ARCHIVES = ( APPLICATION_ZIP, APPLICATION_HYDRUS_ENCRYPTED_ZIP ) -MIMES_WITH_THUMBNAILS = ( APPLICATION_FLASH, IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_WMV, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG ) +MIMES_WITH_THUMBNAILS = ( APPLICATION_FLASH, IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP, VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_WMV, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG ) # mp3 header is complicated @@ -340,6 +341,7 @@ mime_enum_lookup[ 'audio/flac' ] = AUDIO_FLAC mime_enum_lookup[ 'audio/x-ms-wma' ] = AUDIO_WMA mime_enum_lookup[ 'text/html' ] = TEXT_HTML mime_enum_lookup[ 'video/png' ] = VIDEO_APNG +mime_enum_lookup[ 'video/x-msvideo' ] = VIDEO_AVI mime_enum_lookup[ 'video/x-flv' ] = VIDEO_FLV mime_enum_lookup[ 'video/quicktime' ] = VIDEO_MOV mime_enum_lookup[ 'video/mp4' ] = VIDEO_MP4 @@ -374,6 +376,7 @@ mime_string_lookup[ AUDIO_WMA ] = 'audio/x-ms-wma' mime_string_lookup[ AUDIO ] = 'audio' mime_string_lookup[ TEXT_HTML ] = 'text/html' mime_string_lookup[ VIDEO_APNG ] = 'video/png' +mime_string_lookup[ VIDEO_AVI ] = 'video/x-msvideo' mime_string_lookup[ VIDEO_FLV ] = 'video/x-flv' mime_string_lookup[ VIDEO_MOV ] = 'video/quicktime' mime_string_lookup[ VIDEO_MP4 ] = 'video/mp4' @@ -406,6 +409,7 @@ mime_ext_lookup[ AUDIO_FLAC ] = '.flac' mime_ext_lookup[ AUDIO_WMA ] = '.wma' mime_ext_lookup[ TEXT_HTML ] = '.html' mime_ext_lookup[ VIDEO_APNG ] = '.png' +mime_ext_lookup[ VIDEO_AVI ] = '.avi' mime_ext_lookup[ VIDEO_FLV ] = '.flv' mime_ext_lookup[ VIDEO_MOV ] = '.mov' mime_ext_lookup[ VIDEO_MP4 ] = '.mp4' diff --git a/include/HydrusController.py b/include/HydrusController.py index ffa0ea66..c6b849b4 100644 --- a/include/HydrusController.py +++ b/include/HydrusController.py @@ -19,18 +19,13 @@ class HydrusController( object ): pubsub_binding_errors_to_ignore = [] - def __init__( self, db_dir ): + def __init__( self, db_dir, no_daemons, no_wal ): HydrusGlobals.controller = self self._db_dir = db_dir - - self._db = None - - self._no_daemons = False - self._no_wal = False - - self._InitArgsBools() + self._no_daemons = no_daemons + self._no_wal = no_wal self._no_wal_path = os.path.join( self._db_dir, 'no-wal' ) @@ -39,6 +34,8 @@ class HydrusController( object ): self._no_wal = True + self._db = None + self._model_shutdown = False self._view_shutdown = False @@ -84,29 +81,6 @@ class HydrusController( object ): return call_to_thread - def _InitArgsBools( self ): - - args = sys.argv[1:] - - for arg in args: - - while arg.startswith( '-' ): - - arg = arg[ 1: ] - - - if arg in ( 'no-daemon', 'no-daemons' ): - - self._no_daemons = True - - - if arg == 'no-wal': - - self._no_wal = True - - - - def _InitDB( self ): raise NotImplementedError() @@ -190,9 +164,20 @@ class HydrusController( object ): - def GetCache( self, name ): return self._caches[ name ] + def GetCache( self, name ): + + return self._caches[ name ] + - def GetManager( self, name ): return self._managers[ name ] + def GetDBDir( self ): + + return self._db_dir + + + def GetManager( self, name ): + + return self._managers[ name ] + def GoodTimeToDoBackgroundWork( self ): diff --git a/include/HydrusDB.py b/include/HydrusDB.py index 083887c2..7d994e59 100644 --- a/include/HydrusDB.py +++ b/include/HydrusDB.py @@ -403,7 +403,7 @@ class HydrusDB( object ): except sqlite3.OperationalError: - traceback.print_exc() + HydrusData.DebugPrint( traceback.format_exc() ) def create_no_wal_file(): @@ -535,11 +535,6 @@ class HydrusDB( object ): return self._currently_doing_job - def GetDBDir( self ): - - return self._db_dir - - def IsDBUpdated( self ): return self._is_db_updated diff --git a/include/HydrusData.py b/include/HydrusData.py index 6be09bbe..269945e6 100644 --- a/include/HydrusData.py +++ b/include/HydrusData.py @@ -186,6 +186,29 @@ def ConvertIntToPixels( i ): elif i == 1000000: return 'megapixels' else: return 'megapixels' +def ConvertIntToPrettyOrdinalString( num ): + + remainder = num % 10 + + if remainder == 1: + + ordinal = 'st' + + elif remainder == 2: + + ordinal = 'nd' + + elif remainder == 3: + + ordinal = 'rd' + + else: + + ordinal = 'th' + + + return ConvertIntToPrettyString( num ) + ordinal + def ConvertIntToPrettyString( num ): # don't feed this a unicode string u'%d'--locale can't handle it diff --git a/include/HydrusFileHandling.py b/include/HydrusFileHandling.py index 12beb6be..9c02feb5 100644 --- a/include/HydrusFileHandling.py +++ b/include/HydrusFileHandling.py @@ -40,6 +40,7 @@ header_and_mime = [ ( 4, 'ftypMSNV', HC.VIDEO_MP4 ), ( 4, 'ftypqt', HC.VIDEO_MOV ), ( 0, 'fLaC', HC.AUDIO_FLAC ), + ( 8, 'AVI ', HC.VIDEO_AVI ), ( 0, '\x30\x26\xB2\x75\x8E\x66\xCF\x11\xA6\xD9\x00\xAA\x00\x62\xCE\x6C', HC.UNDETERMINED_WM ) ] @@ -175,7 +176,7 @@ def GetFileInfo( path ): ( ( width, height ), duration, num_frames ) = HydrusFlashHandling.GetFlashProperties( path ) - elif mime in ( HC.VIDEO_FLV, HC.VIDEO_WMV, HC.VIDEO_MOV, HC.VIDEO_MP4, HC.VIDEO_MKV, HC.VIDEO_WEBM, HC.VIDEO_MPEG ): + elif mime in ( HC.VIDEO_AVI, HC.VIDEO_FLV, HC.VIDEO_WMV, HC.VIDEO_MOV, HC.VIDEO_MP4, HC.VIDEO_MKV, HC.VIDEO_WEBM, HC.VIDEO_MPEG ): ( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetFFMPEGVideoProperties( path ) diff --git a/include/HydrusParsing.py b/include/HydrusParsing.py index 655b4d4a..270153cf 100644 --- a/include/HydrusParsing.py +++ b/include/HydrusParsing.py @@ -2,6 +2,7 @@ import bs4 import HydrusConstants as HC import HydrusData import HydrusSerialisable +import os def ConvertParsableContentToPrettyString( parsable_content ): @@ -11,9 +12,32 @@ def ConvertParsableContentToPrettyString( parsable_content ): else: - # make this prettier + pretty_strings = [] - return ', '.join( parsable_content ) + content_type_to_additional_infos = HydrusData.BuildKeyToSetDict( parsable_content ) + + for ( content_type, additional_infos ) in content_type_to_additional_infos.items(): + + if content_type == HC.CONTENT_TYPE_MAPPINGS: + + namespaces = [ namespace for namespace in additional_infos if namespace != '' ] + + if '' in additional_infos: + + namespaces.append( 'unnamespaced' ) + + + pretty_strings.append( 'tags: ' + ', '.join( namespaces ) ) + + if content_type == HC.CONTENT_TYPE_RATINGS: + + # I assume additional_info will have star info or whatever + + pass + + + + return ', '.join( pretty_strings ) def RenderTagRule( ( name, attrs, index ) ): @@ -55,6 +79,8 @@ class ParseFormulaHTML( HydrusSerialisable.SerialisableBase ): self._content_rule = content_rule + # I need extra rules here for chopping stuff off the beginning or end and appending or prepending strings + def _GetSerialisableInfo( self ): @@ -124,6 +150,51 @@ class ParseFormulaHTML( HydrusSerialisable.SerialisableBase ): return contents + def ToPrettyMultilineString( self ): + + pretty_strings = [] + + for ( name, attrs, index ) in self._tag_rules: + + s = '' + + if index is None: + + s += 'get every ' + + else: + + num = index + 1 + + s += 'get the ' + HydrusData.ConvertIntToPrettyOrdinalString( num ) + ' ' + + + s += '<' + name + '> tag' + + if len( attrs ) > 0: + + s += 'with attributes ' + ', '.join( key + '=' + value for ( key, value ) in attrs.items() ) + + + pretty_strings.append( s ) + + + if self._content_rule is None: + + pretty_strings.append( 'get the text content of those tags' ) + + else: + + pretty_strings.append( 'get the ' + self._content_rule + ' attribute of those tags' ) + + + separator = os.linesep + 'and then ' + + pretty_multiline_string = separator.join( pretty_strings ) + + return pretty_multiline_string + + def ToTuple( self ): return ( self._tag_rules, self._content_rule ) @@ -138,6 +209,29 @@ class ParseNodeContent( HydrusSerialisable.SerialisableBase ): def __init__( self, name = None, content_type = None, formula = None, additional_info = None ): + if name is None: + + name = '' + + + if content_type is None: + + content_type = HC.CONTENT_TYPE_MAPPINGS + + + if formula is None: + + formula = ParseFormulaHTML() + + + if additional_info is None: + + if content_type == HC.CONTENT_TYPE_MAPPINGS: + + additional_info = '' + + + self._name = name self._content_type = content_type self._formula = formula @@ -160,7 +254,7 @@ class ParseNodeContent( HydrusSerialisable.SerialisableBase ): def GetParsableContent( self ): - return ( self._name, self._content_type ) + return [ ( self._content_type, self._additional_info ) ] def Parse( self, data, referral_url, desired_content ): @@ -178,9 +272,14 @@ class ParseNodeContent( HydrusSerialisable.SerialisableBase ): return [ ( self._name, self._content_type, parsed_text, self._additional_info ) for parsed_text in parsed_texts ] - def SetChildren( self, children ): + def ToPrettyStrings( self ): - self._children = children + return ( self._name, 'content', ConvertParsableContentToPrettyString( self.GetParsableContent() ) ) + + + def ToTuple( self ): + + return ( self._name, self._content_type, self._formula, self._additional_info ) HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_PARSE_NODE_CONTENT ] = ParseNodeContent @@ -190,11 +289,26 @@ class ParseNodeContentLink( HydrusSerialisable.SerialisableBase ): SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_PARSE_NODE_CONTENT_LINK SERIALISABLE_VERSION = 1 - def __init__( self, formula = None, children = None, description = None ): + def __init__( self, name = None, formula = None, children = None ): + if name is None: + + name = '' + + + if formula is None: + + formula = ParseFormulaHTML() + + + if children is None: + + children = [] + + + self._name = name self._formula = formula self._children = children - self._description = description def _GetSerialisableInfo( self ): @@ -202,12 +316,12 @@ class ParseNodeContentLink( HydrusSerialisable.SerialisableBase ): serialisable_formula = self._formula.GetSerialisableTuple() serialisable_children = [ child.GetSerialisableTuple() for child in self._children ] - return ( serialisable_formula, serialisable_children, self._description ) + return ( self._name, serialisable_formula, serialisable_children ) def _InitialiseFromSerialisableInfo( self, serialisable_info ): - ( serialisable_formula, serialisable_children, self._description ) = serialisable_info + ( self._name, serialisable_formula, serialisable_children ) = serialisable_info self._formula = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_formula ) self._children = [ HydrusSerialisable.CreateFromSerialisableTuple( serialisable_child ) for serialisable_child in serialisable_children ] @@ -249,9 +363,9 @@ class ParseNodeContentLink( HydrusSerialisable.SerialisableBase ): return content - def SetChildren( self, children ): + def ToPrettyStrings( self ): - self._children = children + return ( self._name, 'link', ConvertParsableContentToPrettyString( self.GetParsableContent() ) ) HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_PARSE_NODE_CONTENT_LINK ] = ParseNodeContentLink @@ -336,6 +450,16 @@ class ParseRootFileLookup( HydrusSerialisable.SerialisableBaseNamed ): data = self.FetchData( url, file_identifier ) + return self.Parse( data, url, desired_content ) + + + def GetFileIdentifierInfo( self ): + + return ( self._file_identifier_type, self._file_identifier_encoding ) + + + def Parse( self, data, url, desired_content ): + content = [] for child in self._children: @@ -348,11 +472,6 @@ class ParseRootFileLookup( HydrusSerialisable.SerialisableBaseNamed ): return content - def GetFileIdentifierInfo( self ): - - return ( self._file_identifier_type, self._file_identifier_encoding ) - - def SetChildren( self, children ): self._children = children @@ -360,7 +479,7 @@ class ParseRootFileLookup( HydrusSerialisable.SerialisableBaseNamed ): def ToPrettyStrings( self ): - return ( self._name, HC.query_type_string_lookup[ self._query_type ], 'File Lookup returning ' + ConvertParsableContentToPrettyString( self.GetParsableContent() ) ) + return ( self._name, HC.query_type_string_lookup[ self._query_type ], 'File Lookup', ConvertParsableContentToPrettyString( self.GetParsableContent() ) ) def ToTuple( self ): diff --git a/include/HydrusPaths.py b/include/HydrusPaths.py index 9c43f8d2..cd79f828 100644 --- a/include/HydrusPaths.py +++ b/include/HydrusPaths.py @@ -175,8 +175,17 @@ def DeletePath( path ): except Exception as e: - HydrusData.ShowText( 'Trying to delete ' + path + ' caused the following error:' ) - HydrusData.ShowException( e ) + if 'Error 32' in str( e ): + + # file in use by another process + + HydrusData.DebugPrint( 'Trying to delete ' + path + ' failed because it was in use by another process.' ) + + else: + + HydrusData.ShowText( 'Trying to delete ' + path + ' caused the following error:' ) + HydrusData.ShowException( e ) + @@ -506,7 +515,8 @@ def RecyclePath( path ): except: HydrusData.Print( 'Trying to prepare ' + path + ' for recycling created this error:' ) - traceback.print_exc() + + HydrusData.DebugPrint( traceback.format_exc() ) return @@ -524,7 +534,8 @@ def RecyclePath( path ): except: HydrusData.Print( 'Trying to recycle ' + path + ' created this error:' ) - traceback.print_exc() + + HydrusData.DebugPrint( traceback.format_exc() ) HydrusData.Print( 'It has been fully deleted instead.' ) diff --git a/include/ServerController.py b/include/ServerController.py index 542099eb..a419580f 100755 --- a/include/ServerController.py +++ b/include/ServerController.py @@ -155,9 +155,9 @@ def ShutdownSiblingInstance( db_dir ): class Controller( HydrusController.HydrusController ): - def __init__( self, db_dir ): + def __init__( self, db_dir, no_daemons, no_wal ): - HydrusController.HydrusController.__init__( self, db_dir ) + HydrusController.HydrusController.__init__( self, db_dir, no_daemons, no_wal ) HydrusGlobals.server_controller = self diff --git a/server.py b/server.py index 3e14ec97..cb299519 100644 --- a/server.py +++ b/server.py @@ -29,15 +29,21 @@ try: from include import HydrusLogger import traceback + # + import argparse argparser = argparse.ArgumentParser( description = 'hydrus network server' ) argparser.add_argument( 'action', default = 'start', nargs = '?', choices = [ 'start', 'stop', 'restart' ], help = 'either start this server (default), or stop an existing server, or both' ) argparser.add_argument( '-d', '--db_dir', help = 'set an external db location' ) + argparser.add_argument( '--no_daemons', action='store_true', help = 'run without background daemons' ) + argparser.add_argument( '--no_wal', action='store_true', help = 'run without WAL db journalling' ) result = argparser.parse_args() + action = result.action + if result.db_dir is None: db_dir = os.path.join( HC.BASE_DIR, 'db' ) @@ -47,6 +53,8 @@ try: db_dir = result.db_dir + db_dir = HydrusPaths.ConvertPortablePathToAbsPath( db_dir ) + try: HydrusPaths.MakeSureDirectoryExists( db_dir ) @@ -56,7 +64,12 @@ try: raise Exception( 'Could not ensure db path ' + db_dir + ' exists! Check the location is correct and that you have permission to write to it!' ) - action = ServerController.ProcessStartingAction( db_dir, result.action ) + no_daemons = result.no_daemons + no_wal = result.no_wal + + # + + action = ServerController.ProcessStartingAction( db_dir, action ) log_path = os.path.join( db_dir, 'server.log' ) @@ -75,7 +88,7 @@ try: threading.Thread( target = reactor.run, kwargs = { 'installSignalHandlers' : 0 } ).start() - controller = ServerController.Controller( db_dir ) + controller = ServerController.Controller( db_dir, no_daemons, no_wal ) controller.Run() diff --git a/test.py b/test.py index 31cec11e..04c601b2 100644 --- a/test.py +++ b/test.py @@ -338,7 +338,7 @@ if __name__ == '__main__': import traceback - traceback.print_exc() + HydrusData.DebugPrint( traceback.format_exc() ) finally: @@ -357,7 +357,7 @@ if __name__ == '__main__': import traceback - traceback.print_exc() + HydrusData.DebugPrint( traceback.format_exc() ) finally: