changelog
-
+
version 271
+ - fixed an issue where the notebook 'motion' event was being consumed, disallowing the regular OS 'highlighting' on tab mouseover +
- fixed page tab drag and drop crash for Linux +
- fixed page tab drag and drop coordinate calculation for OS X +
- page tab drag and drop should now work for all platforms +
- new 'page of pages' pages now start with a blank file search page +
- fixed several issues with row display in the new listctrl +
- added a # index column to the file import status window, so you can resort the 'to be imported' order reliably +
- fixed an issue where the 'delete_file' application command was not firing on an appropriately mapped shortcut +
- fixed an issue where automatic bandwidth override would only fire if its page was in view. it now occurs at all times +
- fixed an issue where the file import status frame, as launched from manage subscriptions, would sometimes be non-responsive (typically for Linux) +
- fixed the 'pattern shortcuts' button on the export files dialog, which was misbehaving for certain choices for some window managers +
- fixed an unusual error where middle-clicking to create a new page while a menu was open would lock the application from receiving new key/mouse events until program focus was lost, wew lad +
- updated some old network code to the new engine +
- added some delay to network connection error reattempts so that instant fails due to local network disconnect (e.g. while computer is waking from sleep) do not consume all the reattempts too quickly +
- greatly simplified low-time-delta bandwidth testing and reporting, improving essential accuracy and overall feel +
- gallery download errors will now print additional info to the log--let's see if we can gather more info on the gelbooru 404s +
- wrote a patch for users of 2.X opencv who were running from source, who were unable to boot +
- due to hardware failure, had to make a new Linux build computer. some libraries are updated, but build _should_ be the same--let me know if not +
- updated to opencv 3.3 for all releases +
- misc old menu code updates +
version 270
- ?added page tab drag and drop (windows only, have to iron out critical bugs for linux/os x). +
- added page tab drag and drop (windows only, have to iron out critical bugs for linux/os x).
- dragging onto the middle of a normal tab will put the source tab there
- dragging onto the edge of a tab will try to insert the tab there
- dragging onto the middle of a page of pages tab will insert the tab onto the end of that page's list diff --git a/help/running_from_source.html b/help/running_from_source.html index e4486069..a2db0f1c 100755 --- a/help/running_from_source.html +++ b/help/running_from_source.html @@ -11,16 +11,15 @@
- pip install beautifulsoup4 hsaudiotag lxml lz4 nose numpy pafy Pillow psutil pycrypto PyOpenSSL PyPDF2 PySocks python-potr PyYAML requests Send2Trash service_identity twisted +
- pip install beautifulsoup4 hsaudiotag lxml lz4 nose numpy opencv-python pafy Pillow psutil pycrypto PyOpenSSL PyPDF2 PySocks python-potr PyYAML requests Send2Trash service_identity twisted youtube-dl
- sudo apt-get install python-opencv (or you can build a newer version yourself, as here)
- sudo apt-get install python-wxgtk3.0
- brew tap homebrew/homebrew-science
- brew install opencv diff --git a/include/ClientDownloading.py b/include/ClientDownloading.py index df185673..93158a6f 100644 --- a/include/ClientDownloading.py +++ b/include/ClientDownloading.py @@ -155,9 +155,27 @@ def GetImageboardFileURL( thread_url, filename, ext ): html_url = 'https://8ch.net/' + board + '/res/' + thread_id + '.html' - response = ClientNetworking.RequestsGet( html_url ) + network_job = ClientNetworking.NetworkJob( 'GET', html_url ) - thread_html = response.content + network_job.OverrideBandwidth() + + HG.client_controller.network_engine.AddJob( network_job ) + + while not network_job.IsDone(): + + time.sleep( 0.1 ) + + + if HG.view_shutdown: + + raise HydrusExceptions.ShutdownException() + + elif network_job.HasError(): + + raise network_job.GetErrorException() + + + thread_html = network_job.GetContent() soup = GetSoup( thread_html ) @@ -539,6 +557,9 @@ class Gallery( object ): e = network_job.GetErrorException() + HydrusData.Print( 'The url ' + url + ' gave the following problem:' ) + HydrusData.PrintException( e ) + raise e elif network_job.IsCancelled(): @@ -756,13 +777,7 @@ class GalleryBooru( Gallery ): for bad_url in bad_urls: - # turns out the garbage after the redirect is the redirect in base64, so let's not waste time doing this - - #url = ClientNetworking.RequestsGetRedirectURL( bad_url, session ) - # - #urls.append( url ) - # - #time.sleep( 0.5 ) + # the garbage after the redirect.php is the redirect in base64 # https://gelbooru.com/redirect.php?s=Ly9nZWxib29ydS5jb20vaW5kZXgucGhwP3BhZ2U9cG9zdCZzPXZpZXcmaWQ9MzY5NDEyMg== @@ -783,11 +798,9 @@ class GalleryBooru( Gallery ): HydrusData.ShowText( 'gelbooru parsing problem!' ) HydrusData.ShowException( e ) - url = ClientNetworking.RequestsGetRedirectURL( bad_url, session ) + time.sleep( 2 ) - urls.append( url ) - - time.sleep( 0.5 ) + break else: diff --git a/include/ClientGUI.py b/include/ClientGUI.py index a4936821..91ea1bee 100755 --- a/include/ClientGUI.py +++ b/include/ClientGUI.py @@ -2824,14 +2824,19 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p def EventFrameNewPage( self, event ): - screen_position = self.ClientToScreen( event.GetPosition() ) + if self._controller.MenuIsOpen(): + + return + + + screen_position = wx.GetMousePosition() self._notebook.EventNewPageFromScreenPosition( screen_position ) def EventFrameNotebookMenu( self, event ): - screen_position = self.ClientToScreen( event.GetPosition() ) + screen_position = wx.GetMousePosition() self._notebook.EventMenuFromScreenPosition( screen_position ) @@ -2848,7 +2853,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p bandwidth_status = HydrusData.ConvertIntToBytes( usage_since_boot ) - current_usage = global_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 1 ) + current_usage = global_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 1, for_user = True ) if current_usage > 0: @@ -3343,7 +3348,7 @@ class FrameSplash( wx.Frame ): ( old_x, old_y ) = self._last_drag_coordinates - ( x, y ) = event.GetPosition() + ( x, y ) = wx.GetMousePosition() ( delta_x, delta_y ) = ( x - old_x, y - old_y ) @@ -3361,7 +3366,7 @@ class FrameSplash( wx.Frame ): def EventDragBegin( self, event ): - self._last_drag_coordinates = event.GetPosition() + self._last_drag_coordinates = wx.GetMousePosition() event.Skip() diff --git a/include/ClientGUICanvas.py b/include/ClientGUICanvas.py index 193b6c6d..41948fc8 100755 --- a/include/ClientGUICanvas.py +++ b/include/ClientGUICanvas.py @@ -1673,6 +1673,10 @@ class Canvas( wx.Window ): self._Archive() + elif action == 'delete_file': + + self._Delete() + elif action == 'inbox_file': self._Inbox() @@ -4717,7 +4721,7 @@ class CanvasMediaListBrowser( CanvasMediaListNavigable ): for line in self._current_media.GetPrettyInfoLines(): - menu.Append( CC.ID_NULL, line ) + ClientGUIMenus.AppendMenuLabel( menu, line ) ClientGUIMenus.AppendSeparator( menu ) @@ -4772,12 +4776,12 @@ class CanvasMediaListBrowser( CanvasMediaListNavigable ): if CC.LOCAL_FILE_SERVICE_KEY in locations_manager.GetCurrent(): - menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'delete_file', CC.LOCAL_FILE_SERVICE_KEY ), '&delete' ) + ClientGUIMenus.AppendMenuItem( self, menu, 'delete', 'Send this file to the trash.', self._Delete, CC.LOCAL_FILE_SERVICE_KEY ) elif CC.TRASH_SERVICE_KEY in locations_manager.GetCurrent(): - menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'delete_file', CC.TRASH_SERVICE_KEY ), '&delete from trash now' ) - menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'undelete' ), '&undelete' ) + ClientGUIMenus.AppendMenuItem( self, menu, 'delete from trash now', 'Delete this file immediately. This cannot be undone.', self._Delete, CC.TRASH_SERVICE_KEY ) + ClientGUIMenus.AppendMenuItem( self, menu, 'undelete', 'Take this file out of the trash, returning it to its original file service.', self._Undelete ) ClientGUIMenus.AppendSeparator( menu ) diff --git a/include/ClientGUICommon.py b/include/ClientGUICommon.py index 8dacada6..22b637c3 100755 --- a/include/ClientGUICommon.py +++ b/include/ClientGUICommon.py @@ -756,57 +756,32 @@ class ChoiceSort( wx.Panel ): self._UpdateAscLabels() -class ExportPatternButton( wx.Button ): - - ID_HASH = 0 - ID_TAGS = 1 - ID_NN_TAGS = 2 - ID_NAMESPACE = 3 - ID_TAG = 4 +class ExportPatternButton( BetterButton ): def __init__( self, parent ): - wx.Button.__init__( self, parent, label = 'pattern shortcuts' ) - - self.Bind( wx.EVT_BUTTON, self.EventButton ) - self.Bind( wx.EVT_MENU, self.EventMenu ) + BetterButton.__init__( self, parent, 'pattern shortcuts', self._Hit ) - def EventMenu( self, event ): - - id = event.GetId() - - phrase = None - - if id == self.ID_HASH: phrase = '{hash}' - if id == self.ID_TAGS: phrase = '{tags}' - if id == self.ID_NN_TAGS: phrase = '{nn tags}' - if id == self.ID_NAMESPACE: phrase = u'[\u2026]' - if id == self.ID_TAG: phrase = u'(\u2026)' - else: event.Skip() - - if phrase is not None: HG.client_controller.pub( 'clipboard', 'text', phrase ) - - - def EventButton( self, event ): + def _Hit( self ): menu = wx.Menu() - menu.Append( -1, 'click on a phrase to copy to clipboard' ) + ClientGUIMenus.AppendMenuLabel( menu, 'click on a phrase to copy to clipboard' ) ClientGUIMenus.AppendSeparator( menu ) - menu.Append( self.ID_HASH, 'the file\'s hash - {hash}' ) - menu.Append( self.ID_TAGS, 'all the file\'s tags - {tags}' ) - menu.Append( self.ID_NN_TAGS, 'all the file\'s non-namespaced tags - {nn tags}' ) + ClientGUIMenus.AppendMenuItem( self, menu, 'the file\'s hash - {hash}', 'copy "{hash}" to the clipboard', HG.client_controller.pub, 'clipboard', 'text', '{hash}' ) + ClientGUIMenus.AppendMenuItem( self, menu, 'all the file\'s tags - {tags}', 'copy "{tags}" to the clipboard', HG.client_controller.pub, 'clipboard', 'text', '{tags}' ) + ClientGUIMenus.AppendMenuItem( self, menu, 'all the file\'s non-namespaced tags - {nn tags}', 'copy "{nn tags}" to the clipboard', HG.client_controller.pub, 'clipboard', 'text', '{nn tags}' ) ClientGUIMenus.AppendSeparator( menu ) - menu.Append( self.ID_NAMESPACE, u'all instances of a particular namespace - [\u2026]' ) + ClientGUIMenus.AppendMenuItem( self, menu, u'all instances of a particular namespace - [\u2026]', u'copy "[\u2026]" to the clipboard', HG.client_controller.pub, 'clipboard', 'text', u'[\u2026]' ) ClientGUIMenus.AppendSeparator( menu ) - menu.Append( self.ID_TAG, u'a particular tag, if the file has it - (\u2026)' ) + ClientGUIMenus.AppendMenuItem( self, menu, u'a particular tag, if the file has it - (\u2026)', u'copy "(\u2026)" to the clipboard', HG.client_controller.pub, 'clipboard', 'text', u'(\u2026)' ) HG.client_controller.PopupMenu( self, menu ) @@ -2174,7 +2149,7 @@ class RatingNumericalCanvas( RatingNumerical ): class RegexButton( wx.Button ): - ID_REGEX_WHITESPACE = 0 + ID_REGEX_WHITESPACE = 9001 # temp fix, 0 is buggy ID_REGEX_NUMBER = 1 ID_REGEX_ALPHANUMERIC = 2 ID_REGEX_ANY = 3 @@ -2213,7 +2188,7 @@ class RegexButton( wx.Button ): menu = wx.Menu() - menu.Append( -1, 'click on a phrase to copy to clipboard' ) + ClientGUIMenus.AppendMenuLabel( menu, 'click on a phrase to copy to clipboard' ) ClientGUIMenus.AppendSeparator( menu ) diff --git a/include/ClientGUIControls.py b/include/ClientGUIControls.py index d75bd82b..6be06c43 100644 --- a/include/ClientGUIControls.py +++ b/include/ClientGUIControls.py @@ -414,6 +414,21 @@ class NetworkJobControl( wx.Panel ): self._update_timer = wx.Timer( self ) + def _OverrideBandwidthIfAppropriate( self ): + + if self._network_job is None or self._network_job.NoEngineYet(): + + return + + else: + + if self._auto_override_bandwidth_rules and HydrusData.TimeHasPassed( self._network_job.GetCreationTime() + 5 ): + + self._network_job.OverrideBandwidth() + + + + def _Update( self ): if self._network_job is None or self._network_job.NoEngineYet(): @@ -427,11 +442,6 @@ class NetworkJobControl( wx.Panel ): else: - if self._auto_override_bandwidth_rules and HydrusData.TimeHasPassed( self._network_job.GetCreationTime() + 5 ): - - self._network_job.OverrideBandwidth() - - if self._network_job.IsDone(): can_cancel = False @@ -561,6 +571,8 @@ class NetworkJobControl( wx.Panel ): def TIMEREventUpdate( self, event ): + self._OverrideBandwidthIfAppropriate() + if HG.client_controller.gui.IShouldRegularlyUpdate( self ): self._Update() diff --git a/include/ClientGUIListCtrl.py b/include/ClientGUIListCtrl.py index b814dc0a..9f14b299 100644 --- a/include/ClientGUIListCtrl.py +++ b/include/ClientGUIListCtrl.py @@ -547,7 +547,7 @@ class BetterListCtrl( wx.ListCtrl, ListCtrlAutoWidthMixin ): self.Bind( wx.EVT_LIST_COL_CLICK, self.EventColumnClick ) - def _AddDataInfo( self, data_info, select = False ): + def _AddDataInfo( self, data_info ): ( data, display_tuple, sort_tuple ) = data_info @@ -556,11 +556,6 @@ class BetterListCtrl( wx.ListCtrl, ListCtrlAutoWidthMixin ): self._indices_to_data_info[ index ] = data_info self._data_to_indices[ data ] = index - if select: - - self.Select( index ) - - def _GetSelected( self ): @@ -580,12 +575,14 @@ class BetterListCtrl( wx.ListCtrl, ListCtrlAutoWidthMixin ): def _RecalculateIndicesAfterDelete( self ): - sorted_data_info = self._SortDataInfo() + indices_and_data_info = list( self._indices_to_data_info.items() ) + + indices_and_data_info.sort() self._indices_to_data_info = {} self._data_to_indices = {} - for ( index, data_info ) in enumerate( sorted_data_info ): + for ( index, ( old_index, data_info ) ) in enumerate( indices_and_data_info ): ( data, display_tuple, sort_tuple ) = data_info @@ -612,34 +609,53 @@ class BetterListCtrl( wx.ListCtrl, ListCtrlAutoWidthMixin ): def _SortAndRefreshRows( self ): - scroll_pos = self.GetScrollPos( wx.VERTICAL ) - selected_data = self.GetData( only_selected = True ) - self.DeleteAllItems() + selected_indices = self._GetSelected() + + for selected_index in selected_indices: + + self.Select( selected_index, False ) + sorted_data_info = self._SortDataInfo() self._indices_to_data_info = {} self._data_to_indices = {} - for data_info in sorted_data_info: + for ( index, data_info ) in enumerate( sorted_data_info ): + + self._indices_to_data_info[ index ] = data_info ( data, display_tuple, sort_tuple ) = data_info - select = data in selected_data + self._data_to_indices[ data ] = index - self._AddDataInfo( data_info, select = select ) + self._UpdateRow( index, display_tuple ) + + if data in selected_data: + + self.Select( index ) + - - self.SetScrollPos( wx.VERTICAL, scroll_pos ) - def AddData( self, data, select = False ): + def _UpdateRow( self, index, display_tuple ): - ( display_tuple, sort_tuple ) = self._data_to_tuples_func( data ) + for ( column_index, value ) in enumerate( display_tuple ): + + self.SetStringItem( index, column_index, value ) + - self._AddDataInfo( ( data, display_tuple, sort_tuple ), select = select ) + + def AddDatas( self, datas ): + + for data in datas: + + ( display_tuple, sort_tuple ) = self._data_to_tuples_func( data ) + + self._AddDataInfo( ( data, display_tuple, sort_tuple ) ) + def DeleteDatas( self, datas ): @@ -659,6 +675,7 @@ class BetterListCtrl( wx.ListCtrl, ListCtrlAutoWidthMixin ): self._RecalculateIndicesAfterDelete() + def DeleteSelected( self ): @@ -820,10 +837,7 @@ class BetterListCtrl( wx.ListCtrl, ListCtrlAutoWidthMixin ): if len( datas_to_add ) > 0: - for data in datas_to_add: - - self.AddData( data ) - + self.AddDatas( datas_to_add ) self._SortAndRefreshRows() @@ -858,10 +872,7 @@ class BetterListCtrl( wx.ListCtrl, ListCtrlAutoWidthMixin ): self._indices_to_data_info[ index ] = data_info - for ( column_index, value ) in enumerate( display_tuple ): - - self.SetStringItem( index, column_index, value ) - + self._UpdateRow( index, display_tuple ) diff --git a/include/ClientGUIMedia.py b/include/ClientGUIMedia.py index 7e4d32eb..974cdf24 100755 --- a/include/ClientGUIMedia.py +++ b/include/ClientGUIMedia.py @@ -1034,6 +1034,10 @@ class MediaPanel( ClientMedia.ListeningMediaList, wx.ScrolledWindow ): self._Archive() + elif action == 'delete_file': + + self._Delete() + elif action == 'inbox_file': self._Inbox() @@ -2348,7 +2352,7 @@ class MediaPanelThumbnails( MediaPanel ): ( old_x, old_y ) = self._drag_init_coordinates - ( x, y ) = event.GetPosition() + ( x, y ) = wx.GetMousePosition() ( delta_x, delta_y ) = ( x - old_x, y - old_y ) @@ -2399,7 +2403,10 @@ class MediaPanelThumbnails( MediaPanel ): - def EventEraseBackground( self, event ): pass + def EventEraseBackground( self, event ): + + pass + def EventKeyDown( self, event ): @@ -2596,7 +2603,7 @@ class MediaPanelThumbnails( MediaPanel ): def EventSelection( self, event ): - self._drag_init_coordinates = event.GetPosition() + self._drag_init_coordinates = wx.GetMousePosition() self._HitMedia( self._GetThumbnailUnderMouse( event ), event.CmdDown(), event.ShiftDown() ) diff --git a/include/ClientGUIPages.py b/include/ClientGUIPages.py index cf707114..77ad412e 100755 --- a/include/ClientGUIPages.py +++ b/include/ClientGUIPages.py @@ -730,11 +730,7 @@ class PagesNotebook( wx.Notebook ): self._controller.sub( self, 'RefreshPageName', 'refresh_page_name' ) self._controller.sub( self, 'NotifyPageUnclosed', 'notify_page_unclosed' ) - if HC.PLATFORM_WINDOWS: - - self.Bind( wx.EVT_MOTION, self.EventDrag ) - - + self.Bind( wx.EVT_MOTION, self.EventDrag ) self.Bind( wx.EVT_LEFT_DOWN, self.EventLeftDown ) self.Bind( wx.EVT_LEFT_DCLICK, self.EventLeftDoubleClick ) self.Bind( wx.EVT_MIDDLE_DOWN, self.EventMiddleClick ) @@ -962,7 +958,21 @@ class PagesNotebook( wx.Notebook ): return self - if flags & wx.NB_HITTEST_NOWHERE and flags & wx.NB_HITTEST_ONPAGE: # not on a label but inside my client area + not_on_my_label = flags & wx.NB_HITTEST_NOWHERE + + if HC.PLATFORM_OSX: + + ( x, y ) = screen_position + ( child_x, child_y ) = current_page.GetPosition() + + on_child_notebook_somewhere = y > child_y # wew lad, OSX not delivering onpage maybe? + + else: + + on_child_notebook_somewhere = flags & wx.NB_HITTEST_ONPAGE + + + if not_on_my_label and on_child_notebook_somewhere: return current_page._GetNotebookFromScreenPosition( screen_position ) @@ -1212,7 +1222,7 @@ class PagesNotebook( wx.Notebook ): try: - page = self.NewPagesNotebook( name, forced_insertion_index ) + page = self.NewPagesNotebook( name, forced_insertion_index = forced_insertion_index, give_it_a_blank_page = False ) page.AppendSessionPageTuples( subpage_tuples ) @@ -1298,9 +1308,7 @@ class PagesNotebook( wx.Notebook ): if event.LeftIsDown() and self._potential_drag_page is not None: - drop_source = wx.DropSource( self ) - - data_object = wx.DataObjectComposite() + drop_source = wx.DropSource( self._controller.gui ) # @@ -1310,17 +1318,17 @@ class PagesNotebook( wx.Notebook ): hydrus_page_tab_data_object.SetData( data ) - data_object.Add( hydrus_page_tab_data_object, True ) - # - drop_source.SetData( data_object ) + drop_source.SetData( hydrus_page_tab_data_object ) drop_source.DoDragDrop() self._potential_drag_page = None + event.Skip() + def EventLeftDown( self, event ): @@ -1377,6 +1385,11 @@ class PagesNotebook( wx.Notebook ): def EventMiddleClick( self, event ): + if self._controller.MenuIsOpen(): + + return + + position = event.GetPosition() ( tab_index, flags ) = self.HitTest( position ) @@ -1753,13 +1766,13 @@ class PagesNotebook( wx.Notebook ): return self.NewPage( management_controller, initial_hashes = initial_hashes, on_deepest_notebook = on_deepest_notebook ) - def NewPagesNotebook( self, name = 'pages', forced_insertion_index = None, on_deepest_notebook = False ): + def NewPagesNotebook( self, name = 'pages', forced_insertion_index = None, on_deepest_notebook = False, give_it_a_blank_page = True ): current_page = self.GetCurrentPage() if on_deepest_notebook and isinstance( current_page, PagesNotebook ): - current_page.NewPagesNotebook( name = name, forced_insertion_index = forced_insertion_index, on_deepest_notebook = on_deepest_notebook ) + current_page.NewPagesNotebook( name = name, forced_insertion_index = forced_insertion_index, on_deepest_notebook = on_deepest_notebook, give_it_a_blank_page = give_it_a_blank_page ) return @@ -1793,6 +1806,11 @@ class PagesNotebook( wx.Notebook ): self._controller.pub( 'refresh_page_name', page.GetPageKey() ) + if give_it_a_blank_page: + + page.NewPageQuery( CC.LOCAL_FILE_SERVICE_KEY ) + + return page @@ -1854,9 +1872,24 @@ class PagesNotebook( wx.Notebook ): screen_position = wx.GetMousePosition() + if HC.PLATFORM_OSX: + + # idk why os x is giving problems here + # something about the coordinates of notebook stuff is all off, like the tab area is not included as part of the client area for HitTest calc, so all screentoclient calcs are wrong + # in the mouse events, going self.ClientToScreen( event.GetPosition() ) gives a different value to wx.GetMousePosition(), wew lad + # and yet other mouse->client comparisons are ok with other widgets, so this is presumably an OS X notebook issue + # this is a fuzzy fix for now, just filling in the different amounts I discovered with that + + ( x, y ) = screen_position + + screen_position = ( x + 10, y + 33 ) + + dest_notebook = self._GetNotebookFromScreenPosition( screen_position ) - ( x, y ) = dest_notebook.ScreenToClient( screen_position ) + position = dest_notebook.ScreenToClient( screen_position ) + + ( x, y ) = position ( tab_index, flags ) = dest_notebook.HitTest( ( x, y ) ) diff --git a/include/ClientGUIParsing.py b/include/ClientGUIParsing.py index b3ab314e..329261b5 100644 --- a/include/ClientGUIParsing.py +++ b/include/ClientGUIParsing.py @@ -13,11 +13,13 @@ import ClientSerialisable import ClientThreading import HydrusConstants as HC import HydrusData +import HydrusExceptions import HydrusGlobals as HG import HydrusSerialisable import HydrusTags import os import threading +import time import webbrowser import wx @@ -1141,28 +1143,45 @@ The formula should attempt to parse full or relative urls. If the url is relativ def EventTestFetchResult( self, event ): - try: + # this should be published to a job key panel or something so user can see it and cancel if needed + + network_job = ClientNetworking.NetworkJob( 'GET', self._my_example_url, referral_url = self._referral_url ) + + network_job.OverrideBandwidth() + + HG.client_controller.network_engine.AddJob( network_job ) + + while not network_job.IsDone(): - headers = { 'Referer' : self._referral_url } + time.sleep( 0.1 ) - response = ClientNetworking.RequestsGet( self._my_example_url, headers = headers ) + + if HG.view_shutdown: - example_data = response.text + raise HydrusExceptions.ShutdownException() - try: - - self._example_data.SetValue( example_data ) - - except UnicodeDecodeError: - - self._example_data.SetValue( 'The fetched data, which had length ' + HydrusData.ConvertIntToBytes( len( example_data ) ) + ', did not appear to be displayable text.' ) - - - except Exception as e: + elif network_job.HasError(): self._my_example_data.SetValue( 'fetch failed' ) - raise + raise network_job.GetErrorException() + + elif network_job.IsCancelled(): + + self._my_example_data.SetValue( 'fetch cancelled' ) + + return + + + example_data = network_job.GetContent() + + try: + + self._example_data.SetValue( example_data ) + + except UnicodeDecodeError: + + self._example_data.SetValue( 'The fetched data, which had length ' + HydrusData.ConvertIntToBytes( len( example_data ) ) + ', did not appear to be displayable text.' ) diff --git a/include/ClientGUIPopupMessages.py b/include/ClientGUIPopupMessages.py index ce042e6f..e328ee2f 100644 --- a/include/ClientGUIPopupMessages.py +++ b/include/ClientGUIPopupMessages.py @@ -686,6 +686,8 @@ class PopupMessageManager( wx.Frame ): self.Layout() + self.Refresh() + else: diff --git a/include/ClientGUIScrolledPanelsManagement.py b/include/ClientGUIScrolledPanelsManagement.py index 8e26fbfb..dd9cb5ec 100644 --- a/include/ClientGUIScrolledPanelsManagement.py +++ b/include/ClientGUIScrolledPanelsManagement.py @@ -4670,10 +4670,7 @@ class ManageSubscriptionsPanel( ClientGUIScrolledPanels.ManagePanel ): # - for subscription in subscriptions: - - self._subscriptions.AddData( subscription ) - + self._subscriptions.AddDatas( subscriptions ) # @@ -4822,7 +4819,7 @@ class ManageSubscriptionsPanel( ClientGUIScrolledPanels.ManagePanel ): subscription.SetNonDupeName( self._GetExistingNames() ) - self._subscriptions.AddData( subscription ) + self._subscriptions.AddDatas( ( subscription, ) ) else: @@ -4847,7 +4844,7 @@ class ManageSubscriptionsPanel( ClientGUIScrolledPanels.ManagePanel ): new_subscription.SetNonDupeName( self._GetExistingNames() ) - self._subscriptions.AddData( new_subscription ) + self._subscriptions.AddDatas( ( new_subscription, ) ) @@ -4894,7 +4891,7 @@ class ManageSubscriptionsPanel( ClientGUIScrolledPanels.ManagePanel ): dupe_subscription.SetNonDupeName( self._GetExistingNames() ) - self._subscriptions.AddData( dupe_subscription ) + self._subscriptions.AddDatas( ( dupe_subscription, ) ) @@ -4922,7 +4919,7 @@ class ManageSubscriptionsPanel( ClientGUIScrolledPanels.ManagePanel ): edited_subscription.SetNonDupeName( self._GetExistingNames() ) - self._subscriptions.AddData( edited_subscription ) + self._subscriptions.AddDatas( ( edited_subscription, ) ) elif result == wx.ID_CANCEL: diff --git a/include/ClientGUIScrolledPanelsReview.py b/include/ClientGUIScrolledPanelsReview.py index 2da0ba45..aa62a1b3 100644 --- a/include/ClientGUIScrolledPanelsReview.py +++ b/include/ClientGUIScrolledPanelsReview.py @@ -369,7 +369,7 @@ class ReviewAllBandwidthPanel( ClientGUIScrolledPanels.ReviewPanel ): sortable_network_context = ( network_context.context_type, network_context.context_data ) sortable_context_type = CC.network_context_type_string_lookup[ network_context.context_type ] - current_usage = bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 1 ) + current_usage = bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 1, for_user = True ) day_usage = bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 86400 ) month_usage = bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, None ) @@ -688,7 +688,7 @@ class ReviewNetworkContextBandwidthPanel( ClientGUIScrolledPanels.ReviewPanel ): def _Update( self ): - current_usage = self._bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 1 ) + current_usage = self._bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 1, for_user = True ) pretty_current_usage = 'current usage: ' + HydrusData.ConvertIntToBytes( current_usage ) + '/s' diff --git a/include/ClientGUISeedCache.py b/include/ClientGUISeedCache.py index ea0e00b8..28d8defd 100644 --- a/include/ClientGUISeedCache.py +++ b/include/ClientGUISeedCache.py @@ -24,7 +24,7 @@ class EditSeedCachePanel( ClientGUIScrolledPanels.EditPanel ): # add index control row here, hide it if needed and hook into showing/hiding and postsizechangedevent on seed add/remove - columns = [ ( 'source', -1 ), ( 'status', 12 ), ( 'added', 20 ), ( 'last modified', 20 ), ( 'note', 30 ) ] + columns = [ ( '#', 3 ), ( 'source', -1 ), ( 'status', 12 ), ( 'added', 20 ), ( 'last modified', 20 ), ( 'note', 30 ) ] self._list_ctrl = ClientGUIListCtrl.BetterListCtrl( self, 'seed_cache', 30, 30, columns, self._ConvertSeedToListCtrlTuples ) @@ -32,6 +32,8 @@ class EditSeedCachePanel( ClientGUIScrolledPanels.EditPanel ): self._AddSeeds( self._seed_cache.GetSeeds() ) + self._list_ctrl.Sort( 0 ) + # vbox = wx.BoxSizer( wx.VERTICAL ) @@ -50,25 +52,23 @@ class EditSeedCachePanel( ClientGUIScrolledPanels.EditPanel ): def _AddSeeds( self, seeds ): - for seed in seeds: - - self._list_ctrl.AddData( seed ) - + self._list_ctrl.AddDatas( seeds ) def _ConvertSeedToListCtrlTuples( self, seed ): sort_tuple = self._seed_cache.GetSeedInfo( seed ) - ( seed, status, added_timestamp, last_modified_timestamp, note ) = sort_tuple + ( seed_index, seed, status, added_timestamp, last_modified_timestamp, note ) = sort_tuple + pretty_seed_index = HydrusData.ConvertIntToPrettyString( seed_index ) pretty_seed = HydrusData.ToUnicode( seed ) pretty_status = CC.status_string_lookup[ status ] pretty_added = HydrusData.ConvertTimestampToPrettyAgo( added_timestamp ) pretty_modified = HydrusData.ConvertTimestampToPrettyAgo( last_modified_timestamp ) pretty_note = note.split( os.linesep )[0] - display_tuple = ( pretty_seed, pretty_status, pretty_added, pretty_modified, pretty_note ) + display_tuple = ( pretty_seed_index, pretty_seed, pretty_status, pretty_added, pretty_modified, pretty_note ) return ( display_tuple, sort_tuple ) @@ -79,7 +79,7 @@ class EditSeedCachePanel( ClientGUIScrolledPanels.EditPanel ): for seed in self._list_ctrl.GetData( only_selected = True ): - ( seed, status, added_timestamp, last_modified_timestamp, note ) = self._seed_cache.GetSeedInfo( seed ) + ( seed_index, seed, status, added_timestamp, last_modified_timestamp, note ) = self._seed_cache.GetSeedInfo( seed ) if note != '': @@ -266,14 +266,30 @@ class SeedCacheStatusControl( wx.Panel ): def _ShowSeedCacheFrame( self ): - title = 'file import status' - frame_key = 'file_import_status' + tlp = ClientGUICommon.GetTLP( self ) - frame = ClientGUITopLevelWindows.FrameThatTakesScrollablePanel( self, title, frame_key ) - - panel = EditSeedCachePanel( frame, self._controller, self._seed_cache ) - - frame.SetPanel( panel ) + if isinstance( tlp, wx.Dialog ): + + with ClientGUITopLevelWindows.DialogNullipotent( self, 'file import status' ) as dlg: + + panel = EditSeedCachePanel( dlg, self._controller, self._seed_cache ) + + dlg.SetPanel( panel ) + + dlg.ShowModal() + + + else: + + title = 'file import status' + frame_key = 'file_import_status' + + frame = ClientGUITopLevelWindows.FrameThatTakesScrollablePanel( self, title, frame_key ) + + panel = EditSeedCachePanel( frame, self._controller, self._seed_cache ) + + frame.SetPanel( panel ) + def _Update( self ): diff --git a/include/ClientImageHandling.py b/include/ClientImageHandling.py index 3b3d8b7b..f23fdc26 100644 --- a/include/ClientImageHandling.py +++ b/include/ClientImageHandling.py @@ -8,7 +8,10 @@ import HydrusGlobals as HG if cv2.__version__.startswith( '2' ): CV_IMREAD_FLAGS_SUPPORTS_ALPHA = cv2.CV_LOAD_IMAGE_UNCHANGED - CV_IMREAD_FLAGS_SUPPORTS_EXIF_REORIENTATION = cv2.CV_LOAD_IMAGE_ANYDEPTH | cv2.CV_LOAD_IMAGE_ANYCOLOR + CV_IMREAD_FLAGS_SUPPORTS_EXIF_REORIENTATION = CV_IMREAD_FLAGS_SUPPORTS_ALPHA + + # there's something wrong with these, but I don't have an easy test env for it atm + # CV_IMREAD_FLAGS_SUPPORTS_EXIF_REORIENTATION = cv2.CV_LOAD_IMAGE_ANYDEPTH | cv2.CV_LOAD_IMAGE_ANYCOLOR else: diff --git a/include/ClientImporting.py b/include/ClientImporting.py index 6512b87d..70b80a5a 100644 --- a/include/ClientImporting.py +++ b/include/ClientImporting.py @@ -37,13 +37,36 @@ def THREADDownloadURL( job_key, url, url_string ): try: - response = ClientNetworking.RequestsGet( url, stream = True ) + network_job = ClientNetworking.NetworkJob( 'GET', url, temp_path = temp_path ) - with open( temp_path, 'wb' ) as f: + network_job.OverrideBandwidth() + + HG.client_controller.network_engine.AddJob( network_job ) + + job_key.SetVariable( 'popup_network_job', network_job ) + + while not network_job.IsDone(): - ClientNetworking.StreamResponseToFile( job_key, response, f ) + time.sleep( 0.1 ) + if HG.view_shutdown: + + raise HydrusExceptions.ShutdownException() + + elif network_job.HasError(): + + job_key.Cancel() + + raise network_job.GetErrorException() + + elif network_job.IsCancelled(): + + return + + + job_key.DeleteVariable( 'popup_network_job' ) + job_key.SetVariable( 'popup_text_1', 'importing' ) file_import_job = FileImportJob( temp_path ) @@ -52,16 +75,6 @@ def THREADDownloadURL( job_key, url, url_string ): ( result, hash ) = client_files_manager.ImportFile( file_import_job ) - except HydrusExceptions.CancelledException: - - return - - except HydrusExceptions.NetworkException: - - job_key.Cancel() - - raise - finally: HydrusPaths.CleanUpTempPath( os_file_handle, temp_path ) @@ -115,24 +128,30 @@ def THREADDownloadURLs( job_key, urls, title ): try: - try: + network_job = ClientNetworking.NetworkJob( 'GET', url, temp_path = temp_path ) + + network_job.OverrideBandwidth() + + HG.client_controller.network_engine.AddJob( network_job ) + + job_key.SetVariable( 'popup_network_job', network_job ) + + while not network_job.IsDone(): - response = ClientNetworking.RequestsGet( url, stream = True ) + time.sleep( 0.1 ) - with open( temp_path, 'wb' ) as f: - - ClientNetworking.StreamResponseToFile( job_key, response, f ) - + + if HG.view_shutdown: - except HydrusExceptions.CancelledException: + raise HydrusExceptions.ShutdownException() - return + elif network_job.HasError(): - except HydrusExceptions.NetworkException: + raise network_job.GetErrorException() - job_key.Cancel() + elif network_job.IsCancelled(): - raise + break try: @@ -181,6 +200,8 @@ def THREADDownloadURLs( job_key, urls, title ): + job_key.DeleteVariable( 'popup_network_job' ) + text_components = [] if num_successful > 0: @@ -2318,6 +2339,8 @@ class SeedCache( HydrusSerialisable.SerialisableBase ): self._seeds_ordered = [] self._seeds_to_info = {} + self._seeds_to_indices = {} + self._seed_cache_key = HydrusData.GenerateKey() self._status_cache = None @@ -2356,9 +2379,10 @@ class SeedCache( HydrusSerialisable.SerialisableBase ): status = ', '.join( status_strings ) - total_processed = len( self._seeds_ordered ) - num_unknown total = len( self._seeds_ordered ) + total_processed = total - num_unknown + self._status_cache = ( status, ( total_processed, total ) ) self._dirty = False @@ -2366,6 +2390,8 @@ class SeedCache( HydrusSerialisable.SerialisableBase ): def _GetSeedTuple( self, seed ): + seed_index = self._seeds_to_indices[ seed ] + seed_info = self._seeds_to_info[ seed ] status = seed_info[ 'status' ] @@ -2373,7 +2399,7 @@ class SeedCache( HydrusSerialisable.SerialisableBase ): last_modified_timestamp = seed_info[ 'last_modified_timestamp' ] note = seed_info[ 'note' ] - return ( seed, status, added_timestamp, last_modified_timestamp, note ) + return ( seed_index, seed, status, added_timestamp, last_modified_timestamp, note ) def _GetSerialisableInfo( self ): @@ -2404,6 +2430,8 @@ class SeedCache( HydrusSerialisable.SerialisableBase ): self._seeds_to_info[ seed ] = seed_info + self._seeds_to_indices = { seed : index for ( index, seed ) in enumerate( self._seeds_ordered ) } + def _SetDirty( self ): @@ -2575,6 +2603,8 @@ class SeedCache( HydrusSerialisable.SerialisableBase ): self._seeds_ordered.append( seed ) + self._seeds_to_indices[ seed ] = len( self._seeds_ordered ) - 1 + now = HydrusData.GetNow() seed_info = {} @@ -2599,7 +2629,7 @@ class SeedCache( HydrusSerialisable.SerialisableBase ): if seed in self._seeds_to_info: - index = self._seeds_ordered.index( seed ) + index = self._seeds_to_indices[ seed ] if index > 0: @@ -2608,6 +2638,8 @@ class SeedCache( HydrusSerialisable.SerialisableBase ): self._seeds_ordered.insert( index - 1, seed ) + self._seeds_to_indices = { seed : index for ( index, seed ) in enumerate( self._seeds_ordered ) } + HG.client_controller.pub( 'seed_cache_seeds_updated', self._seed_cache_key, ( seed, ) ) @@ -2619,7 +2651,7 @@ class SeedCache( HydrusSerialisable.SerialisableBase ): if seed in self._seeds_to_info: - index = self._seeds_ordered.index( seed ) + index = self._seeds_to_indices[ seed ] if index < len( self._seeds_ordered ) - 1: @@ -2628,6 +2660,8 @@ class SeedCache( HydrusSerialisable.SerialisableBase ): self._seeds_ordered.insert( index + 1, seed ) + self._seeds_to_indices = { seed : index for ( index, seed ) in enumerate( self._seeds_ordered ) } + HG.client_controller.pub( 'seed_cache_seeds_updated', self._seed_cache_key, ( seed, ) ) @@ -2710,23 +2744,6 @@ class SeedCache( HydrusSerialisable.SerialisableBase ): - def GetSeedsWithInfo( self ): - - with self._lock: - - all_info = [] - - for seed in self._seeds_ordered: - - seed_tuple = self._GetSeedTuple( seed ) - - all_info.append( seed_tuple ) - - - return all_info - - - def GetSeedInfo( self, seed ): with self._lock: @@ -2787,6 +2804,8 @@ class SeedCache( HydrusSerialisable.SerialisableBase ): + self._seeds_to_indices = { seed : index for ( index, seed ) in enumerate( self._seeds_ordered ) } + self._SetDirty() @@ -2814,6 +2833,8 @@ class SeedCache( HydrusSerialisable.SerialisableBase ): self._seeds_ordered.remove( seed ) + self._seeds_to_indices = { seed : index for ( index, seed ) in enumerate( self._seeds_ordered ) } + self._SetDirty() diff --git a/include/ClientNetworking.py b/include/ClientNetworking.py index b33ab82c..04dc5926 100644 --- a/include/ClientNetworking.py +++ b/include/ClientNetworking.py @@ -180,6 +180,7 @@ def RequestsGet( url, params = None, stream = False, headers = None ): return response +# this is an old redirect thing to figure out redirected gallery page destinations without hitting them now. note the allow_redirects param def RequestsGetRedirectURL( url, session = None ): if session is None: @@ -1927,6 +1928,8 @@ class NetworkJob( object ): self._status_text = u'connection failed--retrying' + time.sleep( 3 ) + return response diff --git a/include/ClientParsing.py b/include/ClientParsing.py index cd68ddad..3098735f 100644 --- a/include/ClientParsing.py +++ b/include/ClientParsing.py @@ -579,35 +579,57 @@ class ParseNodeContentLink( HydrusSerialisable.SerialisableBase ): for search_url in search_urls: - try: + job_key.SetVariable( 'script_status', 'fetching ' + search_url ) + + network_job = ClientNetworking.NetworkJob( 'GET', search_url, referral_url = referral_url ) + + network_job.OverrideBandwidth() + + HG.client_controller.network_engine.AddJob( network_job ) + + while not network_job.IsDone(): - job_key.SetVariable( 'script_status', 'fetching ' + search_url ) - - headers = { 'Referer' : referral_url } - - response = ClientNetworking.RequestsGet( search_url, headers = headers ) - - except HydrusExceptions.NotFoundException: - - job_key.SetVariable( 'script_status', '404 - nothing found' ) - - time.sleep( 2 ) - - continue - - except HydrusExceptions.NetworkException as e: - - job_key.SetVariable( 'script_status', 'Network error! Details written to log.' ) - - HydrusData.Print( 'Problem fetching ' + search_url + ':' ) - HydrusData.PrintException( e ) - - time.sleep( 2 ) - - continue + time.sleep( 0.1 ) - linked_data = response.text + if HG.view_shutdown: + + raise HydrusExceptions.ShutdownException() + + elif network_job.HasError(): + + e = network_job.GetErrorException() + + if isinstance( e, HydrusExceptions.NotFoundException ): + + job_key.SetVariable( 'script_status', '404 - nothing found' ) + + time.sleep( 2 ) + + continue + + elif isinstance( e, HydrusExceptions.NetworkException ): + + job_key.SetVariable( 'script_status', 'Network error! Details written to log.' ) + + HydrusData.Print( 'Problem fetching ' + search_url + ':' ) + HydrusData.PrintException( e ) + + time.sleep( 2 ) + + continue + + else: + + raise e + + + elif network_job.IsCancelled(): + + break + + + linked_data = network_job.GetContent() children_content = GetChildrenContent( job_key, self._children, linked_data, search_url, desired_content ) diff --git a/include/HydrusConstants.py b/include/HydrusConstants.py index 5cd0ec30..08eef5bf 100755 --- a/include/HydrusConstants.py +++ b/include/HydrusConstants.py @@ -49,7 +49,7 @@ options = {} # Misc NETWORK_VERSION = 18 -SOFTWARE_VERSION = 270 +SOFTWARE_VERSION = 271 UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 ) diff --git a/include/HydrusNetworking.py b/include/HydrusNetworking.py index 3b5e9504..e9015a27 100644 --- a/include/HydrusNetworking.py +++ b/include/HydrusNetworking.py @@ -291,6 +291,8 @@ class BandwidthTracker( HydrusSerialisable.SerialisableBase ): CACHE_MAINTENANCE_TIME_DELTA = 120 + MIN_TIME_DELTA_FOR_USER = 10 + def __init__( self ): HydrusSerialisable.SerialisableBase.__init__( self ) @@ -481,9 +483,9 @@ class BandwidthTracker( HydrusSerialisable.SerialisableBase ): return ( month_time, day_time, hour_time, minute_time, second_time ) - def _GetUsage( self, bandwidth_type, time_delta ): + def _GetUsage( self, bandwidth_type, time_delta, for_user ): - if time_delta is not None and bandwidth_type == HC.BANDWIDTH_TYPE_DATA and time_delta <= 5: + if for_user and time_delta is not None and bandwidth_type == HC.BANDWIDTH_TYPE_DATA and time_delta <= self.MIN_TIME_DELTA_FOR_USER: usage = self._GetWeightedApproximateUsage( time_delta ) @@ -499,13 +501,10 @@ class BandwidthTracker( HydrusSerialisable.SerialisableBase ): def _GetWeightedApproximateUsage( self, time_delta ): - SEARCH_DELTA = time_delta * 5 + SEARCH_DELTA = self.MIN_TIME_DELTA_FOR_USER - window = 0 counter = self._seconds_bytes - SEARCH_DELTA += window - now = HydrusData.GetNow() since = now - SEARCH_DELTA @@ -526,9 +525,9 @@ class BandwidthTracker( HydrusSerialisable.SerialisableBase ): total_bytes = sum( ( counter[ key ] for key in valid_keys ) ) - time_delta_average = total_bytes / SAMPLE_DELTA + time_delta_average_per_sec = total_bytes / SAMPLE_DELTA - return time_delta_average + return time_delta_average_per_sec * time_delta def _MaintainCache( self ): @@ -569,8 +568,8 @@ class BandwidthTracker( HydrusSerialisable.SerialisableBase ): with self._lock: - num_bytes = self._GetUsage( HC.BANDWIDTH_TYPE_DATA, None ) - num_requests = self._GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, None ) + num_bytes = self._GetUsage( HC.BANDWIDTH_TYPE_DATA, None, True ) + num_requests = self._GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, None, True ) return 'used ' + HydrusData.ConvertIntToBytes( num_bytes ) + ' in ' + HydrusData.ConvertIntToPrettyString( num_requests ) + ' requests this month' @@ -599,7 +598,7 @@ class BandwidthTracker( HydrusSerialisable.SerialisableBase ): - def GetUsage( self, bandwidth_type, time_delta ): + def GetUsage( self, bandwidth_type, time_delta, for_user = False ): with self._lock: @@ -608,7 +607,7 @@ class BandwidthTracker( HydrusSerialisable.SerialisableBase ): return 0 - return self._GetUsage( bandwidth_type, time_delta ) + return self._GetUsage( bandwidth_type, time_delta, for_user )
-
+
-
-
what you will need
You will need basic python experience, python 2.7 and a number of python modules. Most of it you can get through pip. I think this will do for most systems:
-
-
Although you may want to do all that in smaller batches. Ultimately, the best way to figure out if you have enough is to just keep running client.pyw and see what it complains about missing.
I use Ubuntu 17.04, which also requires something like:
-
-
YMMV. Feel free to email me if you run into trouble or discover any neat tricks.
-OS X 10.9 is similar, though wx is simpler. I use the cocoa package straight from wxPython's site. This should do the rest:
+OS X 10.9 is similar, though wx is simpler. I use the cocoa package straight from wxPython's site. If you can't get opencv working from pip, this should do the rest: