From e73f2d704d67c7bceec54922e430e2a31c4c23c3 Mon Sep 17 00:00:00 2001 From: Hydrus Network Developer Date: Tue, 4 May 2021 23:41:08 -0500 Subject: [PATCH 1/2] creating new canvas submodule This is a non-functioning commit before tomorrow's release with just file moves, no changes, to help github track file history. --- hydrus/client/gui/{ => canvas}/ClientGUICanvas.py | 0 hydrus/client/gui/{ => canvas}/ClientGUICanvasFrame.py | 0 hydrus/client/gui/{ => canvas}/ClientGUICanvasHoverFrames.py | 0 hydrus/client/gui/{ => canvas}/ClientGUICanvasMedia.py | 0 hydrus/client/gui/canvas/__init__.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename hydrus/client/gui/{ => canvas}/ClientGUICanvas.py (100%) rename hydrus/client/gui/{ => canvas}/ClientGUICanvasFrame.py (100%) rename hydrus/client/gui/{ => canvas}/ClientGUICanvasHoverFrames.py (100%) rename hydrus/client/gui/{ => canvas}/ClientGUICanvasMedia.py (100%) create mode 100644 hydrus/client/gui/canvas/__init__.py diff --git a/hydrus/client/gui/ClientGUICanvas.py b/hydrus/client/gui/canvas/ClientGUICanvas.py similarity index 100% rename from hydrus/client/gui/ClientGUICanvas.py rename to hydrus/client/gui/canvas/ClientGUICanvas.py diff --git a/hydrus/client/gui/ClientGUICanvasFrame.py b/hydrus/client/gui/canvas/ClientGUICanvasFrame.py similarity index 100% rename from hydrus/client/gui/ClientGUICanvasFrame.py rename to hydrus/client/gui/canvas/ClientGUICanvasFrame.py diff --git a/hydrus/client/gui/ClientGUICanvasHoverFrames.py b/hydrus/client/gui/canvas/ClientGUICanvasHoverFrames.py similarity index 100% rename from hydrus/client/gui/ClientGUICanvasHoverFrames.py rename to hydrus/client/gui/canvas/ClientGUICanvasHoverFrames.py diff --git a/hydrus/client/gui/ClientGUICanvasMedia.py b/hydrus/client/gui/canvas/ClientGUICanvasMedia.py similarity index 100% rename from hydrus/client/gui/ClientGUICanvasMedia.py rename to hydrus/client/gui/canvas/ClientGUICanvasMedia.py diff --git a/hydrus/client/gui/canvas/__init__.py b/hydrus/client/gui/canvas/__init__.py new file mode 100644 index 00000000..e69de29b From 63abf752e90b585f679e7ee2219323e403d6d449 Mon Sep 17 00:00:00 2001 From: Hydrus Network Developer Date: Wed, 5 May 2021 15:12:11 -0500 Subject: [PATCH 2/2] Version 438 --- help/changelog.html | 34 ++ help/client_api.html | 1 + hydrus/client/ClientCaches.py | 94 +++++- hydrus/client/ClientController.py | 4 +- hydrus/client/ClientOptions.py | 9 +- hydrus/client/ClientRendering.py | 71 +++- hydrus/client/ClientSerialisable.py | 16 +- hydrus/client/db/ClientDB.py | 312 ++++++++++-------- hydrus/client/gui/ClientGUI.py | 117 ++++++- hydrus/client/gui/ClientGUIManagement.py | 4 +- hydrus/client/gui/ClientGUIPages.py | 9 +- hydrus/client/gui/ClientGUIResults.py | 4 +- .../gui/ClientGUIScrolledPanelsManagement.py | 84 ++++- .../gui/ClientGUIScrolledPanelsReview.py | 2 +- hydrus/client/gui/ClientGUITags.py | 56 +++- hydrus/client/gui/canvas/ClientGUICanvas.py | 94 +++--- .../client/gui/canvas/ClientGUICanvasFrame.py | 2 +- .../client/gui/canvas/ClientGUICanvasMedia.py | 193 +++++++++-- hydrus/client/gui/canvas/__init__.py | 1 + .../services/ClientGUIClientsideServices.py | 2 +- .../client/gui/widgets/ClientGUIControls.py | 1 + .../client/importing/ClientImportFileSeeds.py | 9 +- .../client/networking/ClientNetworkingJobs.py | 25 ++ hydrus/core/HydrusConstants.py | 2 +- hydrus/core/HydrusFlashHandling.py | 6 +- hydrus/core/HydrusGlobals.py | 1 + hydrus/core/networking/HydrusNetwork.py | 8 + hydrus/test/TestClientDB.py | 8 +- hydrus/test/TestController.py | 12 + requirements.txt | 2 +- 30 files changed, 908 insertions(+), 275 deletions(-) diff --git a/help/changelog.html b/help/changelog.html index 56fd1596..5834d7db 100755 --- a/help/changelog.html +++ b/help/changelog.html @@ -8,6 +8,40 @@

changelog

    +

  • version 438

    • +
      +
    • media viewer:
      • +
    • I have hacked in tile-based image rendering for the media viewer. this has always been planned as a larger, longer-term job, but the problem of large images is only getting worse, so I decided to just slam out a prototype in a week. if you have a steam-powered GPU or 4GB ram, you might like to wait until next week to update so I can iron out any surprise bugs or performance problems
      • +
    • images are now cut into tiles that are rendered on demand, so whenever the image is zoomed larger than the media viewer window, only those tiles currently in view have CPU and memory spent on resizing and storage. as you pan around, new tiles are rendered as needed, and old discarded. this makes zooming in super fast and low memory, even for large images!
      • +
    • although I am happy with this, and overall we are talking a huge improvement on previous performance, it is ugly fast code. it may fail for some unusual files. it slices and blits bitmaps around your video memory much faster than before, so some odd GPUs may also have problems. I haven't seen any alignment artifacts (1-pixel thick missing columns or rows), but some images may produce them. more apparent are some pretty ugly tile artifacts that show up between 200% and 500% zoom (interpolation algorithms, which rely on neighbour pixels, are missing border data with my simple system). I will consider how best to implement more complicated but stitch-correct overlapping tiles in future
      • +
    • futhermore, a new 'image tile' cache is added. you can customise size and timeout under _options->speed and memory_ like for images and thumbnails. this is a dedicated cache for remembering image resize computation across images and zooms. once you have seen both situations once, flicking back and forth between two images or zoom levels is now generally always instant! this new cache starts at a healthy default of 256MB. let's see how that amount works out IRL--I think it will be plenty
      • +
    • I tuned the image renderer cache--it no longer caches huge images that eat more than 25% its total size--meaning these images only hang around as long as you are looking at them--and the prefetch call that pre-renders several files previous/next to the current image no longer occurs on images that would eat more than 10% the cache size. this should greatly reduce weird flicker and other lag when browsing through a series of mega-images (which before would stomp through the cache in quick succession, barging each other out of the way and wasting a bunch of CPU). in real world terms, this basically means that with an image cache of 200MB, you should have slower individual image performance but much better overall performance looking at images with more than about 5k resolution. the dreaded 14,000x12,000 png will still bonk you on the head to do the first render, but it won't try to uselessly prefetch or flush the whole cache any more
      • +
    • if you are currently looking at a static image, neighbour prefetch now only starts once the image is rendered, giving the task in front of you a bit more CPU time
      • +
    • new options for prefetch delay and previous/next distance are added to 'speed and memory'
      • +
    • note this does not yet apply to the old hydrus animation renderer. that still sucks at high zoom!
      • +
    • another future step here is to expand prefetch to tiles so the first view of the 'next' media is instant, but let's let all this breathe for a bit. if you get bugs, let me know!
      • +
    • due to a Qt issue, I am stopping zoom-in events that would make the 'virtual' size of the image greater than 32,000x32,000
      • +
    • .
      • +
    • account permission improvements:
      • +
    • to group sibling and parent petitions by uploader (and thus help janitor workflow), the PTR is moving to a system where the public account is download-only and accounts that can upload content are auto-generated in manage services. this code has not been tested much before, and it revealed some very bad reporting and handling of current permissions. I move this forward this week:
      • +
    • if your repository account is currently unsynced from a serious previous error, any attempt to upload pending data will result in a little popup and the upload being abandoned
      • +
    • manage tag siblings and parents will now show service tabs even if the account for those services does not seem currently able to upload tags or siblngs
      • +
    • if your repository account is currently unsynced from a serious previous error, this is now noted in red text in manage siblings and manage parents
      • +
    • if your repository account does not have sibling/parent upload permission, this is now noted in red text in manage siblings and manage parents. you will be able to pend and petition siblings and parents ok
      • +
    • if your repository account does not have mapping/sibling/parent upload permission of the right kind, your client will no longer attempt to upload these content types, and if there is pending count for one of these types, a popup will note this on an upload attempt
      • +
    • .
      • +
    • the rest:
      • +
    • added https://github.com/NO-ob/LoliSnatcher_Droid to the Client API help!
      • +
    • improved some error handling, reporting, and recovery when importing serialised pngs. specific error info is now written to the log as well
      • +
    • fixed a secondary error when dropping non-list, non-downloader pngs on Lain's easy downloader import window, and fixed a 'no interesting objects' reporting test when dropping multiple pngs
      • +
    • added a 'cache report mode' to help debug image and thumb caching issues
      • +
    • refactored the media viewer code to a new 'canvas' submodule
      • +
    • improved the error reporting when a thumbnail cannot be generated for a file being imported
      • +
    • fixed an error in zoom center calculation when a change zoom event was sent in the split-second during media viewer initialisation
      • +
    • I think I fixed an issue where pages could sometimes not automatically move on from 'loading initial files' statusbar text when initialising the session
      • +
    • the requirements.txt now specifies 'requests' 2.23.0 exactly, as newer versions seemed to be giving odd urllib3 attribute binding errors (seems maybe a session thread safety thing) when recovering from connection failures. this should update the macOS build as well as anyone running from source who wants to re-run the requirements.txt. I hacked in a catch for this error case anyway, just a manual retry like a normal connection error, we'll see how it goes (issue #665)
      • +
    • patched an unusual file import bug for a flash file with an inverted bounding box that resulted in negative reported resolution. flash now takes absolute values for width and height
      • +

  • version 437

    • misc:
      • diff --git a/help/client_api.html b/help/client_api.html index 8dd37e31..4bc196a4 100644 --- a/help/client_api.html +++ b/help/client_api.html @@ -18,6 +18,7 @@
      • https://gitgud.io/prkc/hydrus-companion - Hydrus Companion, a Chrome/Firefox extension for hydrus that allows easy download queueing as you browse and advanced login support
      • https://github.com/floogulinc/hydrus-web - Hydrus Web, a web client for hydrus (allows phone browsing of hydrus)
        • +
      • https://github.com/NO-ob/LoliSnatcher_Droid - LoliSnatcher, a booru client for Android that can talk to hydrus
      • https://www.animebox.es/ - Anime Boxes, a booru browser, now supports adding your client as a Hydrus Server
      • https://ififfy.github.io/flipflip/#/ - FlipFlip, an advanced slideshow interface, now supports hydrus as a source
      • https://gitgud.io/koto/hydrus-archive-delete - Archive/Delete filter in your web browser
        • diff --git a/hydrus/client/ClientCaches.py b/hydrus/client/ClientCaches.py index a2b4c3d0..ace76c98 100644 --- a/hydrus/client/ClientCaches.py +++ b/hydrus/client/ClientCaches.py @@ -19,9 +19,10 @@ from hydrus.client import ClientRendering class DataCache( object ): - def __init__( self, controller, cache_size, timeout = 1200 ): + def __init__( self, controller, name, cache_size, timeout = 1200 ): self._controller = controller + self._name = name self._cache_size = cache_size self._timeout = timeout @@ -46,6 +47,11 @@ class DataCache( object ): self._RecalcMemoryUsage() + if HG.cache_report_mode: + + HydrusData.ShowText( 'Cache "{}" removing "{}". Current size {}.'.format( self._name, key, HydrusData.ConvertValueRangeToBytes( self._total_estimated_memory_footprint, self._cache_size ) ) ) + + def _DeleteItem( self ): @@ -98,6 +104,18 @@ class DataCache( object ): self._RecalcMemoryUsage() + if HG.cache_report_mode: + + HydrusData.ShowText( + 'Cache "{}" adding "{}" ({}). Current size {}.'.format( + self._name, + key, + HydrusData.ToHumanBytes( data.GetEstimatedMemoryFootprint() ), + HydrusData.ConvertValueRangeToBytes( self._total_estimated_memory_footprint, self._cache_size ) + ) + ) + + @@ -141,6 +159,14 @@ class DataCache( object ): + def GetSizeLimit( self ): + + with self._lock: + + return self._cache_size + + + def HasData( self, key ): with self._lock: @@ -433,7 +459,7 @@ class ParsingCache( object ): -class RenderedImageCache( object ): +class ImageRendererCache( object ): def __init__( self, controller ): @@ -442,7 +468,7 @@ class RenderedImageCache( object ): cache_size = self._controller.options[ 'fullscreen_cache_size' ] cache_timeout = self._controller.new_options.GetInteger( 'image_cache_timeout' ) - self._data_cache = DataCache( self._controller, cache_size, timeout = cache_timeout ) + self._data_cache = DataCache( self._controller, 'image cache', cache_size, timeout = cache_timeout ) def Clear( self ): @@ -462,7 +488,12 @@ class RenderedImageCache( object ): image_renderer = ClientRendering.ImageRenderer( media ) - self._data_cache.AddData( key, image_renderer ) + # we are no longer going to let big lads flush the whole cache. they can render on demand + + if image_renderer.GetEstimatedMemoryFootprint() < self._data_cache.GetSizeLimit() / 4: + + self._data_cache.AddData( key, image_renderer ) + else: @@ -479,6 +510,59 @@ class RenderedImageCache( object ): return self._data_cache.HasData( key ) + def PrefetchImageRenderer( self, media ): + + ( width, height ) = media.GetResolution() + + # essentially, we are not going to prefetch giganto images any more. they can render on demand and not mess our queue + + if width * height * 3 < self._data_cache.GetSizeLimit() / 10: + + self.GetImageRenderer( media ) + + + +class ImageTileCache( object ): + + def __init__( self, controller ): + + self._controller = controller + + cache_size = self._controller.new_options.GetInteger( 'image_tile_cache_size' ) + cache_timeout = self._controller.new_options.GetInteger( 'image_tile_cache_timeout' ) + + self._data_cache = DataCache( self._controller, 'image tile cache', cache_size, timeout = cache_timeout ) + + + def Clear( self ): + + self._data_cache.Clear() + + + def GetTile( self, image_renderer, media, clip_rect, target_resolution ): + + hash = media.GetHash() + + key = ( hash, clip_rect, target_resolution ) + + result = self._data_cache.GetIfHasData( key ) + + if result is None: + + qt_pixmap = image_renderer.GetQtPixmap( clip_rect = clip_rect, target_resolution = target_resolution ) + + tile = ClientRendering.ImageTile( hash, clip_rect, qt_pixmap ) + + self._data_cache.AddData( key, tile ) + + else: + + tile = result + + + return tile + + class ThumbnailCache( object ): def __init__( self, controller ): @@ -488,7 +572,7 @@ class ThumbnailCache( object ): cache_size = self._controller.options[ 'thumbnail_cache_size' ] cache_timeout = self._controller.new_options.GetInteger( 'thumbnail_cache_timeout' ) - self._data_cache = DataCache( self._controller, cache_size, timeout = cache_timeout ) + self._data_cache = DataCache( self._controller, 'thumbnail cache', cache_size, timeout = cache_timeout ) self._magic_mime_thumbnail_ease_score_lookup = {} diff --git a/hydrus/client/ClientController.py b/hydrus/client/ClientController.py index ffe712b8..2de86bfb 100644 --- a/hydrus/client/ClientController.py +++ b/hydrus/client/ClientController.py @@ -1066,7 +1066,8 @@ class Controller( HydrusController.HydrusController ): self.frame_splash_status.SetSubtext( 'image caches' ) # careful: outside of qt since they don't need qt for init, seems ok _for now_ - self._caches[ 'images' ] = ClientCaches.RenderedImageCache( self ) + self._caches[ 'images' ] = ClientCaches.ImageRendererCache( self ) + self._caches[ 'image_tiles' ] = ClientCaches.ImageTileCache( self ) self._caches[ 'thumbnail' ] = ClientCaches.ThumbnailCache( self ) self.bitmap_manager = ClientManagers.BitmapManager( self ) @@ -2068,6 +2069,7 @@ class Controller( HydrusController.HydrusController ): def CopyToClipboard(): + # this is faster than qpixmap, which converts to a qimage anyway qt_image = image_renderer.GetQtImage().copy() QW.QApplication.clipboard().setImage( qt_image ) diff --git a/hydrus/client/ClientOptions.py b/hydrus/client/ClientOptions.py index 51ce9cd4..f05476f0 100644 --- a/hydrus/client/ClientOptions.py +++ b/hydrus/client/ClientOptions.py @@ -312,7 +312,7 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ): self._dictionary[ 'integers' ][ 'wake_delay_period' ] = 15 - from hydrus.client.gui import ClientGUICanvas + from hydrus.client.gui.canvas import ClientGUICanvas self._dictionary[ 'integers' ][ 'media_viewer_zoom_center' ] = ClientGUICanvas.ZOOM_CENTERPOINT_VIEWER_CENTER @@ -348,8 +348,15 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ): self._dictionary[ 'integers' ][ 'duplicate_comparison_score_more_tags' ] = 8 self._dictionary[ 'integers' ][ 'duplicate_comparison_score_older' ] = 4 + self._dictionary[ 'integers' ][ 'image_tile_cache_size' ] = 1024 * 1024 * 256 + self._dictionary[ 'integers' ][ 'thumbnail_cache_timeout' ] = 86400 self._dictionary[ 'integers' ][ 'image_cache_timeout' ] = 600 + self._dictionary[ 'integers' ][ 'image_tile_cache_timeout' ] = 300 + + self._dictionary[ 'integers' ][ 'media_viewer_prefetch_delay_base_ms' ] = 100 + self._dictionary[ 'integers' ][ 'media_viewer_prefetch_num_previous' ] = 2 + self._dictionary[ 'integers' ][ 'media_viewer_prefetch_num_next' ] = 3 self._dictionary[ 'integers' ][ 'thumbnail_border' ] = 1 self._dictionary[ 'integers' ][ 'thumbnail_margin' ] = 2 diff --git a/hydrus/client/ClientRendering.py b/hydrus/client/ClientRendering.py index 05269ce8..e1dbe6f0 100644 --- a/hydrus/client/ClientRendering.py +++ b/hydrus/client/ClientRendering.py @@ -98,15 +98,34 @@ class ImageRenderer( object ): HG.client_controller.CallToThread( self._Initialise ) - def _GetNumPyImage( self, target_resolution = None ): + def _GetNumPyImage( self, clip_rect: QC.QRect, target_resolution: QC.QSize ): - if target_resolution is None: + clip_topleft = clip_rect.topLeft() + clip_size = clip_rect.size() + + ( my_width, my_height ) = self.GetResolution() + + my_full_rect = QC.QRect( 0, 0, my_width, my_height ) + + if clip_rect == my_full_rect: - numpy_image = self._numpy_image + source = self._numpy_image else: - numpy_image = ClientImageHandling.ResizeNumPyImageForMediaViewer( self._mime, self._numpy_image, ( target_resolution.width(), target_resolution.height() ) ) + ( x, y ) = ( clip_topleft.x(), clip_topleft.y() ) + ( clip_width, clip_height ) = ( clip_size.width(), clip_size.height() ) + + source = self._numpy_image[ y : y + clip_height, x : x + clip_width ] + + + if target_resolution == clip_size: + + return source.copy() + + else: + + numpy_image = ClientImageHandling.ResizeNumPyImageForMediaViewer( self._mime, source, ( target_resolution.width(), target_resolution.height() ) ) return numpy_image @@ -142,11 +161,19 @@ class ImageRenderer( object ): def GetResolution( self ): return self._resolution - def GetQtImage( self, target_resolution = None ): + def GetQtImage( self, clip_rect = None, target_resolution = None ): - # add region param to this to allow clipping before resize + if clip_rect is None: + + clip_rect = QC.QRect( QC.QPoint( 0, 0 ), QC.QSize( self._resolution ) ) + - numpy_image = self._GetNumPyImage( target_resolution = target_resolution ) + if target_resolution is None: + + target_resolution = clip_rect.size() + + + numpy_image = self._GetNumPyImage( clip_rect, target_resolution ) ( height, width, depth ) = numpy_image.shape @@ -155,11 +182,19 @@ class ImageRenderer( object ): return HG.client_controller.bitmap_manager.GetQtImageFromBuffer( width, height, depth * 8, data ) - def GetQtPixmap( self, target_resolution = None ): + def GetQtPixmap( self, clip_rect = None, target_resolution = None ): - # add region param to this to allow clipping before resize + if clip_rect is None: + + clip_rect = QC.QRect( QC.QPoint( 0, 0 ), QC.QSize( self._resolution ) ) + - numpy_image = self._GetNumPyImage( target_resolution = target_resolution ) + if target_resolution is None: + + target_resolution = clip_rect.size() + + + numpy_image = self._GetNumPyImage( clip_rect, target_resolution ) ( height, width, depth ) = numpy_image.shape @@ -173,6 +208,22 @@ class ImageRenderer( object ): return self._numpy_image is not None +class ImageTile( object ): + + def __init__( self, hash: bytes, clip_rect: QC.QRect, qt_pixmap: QG.QPixmap ): + + self.hash = hash + self.clip_rect = clip_rect + self.qt_pixmap = qt_pixmap + + self._num_bytes = self.qt_pixmap.width() * self.qt_pixmap.height() * 3 + + + def GetEstimatedMemoryFootprint( self ): + + return self._num_bytes + + class RasterContainer( object ): def __init__( self, media, target_resolution = None ): diff --git a/hydrus/client/ClientSerialisable.py b/hydrus/client/ClientSerialisable.py index 8ee83337..90991f12 100644 --- a/hydrus/client/ClientSerialisable.py +++ b/hydrus/client/ClientSerialisable.py @@ -264,13 +264,17 @@ def LoadFromPNG( path ): try: - try: + height = numpy_image.shape[0] + width = numpy_image.shape[1] + + if len( numpy_image.shape ) > 2: - ( height, width ) = numpy_image.shape + depth = numpy_image.shape[2] - except: - - raise Exception( 'The file did not appear to be monochrome!' ) + if depth != 1: + + raise Exception( 'The file did not appear to be monochrome!' ) + try: @@ -303,6 +307,8 @@ def LoadFromPNG( path ): except Exception as e: + HydrusData.PrintException( e ) + message = 'The image loaded, but it did not seem to be a hydrus serialised png! The error was: {}'.format( str( e ) ) message += os.linesep * 2 message += 'If you believe this is a legit non-resized, non-converted hydrus serialised png, please send it to hydrus_dev.' diff --git a/hydrus/client/db/ClientDB.py b/hydrus/client/db/ClientDB.py index d4bd7518..a7eb3bd9 100644 --- a/hydrus/client/db/ClientDB.py +++ b/hydrus/client/db/ClientDB.py @@ -11209,12 +11209,14 @@ class DB( HydrusDB.HydrusDB ): return options - def _GetPending( self, service_key ): + def _GetPending( self, service_key, content_types ): service_id = self.modules_services.GetServiceId( service_key ) service = self.modules_services.GetService( service_id ) + account = service.GetAccount() + service_type = service.GetServiceType() client_to_server_update = HydrusNetwork.ClientToServerUpdate() @@ -11223,162 +11225,186 @@ class DB( HydrusDB.HydrusDB ): if service_type == HC.TAG_REPOSITORY: - # mappings - - ( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = ClientDBMappingsStorage.GenerateMappingsTableNames( service_id ) - - pending_dict = HydrusData.BuildKeyToListDict( self._c.execute( 'SELECT tag_id, hash_id FROM ' + pending_mappings_table_name + ' ORDER BY tag_id LIMIT 100;' ) ) - - pending_mapping_ids = list( pending_dict.items() ) - - # dealing with a scary situation when (due to some bug) mappings are current and pending. they get uploaded, but the content update makes no changes, so we cycle infitely! - addable_pending_mapping_ids = self._FilterExistingUpdateMappings( service_id, pending_mapping_ids, HC.CONTENT_UPDATE_ADD ) - - pending_mapping_weight = sum( ( len( hash_ids ) for ( tag_id, hash_ids ) in pending_mapping_ids ) ) - addable_pending_mapping_weight = sum( ( len( hash_ids ) for ( tag_id, hash_ids ) in addable_pending_mapping_ids ) ) - - if pending_mapping_weight != addable_pending_mapping_weight: + if HC.CONTENT_TYPE_MAPPINGS in content_types: - message = 'Hey, while going through the pending tags to upload, it seemed some were simultaneously already in the \'current\' state. This looks like a bug.' - message += os.linesep * 2 - message += 'Please run _database->check and repair->fix logically inconsistent mappings_. If everything seems good after that and you do not get this message again, you should be all fixed. If not, you may need to regenerate your mappings storage cache under the \'database\' menu. If that does not work, hydev would like to know about it!' + if account.HasPermission( HC.CONTENT_TYPE_MAPPINGS, HC.PERMISSION_ACTION_CREATE ): + + ( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = ClientDBMappingsStorage.GenerateMappingsTableNames( service_id ) + + pending_dict = HydrusData.BuildKeyToListDict( self._c.execute( 'SELECT tag_id, hash_id FROM ' + pending_mappings_table_name + ' ORDER BY tag_id LIMIT 100;' ) ) + + pending_mapping_ids = list( pending_dict.items() ) + + # dealing with a scary situation when (due to some bug) mappings are current and pending. they get uploaded, but the content update makes no changes, so we cycle infitely! + addable_pending_mapping_ids = self._FilterExistingUpdateMappings( service_id, pending_mapping_ids, HC.CONTENT_UPDATE_ADD ) + + pending_mapping_weight = sum( ( len( hash_ids ) for ( tag_id, hash_ids ) in pending_mapping_ids ) ) + addable_pending_mapping_weight = sum( ( len( hash_ids ) for ( tag_id, hash_ids ) in addable_pending_mapping_ids ) ) + + if pending_mapping_weight != addable_pending_mapping_weight: + + message = 'Hey, while going through the pending tags to upload, it seemed some were simultaneously already in the \'current\' state. This looks like a bug.' + message += os.linesep * 2 + message += 'Please run _database->check and repair->fix logically inconsistent mappings_. If everything seems good after that and you do not get this message again, you should be all fixed. If not, you may need to regenerate your mappings storage cache under the \'database\' menu. If that does not work, hydev would like to know about it!' + + HydrusData.ShowText( message ) + + raise HydrusExceptions.VetoException( 'Logically inconsistent mappings detected!' ) + + + for ( tag_id, hash_ids ) in pending_mapping_ids: + + tag = self.modules_tags_local_cache.GetTag( tag_id ) + hashes = self.modules_hashes_local_cache.GetHashes( hash_ids ) + + content = HydrusNetwork.Content( HC.CONTENT_TYPE_MAPPINGS, ( tag, hashes ) ) + + client_to_server_update.AddContent( HC.CONTENT_UPDATE_PEND, content ) + + - HydrusData.ShowText( message ) - - raise HydrusExceptions.VetoException( 'Logically inconsistent mappings detected!' ) + if account.HasPermission( HC.CONTENT_TYPE_MAPPINGS, HC.PERMISSION_ACTION_PETITION ): + + petitioned_dict = HydrusData.BuildKeyToListDict( [ ( ( tag_id, reason_id ), hash_id ) for ( tag_id, hash_id, reason_id ) in self._c.execute( 'SELECT tag_id, hash_id, reason_id FROM ' + petitioned_mappings_table_name + ' ORDER BY reason_id LIMIT 100;' ) ] ) + + petitioned_mapping_ids = list( petitioned_dict.items() ) + + # dealing with a scary situation when (due to some bug) mappings are deleted and petitioned. they get uploaded, but the content update makes no changes, so we cycle infitely! + deletable_and_petitioned_mappings = self._FilterExistingUpdateMappings( + service_id, + [ ( tag_id, hash_ids ) for ( ( tag_id, reason_id ), hash_ids ) in petitioned_mapping_ids ], + HC.CONTENT_UPDATE_DELETE + ) + + petitioned_mapping_weight = sum( ( len( hash_ids ) for ( tag_id, hash_ids ) in petitioned_mapping_ids ) ) + deletable_petitioned_mapping_weight = sum( ( len( hash_ids ) for ( tag_id, hash_ids ) in deletable_and_petitioned_mappings ) ) + + if petitioned_mapping_weight != deletable_petitioned_mapping_weight: + + message = 'Hey, while going through the petitioned tags to upload, it seemed some were simultaneously already in the \'deleted\' state. This looks like a bug.' + message += os.linesep * 2 + message += 'Please run _database->check and repair->fix logically inconsistent mappings_. If everything seems good after that and you do not get this message again, you should be all fixed. If not, you may need to regenerate your mappings storage cache under the \'database\' menu. If that does not work, hydev would like to know about it!' + + HydrusData.ShowText( message ) + + raise HydrusExceptions.VetoException( 'Logically inconsistent mappings detected!' ) + + + for ( ( tag_id, reason_id ), hash_ids ) in petitioned_mapping_ids: + + tag = self.modules_tags_local_cache.GetTag( tag_id ) + hashes = self.modules_hashes_local_cache.GetHashes( hash_ids ) + + reason = self.modules_texts.GetText( reason_id ) + + content = HydrusNetwork.Content( HC.CONTENT_TYPE_MAPPINGS, ( tag, hashes ) ) + + client_to_server_update.AddContent( HC.CONTENT_UPDATE_PETITION, content, reason ) + + - for ( tag_id, hash_ids ) in pending_mapping_ids: + if HC.CONTENT_TYPE_TAG_PARENTS in content_types: - tag = self.modules_tags_local_cache.GetTag( tag_id ) - hashes = self.modules_hashes_local_cache.GetHashes( hash_ids ) - - content = HydrusNetwork.Content( HC.CONTENT_TYPE_MAPPINGS, ( tag, hashes ) ) - - client_to_server_update.AddContent( HC.CONTENT_UPDATE_PEND, content ) + if account.HasPermission( HC.CONTENT_TYPE_TAG_PARENTS, HC.PERMISSION_ACTION_PETITION ): + + pending = self._c.execute( 'SELECT child_tag_id, parent_tag_id, reason_id FROM tag_parent_petitions WHERE service_id = ? AND status = ? ORDER BY reason_id LIMIT 1;', ( service_id, HC.CONTENT_STATUS_PENDING ) ).fetchall() + + for ( child_tag_id, parent_tag_id, reason_id ) in pending: + + child_tag = self.modules_tags_local_cache.GetTag( child_tag_id ) + parent_tag = self.modules_tags_local_cache.GetTag( parent_tag_id ) + + reason = self.modules_texts.GetText( reason_id ) + + content = HydrusNetwork.Content( HC.CONTENT_TYPE_TAG_PARENTS, ( child_tag, parent_tag ) ) + + client_to_server_update.AddContent( HC.CONTENT_UPDATE_PEND, content, reason ) + + + petitioned = self._c.execute( 'SELECT child_tag_id, parent_tag_id, reason_id FROM tag_parent_petitions WHERE service_id = ? AND status = ? ORDER BY reason_id LIMIT 100;', ( service_id, HC.CONTENT_STATUS_PETITIONED ) ).fetchall() + + for ( child_tag_id, parent_tag_id, reason_id ) in petitioned: + + child_tag = self.modules_tags_local_cache.GetTag( child_tag_id ) + parent_tag = self.modules_tags_local_cache.GetTag( parent_tag_id ) + + reason = self.modules_texts.GetText( reason_id ) + + content = HydrusNetwork.Content( HC.CONTENT_TYPE_TAG_PARENTS, ( child_tag, parent_tag ) ) + + client_to_server_update.AddContent( HC.CONTENT_UPDATE_PETITION, content, reason ) + + - petitioned_dict = HydrusData.BuildKeyToListDict( [ ( ( tag_id, reason_id ), hash_id ) for ( tag_id, hash_id, reason_id ) in self._c.execute( 'SELECT tag_id, hash_id, reason_id FROM ' + petitioned_mappings_table_name + ' ORDER BY reason_id LIMIT 100;' ) ] ) - - petitioned_mapping_ids = list( petitioned_dict.items() ) - - # dealing with a scary situation when (due to some bug) mappings are deleted and petitioned. they get uploaded, but the content update makes no changes, so we cycle infitely! - deletable_and_petitioned_mappings = self._FilterExistingUpdateMappings( - service_id, - [ ( tag_id, hash_ids ) for ( ( tag_id, reason_id ), hash_ids ) in petitioned_mapping_ids ], - HC.CONTENT_UPDATE_DELETE - ) - - petitioned_mapping_weight = sum( ( len( hash_ids ) for ( tag_id, hash_ids ) in petitioned_mapping_ids ) ) - deletable_petitioned_mapping_weight = sum( ( len( hash_ids ) for ( tag_id, hash_ids ) in deletable_and_petitioned_mappings ) ) - - if petitioned_mapping_weight != deletable_petitioned_mapping_weight: + if HC.CONTENT_TYPE_TAG_SIBLINGS in content_types: - message = 'Hey, while going through the petitioned tags to upload, it seemed some were simultaneously already in the \'deleted\' state. This looks like a bug.' - message += os.linesep * 2 - message += 'Please run _database->check and repair->fix logically inconsistent mappings_. If everything seems good after that and you do not get this message again, you should be all fixed. If not, you may need to regenerate your mappings storage cache under the \'database\' menu. If that does not work, hydev would like to know about it!' - - HydrusData.ShowText( message ) - - raise HydrusExceptions.VetoException( 'Logically inconsistent mappings detected!' ) - - - for ( ( tag_id, reason_id ), hash_ids ) in petitioned_mapping_ids: - - tag = self.modules_tags_local_cache.GetTag( tag_id ) - hashes = self.modules_hashes_local_cache.GetHashes( hash_ids ) - - reason = self.modules_texts.GetText( reason_id ) - - content = HydrusNetwork.Content( HC.CONTENT_TYPE_MAPPINGS, ( tag, hashes ) ) - - client_to_server_update.AddContent( HC.CONTENT_UPDATE_PETITION, content, reason ) - - - # tag parents - - pending = self._c.execute( 'SELECT child_tag_id, parent_tag_id, reason_id FROM tag_parent_petitions WHERE service_id = ? AND status = ? ORDER BY reason_id LIMIT 1;', ( service_id, HC.CONTENT_STATUS_PENDING ) ).fetchall() - - for ( child_tag_id, parent_tag_id, reason_id ) in pending: - - child_tag = self.modules_tags_local_cache.GetTag( child_tag_id ) - parent_tag = self.modules_tags_local_cache.GetTag( parent_tag_id ) - - reason = self.modules_texts.GetText( reason_id ) - - content = HydrusNetwork.Content( HC.CONTENT_TYPE_TAG_PARENTS, ( child_tag, parent_tag ) ) - - client_to_server_update.AddContent( HC.CONTENT_UPDATE_PEND, content, reason ) - - - petitioned = self._c.execute( 'SELECT child_tag_id, parent_tag_id, reason_id FROM tag_parent_petitions WHERE service_id = ? AND status = ? ORDER BY reason_id LIMIT 100;', ( service_id, HC.CONTENT_STATUS_PETITIONED ) ).fetchall() - - for ( child_tag_id, parent_tag_id, reason_id ) in petitioned: - - child_tag = self.modules_tags_local_cache.GetTag( child_tag_id ) - parent_tag = self.modules_tags_local_cache.GetTag( parent_tag_id ) - - reason = self.modules_texts.GetText( reason_id ) - - content = HydrusNetwork.Content( HC.CONTENT_TYPE_TAG_PARENTS, ( child_tag, parent_tag ) ) - - client_to_server_update.AddContent( HC.CONTENT_UPDATE_PETITION, content, reason ) - - - # tag siblings - - pending = self._c.execute( 'SELECT bad_tag_id, good_tag_id, reason_id FROM tag_sibling_petitions WHERE service_id = ? AND status = ? ORDER BY reason_id LIMIT 100;', ( service_id, HC.CONTENT_STATUS_PENDING ) ).fetchall() - - for ( bad_tag_id, good_tag_id, reason_id ) in pending: - - bad_tag = self.modules_tags_local_cache.GetTag( bad_tag_id ) - good_tag = self.modules_tags_local_cache.GetTag( good_tag_id ) - - reason = self.modules_texts.GetText( reason_id ) - - content = HydrusNetwork.Content( HC.CONTENT_TYPE_TAG_SIBLINGS, ( bad_tag, good_tag ) ) - - client_to_server_update.AddContent( HC.CONTENT_UPDATE_PEND, content, reason ) - - - petitioned = self._c.execute( 'SELECT bad_tag_id, good_tag_id, reason_id FROM tag_sibling_petitions WHERE service_id = ? AND status = ? ORDER BY reason_id LIMIT 100;', ( service_id, HC.CONTENT_STATUS_PETITIONED ) ).fetchall() - - for ( bad_tag_id, good_tag_id, reason_id ) in petitioned: - - bad_tag = self.modules_tags_local_cache.GetTag( bad_tag_id ) - good_tag = self.modules_tags_local_cache.GetTag( good_tag_id ) - - reason = self.modules_texts.GetText( reason_id ) - - content = HydrusNetwork.Content( HC.CONTENT_TYPE_TAG_SIBLINGS, ( bad_tag, good_tag ) ) - - client_to_server_update.AddContent( HC.CONTENT_UPDATE_PETITION, content, reason ) + if account.HasPermission( HC.CONTENT_TYPE_TAG_SIBLINGS, HC.PERMISSION_ACTION_PETITION ): + + pending = self._c.execute( 'SELECT bad_tag_id, good_tag_id, reason_id FROM tag_sibling_petitions WHERE service_id = ? AND status = ? ORDER BY reason_id LIMIT 100;', ( service_id, HC.CONTENT_STATUS_PENDING ) ).fetchall() + + for ( bad_tag_id, good_tag_id, reason_id ) in pending: + + bad_tag = self.modules_tags_local_cache.GetTag( bad_tag_id ) + good_tag = self.modules_tags_local_cache.GetTag( good_tag_id ) + + reason = self.modules_texts.GetText( reason_id ) + + content = HydrusNetwork.Content( HC.CONTENT_TYPE_TAG_SIBLINGS, ( bad_tag, good_tag ) ) + + client_to_server_update.AddContent( HC.CONTENT_UPDATE_PEND, content, reason ) + + + petitioned = self._c.execute( 'SELECT bad_tag_id, good_tag_id, reason_id FROM tag_sibling_petitions WHERE service_id = ? AND status = ? ORDER BY reason_id LIMIT 100;', ( service_id, HC.CONTENT_STATUS_PETITIONED ) ).fetchall() + + for ( bad_tag_id, good_tag_id, reason_id ) in petitioned: + + bad_tag = self.modules_tags_local_cache.GetTag( bad_tag_id ) + good_tag = self.modules_tags_local_cache.GetTag( good_tag_id ) + + reason = self.modules_texts.GetText( reason_id ) + + content = HydrusNetwork.Content( HC.CONTENT_TYPE_TAG_SIBLINGS, ( bad_tag, good_tag ) ) + + client_to_server_update.AddContent( HC.CONTENT_UPDATE_PETITION, content, reason ) + + elif service_type == HC.FILE_REPOSITORY: - result = self._c.execute( 'SELECT hash_id FROM file_transfers WHERE service_id = ?;', ( service_id, ) ).fetchone() - - if result is not None: + if HC.CONTENT_TYPE_FILES in content_types: - ( hash_id, ) = result + if account.HasPermission( HC.CONTENT_TYPE_FILES, HC.PERMISSION_ACTION_CREATE ): + + result = self._c.execute( 'SELECT hash_id FROM file_transfers WHERE service_id = ?;', ( service_id, ) ).fetchone() + + if result is not None: + + ( hash_id, ) = result + + media_result = self._GetMediaResults( ( hash_id, ) )[ 0 ] + + return media_result + + - media_result = self._GetMediaResults( ( hash_id, ) )[ 0 ] - - return media_result - - - petitioned = list( HydrusData.BuildKeyToListDict( self._c.execute( 'SELECT reason_id, hash_id FROM file_petitions WHERE service_id = ? ORDER BY reason_id LIMIT 100;', ( service_id, ) ) ).items() ) - - for ( reason_id, hash_ids ) in petitioned: - - hashes = self.modules_hashes_local_cache.GetHashes( hash_ids ) - - reason = self.modules_texts.GetText( reason_id ) - - content = HydrusNetwork.Content( HC.CONTENT_TYPE_FILES, hashes ) - - client_to_server_update.AddContent( HC.CONTENT_UPDATE_PETITION, content, reason ) + if account.HasPermission( HC.CONTENT_TYPE_FILES, HC.PERMISSION_ACTION_PETITION ): + + petitioned = list( HydrusData.BuildKeyToListDict( self._c.execute( 'SELECT reason_id, hash_id FROM file_petitions WHERE service_id = ? ORDER BY reason_id LIMIT 100;', ( service_id, ) ) ).items() ) + + for ( reason_id, hash_ids ) in petitioned: + + hashes = self.modules_hashes_local_cache.GetHashes( hash_ids ) + + reason = self.modules_texts.GetText( reason_id ) + + content = HydrusNetwork.Content( HC.CONTENT_TYPE_FILES, hashes ) + + client_to_server_update.AddContent( HC.CONTENT_UPDATE_PETITION, content, reason ) + + diff --git a/hydrus/client/gui/ClientGUI.py b/hydrus/client/gui/ClientGUI.py index de318d77..9aacb33b 100644 --- a/hydrus/client/gui/ClientGUI.py +++ b/hydrus/client/gui/ClientGUI.py @@ -104,6 +104,15 @@ def THREADUploadPending( service_key ): service = HG.client_controller.services_manager.GetService( service_key ) + account = service.GetAccount() + + if account.IsUnknown(): + + HydrusData.ShowText( 'Your account is currently unsynced, so the upload was cancelled. Please refresh the account under _review services_.' ) + + return + + service_name = service.GetName() service_type = service.GetServiceType() @@ -113,11 +122,92 @@ def THREADUploadPending( service_key ): nums_pending = HG.client_controller.Read( 'nums_pending' ) - info = nums_pending[ service_key ] + nums_pending_for_this_service = nums_pending[ service_key ] - initial_num_pending = sum( info.values() ) + content_types_for_this_service = set() - result = HG.client_controller.Read( 'pending', service_key ) + if service_type in ( HC.IPFS, HC.FILE_REPOSITORY ): + + content_types_for_this_service = { HC.CONTENT_TYPE_FILES } + + elif service_type == HC.TAG_REPOSITORY: + + content_types_for_this_service = { HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_TYPE_TAG_PARENTS, HC.CONTENT_TYPE_TAG_SIBLINGS } + + + if service_type in HC.REPOSITORIES: + + unauthorised_content_types = set() + content_types_to_request = set() + + content_types_to_count_types_and_permissions = { + HC.CONTENT_TYPE_FILES : ( ( HC.SERVICE_INFO_NUM_PENDING_FILES, HC.PERMISSION_ACTION_CREATE ), ( HC.SERVICE_INFO_NUM_PETITIONED_FILES, HC.PERMISSION_ACTION_PETITION ) ), + HC.CONTENT_TYPE_MAPPINGS : ( ( HC.SERVICE_INFO_NUM_PENDING_MAPPINGS, HC.PERMISSION_ACTION_CREATE ), ( HC.SERVICE_INFO_NUM_PETITIONED_MAPPINGS, HC.PERMISSION_ACTION_PETITION ) ), + HC.CONTENT_TYPE_TAG_PARENTS : ( ( HC.SERVICE_INFO_NUM_PENDING_TAG_PARENTS, HC.PERMISSION_ACTION_PETITION ), ( HC.SERVICE_INFO_NUM_PETITIONED_TAG_PARENTS, HC.PERMISSION_ACTION_PETITION ) ), + HC.CONTENT_TYPE_TAG_SIBLINGS : ( ( HC.SERVICE_INFO_NUM_PENDING_TAG_SIBLINGS, HC.PERMISSION_ACTION_PETITION ), ( HC.SERVICE_INFO_NUM_PETITIONED_TAG_SIBLINGS, HC.PERMISSION_ACTION_PETITION ) ) + } + + for content_type in content_types_for_this_service: + + for ( count_type, permission ) in content_types_to_count_types_and_permissions[ content_type ]: + + if count_type not in nums_pending_for_this_service: + + continue + + + num_pending = nums_pending_for_this_service[ count_type ] + + if num_pending == 0: + + continue + + + if account.HasPermission( content_type, permission ): + + content_types_to_request.add( content_type ) + + else: + + unauthorised_content_types.add( content_type ) + + + + + if len( unauthorised_content_types ) > 0: + + message = 'Unfortunately, your account ({}) does not have full permission to upload all your pending content of type ({})!'.format( + account.GetAccountType().GetTitle(), + ', '.join( ( HC.content_type_string_lookup[ content_type ] for content_type in unauthorised_content_types ) ) + ) + + message += os.linesep * 2 + message += 'If you are currently using a public, read-only account (such as with the PTR), please check this service under _manage services_ and see if the server allows you to auto-create a more powerful account to replace the public one. If accounts cannot be automatically created, you may have to contact the server owner directly.' + message += os.linesep * 2 + message += 'If you think your account does have this permission, try refreshing it under _review services_.' + + unauthorised_job_key = ClientThreading.JobKey() + + unauthorised_job_key.SetVariable( 'popup_title', 'some data was not uploaded!' ) + + unauthorised_job_key.SetVariable( 'popup_text_1', message ) + + if len( content_types_to_request ) > 0: + + unauthorised_job_key.Delete( 5 ) + + + HG.client_controller.pub( 'message', unauthorised_job_key ) + + + else: + + content_types_to_request = content_types_for_this_service + + + initial_num_pending = sum( nums_pending_for_this_service.values() ) + + result = HG.client_controller.Read( 'pending', service_key, content_types_to_request ) HG.client_controller.pub( 'message', job_key ) @@ -125,9 +215,9 @@ def THREADUploadPending( service_key ): nums_pending = HG.client_controller.Read( 'nums_pending' ) - info = nums_pending[ service_key ] + nums_pending_for_this_service = nums_pending[ service_key ] - remaining_num_pending = sum( info.values() ) + remaining_num_pending = sum( nums_pending_for_this_service.values() ) # sometimes more come in while we are pending, -754/1,234 ha ha num_to_do = max( initial_num_pending, remaining_num_pending ) @@ -238,7 +328,7 @@ def THREADUploadPending( service_key ): HG.client_controller.WaitUntilViewFree() - result = HG.client_controller.Read( 'pending', service_key ) + result = HG.client_controller.Read( 'pending', service_key, content_types_to_request ) job_key.DeleteVariable( 'popup_gauge_1' ) @@ -247,7 +337,15 @@ def THREADUploadPending( service_key ): HydrusData.Print( job_key.ToString() ) job_key.Finish() - job_key.Delete( 5 ) + + if len( content_types_to_request ) == 0: + + job_key.Delete() + + else: + + job_key.Delete( 5 ) + except Exception as e: @@ -4455,6 +4553,10 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p HG.daemon_report_mode = not HG.daemon_report_mode + elif name == 'cache_report_mode': + + HG.cache_report_mode = not HG.cache_report_mode + elif name == 'callto_profile_mode': HG.callto_profile_mode = not HG.callto_profile_mode @@ -5638,6 +5740,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p report_modes = QW.QMenu( debug ) ClientGUIMenus.AppendMenuCheckItem( report_modes, 'callto report mode', 'Report whenever the thread pool is given a task.', HG.callto_report_mode, self._SwitchBoolean, 'callto_report_mode' ) + ClientGUIMenus.AppendMenuCheckItem( report_modes, 'cache report mode', 'Have the image and thumb caches report their operation.', HG.cache_report_mode, self._SwitchBoolean, 'cache_report_mode' ) ClientGUIMenus.AppendMenuCheckItem( report_modes, 'daemon report mode', 'Have the daemons report whenever they fire their jobs.', HG.daemon_report_mode, self._SwitchBoolean, 'daemon_report_mode' ) ClientGUIMenus.AppendMenuCheckItem( report_modes, 'db report mode', 'Have the db report query information, where supported.', HG.db_report_mode, self._SwitchBoolean, 'db_report_mode' ) ClientGUIMenus.AppendMenuCheckItem( report_modes, 'file import report mode', 'Have the db and file manager report file import progress.', HG.file_import_report_mode, self._SwitchBoolean, 'file_import_report_mode' ) diff --git a/hydrus/client/gui/ClientGUIManagement.py b/hydrus/client/gui/ClientGUIManagement.py index 14baeb78..8733371d 100644 --- a/hydrus/client/gui/ClientGUIManagement.py +++ b/hydrus/client/gui/ClientGUIManagement.py @@ -23,8 +23,6 @@ from hydrus.client import ClientParsing from hydrus.client import ClientPaths from hydrus.client import ClientSearch from hydrus.client import ClientThreading -from hydrus.client.gui import ClientGUICanvas -from hydrus.client.gui import ClientGUICanvasFrame from hydrus.client.gui import ClientGUICore as CGC from hydrus.client.gui import ClientGUIDialogs from hydrus.client.gui import ClientGUIDialogsQuick @@ -40,6 +38,8 @@ from hydrus.client.gui import ClientGUIGallerySeedLog from hydrus.client.gui import ClientGUIScrolledPanelsEdit from hydrus.client.gui import ClientGUITopLevelWindowsPanels from hydrus.client.gui import QtPorting as QP +from hydrus.client.gui.canvas import ClientGUICanvas +from hydrus.client.gui.canvas import ClientGUICanvasFrame from hydrus.client.gui.lists import ClientGUIListBoxes from hydrus.client.gui.lists import ClientGUIListConstants as CGLC from hydrus.client.gui.lists import ClientGUIListCtrl diff --git a/hydrus/client/gui/ClientGUIPages.py b/hydrus/client/gui/ClientGUIPages.py index f8843f3a..3e81dbe4 100644 --- a/hydrus/client/gui/ClientGUIPages.py +++ b/hydrus/client/gui/ClientGUIPages.py @@ -16,7 +16,6 @@ from hydrus.client import ClientConstants as CC from hydrus.client import ClientSearch from hydrus.client import ClientThreading from hydrus.client.gui import ClientGUIAsync -from hydrus.client.gui import ClientGUICanvas from hydrus.client.gui import ClientGUICore as CGC from hydrus.client.gui import ClientGUIDialogs from hydrus.client.gui import ClientGUIDialogsQuick @@ -26,6 +25,7 @@ from hydrus.client.gui import ClientGUIMenus from hydrus.client.gui import ClientGUIResults from hydrus.client.gui import ClientGUIShortcuts from hydrus.client.gui import QtPorting as QP +from hydrus.client.gui.canvas import ClientGUICanvas RESERVED_SESSION_NAMES = { '', 'just a blank page', 'last session', 'exit session' } @@ -853,7 +853,10 @@ class Page( QW.QSplitter ): def qt_code_status( status ): - self._SetPrettyStatus( status ) + if not self._initialised: + + self._SetPrettyStatus( status ) + controller = self._controller @@ -885,6 +888,8 @@ class Page( QW.QSplitter ): def publish_callable( media_results ): + self._SetPrettyStatus( '' ) + if self._management_controller.IsImporter(): file_service_key = CC.LOCAL_FILE_SERVICE_KEY diff --git a/hydrus/client/gui/ClientGUIResults.py b/hydrus/client/gui/ClientGUIResults.py index 760173b5..f5c5295d 100644 --- a/hydrus/client/gui/ClientGUIResults.py +++ b/hydrus/client/gui/ClientGUIResults.py @@ -24,8 +24,6 @@ from hydrus.client.media import ClientMedia from hydrus.client import ClientPaths from hydrus.client import ClientSearch from hydrus.client.gui import ClientGUIDragDrop -from hydrus.client.gui import ClientGUICanvas -from hydrus.client.gui import ClientGUICanvasFrame from hydrus.client.gui import ClientGUICore as CGC from hydrus.client.gui import ClientGUIDialogs from hydrus.client.gui import ClientGUIDialogsManage @@ -42,6 +40,8 @@ from hydrus.client.gui import ClientGUIShortcuts from hydrus.client.gui import ClientGUITags from hydrus.client.gui import ClientGUITopLevelWindowsPanels from hydrus.client.gui import QtPorting as QP +from hydrus.client.gui.canvas import ClientGUICanvas +from hydrus.client.gui.canvas import ClientGUICanvasFrame from hydrus.client.gui.networking import ClientGUIHydrusNetwork from hydrus.client.metadata import ClientTags diff --git a/hydrus/client/gui/ClientGUIScrolledPanelsManagement.py b/hydrus/client/gui/ClientGUIScrolledPanelsManagement.py index 5eddd001..09c7117b 100644 --- a/hydrus/client/gui/ClientGUIScrolledPanelsManagement.py +++ b/hydrus/client/gui/ClientGUIScrolledPanelsManagement.py @@ -1950,7 +1950,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): self._anchor_and_hide_canvas_drags = QW.QCheckBox( self ) self._touchscreen_canvas_drags_unanchor = QW.QCheckBox( self ) - from hydrus.client.gui import ClientGUICanvas + from hydrus.client.gui.canvas import ClientGUICanvas self._media_viewer_zoom_center = ClientGUICommon.BetterChoice() @@ -2502,12 +2502,24 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): self._estimated_number_fullscreens = QW.QLabel( '', media_panel ) + self._image_tile_cache_size = ClientGUIControls.BytesControl( media_panel ) + self._image_tile_cache_size.valueChanged.connect( self.EventImageTilesUpdate ) + + self._estimated_number_image_tiles = QW.QLabel( '', media_panel ) + self._thumbnail_cache_timeout = ClientGUITime.TimeDeltaButton( media_panel, min = 300, days = True, hours = True, minutes = True ) self._thumbnail_cache_timeout.setToolTip( 'The amount of time after which a thumbnail in the cache will naturally be removed, if it is not shunted out due to a new member exceeding the size limit. Requires restart to kick in.' ) self._image_cache_timeout = ClientGUITime.TimeDeltaButton( media_panel, min = 300, days = True, hours = True, minutes = True ) self._image_cache_timeout.setToolTip( 'The amount of time after which a rendered image in the cache will naturally be removed, if it is not shunted out due to a new member exceeding the size limit. Requires restart to kick in.' ) + self._image_tile_cache_timeout = ClientGUITime.TimeDeltaButton( media_panel, min = 300, hours = True, minutes = True ) + self._image_tile_cache_timeout.setToolTip( 'The amount of time after which a rendered image tile in the cache will naturally be removed, if it is not shunted out due to a new member exceeding the size limit. Requires restart to kick in.' ) + + self._media_viewer_prefetch_delay_base_ms = QP.MakeQSpinBox( media_panel, min = 0, max = 2000 ) + self._media_viewer_prefetch_num_previous = QP.MakeQSpinBox( media_panel, min = 0, max = 5 ) + self._media_viewer_prefetch_num_next = QP.MakeQSpinBox( media_panel, min = 0, max = 5 ) + # buffer_panel = ClientGUICommon.StaticBox( self, 'video buffer' ) @@ -2529,13 +2541,20 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): self._fullscreen_cache_size.setValue( int( HC.options['fullscreen_cache_size'] // 1048576 ) ) + self._image_tile_cache_size.SetValue( self._new_options.GetInteger( 'image_tile_cache_size' ) ) + self._thumbnail_cache_timeout.SetValue( self._new_options.GetInteger( 'thumbnail_cache_timeout' ) ) self._image_cache_timeout.SetValue( self._new_options.GetInteger( 'image_cache_timeout' ) ) + self._image_tile_cache_timeout.SetValue( self._new_options.GetInteger( 'image_tile_cache_timeout' ) ) self._video_buffer_size_mb.setValue( self._new_options.GetInteger( 'video_buffer_size_mb' ) ) self._forced_search_limit.SetValue( self._new_options.GetNoneableInteger( 'forced_search_limit' ) ) + self._media_viewer_prefetch_delay_base_ms.setValue( self._new_options.GetInteger( 'media_viewer_prefetch_delay_base_ms' ) ) + self._media_viewer_prefetch_num_previous.setValue( self._new_options.GetInteger( 'media_viewer_prefetch_num_previous' ) ) + self._media_viewer_prefetch_num_next.setValue( self._new_options.GetInteger( 'media_viewer_prefetch_num_next' ) ) + # vbox = QP.VBoxLayout() @@ -2552,17 +2571,39 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): QP.AddToLayout( fullscreens_sizer, self._fullscreen_cache_size, CC.FLAGS_CENTER_PERPENDICULAR ) QP.AddToLayout( fullscreens_sizer, self._estimated_number_fullscreens, CC.FLAGS_CENTER_PERPENDICULAR ) + image_tiles_sizer = QP.HBoxLayout() + + QP.AddToLayout( image_tiles_sizer, self._image_tile_cache_size, CC.FLAGS_CENTER_PERPENDICULAR ) + QP.AddToLayout( image_tiles_sizer, self._estimated_number_image_tiles, CC.FLAGS_CENTER_PERPENDICULAR ) + video_buffer_sizer = QP.HBoxLayout() QP.AddToLayout( video_buffer_sizer, self._video_buffer_size_mb, CC.FLAGS_CENTER_PERPENDICULAR ) QP.AddToLayout( video_buffer_sizer, self._estimated_number_video_frames, CC.FLAGS_CENTER_PERPENDICULAR ) + text = 'These options are advanced!' + text += os.linesep + text += 'If your navigation back and forth or between zooms is sluggish, the \'tile\' cache is probably the best one to try boosting.' + text += os.linesep + text += 'PROTIP: Do not go crazy here.' + + st = ClientGUICommon.BetterStaticText( media_panel, text ) + + st.setWordWrap( True ) + + media_panel.Add( st, CC.FLAGS_EXPAND_PERPENDICULAR ) + rows = [] - rows.append( ( 'MB memory reserved for thumbnail cache: ', thumbnails_sizer ) ) - rows.append( ( 'MB memory reserved for image cache: ', fullscreens_sizer ) ) - rows.append( ( 'Thumbnail cache timeout: ', self._thumbnail_cache_timeout ) ) - rows.append( ( 'Image cache timeout: ', self._image_cache_timeout ) ) + rows.append( ( 'MB memory reserved for thumbnail cache:', thumbnails_sizer ) ) + rows.append( ( 'MB memory reserved for image cache:', fullscreens_sizer ) ) + rows.append( ( 'MB memory reserved for image tile cache:', image_tiles_sizer ) ) + rows.append( ( 'Thumbnail cache timeout:', self._thumbnail_cache_timeout ) ) + rows.append( ( 'Image cache timeout:', self._image_cache_timeout ) ) + rows.append( ( 'Image tile cache timeout:', self._image_tile_cache_timeout ) ) + rows.append( ( 'Base ms delay for media viewer neighbour render prefetch:', self._media_viewer_prefetch_delay_base_ms ) ) + rows.append( ( 'Num previous to prefetch:', self._media_viewer_prefetch_num_previous ) ) + rows.append( ( 'Num next to prefetch:', self._media_viewer_prefetch_num_next ) ) gridbox = ClientGUICommon.WrapInGrid( media_panel, rows ) @@ -2572,7 +2613,9 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): # - text = 'Hydrus video rendering is CPU intensive.' + text = 'This old option does not apply to mpv! It only applies to the native hydrus animation renderer!' + text += os.linesep + text += 'Hydrus video rendering is CPU intensive.' text += os.linesep text += 'If you have a lot of memory, you can set a generous potential video buffer to compensate.' text += os.linesep @@ -2580,7 +2623,11 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): text += os.linesep text += 'PROTIP: Do not go crazy here.' - buffer_panel.Add( QW.QLabel( text, buffer_panel ), CC.FLAGS_CENTER_PERPENDICULAR ) + st = ClientGUICommon.BetterStaticText( buffer_panel, text ) + + st.setWordWrap( True ) + + buffer_panel.Add( st, CC.FLAGS_EXPAND_PERPENDICULAR ) rows = [] @@ -2614,6 +2661,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): self.EventFullscreensUpdate( self._fullscreen_cache_size.value() ) self.EventThumbnailsUpdate( self._thumbnail_cache_size.value() ) + self.EventImageTilesUpdate() self.EventVideoBufferUpdate( self._video_buffer_size_mb.value() ) @@ -2625,7 +2673,20 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): estimate = ( value * 1048576 ) // estimated_bytes_per_fullscreen - self._estimated_number_fullscreens.setText( '(about {}-{} images)'.format( HydrusData.ToHumanInt( estimate ), HydrusData.ToHumanInt( estimate * 4 ) ) ) + self._estimated_number_fullscreens.setText( '(about {}-{} images the size of your screen)'.format( HydrusData.ToHumanInt( estimate // 2 ), HydrusData.ToHumanInt( estimate * 2 ) ) ) + + + def EventImageTilesUpdate( self ): + + value = self._image_tile_cache_size.GetValue() + + display_size = ClientGUIFunctions.GetDisplaySize( self ) + + estimated_bytes_per_fullscreen = 3 * display_size.width() * display_size.height() + + estimate = value // estimated_bytes_per_fullscreen + + self._estimated_number_image_tiles.setText( '(about {} fullscreens)'.format( HydrusData.ToHumanInt( estimate ) ) ) def EventThumbnailsUpdate( self, value ): @@ -2653,8 +2714,15 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): HC.options[ 'thumbnail_cache_size' ] = self._thumbnail_cache_size.value() * 1048576 HC.options[ 'fullscreen_cache_size' ] = self._fullscreen_cache_size.value() * 1048576 + self._new_options.SetInteger( 'image_tile_cache_size', self._image_tile_cache_size.GetValue() ) + self._new_options.SetInteger( 'thumbnail_cache_timeout', self._thumbnail_cache_timeout.GetValue() ) self._new_options.SetInteger( 'image_cache_timeout', self._image_cache_timeout.GetValue() ) + self._new_options.SetInteger( 'image_tile_cache_timeout', self._image_tile_cache_timeout.GetValue() ) + + self._new_options.SetInteger( 'media_viewer_prefetch_delay_base_ms', self._media_viewer_prefetch_delay_base_ms.value() ) + self._new_options.SetInteger( 'media_viewer_prefetch_num_previous', self._media_viewer_prefetch_num_previous.value() ) + self._new_options.SetInteger( 'media_viewer_prefetch_num_next', self._media_viewer_prefetch_num_next.value() ) self._new_options.SetInteger( 'video_buffer_size_mb', self._video_buffer_size_mb.value() ) diff --git a/hydrus/client/gui/ClientGUIScrolledPanelsReview.py b/hydrus/client/gui/ClientGUIScrolledPanelsReview.py index 216236b7..6f32e781 100644 --- a/hydrus/client/gui/ClientGUIScrolledPanelsReview.py +++ b/hydrus/client/gui/ClientGUIScrolledPanelsReview.py @@ -1880,7 +1880,7 @@ class ReviewDownloaderImport( ClientGUIScrolledPanels.ReviewPanel ): - if len( obj_list ) - num_misc_objects == 0: + if len( gugs ) + len( url_classes ) + len( parsers ) + len( domain_metadatas ) + len( login_scripts ) == 0: if num_misc_objects > 0: diff --git a/hydrus/client/gui/ClientGUITags.py b/hydrus/client/gui/ClientGUITags.py index e0738d0c..d1a1134e 100644 --- a/hydrus/client/gui/ClientGUITags.py +++ b/hydrus/client/gui/ClientGUITags.py @@ -1940,7 +1940,7 @@ class ManageTagsPanel( ClientGUIScrolledPanels.ManagePanel ): tlws = ClientGUIFunctions.GetTLWParents( self ) - from hydrus.client.gui import ClientGUICanvasFrame + from hydrus.client.gui.canvas import ClientGUICanvasFrame command_processed = False @@ -2741,7 +2741,7 @@ class ManageTagParents( ClientGUIScrolledPanels.ManagePanel ): services = list( HG.client_controller.services_manager.GetServices( ( HC.LOCAL_TAG, ) ) ) - services.extend( [ service for service in HG.client_controller.services_manager.GetServices( ( HC.TAG_REPOSITORY, ) ) if service.HasPermission( HC.CONTENT_TYPE_TAG_PARENTS, HC.PERMISSION_ACTION_PETITION ) ] ) + services.extend( HG.client_controller.services_manager.GetServices( ( HC.TAG_REPOSITORY, ) ) ) for service in services: @@ -3528,7 +3528,11 @@ class ManageTagParents( ClientGUIScrolledPanels.ManagePanel ): self._original_statuses_to_pairs = original_statuses_to_pairs self._current_statuses_to_pairs = current_statuses_to_pairs - self._status_st.setText( 'Files with a tag on the left will also be given the tag on the right.' + os.linesep + 'As an experiment, this panel will only display the \'current\' pairs for those tags entered below.' ) + simple_status_text = 'Files with a tag on the left will also be given the tag on the right.' + simple_status_text += os.linesep + simple_status_text += 'As an experiment, this panel will only display the \'current\' pairs for those tags entered below.' + + self._status_st.setText( simple_status_text ) looking_good = True @@ -3597,6 +3601,28 @@ class ManageTagParents( ClientGUIScrolledPanels.ManagePanel ): status_text = s.join( ( service_part, maintenance_part, changes_part ) ) + if not self._i_am_local_tag_service: + + account = self._service.GetAccount() + + if account.IsUnknown(): + + looking_good = False + + s = 'The account for this service is currently unsynced! It is uncertain if you have permission to upload parents! Please try to refresh the account in _review services_.' + + status_text = '{}{}{}'.format( s, os.linesep * 2, status_text ) + + elif not account.HasPermission( HC.CONTENT_TYPE_TAG_PARENTS, HC.PERMISSION_ACTION_PETITION ): + + looking_good = False + + s = 'The account for this service does not seem to have permission to upload parents! You can edit them here for now, but the pending menu will not try to upload any changes you make.' + + status_text = '{}{}{}'.format( s, os.linesep * 2, status_text ) + + + self._sync_status_st.setText( status_text ) if looking_good: @@ -3664,7 +3690,7 @@ class ManageTagSiblings( ClientGUIScrolledPanels.ManagePanel ): services = list( HG.client_controller.services_manager.GetServices( ( HC.LOCAL_TAG, ) ) ) - services.extend( [ service for service in HG.client_controller.services_manager.GetServices( ( HC.TAG_REPOSITORY, ) ) if service.HasPermission( HC.CONTENT_TYPE_TAG_SIBLINGS, HC.PERMISSION_ACTION_PETITION ) ] ) + services.extend( HG.client_controller.services_manager.GetServices( ( HC.TAG_REPOSITORY, ) ) ) for service in services: @@ -4604,6 +4630,28 @@ class ManageTagSiblings( ClientGUIScrolledPanels.ManagePanel ): status_text = s.join( ( service_part, maintenance_part, changes_part ) ) + if not self._i_am_local_tag_service: + + account = self._service.GetAccount() + + if account.IsUnknown(): + + looking_good = False + + s = 'The account for this service is currently unsynced! It is uncertain if you have permission to upload parents! Please try to refresh the account in _review services_.' + + status_text = '{}{}{}'.format( s, os.linesep * 2, status_text ) + + elif not account.HasPermission( HC.CONTENT_TYPE_TAG_SIBLINGS, HC.PERMISSION_ACTION_PETITION ): + + looking_good = False + + s = 'The account for this service does not seem to have permission to upload parents! You can edit them here for now, but the pending menu will not try to upload any changes you make.' + + status_text = '{}{}{}'.format( s, os.linesep * 2, status_text ) + + + self._sync_status_st.setText( status_text ) if looking_good: diff --git a/hydrus/client/gui/canvas/ClientGUICanvas.py b/hydrus/client/gui/canvas/ClientGUICanvas.py index 7b71e7ae..5afa011f 100644 --- a/hydrus/client/gui/canvas/ClientGUICanvas.py +++ b/hydrus/client/gui/canvas/ClientGUICanvas.py @@ -19,13 +19,11 @@ from hydrus.client import ClientData from hydrus.client import ClientDuplicates from hydrus.client import ClientPaths from hydrus.client import ClientSearch -from hydrus.client.gui import ClientGUICanvasMedia from hydrus.client.gui import ClientGUICore as CGC from hydrus.client.gui import ClientGUIDialogs from hydrus.client.gui import ClientGUIDialogsManage from hydrus.client.gui import ClientGUIDialogsQuick from hydrus.client.gui import ClientGUIFunctions -from hydrus.client.gui import ClientGUICanvasHoverFrames from hydrus.client.gui import ClientGUIMedia from hydrus.client.gui import ClientGUIMediaActions from hydrus.client.gui import ClientGUIMediaControls @@ -37,6 +35,8 @@ from hydrus.client.gui import ClientGUIShortcuts from hydrus.client.gui import ClientGUITags from hydrus.client.gui import ClientGUITopLevelWindowsPanels from hydrus.client.gui import QtPorting as QP +from hydrus.client.gui.canvas import ClientGUICanvasHoverFrames +from hydrus.client.gui.canvas import ClientGUICanvasMedia from hydrus.client.media import ClientMedia from hydrus.client.metadata import ClientRatings from hydrus.client.metadata import ClientTags @@ -395,6 +395,8 @@ class Canvas( QW.QWidget ): self._widget_event_filter = QP.WidgetEventFilter( self ) + self._media_container.readyForNeighbourPrefetch.connect( self._PrefetchNeighbours ) + HG.client_controller.sub( self, 'ZoomIn', 'canvas_zoom_in' ) HG.client_controller.sub( self, 'ZoomOut', 'canvas_zoom_out' ) HG.client_controller.sub( self, 'ZoomSwitch', 'canvas_zoom_switch' ) @@ -1051,6 +1053,11 @@ class Canvas( QW.QWidget ): new_media_window_width = new_media_window_size.width() new_media_window_height = new_media_window_size.height() + if new_media_window_width > 32000 or new_media_window_height > 32000: + + return + + my_size = self.size() old_size_bigger = my_size.width() < media_window_width or my_size.height() < media_window_height @@ -1058,46 +1065,49 @@ class Canvas( QW.QWidget ): # - if zoom_center_type_override is None: + if media_window_width > 0 and media_window_height > 0: - zoom_center_type = HG.client_controller.new_options.GetInteger( 'media_viewer_zoom_center' ) - - else: - - zoom_center_type = zoom_center_type_override - - - # viewer center is the default - zoom_centerpoint = QC.QPoint( my_size.width() // 2, my_size.height() // 2 ) - - if zoom_center_type == ZOOM_CENTERPOINT_MEDIA_CENTER: - - zoom_centerpoint = self._media_window_pos + QC.QPoint( media_window_width // 2, media_window_height // 2 ) - - elif zoom_center_type == ZOOM_CENTERPOINT_MEDIA_TOP_LEFT: - - zoom_centerpoint = self._media_window_pos - - elif zoom_center_type == ZOOM_CENTERPOINT_MOUSE: - - mouse_pos = self.mapFromGlobal( QG.QCursor.pos() ) - - if self.rect().contains( mouse_pos ): + if zoom_center_type_override is None: - zoom_centerpoint = mouse_pos + zoom_center_type = HG.client_controller.new_options.GetInteger( 'media_viewer_zoom_center' ) + + else: + + zoom_center_type = zoom_center_type_override + # viewer center is the default + zoom_centerpoint = QC.QPoint( my_size.width() // 2, my_size.height() // 2 ) + + if zoom_center_type == ZOOM_CENTERPOINT_MEDIA_CENTER: + + zoom_centerpoint = self._media_window_pos + QC.QPoint( media_window_width // 2, media_window_height // 2 ) + + elif zoom_center_type == ZOOM_CENTERPOINT_MEDIA_TOP_LEFT: + + zoom_centerpoint = self._media_window_pos + + elif zoom_center_type == ZOOM_CENTERPOINT_MOUSE: + + mouse_pos = self.mapFromGlobal( QG.QCursor.pos() ) + + if self.rect().contains( mouse_pos ): + + zoom_centerpoint = mouse_pos + + - # probably a simpler way to calc this, but hey - widths_centerpoint_is_from_pos = ( zoom_centerpoint.x() - self._media_window_pos.x() ) / media_window_width - heights_centerpoint_is_from_pos = ( zoom_centerpoint.y() - self._media_window_pos.y() ) / media_window_height - - zoom_width_delta = media_window_width - new_media_window_width - zoom_height_delta = media_window_height - new_media_window_height - - centerpoint_adjusted_delta = QC.QPoint( int( zoom_width_delta * widths_centerpoint_is_from_pos ), int( zoom_height_delta * heights_centerpoint_is_from_pos ) ) - - self._media_window_pos += centerpoint_adjusted_delta + # probably a simpler way to calc this, but hey + widths_centerpoint_is_from_pos = ( zoom_centerpoint.x() - self._media_window_pos.x() ) / media_window_width + heights_centerpoint_is_from_pos = ( zoom_centerpoint.y() - self._media_window_pos.y() ) / media_window_height + + zoom_width_delta = media_window_width - new_media_window_width + zoom_height_delta = media_window_height - new_media_window_height + + centerpoint_adjusted_delta = QC.QPoint( int( zoom_width_delta * widths_centerpoint_is_from_pos ), int( zoom_height_delta * heights_centerpoint_is_from_pos ) ) + + self._media_window_pos += centerpoint_adjusted_delta + # @@ -1669,8 +1679,6 @@ class Canvas( QW.QWidget ): self._media_container.SetMedia( self._current_media, initial_size, self._media_window_pos, media_show_action, media_start_paused, media_start_with_embed ) - self._PrefetchNeighbours() - else: self._current_media = None @@ -3448,10 +3456,10 @@ class CanvasMediaList( ClientMedia.ListeningMediaList, CanvasWithHovers ): previous = self._current_media next = self._current_media - delay_base = 0.1 + delay_base = HG.client_controller.new_options.GetInteger( 'media_viewer_prefetch_delay_base_ms' ) / 1000 - num_to_go_back = 3 - num_to_go_forward = 5 + num_to_go_back = HG.client_controller.new_options.GetInteger( 'media_viewer_prefetch_num_previous' ) + num_to_go_forward = HG.client_controller.new_options.GetInteger( 'media_viewer_prefetch_num_next' ) # if media_looked_at nukes the list, we want shorter delays, so do next first @@ -3502,7 +3510,7 @@ class CanvasMediaList( ClientMedia.ListeningMediaList, CanvasWithHovers ): if not image_cache.HasImageRenderer( hash ): - HG.client_controller.CallLaterQtSafe( self, delay, image_cache.GetImageRenderer, media ) + HG.client_controller.CallLaterQtSafe( self, delay, image_cache.PrefetchImageRenderer, media ) diff --git a/hydrus/client/gui/canvas/ClientGUICanvasFrame.py b/hydrus/client/gui/canvas/ClientGUICanvasFrame.py index 4d37b459..78cfc701 100644 --- a/hydrus/client/gui/canvas/ClientGUICanvasFrame.py +++ b/hydrus/client/gui/canvas/ClientGUICanvasFrame.py @@ -6,11 +6,11 @@ from hydrus.core import HydrusConstants as HC from hydrus.core import HydrusGlobals as HG from hydrus.client import ClientApplicationCommand as CAC -from hydrus.client.gui import ClientGUICanvas from hydrus.client.gui import ClientGUIMediaControls from hydrus.client.gui import ClientGUIShortcuts from hydrus.client.gui import ClientGUITopLevelWindows from hydrus.client.gui import QtPorting as QP +from hydrus.client.gui.canvas import ClientGUICanvas class CanvasFrame( ClientGUITopLevelWindows.FrameThatResizesWithHovers ): diff --git a/hydrus/client/gui/canvas/ClientGUICanvasMedia.py b/hydrus/client/gui/canvas/ClientGUICanvasMedia.py index eeedcb0f..8608f505 100644 --- a/hydrus/client/gui/canvas/ClientGUICanvasMedia.py +++ b/hydrus/client/gui/canvas/ClientGUICanvasMedia.py @@ -1,3 +1,4 @@ +import itertools import typing from qtpy import QtCore as QC @@ -871,6 +872,7 @@ class AnimationBar( QW.QWidget ): class MediaContainer( QW.QWidget ): launchMediaViewer = QC.Signal() + readyForNeighbourPrefetch = QC.Signal() def __init__( self, parent, canvas_type, additional_event_filter: QC.QObject ): @@ -906,6 +908,8 @@ class MediaContainer( QW.QWidget ): self._volume_control = ClientGUIMediaControls.VolumeControl( self, self._canvas_type, direction = 'up' ) self._static_image_window = StaticImage( self, self._canvas_type ) + self._static_image_window.readyForNeighbourPrefetch.connect( self.readyForNeighbourPrefetch ) + self._volume_control.adjustSize() self._volume_control.setCursor( QC.Qt.ArrowCursor ) @@ -970,6 +974,8 @@ class MediaContainer( QW.QWidget ): old_media_window = self._media_window destroy_old_media_window = True + do_neighbour_prefetch_emit = True + if self._show_action == CC.MEDIA_VIEWER_ACTION_SHOW_WITH_MPV and not ClientGUIMPV.MPV_IS_AVAILABLE: self._show_action = CC.MEDIA_VIEWER_ACTION_SHOW_OPEN_EXTERNALLY_BUTTON @@ -1007,6 +1013,8 @@ class MediaContainer( QW.QWidget ): self._media_window.SetMedia( self._media ) + do_neighbour_prefetch_emit = False + else: if isinstance( self._media_window, Animation ): @@ -1074,6 +1082,11 @@ class MediaContainer( QW.QWidget ): self.repaint() + if do_neighbour_prefetch_emit: + + self.readyForNeighbourPrefetch.emit() + + def _SizeAndPositionChildren( self ): @@ -1578,6 +1591,7 @@ class OpenExternallyPanel( QW.QWidget ): class StaticImage( QW.QWidget ): launchMediaViewer = QC.Signal() + readyForNeighbourPrefetch = QC.Signal() def __init__( self, parent, canvas_type ): @@ -1592,13 +1606,19 @@ class StaticImage( QW.QWidget ): self._media = None - self._first_background_drawn = False - self._image_renderer = None + self._tile_cache = HG.client_controller.GetCache( 'image_tiles' ) + + self._canvas_tiles = {} + self._is_rendered = False - self._canvas_qt_pixmap = None + self._first_background_drawn = False + + self._canvas_tile_size = QC.QSize( 768, 768 ) + + self._zoom = 1.0 if self._canvas_type == ClientGUICommon.CANVAS_MEDIA_VIEWER: @@ -1612,9 +1632,18 @@ class StaticImage( QW.QWidget ): self._my_shortcut_handler = ClientGUIShortcuts.ShortcutsHandler( self, [ shortcut_set ], catch_mouse = True ) - def _ClearCanvasBitmap( self ): + def _ClearCanvasTileCache( self ): - self._canvas_qt_pixmap = None + if self._media is None: + + self._zoom = 1.0 + + else: + + self._zoom = self.width() / self._media.GetResolution()[ 0 ] + + + self._canvas_tiles = {} self._is_rendered = False @@ -1632,61 +1661,156 @@ class StaticImage( QW.QWidget ): self._first_background_drawn = True - def _TryToDrawCanvasBitmap( self ): + def _DrawTile( self, tile_coordinate ): - if self._image_renderer is not None and self._image_renderer.IsReady(): + ( native_clip_rect, canvas_clip_rect ) = self._GetClipRectsFromTileCoordinates( tile_coordinate ) + + width = canvas_clip_rect.width() + height = canvas_clip_rect.height() + + tile_pixmap = HG.client_controller.bitmap_manager.GetQtPixmap( width, height ) + + painter = QG.QPainter( tile_pixmap ) + + self._DrawBackground( painter ) + + tile = self._tile_cache.GetTile( self._image_renderer, self._media, native_clip_rect, canvas_clip_rect.size() ) + + painter.drawPixmap( 0, 0, tile.qt_pixmap ) + + self._canvas_tiles[ tile_coordinate ] = ( tile_pixmap, canvas_clip_rect.topLeft() ) + + + def _GetClipRectsFromTileCoordinates( self, tile_coordinate ) -> typing.Tuple[ QC.QRect, QC.QRect ]: + + ( tile_x, tile_y ) = tile_coordinate + + ( my_width, my_height ) = ( self.width(), self.height() ) + + ( normal_canvas_width, normal_canvas_height ) = ( self._canvas_tile_size.width(), self._canvas_tile_size.height() ) + + ( media_width, media_height ) = self._media.GetResolution() + + canvas_x = tile_x * self._canvas_tile_size.width() + canvas_y = tile_y * self._canvas_tile_size.height() + + canvas_topLeft = QC.QPoint( canvas_x, canvas_y ) + + canvas_width = normal_canvas_width + + if canvas_x + normal_canvas_width > my_width: - my_size = self.size() + # this is the rightmost tile and should be shrunk - width = my_size.width() - height = my_size.height() + canvas_width = my_width % normal_canvas_width - self._canvas_qt_pixmap = HG.client_controller.bitmap_manager.GetQtPixmap( width, height ) + + canvas_height = normal_canvas_height + + if canvas_y + normal_canvas_height > my_height: - painter = QG.QPainter( self._canvas_qt_pixmap ) + # this is the bottommost tile and should be shrunk - self._DrawBackground( painter ) - - qt_bitmap = self._image_renderer.GetQtImage( self.size() ) - - painter.drawImage( 0, 0, qt_bitmap ) - - self._is_rendered = True + canvas_height = my_height % normal_canvas_height + native_width = canvas_width * self._zoom + + # if we are the last row/column our size is not this! + + canvas_size = QC.QSize( canvas_width, canvas_height ) + + canvas_clip_rect = QC.QRect( canvas_topLeft, canvas_size ) + + native_clip_rect = QC.QRect( canvas_topLeft / self._zoom, canvas_size / self._zoom ) + + return ( native_clip_rect, canvas_clip_rect ) + + + def _GetTileCoordinateFromPoint( self, pos: QC.QPoint ): + + tile_x = pos.x() // self._canvas_tile_size.width() + tile_y = pos.y() // self._canvas_tile_size.height() + + return ( tile_x, tile_y ) + + + def _GetTileCoordinatesInView( self, rect: QC.QRect ): + + topLeft_tile_coordinate = self._GetTileCoordinateFromPoint( rect.topLeft() ) + bottomRight_tile_coordinate = self._GetTileCoordinateFromPoint( rect.bottomRight() ) + + i = itertools.product( + range( topLeft_tile_coordinate[0], bottomRight_tile_coordinate[0] + 1 ), + range( topLeft_tile_coordinate[1], bottomRight_tile_coordinate[1] + 1 ) + ) + + return list( i ) + def ClearMedia( self ): self._media = None self._image_renderer = None - self._ClearCanvasBitmap() + self._ClearCanvasTileCache() self.update() - def paintEvent( self, event ): - - if self._canvas_qt_pixmap is None: - - self._TryToDrawCanvasBitmap() - + def paintEvent( self, event ): painter = QG.QPainter( self ) - if self._canvas_qt_pixmap is None: + if self._image_renderer is None or not self._image_renderer.IsReady(): self._DrawBackground( painter ) - else: + return - painter.drawPixmap( 0, 0, self._canvas_qt_pixmap ) + + dirty_tile_coordinates = self._GetTileCoordinatesInView( event.rect() ) + + for dirty_tile_coordinate in dirty_tile_coordinates: + + if dirty_tile_coordinate not in self._canvas_tiles: + + self._DrawTile( dirty_tile_coordinate ) + + + + for dirty_tile_coordinate in dirty_tile_coordinates: + + ( tile, pos ) = self._canvas_tiles[ dirty_tile_coordinate ] + + painter.drawPixmap( pos, tile ) + + + all_visible_tile_coordinates = self._GetTileCoordinatesInView( self.visibleRegion().boundingRect() ) + + deletee_tile_coordinates = set( self._canvas_tiles.keys() ).difference( all_visible_tile_coordinates ) + + for deletee_tile_coordinate in deletee_tile_coordinates: + + del self._canvas_tiles[ deletee_tile_coordinate ] + + + if not self._is_rendered: + + self.readyForNeighbourPrefetch.emit() + + self._is_rendered = True def resizeEvent( self, event ): - self._ClearCanvasBitmap() + self._ClearCanvasTileCache() + + + def showEvent( self, event ): + + self._ClearCanvasTileCache() def IsRendered( self ): @@ -1734,14 +1858,19 @@ class StaticImage( QW.QWidget ): def SetMedia( self, media ): + if media == self._media: + + return + + + self._ClearCanvasTileCache() + self._media = media image_cache = HG.client_controller.GetCache( 'images' ) self._image_renderer = image_cache.GetImageRenderer( self._media ) - self._ClearCanvasBitmap() - if not self._image_renderer.IsReady(): HG.client_controller.gui.RegisterAnimationUpdateWindow( self ) diff --git a/hydrus/client/gui/canvas/__init__.py b/hydrus/client/gui/canvas/__init__.py index e69de29b..8b137891 100644 --- a/hydrus/client/gui/canvas/__init__.py +++ b/hydrus/client/gui/canvas/__init__.py @@ -0,0 +1 @@ + diff --git a/hydrus/client/gui/services/ClientGUIClientsideServices.py b/hydrus/client/gui/services/ClientGUIClientsideServices.py index e5253b1d..9f2c3094 100644 --- a/hydrus/client/gui/services/ClientGUIClientsideServices.py +++ b/hydrus/client/gui/services/ClientGUIClientsideServices.py @@ -2271,7 +2271,7 @@ class ReviewServiceRestrictedSubPanel( ClientGUICommon.StaticBox ): self._refresh_account_button = ClientGUICommon.BetterButton( self, 'refresh account', self._RefreshAccount ) self._copy_account_key_button = ClientGUICommon.BetterButton( self, 'copy account id', self._CopyAccountKey ) - self._permissions_button = ClientGUIMenuButton.MenuButton( self, 'see special permissions', [] ) + self._permissions_button = ClientGUIMenuButton.MenuButton( self, 'see account permissions', [] ) # diff --git a/hydrus/client/gui/widgets/ClientGUIControls.py b/hydrus/client/gui/widgets/ClientGUIControls.py index 32a529ef..31104edb 100644 --- a/hydrus/client/gui/widgets/ClientGUIControls.py +++ b/hydrus/client/gui/widgets/ClientGUIControls.py @@ -291,6 +291,7 @@ class BytesControl( QW.QWidget ): self._spin.valueChanged.connect( self._HandleValueChanged ) self._unit.currentIndexChanged.connect( self._HandleValueChanged ) + def _HandleValueChanged( self, val ): self.valueChanged.emit() diff --git a/hydrus/client/importing/ClientImportFileSeeds.py b/hydrus/client/importing/ClientImportFileSeeds.py index d47a2f6e..d3221879 100644 --- a/hydrus/client/importing/ClientImportFileSeeds.py +++ b/hydrus/client/importing/ClientImportFileSeeds.py @@ -210,7 +210,14 @@ class FileImportJob( object ): percentage_in = HG.client_controller.new_options.GetInteger( 'video_thumbnail_percentage_in' ) - self._thumbnail_bytes = HydrusFileHandling.GenerateThumbnailBytes( self._temp_path, target_resolution, mime, duration, num_frames, percentage_in = percentage_in ) + try: + + self._thumbnail_bytes = HydrusFileHandling.GenerateThumbnailBytes( self._temp_path, target_resolution, mime, duration, num_frames, percentage_in = percentage_in ) + + except Exception as e: + + raise HydrusExceptions.DamagedOrUnusualFileException( 'Could not render a thumbnail: {}'.format( str( e ) ) ) + if mime in HC.MIMES_WE_CAN_PHASH: diff --git a/hydrus/client/networking/ClientNetworkingJobs.py b/hydrus/client/networking/ClientNetworkingJobs.py index dd5f1f98..4670e255 100644 --- a/hydrus/client/networking/ClientNetworkingJobs.py +++ b/hydrus/client/networking/ClientNetworkingJobs.py @@ -1334,6 +1334,31 @@ class NetworkJob( object ): self._WaitOnConnectionError( 'read timed out' ) + except Exception as e: + + if '\'Retry\' has no attribute' in str( e ): + + # this is that weird requests 2.25.x(?) urllib3 maybe thread safety error + # we'll just try and pause a bit I guess! + + self._current_connection_attempt_number += 1 + + if self._CanReattemptConnection(): + + self.engine.domain_manager.ReportNetworkInfrastructureError( self._url ) + + else: + + raise HydrusExceptions.ConnectionException( 'Could not connect!' ) + + + self._WaitOnConnectionError( 'connection failed, and could not recover neatly' ) + + else: + + raise + + finally: with self._lock: diff --git a/hydrus/core/HydrusConstants.py b/hydrus/core/HydrusConstants.py index 05fff234..2e75c812 100644 --- a/hydrus/core/HydrusConstants.py +++ b/hydrus/core/HydrusConstants.py @@ -81,7 +81,7 @@ options = {} # Misc NETWORK_VERSION = 20 -SOFTWARE_VERSION = 437 +SOFTWARE_VERSION = 438 CLIENT_API_VERSION = 16 SERVER_THUMBNAIL_DIMENSIONS = ( 200, 200 ) diff --git a/hydrus/core/HydrusFlashHandling.py b/hydrus/core/HydrusFlashHandling.py index 36cd732e..8e87da4b 100644 --- a/hydrus/core/HydrusFlashHandling.py +++ b/hydrus/core/HydrusFlashHandling.py @@ -28,8 +28,10 @@ def GetFlashProperties( path ): metadata = hexagonitswfheader.parse( f ) - width = metadata[ 'width' ] - height = metadata[ 'height' ] + # abs since one flash delivered negatives, and hexagonit calcs by going width = ( xmax - xmin ) etc... + + width = abs( metadata[ 'width' ] ) + height = abs( metadata[ 'height' ] ) num_frames = metadata[ 'frames' ] fps = metadata[ 'fps' ] diff --git a/hydrus/core/HydrusGlobals.py b/hydrus/core/HydrusGlobals.py index ec440117..862b709f 100644 --- a/hydrus/core/HydrusGlobals.py +++ b/hydrus/core/HydrusGlobals.py @@ -36,6 +36,7 @@ file_report_mode = False media_load_report_mode = False gui_report_mode = False shortcut_report_mode = False +cache_report_mode = False subprocess_report_mode = False subscription_report_mode = False hover_window_report_mode = False diff --git a/hydrus/core/networking/HydrusNetwork.py b/hydrus/core/networking/HydrusNetwork.py index 09117b03..4cc0f78a 100644 --- a/hydrus/core/networking/HydrusNetwork.py +++ b/hydrus/core/networking/HydrusNetwork.py @@ -473,6 +473,14 @@ class Account( object ): + def IsUnknown( self ): + + with self._lock: + + return self._created == 0 + + + def ReportDataUsed( self, num_bytes ): with self._lock: diff --git a/hydrus/test/TestClientDB.py b/hydrus/test/TestClientDB.py index fc426a9b..0c43d051 100644 --- a/hydrus/test/TestClientDB.py +++ b/hydrus/test/TestClientDB.py @@ -1392,7 +1392,11 @@ class TestClientDB( unittest.TestCase ): old_services = list( services ) - services.append( ClientServices.GenerateService( service_key, HC.TAG_REPOSITORY, 'new tag repo' ) ) + service = ClientServices.GenerateService( service_key, HC.TAG_REPOSITORY, 'new tag repo' ) + + service._account._account_type = HydrusNetwork.AccountType.GenerateAdminAccountType( HC.TAG_REPOSITORY ) + + services.append( service ) self._write( 'update_services', services ) @@ -1408,7 +1412,7 @@ class TestClientDB( unittest.TestCase ): self._write( 'content_updates', service_keys_to_content_updates ) - result = self._read( 'pending', service_key ) + result = self._read( 'pending', service_key, ( HC.CONTENT_TYPE_MAPPINGS, ) ) self.assertIsInstance( result, HydrusNetwork.ClientToServerUpdate ) diff --git a/hydrus/test/TestController.py b/hydrus/test/TestController.py index bd1aa1f7..590e887d 100644 --- a/hydrus/test/TestController.py +++ b/hydrus/test/TestController.py @@ -282,6 +282,13 @@ class Controller( object ): self.tag_display_manager = ClientTagsHandling.TagDisplayManager() self._managers[ 'undo' ] = ClientManagers.UndoManager( self ) + + self._caches = {} + + self._caches[ 'images' ] = ClientCaches.ImageRendererCache( self ) + self._caches[ 'image_tiles' ] = ClientCaches.ImageTileCache( self ) + self._caches[ 'thumbnail' ] = ClientCaches.ThumbnailCache( self ) + self.server_session_manager = HydrusSessions.HydrusSessionManagerServer() self.bitmap_manager = ClientManagers.BitmapManager( self ) @@ -492,6 +499,11 @@ class Controller( object ): return False + def GetCache( self, name ): + + return self._caches[ name ] + + def GetCurrentSessionPageAPIInfoDict( self ): return { diff --git a/requirements.txt b/requirements.txt index ef19dccb..58dc7502 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ PySocks>=1.7.0 python-mpv>=0.4.5 PyYAML>=5.0.0 QtPy>=1.9.0 -requests>=2.23.0 +requests==2.23.0 Send2Trash>=1.5.0 service-identity>=18.1.0 six>=1.14.0