From 8b3ae4ac1a006e955addefb7a8af47a4e2b6336c Mon Sep 17 00:00:00 2001 From: Hydrus Network Developer Date: Wed, 28 Jun 2023 15:29:14 -0500 Subject: [PATCH] Version 533 closes #1379 --- docs/changelog.md | 89 +++--- docs/client_api.md | 9 +- docs/getting_started_searching.md | 15 ++ docs/old_changelog.html | 35 +++ hydrus/client/ClientController.py | 15 +- hydrus/client/ClientFiles.py | 2 +- hydrus/client/ClientParsing.py | 2 +- hydrus/client/ClientRendering.py | 16 +- hydrus/client/ClientVideoHandling.py | 8 - hydrus/client/{ => caches}/ClientCaches.py | 222 +-------------- hydrus/client/caches/ClientCachesBase.py | 238 ++++++++++++++++ hydrus/client/caches/__init__.py | 0 hydrus/client/db/ClientDB.py | 36 ++- hydrus/client/db/ClientDBFilesSearch.py | 17 +- hydrus/client/gui/ClientGUI.py | 50 ++-- hydrus/client/gui/ClientGUIFunctions.py | 2 +- .../gui/ClientGUILocatorSearchProviders.py | 7 +- hydrus/client/gui/ClientGUIPopupMessages.py | 1 + .../client/gui/ClientGUIScrolledPanelsEdit.py | 2 +- hydrus/client/gui/QtInit.py | 56 +--- hydrus/client/gui/QtInitImportTest.py | 54 ++++ hydrus/client/gui/QtPorting.py | 19 +- hydrus/client/gui/canvas/ClientGUICanvas.py | 27 +- .../client/gui/canvas/ClientGUICanvasMedia.py | 17 ++ hydrus/client/gui/canvas/ClientGUIMPV.py | 51 +--- .../client/gui/canvas/ClientGUIMediaVolume.py | 44 +++ .../networking/ClientGUINetworkJobControl.py | 9 +- .../pages/ClientGUIManagementController.py | 14 +- hydrus/client/gui/pages/ClientGUIPages.py | 2 +- hydrus/client/gui/pages/ClientGUIResults.py | 16 +- .../gui/search/ClientGUIPredicatesSingle.py | 2 +- .../services/ClientGUIClientsideServices.py | 6 +- hydrus/client/gui/widgets/ClientGUICommon.py | 5 +- .../client/importing/ClientImportControl.py | 4 +- .../client/importing/ClientImportFileSeeds.py | 36 +-- .../importing/ClientImportGallerySeeds.py | 106 +++++--- hydrus/client/importing/ClientImportLocal.py | 4 +- .../importing/ClientImportSimpleURLs.py | 8 +- .../client/importing/ClientImportWatchers.py | 70 ++--- hydrus/client/importing/ClientImporting.py | 2 +- hydrus/client/media/ClientMedia.py | 12 +- hydrus/client/media/ClientMediaResultCache.py | 26 ++ .../ClientMetadataMigrationImporters.py | 2 +- .../networking/ClientLocalServerResources.py | 2 +- .../networking/ClientNetworkingLogin.py | 2 +- hydrus/core/HydrusAudioHandling.py | 2 +- hydrus/core/HydrusConstants.py | 2 +- hydrus/core/HydrusController.py | 5 +- hydrus/core/HydrusData.py | 9 + hydrus/core/HydrusGlobals.py | 13 +- hydrus/core/HydrusProfiling.py | 4 +- hydrus/core/HydrusThreading.py | 32 +-- hydrus/core/HydrusVideoHandling.py | 6 +- .../interfaces/HydrusControllerInterface.py | 255 ++++++++++++++++++ .../interfaces/HydrusThreadingInterface.py | 100 +++++++ hydrus/core/interfaces/__init__.py | 0 hydrus/core/networking/HydrusNATPunch.py | 2 +- hydrus/test/TestController.py | 4 +- requirements.txt | 4 +- static/build_files/linux/requirements.txt | 5 +- static/build_files/macos/requirements.txt | 7 +- static/build_files/windows/requirements.txt | 5 +- 62 files changed, 1223 insertions(+), 594 deletions(-) rename hydrus/client/{ => caches}/ClientCaches.py (84%) create mode 100644 hydrus/client/caches/ClientCachesBase.py create mode 100644 hydrus/client/caches/__init__.py create mode 100644 hydrus/client/gui/QtInitImportTest.py create mode 100644 hydrus/client/gui/canvas/ClientGUIMediaVolume.py create mode 100644 hydrus/core/interfaces/HydrusControllerInterface.py create mode 100644 hydrus/core/interfaces/HydrusThreadingInterface.py create mode 100644 hydrus/core/interfaces/__init__.py diff --git a/docs/changelog.md b/docs/changelog.md index 178b8780..76774fa2 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,6 +7,46 @@ title: Changelog !!! note This is the new changelog, only the most recent builds. For all versions, see the [old changelog](old_changelog.html). +## [Version 533](https://github.com/hydrusnetwork/hydrus/releases/tag/v533) + +### macOS App crashes + +* unfortunately, last week's eventFilter work did not fix the macOS build's crashing--however, thanks to user help, we figured out that it was some half-hidden auxiliary Qt library that updated in the background starting v530 (the excellently named `PyQt6-Qt6` package). the build script is updated to roll back this version and it seems like things are fixed. this particular issue shouldn't happen again. sorry for the trouble, and let me know if there are any new issues! (issue #1379) + +### misc + +* the download panels in subscription popup windows are now significantly more responsive. ever since the popup manager was embedded into the gui, popup messages were not doing the 'should I update myself?' test correctly, and their network UI was not being updated without other events like surrounding widgets resizing. I was wondering what was going on here for ages--turns out it was regular stupidity +* if an image has width or height > 1024, the 'share->copy' menu now shows a second, 'source lookup' bitmap, with the resolution clipped to 1024x1024 +* 'sort files by hash' can now be sorted asc or desc. this also fixes a bug where it was secretly either sorting asc or desc based on the previous selection. well done to the user who noticed and tested this +* if system:limit=0 is in a search, the search is no longer run--it comes back immediately empty. also, the system:limit edit panel now has a minimum value of 1 to dissuade this state +* the experimental QtMediaPlayer now initialises with the correct volume/mute and updates on volume/mute events. the scanbar and volume control UI are still hidden behind the OpenGL frame for now, but one step forward +* the system that caches media results now hangs on to the most recent 2048 files aggressively for two minutes after initial load. previously, if you refreshed a page of unique files, or did some repeated client api work on files that were not loaded somewhere as thumbs, in the interim periods those media objects were not strictly in non-weak memory anywhere in the client and could have been eligible for clearing out of the cache. now they are a bit more sticky +* added some info on editing predicates and the various undocumented click shortcuts the taglist supports (e.g. ctrl+double-left-click) to the 'getting started with searching and sorting' help page +* added a link to the Client API help for 'Hydrus Video Deduplicator' (https://github.com/appleappleapplenanner/hydrus-video-deduplicator), which neatly discovers duplicate videos in your client and queues them up in the duplicate filter by marking them as 'potential dupes' + +### sub-gallery url network parsing gubbins + +* sub-gallery import objects now get the tags and custom headers that are parsed with them. if the sub-gallery urls are parsed in 'posts' using a subsidiary parser, they only inherit the metadata within their post +* sub-gallery import objects now use their parent gallery urls as referral header +* sub-gallery import objects now inherit the 'can generate more pages' state of their parents (previously it was always 'yes') +* 'next page' gallery urls do not get the tags they are parsed with. this behaviour makes a little less sense, and I suspect it _could_ cause various troubles, so I'll wait for more input, bug reports, and a larger cleanup and overhaul of how metadata is managed and passed down from one item to the next in the downloader system +* generally speaking, when file and gallery import objects have the opportunity to acquire tags or headers, they'll now add to the existing store rather than replace it. this should mean if they both inherit and parse stuff, it won't all overwrite each other. this is all a giant mess so I have a cleanup overhaul planned + +### boring stuff + +* if a critical drive error occurs (e.g. running out of space on a storage drive), the popup message saying 'hey everything just got mega-paused' is now a little clearer about what has happened and how to fix it +* similarly, the specific 'all watchers are paused'-style messages now specifically state 'network->pause to resume!' to point users to this menu to fix this tricky issue. this has frustrated a couple of newer users before +* to reduce confusion, the 'clear orphan files' pre-job now only presents the user one combined dialog +* improved how pages test and recognise that they have changes and thus should be saved--it works faster, and a bunch of false negatives should be removed +* improved the safety routine that ensures multiple-column list data is read-only +* fixed .txt sidecar importers' description labels, which were missing extra text munging +* to relieve some latency stress on session load, pages that are loading their initial files in the background now do so in batches of 64 rather than 256 +* fixed some bad error handling in the master client db update routine +* fixed a scatter of linting problems a user found +* last week's pixiv parser hotfix is reinforced this week for anyone who got the early 532 release +* made some primitive interfaces for the main controller and schedulable job classes and ensured the main hydrusglobals has type hinting for these--now everything that talks to the controller has a _bit_ of an idea what it is actually doing, and as I continue to work on this, we'll get better linting +* moved the client DataCache object and its friends to a new 'caches' module and cleaned some of the code + ## [Version 532](https://github.com/hydrusnetwork/hydrus/releases/tag/v532) ### misc @@ -324,52 +364,3 @@ title: Changelog * fixed an error popup that still said 'run repair invalid tags' instead of 'run fix invalid tags' * the FILE_SERVICES constant now holds the 'all deleted files' virtual domain. this domain keeps slipping my logic, so fingers crossed this helps. also means you can select it in 'system:file service' and stuff now * misc cleaning and linting work - -## [Version 523](https://github.com/hydrusnetwork/hydrus/releases/tag/v523) - -### timestamp editing - -* you can now _right-click->manage->times_ on any file to edit its archived, imported, deleted, previously imported (for undelete), file modified, domain modified, and last viewed times. there's a whole new dialog with new datetime buttons and everything. it only works on single files atm, so it is currently only appropriate for little fixes, and there's a couple advanced things like setting a currently missing deletion time that it can't do yet, but I expect to expand it in future (also ideally with some kind of 'cascade' option for multi-files so you can set a timestamp iteratively (e.g. +1 second per file) over a series of thumbs to force a certain import order sort etc...) -* I added a new shortcut action 'manage file times', for this dialog. like the other media 'manage' shortcuts, you can hit it on the dialog to ok it, too -* when you edit a saved file modified date, I have made it to update the actual file modified date on your disk too. a statement is printed to the log with old/new timestamps, just in case you ever need to recover this -* added system:archived time search predicate! it is under the system:time stub like the other time-based search preds. it works in the system predicate parser too - -### misc - -* fixed a stupid logical typo from 521's overhaul that was causing the advanced file deletion dialog to always set the default file deletion reason! sorry for the trouble, this one slipped through due to a tricky test situation (this data is actually calculated twice on dialog ok, and on the first run it was correct -\_-) -* in the edit system predicate dialogs, when you have a list of 'recent' preds and static useful preds, if one of the recent is supposed to also appear in the statics, it now won't be duped -* fixed a bug in the media object's file locations manager's deletion routine, which wasn't adding and removing the special 'all deleted files' domain at the UI level--not that this shows up in UI much, but the new timestamps UI revealed this -* in the janitorial 'petitions processing' page, the add and delete checkbox lists now no longer have horizontal scrollbars in any situation. previously, either list, but particularly the 'delete', at height 1, could be deceptively obscured by a pop-in scrollbar -* when you change your internal backup location, the dialog now states your current location beforehand. this information was previously not viewable! also, if you select the same location again, the process notes this and exits with no changes made -* all multi-column lists across the program now show a ▲ or ▼ on the column they are currently sorted on! this is one of those things I meant to do for ages; now it is done. -* also, you can now right-click any multi-column list's header for a stub menu. for now it just says the thing's identifier name, but I'll start hanging things off here like individual section-size reset and, in time, finally play around with 'select columns' tech -* all menus across the program now send their longer description text to the main window status bar. until now (at least in Qt, I forget wx), this has only been true for the menubar menus -* all menus across the program now have tooltips turned on. any command with description text, which is I think pretty much all of them, will present its full written description on hover. this may end up being annoying, so let me know what you think - -### client api - -* fixed an issue in the client api where it wasn't returning `file_metadata` results in the same file order you asked for. sorry for the trouble--this was softly intended, previously, but I forgot to make sure it stayed true. it also now folds in 'missing' hashes with null ids in the same position you asked for -* a new suite of unit tests check this explicitly for all the typical parameter/response types, and the new missing-hash insertion order--it shouldn't happen again! -* just to be safe, since this is a new feature, client api version is now 44 - -### boring code updates/cleanup - -* wrote a new serialisable 'timestamp data' object to hold the various hydrus timestamps: archived, imported, deleted, previously imported, file modified, domain modified, aggregate modified, and last viewed time -* rewrote the timestamp content update pipeline to use 'timestamp data' object -* wrote a new database module for timestamp management off the file metadata module and migrated the domain-based modified timestamp code to it -* migrated the 'archive time' timestamp-handling from the inbox module to the new timestamp module -* migrated the media result timestamp-manager construction routine all down to the new timestamp module -* migrated the aggregate modified time file search code to the new timestamp module and added archived time search too -* wrote some UI for timestamp editing, whacked some copy/paste buttons on it too -* moved all current/deleted timestamp handling down from the locations manager to the timestamp manager and split off 'previously imported' time, which is used to preserve import timestamp for undelete events, into its own thing rather than a tacked-on hack for deleted timestamps -* moved all the location manager location timestamp tracking down to the timestamp manager -* the media result is now initialised with and handles an explicit copy of the timestamp manager, which is now shared to both location manager and file viewing stats manager, with duplication and merging code updated to handle this shared situation -* moved all the media/preview 'last view time' tracking down from the file viewing stats manager to the timestamp manager, which FVS now received on initialisation -* all media-based timestamp inspection now goes through the timestamp manager -* collections now track some aggregate timestamps a bit better, and they now calculate a archived time--not sure if it is useful, but they know it now -* updated all parts of the timestamp system to use the same shared enums -* cleaned the timestamp code generally -* cleaned some file service update code generally -* moved the main file viewing stats fetching routine for MediaResult building down to the file viewing stats module -* updated the old custom gridbox layout to handle multiple-column-spanning controls -* went through all the bash scripts and fixed some issues my IDE linter was moaning about. -r on reads, quotes around variable names, 4-space indenting, and neater testing of program return states diff --git a/docs/client_api.md b/docs/client_api.md index 7df46853..fac20a6c 100644 --- a/docs/client_api.md +++ b/docs/client_api.md @@ -29,11 +29,12 @@ Once the API is running, go to its entry in _services->review services_. Each ex * [LoliSnatcher](https://github.com/NO-ob/LoliSnatcher_Droid): a booru client for Android that can talk to hydrus * [Anime Boxes](https://www.animebox.es/): a booru browser, now supports adding your client as a Hydrus Server * [FlipFlip](https://ififfy.github.io/flipflip/#/): an advanced slideshow interface, now supports hydrus as a source +* [hydrus-dd](https://gitgud.io/koto/hydrus-dd): DeepDanbooru neural network tagging for Hydrus +* [Hydrus Video Deduplicator](https://github.com/appleappleapplenanner/hydrus-video-deduplicator): Discovers duplicate videos in your client and queues them for the duplicate filter +* [Hydrus Archive Delete](https://gitgud.io/koto/hydrus-archive-delete): Archive/Delete filter in your web browser +* [tagrank](https://github.com/matjojo/tagrank): Shows you comparison images and cleverly ranks your favourite tag. +* [hyextract](https://github.com/floogulinc/hyextract): Extract archives from Hydrus and reimport with tags and URL associations * [Send to Hydrus](https://github.com/Wyrrrd/send-to-hydrus): send URLs from your Android device to your client * [Iwara-Hydrus](https://github.com/GoAwayNow/Iwara-Hydrus): a userscript to simplify sending Iwara videos to Hydrus Network -* [tagrank](https://github.com/matjojo/tagrank): Shows you comparison images and cleverly ranks your favourite tag. -* [Hydrus Archive Delete](https://gitgud.io/koto/hydrus-archive-delete): Archive/Delete filter in your web browser -* [hydrus-dd](https://gitgud.io/koto/hydrus-dd): DeepDanbooru neural network tagging for Hydrus -* [hyextract](https://github.com/floogulinc/hyextract): Extract archives from Hydrus and reimport with tags and URL associations * [dolphin-hydrus-actions](https://gitgud.io/prkc/dolphin-hydrus-actions): Adds Hydrus right-click context menu actions to Dolphin file manager. * [more projects on github](https://github.com/stars/hydrusnetwork/lists/hydrus-related-projects) diff --git a/docs/getting_started_searching.md b/docs/getting_started_searching.md index a3fef00d..5235b73a 100644 --- a/docs/getting_started_searching.md +++ b/docs/getting_started_searching.md @@ -67,6 +67,21 @@ This is particularly useful if you have a number of files with commonly structur In this case, selecting the `title:cool pic*` predicate will return all three images in the same search, where you can conveniently give them some more-easily searched tags like `series:cool pic` and `page:1`, `page:2`, `page:3`. +### Editing Predicates + +You can edit any selected 'active' search predicates by either its Right-Click menu or through Shift+Double-Left-Click on the selection. For simple tags, this means just changing the text (and, say, adding/removing a leading hyphen for negation/inclusion), but any 'system' predicate can be fully edited with its original panel. If you entered 'system:filesize < 200KB' and want to make it a little bigger, don't delete and re-add--just edit the existing one in place. + +### Other Shortcuts + +These will eventually be migrated to the shortcut system where they will be more visible and changeable, but for now: + +- Left-Click on any taglist is draggable, if you want to select multiple tags quickly. +- Shift+Left-Click across any taglist will do a multi-select. This click is also draggable. +- Ctrl+Left-Click on any taglist will add to or remove from the selection. This is draggable, and if you start on a 'remove', the drag will be a 'remove' drag. Play with it--you'll see how it works. +- Double-Left-Click on one or more tags in the 'selection tags' box moves them to the active search box. Doing the same on the active search box removes them. +- Ctrl+Double-Left-Click on one or more tags in the 'selection tags' box will add their negation (i.e. '-skirt'). +- Shift+Double-Left-Click on more than one tags in the 'selection tags' box will add their 'OR' to the active search box. What's an OR? Well: + ## OR searching Searches find files that match every search 'predicate' in the list (it is an **AND** search), which makes it difficult to search for files that include one **OR** another tag. For example the query `red eyes` **AND** `green eyes` (aka what you get if you enter each tag by itself) will only find files that has both tags. While the query `red eyes` **OR** `green eyes` will present you with files that are tagged with red eyes or green eyes, or both. diff --git a/docs/old_changelog.html b/docs/old_changelog.html index 1a186427..51a31f59 100644 --- a/docs/old_changelog.html +++ b/docs/old_changelog.html @@ -34,6 +34,41 @@

changelog