From b1b841cb112182fe6d807d47caed37e60dfe1952 Mon Sep 17 00:00:00 2001 From: Hydrus Network Developer Date: Wed, 23 Nov 2022 15:01:41 -0600 Subject: [PATCH] Version 507 --- client.bat | 12 + client.command | 6 + client.sh | 6 + docs/changelog.md | 79 +++-- docs/client_api.md | 1 + docs/developer_api.md | 136 ++++++-- docs/old_changelog.html | 36 ++ docs/running_from_source.md | 4 +- git_pull.bat | 9 +- git_pull.command | 4 + git_pull.sh | 4 + hydrus/client/ClientDuplicates.py | 7 +- hydrus/client/ClientServices.py | 12 +- hydrus/client/gui/QtInit.py | 135 +++++++- .../metadata/ClientGUIMetadataMigration.py | 11 + .../ClientGUIMetadataMigrationExporters.py | 20 +- .../gui/networking/ClientGUIHydrusNetwork.py | 23 +- .../client/gui/pages/ClientGUIManagement.py | 168 ++++++++- hydrus/client/gui/pages/ClientGUIResults.py | 76 ++++- .../client/gui/search/ClientGUIACDropdown.py | 2 +- .../gui/search/ClientGUIPredicatesSingle.py | 2 + .../networking/ClientLocalServerResources.py | 26 +- hydrus/core/HydrusConstants.py | 4 +- .../HydrusNetworkVariableHandling.py | 12 +- hydrus/server/ServerController.py | 2 +- hydrus/server/ServerDB.py | 319 ++++++++++++++---- hydrus/server/networking/ServerServer.py | 2 + .../networking/ServerServerResources.py | 190 +++++------ hydrus/test/TestClientAPI.py | 122 ++++--- hydrus/test/TestClientDBDuplicates.py | 4 +- hydrus/test/TestController.py | 4 + hydrus/test/TestHydrusServer.py | 78 +---- hydrus/test/TestServerDB.py | 4 +- open_venv.bat | 9 +- setup_desktop.sh | 44 +++ setup_help.bat | 7 + setup_help.command | 5 + setup_help.sh | 5 + setup_venv.bat | 14 + setup_venv.command | 9 + setup_venv.sh | 8 + static/hydrus.desktop | 2 +- 42 files changed, 1219 insertions(+), 404 deletions(-) create mode 100644 setup_desktop.sh diff --git a/client.bat b/client.bat index befdbd6f..9130c4b0 100644 --- a/client.bat +++ b/client.bat @@ -1,8 +1,13 @@ @ECHO off +pushd "%~dp0" + IF NOT EXIST "venv\" ( SET /P gumpf=You need to set up a venv! Check the running from source help for more info! + + popd + EXIT /B 1 ) @@ -12,6 +17,9 @@ CALL venv\Scripts\activate.bat IF ERRORLEVEL 1 ( SET /P gumpf=The venv failed to activate, stopping now! + + popd + EXIT /B 1 ) @@ -24,3 +32,7 @@ start "" "pythonw" client.pyw REM Here is an alternate line that will keep the console open and see live log updates. Useful for boot/live debugging. REM python client.py + +CALL venv\Scripts\deactivate.bat + +popd diff --git a/client.command b/client.command index 7589a11b..d12171b4 100644 --- a/client.command +++ b/client.command @@ -1,7 +1,10 @@ #!/bin/bash +pushd "$(dirname "$0")" + if [ ! -d "venv" ]; then echo "You need to set up a venv! Check the running from source help for more info!" + popd exit 1 fi @@ -9,6 +12,7 @@ source venv/bin/activate if [ $? -ne 0 ]; then echo "The venv failed to activate, stopping now!" + popd exit 1 fi @@ -19,3 +23,5 @@ fi python client.py deactivate + +popd diff --git a/client.sh b/client.sh index 7589a11b..d12171b4 100644 --- a/client.sh +++ b/client.sh @@ -1,7 +1,10 @@ #!/bin/bash +pushd "$(dirname "$0")" + if [ ! -d "venv" ]; then echo "You need to set up a venv! Check the running from source help for more info!" + popd exit 1 fi @@ -9,6 +12,7 @@ source venv/bin/activate if [ $? -ne 0 ]; then echo "The venv failed to activate, stopping now!" + popd exit 1 fi @@ -19,3 +23,5 @@ fi python client.py deactivate + +popd diff --git a/docs/changelog.md b/docs/changelog.md index cd992625..84e13310 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,7 +7,48 @@ title: Changelog !!! note This is the new changelog, only the most recent builds. For all versions, see the [old changelog](old_changelog.html). -## [Version 50](https://github.com/hydrusnetwork/hydrus/releases/tag/v506) +## [Version 507](https://github.com/hydrusnetwork/hydrus/releases/tag/v506) + +### misc + +* fixed an issue where you could set 'all known tags' in the media-tag exporter box in the sidecars system +* if a media-tag exporter in the sidecars system is set to an invalid (missing) tag service, the dialog now protests when you try to OK it. also, when you boot into this dialog, it will now moan about the invalid service. also, new media-tag exporters will always start with a valid local tag service. +* Qt import error states are handled better. when the client boots, the various 'could not find Qt' errors at different qtpy and QtCore import stages are now handled separately. the Qt selected by qtpy, if any, is reported, as is the state of QT_API and whether hydrus thought it was importable. it seems like there have been a couple of users caught by something like system-wide QT_API env variables here, which this should reveal better in boot-crash logs from now on +* all the new setup scripts in the base directory now push their location as the new CWD when they start, and they pop back to your original when they exit. you should be able to call them from anywhere now! +* I've written a 'setup_desktop.sh' install script for Linux users to 'install' a hydrus.desktop file for the current install location to your applications directory. thanks to the user who made the original hydrus.desktop file for the help here +* I fixed the focus when you open a 'edit predicate' panel that only has buttons, like 'has audio'/'no audio'. top button should have focus again, so you can hit enter quick +* added updated link to hydownloader on the client api page + +### dupes apply better to groups of thumbs + +* tl;dr: when the user sets a 'copy both ways' duplicate file status on more than two thumbnails, the duplicate metadata merge options are applied better now +* advanced explanation: previously, all merge updates were calculated before applying the updates, so when applied to a group of interconnected relationships, the nodes that were not directly connected to each other were not syncing data. now, all merge updates are calculated and applied to each pair in turn, and then the whole batch is repeated once more, ensuring two-way transitivity. for instance, if you are set to copy tags in both directions and set 'A is the best' of three files 'ABC', and B has tag 'x' and C has 'y', then previously A would get 'x' and 'y', but B would not get 'y' and C would not get 'x'. now, A gets 'x' before the AC merge is calculated, so A and C get x, and then the whole operation is repeated, so when AB is re-calculated, B now gets 'y' from the updated A. same thing if you set to archive if either file is archived--now that archived status will propagate across the whole group in one action + +### client api + +* the new 'tags' structure in `/get_files/file_metadata` now has the 'all known tags' service's tags +* the 'file_services' structure in `/get_files/file_metadata` now states service name, type, and pretty type, like 'tags' +* `/get_services` now says the service `type` and `type_pretty`, like 'tags'. `/get_services` may be reformatted to a service_key key'd Object at some point, since it uses an old custom human-readable service type as Object key atm and I'd rather we move to the same labels and references for everything, but we'll see +* updated the client api help with more example result data for the above changes (and other stuff like 'all my files') +* updated the client api unit tests to deal with the above changes +* client api version is now 36 + +### server/janitor improvements + +* I recommend server admins update their servers this week! everything old still works, but jannies who update have new abilities that won't work until you update +* the petition processing page now has an 'account id' text field. paste an account id in there, and you'll get the petition counts just for that account! the petitions requested will also only be for that account! +* if you get a 404 on a 'get petition' call (either due to another janitor clearing the last, or from a server count cache miscount), it no longer throws an error. instead, a popup appears for five seconds saying 'hey, there wasn't one after all, please hit refresh counts' + +### boring server improvements + +* refactored the account-fetching routine a little. some behind the scenes account identifier code, which determines an account from a mapping or file record, is now cleaner and more cleanly separated from the 'fetch account from account key' calls. account key is the master account identifier henceforth, and any content lookups will look up the account key and then do normal account lookup after. I will clean this further in the near future +* a new server call looks up the account key from a content object explicitly; this will get more use in future +* all the 'get number of x' server calls now support 'get number of x made by y' for account-specific counting. these numbers aren't cached, but should be fairly quick for janitorial purposes +* same deal for petitions, the server can now fetch petitions by a particular user, if any +* added/updated unit tests for these changes +* general server code cleanup + +## [Version 506](https://github.com/hydrusnetwork/hydrus/releases/tag/v506) ### misc @@ -433,39 +474,3 @@ _almost all the changes this week are only important to server admins and janito * to match the new change in the server, in the client, tag and rating services now store their 'num_files' service info count as the new 'num_file_hashes'. existing numbers will be converted over during update * fixed a probably ten year old bug where 'num pending/petitioned files' had the same enum as 'num pending/petitioned mappings'. never noticed, since no service has done both those things * if the upload pending process fails due to an unusual permission error or similar, the pending menu should now recover and update itself (previously it stayed greyed out) - -## [Version 497](https://github.com/hydrusnetwork/hydrus/releases/tag/v497) - -### misc -* I bulked out the 'star' rating shape a bit more, since the new pentragram, while it looked better than my old 'by-eye' star, was a bit thin. if you prefer the pentagram, this is now selectable as a new shape type under manage services -* the Windows installer is now Qt6 exclusively. there are no special update instructions, it should all just work™ -* the 'manage tag siblings/parents' dialogs now have explicit delete buttons, which should make mass-deletes a little easier to do. some of the background code is cleaned up too, and the 'add' button is moved up to the main button row -* you can now hide all sibling and/or parent text-suffix 'decorators' in the manage tags and autocomplete dropdown taglists, with four new checkboxes under _options->tags_. the right-click menus of these lists let you temporarily show/hide too, just like 'hide/show parent rows' -* when you change the namespace sort in the options, the existing collect-by dropdowns now update instantly (previously, existing pages needed a client restart to see any changes) -* I updated how the media viewer 'note' hover window lays out and does its 'how tall should I be?' estimate. it fits better, being exactly just tall enough in more cases, but it still seems to have trouble with multiple notes that include wrapping text -* added a link to the new flatpak release (easy Linux running-from-source setup) that a user made to the install help -* fixed an issue with the new 'default' file import options when you right-click a watcher/gallery download--the 'show files' menu now correctly adapts to you having a default file import options -* if you are set to elide page tab names, then all pages will tooltip their names on mouseover -* new clients now start with (ctrl+page up/down) as 'move page selection left/right' - -### client api -* the Client API routine that fetches file statuses for a given URL no longer double-checks 'already in db' results against your actual file system. this check is more appropriate to an actual working import process, so it now defaults off in the Client API -* if you want to do this check because you are searching for missing files, you can turn it back on with the new 'doublecheck_file_system' parameter. -* the client api help has been updated to reference this -* the client api's Server header is now "client api/32 (497)". NOT "client api/17". it was stating the hydrus network version erroneously. it now states client api version and software version. if you are able to parse this header, it makes '/api_version' request superfluous -* the client api version is now 32 - -### multiline parsing -* the parser now supports limited multiline parsing. the main changes are hardcoded: the formulae beneath note content parsers and those that do subsidiary page parser splitting no longer remove newlines when they parse. all the parsing UI and the test panels and so on are now aware of this and set flags in all the right places, and parsed notes are now washed through the new trimming/cleaning method, and everything _seems_ to basically work. the main remaining problems is the complicated string processing UI has mixed single/multi-line testing support. some looks great, most gets coerced to single-line just for the previewed test results -* as an example, the default hentai foundry downloader now grabs the artist description as a multi-line note -* the parsing sub-system that extracts cohesive strings from complex html blocks now inserts newlines at 'p' and 'br' tags -* trying to parse clean multiline notes still caused several formatting issues this week, so I have updated the automatic note-washing routine to standardise hydrus notes in several new ways that I hope will not be too disruptive to manually written notes: -* the note washing routine now coerces all newline characters to 'backslash-n', regardless of platform -* the note washing routine now trims each line, so no leading or trailing whitespace anywhere. I am open to changing this in future, maybe for handwritten notes where you really want an indent somewhere, but parsing from complex nested html tags is making a heap of weird extra whitespace, for which this is a clean solution -* the note washing routine now trims newline gaps that are greater than two-newlines. you can split paragraphs by one empty line, but no more -* there may be other issues figuring out cleanly formatted strings from nested html tags--so give it a go and let me know what you think. maybe p and br blocks should always make two newlines, so we have separated paragraphs, maybe I need to parse more blocks, like h1 and friends. any specific example html blocks would also be helpful - -### cleanup -* refactored ClientGUIParsing to its own 'parsing' module and split everything into four less tangled files -* cleaned up a bunch of taglist text presentation code, mostly simplicity and clarity in prep for future updates -* updated the checker options button to use a Qt signal instead of a callable diff --git a/docs/client_api.md b/docs/client_api.md index 2f67af32..60cd4076 100644 --- a/docs/client_api.md +++ b/docs/client_api.md @@ -25,6 +25,7 @@ Once the API is running, go to its entry in _services->review services_. Each ex * [Hydrus Companion](https://gitgud.io/prkc/hydrus-companion): a Chrome/Firefox extension for hydrus that allows easy download queueing as you browse and advanced login support * [Hydrus Web](https://github.com/floogulinc/hydrus-web): a web client for hydrus (allows phone browsing of hydrus) * [Hyshare](https://github.com/floogulinc/hyshare): a way to share small galleries with friends--a replacement for the old 'local booru' system +* [hydownloader](https://gitgud.io/thatfuckingbird/hydownloader): Hydrus-like download system based on gallery-dl. * [LoliSnatcher](https://github.com/NO-ob/LoliSnatcher_Droid): a booru client for Android that can talk to hydrus * [Anime Boxes](https://www.animebox.es/): a booru browser, now supports adding your client as a Hydrus Server * [FlipFlip](https://ififfy.github.io/flipflip/#/): an advanced slideshow interface, now supports hydrus as a source diff --git a/docs/developer_api.md b/docs/developer_api.md index 02c61a38..470a522a 100644 --- a/docs/developer_api.md +++ b/docs/developer_api.md @@ -225,65 +225,94 @@ Response: "local_tags" : [ { "name" : "my tags", - "service_key" : "6c6f63616c2074616773" + "service_key" : "6c6f63616c2074616773", + "type" : 5, + "type_pretty" : "local tag service" }, { "name" : "filenames", - "service_key" : "231a2e992b67101318c410abb6e7d98b6e32050623f138ca93bd4ad2993de31b" + "service_key" : "231a2e992b67101318c410abb6e7d98b6e32050623f138ca93bd4ad2993de31b", + "type" : 5, + "type_pretty" : "local tag service" } ], "tag_repositories" : [ { "name" : "PTR", - "service_key" : "ccb0cf2f9e92c2eb5bd40986f72a339ef9497014a5fb8ce4cea6d6c9837877d9" + "service_key" : "ccb0cf2f9e92c2eb5bd40986f72a339ef9497014a5fb8ce4cea6d6c9837877d9", + "type" : 0, + "type_pretty" : "hydrus tag repository" + } + ], + "file_repositories" : [ + { + "name" : "kamehameha central", + "service_key" : "89295dc26dae3ea7d395a1746a8fe2cb836b9472b97db48024bd05587f32ab0b", + "type" : 1, + "type_pretty" : "hydrus file repository" } ], "local_files" : [ { "name" : "my files", - "service_key" : "6c6f63616c2066696c6573" - } - ], - "local_updates" : [ - { - "name" : "repository updates", - "service_key" : "7265706f7369746f72792075706461746573" - } - ], - "file_repositories" : [], - "all_local_files" : [ - { - "name" : "all local files", - "service_key" : "616c6c206c6f63616c2066696c6573" + "service_key" : "6c6f63616c2066696c6573", + "type" : 2, + "type_pretty" : "local file domain" } ], "all_local_media" : [ { "name" : "all my files", - "service_key" : "616c6c206c6f63616c206d65646961" - } - ], - "all_known_files" : [ - { - "name" : "all known files", - "service_key" : "616c6c206b6e6f776e2066696c6573" - } - ], - "all_known_tags" : [ - { - "name" : "all known tags", - "service_key" : "616c6c206b6e6f776e2074616773" + "service_key" : "616c6c206c6f63616c206d65646961", + "type" : 21, + "type_pretty" : "virtual combined local media service" } ], "trash" : [ { "name" : "trash", - "service_key" : "7472617368" + "service_key" : "7472617368", + "type" : 14, + "type_pretty" : "local trash file domain" + } + ], + "local_updates" : [ + { + "name" : "repository updates", + "service_key" : "7265706f7369746f72792075706461746573", + "type" : 20, + "type_pretty" : "local update file domain" + } + ], + "all_local_files" : [ + { + "name" : "all local files", + "service_key" : "616c6c206c6f63616c2066696c6573", + "type" : 15, + "type_pretty" : "virtual combined local file service" + } + ], + "all_known_files" : [ + { + "name" : "all known files", + "service_key" : "616c6c206b6e6f776e2066696c6573", + "type" : 11, + "type_pretty" : "virtual combined file service" + } + ], + "all_known_tags" : [ + { + "name" : "all known tags", + "service_key" : "616c6c206b6e6f776e2074616773", + "type" : 10, + "type_pretty" : "virtual combined tag service" } ] } ``` - These services may be referred to in various metadata responses or required in request parameters (e.g. where to add tag mappings). Note that a user can rename their services. Much of this Client API uses this renameable 'service name' as service identifier, but I may start using service key, which is non-mutable ID specific to each client. The hardcoded services have shorter service key strings (it is usually just 'all known files' etc.. ASCII-converted to hex), but user-made stuff will have 64-character hex. + These services may be referred to in various metadata responses or required in request parameters (e.g. where to add tag mappings). Note that a user can rename their services. The older parts of the Client API use the renameable 'service name' as service identifier, but wish to move away from this. Please use the hex 'service_key', which is a non-mutable ID specific to each client. The hardcoded services have shorter service key strings (it is usually just 'all known files' etc.. ASCII-converted to hex), but user-made stuff will have 64-character hex. + + Now that I state `type` and `type_pretty` here, I may rearrange this call, probably to make the `service_key` the Object key, rather than the arbitrary 'all_known_tags' strings. ## Adding Files @@ -1537,6 +1566,13 @@ Response: "type_pretty" : "hydrus tag repository", "storage_tags" : {}, "display_tags" : {} + }, + "616c6c206b6e6f776e2074616773" : { + "name" : "all known tags", + "type" : 10, + "type_pretty" : "virtual combined tag service", + "storage_tags" : {}, + "display_tags" : {} } } }, @@ -1559,14 +1595,29 @@ Response: "file_services" : { "current" : { "616c6c206c6f63616c2066696c6573" : { + "name" : "all local files", + "type" : 15, + "type_pretty" : "virtual combined local file service", + "time_imported" : 1641044491 + }, + "616c6c206c6f63616c2066696c6573" : { + "name" : "all my files", + "type" : 21, + "type_pretty" : "virtual combined local media service", "time_imported" : 1641044491 }, "cb072cffbd0340b67aec39e1953c074e7430c2ac831f8e78fb5dfbda6ec8dcbd" : { + "name" : "cool space babes", + "type" : 2, + "type_pretty" : "local file domain", "time_imported" : 1641204220 } }, "deleted" : { "6c6f63616c2066696c6573" : { + "name" : "my files", + "type" : 2, + "type_pretty" : "local file domain", "time_deleted" : 1641204274, "time_imported" : 1641044491 } @@ -1598,6 +1649,10 @@ Response: "37e3849bda234f53b0e9792a036d14d4f3a9a136d1cb939705dbcd5287941db4" : { "0" : ["blonde_hair", "blue_eyes", "looking_at_viewer"], "1" : ["bodysuit"] + }, + "616c6c206b6e6f776e2074616773" : { + "0" : ["samus favourites", "blonde_hair", "blue_eyes", "looking_at_viewer"], + "1" : ["bodysuit"] } }, "service_keys_to_statuses_to_display_tags" : { @@ -1608,6 +1663,10 @@ Response: "37e3849bda234f53b0e9792a036d14d4f3a9a136d1cb939705dbcd5287941db4" : { "0" : ["blonde hair", "blue_eyes", "looking at viewer"], "1" : ["bodysuit", "clothing"] + }, + "616c6c206b6e6f776e2074616773" : { + "0" : ["samus favourites", "favourites", "blonde hair", "blue_eyes", "looking at viewer"], + "1" : ["bodysuit", "clothing"] } }, "tags" : { @@ -1636,6 +1695,19 @@ Response: "0" : ["blonde hair", "blue_eyes", "looking at viewer"], "1" : ["bodysuit", "clothing"] } + }, + "616c6c206b6e6f776e2074616773" : { + "name" : "all known tags", + "type" : 10, + "type_pretty" : "virtual combined tag service", + "storage_tags" : { + "0" : ["samus favourites", "blonde_hair", "blue_eyes", "looking_at_viewer"], + "1" : ["bodysuit"] + }, + "display_tags" : { + "0" : ["samus favourites", "favourites", "blonde hair", "blue_eyes", "looking at viewer"], + "1" : ["bodysuit", "clothing"] + } } } } diff --git a/docs/old_changelog.html b/docs/old_changelog.html index fdf3f540..ea82b5db 100644 --- a/docs/old_changelog.html +++ b/docs/old_changelog.html @@ -33,6 +33,42 @@

changelog