From 43ae2c2593c8e900b40460191b8379229578f52c Mon Sep 17 00:00:00 2001 From: Hydrus Network Developer Date: Wed, 19 Apr 2023 15:38:13 -0500 Subject: [PATCH] Version 524 closes #1203 --- docs/changelog.md | 145 ++--- docs/getting_started_downloading.md | 21 +- docs/old_changelog.html | 74 ++- hydrus/client/ClientAPI.py | 13 +- hydrus/client/ClientApplicationCommand.py | 3 +- hydrus/client/ClientCaches.py | 23 +- hydrus/client/ClientController.py | 75 ++- hydrus/client/ClientDaemons.py | 1 + hydrus/client/ClientData.py | 12 - hydrus/client/ClientDefaults.py | 7 +- hydrus/client/ClientDownloading.py | 1 + hydrus/client/ClientDuplicates.py | 156 ++++- hydrus/client/ClientFiles.py | 28 +- hydrus/client/ClientImageHandling.py | 1 + hydrus/client/ClientLocation.py | 3 + hydrus/client/ClientMaintenance.py | 7 +- hydrus/client/ClientManagers.py | 3 +- hydrus/client/ClientMigration.py | 30 +- hydrus/client/ClientOptions.py | 5 +- hydrus/client/ClientParsing.py | 5 +- hydrus/client/ClientRendering.py | 1 + hydrus/client/ClientSearch.py | 44 +- .../ClientSearchParseSystemPredicates.py | 2 + hydrus/client/ClientSerialisable.py | 1 + hydrus/client/ClientServices.py | 53 +- hydrus/client/ClientStrings.py | 56 +- hydrus/client/ClientThreading.py | 41 +- hydrus/client/ClientTime.py | 105 ++- hydrus/client/ClientVideoHandling.py | 1 + hydrus/client/db/ClientDB.py | 118 ++-- hydrus/client/db/ClientDBDefinitionsCache.py | 4 +- hydrus/client/db/ClientDBFilesInbox.py | 3 +- hydrus/client/db/ClientDBFilesMaintenance.py | 1 + .../db/ClientDBFilesMaintenanceQueue.py | 5 +- hydrus/client/db/ClientDBFilesMetadataRich.py | 7 +- .../client/db/ClientDBFilesPhysicalStorage.py | 1 + hydrus/client/db/ClientDBFilesSearch.py | 1 + hydrus/client/db/ClientDBFilesStorage.py | 3 +- hydrus/client/db/ClientDBFilesTimestamps.py | 1 + hydrus/client/db/ClientDBFilesViewingStats.py | 1 + hydrus/client/db/ClientDBMaintenance.py | 15 +- ...ientDBMappingsCacheCombinedFilesDisplay.py | 1 + ...ientDBMappingsCacheCombinedFilesStorage.py | 1 + .../ClientDBMappingsCacheSpecificDisplay.py | 1 + .../ClientDBMappingsCacheSpecificStorage.py | 4 +- hydrus/client/db/ClientDBMaster.py | 1 + hydrus/client/db/ClientDBModule.py | 1 + hydrus/client/db/ClientDBNotesMap.py | 1 + hydrus/client/db/ClientDBRepositories.py | 12 +- hydrus/client/db/ClientDBSerialisable.py | 7 +- hydrus/client/db/ClientDBServicePaths.py | 1 + hydrus/client/db/ClientDBServices.py | 1 + hydrus/client/db/ClientDBSimilarFiles.py | 8 +- hydrus/client/db/ClientDBTagDisplay.py | 7 +- hydrus/client/db/ClientDBTagParents.py | 1 + hydrus/client/db/ClientDBTagSearch.py | 8 +- hydrus/client/db/ClientDBTagSiblings.py | 1 + hydrus/client/db/ClientDBURLMap.py | 1 + .../client/exporting/ClientExportingFiles.py | 189 +++++- hydrus/client/gui/ClientGUI.py | 68 +- hydrus/client/gui/ClientGUIAPI.py | 3 +- hydrus/client/gui/ClientGUIAsync.py | 1 + hydrus/client/gui/ClientGUIDialogs.py | 7 +- hydrus/client/gui/ClientGUIDialogsManage.py | 3 +- hydrus/client/gui/ClientGUIDownloaders.py | 1 + hydrus/client/gui/ClientGUIDragDrop.py | 1 + hydrus/client/gui/ClientGUIDuplicates.py | 1 + hydrus/client/gui/ClientGUIFileSeedCache.py | 9 +- hydrus/client/gui/ClientGUIGallerySeedLog.py | 6 +- hydrus/client/gui/ClientGUILogin.py | 11 +- hydrus/client/gui/ClientGUIMedia.py | 1 + hydrus/client/gui/ClientGUIMediaActions.py | 9 +- hydrus/client/gui/ClientGUIMediaMenus.py | 1 + hydrus/client/gui/ClientGUIMenus.py | 4 +- hydrus/client/gui/ClientGUIPopupMessages.py | 5 +- hydrus/client/gui/ClientGUIRatings.py | 1 + hydrus/client/gui/ClientGUIScrolledPanels.py | 5 +- .../client/gui/ClientGUIScrolledPanelsEdit.py | 61 +- .../gui/ClientGUIScrolledPanelsManagement.py | 1 + .../gui/ClientGUIScrolledPanelsReview.py | 13 +- hydrus/client/gui/ClientGUISerialisable.py | 1 + .../client/gui/ClientGUIShortcutControls.py | 1 + hydrus/client/gui/ClientGUIShortcuts.py | 3 +- hydrus/client/gui/ClientGUISplash.py | 1 + hydrus/client/gui/ClientGUIStringPanels.py | 5 +- hydrus/client/gui/ClientGUIStyle.py | 1 + hydrus/client/gui/ClientGUISubscriptions.py | 29 +- hydrus/client/gui/ClientGUITagSuggestions.py | 9 +- hydrus/client/gui/ClientGUITags.py | 1 + hydrus/client/gui/ClientGUITime.py | 207 +++++- hydrus/client/gui/ClientGUITopLevelWindows.py | 5 +- hydrus/client/gui/QtPorting.py | 4 +- hydrus/client/gui/canvas/ClientGUICanvas.py | 19 +- .../gui/canvas/ClientGUICanvasHoverFrames.py | 1 + .../client/gui/canvas/ClientGUICanvasMedia.py | 15 +- hydrus/client/gui/canvas/ClientGUIMPV.py | 1 + .../client/gui/exporting/ClientGUIExport.py | 64 +- .../client/gui/importing/ClientGUIImport.py | 8 +- .../gui/importing/ClientGUIImportFolders.py | 5 +- .../gui/importing/ClientGUIImportOptions.py | 1 + hydrus/client/gui/lists/ClientGUIListBoxes.py | 4 +- .../gui/lists/ClientGUIListBoxesData.py | 1 + .../gui/lists/ClientGUIListConstants.py | 4 +- hydrus/client/gui/lists/ClientGUIListCtrl.py | 5 +- .../client/gui/lists/ClientGUIListManager.py | 6 +- .../client/gui/lists/ClientGUIListStatus.py | 44 ++ .../metadata/ClientGUIMetadataMigration.py | 1 + .../ClientGUIMetadataMigrationExporters.py | 29 + .../ClientGUIMetadataMigrationImporters.py | 29 + .../gui/networking/ClientGUIHydrusNetwork.py | 11 +- .../client/gui/networking/ClientGUINetwork.py | 15 +- .../networking/ClientGUINetworkJobControl.py | 5 +- .../client/gui/pages/ClientGUIManagement.py | 21 +- hydrus/client/gui/pages/ClientGUIPages.py | 14 +- hydrus/client/gui/pages/ClientGUIResults.py | 74 +-- .../gui/pages/ClientGUIResultsSortCollect.py | 1 + .../gui/pages/ClientGUISessionLegacy.py | 1 + hydrus/client/gui/parsing/ClientGUIParsing.py | 1 + .../gui/parsing/ClientGUIParsingFormulae.py | 1 + .../gui/parsing/ClientGUIParsingLegacy.py | 7 +- .../gui/parsing/ClientGUIParsingTest.py | 1 + .../client/gui/search/ClientGUIACDropdown.py | 4 +- .../gui/search/ClientGUIPredicatesOR.py | 1 + .../gui/search/ClientGUIPredicatesSingle.py | 1 + hydrus/client/gui/search/ClientGUISearch.py | 2 + .../gui/search/ClientGUISearchPanels.py | 1 + .../services/ClientGUIClientsideServices.py | 137 ++-- .../services/ClientGUIServersideServices.py | 1 + .../gui/widgets/ClientGUIColourPicker.py | 1 + hydrus/client/gui/widgets/ClientGUICommon.py | 1 + .../client/gui/widgets/ClientGUIControls.py | 5 +- .../client/importing/ClientImportControl.py | 9 +- .../client/importing/ClientImportFileSeeds.py | 31 +- hydrus/client/importing/ClientImportFiles.py | 3 +- .../client/importing/ClientImportGallery.py | 19 +- .../importing/ClientImportGallerySeeds.py | 5 +- hydrus/client/importing/ClientImportLocal.py | 23 +- .../importing/ClientImportSimpleURLs.py | 9 +- .../ClientImportSubscriptionLegacy.py | 25 +- .../ClientImportSubscriptionQuery.py | 15 +- .../importing/ClientImportSubscriptions.py | 22 +- .../client/importing/ClientImportWatchers.py | 60 +- hydrus/client/importing/ClientImporting.py | 1 + .../importing/options/ClientImportOptions.py | 13 +- .../importing/options/FileImportOptions.py | 1 + .../importing/options/NoteImportOptions.py | 1 + .../importing/options/TagImportOptions.py | 1 + hydrus/client/media/ClientMedia.py | 455 +------------ hydrus/client/media/ClientMediaFileFilter.py | 432 +++++++++++++ hydrus/client/media/ClientMediaManagers.py | 50 +- hydrus/client/media/ClientMediaResultCache.py | 4 +- .../metadata/ClientMetadataMigrationCore.py | 1 + .../ClientMetadataMigrationExporters.py | 92 +++ .../ClientMetadataMigrationImporters.py | 97 +++ hydrus/client/metadata/ClientTagsHandling.py | 9 +- .../networking/ClientLocalServerResources.py | 17 +- hydrus/client/networking/ClientNetworking.py | 1 + .../networking/ClientNetworkingBandwidth.py | 5 +- .../ClientNetworkingBandwidthLegacy.py | 1 + .../networking/ClientNetworkingDomain.py | 5 +- .../client/networking/ClientNetworkingGUG.py | 1 + .../client/networking/ClientNetworkingJobs.py | 53 +- .../networking/ClientNetworkingLogin.py | 7 +- .../networking/ClientNetworkingSessions.py | 5 +- .../ClientNetworkingSessionsLegacy.py | 1 + .../networking/ClientNetworkingURLClass.py | 1 + hydrus/core/HydrusAudioHandling.py | 1 + hydrus/core/HydrusConstants.py | 18 +- hydrus/core/HydrusController.py | 25 +- hydrus/core/HydrusDB.py | 8 +- hydrus/core/HydrusDBBase.py | 29 +- hydrus/core/HydrusData.py | 605 +----------------- hydrus/core/HydrusFileHandling.py | 1 + hydrus/core/HydrusFlashHandling.py | 5 +- hydrus/core/HydrusImageHandling.py | 1 + hydrus/core/HydrusLists.py | 115 ++++ hydrus/core/HydrusPaths.py | 44 +- hydrus/core/HydrusProfiling.py | 71 ++ hydrus/core/HydrusPubSub.py | 4 +- hydrus/core/HydrusSerialisable.py | 3 + hydrus/core/HydrusSessions.py | 5 +- hydrus/core/HydrusTags.py | 2 +- hydrus/core/HydrusTemp.py | 7 +- hydrus/core/HydrusThreading.py | 71 +- hydrus/core/HydrusTime.py | 477 ++++++++++++++ hydrus/core/HydrusVideoHandling.py | 1 + hydrus/core/networking/HydrusNATPunch.py | 13 +- hydrus/core/networking/HydrusNetwork.py | 50 +- hydrus/core/networking/HydrusNetworking.py | 27 +- hydrus/core/networking/HydrusServerRequest.py | 5 +- .../core/networking/HydrusServerResources.py | 18 +- hydrus/external/SystemPredicateParser.py | 18 +- hydrus/hydrus_client.py | 3 +- hydrus/hydrus_server.py | 3 +- hydrus/hydrus_test.py | 2 + hydrus/server/ServerController.py | 6 +- hydrus/server/ServerDB.py | 68 +- hydrus/server/ServerFiles.py | 1 + .../networking/ServerServerResources.py | 19 +- hydrus/test/HelperFunctions.py | 3 +- hydrus/test/TestClientAPI.py | 11 +- hydrus/test/TestClientConstants.py | 1 + hydrus/test/TestClientDB.py | 7 +- hydrus/test/TestClientDBDuplicates.py | 1 + hydrus/test/TestClientDBTags.py | 4 +- hydrus/test/TestClientImportOptions.py | 3 +- hydrus/test/TestClientMetadataMigration.py | 115 ++++ hydrus/test/TestClientMigration.py | 2 + hydrus/test/TestClientNetworking.py | 9 +- hydrus/test/TestClientParsing.py | 22 +- hydrus/test/TestClientTags.py | 6 +- hydrus/test/TestController.py | 1 + hydrus/test/TestFunctions.py | 1 + hydrus/test/TestHydrusData.py | 1 + hydrus/test/TestHydrusNetworking.py | 67 +- hydrus/test/TestHydrusSerialisable.py | 3 +- hydrus/test/TestHydrusServer.py | 9 +- hydrus/test/TestHydrusSessions.py | 15 +- hydrus/test/TestServerDB.py | 69 +- 219 files changed, 3716 insertions(+), 2180 deletions(-) create mode 100644 hydrus/client/media/ClientMediaFileFilter.py create mode 100644 hydrus/core/HydrusLists.py create mode 100644 hydrus/core/HydrusProfiling.py create mode 100644 hydrus/core/HydrusTime.py diff --git a/docs/changelog.md b/docs/changelog.md index c5ea5ffd..c78b4484 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,6 +7,63 @@ title: Changelog !!! note This is the new changelog, only the most recent builds. For all versions, see the [old changelog](old_changelog.html). +## [Version 524](https://github.com/hydrusnetwork/hydrus/releases/tag/v524) + +### timestamp sidecars + +* the sidecars system now supports timestamps. it just uses the unix timestamp number, but if you need it, you can use string conversion to create a full datestring. each sidecar node only selects/sets that one timestamp, so this may get spammy if you want to migrate everything, but you can now migrate archived/imported/whatever time from one client to another! the content updates from sidecar imports apply immediately _after_ the file is fully imported, so it is safe and good to sidecar-import 'my files imported time' etc.. for new files, and it should all get set correctly, but obviously let me know otherwise. if you set 'archived time', the files have to be in an archived state immediately after import, which means importing and archiving them previously, or hitting 'archive all imports' on the respective file import options +* sidecars are getting complex, so I expect I will soon add a button that sets up a 'full' JSON sidecar import/export in one click, basically just spamming/sucking everything the sidecar system can do, pretty soon, so it is easier to set up larger migrations + +### timestamp merge + +* the duplicate merge options now have an action for 'sync file modified date?'. you can set so both files get their earliest (the new default for 'they are the same'), or that the earlier worse can be applied to the later better (the new default for 'this is better') (issue #1203) +* in the duplicate system, when URLs are merged, their respective domain-based timestamps are also merged according to the earliest, as above + +### more timestamps + +* hydrus now supports timestamps before 1970. should be good now, lol, back to 1AD (and my tests show BC dates seem to be working too?). it is probably a meme to apply a modified date of 1505 to some painting, but when I add timestamps to the API maybe we can have some fun. btw calendar calculations and timezones are hell on earth at times, and there's a decent chance that your pre-1970 dates may show up on hour out of phase in labels (a daylight savings time thing) of what you enter in some other area of UI. in either case, my code is not clever enough to apply DST schedules retroactively to older dates, so your search ranges may simply be an hour out back in 1953. it sounds stupid, but it may matter if we are talking midnight boundaries, so let me know how you find it +* when you set a new file modified date, the file on disk's modified date will only be updated if the date set is after 1980-01-01 (Windows) or 1970-01-01 (Linux) due to system limitations +* fixed a typo bug in last week's work that meant file service timestamp editing was not updating the media object (i.e. changes were not visible until a restart) +* fixed a bug where collections that contained files with delete timestamps were throwing errors on display. (they were calculating aggregate timestamp data wrong) +* I rejiggered how the 'is this timestamp sensible?' test applies. this test essentially discounts any timestamp before 1970-01-08 to catch any weird mis-parses and stop them nuking your aggregate modified timestamp values. it now won't apply to internal duplicate merge and so on, but it still applies when you parse timestamps in the downloader system, so you still can't parse anything pre-1970 for now +* one thing I noticed is my '5 years 1 months ago' calculation, which uses a fixed 30 day month and doesn't count the extra day of leap years, is showing obviously increasingly inaccurate numbers here. I'll fix it up + +### export folders + +* export folders can now show a popup while they work. there's a new checkbox for it in their edit UI. default is ON, so you'll start seeing popups for export folders that run in the background. this popup is cancellable, too, so you can now stop in-progress export runs if things seem wrong +* both import and export folders will force-show working popups whenever you trigger them manually +* export folders no longer have the weird and confusing 'paused' and 'run regularly?' duality. this was a legacy error handling thing, now cleaned up and merged into 'run regularly?' +* when 'run regularly?' is unchecked, the run period and new 'show popup while working regularly?' checkboxes are now disabled + +### misc + +* added 'system:ratio is square/portrait/landscape' nicer label aliases for =/taller/wider 1:1 ratio. I added them to the quick-select list on the edit panel, too. they also parse in the system predicate parser! +* I added a bit to the 'getting started with downloading' help page about getting access to difficult sites. I refer to Hydrus Companion as a good internal login solution, and link to yt-dlp, gallery-dl, and imgbrd-grabber with a little discussion on setting up external import workflows. I tried gallery-dl on twitter this week and it was excellent. it can also take your login credentials as either user/pass or cookies.txt (or pull cookies straight from firefox/safari) and give access to nsfw. since twitter has rapidly become a pain for us recently, I will be pointing people to gallery-dl for now +* fixed my Qt subclass definitions for PySide6 6.5.0, which strictly requires the Qt object to be the rightmost base class in multiple inheritance subclasses, wew. this his AUR users last week, I understand! + +### client api (and local booru lol) + +* if you set the Client API to not allow non-local connections, it now binds to 127.0.0.1 and ::1 specifically, which tell your OS we only want the loopback interface. this increases security, and on Windows _should_ mean it only does that first-time firewall dialog popup when 'allow non-local connections' is unchecked +* I brushed up the manage services UI for the Client API. the widgets all line up better now, and turning the service on and off isn't the awkward '[] do not run the service' any more +* fixed the 'disable idle mode if the client api does stuff' check, which was wired up wrong! also, the reset here now fires as a request starts, not when it is complete, meaning if you are already in idle mode, a client api request will now quickly cancel idle mode and hopefully free up any locked database situation promptly + +### boring cleanup and stuff + +* reworked all timestamp-datetime conversion to be happier with pre-1970 dates regardless of system/python support. it is broadly improved all around +* refactored all of the HydrusData time functions and much of ClientTime to a new HydrusTime module +* refactored the ClientData time stuff to ClientTime +* refactored some thread/process functions from HydrusData to HydrusThreading +* refactored some list splitting/throttling functions from HydrusData to a new HydrusLists module +* refactored the file filter out of ClientMedia and into the new ClientMediaFileFilter, and reworked things so the medialist filter jobs now happen at the filter level. this was probably done the wrong way around, but oh well +* expanded the new TimestampData object a bit, it can now give a nice descriptive string of itself +* wrote a new widget to edit TimestampData stubs +* wrote some unit tests for the new timestamp sidecar importer and exporter +* updated my multi-column list system to handle the deprecation of a column definition (today it was the 'paused' column in manage export folders list) +* it should also be able to handle new column definitions appearing +* fixed an error popup that still said 'run repair invalid tags' instead of 'run fix invalid tags' +* the FILE_SERVICES constant now holds the 'all deleted files' virtual domain. this domain keeps slipping my logic, so fingers crossed this helps. also means you can select it in 'system:file service' and stuff now +* misc cleaning and linting work + ## [Version 523](https://github.com/hydrusnetwork/hydrus/releases/tag/v523) ### timestamp editing @@ -325,91 +382,3 @@ title: Changelog * thanks to a user, updated the recently note-and-ai-updated pixiv parser again to grab the canonical pixiv URL and translated tags, if present * thanks to a user, updated the sankaku parser to grab some more tags * the file location context and tag context buttons under tag autocompletes now put menu separators between each type of file/tag service in their menus. for basic users, this'll be a separator for every row, but for advanced users with multiple local domains, it will help categorise the list a bit - -## [Version 514](https://github.com/hydrusnetwork/hydrus/releases/tag/v514) - -### downloaders - -* twitter took down the API we were using, breaking all our nice twitter downloaders! argh! -* a user has figured out a basic new downloader that grabs the tweets amongst the first twenty tweets-and-retweets of an account. yes, only the first twenty max, and usually fewer. because this is a big change, the client will ask about it when you update. if you have some complicated situation where you are working on the old default twitter downloaders and don't want them deleted, you can select 'no' on the dialog it throws up, but everyone else wants to say 'yes'. then check your twitter subs: make sure they moved to the new downloader, and you probably want to make them check more frequently too. -* given the rate of changes at twitter, I think we can expect more changes and blocks in future. I don't know whether nitter will be viable alternative, so if the artists you like end up on a nice simple booru _anywhere_, I strongly recommend just moving there. twitter appears to be explicitly moving to non-third-party-friendly -* thanks to a user's work, the 'danbooru - get webm ugoira' parser is fixed! -* thanks to a user's work, the deviant art parser is updated to get the highest res image in more situations! -* thanks to a user's work, the pixiv downloader now gets the artist note, in japanese (and translated, if there is one), and a 'medium:ai generated' tag! - -### sidecars - -* I wrote some sidecar help here! https://hydrusnetwork.github.io/hydrus/advanced_sidecars.html -* when the client parses files for import, the 'does this look like a sidecar?' test now also checks that the base component of the base filename (e.g. 'Image123' from 'Image123.jpg.txt') actually appears in the list of non-txt/json/xml ext files. a random yo.txt file out of nowhere will now be inspected in case it is secretly a jpeg again, for good or ill -* when you drop some files on the client, the number of files skipped because they looked like sidecars is now stated in the status label -* fixed a typo bug that meant tags imported from sidecars were not being properly cleaned, despite preview appearance otherwise, for instance ':)', which in hydrus needs to be secretly stored as '::)' was being imported as ')' -* as a special case, tags that in hydrus are secretly '::)' will be converted to ':)' on export to sidecar too, the inverse of the above problem. there may be some other tag cleaning quirks to undo here, so let me know what you run into - -### related tags overhaul - -* the 'related tags' suggestion system, turned on under _options->tag suggestions_, has several changes, including some prototype tech I'd love feedback on -* first off, there are two new search buttons, 'new 1' and 'new 2' ('2' is available on repositories only).. these use an upgraded statistical search and scoring system that a user worked on and sent in. I have butchered his specific namespace searching system to something more general/flexible and easy for me to maintain, but it works better and more comprehensibly than my old method! give it a go and let me know how each button does--the first one will be fast but less useful on the PTR, the second will be slower but generally give richer results (although it cannot do tags with too-high count) -* the new search routine works on multiple files, so 'related tags' now shows on tag dialogs launched from a selection of thumbnails! -* also, all the related search buttons now search any selection of tags you make!!! so if you can't remember that character's name, just click on the series or another character they are often with and hit the search, and you should get a whole bunch appear -* I am going to keep working on this in the future. the new buttons will become the only buttons, I'll try and mitigate the prototype search limitations, add some cancel tech, move to a time-based search length like the current buttons, and I'll add more settings, including for filtering so we aren't looking up related tags for 'page:x' and so on. I'm interested in knowing how you get on with IRL data. are there too many recommendations (is the tolerance too high?)? is the sorting good (is the stuff at the top relevant or often just noise?)? - -### misc - -* all users can now copy their service keys (which are a technical non-changing hex identifier for your client's services) from the review services window--advanced mode is no longer needed. this may be useful as the client api transitions to service keys -* when a job in the downloader search log generates new jobs (e.g. fetches the next page), the new job(s) are now inserted after the parent. previously, they were appended to the end of the list. this changes how ngugs operate, converting their searches from interleaved to sequential! -* restarting search log jobs now also places the new job after the restarted job -* when you create a new export folder, if you have default metadata export sidecar settings from a previous manual file export, the program now asks if you want those for the new export folder or an empty list. previously, it just assigned the saved default, which could be jarring if it was saved from ages ago -* added a migration guide to the running from source help. also brushed up some language and fixed a bunch of borked title weights in that document -* the max initial and periodic file limits in subscriptions is now 50k when in advanced mode. I can't promise that would be nice though! -* the file history chart no longer says that inbox and delete time tracking are new - -### misc fixes - -* fixed a cursor type detection test that was stopping the cursor from hiding immediately when you do a media viewer drag in Qt6 -* fixed an issue where 'clear deletion record' calls were not deleting from the newer 'all my files' domain. the erroneous extra records will be searched for and scrubbed on update -* fixed the issue where if you had the new 'unnamespaced input gives (any namespace) wildcard results' search option on, you couldn't add any novel tags in WRITE autocomplete contexts like 'manage tags'!!! it could only offer the automatically converted wildcard tags as suggested input, which of course aren't appropriate for a WRITE context. the way I ultimately fixed this was horrible; the whole thing needs more work to deal with clever logic like this better, so let me know if you get any more trouble here -* I think I fixed an infinite hang when trying to add certain siblings in manage tag siblings. I believe this was occuring when the dialog was testing if the new pair would create a loop when the sibling structure already contains a loop. now it throws up a message and breaks the test -* fixed an issue where certain system:filetype predicates would spawn apparent duplicates of themselves instead of removing on double-click. images+audio+video+swf+pdf was one example. it was a 'all the image types' vs 'list of (all the) image types' conversion/comparison/sorting issue - -### client api - -* **this is later than I expected, but as was planned last year, I am clearing up several obsolete parameters and data structures this week. mostly it is bad service name-identification that seemed simple or flexible to support but just added maintenance debt, induced bad implementation practises, and hindered future expansions. if you have a custom api script, please read on--and if you have not yet moved to the alternatives, do so before updating!** -* **all `...service_name...` parameters are officially obsolete! they will still work via some legacy hacks, so old scripts shouldn't break, but they are no longer documented. please move to the `...service_key...` alternates as soon as reasonably possible (check out `/get_services` if you need to learn about service keys)** -* **`/add_tags/get_tag_services` is removed! use `/get_services` instead!** -* **`hide_service_names_tags`, previously made default true, is removed and its data structures `service_names_to_statuses_to_...` are also gone! move to the new `tags` structure.** -* **`hide_service_keys_tags` is now default true. it will be removed in 4 weeks or so. same deal as with `service_names_to_statuses_to_...`--move to `tags`** -* **`system_inbox` and `system_archive` are removed from `/get_files/search_files`! just use 'system:inbox/archive' in the tags list** -* **the 'set_file_relationships' command from last week has been reworked to have a nicer Object parameter with a new name. please check the updated help!** normally I wouldn't change something so quick, but we are still in early prototype, so I'm ok shifting it (and the old method still works lmao, but I'll clear that code out in a few weeks, so please move over--the Object will be much nicer to expand in future, which I forgot about in v513) -* many Client API commands now support modern file domain objects, meaning you can search a UNION of file services and 'deleted-from' file services. affected commands are -* * /add_files/delete_files -* * /add_files/undelete_files -* * /add_tags/search_tags -* * /get_files/search_files -* * /manage_file_relationships/get_everything -* a new `/get_service` call now lets you ask about an individual service by service name or service key, basically a parameterised /get_services -* the `/manage_pages/get_pages` and `/manage_pages/get_page_info` calls now give the `page_state`, a new enum that says if the page is ready, initialised, searching, or search-cancelled -* to reduce duplicate argument spam, the client api help now specifies the complicated 'these files' and now 'this file domain' arguments into sub-sections, and the commands that use them just point to the subsections. check it out--it makes sense when you look at it. -* `/add_tags/add_tags` now raises 400 if you give an invalid content action (e.g. pending to a local tag service). previously it skipped these rows silently -* added and updated unit tests and help for the above changes -* client api version is now 41 - -### boring optimisation - -* when you are looking at a search log or file log, if entries are added, removed, or moved around, all the log entries that have changed row # now update (previously it just sent a redraw signal for the new rows, not the second-order affected rows that were shuffled up/down. many access routines for these logs are sped up -* file log status checking is completely rewritten. the ways it searches, caches and optimises the 'which is the next item with x status' queues is faster and requires far less maintenance. large import queues have less overhead, so the in and outs of general download work should scale up much better now -* the main data cache that stores rendered images, image tiles, and thumbnails now maintains itself far more efficiently. there was a hellish O(n) overhead when adding or removing an item which has been reduced to constant time. this gonk was being spammed every few minutes during normal memory maintenance, when hundreds of thumbs can be purged at once. clients with tens of thousands of thumbnails in memory will maintain that list far more smoothly -* physical file delete is now more efficient, requiring far fewer hard drive hits to delete a media file. it is also far less aggressive, with a new setting in _options->files and trash_ that sets how long to wait between individual file deletes, default 250ms. before, it was full LFG mode with minor delays every hundred/thousand jobs, and since it takes a write lock, it was lagging out thumbnail load when hitting a lot of work. the daemon here also shuts down faster if caught working during program shut down - -### boring code cleanup - -* refactored some parsing routines to be more flexible -* added some more dictionary and enum type testing to the client api parameter parsing routines. error messages should be better! -* improved how `/add_tags/add_tags` parsing works. ensuring both access methods check all types and report nicer errors -* cleaned up the `/search_files/file_metadata` call's parsing, moving to the new generalised method and smoothing out some old code flow. it now checks hashes against the last search, too -* cleaned up `/manage_pages/add_files` similarly -* cleaned up how tag services are parsed and their errors reported in the client api -* the client api is better about processing the file identifiers you give it in the same order you gave -* fixed bad 'potentials_search_type'/'search_type' inconsistency in the client api help examples -* obviously a bunch of client api unit test and help cleanup to account for the obsolete stuff and various other changes here -* updated a bunch of the client api unit tests to handle some of the new parsing -* fixed the remaining 'randomly fail due to complex counting logic' potential count unit tests. turns out there were like seven more of them diff --git a/docs/getting_started_downloading.md b/docs/getting_started_downloading.md index f0606515..301eaa62 100644 --- a/docs/getting_started_downloading.md +++ b/docs/getting_started_downloading.md @@ -169,6 +169,23 @@ To start using a login script, select the domain and click 'edit credentials'. Y Most sites only have one way of logging in, but hydrus does support more. Hentai Foundry is a good example--by default, the client performs the 'click-through' login as a guest, which requires no credentials and means any hydrus client can get any content from the start. But this way of logging in only lasts about 60 minutes or so before having to be refreshed, and it does not hide any spicy stuff, so if you use HF a lot, I recommend you create a throwaway account, set the filters you like in your HF profile (e.g. no guro content), and then click the 'change login script' in the client to the proper username/pass login. -The login system is new and still a bit experimental. Don't try to pull off anything too weird with it! If anything goes wrong, it will likely delay the script (and hence the whole domain) from working for a while, or invalidate it entirely. If the error is something simple, like a password typo or current server maintenance, go back to this dialog to fix and scrub the error and try again. If the site just changed its layout, you may need to update the login script. If it is more complicated, please contact me, hydrus_dev, with the details! +The login system is not very clever. Don't try to pull off anything too weird with it! If anything goes wrong, it will likely delay the script (and hence the whole domain) from working for a while, or invalidate it entirely. If the error is something simple, like a password typo or current server maintenance, go back to this dialog to fix and scrub the error and try again. If the site just changed its layout, you may need to update the login script. If it is more complicated, please contact me, hydrus_dev, with the details! -If you would like to login to a site that is not yet supported by hydrus (usually ones with a Captcha in the login page), see about getting a web browser add-on that lets you export a cookies.txt (either for the whole browser or just for that domain) and then drag and drop that file onto the hydrus _network->data->review session cookies_ dialog. This sometimes does not work if your add-on's export formatting is unusual. If it does work, hydrus will import and use those cookies, which skips the login by making your hydrus pretend to be your browser directly. This is obviously advanced and hacky, so if you need to do it, let me know how you get on and what tools you find work best! +If you would like to login to a site that is not yet supported by hydrus (usually ones with a Captcha in the login page), you have two options: + +1. Get a web browser add-on that lets you export a cookies.txt (either for the whole browser or just for that domain) and then drag and drop that cookies.txt file onto the hydrus _network->data->review session cookies_ dialog. This sometimes does not work if your add-on's export formatting is unusual. If it does work, hydrus will import and use those cookies, which skips the login by making your hydrus pretend to be your browser directly. This is obviously advanced and hacky, so if you need to do it, let me know how you get on and what tools you find work best! +2. Use [Hydrus Companion](https://gitgud.io/prkc/hydrus-companion) browser add-on to do the same basic thing automatically. + +## Difficult Sites + +Boorus are usually easy to parse from, and there are many hydrus downloaders available that work well. Other sites are less easy to download from. Some will purposefully disguise access behind captchas or difficult login tokens that the hydrus downloader just isn't clever enough to handle. In these cases, it can be best just to go to an external downloader program that is specially tuned for these complex sites. + +It takes a bit of time to set up these sorts of programs--and if you get into them, you'll likely want to make a script to help automate their use--but if you know they solve your problem, it is well worth it! + +- [yt-dlp](https://github.com/yt-dlp/yt-dlp) - This is an excellent video downloader that can download from hundreds of different websites. Learn how it works, it is useful for all sorts of things! +- [gallery-dl](https://github.com/mikf/gallery-dl) - This is an excellent image and small-vid downloader that works for pretty much any booru and many larger/professional gallery sites, particularly when those sites need logins. Check the documentation, since you may be able to get it to rip cookies right out of your firefox, or you can give it your actual user/password for many sites and it'll handle all the login for you. +- [imgbrd-grabber](https://github.com/Bionus/imgbrd-grabber) - Another excellent, mostly booru downloader, with an UI. You can export some metadata to filenames, which you might like to then suck up with hydrus filename-import-parsing. + +With these tools, used manually and/or with some scripts you set up, you may be able to set up a regular import workflow to hydrus (especilly with an `Import Folder` as under the `file` menu) and get _most_ of what you would with an internal downloader. Some things like known URLs and tag parsing may be limited or non-existant, but it is better than nothing, and if you only need to do it for a couple sources on a couple sites every month, you can fill in the most of the gap manually yourself. + +Hydev is planning to roll yt-dlp and gallery-dl support into the program natively in a future update of the downloader engine. diff --git a/docs/old_changelog.html b/docs/old_changelog.html index 0ce00401..c7420a9d 100644 --- a/docs/old_changelog.html +++ b/docs/old_changelog.html @@ -34,6 +34,52 @@

changelog