From aa0ed5db3d20fc144fe405201eebf6e1477a8a24 Mon Sep 17 00:00:00 2001 From: Paul Friederichsen Date: Sat, 30 Mar 2024 13:15:09 -0500 Subject: [PATCH 1/5] Fix importing posts with multiple file urls (#1537) --- hydrus/client/importing/ClientImportFileSeeds.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hydrus/client/importing/ClientImportFileSeeds.py b/hydrus/client/importing/ClientImportFileSeeds.py index 196f6d78..e79873d0 100644 --- a/hydrus/client/importing/ClientImportFileSeeds.py +++ b/hydrus/client/importing/ClientImportFileSeeds.py @@ -1590,6 +1590,8 @@ class FileSeed( HydrusSerialisable.SerialisableBase ): duplicate_file_seed.file_seed_data = child_url + duplicate_file_seed.Normalise() + duplicate_file_seed.SetReferralURL( url_for_child_referral ) duplicate_file_seed.AddRequestHeaders( self._request_headers ) From f9ff2829b64ee3ee4483e202985b29d23c15066d Mon Sep 17 00:00:00 2001 From: Paul Friederichsen Date: Sat, 30 Mar 2024 13:19:13 -0500 Subject: [PATCH 2/5] Fix KDE 6 file picker causing hydrus to close (#1535) --- hydrus/client/ClientController.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hydrus/client/ClientController.py b/hydrus/client/ClientController.py index d7fcc1f6..443f5720 100644 --- a/hydrus/client/ClientController.py +++ b/hydrus/client/ClientController.py @@ -112,6 +112,8 @@ class App( QW.QApplication ): self.setQuitOnLastWindowClosed( False ) + self.setQuitLockEnabled( False ) + self.call_after_catcher = QP.CallAfterEventCatcher( self ) self.pubsub_catcher = PubSubEventCatcher( self, self._pubsub ) From fc0487e537b647292b2bc62b833aff8e98d3572a Mon Sep 17 00:00:00 2001 From: Paul Friederichsen Date: Sat, 30 Mar 2024 13:38:03 -0500 Subject: [PATCH 3/5] Add ellipsis to menu items where appropriate (#1538) --- hydrus/client/gui/ClientGUI.py | 148 ++++++++++++++++----------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/hydrus/client/gui/ClientGUI.py b/hydrus/client/gui/ClientGUI.py index b3506815..026fa557 100644 --- a/hydrus/client/gui/ClientGUI.py +++ b/hydrus/client/gui/ClientGUI.py @@ -2688,7 +2688,7 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendMenuItem( save, name, 'Save the existing open pages as a session.', self.ProposeSaveGUISession, name ) - ClientGUIMenus.AppendMenuItem( save, 'as new session', 'Save the existing open pages as a session.', self.ProposeSaveGUISession ) + ClientGUIMenus.AppendMenuItem( save, 'as new session' + HC.UNICODE_ELLIPSIS, 'Save the existing open pages as a session.', self.ProposeSaveGUISession ) ClientGUIMenus.AppendMenu( self._menubar_pages_sessions_submenu, save, 'save' ) @@ -2955,20 +2955,20 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M if can_overrule_accounts: - ClientGUIMenus.AppendMenuItem( submenu, 'review all accounts', 'See all accounts.', self._STARTReviewAllAccounts, service_key ) - ClientGUIMenus.AppendMenuItem( submenu, 'modify an account', 'Modify a specific account\'s type and expiration.', self._ModifyAccount, service_key ) + ClientGUIMenus.AppendMenuItem( submenu, 'review all accounts' + HC.UNICODE_ELLIPSIS, 'See all accounts.', self._STARTReviewAllAccounts, service_key ) + ClientGUIMenus.AppendMenuItem( submenu, 'modify an account' + HC.UNICODE_ELLIPSIS, 'Modify a specific account\'s type and expiration.', self._ModifyAccount, service_key ) if can_overrule_accounts and service_type == HC.FILE_REPOSITORY: - ClientGUIMenus.AppendMenuItem( submenu, 'get an uploader\'s ip address', 'Fetch the ip address that uploaded a specific file, if the service knows it.', self._FetchIP, service_key ) + ClientGUIMenus.AppendMenuItem( submenu, 'get an uploader\'s ip address' + HC.UNICODE_ELLIPSIS, 'Fetch the ip address that uploaded a specific file, if the service knows it.', self._FetchIP, service_key ) if can_create_accounts: ClientGUIMenus.AppendSeparator( submenu ) - ClientGUIMenus.AppendMenuItem( submenu, 'create new accounts', 'Create new accounts for this service.', self._GenerateNewAccounts, service_key ) + ClientGUIMenus.AppendMenuItem( submenu, 'create new accounts' + HC.UNICODE_ELLIPSIS, 'Create new accounts for this service.', self._GenerateNewAccounts, service_key ) if can_overrule_account_types: @@ -2982,13 +2982,13 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendSeparator( submenu ) - ClientGUIMenus.AppendMenuItem( submenu, 'change update period', 'Change the update period for this service.', self._ManageServiceOptionsUpdatePeriod, service_key ) + ClientGUIMenus.AppendMenuItem( submenu, 'change update period' + HC.UNICODE_ELLIPSIS, 'Change the update period for this service.', self._ManageServiceOptionsUpdatePeriod, service_key ) - ClientGUIMenus.AppendMenuItem( submenu, 'change anonymisation period', 'Change the account history nullification period for this service.', self._ManageServiceOptionsNullificationPeriod, service_key ) + ClientGUIMenus.AppendMenuItem( submenu, 'change anonymisation period' + HC.UNICODE_ELLIPSIS, 'Change the account history nullification period for this service.', self._ManageServiceOptionsNullificationPeriod, service_key ) if service_type == HC.TAG_REPOSITORY: - ClientGUIMenus.AppendMenuItem( submenu, 'edit tag filter', 'Change the tag filter for this service.', self._ManageServiceOptionsTagFilter, service_key ) + ClientGUIMenus.AppendMenuItem( submenu, 'edit tag filter' + HC.UNICODE_ELLIPSIS, 'Change the tag filter for this service.', self._ManageServiceOptionsTagFilter, service_key ) ClientGUIMenus.AppendSeparator( submenu ) @@ -3000,7 +3000,7 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendSeparator( submenu ) - ClientGUIMenus.AppendMenuItem( submenu, 'manage services', 'Add, edit, and delete this server\'s services.', self._ManageServer, service_key ) + ClientGUIMenus.AppendMenuItem( submenu, 'manage services' + HC.UNICODE_ELLIPSIS, 'Add, edit, and delete this server\'s services.', self._ManageServer, service_key ) ClientGUIMenus.AppendSeparator( submenu ) ClientGUIMenus.AppendMenuItem( submenu, 'backup server', 'Command the server to temporarily pause and back up its database.', self._BackupServer, service_key ) ClientGUIMenus.AppendSeparator( submenu ) @@ -3088,7 +3088,7 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M if have_closed_pages: - ClientGUIMenus.AppendMenuItem( self._menubar_undo_closed_pages_submenu, 'clear all', 'Remove all closed pages from memory.', self.AskToDeleteAllClosedPages ) + ClientGUIMenus.AppendMenuItem( self._menubar_undo_closed_pages_submenu, 'clear all' + HC.UNICODE_ELLIPSIS, 'Remove all closed pages from memory.', self.AskToDeleteAllClosedPages ) self._menubar_undo_closed_pages_submenu.addSeparator() @@ -3120,17 +3120,17 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M menu = ClientGUIMenus.GenerateMenu( self ) - ClientGUIMenus.AppendMenuItem( menu, 'set a password', 'Set a simple password for the database so only you can open it in the client.', self._SetPassword ) + ClientGUIMenus.AppendMenuItem( menu, 'set a password' + HC.UNICODE_ELLIPSIS, 'Set a simple password for the database so only you can open it in the client.', self._SetPassword ) ClientGUIMenus.AppendSeparator( menu ) - self._menubar_database_set_up_backup_path = ClientGUIMenus.AppendMenuItem( menu, 'set up a database backup location', 'Choose a path to back the database up to.', self._SetupBackupPath ) - self._menubar_database_update_backup = ClientGUIMenus.AppendMenuItem( menu, 'update database backup', 'Back the database up to an external location.', self._BackupDatabase ) - self._menubar_database_change_backup_path = ClientGUIMenus.AppendMenuItem( menu, 'change database backup location', 'Choose a path to back the database up to.', self._SetupBackupPath ) + self._menubar_database_set_up_backup_path = ClientGUIMenus.AppendMenuItem( menu, 'set up a database backup location' + HC.UNICODE_ELLIPSIS, 'Choose a path to back the database up to.', self._SetupBackupPath ) + self._menubar_database_update_backup = ClientGUIMenus.AppendMenuItem( menu, 'update database backup' + HC.UNICODE_ELLIPSIS, 'Back the database up to an external location.', self._BackupDatabase ) + self._menubar_database_change_backup_path = ClientGUIMenus.AppendMenuItem( menu, 'change database backup location' + HC.UNICODE_ELLIPSIS, 'Choose a path to back the database up to.', self._SetupBackupPath ) ClientGUIMenus.AppendSeparator( menu ) - self._menubar_database_restore_backup = ClientGUIMenus.AppendMenuItem( menu, 'restore from a database backup', 'Restore the database from an external location.', self._controller.RestoreDatabase ) + self._menubar_database_restore_backup = ClientGUIMenus.AppendMenuItem( menu, 'restore from a database backup' + HC.UNICODE_ELLIPSIS, 'Restore the database from an external location.', self._controller.RestoreDatabase ) message = 'Your database is stored across multiple locations. The in-client backup routine can only handle simple databases (in one location), so the menu commands to backup have been hidden. To back up, please use a third-party program that will work better than anything I can write.' message += os.linesep * 2 @@ -3140,7 +3140,7 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendSeparator( menu ) - ClientGUIMenus.AppendMenuItem( menu, 'move media files', 'Review and manage the locations your database is stored.', self._MoveMediaFiles ) + ClientGUIMenus.AppendMenuItem( menu, 'move media files' + HC.UNICODE_ELLIPSIS, 'Review and manage the locations your database is stored.', self._MoveMediaFiles ) ClientGUIMenus.AppendSeparator( menu ) @@ -3153,7 +3153,7 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M file_maintenance_menu = ClientGUIMenus.GenerateMenu( menu ) - ClientGUIMenus.AppendMenuItem( file_maintenance_menu, 'manage scheduled jobs', 'Review outstanding jobs, and schedule new ones.', self._ReviewFileMaintenance ) + ClientGUIMenus.AppendMenuItem( file_maintenance_menu, 'manage scheduled jobs' + HC.UNICODE_ELLIPSIS, 'Review outstanding jobs, and schedule new ones.', self._ReviewFileMaintenance ) ClientGUIMenus.AppendSeparator( file_maintenance_menu ) check_manager = ClientGUICommon.CheckboxManagerOptions( 'file_maintenance_during_idle' ) @@ -3172,7 +3172,7 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendSeparator( file_maintenance_menu ) - ClientGUIMenus.AppendMenuItem( file_maintenance_menu, 'clear orphan files', 'Clear out surplus files that have found their way into the file structure.', self._ClearOrphanFiles ) + ClientGUIMenus.AppendMenuItem( file_maintenance_menu, 'clear orphan files' + HC.UNICODE_ELLIPSIS, 'Clear out surplus files that have found their way into the file structure.', self._ClearOrphanFiles ) ClientGUIMenus.AppendMenu( menu, file_maintenance_menu, 'file maintenance' ) @@ -3202,53 +3202,53 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendSeparator( db_maintenance_submenu ) - ClientGUIMenus.AppendMenuItem( db_maintenance_submenu, 'analyze', 'Optimise slow queries by running statistical analyses on the database.', self._AnalyzeDatabase ) - ClientGUIMenus.AppendMenuItem( db_maintenance_submenu, 'review vacuum data', 'See whether it is worth rebuilding the database to reformat tables and recover disk space.', self._ReviewVacuumData ) + ClientGUIMenus.AppendMenuItem( db_maintenance_submenu, 'analyze' + HC.UNICODE_ELLIPSIS, 'Optimise slow queries by running statistical analyses on the database.', self._AnalyzeDatabase ) + ClientGUIMenus.AppendMenuItem( db_maintenance_submenu, 'review vacuum data' + HC.UNICODE_ELLIPSIS, 'See whether it is worth rebuilding the database to reformat tables and recover disk space.', self._ReviewVacuumData ) ClientGUIMenus.AppendSeparator( db_maintenance_submenu ) - ClientGUIMenus.AppendMenuItem( db_maintenance_submenu, 'clear/fix orphan file records', 'Clear out surplus file records that have not been deleted correctly.', self._ClearOrphanFileRecords ) + ClientGUIMenus.AppendMenuItem( db_maintenance_submenu, 'clear/fix orphan file records' + HC.UNICODE_ELLIPSIS, 'Clear out surplus file records that have not been deleted correctly.', self._ClearOrphanFileRecords ) - ClientGUIMenus.AppendMenuItem( db_maintenance_submenu, 'clear orphan tables', 'Clear out surplus db tables that have not been deleted correctly.', self._ClearOrphanTables ) + ClientGUIMenus.AppendMenuItem( db_maintenance_submenu, 'clear orphan tables' + HC.UNICODE_ELLIPSIS, 'Clear out surplus db tables that have not been deleted correctly.', self._ClearOrphanTables ) - ClientGUIMenus.AppendMenuItem( db_maintenance_submenu, 'clear orphan hashed serialisables', 'Clear non-needed cached hashed serialisable objects.', self._ClearOrphanHashedSerialisables ) + ClientGUIMenus.AppendMenuItem( db_maintenance_submenu, 'clear orphan hashed serialisables' + HC.UNICODE_ELLIPSIS, 'Clear non-needed cached hashed serialisable objects.', self._ClearOrphanHashedSerialisables ) ClientGUIMenus.AppendMenu( menu, db_maintenance_submenu, 'db maintenance' ) check_submenu = ClientGUIMenus.GenerateMenu( menu ) - ClientGUIMenus.AppendMenuItem( check_submenu, 'database integrity', 'Have the database examine all its records for internal consistency.', self._CheckDBIntegrity ) - ClientGUIMenus.AppendMenuItem( check_submenu, 'repopulate truncated mappings tables', 'Use the mappings cache to try to repair a previously damaged mappings file.', self._RepopulateMappingsTables ) - ClientGUIMenus.AppendMenuItem( check_submenu, 'resync tag mappings cache files', 'Check the tag mappings cache for surplus or missing files.', self._ResyncTagMappingsCacheFiles ) - ClientGUIMenus.AppendMenuItem( check_submenu, 'fix logically inconsistent mappings', 'Remove tags that are occupying two mutually exclusive states.', self._FixLogicallyInconsistentMappings ) - ClientGUIMenus.AppendMenuItem( check_submenu, 'fix invalid tags', 'Scan the database for invalid tags.', self._RepairInvalidTags ) + ClientGUIMenus.AppendMenuItem( check_submenu, 'database integrity' + HC.UNICODE_ELLIPSIS, 'Have the database examine all its records for internal consistency.', self._CheckDBIntegrity ) + ClientGUIMenus.AppendMenuItem( check_submenu, 'repopulate truncated mappings tables' + HC.UNICODE_ELLIPSIS, 'Use the mappings cache to try to repair a previously damaged mappings file.', self._RepopulateMappingsTables ) + ClientGUIMenus.AppendMenuItem( check_submenu, 'resync tag mappings cache files' + HC.UNICODE_ELLIPSIS, 'Check the tag mappings cache for surplus or missing files.', self._ResyncTagMappingsCacheFiles ) + ClientGUIMenus.AppendMenuItem( check_submenu, 'fix logically inconsistent mappings' + HC.UNICODE_ELLIPSIS, 'Remove tags that are occupying two mutually exclusive states.', self._FixLogicallyInconsistentMappings ) + ClientGUIMenus.AppendMenuItem( check_submenu, 'fix invalid tags' + HC.UNICODE_ELLIPSIS, 'Scan the database for invalid tags.', self._RepairInvalidTags ) ClientGUIMenus.AppendMenu( menu, check_submenu, 'check and repair' ) regen_submenu = ClientGUIMenus.GenerateMenu( menu ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'total pending count, in the pending menu', 'Regenerate the pending count up top.', self._DeleteServiceInfo, only_pending = True ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag storage mappings cache (all, with deferred siblings & parents calculation)', 'Delete and recreate the tag mappings cache, fixing bad tags or miscounts.', self._RegenerateTagMappingsCache ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag storage mappings cache (just pending tags, instant calculation)', 'Delete and recreate the tag pending mappings cache, fixing bad tags or miscounts.', self._RegenerateTagPendingMappingsCache ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag display mappings cache (all, deferred siblings & parents calculation)', 'Delete and recreate the tag display mappings cache, fixing bad tags or miscounts.', self._RegenerateTagDisplayMappingsCache ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag display mappings cache (just pending tags, instant calculation)', 'Delete and recreate the tag display pending mappings cache, fixing bad tags or miscounts.', self._RegenerateTagDisplayPendingMappingsCache ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag display mappings cache (missing file repopulation)', 'Repopulate the mappings cache if you know it is lacking files, fixing bad tags or miscounts.', self._RepopulateTagDisplayMappingsCache ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag siblings lookup cache', 'Delete and recreate the tag siblings cache.', self._RegenerateTagSiblingsLookupCache ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag parents lookup cache', 'Delete and recreate the tag siblings cache.', self._RegenerateTagParentsLookupCache ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag text search cache', 'Delete and regenerate the cache hydrus uses for fast tag search.', self._RegenerateTagCache ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag text search cache (subtags repopulation)', 'Repopulate the subtags for the cache hydrus uses for fast tag search.', self._RepopulateTagCacheMissingSubtags ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag text search cache (searchable subtag maps)', 'Regenerate the searchable subtag maps.', self._RegenerateTagCacheSearchableSubtagsMaps ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'total pending count, in the pending menu' + HC.UNICODE_ELLIPSIS, 'Regenerate the pending count up top.', self._DeleteServiceInfo, only_pending = True ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag storage mappings cache (all, with deferred siblings & parents calculation)' + HC.UNICODE_ELLIPSIS, 'Delete and recreate the tag mappings cache, fixing bad tags or miscounts.', self._RegenerateTagMappingsCache ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag storage mappings cache (just pending tags, instant calculation)' + HC.UNICODE_ELLIPSIS, 'Delete and recreate the tag pending mappings cache, fixing bad tags or miscounts.', self._RegenerateTagPendingMappingsCache ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag display mappings cache (all, deferred siblings & parents calculation)' + HC.UNICODE_ELLIPSIS, 'Delete and recreate the tag display mappings cache, fixing bad tags or miscounts.', self._RegenerateTagDisplayMappingsCache ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag display mappings cache (just pending tags, instant calculation)' + HC.UNICODE_ELLIPSIS, 'Delete and recreate the tag display pending mappings cache, fixing bad tags or miscounts.', self._RegenerateTagDisplayPendingMappingsCache ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag display mappings cache (missing file repopulation)' + HC.UNICODE_ELLIPSIS, 'Repopulate the mappings cache if you know it is lacking files, fixing bad tags or miscounts.', self._RepopulateTagDisplayMappingsCache ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag siblings lookup cache' + HC.UNICODE_ELLIPSIS, 'Delete and recreate the tag siblings cache.', self._RegenerateTagSiblingsLookupCache ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag parents lookup cache' + HC.UNICODE_ELLIPSIS, 'Delete and recreate the tag siblings cache.', self._RegenerateTagParentsLookupCache ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag text search cache' + HC.UNICODE_ELLIPSIS, 'Delete and regenerate the cache hydrus uses for fast tag search.', self._RegenerateTagCache ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag text search cache (subtags repopulation)' + HC.UNICODE_ELLIPSIS, 'Repopulate the subtags for the cache hydrus uses for fast tag search.', self._RepopulateTagCacheMissingSubtags ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'tag text search cache (searchable subtag maps)' + HC.UNICODE_ELLIPSIS, 'Regenerate the searchable subtag maps.', self._RegenerateTagCacheSearchableSubtagsMaps ) ClientGUIMenus.AppendSeparator( regen_submenu ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'all deleted files', 'Resynchronise the store of all known deleted files.', self._RegenerateCombinedDeletedFiles ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'local hash cache', 'Repopulate the cache hydrus uses for fast hash lookup for local files.', self._RegenerateLocalHashCache ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'local tag cache', 'Repopulate the cache hydrus uses for fast tag lookup for local files.', self._RegenerateLocalTagCache ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'all deleted files' + HC.UNICODE_ELLIPSIS, 'Resynchronise the store of all known deleted files.', self._RegenerateCombinedDeletedFiles ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'local hash cache' + HC.UNICODE_ELLIPSIS, 'Repopulate the cache hydrus uses for fast hash lookup for local files.', self._RegenerateLocalHashCache ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'local tag cache' + HC.UNICODE_ELLIPSIS, 'Repopulate the cache hydrus uses for fast tag lookup for local files.', self._RegenerateLocalTagCache ) ClientGUIMenus.AppendSeparator( regen_submenu ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'service info numbers', 'Delete all cached service info like total number of mappings or files, in case it has become desynchronised. Some parts of the gui may be laggy immediately after this as these numbers are recalculated.', self._DeleteServiceInfo ) - ClientGUIMenus.AppendMenuItem( regen_submenu, 'similar files search tree', 'Delete and recreate the similar files search tree.', self._RegenerateSimilarFilesTree ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'service info numbers' + HC.UNICODE_ELLIPSIS, 'Delete all cached service info like total number of mappings or files, in case it has become desynchronised. Some parts of the gui may be laggy immediately after this as these numbers are recalculated.', self._DeleteServiceInfo ) + ClientGUIMenus.AppendMenuItem( regen_submenu, 'similar files search tree' + HC.UNICODE_ELLIPSIS, 'Delete and recreate the similar files search tree.', self._RegenerateSimilarFilesTree ) ClientGUIMenus.AppendMenu( menu, regen_submenu, 'regenerate' ) @@ -3256,8 +3256,8 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M file_viewing_submenu = ClientGUIMenus.GenerateMenu( menu ) - ClientGUIMenus.AppendMenuItem( file_viewing_submenu, 'clear all file viewing statistics', 'Delete all file viewing records from the database.', self._ClearFileViewingStats ) - ClientGUIMenus.AppendMenuItem( file_viewing_submenu, 'cull file viewing statistics based on current min/max values', 'Cull your file viewing statistics based on minimum and maximum permitted time deltas.', self._CullFileViewingStats ) + ClientGUIMenus.AppendMenuItem( file_viewing_submenu, 'clear all file viewing statistics' + HC.UNICODE_ELLIPSIS, 'Delete all file viewing records from the database.', self._ClearFileViewingStats ) + ClientGUIMenus.AppendMenuItem( file_viewing_submenu, 'cull file viewing statistics based on current min/max values' + HC.UNICODE_ELLIPSIS, 'Cull your file viewing statistics based on minimum and maximum permitted time deltas.', self._CullFileViewingStats ) ClientGUIMenus.AppendMenu( menu, file_viewing_submenu, 'file viewing statistics' ) @@ -3268,7 +3268,7 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M menu = ClientGUIMenus.GenerateMenu( self ) - ClientGUIMenus.AppendMenuItem( menu, 'import files', 'Add new files to the database.', self._ImportFiles ) + ClientGUIMenus.AppendMenuItem( menu, 'import files' + HC.UNICODE_ELLIPSIS, 'Add new files to the database.', self._ImportFiles ) ClientGUIMenus.AppendSeparator( menu ) @@ -3295,8 +3295,8 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendSeparator( i_and_e_submenu ) - ClientGUIMenus.AppendMenuItem( i_and_e_submenu, 'manage import folders', 'Manage folders from which the client can automatically import.', self._ManageImportFolders ) - ClientGUIMenus.AppendMenuItem( i_and_e_submenu, 'manage export folders', 'Manage folders to which the client can automatically export.', self._ManageExportFolders ) + ClientGUIMenus.AppendMenuItem( i_and_e_submenu, 'manage import folders' + HC.UNICODE_ELLIPSIS, 'Manage folders from which the client can automatically import.', self._ManageImportFolders ) + ClientGUIMenus.AppendMenuItem( i_and_e_submenu, 'manage export folders' + HC.UNICODE_ELLIPSIS, 'Manage folders to which the client can automatically export.', self._ManageExportFolders ) ClientGUIMenus.AppendMenu( menu, i_and_e_submenu, 'import and export folders' ) @@ -3314,8 +3314,8 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendSeparator( menu ) - ClientGUIMenus.AppendMenuItem( menu, 'options', 'Change how the client operates.', self._ManageOptions, role = QW.QAction.MenuRole.PreferencesRole ) - ClientGUIMenus.AppendMenuItem( menu, 'shortcuts', 'Edit the shortcuts your client responds to.', ClientGUIShortcutControls.ManageShortcuts, self, role = QW.QAction.MenuRole.ApplicationSpecificRole ) + ClientGUIMenus.AppendMenuItem( menu, 'options' + HC.UNICODE_ELLIPSIS, 'Change how the client operates.', self._ManageOptions, role = QW.QAction.MenuRole.PreferencesRole ) + ClientGUIMenus.AppendMenuItem( menu, 'shortcuts' + HC.UNICODE_ELLIPSIS, 'Edit the shortcuts your client responds to.', ClientGUIShortcutControls.ManageShortcuts, self, role = QW.QAction.MenuRole.ApplicationSpecificRole ) ClientGUIMenus.AppendSeparator( menu ) @@ -3369,7 +3369,7 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendSeparator( menu ) - ClientGUIMenus.AppendMenuItem( menu, 'add the public tag repository', 'This will add the public tag repository to your client.', self._AutoRepoSetup ) + ClientGUIMenus.AppendMenuItem( menu, 'add the public tag repository' + HC.UNICODE_ELLIPSIS, 'This will add the public tag repository to your client.', self._AutoRepoSetup ) ClientGUIMenus.AppendSeparator( menu ) @@ -3555,7 +3555,7 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendSeparator( menu ) - ClientGUIMenus.AppendMenuItem( menu, 'manage subscriptions', 'Change the queries you want the client to regularly import from.', self._ManageSubscriptions ) + ClientGUIMenus.AppendMenuItem( menu, 'manage subscriptions' + HC.UNICODE_ELLIPSIS, 'Change the queries you want the client to regularly import from.', self._ManageSubscriptions ) ClientGUIMenus.AppendSeparator( menu ) @@ -3564,11 +3564,11 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendMenuItem( submenu, 'review bandwidth usage and edit rules', 'See where you are consuming data.', self._ReviewBandwidth ) ClientGUIMenus.AppendMenuItem( submenu, 'review current network jobs', 'Review the jobs currently running in the network engine.', self._ReviewNetworkJobs ) ClientGUIMenus.AppendMenuItem( submenu, 'review session cookies', 'Review and edit which cookies you have for which network contexts.', self._ReviewNetworkSessions ) - ClientGUIMenus.AppendMenuItem( submenu, 'manage http headers', 'Configure how the client talks to the network.', self._ManageNetworkHeaders ) + ClientGUIMenus.AppendMenuItem( submenu, 'manage http headers' + HC.UNICODE_ELLIPSIS, 'Configure how the client talks to the network.', self._ManageNetworkHeaders ) ClientGUIMenus.AppendSeparator( submenu ) - ClientGUIMenus.AppendMenuItem( submenu, 'manage upnp', 'If your router supports it, see and edit your current UPnP NAT traversal mappings.', self._ManageUPnP ) + ClientGUIMenus.AppendMenuItem( submenu, 'manage upnp' + HC.UNICODE_ELLIPSIS, 'If your router supports it, see and edit your current UPnP NAT traversal mappings.', self._ManageUPnP ) ClientGUIMenus.AppendMenu( menu, submenu, 'data' ) @@ -3587,13 +3587,13 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendSeparator( submenu ) - ClientGUIMenus.AppendMenuItem( submenu, 'import downloaders', 'Import new download capability through encoded pngs from other users.', self._ImportDownloaders ) - ClientGUIMenus.AppendMenuItem( submenu, 'export downloaders', 'Export downloader components to easy-import pngs.', self._ExportDownloader ) + ClientGUIMenus.AppendMenuItem( submenu, 'import downloaders' + HC.UNICODE_ELLIPSIS, 'Import new download capability through encoded pngs from other users.', self._ImportDownloaders ) + ClientGUIMenus.AppendMenuItem( submenu, 'export downloaders' + HC.UNICODE_ELLIPSIS, 'Export downloader components to easy-import pngs.', self._ExportDownloader ) ClientGUIMenus.AppendSeparator( submenu ) - ClientGUIMenus.AppendMenuItem( submenu, 'manage default import options', 'Change the default import options for each of your linked url matches.', self._ManageDefaultImportOptions ) - ClientGUIMenus.AppendMenuItem( submenu, 'manage downloader and url display', 'Configure how downloader objects present across the client.', self._ManageDownloaderDisplay ) + ClientGUIMenus.AppendMenuItem( submenu, 'manage default import options' + HC.UNICODE_ELLIPSIS, 'Change the default import options for each of your linked url matches.', self._ManageDefaultImportOptions ) + ClientGUIMenus.AppendMenuItem( submenu, 'manage downloader and url display' + HC.UNICODE_ELLIPSIS, 'Configure how downloader objects present across the client.', self._ManageDownloaderDisplay ) ClientGUIMenus.AppendSeparator( submenu ) @@ -3610,17 +3610,17 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M submenu = ClientGUIMenus.GenerateMenu( menu ) - ClientGUIMenus.AppendMenuItem( submenu, 'manage url class links', 'Configure how URLs present across the client.', self._ManageURLClassLinks ) + ClientGUIMenus.AppendMenuItem( submenu, 'manage url class links' + HC.UNICODE_ELLIPSIS, 'Configure how URLs present across the client.', self._ManageURLClassLinks ) ClientGUIMenus.AppendSeparator( submenu ) - ClientGUIMenus.AppendMenuItem( submenu, 'manage gallery url generators', 'Manage the client\'s GUGs, which convert search terms into URLs.', self._ManageGUGs ) - ClientGUIMenus.AppendMenuItem( submenu, 'manage url classes', 'Configure which URLs the client can recognise.', self._ManageURLClasses ) - ClientGUIMenus.AppendMenuItem( submenu, 'manage parsers', 'Manage the client\'s parsers, which convert URL content into hydrus metadata.', self._ManageParsers ) + ClientGUIMenus.AppendMenuItem( submenu, 'manage gallery url generators' + HC.UNICODE_ELLIPSIS, 'Manage the client\'s GUGs, which convert search terms into URLs.', self._ManageGUGs ) + ClientGUIMenus.AppendMenuItem( submenu, 'manage url classes' + HC.UNICODE_ELLIPSIS, 'Configure which URLs the client can recognise.', self._ManageURLClasses ) + ClientGUIMenus.AppendMenuItem( submenu, 'manage parsers' + HC.UNICODE_ELLIPSIS, 'Manage the client\'s parsers, which convert URL content into hydrus metadata.', self._ManageParsers ) ClientGUIMenus.AppendSeparator( submenu ) - ClientGUIMenus.AppendMenuItem( submenu, 'SEMI-LEGACY: manage file lookup scripts', 'Manage how the client parses different types of web content.', self._ManageParsingScripts ) + ClientGUIMenus.AppendMenuItem( submenu, 'SEMI-LEGACY: manage file lookup scripts' + HC.UNICODE_ELLIPSIS, 'Manage how the client parses different types of web content.', self._ManageParsingScripts ) ClientGUIMenus.AppendMenu( menu, submenu, 'downloader components' ) @@ -3628,11 +3628,11 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M submenu = ClientGUIMenus.GenerateMenu( menu ) - ClientGUIMenus.AppendMenuItem( submenu, 'manage logins', 'Edit which domains you wish to log in to.', self._ManageLogins ) + ClientGUIMenus.AppendMenuItem( submenu, 'manage logins' + HC.UNICODE_ELLIPSIS, 'Edit which domains you wish to log in to.', self._ManageLogins ) ClientGUIMenus.AppendSeparator( submenu ) - ClientGUIMenus.AppendMenuItem( submenu, 'manage login scripts', 'Manage the client\'s login scripts, which define how to log in to different sites.', self._ManageLoginScripts ) + ClientGUIMenus.AppendMenuItem( submenu, 'manage login scripts' + HC.UNICODE_ELLIPSIS, 'Manage the client\'s login scripts, which define how to log in to different sites.', self._ManageLoginScripts ) ClientGUIMenus.AppendSeparator( submenu ) @@ -3683,7 +3683,7 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendSeparator( menu ) - ClientGUIMenus.AppendMenuItem( menu, 'pick a new page', 'Choose a new page to open.', self.ProcessApplicationCommand, CAC.ApplicationCommand.STATICCreateSimpleCommand( CAC.SIMPLE_NEW_PAGE ) ) + ClientGUIMenus.AppendMenuItem( menu, 'pick a new page' + HC.UNICODE_ELLIPSIS, 'Choose a new page to open.', self.ProcessApplicationCommand, CAC.ApplicationCommand.STATICCreateSimpleCommand( CAC.SIMPLE_NEW_PAGE ) ) # @@ -3753,7 +3753,7 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendSeparator( menu ) ClientGUIMenus.AppendMenuItem( menu, 'review services', 'Look at the services your client connects to.', self._ReviewServices ) - ClientGUIMenus.AppendMenuItem( menu, 'manage services', 'Edit the services your client connects to.', self._ManageServices ) + ClientGUIMenus.AppendMenuItem( menu, 'manage services' + HC.UNICODE_ELLIPSIS, 'Edit the services your client connects to.', self._ManageServices ) self._menubar_services_admin_submenu = ClientGUIMenus.GenerateMenu( menu ) @@ -3761,7 +3761,7 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M ClientGUIMenus.AppendSeparator( menu ) - ClientGUIMenus.AppendMenuItem( menu, 'import repository update files', 'Add repository update files to the database.', self._ImportUpdateFiles ) + ClientGUIMenus.AppendMenuItem( menu, 'import repository update files' + HC.UNICODE_ELLIPSIS, 'Add repository update files to the database.', self._ImportUpdateFiles ) return ( menu, '&services' ) @@ -3770,18 +3770,18 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M menu = ClientGUIMenus.GenerateMenu( self ) - ClientGUIMenus.AppendMenuItem( menu, 'migrate tags', 'Migrate tags from one place to another.', self._MigrateTags ) + ClientGUIMenus.AppendMenuItem( menu, 'migrate tags' + HC.UNICODE_ELLIPSIS, 'Migrate tags from one place to another.', self._MigrateTags ) ClientGUIMenus.AppendSeparator( menu ) - ClientGUIMenus.AppendMenuItem( menu, 'manage tag display and search', 'Set which tags you want to see from which services.', self._ManageTagDisplay ) + ClientGUIMenus.AppendMenuItem( menu, 'manage tag display and search' + HC.UNICODE_ELLIPSIS, 'Set which tags you want to see from which services.', self._ManageTagDisplay ) ClientGUIMenus.AppendSeparator( menu ) - ClientGUIMenus.AppendMenuItem( menu, 'manage tag siblings', 'Set certain tags to be automatically replaced with other tags.', self._ManageTagSiblings ) - ClientGUIMenus.AppendMenuItem( menu, 'manage tag parents', 'Set certain tags to be automatically added with other tags.', self._ManageTagParents ) + ClientGUIMenus.AppendMenuItem( menu, 'manage tag siblings' + HC.UNICODE_ELLIPSIS, 'Set certain tags to be automatically replaced with other tags.', self._ManageTagSiblings ) + ClientGUIMenus.AppendMenuItem( menu, 'manage tag parents' + HC.UNICODE_ELLIPSIS, 'Set certain tags to be automatically added with other tags.', self._ManageTagParents ) - ClientGUIMenus.AppendMenuItem( menu, 'manage where tag siblings and parents apply', 'Set which services\' siblings and parents apply where.', self._ManageTagDisplayApplication ) + ClientGUIMenus.AppendMenuItem( menu, 'manage where tag siblings and parents apply' + HC.UNICODE_ELLIPSIS, 'Set which services\' siblings and parents apply where.', self._ManageTagDisplayApplication ) # From a99ab83ecb9911d9980aca919be5511e606884fa Mon Sep 17 00:00:00 2001 From: Aidan Harris Date: Sat, 30 Mar 2024 18:43:16 +0000 Subject: [PATCH 4/5] Fix the application icon on KDE Plasma Wayland (#1531) * Fix the application icon on KDE Plasma Wayland This fixes the application icon on KDE Plasma Wayland (tested on Plasma 6) and possibly other Wayland compositors too. The desktop filename was chosen because this is the identifier that the Hydrus Flatpak application uses. Distribution packagers should make sure that this name matches with the desktop file that they install (for example, Gentoo installs a /usr/share/applications/hydrus-client-hydrus.desktop file). It is strongly suggested that distributions install their Hydrus .desktop file at /usr/share/applications/io.github.hydrusnetwork.hydrus.desktop to avoid needing to apply extra patches, etc. * Rename .desktop file and update setup_desktop.sh --- docs/running_from_source.md | 2 +- hydrus/client/ClientController.py | 1 + setup_desktop.sh | 8 ++++---- ...rus.desktop => io.github.hydrusnetwork.hydrus.desktop} | 0 4 files changed, 6 insertions(+), 5 deletions(-) rename static/{hydrus.desktop => io.github.hydrusnetwork.hydrus.desktop} (100%) diff --git a/docs/running_from_source.md b/docs/running_from_source.md index 812d46cf..b9ed1b4f 100644 --- a/docs/running_from_source.md +++ b/docs/running_from_source.md @@ -169,7 +169,7 @@ There are three special external libraries. You just have to get them and put th If you get an error about the venv failing to activate during `setup_venv.sh`, you may need to install venv especially for your system. The specific error message should help you out, but you'll be looking at something along the lines of `apt install python3.10-venv`. - If you like, you can run the `setup_desktop.sh` file to install a hydrus.desktop file to your applications folder. (Or check the template in `install_dir/static/hydrus.desktop` and do it yourself!) + If you like, you can run the `setup_desktop.sh` file to install an io.github.hydrusnetwork.hydrus.desktop file to your applications folder. (Or check the template in `install_dir/static/io.github.hydrusnetwork.hydrus.desktop` and do it yourself!) === "macOS" diff --git a/hydrus/client/ClientController.py b/hydrus/client/ClientController.py index 443f5720..47a94cd2 100644 --- a/hydrus/client/ClientController.py +++ b/hydrus/client/ClientController.py @@ -105,6 +105,7 @@ class App( QW.QApplication ): self._pubsub = pubsub self.setApplicationName( 'Hydrus Client' ) + self.setDesktopFileName( 'io.github.hydrusnetwork.hydrus' ) self.setApplicationVersion( str( HC.SOFTWARE_VERSION ) ) diff --git a/setup_desktop.sh b/setup_desktop.sh index c0e1df12..35cc5407 100755 --- a/setup_desktop.sh +++ b/setup_desktop.sh @@ -3,8 +3,8 @@ pushd "$(dirname "$0")" || exit 1 INSTALL_DIR="$(readlink -f .)" -DESKTOP_SOURCE_PATH=$INSTALL_DIR/static/hydrus.desktop -DESKTOP_DEST_PATH=$HOME/.local/share/applications/hydrus.desktop +DESKTOP_SOURCE_PATH=$INSTALL_DIR/static/io.github.hydrusnetwork.hydrus.desktop +DESKTOP_DEST_PATH=$HOME/.local/share/applications/io.github.hydrusnetwork.hydrus.desktop echo "Install folder appears to be $INSTALL_DIR" @@ -16,11 +16,11 @@ fi if [ -f "$DESKTOP_DEST_PATH" ]; then - echo "You already have a hydrus.desktop file at $DESKTOP_DEST_PATH. Would you like to overwrite it? y/n " + echo "You already have an io.github.hydrusnetwork.hydrus.desktop file at $DESKTOP_DEST_PATH. Would you like to overwrite it? y/n " else - echo "Create a hydrus.desktop file at $DESKTOP_DEST_PATH? y/n " + echo "Create an io.github.hydrusnetwork.hydrus.desktop file at $DESKTOP_DEST_PATH? y/n " fi diff --git a/static/hydrus.desktop b/static/io.github.hydrusnetwork.hydrus.desktop similarity index 100% rename from static/hydrus.desktop rename to static/io.github.hydrusnetwork.hydrus.desktop From 3aa8702972e73eb8e6b6c30efcf8694cddea69e5 Mon Sep 17 00:00:00 2001 From: Hydrus Network Developer Date: Wed, 3 Apr 2024 16:15:48 -0500 Subject: [PATCH 5/5] Version 569 --- docs/changelog.md | 40 ++++++ docs/developer_api.md | 4 +- docs/old_changelog.html | 35 ++++++ hydrus/client/ClientController.py | 56 +++++++-- hydrus/client/ClientParsing.py | 46 +++---- .../client/exporting/ClientExportingFiles.py | 4 +- hydrus/client/gui/ClientGUI.py | 5 +- hydrus/client/gui/ClientGUIDownloaders.py | 21 ++-- hydrus/client/gui/ClientGUIFileSeedCache.py | 16 ++- .../gui/ClientGUIScrolledPanelsManagement.py | 19 +-- .../client/gui/exporting/ClientGUIExport.py | 17 ++- .../gui/pages/ClientGUIManagementPanels.py | 9 +- .../client/importing/ClientImportFileSeeds.py | 24 ++-- .../ClientMetadataMigrationExporters.py | 5 +- .../networking/ClientNetworkingDomain.py | 20 +-- .../networking/ClientNetworkingFunctions.py | 114 +++++++++++++++--- .../client/networking/ClientNetworkingGUG.py | 8 +- .../networking/ClientNetworkingURLClass.py | 89 ++++++-------- hydrus/client/search/ClientSearch.py | 92 ++++++++++++-- hydrus/core/HydrusConstants.py | 2 +- hydrus/test/TestClientAPI.py | 4 +- hydrus/test/TestClientDB.py | 4 +- hydrus/test/TestClientNetworking.py | 52 ++------ hydrus_client.sh | 13 +- setup_venv.bat | 47 ++++++-- 25 files changed, 493 insertions(+), 253 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 7dcf9450..87466e54 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,6 +7,46 @@ title: Changelog !!! note This is the new changelog, only the most recent builds. For all versions, see the [old changelog](old_changelog.html). +## [Version 569](https://github.com/hydrusnetwork/hydrus/releases/tag/v569) + +### user contributions + +* thanks to a user, fixed a problem with the recent URL changes that caused downloaders examining multi-file posts to only grab the first file +* thanks to a user, all the menubar commands that launch a modal dialog are now suffix'd by an ellipsis +* thanks to a user, fixed an issue regarding KDE 6 quitting the program as soon as the pre-boot 'your database is missing a location, let's find it' repair dialog was ok'd +* thanks to a user, the application icon is fixed in KDE Plasma Wayland (and anything else that pulls icon from .desktop file). if you have been using a hydrus.desktop file and don't see a program icon, you should rename it to `/usr/share/applications/io.github.hydrusnetwork.hydrus.desktop` . more importantly, if you manage a package for hydrus--please output to this file path instead of `hydrus.desktop` if you make one +* thanks to a user, updated the `hydrus_client.sh` file to include `"$@"`, which passes parameters given to the .sh file to the .py call + +### more on last week's URL work + +* fixed the 'show the Request URL under "additional urls" submenu' thing on the file log list menu. I screwed up the logic and was effectively testing for when `1 != 1` +* the converter that generates a Referral URL now operates on the API/redirect conversion principle too--it normalises the Source URL to its 'Request URL' state--keeping defined ephemeral params and filling in defaults but dropping any extra gubbins not asked for--before applying the conversion +* fixed the 'manage url class' dialog to correctly display an example API/redirect-converted URL based on the new _request url_, not the _normalised url_ (so the api/redirect example will now show the new ephemeral params properly). this was working in requests correctly behind the scenes, it was just the example text box in the dialog that was showing wrong +* improved the 'is this query text pre-encoded?' test to check for `%hh`, where `h` is a hexadecimal character, instead of the hackier 'is % in it while not followed by whitespace or end of string?' +* improved/simplified/optimised the overall procedure that figures out if an entered URL is pre-encoded or not. this routine now only runs at the stage where a URL is ingested and it obeys the `%hh` rule. these ingestion points are currently: the text boxes in a urls downloader/simple downloader page; the 'import new sources' function of file log menus; a URL `ContentParser` in the parsing system; the test box in `manage url classes`; and the main gui's 'import url' landing pad, which is used by the drag and drop system, the clipboard watcher, and the client api's 'import url' command. note that this does not occur on 'manage known urls' editing, where you can do what you want with whatever, and I won't coerce it to anything + +### misc + +* fixed a variety of logical cases around >0, =0, !=0, <0 for the `NumberTest` objects I recently applied to system:duration and elsewhere. when it comes to file searching, files that have 'None' duration are now considered equivalent to files that have an explicit 0 duration in all cases. previously, I was trying to thread a needle where '=0' would find null results but <x would not, and it was a mess. now it all works the same way. if you want to search for 'duration < x' and want to exclude still images, either add a filetype pred or slap on 'has duration' +* improved the stability of the manual file exporter process. it was consulting an object in a thread that it shouldn't have +* improved the ability of the manual file exporter process to report errors on a very large export that encounters errors after the dialog has closed +* fixed the 'remember last used default tag service in manage tag dialogs' and its accompanying dropdown not saving their current value on options dialog ok. sorry for the trouble! +* fixed the system that truncates very long filenames (for export folders and drag and drop exports) on Linux when the exporter is also outputting a sidecar that has a long extra suffix +* the 'find potential duplicate pairs' routine that runs in idle time now properly obeys the work/rest times in `options->maintenance and processing`. previously, it was just the 'run now' routine that was resting in that way, and the idle thing was just doing a hardcoded 'work for 60 seconds every 10 mins or so'. thanks to the reporting user who cleverly noticed this +* the `options->connection` page now mentions your proxy needs to be `http://` + +### boring stuff + +* updated the windows setup_venv.bat to allow for custom python or venv locations using parameters. this was so I could set up a multi-python testing situation easier +* added some unit tests for the new URL encoding gubbins +* improved un-encoded URL parsing in the downloader when the URL is relative and needs to be joined to the source url +* improved some URL parsing and ingestion to better handle urls with non-ascii characters in the domain +* replaced several 'does it start with "http"?' areas with a better and unified scheme/netloc test +* wrote a routine to split URL paths into path components, and spammed it everywhere so this code is now unified. I expect we'll get a `PathComponent` class at some point, too. there will be a future question about what to do with double slashes, `//` in paths--it turns out the logic has been mixed here, and I think I will probably collapse them to `/` in all cases +* rewrote an unhealthy call that indirectly caused the above multi-file post parsing problem +* fixed some None/0 `NumberTest` stuff if you manage to enter '<0' or >-5 and similar +* I figured out the problems with PyInstaller 6.x and some other stuff, there should be a 'Future Build' alongside this release in github for advanced users to test with + ## [Version 568](https://github.com/hydrusnetwork/hydrus/releases/tag/v568) ### user contributions diff --git a/docs/developer_api.md b/docs/developer_api.md index f365e93f..5473b941 100644 --- a/docs/developer_api.md +++ b/docs/developer_api.md @@ -1783,7 +1783,7 @@ Response: "has_transparency" : false, "known_urls" : [ "https://gelbooru.com/index.php?page=post&s=view&id=4841557", - "https://img2.gelbooru.com//images/80/c8/80c8646b4a49395fb36c805f316c49a9.jpg", + "https://img2.gelbooru.com/images/80/c8/80c8646b4a49395fb36c805f316c49a9.jpg", "http://origin-orig.deviantart.net/ed31/f/2019/210/7/8/beachqueen_samus_by_dandonfuga-ddcu1xg.jpg" ], "ratings" : { @@ -1971,7 +1971,7 @@ If you add `detailed_url_information=true`, a new entry, `detailed_known_urls`, "can_parse": true }, { - "normalised_url": "https://img2.gelbooru.com//images/80/c8/80c8646b4a49395fb36c805f316c49a9.jpg", + "normalised_url": "https://img2.gelbooru.com/images/80/c8/80c8646b4a49395fb36c805f316c49a9.jpg", "url_type": 5, "url_type_string": "unknown url", "match_name": "unknown url", diff --git a/docs/old_changelog.html b/docs/old_changelog.html index 0c3fdf35..07d49251 100644 --- a/docs/old_changelog.html +++ b/docs/old_changelog.html @@ -34,6 +34,41 @@

changelog

    +
  • +

    version 569

    +
      +

    • user contributions

      • +
    • thanks to a user, fixed a problem with the recent URL changes that caused downloaders examining multi-file posts to only grab the first file
      • +
    • thanks to a user, all the menubar commands that launch a modal dialog are now suffix'd by an ellipsis
      • +
    • thanks to a user, fixed an issue regarding KDE 6 quitting the program as soon as the pre-boot 'your database is missing a location, let's find it' repair dialog was ok'd
      • +
    • thanks to a user, the application icon is fixed in KDE Plasma Wayland (and anything else that pulls icon from .desktop file). if you have been using a hydrus.desktop file and don't see a program icon, you should rename it to `/usr/share/applications/io.github.hydrusnetwork.hydrus.desktop` . more importantly, if you manage a package for hydrus--please output to this file path instead of `hydrus.desktop` if you make one
      • +
    • thanks to a user, updated the `hydrus_client.sh` file to include `"$@"`, which passes parameters given to the .sh file to the .py call
      • +

    • more on last week's URL work

      • +
    • fixed the 'show the Request URL under "additional urls" submenu' thing on the file log list menu. I screwed up the logic and was effectively testing for when `1 != 1`
      • +
    • the converter that generates a Referral URL now operates on the API/redirect conversion principle too--it normalises the Source URL to its 'Request URL' state--keeping defined ephemeral params and filling in defaults but dropping any extra gubbins not asked for--before applying the conversion
      • +
    • fixed the 'manage url class' dialog to correctly display an example API/redirect-converted URL based on the new _request url_, not the _normalised url_ (so the api/redirect example will now show the new ephemeral params properly). this was working in requests correctly behind the scenes, it was just the example text box in the dialog that was showing wrong
      • +
    • improved the 'is this query text pre-encoded?' test to check for `%hh`, where `h` is a hexadecimal character, instead of the hackier 'is % in it while not followed by whitespace or end of string?'
      • +
    • improved/simplified/optimised the overall procedure that figures out if an entered URL is pre-encoded or not. this routine now only runs at the stage where a URL is ingested and it obeys the `%hh` rule. these ingestion points are currently: the text boxes in a urls downloader/simple downloader page; the 'import new sources' function of file log menus; a URL `ContentParser` in the parsing system; the test box in `manage url classes`; and the main gui's 'import url' landing pad, which is used by the drag and drop system, the clipboard watcher, and the client api's 'import url' command. note that this does not occur on 'manage known urls' editing, where you can do what you want with whatever, and I won't coerce it to anything
      • +

    • misc

      • +
    • fixed a variety of logical cases around >0, =0, !=0, <0 for the `NumberTest` objects I recently applied to system:duration and elsewhere. when it comes to file searching, files that have 'None' duration are now considered equivalent to files that have an explicit 0 duration in all cases. previously, I was trying to thread a needle where '=0' would find null results but <x would not, and it was a mess. now it all works the same way. if you want to search for 'duration < x' and want to exclude still images, either add a filetype pred or slap on 'has duration'
      • +
    • improved the stability of the manual file exporter process. it was consulting an object in a thread that it shouldn't have
      • +
    • improved the ability of the manual file exporter process to report errors on a very large export that encounters errors after the dialog has closed
      • +
    • fixed the 'remember last used default tag service in manage tag dialogs' and its accompanying dropdown not saving their current value on options dialog ok. sorry for the trouble!
      • +
    • fixed the system that truncates very long filenames (for export folders and drag and drop exports) on Linux when the exporter is also outputting a sidecar that has a long extra suffix
      • +
    • the 'find potential duplicate pairs' routine that runs in idle time now properly obeys the work/rest times in `options->maintenance and processing`. previously, it was just the 'run now' routine that was resting in that way, and the idle thing was just doing a hardcoded 'work for 60 seconds every 10 mins or so'. thanks to the reporting user who cleverly noticed this
      • +
    • the `options->connection` page now mentions your proxy needs to be `http://`
      • +

    • boring stuff

      • +
    • updated the windows setup_venv.bat to allow for custom python or venv locations using parameters. this was so I could set up a multi-python testing situation easier
      • +
    • added some unit tests for the new URL encoding gubbins
      • +
    • improved un-encoded URL parsing in the downloader when the URL is relative and needs to be joined to the source url
      • +
    • improved some URL parsing and ingestion to better handle urls with non-ascii characters in the domain
      • +
    • replaced several 'does it start with "http"?' areas with a better and unified scheme/netloc test
      • +
    • wrote a routine to split URL paths into path components, and spammed it everywhere so this code is now unified. I expect we'll get a `PathComponent` class at some point, too. there will be a future question about what to do with double slashes, `//` in paths--it turns out the logic has been mixed here, and I think I will probably collapse them to `/` in all cases
      • +
    • rewrote an unhealthy call that indirectly caused the above multi-file post parsing problem
      • +
    • fixed some None/0 `NumberTest` stuff if you manage to enter '<0' or >-5 and similar
      • +
    • I figured out the problems with PyInstaller 6.x and some other stuff, there should be a 'Future Build' alongside this release in github for advanced users to test with
      • +
    +

  • version 568

    Version 567 was cancelled, its changes folded into 568

    diff --git a/hydrus/client/ClientController.py b/hydrus/client/ClientController.py index 47a94cd2..81816177 100644 --- a/hydrus/client/ClientController.py +++ b/hydrus/client/ClientController.py @@ -105,7 +105,11 @@ class App( QW.QApplication ): self._pubsub = pubsub self.setApplicationName( 'Hydrus Client' ) - self.setDesktopFileName( 'io.github.hydrusnetwork.hydrus' ) + + if HC.PLATFORM_LINUX: + + self.setDesktopFileName( 'io.github.hydrusnetwork.hydrus' ) + self.setApplicationVersion( str( HC.SOFTWARE_VERSION ) ) @@ -1426,20 +1430,52 @@ class Controller( ClientControllerInterface.ClientControllerInterface, HydrusCon if self.new_options.GetBoolean( 'maintain_similar_files_duplicate_pairs_during_idle' ): - search_distance = self.new_options.GetInteger( 'similar_files_duplicate_pairs_search_distance' ) + work_done = False - search_stop_time = stop_time + still_work_to_do = True - if search_stop_time is None: + while still_work_to_do: - search_stop_time = HydrusTime.GetNow() + 60 + search_distance = CG.client_controller.new_options.GetInteger( 'similar_files_duplicate_pairs_search_distance' ) + + start_time = HydrusTime.GetNowPrecise() + + work_time_ms = CG.client_controller.new_options.GetInteger( 'potential_duplicates_search_work_time_ms' ) + + work_time = work_time_ms / 1000 + + ( still_work_to_do, num_done ) = CG.client_controller.WriteSynchronous( 'maintain_similar_files_search_for_potential_duplicates', search_distance, maintenance_mode = maintenance_mode, work_time_float = work_time ) + + if num_done > 0: + + work_done = True + + + if not still_work_to_do: + + break + + + if self.ShouldStopThisWork( maintenance_mode, stop_time = stop_time ): + + break + + + time_it_took = HydrusTime.GetNowPrecise() - start_time + + rest_ratio = CG.client_controller.new_options.GetInteger( 'potential_duplicates_search_rest_percentage' ) / 100 + + reasonable_work_time = min( 5 * work_time, time_it_took ) + + time.sleep( reasonable_work_time * rest_ratio ) - self.WriteSynchronous( 'maintain_similar_files_search_for_potential_duplicates', search_distance, maintenance_mode = maintenance_mode, stop_time = search_stop_time ) - - from hydrus.client import ClientDuplicates - - ClientDuplicates.DuplicatesManager.instance().RefreshMaintenanceNumbers() + if work_done: + + from hydrus.client import ClientDuplicates + + ClientDuplicates.DuplicatesManager.instance().RefreshMaintenanceNumbers() + if self.ShouldStopThisWork( maintenance_mode, stop_time = stop_time ): diff --git a/hydrus/client/ClientParsing.py b/hydrus/client/ClientParsing.py index e82e1eab..5b0101d2 100644 --- a/hydrus/client/ClientParsing.py +++ b/hydrus/client/ClientParsing.py @@ -2335,12 +2335,12 @@ class ContentParser( HydrusSerialisable.SerialisableBase ): base_url = parsing_context[ 'url' ] - def clean_url( u ): + def remove_pre_url_gubbins( u ): # clears up when a source field starts with gubbins for some reason. e.g.: - # (jap characters).avi | ranken [pixiv] http:/www.pixiv.net/member_illust.php?illust_id=48114073&mode=medium + # (jap characters).avi | ranken [pixiv] http://www.pixiv.net/member_illust.php?illust_id=48114073&mode=medium # -> - # http:/www.pixiv.net/member_illust.php?illust_id=48114073&mode=medium + # http://www.pixiv.net/member_illust.php?illust_id=48114073&mode=medium gumpf_until_scheme = r'^.*\s(?Phttps?://)' @@ -2368,33 +2368,33 @@ class ContentParser( HydrusSerialisable.SerialisableBase ): return u - clean_parsed_texts = [] + clean_urls = [] - for parsed_text in parsed_texts: + for unclean_url in parsed_texts: - if not ClientNetworkingFunctions.LooksLikeAFullURL( parsed_text ) and ( 'http://' in parsed_text or 'https://' in parsed_text ): + if not ClientNetworkingFunctions.LooksLikeAFullURL( unclean_url ) and ( 'http://' in unclean_url or 'https://' in unclean_url ): - parsed_text = clean_url( parsed_text ) + unclean_url = remove_pre_url_gubbins( unclean_url ) - clean_parsed_texts.append( parsed_text ) + if not ClientNetworkingFunctions.LooksLikeAFullURL( unclean_url ): + + try: + + unclean_url = urllib.parse.urljoin( base_url, unclean_url ) + + except: + + pass # welp + + + + clean_url = ClientNetworkingFunctions.WashURL( unclean_url ) + + clean_urls.append( clean_url ) - parsed_texts = [] - - for clean_parsed_text in clean_parsed_texts: - - try: - - parsed_text = urllib.parse.urljoin( base_url, clean_parsed_text ) - - except: - - parsed_text = clean_parsed_text - - - parsed_texts.append( parsed_text ) - + parsed_texts = clean_urls diff --git a/hydrus/client/exporting/ClientExportingFiles.py b/hydrus/client/exporting/ClientExportingFiles.py index 5ea5950b..d8ef0db5 100644 --- a/hydrus/client/exporting/ClientExportingFiles.py +++ b/hydrus/client/exporting/ClientExportingFiles.py @@ -154,9 +154,9 @@ def GenerateExportFilename( destination_directory, media, terms, file_index, do_ # sidecar suffixes, and in general we don't want to spam giganto strings to people's hard drives - extra_characters_and_padding = 64 + extra_characters_and_padding = 64 + len( ext ) - filename = HydrusPaths.ElideFilenameOrDirectorySafely( filename, num_characters_used_in_other_components = destination_directory_num_characters_in_filesystem + extra_characters_and_padding ) + filename = HydrusPaths.ElideFilenameOrDirectorySafely( filename, num_characters_already_used_in_this_component = extra_characters_and_padding, num_characters_used_in_other_components = destination_directory_num_characters_in_filesystem ) if do_not_use_filenames is not None: diff --git a/hydrus/client/gui/ClientGUI.py b/hydrus/client/gui/ClientGUI.py index 026fa557..a237cc64 100644 --- a/hydrus/client/gui/ClientGUI.py +++ b/hydrus/client/gui/ClientGUI.py @@ -98,6 +98,7 @@ from hydrus.client.interfaces import ClientControllerInterface from hydrus.client.media import ClientMediaResult from hydrus.client.metadata import ClientContentUpdates from hydrus.client.metadata import ClientTags +from hydrus.client.networking import ClientNetworkingFunctions MENU_ORDER = [ 'file', 'undo', 'pages', 'database', 'network', 'services', 'tags', 'pending', 'help' ] @@ -2192,7 +2193,7 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M def _ImportURL( self, - url, + unclean_url, filterable_tags = None, additional_service_keys_to_tags = None, destination_page_name = None, @@ -2213,6 +2214,8 @@ class FrameGUI( CAC.ApplicationCommandProcessorMixin, ClientGUITopLevelWindows.M additional_service_keys_to_tags = ClientTags.ServiceKeysToTags() + url = ClientNetworkingFunctions.WashURL( unclean_url ) + url = CG.client_controller.network_engine.domain_manager.NormaliseURL( url, for_server = True ) ( url_type, match_name, can_parse, cannot_parse_reason ) = self._controller.network_engine.domain_manager.GetURLParseCapability( url ) diff --git a/hydrus/client/gui/ClientGUIDownloaders.py b/hydrus/client/gui/ClientGUIDownloaders.py index d0b11523..2d9d0631 100644 --- a/hydrus/client/gui/ClientGUIDownloaders.py +++ b/hydrus/client/gui/ClientGUIDownloaders.py @@ -30,6 +30,7 @@ from hydrus.client.gui.lists import ClientGUIListCtrl from hydrus.client.gui.widgets import ClientGUICommon from hydrus.client.gui.widgets import ClientGUIMenuButton from hydrus.client.networking import ClientNetworkingDomain +from hydrus.client.networking import ClientNetworkingFunctions from hydrus.client.networking import ClientNetworkingGUG from hydrus.client.networking import ClientNetworkingURLClass @@ -2055,12 +2056,16 @@ class EditURLClassPanel( ClientGUIScrolledPanels.EditPanel ): self._example_url_classes.setText( 'Example matches ok!' ) self._example_url_classes.setObjectName( 'HydrusValid' ) + for_server_normalised = url_class.Normalise( example_url, for_server = True ) + + self._for_server_normalised_url.setText( for_server_normalised ) + normalised = url_class.Normalise( example_url ) self._normalised_url.setText( normalised ) - self._referral_url_converter.SetExampleString( normalised ) - self._api_lookup_converter.SetExampleString( normalised ) + self._referral_url_converter.SetExampleString( for_server_normalised ) + self._api_lookup_converter.SetExampleString( for_server_normalised ) if url_class.UsesAPIURL(): @@ -2106,15 +2111,11 @@ class EditURLClassPanel( ClientGUIScrolledPanels.EditPanel ): - for_server_normalised = url_class.Normalise( example_url, for_server = True ) - - self._for_server_normalised_url.setText( for_server_normalised ) - try: if url_class.UsesAPIURL(): - api_lookup_url = url_class.GetAPIURL( normalised ) + api_lookup_url = url_class.GetAPIURL( example_url ) if url_class.Matches( api_lookup_url ): @@ -2450,14 +2451,16 @@ class EditURLClassesPanel( ClientGUIScrolledPanels.EditPanel ): def _UpdateURLClassCheckerText( self ): - url = self._url_class_checker.text() + unclean_url = self._url_class_checker.text() - if url == '': + if unclean_url == '': text = '<-- Enter a URL here to see which url class it currently matches!' else: + url = ClientNetworkingFunctions.WashURL( unclean_url ) + url_classes = self.GetValue() domain_manager = ClientNetworkingDomain.NetworkDomainManager() diff --git a/hydrus/client/gui/ClientGUIFileSeedCache.py b/hydrus/client/gui/ClientGUIFileSeedCache.py index d5d27c05..94b9ce44 100644 --- a/hydrus/client/gui/ClientGUIFileSeedCache.py +++ b/hydrus/client/gui/ClientGUIFileSeedCache.py @@ -1,13 +1,10 @@ import os -from qtpy import QtCore as QC from qtpy import QtWidgets as QW -from qtpy import QtGui as QG from hydrus.core import HydrusConstants as HC from hydrus.core import HydrusData from hydrus.core import HydrusExceptions -from hydrus.core import HydrusGlobals as HG from hydrus.core import HydrusPaths from hydrus.core import HydrusText @@ -70,6 +67,8 @@ def GetSourcesFromSourcesString( sources_string ): sources = HydrusText.DeserialiseNewlinedTexts( sources_string ) + sources = [ ClientNetworkingFunctions.WashURL( source ) for source in sources ] + return sources def ExportToClipboard( file_seed_cache: ClientImportFileSeeds.FileSeedCache ): @@ -528,14 +527,13 @@ class EditFileSeedCachePanel( ClientGUIScrolledPanels.EditPanel ): if selected_file_seed.IsURLFileImport(): - main_url = selected_file_seed.file_seed_data + request_url = selected_file_seed.file_seed_data + normalised_url = selected_file_seed.file_seed_data_for_comparison referral_url = selected_file_seed.GetReferralURL() primary_urls = sorted( selected_file_seed.GetPrimaryURLs() ) source_urls = sorted( selected_file_seed.GetSourceURLs() ) - for_server_url = CG.client_controller.network_engine.domain_manager.GetURLToFetch( main_url ) - - nothing_interesting_going_on = main_url == for_server_url and referral_url is None and len( primary_urls ) == 0 and len( source_urls ) == 0 + nothing_interesting_going_on = request_url == normalised_url and referral_url is None and len( primary_urls ) == 0 and len( source_urls ) == 0 if nothing_interesting_going_on: @@ -545,9 +543,9 @@ class EditFileSeedCachePanel( ClientGUIScrolledPanels.EditPanel ): url_submenu = ClientGUIMenus.GenerateMenu( menu ) - if main_url != for_server_url: + if request_url != normalised_url: - ClientGUIMenus.AppendMenuLabel( url_submenu, f'request url: {for_server_url}', copy_text = for_server_url ) + ClientGUIMenus.AppendMenuLabel( url_submenu, f'request url: {request_url}', copy_text = request_url ) if referral_url is not None: diff --git a/hydrus/client/gui/ClientGUIScrolledPanelsManagement.py b/hydrus/client/gui/ClientGUIScrolledPanelsManagement.py index c575b01f..787800b6 100644 --- a/hydrus/client/gui/ClientGUIScrolledPanelsManagement.py +++ b/hydrus/client/gui/ClientGUIScrolledPanelsManagement.py @@ -393,20 +393,22 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): general.Add( gridbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) - text = 'Enter strings such as "http://ip:port" or "http://user:pass@ip:port" to use for http and https traffic. It should take effect immediately on dialog ok.' - text += os.linesep * 2 - text += 'NO PROXY DOES NOT WORK UNLESS YOU HAVE A CUSTOM BUILD OF REQUESTS, SORRY! no_proxy takes the form of comma-separated hosts/domains, just as in curl or the NO_PROXY environment variable. When http and/or https proxies are set, they will not be used for these.' - text += os.linesep * 2 + text = 'PROTIP: Use a system-wide VPN or other software to handle this externally if you can. This tech is old and imperfect.' + text += '\n' * 2 + text += 'Enter strings such as "http://ip:port" or "http://user:pass@ip:port" to use for http and https traffic. It should take effect immediately on dialog ok. Note that you have to enter "http://", not "https://" (an HTTP proxy forwards your traffic, which when you talk to an https:// address will be encrypted, but it does not wrap that in an extra layer of encryption itself).' + text += '\n' * 2 + text += '"NO_PROXY" DOES NOT WORK UNLESS YOU HAVE A CUSTOM BUILD OF REQUESTS, SORRY! no_proxy takes the form of comma-separated hosts/domains, just as in curl or the NO_PROXY environment variable. When http and/or https proxies are set, they will not be used for these.' + text += '\n' * 2 if ClientNetworkingSessions.SOCKS_PROXY_OK: - text += 'It looks like you have socks support! You should also be able to enter (socks4 or) "socks5://ip:port".' - text += os.linesep + text += 'It looks like you have SOCKS support! You should also be able to enter (socks4 or) "socks5://ip:port".' + text += '\n' text += 'Use socks4a or socks5h to force remote DNS resolution, on the proxy server.' else: - text += 'It does not look like you have socks support! If you want it, try adding "pysocks" (or "requests[socks]")!' + text += 'It does not look like you have SOCKS support! If you want it, try adding "pysocks" (or "requests[socks]")!' st = ClientGUICommon.BetterStaticText( proxy_panel, text ) @@ -3215,6 +3217,9 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): self._new_options.SetInteger( 'ac_read_list_height_num_chars', self._ac_read_list_height_num_chars.value() ) self._new_options.SetInteger( 'ac_write_list_height_num_chars', self._ac_write_list_height_num_chars.value() ) + self._new_options.SetBoolean( 'save_default_tag_service_tab_on_change', self._save_default_tag_service_tab_on_change.isChecked() ) + self._new_options.SetKey( 'default_tag_service_tab', self._default_tag_service_tab.GetValue() ) + self._new_options.SetBoolean( 'always_show_system_everything', self._always_show_system_everything.isChecked() ) self._new_options.SetBoolean( 'filter_inbox_and_archive_predicates', self._filter_inbox_and_archive_predicates.isChecked() ) diff --git a/hydrus/client/gui/exporting/ClientGUIExport.py b/hydrus/client/gui/exporting/ClientGUIExport.py index a7e08049..dd248f60 100644 --- a/hydrus/client/gui/exporting/ClientGUIExport.py +++ b/hydrus/client/gui/exporting/ClientGUIExport.py @@ -806,7 +806,7 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ): - def do_it( directory, metadata_routers, delete_afterwards, export_symlinks, quit_afterwards ): + def do_it( directory, metadata_routers, delete_afterwards, export_symlinks, quit_afterwards, media_to_number_indices ): job_status = ClientThreading.JobStatus( cancellable = True ) @@ -818,7 +818,7 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ): for ( index, ( media, dest_path ) ) in enumerate( to_do ): - number = self._media_to_number_indices[ media ] + number = media_to_number_indices[ media ] if job_status.IsCancelled(): @@ -881,7 +881,16 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ): except: - ClientGUIDialogsMessage.ShowCritical( self, 'Problem during file export!', f'Encountered a problem while attempting to export file #{HydrusData.ToHumanInt( number )}:\n\n{traceback.format_exc()}' ) + if QP.isValid( self ): + + win = self + + else: + + win = CG.client_controller.gui + + + ClientGUIDialogsMessage.ShowCritical( win, 'Problem during file export!', f'Encountered a problem while attempting to export file #{HydrusData.ToHumanInt( number )}:\n\n{traceback.format_exc()}' ) break @@ -938,7 +947,7 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ): QP.CallAfter( qt_done, quit_afterwards ) - CG.client_controller.CallToThread( do_it, directory, metadata_routers, delete_afterwards, export_symlinks, quit_afterwards ) + CG.client_controller.CallToThread( do_it, directory, metadata_routers, delete_afterwards, export_symlinks, quit_afterwards, self._media_to_number_indices ) def _GetPath( self, media ): diff --git a/hydrus/client/gui/pages/ClientGUIManagementPanels.py b/hydrus/client/gui/pages/ClientGUIManagementPanels.py index 1b153af2..fc205435 100644 --- a/hydrus/client/gui/pages/ClientGUIManagementPanels.py +++ b/hydrus/client/gui/pages/ClientGUIManagementPanels.py @@ -64,6 +64,7 @@ from hydrus.client.importing.options import PresentationImportOptions from hydrus.client.media import ClientMedia from hydrus.client.metadata import ClientContentUpdates from hydrus.client.metadata import ClientTags +from hydrus.client.networking import ClientNetworkingFunctions from hydrus.client.search import ClientSearch management_panel_types_to_classes = {} @@ -3443,9 +3444,9 @@ class ManagementPanelImporterSimpleDownloader( ManagementPanelImporter ): self._RefreshFormulae() - def _PendPageURLs( self, urls ): + def _PendPageURLs( self, unclean_urls ): - urls = [ url for url in urls if url.startswith( 'http' ) ] + urls = [ ClientNetworkingFunctions.WashURL( unclean_url ) for unclean_url in unclean_urls if ClientNetworkingFunctions.LooksLikeAFullURL( unclean_url ) ] simple_downloader_formula = self._formulae.GetValue() @@ -3757,7 +3758,7 @@ class ManagementPanelImporterURLs( ManagementPanelImporter ): self._import_options_button.tagImportOptionsChanged.connect( self._urls_import.SetTagImportOptions ) - def _PendURLs( self, urls, filterable_tags = None, additional_service_keys_to_tags = None ): + def _PendURLs( self, unclean_urls, filterable_tags = None, additional_service_keys_to_tags = None ): if filterable_tags is None: @@ -3769,7 +3770,7 @@ class ManagementPanelImporterURLs( ManagementPanelImporter ): additional_service_keys_to_tags = ClientTags.ServiceKeysToTags() - urls = [ url for url in urls if url.startswith( 'http' ) ] + urls = [ ClientNetworkingFunctions.WashURL( unclean_url ) for unclean_url in unclean_urls if ClientNetworkingFunctions.LooksLikeAFullURL( unclean_url ) ] self._urls_import.PendURLs( urls, filterable_tags = filterable_tags, additional_service_keys_to_tags = additional_service_keys_to_tags ) diff --git a/hydrus/client/importing/ClientImportFileSeeds.py b/hydrus/client/importing/ClientImportFileSeeds.py index e79873d0..1ccfcda8 100644 --- a/hydrus/client/importing/ClientImportFileSeeds.py +++ b/hydrus/client/importing/ClientImportFileSeeds.py @@ -133,17 +133,7 @@ class FileSeed( HydrusSerialisable.SerialisableBase ): self.file_seed_data = file_seed_data self.file_seed_data_for_comparison = file_seed_data - if self.file_seed_type == FILE_SEED_TYPE_URL: - - try: - - self.file_seed_data_for_comparison = CG.client_controller.network_engine.domain_manager.NormaliseURL( self.file_seed_data ) - - except: - - pass - - + self.Normalise() # this fixes the comparison file seed data and fails safely self.created = HydrusTime.GetNow() self.modified = self.created @@ -1260,6 +1250,14 @@ class FileSeed( HydrusSerialisable.SerialisableBase ): + def SetFileSeedData( self, file_seed_data: str ): + + self.file_seed_data = file_seed_data + self.file_seed_data_for_comparison = file_seed_data + + self.Normalise() # this fixes the comparison file seed data and fails safely + + def SetHash( self, hash ): if hash is not None: @@ -1588,9 +1586,7 @@ class FileSeed( HydrusSerialisable.SerialisableBase ): duplicate_file_seed = self.Duplicate() # inherits all urls and tags from here - duplicate_file_seed.file_seed_data = child_url - - duplicate_file_seed.Normalise() + duplicate_file_seed.SetFileSeedData( child_url ) duplicate_file_seed.SetReferralURL( url_for_child_referral ) diff --git a/hydrus/client/metadata/ClientMetadataMigrationExporters.py b/hydrus/client/metadata/ClientMetadataMigrationExporters.py index c6fe8623..24208f08 100644 --- a/hydrus/client/metadata/ClientMetadataMigrationExporters.py +++ b/hydrus/client/metadata/ClientMetadataMigrationExporters.py @@ -17,6 +17,7 @@ from hydrus.client import ClientTime from hydrus.client.importing.options import NoteImportOptions from hydrus.client.metadata import ClientContentUpdates from hydrus.client.metadata import ClientMetadataMigrationCore +from hydrus.client.networking import ClientNetworkingFunctions class SingleFileMetadataExporter( ClientMetadataMigrationCore.ImporterExporterNode ): @@ -406,7 +407,9 @@ class SingleFileMetadataExporterMediaURLs( SingleFileMetadataExporterMedia, Hydr try: - url = CG.client_controller.network_engine.domain_manager.NormaliseURL( row ) + url = ClientNetworkingFunctions.WashURL( row ) + + url = CG.client_controller.network_engine.domain_manager.NormaliseURL( url ) urls.append( url ) diff --git a/hydrus/client/networking/ClientNetworkingDomain.py b/hydrus/client/networking/ClientNetworkingDomain.py index b4fa1beb..d022cf86 100644 --- a/hydrus/client/networking/ClientNetworkingDomain.py +++ b/hydrus/client/networking/ClientNetworkingDomain.py @@ -1518,24 +1518,8 @@ class NetworkDomainManager( HydrusSerialisable.SerialisableBase ): if url_class is None: - p = ClientNetworkingFunctions.ParseURL( url ) - - scheme = p.scheme - netloc = p.netloc - path = p.path - params = p.params - - # this puts them all in alphabetical order - - ( query_dict, single_value_parameters, param_order ) = ClientNetworkingFunctions.ConvertQueryTextToDict( p.query ) - - query = ClientNetworkingFunctions.ConvertQueryDictToText( query_dict, single_value_parameters ) - - fragment = '' - - r = urllib.parse.ParseResult( scheme, netloc, path, params, query, fragment ) - - normalised_url = r.geturl() + # this is less about washing as it is about stripping the fragment + normalised_url = ClientNetworkingFunctions.WashURL( url, keep_fragment = False ) else: diff --git a/hydrus/client/networking/ClientNetworkingFunctions.py b/hydrus/client/networking/ClientNetworkingFunctions.py index 418f21be..80a875de 100644 --- a/hydrus/client/networking/ClientNetworkingFunctions.py +++ b/hydrus/client/networking/ClientNetworkingFunctions.py @@ -1,5 +1,7 @@ import http.cookiejar import re +import typing + import unicodedata import urllib.parse @@ -102,6 +104,27 @@ def ConvertHTTPToHTTPS( url ): +def ConvertPathTextToList( path: str ) -> typing.List[ str ]: + + # yo sometimes you see a URL with double slashes in a weird place. maybe we should just split( '/' ) and then remove empty '' results? + + # /post/show/1326143/akunim-anthro-armband-armwear-clothed-clothing-fem + + # for a while we've had URLs like this: + # https://img2.gelbooru.com//images/80/c8/80c8646b4a49395fb36c805f316c49a9.jpg + # I was going to be careful as I unified all this to preserve the double-slash to help legacy known url storage matching, but it seems we've been nuking the extra slash for ages in actual db storage, so w/e! + while path.startswith( '/' ): + + path = path[ 1 : ] + + + # post/show/1326143/akunim-anthro-armband-armwear-clothed-clothing-fem + + path_components = path.split( '/' ) + + return path_components + + def ConvertQueryDictToText( query_dict, single_value_parameters, param_order = None ): # we now do everything with requests, which does all the unicode -> %20 business naturally, phew @@ -152,6 +175,7 @@ def ConvertQueryDictToText( query_dict, single_value_parameters, param_order = N return query_text + def ConvertQueryTextToDict( query_text ): # in the old version of this func, we played silly games with character encoding. I made the foolish decision to try to handle/save URLs with %20 stuff decoded @@ -187,11 +211,6 @@ def ConvertQueryTextToDict( query_text ): continue - if '%' not in value: - - value = urllib.parse.quote( value, safe = '+' ) - - single_value_parameters.append( value ) param_order.append( None ) @@ -199,16 +218,6 @@ def ConvertQueryTextToDict( query_text ): ( key, value ) = result - if '%' not in key: - - key = urllib.parse.quote( key, safe = '+' ) - - - if '%' not in value: - - value = urllib.parse.quote( value, safe = '+' ) - - param_order.append( key ) query_dict[ key ] = value @@ -218,6 +227,32 @@ def ConvertQueryTextToDict( query_text ): return ( query_dict, single_value_parameters, param_order ) +def EnsureURLInfoIsEncoded( path_components: typing.List[ str ], query_dict: typing.Dict[ str, str ], single_value_parameters: typing.List[ str ] ): + + # ok so the user just posted a URL at us, and this query dict could either be from a real url, like "tags=skirt%20blonde_hair", or it could be a pretty URL they typed or whatever, "tags=skirt blonde_hair" + # so, let's do our best to figure out if the thing was pre-encoded or not, and wash it through a safe encoding process so it is encoded when we give it back + # what's the potential problem? '+' is a special character that may or may not be encoded, e.g. "tags=6%2Bgirls+skirt" WEW + + percent_encoding_re = re.compile( r'%[0-9A-Fa-f]{2}' ) + + all_gubbins = set( path_components ) + all_gubbins.update( query_dict.keys() ) + all_gubbins.update( query_dict.values() ) + all_gubbins.update( single_value_parameters ) + + there_are_percent_encoding_chars = True in ( percent_encoding_re.search( text ) is not None for text in all_gubbins ) + + # if there are percent-encoded characters anywhere, we have to assume the whole URL is already encoded correctly! + if not there_are_percent_encoding_chars: + + path_components = [ urllib.parse.quote( value, safe = '+' ) for value in path_components ] + query_dict = { urllib.parse.quote( key, safe = '+' ) : urllib.parse.quote( value, safe = '+' ) for ( key, value ) in query_dict.items() } + single_value_parameters = [ urllib.parse.quote( value, safe = '+' ) for value in single_value_parameters ] + + + return ( path_components, query_dict, single_value_parameters ) + + def ConvertURLIntoDomain( url ): parser_result = ParseURL( url ) @@ -372,14 +407,14 @@ def LooksLikeAFullURL( text: str ) -> bool: try: - result = urllib.parse.urlparse( text ) + p = ParseURL( text ) - if result.scheme == '': + if p.scheme == '': return False - if result.netloc == '': + if p.netloc == '': return False @@ -414,6 +449,7 @@ def NormaliseAndFilterAssociableURLs( urls ): return associable_urls + def ParseURL( url: str ) -> urllib.parse.ParseResult: url = url.strip() @@ -430,6 +466,47 @@ def ParseURL( url: str ) -> urllib.parse.ParseResult: +def WashURL( url: str, keep_fragment = True ) -> str: + + if not LooksLikeAFullURL( url ): + + return url + + + try: + + p = ParseURL( url ) + + scheme = p.scheme + netloc = p.netloc + params = p.params # just so you know, this is ancient web semicolon tech, can be ignored + fragment = p.fragment + + path_components = ConvertPathTextToList( p.path ) + ( query_dict, single_value_parameters, param_order ) = ConvertQueryTextToDict( p.query ) + + ( path_components, query_dict, single_value_parameters ) = EnsureURLInfoIsEncoded( path_components, query_dict, single_value_parameters ) + + path = '/' + '/'.join( path_components ) + query = ConvertQueryDictToText( query_dict, single_value_parameters ) + + if not keep_fragment: + + fragment = '' + + + r = urllib.parse.ParseResult( scheme, netloc, path, params, query, fragment ) + + clean_url = r.geturl() + + return clean_url + + except: + + return url + + + OH_NO_NO_NETLOC_CHARACTERS = '?#' OH_NO_NO_NETLOC_CHARACTERS_UNICODE_TRANSLATE = { ord( char ) : '_' for char in OH_NO_NO_NETLOC_CHARACTERS } @@ -442,6 +519,7 @@ def RemoveWWWFromDomain( domain ): return domain + def UnicodeNormaliseURL( url: str ): if url.startswith( 'file:' ): diff --git a/hydrus/client/networking/ClientNetworkingGUG.py b/hydrus/client/networking/ClientNetworkingGUG.py index 8d7febf6..ebee1ebc 100644 --- a/hydrus/client/networking/ClientNetworkingGUG.py +++ b/hydrus/client/networking/ClientNetworkingGUG.py @@ -1,3 +1,4 @@ +import re import urllib.parse from hydrus.core import HydrusConstants as HC @@ -108,12 +109,11 @@ class GalleryURLGenerator( HydrusSerialisable.SerialisableBaseNamed ): raise HydrusExceptions.GUGException( 'Replacement phrase not in URL template!' ) - if '%' in query_text: + if re.search( r'%[0-9A-Fa-f]{2}', query_text ) is not None: - # redundant test but leave it in for now - if ' ' in query_text or '% ' in query_text or query_text.endswith( '%' ): + if ' ' in query_text: - # there is probably a legit % character here that should be encoded + # against probability, there is probably a legit % character here that should be encoded search_terms = query_text.split( ' ' ) diff --git a/hydrus/client/networking/ClientNetworkingURLClass.py b/hydrus/client/networking/ClientNetworkingURLClass.py index c6bbacf4..e26a467f 100644 --- a/hydrus/client/networking/ClientNetworkingURLClass.py +++ b/hydrus/client/networking/ClientNetworkingURLClass.py @@ -42,7 +42,9 @@ def ConvertURLClassesIntoAPIPairs( url_classes ): continue - api_url = url_class.GetAPIURL( url_class.GetExampleURL() ) + example_url = url_class.GetExampleURL() + + api_url = url_class.GetAPIURL( example_url ) for other_url_class in url_classes: @@ -387,19 +389,10 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ): return netloc - def _ClipAndFleshOutPath( self, path: str, for_server: bool ): + def _ClipAndFleshOutPath( self, path_components: typing.List[ str ], for_server: bool ): # /post/show/1326143/akunim-anthro-armband-armwear-clothed-clothing-fem - while path.startswith( '/' ): - - path = path[ 1 : ] - - - # post/show/1326143/akunim-anthro-armband-armwear-clothed-clothing-fem - - path_components = path.split( '/' ) - do_clip = self.UsesAPIURL() or not for_server flesh_out = len( path_components ) < len( self._path_components ) @@ -425,21 +418,17 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ): clipped_path_components.append( clipped_path_component ) - path = '/'.join( clipped_path_components ) + path_components = clipped_path_components - # post/show/1326143 - - path = '/' + path + path = '/' + '/'.join( path_components ) # /post/show/1326143 return path - def _ClipAndFleshOutQuery( self, query: str, for_server: bool ): - - ( query_dict, single_value_parameters, param_order ) = ClientNetworkingFunctions.ConvertQueryTextToDict( query ) + def _ClipAndFleshOutQuery( self, query_dict: typing.Dict[ str, str ], single_value_parameters: typing.List[ str ], param_order: typing.List[ str ], for_server: bool ): query_dict_keys_to_parameters = {} @@ -1034,16 +1023,11 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ): return self._api_lookup_converter - def GetAPIURL( self, url = None ): + def GetAPIURL( self, url ): - if url is None: - - url = self._example_url - + request_url = self.Normalise( url, for_server = True ) - url = self.Normalise( url, for_server = True ) - - return self._api_lookup_converter.Convert( url ) + return self._api_lookup_converter.Convert( request_url ) def GetClassKey( self ): @@ -1093,12 +1077,7 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ): page_index_path_component_index = self._gallery_index_identifier - while path.startswith( '/' ): - - path = path[ 1 : ] - - - path_components = path.split( '/' ) + path_components = ClientNetworkingFunctions.ConvertPathTextToList( path ) try: @@ -1195,9 +1174,11 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ): elif self._send_referral_url in ( SEND_REFERRAL_URL_CONVERTER_IF_NONE_PROVIDED, SEND_REFERRAL_URL_ONLY_CONVERTER ): + request_url = self.Normalise( url, for_server = True ) + try: - converted_referral_url = self._referral_url_converter.Convert( url ) + converted_referral_url = self._referral_url_converter.Convert( request_url ) except HydrusExceptions.StringConvertException: @@ -1331,9 +1312,12 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ): fragment = '' + path_components = ClientNetworkingFunctions.ConvertPathTextToList( p.path ) + ( query_dict, single_value_parameters, param_order ) = ClientNetworkingFunctions.ConvertQueryTextToDict( p.query ) + netloc = self._ClipNetLoc( p.netloc ) - path = self._ClipAndFleshOutPath( p.path, for_server ) - query = self._ClipAndFleshOutQuery( p.query, for_server ) + path = self._ClipAndFleshOutPath( path_components, for_server ) + query = self._ClipAndFleshOutQuery( query_dict, single_value_parameters, param_order, for_server ) r = urllib.parse.ParseResult( scheme, netloc, path, params, query, fragment ) @@ -1442,32 +1426,29 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ): - url_path = p.path + path = p.path + query = p.query - while url_path.startswith( '/' ): - - url_path = url_path[ 1 : ] - - - url_path_components = url_path.split( '/' ) + path_components = ClientNetworkingFunctions.ConvertPathTextToList( path ) + ( query_dict, single_value_parameters, param_order ) = ClientNetworkingFunctions.ConvertQueryTextToDict( query ) if self._no_more_path_components_than_this: - if len( url_path_components ) > len( self._path_components ): + if len( path_components ) > len( self._path_components ): - raise HydrusExceptions.URLClassException( '"{}" has {} path components, but I will not allow more than my defined {}!'.format( url_path, len( url_path_components ), len( self._path_components ) ) ) + raise HydrusExceptions.URLClassException( '"{}" has {} path components, but I will not allow more than my defined {}!'.format( path, len( path_components ), len( self._path_components ) ) ) for ( index, ( string_match, default ) ) in enumerate( self._path_components ): - if len( url_path_components ) > index: + if len( path_components ) > index: - url_path_component = url_path_components[ index ] + path_component = path_components[ index ] try: - string_match.Test( url_path_component ) + string_match.Test( path_component ) except HydrusExceptions.StringMatchException as e: @@ -1478,24 +1459,22 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ): if index + 1 == len( self._path_components ): - message = '"{}" has {} path components, but I was expecting {}!'.format( url_path, len( url_path_components ), len( self._path_components ) ) + message = '"{}" has {} path components, but I was expecting {}!'.format( path, len( path_components ), len( self._path_components ) ) else: - message = '"{}" has {} path components, but I was expecting at least {} and maybe as many as {}!'.format( url_path, len( url_path_components ), index + 1, len( self._path_components ) ) + message = '"{}" has {} path components, but I was expecting at least {} and maybe as many as {}!'.format( path, len( path_components ), index + 1, len( self._path_components ) ) raise HydrusExceptions.URLClassException( message ) - ( url_query_dict, single_value_parameters, param_order ) = ClientNetworkingFunctions.ConvertQueryTextToDict( p.query ) - if self._no_more_parameters_than_this: good_fixed_names = { parameter.GetName() for parameter in self._parameters if isinstance( parameter, URLClassParameterFixedName ) } - for ( name, value ) in url_query_dict.items(): + for ( name, value ) in query_dict.items(): if name not in good_fixed_names: @@ -1510,7 +1489,7 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ): name = parameter.GetName() - if name not in url_query_dict: + if name not in query_dict: if parameter.MustBeInOriginalURL(): @@ -1522,7 +1501,7 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ): - value = url_query_dict[ name ] + value = query_dict[ name ] try: @@ -1537,7 +1516,7 @@ class URLClass( HydrusSerialisable.SerialisableBaseNamed ): if len( single_value_parameters ) > 0 and not self._has_single_value_parameters and self._no_more_parameters_than_this: - raise HydrusExceptions.URLClassException( '"{}" has unexpected single-value parameters, but I am set to not allow any unexpected parameters!'.format( url_path ) ) + raise HydrusExceptions.URLClassException( '"{}" has unexpected single-value parameters, but I am set to not allow any unexpected parameters!'.format( query ) ) if self._has_single_value_parameters: diff --git a/hydrus/client/search/ClientSearch.py b/hydrus/client/search/ClientSearch.py index ec44efa4..1e50b124 100644 --- a/hydrus/client/search/ClientSearch.py +++ b/hydrus/client/search/ClientSearch.py @@ -393,11 +393,25 @@ class NumberTest( HydrusSerialisable.SerialisableBase ): if self.operator == NUMBER_TEST_OPERATOR_LESS_THAN: - return lambda x: x is not None and x < self.value + if self.value > 0: + + return lambda x: x is None or x < self.value + + else: + + return lambda x: x is not None and x < self.value + elif self.operator == NUMBER_TEST_OPERATOR_GREATER_THAN: - return lambda x: x is not None and x > self.value + if self.value < 0: + + return lambda x: x is None or x > self.value + + else: + + return lambda x: x is not None and x > self.value + elif self.operator == NUMBER_TEST_OPERATOR_EQUAL: @@ -415,18 +429,39 @@ class NumberTest( HydrusSerialisable.SerialisableBase ): lower = self.value * ( 1 - self.extra_value ) upper = self.value * ( 1 + self.extra_value ) - return lambda x: x is not None and lower < x < upper + if lower <= 0: + + return lambda x: x is None or x < upper + + else: + + return lambda x: x is not None and lower < x < upper + elif self.operator == NUMBER_TEST_OPERATOR_APPROXIMATE_ABSOLUTE: lower = self.value - self.extra_value upper = self.value + self.extra_value - return lambda x: x is not None and lower < x < upper + if lower <= 0: + + return lambda x: x is None or x < upper + + else: + + return lambda x: x is not None and lower < x < upper + elif self.operator == NUMBER_TEST_OPERATOR_NOT_EQUAL: - return lambda x: x is not None and x != self.value + if self.value == 0: + + return lambda x: x is not None and x != self.value + + else: + + return lambda x: x is None or x != self.value + @@ -434,17 +469,31 @@ class NumberTest( HydrusSerialisable.SerialisableBase ): if self.operator == NUMBER_TEST_OPERATOR_LESS_THAN: - return [ f'{variable_name} < {self.value}' ] + if self.value > 0: + + return [ f'( {variable_name} IS NULL OR {variable_name} < {self.value} )' ] + + else: + + return [ f'{variable_name} < {self.value}' ] + elif self.operator == NUMBER_TEST_OPERATOR_GREATER_THAN: - return [ f'{variable_name} > {self.value}' ] + if self.value < 0: + + return [ f'( {variable_name} IS NULL OR {variable_name} > {self.value} )' ] + + else: + + return [ f'{variable_name} > {self.value}' ] + elif self.operator == NUMBER_TEST_OPERATOR_EQUAL: if self.value == 0: - return [ f'({variable_name} IS NULL OR {variable_name} = {self.value})' ] + return [ f'( {variable_name} IS NULL OR {variable_name} = {self.value} )' ] else: @@ -456,18 +505,39 @@ class NumberTest( HydrusSerialisable.SerialisableBase ): lower = self.value * ( 1 - self.extra_value ) upper = self.value * ( 1 + self.extra_value ) - return [ f'{variable_name} > {lower}', f'{variable_name} < {upper}' ] + if lower <= 0: + + return [ f'( {variable_name} is NULL OR {variable_name} < {upper} )' ] + + else: + + return [ f'{variable_name} > {lower}', f'{variable_name} < {upper}' ] + elif self.operator == NUMBER_TEST_OPERATOR_APPROXIMATE_ABSOLUTE: lower = self.value - self.extra_value upper = self.value + self.extra_value - return [ f'{variable_name} > {lower}', f'{variable_name} < {upper}' ] + if lower <= 0: + + return [ f'( {variable_name} IS NULL OR {variable_name} < {upper} )' ] + + else: + + return [ f'{variable_name} > {lower}', f'{variable_name} < {upper}' ] + elif self.operator == NUMBER_TEST_OPERATOR_NOT_EQUAL: - return [ f'{variable_name} != {self.value}' ] + if self.value == 0: + + return [ f'{variable_name} IS NOT NULL AND {variable_name} != {self.value}' ] + + else: + + return [ f'( {variable_name} IS NULL OR {variable_name} != {self.value} )' ] + return [] diff --git a/hydrus/core/HydrusConstants.py b/hydrus/core/HydrusConstants.py index 9492e5d0..081e92f2 100644 --- a/hydrus/core/HydrusConstants.py +++ b/hydrus/core/HydrusConstants.py @@ -105,7 +105,7 @@ options = {} # Misc NETWORK_VERSION = 20 -SOFTWARE_VERSION = 568 +SOFTWARE_VERSION = 569 CLIENT_API_VERSION = 63 SERVER_THUMBNAIL_DIMENSIONS = ( 200, 200 ) diff --git a/hydrus/test/TestClientAPI.py b/hydrus/test/TestClientAPI.py index de7ad148..1c472af4 100644 --- a/hydrus/test/TestClientAPI.py +++ b/hydrus/test/TestClientAPI.py @@ -5223,7 +5223,7 @@ class TestClientAPI( unittest.TestCase ): media_results = [] file_info_managers = [] - urls = { "https://gelbooru.com/index.php?page=post&s=view&id=4841557", "https://img2.gelbooru.com//images/80/c8/80c8646b4a49395fb36c805f316c49a9.jpg" } + urls = { "https://gelbooru.com/index.php?page=post&s=view&id=4841557", "https://img2.gelbooru.com/images/80/c8/80c8646b4a49395fb36c805f316c49a9.jpg" } sorted_urls = sorted( urls ) @@ -5492,7 +5492,7 @@ class TestClientAPI( unittest.TestCase ): detailed_known_urls_metadata_row[ 'detailed_known_urls' ] = [ {'normalised_url' : 'https://gelbooru.com/index.php?id=4841557&page=post&s=view', 'url_type' : 0, 'url_type_string' : 'post url', 'match_name' : 'gelbooru file page', 'can_parse' : True}, - {'normalised_url' : 'https://img2.gelbooru.com//images/80/c8/80c8646b4a49395fb36c805f316c49a9.jpg', 'url_type' : 5, 'url_type_string' : 'unknown url', 'match_name' : 'unknown url', 'can_parse' : False, 'cannot_parse_reason' : 'unknown url class'} + {'normalised_url' : 'https://img2.gelbooru.com/images/80/c8/80c8646b4a49395fb36c805f316c49a9.jpg', 'url_type' : 5, 'url_type_string' : 'unknown url', 'match_name' : 'unknown url', 'can_parse' : False, 'cannot_parse_reason' : 'unknown url class'} ] detailed_known_urls_metadata.append( detailed_known_urls_metadata_row ) diff --git a/hydrus/test/TestClientDB.py b/hydrus/test/TestClientDB.py index 820b5232..520b75f2 100644 --- a/hydrus/test/TestClientDB.py +++ b/hydrus/test/TestClientDB.py @@ -388,7 +388,7 @@ class TestClientDB( unittest.TestCase ): tests.append( ( ClientSearch.PREDICATE_TYPE_SYSTEM_ARCHIVE, None, 0 ) ) - tests.append( ( ClientSearch.PREDICATE_TYPE_SYSTEM_DURATION, ClientSearch.NumberTest.STATICCreateFromCharacters( '<', 100, ), 0 ) ) + tests.append( ( ClientSearch.PREDICATE_TYPE_SYSTEM_DURATION, ClientSearch.NumberTest.STATICCreateFromCharacters( '<', 100, ), 1 ) ) tests.append( ( ClientSearch.PREDICATE_TYPE_SYSTEM_DURATION, ClientSearch.NumberTest.STATICCreateFromCharacters( '<', 0, ), 0 ) ) tests.append( ( ClientSearch.PREDICATE_TYPE_SYSTEM_DURATION, ClientSearch.NumberTest.STATICCreateFromCharacters( HC.UNICODE_APPROX_EQUAL, 100, ), 0 ) ) tests.append( ( ClientSearch.PREDICATE_TYPE_SYSTEM_DURATION, ClientSearch.NumberTest.STATICCreateFromCharacters( HC.UNICODE_APPROX_EQUAL, 0, ), 1 ) ) @@ -472,7 +472,7 @@ class TestClientDB( unittest.TestCase ): tests.append( ( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_TAGS, ( '', '>', 0 ), 0 ) ) tests.append( ( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_TAGS, ( '', '>', 1 ), 0 ) ) - tests.append( ( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_WORDS, ClientSearch.NumberTest.STATICCreateFromCharacters( '<', 1 ), 0 ) ) + tests.append( ( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_WORDS, ClientSearch.NumberTest.STATICCreateFromCharacters( '<', 1 ), 1 ) ) tests.append( ( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_WORDS, ClientSearch.NumberTest.STATICCreateFromCharacters( '<', 0 ), 0 ) ) tests.append( ( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_WORDS, ClientSearch.NumberTest.STATICCreateFromCharacters( HC.UNICODE_APPROX_EQUAL, 0 ), 1 ) ) tests.append( ( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_WORDS, ClientSearch.NumberTest.STATICCreateFromCharacters( HC.UNICODE_APPROX_EQUAL, 1 ), 0 ) ) diff --git a/hydrus/test/TestClientNetworking.py b/hydrus/test/TestClientNetworking.py index a99e6c7e..afcc687f 100644 --- a/hydrus/test/TestClientNetworking.py +++ b/hydrus/test/TestClientNetworking.py @@ -18,6 +18,7 @@ from hydrus.client.networking import ClientNetworking from hydrus.client.networking import ClientNetworkingBandwidth from hydrus.client.networking import ClientNetworkingContexts from hydrus.client.networking import ClientNetworkingDomain +from hydrus.client.networking import ClientNetworkingFunctions from hydrus.client.networking import ClientNetworkingJobs from hydrus.client.networking import ClientNetworkingLogin from hydrus.client.networking import ClientNetworkingSessions @@ -281,51 +282,20 @@ class TestURLClasses( unittest.TestCase ): def test_encoding( self ): - name = 'test' - url_type = HC.URL_TYPE_POST - preferred_scheme = 'https' - netloc = 'testbooru.cx' + human_url = 'https://testbooru.cx/post/page.php?id=1234 56&s=view' + encoded_url = 'https://testbooru.cx/post/page.php?id=1234%2056&s=view' - alphabetise_get_parameters = True - match_subdomains = False - keep_matched_subdomains = False - can_produce_multiple_files = False - should_be_associated_with_files = True - keep_fragment = False + self.assertEqual( ClientNetworkingFunctions.WashURL( human_url ), encoded_url ) + self.assertEqual( ClientNetworkingFunctions.WashURL( encoded_url ), encoded_url ) - path_components = [] + human_url_with_fragment = 'https://testbooru.cx/post/page.php?id=1234 56&s=view#hello' + encoded_url_with_fragment = 'https://testbooru.cx/post/page.php?id=1234%2056&s=view#hello' - path_components.append( ( ClientStrings.StringMatch( match_type = ClientStrings.STRING_MATCH_FIXED, match_value = 'post', example_string = 'post' ), None ) ) - path_components.append( ( ClientStrings.StringMatch( match_type = ClientStrings.STRING_MATCH_FIXED, match_value = 'page.php', example_string = 'page.php' ), None ) ) + self.assertEqual( ClientNetworkingFunctions.WashURL( human_url_with_fragment ), encoded_url_with_fragment ) + self.assertEqual( ClientNetworkingFunctions.WashURL( encoded_url_with_fragment ), encoded_url_with_fragment ) - parameters = [] - - parameters.append( ClientNetworkingURLClass.URLClassParameterFixedName( name = 's', value_string_match = ClientStrings.StringMatch( match_type = ClientStrings.STRING_MATCH_FIXED, match_value = 'view', example_string = 'view' ) ) ) - parameters.append( ClientNetworkingURLClass.URLClassParameterFixedName( name = 'id', value_string_match = ClientStrings.StringMatch( match_type = ClientStrings.STRING_MATCH_FLEXIBLE, match_value = ClientStrings.NUMERIC, example_string = '123456' ) ) ) - - send_referral_url = ClientNetworkingURLClass.SEND_REFERRAL_URL_ONLY_IF_PROVIDED - referral_url_converter = None - gallery_index_type = None - gallery_index_identifier = None - gallery_index_delta = 1 - example_url = 'https://testbooru.cx/post/page.php?id=123456&s=view' - - # encoding test - - parameters = [] - - parameters.append( ClientNetworkingURLClass.URLClassParameterFixedName( name = 's', value_string_match = ClientStrings.StringMatch( match_type = ClientStrings.STRING_MATCH_FIXED, match_value = 'view', example_string = 'view' ) ) ) - parameters.append( ClientNetworkingURLClass.URLClassParameterFixedName( name = 'id', value_string_match = ClientStrings.StringMatch( match_type = ClientStrings.STRING_MATCH_ANY, example_string = 'hello' ) ) ) - - url_class = ClientNetworkingURLClass.URLClass( name, url_type = url_type, preferred_scheme = preferred_scheme, netloc = netloc, path_components = path_components, parameters = parameters, send_referral_url = send_referral_url, referral_url_converter = referral_url_converter, gallery_index_type = gallery_index_type, gallery_index_identifier = gallery_index_identifier, gallery_index_delta = gallery_index_delta, example_url = example_url ) - - url_class.SetURLBooleans( match_subdomains, keep_matched_subdomains, alphabetise_get_parameters, can_produce_multiple_files, should_be_associated_with_files, keep_fragment ) - - unnormalised_human_url = 'https://testbooru.cx/post/page.php?id=1234 56&s=view' - normalised_encoded_url = 'https://testbooru.cx/post/page.php?id=1234%2056&s=view' - - self.assertEqual( url_class.Normalise( unnormalised_human_url ), normalised_encoded_url ) - self.assertEqual( url_class.Normalise( normalised_encoded_url ), normalised_encoded_url ) + self.assertEqual( ClientNetworkingFunctions.WashURL( human_url_with_fragment, keep_fragment = False ), encoded_url ) + self.assertEqual( ClientNetworkingFunctions.WashURL( encoded_url_with_fragment, keep_fragment = False ), encoded_url ) def test_defaults( self ): diff --git a/hydrus_client.sh b/hydrus_client.sh index 0af06bb7..9c31813f 100755 --- a/hydrus_client.sh +++ b/hydrus_client.sh @@ -15,10 +15,17 @@ if ! source venv/bin/activate; then fi # You can copy this file to 'client-user.sh' and add in your own launch parameters here if you like, and a git pull won't overwrite the file. -# Just tack new params on like this: -# python hydrus_client.py -d="/path/to/hydrus/db" +# Just tack new hardcoded params on like this: +# +# python hydrus_client.py -d="/path/to/hydrus/db" "$@" +# +# The "$@" part also passes on any launch parameters this script was called with, so you can also just go-- +# +# ./hydrus_client.sh -d="/path/to/hydrus/db" +# +# --depending on your needs! -python hydrus_client.py +python hydrus_client.py "$@" deactivate diff --git a/setup_venv.bat b/setup_venv.bat index d7eca37f..170e8c9a 100644 --- a/setup_venv.bat +++ b/setup_venv.bat @@ -2,6 +2,26 @@ pushd "%~dp0" +IF "%1"=="" ( + + set python_bin=python + +) else ( + + set python_bin=%1 + +) + +IF "%2"=="" ( + + set venv_location=venv + +) else ( + + set venv_location=%2 + +) + ECHO r::::::::::::::::::::::::::::::::::r ECHO : : ECHO : :PP. : @@ -27,24 +47,27 @@ ECHO: ECHO hydrus ECHO: -where /q python -IF ERRORLEVEL 1 ( +IF "%python_bin%"=="python" ( - SET /P gumpf="You do not seem to have python installed. Please check the 'running from source' help." + where /q %python_bin% + IF ERRORLEVEL 1 ( - popd + SET /P gumpf="You do not seem to have python installed. Please check the 'running from source' help." - EXIT /B 1 + popd + EXIT /B 1 + + ) ) -IF EXIST "venv\" ( +IF EXIST "%venv_location%\" ( SET /P ready="Virtual environment will be reinstalled. Hit Enter to start." echo Deleting old venv... - rmdir /s /q venv + rmdir /s /q %venv_location% ) ELSE ( @@ -52,7 +75,7 @@ IF EXIST "venv\" ( ) -IF EXIST "venv\" ( +IF EXIST "%venv_location%\" ( SET /P gumpf="It looks like the venv directory did not delete correctly. Do you have it activated in a terminal or IDE anywhere? Please close that and try this again!" @@ -68,7 +91,7 @@ ECHO -------- ECHO Users on older Windows or Python ^>=3.11 need the advanced install. ECHO: ECHO Your Python version is: -python --version +%python_bin% --version ECHO: SET /P install_type="Do you want the (s)imple or (a)dvanced install? " @@ -161,9 +184,9 @@ goto :parse_fail ECHO -------- echo Creating new venv... -python -m venv venv +%python_bin% -m venv %venv_location% -CALL venv\Scripts\activate.bat +CALL %venv_location%\Scripts\activate.bat IF ERRORLEVEL 1 ( @@ -252,7 +275,7 @@ IF "%install_type%" == "a" ( ) -CALL venv\Scripts\deactivate.bat +CALL %venv_location%\Scripts\deactivate.bat ECHO -------- SET /P done="Done!"