diff --git a/.github/workflows/macos_build.yml b/.github/workflows/macos_build.yml index d1d308a8..aaf28e42 100644 --- a/.github/workflows/macos_build.yml +++ b/.github/workflows/macos_build.yml @@ -22,6 +22,7 @@ jobs: - name: Build Hydrus run: | cd $GITHUB_WORKSPACE + cp static/build_files/pyoxidizer.bzl pyoxidizer.bzl basename $(rustc --print sysroot) | sed -e "s/^stable-//" > triple.txt pyoxidizer build --release cd build/$(head -n 1 triple.txt)/release @@ -54,6 +55,9 @@ jobs: True EOF + cp install/static/build_files/ReadMeFirst.rtf ./ReadMeFirst.rtf + cp install/static/build_files/Applications ./Applications + cp install/static/build_files/running_from_app "install/running_from_app" mv install/* "Hydrus Network.app/Contents/MacOS/" rm -rf install cd $GITHUB_WORKSPACE @@ -84,12 +88,12 @@ jobs: echo "::set-output name=version_short::${GITHUB_REF##*/v}" - name: Rename Files run: | - mv MacOS-DMG/HydrusNetwork.dmg HydrusNetwork-${{ steps.meta.outputs.version }}.dmg + mv MacOS-DMG/HydrusNetwork.dmg Hydrus.Network.${{ steps.meta.outputs.version_short }}.-.macOS.-.App.dmg - name: Release new uses: softprops/action-gh-release@v1 if: startsWith(github.ref, 'refs/tags/') with: files: | - HydrusNetwork-${{ steps.meta.outputs.version }}.dmg + Hydrus.Network.${{ steps.meta.outputs.version_short }}.-.macOS.-.App.dmg env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/README.md b/README.md index 67af9af9..abfb8a47 100755 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ The client can do quite a lot! Please check out the help inside the release or [ * [tumblr](http://hydrus.tumblr.com/) * [discord](https://discord.gg/wPHPCUZ) * [patreon](https://www.patreon.com/hydrus_dev) +* [user-run repository and wiki] (https://github.com/CuddleBear92/Hydrus-Presets-and-Scripts) ## Attribution diff --git a/help/changelog.html b/help/changelog.html index ce5ded7c..706fb40d 100755 --- a/help/changelog.html +++ b/help/changelog.html @@ -8,6 +8,40 @@

changelog

diff --git a/help/ipfs.html b/help/ipfs.html index e254adb5..74ff761a 100644 --- a/help/ipfs.html +++ b/help/ipfs.html @@ -17,7 +17,8 @@
  • unpin -- To tell our IPFS daemon to stop hosting a file or group of files.

    • getting ipfs

    -

    Get the prebuilt executable here. Inside should be a very simple 'ipfs' executable that does everything. Extract it somewhere and open up a terminal in the same folder, and then type:

    +

    Note there is now a nicer desktop package here. I haven't used it, but it may be a nicer intro to the program.

    +

    Get the prebuilt executable here. Inside should be a very simple 'ipfs' executable that does everything. Extract it somewhere and open up a terminal in the same folder, and then type:

    +
  • + --db_transaction_commit_period DB_TRANSACTION_COMMIT_PERIOD +

    Change the regular duration at which any database changes are committed to disk. By default this is 30 (seconds) for the client, and 120 for the server. Minimum value is 10. Typically, if hydrus crashes, it may 'forget' what happened up to this duration on the next boot. Increasing the duration will result in fewer overall 'commit' writes during very heavy work that makes several changes to the same database pages (read up on WAL mode for more details here), but it will increase commit time and memory/storage needs. Note that changes can only be committed after a job is complete, so if a single job takes longer than this period, changes will not be saved until it is done.

    +
  • --db_cache_size DB_CACHE_SIZE

    Change the size of the cache SQLite will use for each db file, in MB. By default this is 200, for 200MB, which for the four main client db files could mean 800MB peak use if you run a very heavy client and perform a long period of PTR sync. This does not matter so much (nor should it be fully used) if you have a smaller client.

    diff --git a/hydrus/client/ClientOptions.py b/hydrus/client/ClientOptions.py index 1cf189ce..51ce9cd4 100644 --- a/hydrus/client/ClientOptions.py +++ b/hydrus/client/ClientOptions.py @@ -149,6 +149,7 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ): self._dictionary[ 'booleans' ][ 'maintain_similar_files_duplicate_pairs_during_idle' ] = False self._dictionary[ 'booleans' ][ 'show_namespaces' ] = True + self._dictionary[ 'booleans' ][ 'replace_tag_underscores_with_spaces' ] = False self._dictionary[ 'booleans' ][ 'verify_regular_https' ] = True diff --git a/hydrus/client/ClientServices.py b/hydrus/client/ClientServices.py index 908e1a5c..b6af302f 100644 --- a/hydrus/client/ClientServices.py +++ b/hydrus/client/ClientServices.py @@ -2161,13 +2161,20 @@ class ServiceRepository( ServiceRestricted ): - def GetUpdatePeriod( self ): + def GetUpdatePeriod( self ) -> int: with self._lock: if 'update_period' in self._service_options: - return self._service_options[ 'update_period' ] + update_period = self._service_options[ 'update_period' ] + + if not isinstance( update_period, int ): + + raise HydrusExceptions.DataMissing( 'This service has a bad update period! Try refreshing your account!' ) + + + return update_period else: diff --git a/hydrus/client/db/ClientDB.py b/hydrus/client/db/ClientDB.py index fdc567c2..ffacafca 100644 --- a/hydrus/client/db/ClientDB.py +++ b/hydrus/client/db/ClientDB.py @@ -1109,7 +1109,7 @@ class DB( HydrusDB.HydrusDB ): if status_hook is not None: - message = 'clearing old data' + message = 'clearing old combined display data' status_hook( message ) @@ -1123,8 +1123,8 @@ class DB( HydrusDB.HydrusDB ): del all_pending_storage_tag_ids del storage_tag_ids_to_display_tag_ids - self._c.executemany( 'UPDATE {} SET pending_count = 0 WHERE tag_id = ?;'.format( ac_cache_table_name ), ( ( tag_id, ) for tag_id in all_pending_display_tag_ids ) ) - self._c.executemany( 'DELETE FROM {} WHERE tag_id = ? AND current_count = 0 AND pending_count = 0;'.format( ac_cache_table_name ), ( ( tag_id, ) for tag_id in all_pending_display_tag_ids ) ) + self._c.execute( 'UPDATE {} SET pending_count = 0 WHERE pending_count > 0;'.format( ac_cache_table_name ) ) + self._c.execute( 'DELETE FROM {} WHERE current_count = 0 AND pending_count = 0;'.format( ac_cache_table_name ) ) all_pending_display_tag_ids_to_implied_by_storage_tag_ids = self._CacheTagDisplayGetTagsToImpliedBy( ClientTags.TAG_DISPLAY_ACTUAL, tag_service_id, all_pending_display_tag_ids, tags_are_ideal = True ) @@ -1262,6 +1262,47 @@ class DB( HydrusDB.HydrusDB ): self._CacheCombinedFilesDisplayMappingsGenerate( tag_service_id ) + def _CacheCombinedFilesMappingsRegeneratePending( self, tag_service_id, status_hook = None ): + + ac_cache_table_name = self._CacheMappingsGetACCacheTableName( ClientTags.TAG_DISPLAY_STORAGE, self.modules_services.combined_file_service_id, tag_service_id ) + + ( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = ClientDBMappingsStorage.GenerateMappingsTableNames( tag_service_id ) + + if status_hook is not None: + + message = 'clearing old combined display data' + + status_hook( message ) + + + all_pending_storage_tag_ids = self._STS( self._c.execute( 'SELECT DISTINCT tag_id FROM {};'.format( pending_mappings_table_name ) ) ) + + self._c.execute( 'UPDATE {} SET pending_count = 0 WHERE pending_count > 0;'.format( ac_cache_table_name ) ) + self._c.execute( 'DELETE FROM {} WHERE current_count = 0 AND pending_count = 0;'.format( ac_cache_table_name ) ) + + ac_cache_changes = [] + + num_to_do = len( all_pending_storage_tag_ids ) + + for ( i, storage_tag_id ) in enumerate( all_pending_storage_tag_ids ): + + if i % 100 == 0 and status_hook is not None: + + message = 'regenerating pending tags {}'.format( HydrusData.ConvertValueRangeToPrettyString( i + 1, num_to_do ) ) + + status_hook( message ) + + + ( pending_delta, ) = self._c.execute( 'SELECT COUNT( DISTINCT hash_id ) FROM {} WHERE tag_id = ?;'.format( pending_mappings_table_name ), ( storage_tag_id, ) ).fetchone() + + ac_cache_changes.append( ( storage_tag_id, 0, pending_delta ) ) + + + self._CacheMappingsAddACCounts( ClientTags.TAG_DISPLAY_STORAGE, self.modules_services.combined_file_service_id, tag_service_id, ac_cache_changes ) + + self._CacheCombinedFilesDisplayMappingsRegeneratePending( tag_service_id, status_hook = status_hook ) + + def _CacheLocalHashIdsGenerate( self ): self.modules_hashes_local_cache.ClearCache() @@ -2047,7 +2088,7 @@ class DB( HydrusDB.HydrusDB ): if status_hook is not None: - message = 'clearing old data' + message = 'clearing old specific display data' status_hook( message ) @@ -2061,8 +2102,8 @@ class DB( HydrusDB.HydrusDB ): del all_pending_storage_tag_ids del storage_tag_ids_to_display_tag_ids - self._c.executemany( 'UPDATE {} SET pending_count = 0 WHERE tag_id = ?;'.format( ac_cache_table_name ), ( ( tag_id, ) for tag_id in all_pending_display_tag_ids ) ) - self._c.executemany( 'DELETE FROM {} WHERE tag_id = ? AND current_count = 0 AND pending_count = 0;'.format( ac_cache_table_name ), ( ( tag_id, ) for tag_id in all_pending_display_tag_ids ) ) + self._c.execute( 'UPDATE {} SET pending_count = 0 WHERE pending_count > 0;'.format( ac_cache_table_name ) ) + self._c.execute( 'DELETE FROM {} WHERE current_count = 0 AND pending_count = 0;'.format( ac_cache_table_name ) ) self._c.execute( 'DELETE FROM {};'.format( cache_display_pending_mappings_table_name ) ) @@ -2521,6 +2562,53 @@ class DB( HydrusDB.HydrusDB ): + def _CacheSpecificMappingsRegeneratePending( self, file_service_id, tag_service_id, status_hook = None ): + + ac_cache_table_name = self._CacheMappingsGetACCacheTableName( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id ) + + ( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = ClientDBMappingsStorage.GenerateMappingsTableNames( tag_service_id ) + ( cache_current_mappings_table_name, cache_deleted_mappings_table_name, cache_pending_mappings_table_name ) = GenerateSpecificMappingsCacheTableNames( file_service_id, tag_service_id ) + cache_files_table_name = GenerateSpecificFilesTableName( file_service_id, tag_service_id ) + + if status_hook is not None: + + message = 'clearing old specific data' + + status_hook( message ) + + + all_pending_storage_tag_ids = self._STS( self._c.execute( 'SELECT DISTINCT tag_id FROM {};'.format( pending_mappings_table_name ) ) ) + + self._c.execute( 'UPDATE {} SET pending_count = 0 WHERE pending_count > 0;'.format( ac_cache_table_name ) ) + self._c.execute( 'DELETE FROM {} WHERE current_count = 0 AND pending_count = 0;'.format( ac_cache_table_name ) ) + + self._c.execute( 'DELETE FROM {};'.format( cache_pending_mappings_table_name ) ) + + ac_cache_changes = [] + + num_to_do = len( all_pending_storage_tag_ids ) + + for ( i, storage_tag_id ) in enumerate( all_pending_storage_tag_ids ): + + if i % 100 == 0 and status_hook is not None: + + message = 'regenerating pending tags {}'.format( HydrusData.ConvertValueRangeToPrettyString( i + 1, num_to_do ) ) + + status_hook( message ) + + + self._c.execute( 'INSERT OR IGNORE INTO {} ( tag_id, hash_id ) SELECT tag_id, hash_id FROM {} CROSS JOIN {} USING ( hash_id ) WHERE tag_id = ?;'.format( cache_pending_mappings_table_name, pending_mappings_table_name, cache_files_table_name ), ( storage_tag_id, ) ) + + pending_delta = HydrusDB.GetRowCount( self._c ) + + ac_cache_changes.append( ( storage_tag_id, 0, pending_delta ) ) + + + self._CacheMappingsAddACCounts( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id, ac_cache_changes ) + + self._CacheSpecificDisplayMappingsRegeneratePending( file_service_id, tag_service_id, status_hook = status_hook ) + + def _CacheSpecificMappingsRescindPendingMappings( self, tag_service_id, tag_id, hash_ids, filtered_hashes_generator: FilteredHashesGenerator ): for ( file_service_id, filtered_hash_ids ) in filtered_hashes_generator.IterateHashes( hash_ids ): @@ -11575,6 +11663,16 @@ class DB( HydrusDB.HydrusDB ): hash_ids_to_hashes_and_mimes = { hash_id : ( hash, mime ) for ( hash_id, hash, mime ) in self._c.execute( 'SELECT hash_id, hash, mime FROM {} CROSS JOIN hashes USING ( hash_id ) CROSS JOIN files_info USING ( hash_id );'.format( temp_hash_ids_table_name ) ) } + if len( hash_ids_to_hashes_and_mimes ) < len( hash_ids_i_can_process ): + + self._ScheduleRepositoryUpdateFileMaintenance( service_id, ClientFiles.REGENERATE_FILE_DATA_JOB_FILE_INTEGRITY_DATA ) + self._ScheduleRepositoryUpdateFileMaintenance( service_id, ClientFiles.REGENERATE_FILE_DATA_JOB_FILE_METADATA ) + + self._cursor_transaction_wrapper.CommitAndBegin() + + raise Exception( 'An error was discovered during repository processing--some update files are missing file info or hashes. A maintenance routine will try to scan these files and fix this problem, but it may be more complicated to fix. Please contact hydev and let him know the details!' ) + + for hash_id in hash_ids_i_can_process: ( hash, mime ) = hash_ids_to_hashes_and_mimes[ hash_id ] @@ -15193,6 +15291,87 @@ class DB( HydrusDB.HydrusDB ): self.pub_after_job( 'notify_new_tag_display_application' ) + def _RegenerateTagPendingMappingsCache( self, tag_service_key = None ): + + job_key = ClientThreading.JobKey( cancellable = True ) + + try: + + job_key.SetVariable( 'popup_title', 'regenerating tag pending mappings cache' ) + + self._controller.pub( 'modal_message', job_key ) + + if tag_service_key is None: + + tag_service_ids = self.modules_services.GetServiceIds( HC.REAL_TAG_SERVICES ) + + else: + + tag_service_ids = ( self.modules_services.GetServiceId( tag_service_key ), ) + + + file_service_ids = self.modules_services.GetServiceIds( HC.AUTOCOMPLETE_CACHE_SPECIFIC_FILE_SERVICES ) + + for ( file_service_id, tag_service_id ) in itertools.product( file_service_ids, tag_service_ids ): + + if job_key.IsCancelled(): + + break + + + message = 'regenerating specific cache pending {}_{}'.format( file_service_id, tag_service_id ) + + def status_hook_1( s: str ): + + job_key.SetVariable( 'popup_text_2', s ) + self._controller.frame_splash_status.SetSubtext( '{} - {}'.format( message, s ) ) + + + job_key.SetVariable( 'popup_text_1', message ) + self._controller.frame_splash_status.SetSubtext( message ) + + self._CacheSpecificMappingsRegeneratePending( file_service_id, tag_service_id, status_hook = status_hook_1 ) + + + job_key.SetVariable( 'popup_text_2', '' ) + self._controller.frame_splash_status.SetSubtext( '' ) + + for tag_service_id in tag_service_ids: + + if job_key.IsCancelled(): + + break + + + message = 'regenerating combined cache pending {}'.format( tag_service_id ) + + def status_hook_2( s: str ): + + job_key.SetVariable( 'popup_text_2', s ) + self._controller.frame_splash_status.SetSubtext( '{} - {}'.format( message, s ) ) + + + job_key.SetVariable( 'popup_text_1', message ) + self._controller.frame_splash_status.SetSubtext( message ) + + self._CacheCombinedFilesMappingsRegeneratePending( tag_service_id, status_hook = status_hook_2 ) + + + job_key.SetVariable( 'popup_text_2', '' ) + self._controller.frame_splash_status.SetSubtext( '' ) + + finally: + + job_key.SetVariable( 'popup_text_1', 'done!' ) + + job_key.Finish() + + job_key.Delete( 5 ) + + self.pub_after_job( 'notify_new_force_refresh_tags_data' ) + + + def _RegenerateTagSiblingsCache( self, only_these_service_ids = None ): if only_these_service_ids is None: @@ -18909,6 +19088,34 @@ class DB( HydrusDB.HydrusDB ): + if version == 435: + + try: + + self._RegenerateTagPendingMappingsCache() + + types_to_delete = ( + HC.SERVICE_INFO_NUM_PENDING_MAPPINGS, + HC.SERVICE_INFO_NUM_PENDING_TAG_SIBLINGS, + HC.SERVICE_INFO_NUM_PENDING_TAG_PARENTS, + HC.SERVICE_INFO_NUM_PETITIONED_MAPPINGS, + HC.SERVICE_INFO_NUM_PETITIONED_TAG_SIBLINGS, + HC.SERVICE_INFO_NUM_PETITIONED_TAG_PARENTS, + HC.SERVICE_INFO_NUM_PENDING_FILES, + HC.SERVICE_INFO_NUM_PETITIONED_FILES + ) + + self._DeleteServiceInfo( types_to_delete = types_to_delete ) + + except Exception as e: + + HydrusData.PrintException( e ) + + message = 'Trying to regenerate the pending tag cache failed! This is not a big deal, but you might still have a bad pending count for your pending menu. Error information has been written to the log. Please let hydrus dev know!' + + self.pub_initial_message( message ) + + self._controller.frame_splash_status.SetTitleText( 'updated db to v{}'.format( HydrusData.ToHumanInt( version + 1 ) ) ) self._c.execute( 'UPDATE version SET version = ?;', ( version + 1, ) ) @@ -19467,8 +19674,9 @@ class DB( HydrusDB.HydrusDB ): elif action == 'regenerate_tag_display_mappings_cache': self._RegenerateTagDisplayMappingsCache( *args, **kwargs ) elif action == 'regenerate_tag_display_pending_mappings_cache': self._RegenerateTagDisplayPendingMappingsCache( *args, **kwargs ) elif action == 'regenerate_tag_mappings_cache': self._RegenerateTagMappingsCache( *args, **kwargs ) - elif action == 'regenerate_tag_siblings_cache': self._RegenerateTagSiblingsCache( *args, **kwargs ) elif action == 'regenerate_tag_parents_cache': self._RegenerateTagParentsCache( *args, **kwargs ) + elif action == 'regenerate_tag_pending_mappings_cache': self._RegenerateTagPendingMappingsCache( *args, **kwargs ) + elif action == 'regenerate_tag_siblings_cache': self._RegenerateTagSiblingsCache( *args, **kwargs ) elif action == 'repopulate_mappings_from_cache': self._RepopulateMappingsFromCache( *args, **kwargs ) elif action == 'repopulate_tag_cache_missing_subtags': self._RepopulateTagCacheMissingSubtags( *args, **kwargs ) elif action == 'relocate_client_files': self._RelocateClientFiles( *args, **kwargs ) diff --git a/hydrus/client/gui/ClientGUI.py b/hydrus/client/gui/ClientGUI.py index 28a6df63..418c93e6 100644 --- a/hydrus/client/gui/ClientGUI.py +++ b/hydrus/client/gui/ClientGUI.py @@ -272,7 +272,7 @@ def THREADUploadPending( service_key ): if service_type == HC.TAG_REPOSITORY: - types_to_clear = ( + types_to_delete = ( HC.SERVICE_INFO_NUM_PENDING_MAPPINGS, HC.SERVICE_INFO_NUM_PENDING_TAG_SIBLINGS, HC.SERVICE_INFO_NUM_PENDING_TAG_PARENTS, @@ -283,13 +283,13 @@ def THREADUploadPending( service_key ): elif service_type in ( HC.FILE_REPOSITORY, HC.IPFS ): - types_to_clear = ( + types_to_delete = ( HC.SERVICE_INFO_NUM_PENDING_FILES, HC.SERVICE_INFO_NUM_PETITIONED_FILES ) - HG.client_controller.Write( 'delete_service_info', service_key, types_to_clear ) + HG.client_controller.Write( 'delete_service_info', service_key, types_to_delete ) HG.currently_uploading_pending = False HG.client_controller.pub( 'notify_new_pending' ) @@ -539,6 +539,7 @@ class FrameGUI( ClientGUITopLevelWindows.MainFrameThatResizes ): library_versions.append( ( 'temp dir', HydrusPaths.GetCurrentTempDir() ) ) library_versions.append( ( 'db journal mode', HG.db_journal_mode ) ) library_versions.append( ( 'db cache size per file', '{}MB'.format( HG.db_cache_size ) ) ) + library_versions.append( ( 'db transaction commit period', '{}'.format( HydrusData.TimeDeltaToPrettyTimeDelta( HG.db_cache_size ) ) ) ) library_versions.append( ( 'db synchronous value', str( HG.db_synchronous ) ) ) library_versions.append( ( 'db using memory for temp?', str( HG.no_db_temp_files ) ) ) @@ -1343,17 +1344,37 @@ class FrameGUI( ClientGUITopLevelWindows.MainFrameThatResizes ): - def _DeleteServiceInfo( self ): + def _DeleteServiceInfo( self, only_pending = False ): - message = 'This clears the cached counts for things like the number of files or tags on a service. Due to unusual situations and little counting bugs, these numbers can sometimes become unsynced. Clearing them forces an accurate recount from source.' - message += os.linesep * 2 - message += 'Some GUI elements (review services, mainly) may be slow the next time they launch.' + if only_pending: + + types_to_delete = ( + HC.SERVICE_INFO_NUM_PENDING_MAPPINGS, + HC.SERVICE_INFO_NUM_PENDING_TAG_SIBLINGS, + HC.SERVICE_INFO_NUM_PENDING_TAG_PARENTS, + HC.SERVICE_INFO_NUM_PETITIONED_MAPPINGS, + HC.SERVICE_INFO_NUM_PETITIONED_TAG_SIBLINGS, + HC.SERVICE_INFO_NUM_PETITIONED_TAG_PARENTS, + HC.SERVICE_INFO_NUM_PENDING_FILES, + HC.SERVICE_INFO_NUM_PETITIONED_FILES + ) + + message = 'This will clear and regen the number for the pending menu up top. Due to unusual situations and little counting bugs, these numbers can sometimes become unsynced. It should not take long at all, and will update instantly if changed.' + + else: + + types_to_delete = None + + message = 'This clears the cached counts for things like the number of files or tags on a service. Due to unusual situations and little counting bugs, these numbers can sometimes become unsynced. Clearing them forces an accurate recount from source.' + message += os.linesep * 2 + message += 'Some GUI elements (review services, mainly) may be slow the next time they launch.' + result = ClientGUIDialogsQuick.GetYesNo( self, message ) if result == QW.QDialog.Accepted: - self._controller.Write( 'delete_service_info' ) + self._controller.Write( 'delete_service_info', types_to_delete = types_to_delete ) @@ -3253,7 +3274,7 @@ class FrameGUI( ClientGUITopLevelWindows.MainFrameThatResizes ): message = 'This will delete and then recreate the pending tags on the tag \'display\' mappings cache, which is used for user-presented tag searching, loading, and autocomplete counts. This is useful if you have \'ghost\' pending tags or counts hanging around.' message += os.linesep * 2 - message += 'If you have a millions of pending tags, it can take a long time, during which the gui may hang.' + message += 'If you have a millions of tags, pending or current, it can take a long time, during which the gui may hang.' message += os.linesep * 2 message += 'If you do not have a specific reason to run this, it is pointless.' @@ -3301,6 +3322,31 @@ class FrameGUI( ClientGUITopLevelWindows.MainFrameThatResizes ): + def _RegenerateTagPendingMappingsCache( self ): + + message = 'This will delete and then recreate the pending tags on the whole tag mappings cache, which is used for multiple kinds of tag searching, loading, and autocomplete counts. This is useful if you have \'ghost\' pending tags or counts hanging around.' + message += os.linesep * 2 + message += 'If you have a millions of tags, pending or current, it can take a long time, during which the gui may hang.' + message += os.linesep * 2 + message += 'If you do not have a specific reason to run this, it is pointless.' + + result = ClientGUIDialogsQuick.GetYesNo( self, message, yes_label = 'do it--now choose which service', no_label = 'forget it' ) + + if result == QW.QDialog.Accepted: + + try: + + tag_service_key = GetTagServiceKeyForMaintenance( self ) + + except HydrusExceptions.CancelledException: + + return + + + self._controller.Write( 'regenerate_tag_pending_mappings_cache', tag_service_key = tag_service_key ) + + + def _RegenerateSimilarFilesTree( self ): message = 'This will delete and then recreate the similar files search tree. This is useful if it has somehow become unbalanced and similar files searches are running slow.' @@ -5201,9 +5247,11 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p submenu = QW.QMenu( menu ) - ClientGUIMenus.AppendMenuItem( submenu, 'tag storage mappings cache', 'Delete and recreate the tag mappings cache, fixing any miscounts.', self._RegenerateTagMappingsCache ) - ClientGUIMenus.AppendMenuItem( submenu, 'tag display mappings cache (deferred siblings & parents calculation)', 'Delete and recreate the tag display mappings cache, fixing any miscounts.', self._RegenerateTagDisplayMappingsCache ) - ClientGUIMenus.AppendMenuItem( submenu, 'tag display mappings cache (just pending tags, instant calculation)', 'Delete and recreate the tag display pending mappings cache, fixing any miscounts.', self._RegenerateTagDisplayPendingMappingsCache ) + ClientGUIMenus.AppendMenuItem( submenu, 'total pending count, in the pending menu', 'Regenerate the pending count up top.', self._DeleteServiceInfo, only_pending = True ) + ClientGUIMenus.AppendMenuItem( submenu, 'tag storage mappings cache (all, with deferred siblings & parents calculation)', 'Delete and recreate the tag mappings cache, fixing bad tags or miscounts.', self._RegenerateTagMappingsCache ) + ClientGUIMenus.AppendMenuItem( submenu, 'tag storage mappings cache (just pending tags, instant calculation)', 'Delete and recreate the tag pending mappings cache, fixing bad tags or miscounts.', self._RegenerateTagPendingMappingsCache ) + ClientGUIMenus.AppendMenuItem( submenu, 'tag display mappings cache (all, deferred siblings & parents calculation)', 'Delete and recreate the tag display mappings cache, fixing bad tags or miscounts.', self._RegenerateTagDisplayMappingsCache ) + ClientGUIMenus.AppendMenuItem( submenu, 'tag display mappings cache (just pending tags, instant calculation)', 'Delete and recreate the tag display pending mappings cache, fixing bad tags or miscounts.', self._RegenerateTagDisplayPendingMappingsCache ) ClientGUIMenus.AppendMenuItem( submenu, 'tag siblings lookup cache', 'Delete and recreate the tag siblings cache.', self._RegenerateTagSiblingsLookupCache ) ClientGUIMenus.AppendMenuItem( submenu, 'tag parents lookup cache', 'Delete and recreate the tag siblings cache.', self._RegenerateTagParentsLookupCache ) ClientGUIMenus.AppendMenuItem( submenu, 'tag text search cache', 'Delete and regenerate the cache hydrus uses for fast tag search.', self._RegenerateTagCache ) diff --git a/hydrus/client/gui/ClientGUIImport.py b/hydrus/client/gui/ClientGUIImport.py index 5ffe364f..ae2774c5 100644 --- a/hydrus/client/gui/ClientGUIImport.py +++ b/hydrus/client/gui/ClientGUIImport.py @@ -482,7 +482,7 @@ class FilenameTaggingOptionsPanel( QW.QWidget ): self._tags_panel = ClientGUICommon.StaticBox( self, 'tags for all' ) - self._tags = ClientGUIListBoxes.ListBoxTagsStringsAddRemove( self._tags_panel, self._service_key, ClientTags.TAG_DISPLAY_STORAGE ) + self._tags = ClientGUIListBoxes.ListBoxTagsStringsAddRemove( self._tags_panel, self._service_key, tag_display_type = ClientTags.TAG_DISPLAY_STORAGE ) self._tag_autocomplete_all = ClientGUIACDropdown.AutoCompleteDropdownTagsWrite( self._tags_panel, self.EnterTags, CC.LOCAL_FILE_SERVICE_KEY, service_key, show_paste_button = True ) @@ -494,7 +494,7 @@ class FilenameTaggingOptionsPanel( QW.QWidget ): self._paths_to_single_tags = collections.defaultdict( set ) - self._single_tags = ClientGUIListBoxes.ListBoxTagsStringsAddRemove( self._single_tags_panel, self._service_key, ClientTags.TAG_DISPLAY_STORAGE ) + self._single_tags = ClientGUIListBoxes.ListBoxTagsStringsAddRemove( self._single_tags_panel, self._service_key, tag_display_type = ClientTags.TAG_DISPLAY_STORAGE ) self._single_tags_paste_button = ClientGUICommon.BetterButton( self._single_tags_panel, 'paste tags', self._PasteSingleTags ) @@ -1980,7 +1980,23 @@ class GUGKeyAndNameSelector( ClientGUICommon.BetterButton ): if len( non_functional_gugs ) > 0: - non_functional_choice_tuples = [ ( gug.GetName(), gug ) for gug in non_functional_gugs ] + non_functional_choice_tuples = [] + + for gug in non_functional_gugs: + + s = gug.GetName() + + try: + + gug.CheckFunctional() + + except HydrusExceptions.ParseException as e: + + s = '{} ({})'.format( gug.GetName(), e ) + + + non_functional_choice_tuples.append( ( s, gug ) ) + choice_tuples.append( ( '--non-functional galleries', -2 ) ) diff --git a/hydrus/client/gui/ClientGUIScrolledPanelsManagement.py b/hydrus/client/gui/ClientGUIScrolledPanelsManagement.py index 9bce78d7..5eddd001 100644 --- a/hydrus/client/gui/ClientGUIScrolledPanelsManagement.py +++ b/hydrus/client/gui/ClientGUIScrolledPanelsManagement.py @@ -3060,6 +3060,8 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): self._show_namespaces = QW.QCheckBox( render_panel ) self._namespace_connector = QW.QLineEdit( render_panel ) + self._replace_tag_underscores_with_spaces = QW.QCheckBox( render_panel ) + # namespace_colours_panel = ClientGUICommon.StaticBox( self, 'namespace colours' ) @@ -3076,6 +3078,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): self._show_namespaces.setChecked( new_options.GetBoolean( 'show_namespaces' ) ) self._namespace_connector.setText( new_options.GetString( 'namespace_connector' ) ) + self._replace_tag_underscores_with_spaces.setChecked( new_options.GetBoolean( 'replace_tag_underscores_with_spaces' ) ) # @@ -3105,6 +3108,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): rows.append( ( 'Show namespaces: ', self._show_namespaces ) ) rows.append( ( 'If shown, namespace connecting string: ', self._namespace_connector ) ) + rows.append( ( 'EXPERIMENTAL: Replace all underscores with spaces: ', self._replace_tag_underscores_with_spaces ) ) gridbox = ClientGUICommon.WrapInGrid( render_panel, rows ) @@ -3159,6 +3163,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ): self._new_options.SetBoolean( 'show_namespaces', self._show_namespaces.isChecked() ) self._new_options.SetString( 'namespace_connector', self._namespace_connector.text() ) + self._new_options.SetBoolean( 'replace_tag_underscores_with_spaces', self._replace_tag_underscores_with_spaces.isChecked() ) HC.options[ 'namespace_colours' ] = self._namespace_colours.GetNamespaceColours() diff --git a/hydrus/client/gui/services/ClientGUIClientsideServices.py b/hydrus/client/gui/services/ClientGUIClientsideServices.py index 5cae0141..bccc0ead 100644 --- a/hydrus/client/gui/services/ClientGUIClientsideServices.py +++ b/hydrus/client/gui/services/ClientGUIClientsideServices.py @@ -2432,20 +2432,30 @@ class ReviewServiceRestrictedSubPanel( ClientGUICommon.StaticBox ): def _RefreshAccount( self ): - def do_it( service, my_updater ): + service = self._service + + def work_callable(): - try: + service.SyncAccount( force = True ) + + return 1 + + + def publish_callable( result ): + + self._my_updater.Update() + + + def errback_callable( etype, value, tb ): + + if not isinstance( etype, HydrusExceptions.ServerBusyException ): - service.SyncAccount( force = True ) - - except Exception as e: - - HydrusData.ShowException( e ) - - QP.CallAfter( QW.QMessageBox.critical, None, 'Error', str(e) ) + HydrusData.ShowExceptionTuple( etype, value, tb, do_wait = False ) - my_updater.Update() + QW.QMessageBox.critical( self, 'Error', str( value ) ) + + self._my_updater.Update() if HG.client_controller.options[ 'pause_repo_sync' ]: @@ -2455,7 +2465,7 @@ class ReviewServiceRestrictedSubPanel( ClientGUICommon.StaticBox ): return - if self._service.GetServiceType() in HC.REPOSITORIES and self._service.IsPausedNetworkSync(): + if self._service.IsPausedNetworkSync(): QW.QMessageBox.warning( self, 'Warning', 'Account sync is paused for this service! Please unpause it to refresh its account.' ) @@ -2465,7 +2475,9 @@ class ReviewServiceRestrictedSubPanel( ClientGUICommon.StaticBox ): self._refresh_account_button.setEnabled( False ) self._refresh_account_button.setText( 'fetching\u2026' ) - HG.client_controller.CallToThread( do_it, self._service, self._my_updater ) + job = ClientGUIAsync.AsyncQtJob( self, work_callable, publish_callable, errback_callable = errback_callable ) + + job.start() def ServiceUpdated( self, service ): @@ -2490,6 +2502,7 @@ class ReviewServiceRepositorySubPanel( ClientGUICommon.StaticBox ): self._content_panel = QW.QWidget( self ) + self._update_period_st = ClientGUICommon.BetterStaticText( self ) self._metadata_st = ClientGUICommon.BetterStaticText( self ) self._download_progress = ClientGUICommon.TextAndGauge( self ) @@ -2544,6 +2557,7 @@ class ReviewServiceRepositorySubPanel( ClientGUICommon.StaticBox ): QP.AddToLayout( hbox, self._reset_downloading_button, CC.FLAGS_CENTER_PERPENDICULAR ) QP.AddToLayout( hbox, self._reset_processing_button, CC.FLAGS_CENTER_PERPENDICULAR ) + self.Add( self._update_period_st, CC.FLAGS_EXPAND_PERPENDICULAR ) self.Add( self._metadata_st, CC.FLAGS_EXPAND_PERPENDICULAR ) self.Add( self._download_progress, CC.FLAGS_EXPAND_PERPENDICULAR ) self.Add( self._update_downloading_paused_button, CC.FLAGS_ON_RIGHT ) @@ -2722,6 +2736,17 @@ class ReviewServiceRepositorySubPanel( ClientGUICommon.StaticBox ): # + try: + + update_period = self._service.GetUpdatePeriod() + + self._update_period_st.setText( 'update period: {}'.format( HydrusData.TimeDeltaToPrettyTimeDelta( update_period ) ) ) + + except HydrusExceptions.DataMissing: + + self._update_period_st.setText( 'Unknown update period.' ) + + self._metadata_st.setText( self._service.GetNextUpdateDueString() ) HG.client_controller.CallToThread( self.THREADFetchInfo, self._service ) diff --git a/hydrus/client/importing/ClientImportSubscriptionLegacy.py b/hydrus/client/importing/ClientImportSubscriptionLegacy.py index 25613d00..71839cdb 100644 --- a/hydrus/client/importing/ClientImportSubscriptionLegacy.py +++ b/hydrus/client/importing/ClientImportSubscriptionLegacy.py @@ -1160,16 +1160,24 @@ class SubscriptionLegacy( HydrusSerialisable.SerialisableBaseNamed ): self._paused = True - HydrusData.ShowText( 'The subscription "' + self._name + '" could not find a Gallery URL Generator for "' + self._gug_key_and_name[1] + '"! The sub has paused!' ) + HydrusData.ShowText( 'The subscription "{}" could not find a Gallery URL Generator for "{}"! The sub has paused!'.format( self._name, self._gug_key_and_name[1] ) ) return - if not gug.IsFunctional(): + try: + + gug.CheckFunctional() + + except HydrusExceptions.ParseException as e: self._paused = True - HydrusData.ShowText( 'The subscription "' + self._name + '"\'s Gallery URL Generator, "' + self._gug_key_and_name[1] + '" seems not to be functional! Maybe it needs a gallery url class or a gallery parser? The sub has paused!' ) + message = 'The subscription "{}"\'s Gallery URL Generator, "{}" seems not to be functional! The sub has paused! The given reason was:'.format( self._name, self._gug_key_and_name[1] ) + message += os.linesep * 2 + message += str( e ) + + HydrusData.ShowText( message ) return diff --git a/hydrus/client/importing/ClientImportSubscriptions.py b/hydrus/client/importing/ClientImportSubscriptions.py index 7ebe3a35..b32d6c97 100644 --- a/hydrus/client/importing/ClientImportSubscriptions.py +++ b/hydrus/client/importing/ClientImportSubscriptions.py @@ -563,16 +563,24 @@ class Subscription( HydrusSerialisable.SerialisableBaseNamed ): self._paused = True - HydrusData.ShowText( 'The subscription "' + self._name + '" could not find a Gallery URL Generator for "' + self._gug_key_and_name[1] + '"! The sub has paused!' ) + HydrusData.ShowText( 'The subscription "{}" could not find a Gallery URL Generator for "{}"! The sub has paused!'.format( self._name, self._gug_key_and_name[1] ) ) return - if not gug.IsFunctional(): + try: + + gug.CheckFunctional() + + except HydrusExceptions.ParseException as e: self._paused = True - HydrusData.ShowText( 'The subscription "' + self._name + '"\'s Gallery URL Generator, "' + self._gug_key_and_name[1] + '" seems not to be functional! Maybe it needs a gallery url class or a gallery parser? The sub has paused!' ) + message = 'The subscription "{}"\'s Gallery URL Generator, "{}" seems not to be functional! The sub has paused! The given reason was:'.format( self._name, self._gug_key_and_name[1] ) + message += os.linesep * 2 + message += str( e ) + + HydrusData.ShowText( message ) return diff --git a/hydrus/client/metadata/ClientTags.py b/hydrus/client/metadata/ClientTags.py index a4d7f200..22c6f91a 100644 --- a/hydrus/client/metadata/ClientTags.py +++ b/hydrus/client/metadata/ClientTags.py @@ -28,6 +28,16 @@ def RenderNamespaceForUser( namespace ): def RenderTag( tag, render_for_user: bool ): + if render_for_user: + + new_options = HG.client_controller.new_options + + if new_options.GetBoolean( 'replace_tag_underscores_with_spaces' ): + + tag = tag.replace( '_', ' ' ) + + + ( namespace, subtag ) = HydrusTags.SplitTag( tag ) if namespace == '': @@ -38,8 +48,6 @@ def RenderTag( tag, render_for_user: bool ): if render_for_user: - new_options = HG.client_controller.new_options - if new_options.GetBoolean( 'show_namespaces' ): connector = new_options.GetString( 'namespace_connector' ) diff --git a/hydrus/client/networking/ClientLocalServerResources.py b/hydrus/client/networking/ClientLocalServerResources.py index 89b9a8f3..3f100f8b 100644 --- a/hydrus/client/networking/ClientLocalServerResources.py +++ b/hydrus/client/networking/ClientLocalServerResources.py @@ -1904,6 +1904,8 @@ class HydrusResourceClientAPIRestrictedManageCookiesSetCookies( HydrusResourceCl ClientNetworkingDomain.AddCookieToSession( session, name, value, domain, path, expires ) + HG.client_controller.network_engine.session_manager.SetSessionDirty( network_context ) + if HG.client_controller.new_options.GetBoolean( 'notify_client_api_cookies' ) and len( domains_cleared ) + len( domains_set ) > 0: @@ -1933,8 +1935,6 @@ class HydrusResourceClientAPIRestrictedManageCookiesSetCookies( HydrusResourceCl HG.client_controller.pub( 'message', job_key ) - HG.client_controller.network_engine.session_manager.SetSessionDirty( network_context ) - response_context = HydrusServerResources.ResponseContext( 200 ) return response_context diff --git a/hydrus/client/networking/ClientNetworkingDomain.py b/hydrus/client/networking/ClientNetworkingDomain.py index 8fc730ab..d642c6b6 100644 --- a/hydrus/client/networking/ClientNetworkingDomain.py +++ b/hydrus/client/networking/ClientNetworkingDomain.py @@ -2539,6 +2539,30 @@ class GalleryURLGenerator( HydrusSerialisable.SerialisableBaseNamed ): self._gallery_url_generator_key = bytes.fromhex( serialisable_gallery_url_generator_key ) + def CheckFunctional( self ): + + try: + + example_url = self.GetExampleURL() + + ( url_type, match_name, can_parse ) = HG.client_controller.network_engine.domain_manager.GetURLParseCapability( example_url ) + + except Exception as e: + + raise HydrusExceptions.ParseException( 'Unusual error: {}'.format( e ) ) + + + if url_type == HC.URL_TYPE_UNKNOWN: + + raise HydrusExceptions.ParseException( 'No URL Class for example URL!' ) + + + if not can_parse: + + raise HydrusExceptions.ParseException( 'No Parser for the URL Class {}!'.format( match_name ) ) + + + def GenerateGalleryURL( self, query_text ): if self._replacement_phrase == '': @@ -2641,6 +2665,20 @@ class GalleryURLGenerator( HydrusSerialisable.SerialisableBaseNamed ): return ( self._url_template, self._replacement_phrase, self._search_terms_separator, self._example_search_text ) + def IsFunctional( self ): + + try: + + self.CheckFunctional() + + return True + + except HydrusExceptions.ParseException: + + return False + + + def SetGUGKey( self, gug_key: bytes ): self._gallery_url_generator_key = gug_key @@ -2654,22 +2692,6 @@ class GalleryURLGenerator( HydrusSerialisable.SerialisableBaseNamed ): self._name = name - def IsFunctional( self ): - - try: - - example_url = self.GetExampleURL() - - ( url_type, match_name, can_parse ) = HG.client_controller.network_engine.domain_manager.GetURLParseCapability( example_url ) - - except: - - return False - - - return can_parse - - def RegenerateGUGKey( self ): self._gallery_url_generator_key = HydrusData.GenerateKey() @@ -2723,6 +2745,19 @@ class NestedGalleryURLGenerator( HydrusSerialisable.SerialisableBaseNamed ): self._gug_keys_and_names = [ ( bytes.fromhex( gug_key ), gug_name ) for ( gug_key, gug_name ) in serialisable_gug_keys_and_names ] + def CheckFunctional( self ): + + for gug_key_and_name in self._gug_keys_and_names: + + gug = HG.client_controller.network_engine.domain_manager.GetGUG( gug_key_and_name ) + + if gug is not None: + + gug.CheckFunctional() + + + + def GenerateGalleryURLs( self, query_text ): gallery_urls = [] @@ -2794,20 +2829,16 @@ class NestedGalleryURLGenerator( HydrusSerialisable.SerialisableBaseNamed ): def IsFunctional( self ): - for gug_key_and_name in self._gug_keys_and_names: + try: - gug = HG.client_controller.network_engine.domain_manager.GetGUG( gug_key_and_name ) + self.CheckFunctional() - if gug is not None: - - if gug.IsFunctional(): - - return True - - + return True + + except HydrusExceptions.ParseException: + + return False - - return False def RegenerateGUGKey( self ): diff --git a/hydrus/client/networking/ClientNetworkingJobs.py b/hydrus/client/networking/ClientNetworkingJobs.py index 83adfec1..dd5f1f98 100644 --- a/hydrus/client/networking/ClientNetworkingJobs.py +++ b/hydrus/client/networking/ClientNetworkingJobs.py @@ -271,7 +271,7 @@ class NetworkJob( object ): return ( connect_timeout, read_timeout ) - def _SendRequestAndGetResponse( self ): + def _SendRequestAndGetResponse( self ) -> requests.Response: with self._lock: @@ -1659,7 +1659,7 @@ class NetworkJobHydrus( NetworkJob ): NetworkJob._ReportDataUsed( self, num_bytes ) - def _SendRequestAndGetResponse( self ): + def _SendRequestAndGetResponse( self ) -> requests.Response: service = self.engine.controller.services_manager.GetService( self._service_key ) @@ -1674,7 +1674,7 @@ class NetworkJobHydrus( NetworkJob ): response = NetworkJob._SendRequestAndGetResponse( self ) - if service_type in HC.RESTRICTED_SERVICES: + if response.ok and service_type in HC.RESTRICTED_SERVICES: self._CheckHydrusVersion( service_type, response ) diff --git a/hydrus/core/HydrusConstants.py b/hydrus/core/HydrusConstants.py index ed10d076..db6a88bd 100644 --- a/hydrus/core/HydrusConstants.py +++ b/hydrus/core/HydrusConstants.py @@ -33,10 +33,12 @@ else: BASE_DIR = os.getcwd() -PLATFORM_WINDOWS = sys.platform == 'win32' -PLATFORM_MACOS = sys.platform == 'darwin' -PLATFORM_LINUX = sys.platform == 'linux' -PLATFORM_HAIKU = sys.platform == 'haiku1' +muh_platform = sys.platform.lower() + +PLATFORM_WINDOWS = muh_platform == 'win32' +PLATFORM_MACOS = muh_platform == 'darwin' +PLATFORM_LINUX = muh_platform == 'linux' +PLATFORM_HAIKU = muh_platform == 'haiku1' RUNNING_FROM_SOURCE = sys.argv[0].endswith( '.py' ) or sys.argv[0].endswith( '.pyw' ) RUNNING_FROM_MACOS_APP = os.path.exists( os.path.join( BASE_DIR, 'running_from_app' ) ) @@ -79,7 +81,7 @@ options = {} # Misc NETWORK_VERSION = 20 -SOFTWARE_VERSION = 435 +SOFTWARE_VERSION = 436 CLIENT_API_VERSION = 16 SERVER_THUMBNAIL_DIMENSIONS = ( 200, 200 ) diff --git a/hydrus/core/HydrusDB.py b/hydrus/core/HydrusDB.py index 48e6e54f..19787b56 100644 --- a/hydrus/core/HydrusDB.py +++ b/hydrus/core/HydrusDB.py @@ -264,8 +264,6 @@ class DBCursorTransactionWrapper( object ): class HydrusDB( object ): - TRANSACTION_COMMIT_PERIOD = 30 - READ_WRITE_ACTIONS = [] UPDATE_WAIT = 2 @@ -614,7 +612,7 @@ class HydrusDB( object ): self._c = self._db.cursor() - self._cursor_transaction_wrapper = DBCursorTransactionWrapper( self._c, self.TRANSACTION_COMMIT_PERIOD ) + self._cursor_transaction_wrapper = DBCursorTransactionWrapper( self._c, HG.db_transaction_commit_period ) self._LoadModules() diff --git a/hydrus/core/HydrusGlobals.py b/hydrus/core/HydrusGlobals.py index 7b888b09..ec440117 100644 --- a/hydrus/core/HydrusGlobals.py +++ b/hydrus/core/HydrusGlobals.py @@ -16,6 +16,7 @@ no_db_temp_files = False boot_debug = False db_cache_size = 200 +db_transaction_commit_period = 30 # if this is set to 1, transactions are not immediately synced to the journal so multiple can be undone following a power-loss # if set to 2, all transactions are synced, so once a new one starts you know the last one is on disk diff --git a/hydrus/core/networking/HydrusNetwork.py b/hydrus/core/networking/HydrusNetwork.py index 3636ac76..3d2df800 100644 --- a/hydrus/core/networking/HydrusNetwork.py +++ b/hydrus/core/networking/HydrusNetwork.py @@ -1956,11 +1956,11 @@ class Metadata( HydrusSerialisable.SerialisableBase ): if HydrusData.TimeHasPassed( update_due ): - s = 'next update imminent' + s = 'checking for updates imminently' else: - s = 'next update due {}'.format( HydrusData.TimestampToPrettyTimeDelta( update_due ) ) + s = 'checking for updates {}'.format( HydrusData.TimestampToPrettyTimeDelta( update_due ) ) return 'metadata synced up to {}, {}'.format( HydrusData.TimestampToPrettyTimeDelta( self._biggest_end ), s ) @@ -2077,13 +2077,20 @@ class Metadata( HydrusSerialisable.SerialisableBase ): - def UpdateFromSlice( self, metadata_slice ): + def UpdateFromSlice( self, metadata_slice: "Metadata" ): with self._lock: self._metadata.update( metadata_slice._metadata ) - self._next_update_due = metadata_slice._next_update_due + new_next_update_due = metadata_slice._next_update_due + + if HydrusData.TimeHasPassed( new_next_update_due ): + + new_next_update_due = HydrusData.GetNow() + 100000 + + + self._next_update_due = new_next_update_due self._biggest_end = self._CalculateBiggestEnd() @@ -2500,6 +2507,14 @@ class ServerServiceRepository( ServerServiceRestricted ): + def GetUpdatePeriod( self ) -> int: + + with self._lock: + + return self._service_options[ 'update_period' ] + + + def HasUpdateHash( self, update_hash ): with self._lock: diff --git a/hydrus/core/networking/HydrusServer.py b/hydrus/core/networking/HydrusServer.py index bb146dbb..e213a568 100644 --- a/hydrus/core/networking/HydrusServer.py +++ b/hydrus/core/networking/HydrusServer.py @@ -1,7 +1,9 @@ from twisted.web.http import _GenericHTTPChannelProtocol, HTTPChannel from twisted.web.server import Site +from twisted.web.server import Request from twisted.web.resource import Resource +from hydrus.core import HydrusConstants as HC from hydrus.core.networking import HydrusServerRequest from hydrus.core.networking import HydrusServerResources @@ -18,6 +20,10 @@ class HydrusService( Site ): self._service = service + service_type = self._service.GetServiceType() + + self._server_version_string = HC.service_string_lookup[ service_type ] + '/' + str( HC.NETWORK_VERSION ) + root = self._InitRoot() Site.__init__( self, root ) @@ -49,3 +55,10 @@ class HydrusService( Site ): return _GenericHTTPChannelProtocol( FatHTTPChannel() ) + + def getResourceFor( self, request: Request ): + + request.setHeader( 'Server', self._server_version_string ) + + return Site.getResourceFor( self, request ) + \ No newline at end of file diff --git a/hydrus/core/networking/HydrusServerResources.py b/hydrus/core/networking/HydrusServerResources.py index 906c0397..e02addfc 100644 --- a/hydrus/core/networking/HydrusServerResources.py +++ b/hydrus/core/networking/HydrusServerResources.py @@ -900,8 +900,6 @@ class HydrusResource( Resource ): def render_GET( self, request: HydrusServerRequest.HydrusRequest ): - request.setHeader( 'Server', self._server_version_string ) - d = defer.Deferred() d.addCallback( self._callbackCheckServiceRestrictions ) @@ -929,8 +927,6 @@ class HydrusResource( Resource ): def render_OPTIONS( self, request: HydrusServerRequest.HydrusRequest ): - request.setHeader( 'Server', self._server_version_string ) - d = defer.Deferred() d.addCallback( self._callbackCheckServiceRestrictions ) @@ -950,8 +946,6 @@ class HydrusResource( Resource ): def render_POST( self, request: HydrusServerRequest.HydrusRequest ): - request.setHeader( 'Server', self._server_version_string ) - d = defer.Deferred() d.addCallback( self._callbackCheckServiceRestrictions ) diff --git a/hydrus/hydrus_client.py b/hydrus/hydrus_client.py index a16c7500..f32fce4f 100644 --- a/hydrus/hydrus_client.py +++ b/hydrus/hydrus_client.py @@ -34,6 +34,7 @@ try: argparser.add_argument( '--temp_dir', help = 'override the program\'s temporary directory' ) argparser.add_argument( '--db_journal_mode', default = 'WAL', choices = [ 'WAL', 'TRUNCATE', 'PERSIST', 'MEMORY' ], help = 'change db journal mode (default=WAL)' ) argparser.add_argument( '--db_cache_size', type = int, help = 'override SQLite cache_size per db file, in MB (default=200)' ) + argparser.add_argument( '--db_transaction_commit_period', type = int, help = 'override how often (in seconds) database changes are saved to disk (default=30,min=10)' ) argparser.add_argument( '--db_synchronous_override', type = int, choices = range(4), help = 'override SQLite Synchronous PRAGMA (default=2)' ) argparser.add_argument( '--no_db_temp_files', action='store_true', help = 'run db temp operations entirely in memory' ) argparser.add_argument( '--boot_debug', action='store_true', help = 'print additional bootup information to the log' ) @@ -105,6 +106,15 @@ try: HG.db_cache_size = 200 + if result.db_transaction_commit_period is not None: + + HG.db_transaction_commit_period = max( 10, result.db_transaction_commit_period ) + + else: + + HG.db_transaction_commit_period = 30 + + if result.db_synchronous_override is not None: HG.db_synchronous = int( result.db_synchronous_override ) diff --git a/hydrus/hydrus_server.py b/hydrus/hydrus_server.py index 9c96c207..d28868a2 100644 --- a/hydrus/hydrus_server.py +++ b/hydrus/hydrus_server.py @@ -43,6 +43,7 @@ try: argparser.add_argument( '--temp_dir', help = 'override the program\'s temporary directory' ) argparser.add_argument( '--db_journal_mode', default = 'WAL', choices = [ 'WAL', 'TRUNCATE', 'PERSIST', 'MEMORY' ], help = 'change db journal mode (default=WAL)' ) argparser.add_argument( '--db_cache_size', type = int, help = 'override SQLite cache_size per db file, in MB (default=200)' ) + argparser.add_argument( '--db_transaction_commit_period', type = int, help = 'override how often (in seconds) database changes are saved to disk (default=120,min=10)' ) argparser.add_argument( '--db_synchronous_override', type = int, choices = range(4), help = 'override SQLite Synchronous PRAGMA (default=2)' ) argparser.add_argument( '--no_db_temp_files', action='store_true', help = 'run db temp operations entirely in memory' ) argparser.add_argument( '--boot_debug', action='store_true', help = 'print additional bootup information to the log' ) @@ -116,6 +117,15 @@ try: HG.db_cache_size = 200 + if result.db_transaction_commit_period is not None: + + HG.db_transaction_commit_period = max( 10, result.db_transaction_commit_period ) + + else: + + HG.db_transaction_commit_period = 30 + + if result.db_synchronous_override is not None: HG.db_synchronous = int( result.db_synchronous_override ) diff --git a/hydrus/server/ServerDB.py b/hydrus/server/ServerDB.py index dd335408..0566e1be 100644 --- a/hydrus/server/ServerDB.py +++ b/hydrus/server/ServerDB.py @@ -83,8 +83,6 @@ class DB( HydrusDB.HydrusDB ): READ_WRITE_ACTIONS = [ 'access_key', 'immediate_content_update', 'registration_keys' ] - TRANSACTION_COMMIT_PERIOD = 120 - def __init__( self, controller, db_dir, db_name ): self._files_dir = os.path.join( db_dir, 'server_files' ) diff --git a/hydrus/server/networking/ServerServerResources.py b/hydrus/server/networking/ServerServerResources.py index ba4838c7..44f43baf 100644 --- a/hydrus/server/networking/ServerServerResources.py +++ b/hydrus/server/networking/ServerServerResources.py @@ -17,19 +17,10 @@ from hydrus.server import ServerFiles class HydrusResourceBusyCheck( HydrusServerResources.Resource ): - def __init__( self ): - - HydrusServerResources.Resource.__init__( self ) - - self._server_version_string = HC.service_string_lookup[ HC.SERVER_ADMIN ] + '/' + str( HC.NETWORK_VERSION ) - - def render_GET( self, request: HydrusServerRequest.HydrusRequest ): request.setResponseCode( 200 ) - request.setHeader( 'Server', self._server_version_string ) - if HG.server_busy.locked(): return b'1' @@ -405,7 +396,20 @@ class HydrusResourceRestrictedOptionsModifyUpdatePeriod( HydrusResourceRestricte raise HydrusExceptions.BadRequestException( 'The update period was too high. It needs to be lower than {}.'.format( HydrusData.TimeDeltaToPrettyTimeDelta( HydrusNetwork.MAX_UPDATE_PERIOD ) ) ) - self._service.SetUpdatePeriod( update_period ) + old_update_period = self._service.GetUpdatePeriod() + + if old_update_period != update_period: + + self._service.SetUpdatePeriod( update_period ) + + HydrusData.Print( + 'Account {} changed the update period to from "{}" to "{}".'.format( + request.hydrus_account.GetAccountKey().hex(), + HydrusData.TimeDeltaToPrettyTimeDelta( old_update_period ), + HydrusData.TimeDeltaToPrettyTimeDelta( update_period ) + ) + ) + response_context = HydrusServerResources.ResponseContext( 200 ) diff --git a/static/build_files/Applications b/static/build_files/Applications new file mode 100644 index 00000000..bd2d47fa --- /dev/null +++ b/static/build_files/Applications @@ -0,0 +1 @@ +/Applications \ No newline at end of file diff --git a/static/build_files/ReadMeFirst.rtf b/static/build_files/ReadMeFirst.rtf new file mode 100644 index 00000000..63cc48a1 Binary files /dev/null and b/static/build_files/ReadMeFirst.rtf differ diff --git a/static/build_files/macos_build.yml b/static/build_files/macos_build.yml new file mode 100644 index 00000000..aaf28e42 --- /dev/null +++ b/static/build_files/macos_build.yml @@ -0,0 +1,99 @@ +name: Release +on: + push: + tags: + - 'v*' + +jobs: + build-macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - name: Rust Cache + uses: Swatinem/rust-cache@v1.2.0 + - name: rust-cargo + uses: actions-rs/cargo@v1.0.3 + with: + command: install + args: pyoxidizer + - name: Build Hydrus + run: | + cd $GITHUB_WORKSPACE + cp static/build_files/pyoxidizer.bzl pyoxidizer.bzl + basename $(rustc --print sysroot) | sed -e "s/^stable-//" > triple.txt + pyoxidizer build --release + cd build/$(head -n 1 triple.txt)/release + mkdir -p "Hydrus Network.app/Contents/MacOS" + mkdir -p "Hydrus Network.app/Contents/Resources" + mkdir -p "Hydrus Network.app/Contents/Frameworks" + mv install/static/icon.icns "Hydrus Network.app/Contents/Resources/icon.icns" + cat > "Hydrus Network.app/Contents/Info.plist" < + + + + CFBundleDisplayName + client + CFBundleExecutable + MacOS/client + CFBundleIconFile + icon.icns + CFBundleIdentifier + client + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + client + CFBundlePackageType + APPL + CFBundleShortVersionString + 0.0.0 + NSHighResolutionCapable + True + + EOF + cp install/static/build_files/ReadMeFirst.rtf ./ReadMeFirst.rtf + cp install/static/build_files/Applications ./Applications + cp install/static/build_files/running_from_app "install/running_from_app" + mv install/* "Hydrus Network.app/Contents/MacOS/" + rm -rf install + cd $GITHUB_WORKSPACE + temp_dmg="$(mktemp).dmg" + hdiutil create "$temp_dmg" -ov -volname "HydrusNetwork" -fs HFS+ -srcfolder "$GITHUB_WORKSPACE/build/$(head -n 1 triple.txt)/release" + hdiutil convert "$temp_dmg" -format UDZO -o HydrusNetwork.dmg + - name: Upload a Build Artifact + uses: actions/upload-artifact@v2.2.1 + with: + name: MacOS-DMG + path: HydrusNetwork.dmg + if-no-files-found: error + retention-days: 2 + + create-release: + name: Create Release Entry + runs-on: ubuntu-20.04 + needs: [build-macos] + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Get All Artifacts + uses: actions/download-artifact@v2 + - name: Extract version metadata + id: meta + run: | + echo "::set-output name=version::${GITHUB_REF##*/}" + echo "::set-output name=version_short::${GITHUB_REF##*/v}" + - name: Rename Files + run: | + mv MacOS-DMG/HydrusNetwork.dmg Hydrus.Network.${{ steps.meta.outputs.version_short }}.-.macOS.-.App.dmg + - name: Release new + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + files: | + Hydrus.Network.${{ steps.meta.outputs.version_short }}.-.macOS.-.App.dmg + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/pyoxidizer.bzl b/static/build_files/pyoxidizer.bzl similarity index 100% rename from pyoxidizer.bzl rename to static/build_files/pyoxidizer.bzl diff --git a/static/build_files/running_from_app b/static/build_files/running_from_app new file mode 100644 index 00000000..e69de29b diff --git a/static/qss/CutieDuck_(darkorange).qss b/static/qss/CutieDuck_(darkorange).qss index 5adb156c..13e7547e 100644 --- a/static/qss/CutieDuck_(darkorange).qss +++ b/static/qss/CutieDuck_(darkorange).qss @@ -80,7 +80,6 @@ QPushButton border-color: #000000; border-style: solid; padding: 3px; - font-size: 12px; padding-left: 5px; padding-right: 5px; } diff --git a/static/qss/DarkerDuck_(darkorange).qss b/static/qss/DarkerDuck_(darkorange).qss index a7a48f92..1f7b1373 100644 --- a/static/qss/DarkerDuck_(darkorange).qss +++ b/static/qss/DarkerDuck_(darkorange).qss @@ -80,7 +80,6 @@ QPushButton border-color: #bcbcbc; border-style: solid; padding: 3px; - font-size: 12px; padding-left: 5px; padding-right: 5px; }