From 9d95ee79fe067f4ce0fc072b0ea79159284586dd Mon Sep 17 00:00:00 2001 From: Hydrus Network Developer Date: Wed, 27 Jan 2021 16:14:03 -0600 Subject: [PATCH] Version 427 closes #788 --- client.py | 230 +--- client.pyw | 229 +--- help/changelog.html | 36 +- help/client_api.html | 131 +- help/database_migration.html | 14 +- help/downloader_completion.html | 4 +- help/downloader_gugs.html | 8 +- help/downloader_intro.html | 16 +- help/downloader_login.html | 6 +- help/downloader_parsers.html | 4 +- help/downloader_parsers_content_parsers.html | 12 +- help/downloader_parsers_full_example_api.html | 6 +- ...loader_parsers_full_example_file_page.html | 14 +- ...der_parsers_full_example_gallery_page.html | 12 +- help/downloader_parsers_page_parsers.html | 8 +- help/downloader_sharing.html | 4 +- help/downloader_url_classes.html | 40 +- help/duplicates.html | 6 +- help/faq.html | 20 +- help/getting_started_installing.html | 4 +- help/getting_started_messages.html | 56 - help/getting_started_more_files.html | 2 +- help/getting_started_subscriptions.html | 2 +- help/index.html | 12 +- help/ipfs.html | 10 +- help/launch_arguments.html | 2 +- help/local_booru.html | 17 +- help/privacy.html | 19 +- help/reducing_lag.html | 6 +- help/running_from_source.html | 8 +- help/server.html | 20 +- help/support.html | 2 +- help/tagging_schema.html | 24 +- help/wine.html | 2 +- hydrus/client/ClientApplicationCommand.py | 18 +- hydrus/client/ClientController.py | 2 +- hydrus/client/ClientDefaults.py | 17 +- hydrus/client/ClientLocalServerResources.py | 52 +- hydrus/client/db/ClientDB.py | 1121 ++++++++--------- hydrus/client/db/ClientDBMaster.py | 679 ++++++++++ hydrus/client/db/__init__.py | 1 + hydrus/client/gui/ClientGUI.py | 42 +- hydrus/client/gui/ClientGUICanvas.py | 2 +- hydrus/client/gui/ClientGUIManagement.py | 30 +- hydrus/client/gui/ClientGUIPages.py | 5 - .../client/gui/ClientGUIShortcutControls.py | 2 +- hydrus/client/gui/ClientGUIShortcuts.py | 59 +- hydrus/client/gui/ClientGUITags.py | 2 +- .../client/gui/search/ClientGUIACDropdown.py | 174 ++- .../networking/ClientNetworkingDomain.py | 41 + hydrus/core/HydrusConstants.py | 2 +- hydrus/core/HydrusDB.py | 16 + hydrus/core/HydrusDBModule.py | 79 ++ hydrus/core/HydrusGlobals.py | 2 + hydrus/core/HydrusImageHandling.py | 1 - hydrus/hydrus_client.py | 239 ++++ hydrus/hydrus_server.py | 245 ++++ hydrus/hydrus_test.py | 93 ++ hydrus/test/TestClientAPI.py | 92 +- hydrus/test/TestClientDB.py | 2 +- hydrus/test/TestClientDBDuplicates.py | 2 +- hydrus/test/TestClientDBTags.py | 302 ++++- hydrus/test/TestClientMigration.py | 2 +- hydrus/test/TestController.py | 2 + server.py | 235 +--- .../gelbooru 0.2.5 file page parser.png | Bin 2684 -> 2679 bytes .../url_classes/420chan thread new format.png | Bin 0 -> 2064 bytes test.py | 87 +- 68 files changed, 2827 insertions(+), 1807 deletions(-) delete mode 100755 help/getting_started_messages.html create mode 100644 hydrus/client/db/ClientDBMaster.py create mode 100644 hydrus/core/HydrusDBModule.py create mode 100644 hydrus/hydrus_client.py create mode 100644 hydrus/hydrus_server.py create mode 100644 hydrus/hydrus_test.py create mode 100644 static/default/url_classes/420chan thread new format.png diff --git a/client.py b/client.py index b12e44b7..c46ae740 100644 --- a/client.py +++ b/client.py @@ -4,233 +4,9 @@ # You just DO WHAT THE FUCK YOU WANT TO. # https://github.com/sirkris/WTFPL/blob/master/WTFPL.md -try: - - import locale - - try: locale.setlocale( locale.LC_ALL, '' ) - except: pass - - import os - import argparse - import traceback - - from hydrus.core import HydrusBoot - - HydrusBoot.AddBaseDirToEnvPath() - - # initialise Qt here, important it is done early - from hydrus.client.gui import QtPorting as QP - - from hydrus.core import HydrusConstants as HC - from hydrus.core import HydrusPaths - from hydrus.core import HydrusGlobals as HG - - argparser = argparse.ArgumentParser( description = 'hydrus network client (console)' ) - - argparser.add_argument( '-d', '--db_dir', help = 'set an external db location' ) - argparser.add_argument( '--temp_dir', help = 'override the program\'s temporary directory' ) - argparser.add_argument( '--db_journal_mode', default = 'WAL', choices = [ 'WAL', 'TRUNCATE', 'PERSIST', 'MEMORY' ], help = 'change db journal mode (default=WAL)' ) - argparser.add_argument( '--db_cache_size', type = int, help = 'override SQLite cache_size per db file, in MB (default=200)' ) - argparser.add_argument( '--db_synchronous_override', type = int, choices = range(4), help = 'override SQLite Synchronous PRAGMA (default=2)' ) - argparser.add_argument( '--no_db_temp_files', action='store_true', help = 'run db temp operations entirely in memory' ) - argparser.add_argument( '--boot_debug', action='store_true', help = 'print additional bootup information to the log' ) - argparser.add_argument( '--no_daemons', action='store_true', help = 'run without background daemons' ) - argparser.add_argument( '--no_wal', action='store_true', help = 'OBSOLETE: run using TRUNCATE db journaling' ) - argparser.add_argument( '--db_memory_journaling', action='store_true', help = 'OBSOLETE: run using MEMORY db journaling (DANGEROUS)' ) - - result = argparser.parse_args() - - if result.db_dir is None: - - db_dir = HC.DEFAULT_DB_DIR - - if not HydrusPaths.DirectoryIsWritable( db_dir ) or HC.RUNNING_FROM_MACOS_APP: - - db_dir = HC.USERPATH_DB_DIR - - - else: - - db_dir = result.db_dir - - - db_dir = HydrusPaths.ConvertPortablePathToAbsPath( db_dir, HC.BASE_DIR ) - - try: - - HydrusPaths.MakeSureDirectoryExists( db_dir ) - - except: - - raise Exception( 'Could not ensure db path "{}" exists! Check the location is correct and that you have permission to write to it!'.format( db_dir ) ) - - - if not os.path.isdir( db_dir ): - - raise Exception( 'The given db path "{}" is not a directory!'.format( db_dir ) ) - - - if not HydrusPaths.DirectoryIsWritable( db_dir ): - - raise Exception( 'The given db path "{}" is not a writable-to!'.format( db_dir ) ) - - - HG.no_daemons = result.no_daemons - - HG.db_journal_mode = result.db_journal_mode - - if result.no_wal: - - HG.db_journal_mode = 'TRUNCATE' - - if result.db_memory_journaling: - - HG.db_journal_mode = 'MEMORY' - - - if result.db_cache_size is not None: - - HG.db_cache_size = result.db_cache_size - - else: - - HG.db_cache_size = 200 - - - if result.db_synchronous_override is not None: - - HG.db_synchronous = int( result.db_synchronous_override ) - - else: - - if HG.db_journal_mode == 'WAL': - - HG.db_synchronous = 1 - - else: - - HG.db_synchronous = 2 - - - - HG.no_db_temp_files = result.no_db_temp_files - - HG.boot_debug = result.boot_debug - - if result.temp_dir is not None: - - HydrusPaths.SetEnvTempDir( result.temp_dir ) - - - from hydrus.core import HydrusData - from hydrus.core import HydrusLogger - - try: - - from twisted.internet import reactor - - except: - - HG.twisted_is_broke = True - - -except Exception as e: - - try: - - from hydrus.core import HydrusData - - HydrusData.DebugPrint( 'Critical boot error occurred! Details written to crash.log!' ) - HydrusData.PrintException( e ) - - except: - - pass - - - error_trace = traceback.format_exc() - - print( error_trace ) - - if 'db_dir' in locals() and os.path.exists( db_dir ): - - emergency_dir = db_dir - - else: - - emergency_dir = os.path.expanduser( '~' ) - - possible_desktop = os.path.join( emergency_dir, 'Desktop' ) - - if os.path.exists( possible_desktop ) and os.path.isdir( possible_desktop ): - - emergency_dir = possible_desktop - - - - dest_path = os.path.join( emergency_dir, 'hydrus_crash.log' ) - - with open( dest_path, 'w', encoding = 'utf-8' ) as f: - - f.write( error_trace ) - - - print( 'Critical boot error occurred! Details written to hydrus_crash.log in either db dir or user dir!' ) - - import sys - - sys.exit( 1 ) - -controller = None +from hydrus import hydrus_client -with HydrusLogger.HydrusLogger( db_dir, 'client' ) as logger: +if __name__ == '__main__': - try: - - HydrusData.Print( 'hydrus client started' ) - - if not HG.twisted_is_broke: - - import threading - - threading.Thread( target = reactor.run, name = 'twisted', kwargs = { 'installSignalHandlers' : 0 } ).start() - - - from hydrus.client import ClientController - - controller = ClientController.Controller( db_dir ) - - controller.Run() - - except: - - HydrusData.Print( 'hydrus client failed' ) - - HydrusData.Print( traceback.format_exc() ) - - finally: - - HG.view_shutdown = True - HG.model_shutdown = True - - if controller is not None: - - controller.pubimmediate( 'wake_daemons' ) - - - if not HG.twisted_is_broke: - - reactor.callFromThread( reactor.stop ) - - - HydrusData.Print( 'hydrus client shut down' ) - + hydrus_client.boot() -HG.shutdown_complete = True - -if HG.restart: - - HydrusData.RestartProcess() - - diff --git a/client.pyw b/client.pyw index ffa536a3..c46ae740 100755 --- a/client.pyw +++ b/client.pyw @@ -4,232 +4,9 @@ # You just DO WHAT THE FUCK YOU WANT TO. # https://github.com/sirkris/WTFPL/blob/master/WTFPL.md -try: - - import locale - - try: locale.setlocale( locale.LC_ALL, '' ) - except: pass - - import os - import argparse - import traceback - - from hydrus.core import HydrusBoot - - HydrusBoot.AddBaseDirToEnvPath() - - # initialise Qt here, important it is done early - from hydrus.client.gui import QtPorting as QP - - from hydrus.core import HydrusConstants as HC - from hydrus.core import HydrusPaths - from hydrus.core import HydrusGlobals as HG - - argparser = argparse.ArgumentParser( description = 'hydrus network client (windowed)' ) - - argparser.add_argument( '-d', '--db_dir', help = 'set an external db location' ) - argparser.add_argument( '--temp_dir', help = 'override the program\'s temporary directory' ) - argparser.add_argument( '--db_journal_mode', default = 'WAL', choices = [ 'WAL', 'TRUNCATE', 'PERSIST', 'MEMORY' ], help = 'change db journal mode (default=WAL)' ) - argparser.add_argument( '--db_cache_size', type = int, help = 'override SQLite cache_size per db file, in MB (default=200)' ) - argparser.add_argument( '--db_synchronous_override', type = int, choices = range(4), help = 'override SQLite Synchronous PRAGMA (default=2)' ) - argparser.add_argument( '--no_db_temp_files', action='store_true', help = 'run db temp operations entirely in memory' ) - argparser.add_argument( '--boot_debug', action='store_true', help = 'print additional bootup information to the log' ) - argparser.add_argument( '--no_daemons', action='store_true', help = 'run without background daemons' ) - argparser.add_argument( '--no_wal', action='store_true', help = 'OBSOLETE: run using TRUNCATE db journaling' ) - argparser.add_argument( '--db_memory_journaling', action='store_true', help = 'OBSOLETE: run using MEMORY db journaling (DANGEROUS)' ) - - result = argparser.parse_args() - - if result.db_dir is None: - - db_dir = HC.DEFAULT_DB_DIR - - if not HydrusPaths.DirectoryIsWritable( db_dir ) or HC.RUNNING_FROM_MACOS_APP: - - db_dir = HC.USERPATH_DB_DIR - - - else: - - db_dir = result.db_dir - - - db_dir = HydrusPaths.ConvertPortablePathToAbsPath( db_dir, HC.BASE_DIR ) - - try: - - HydrusPaths.MakeSureDirectoryExists( db_dir ) - - except: - - raise Exception( 'Could not ensure db path "{}" exists! Check the location is correct and that you have permission to write to it!'.format( db_dir ) ) - - - if not os.path.isdir( db_dir ): - - raise Exception( 'The given db path "{}" is not a directory!'.format( db_dir ) ) - - - if not HydrusPaths.DirectoryIsWritable( db_dir ): - - raise Exception( 'The given db path "{}" is not a writable-to!'.format( db_dir ) ) - - - HG.no_daemons = result.no_daemons - - HG.db_journal_mode = result.db_journal_mode - - if result.no_wal: - - HG.db_journal_mode = 'TRUNCATE' - - if result.db_memory_journaling: - - HG.db_journal_mode = 'MEMORY' - - - if result.db_cache_size is not None: - - HG.db_cache_size = result.db_cache_size - - else: - - HG.db_cache_size = 200 - - - if result.db_synchronous_override is not None: - - HG.db_synchronous = int( result.db_synchronous_override ) - - else: - - if HG.db_journal_mode == 'WAL': - - HG.db_synchronous = 1 - - else: - - HG.db_synchronous = 2 - - - - HG.no_db_temp_files = result.no_db_temp_files - - HG.boot_debug = result.boot_debug - - if result.temp_dir is not None: - - HydrusPaths.SetEnvTempDir( result.temp_dir ) - - - from hydrus.core import HydrusData - from hydrus.core import HydrusLogger - - try: - - from twisted.internet import reactor - - except: - - HG.twisted_is_broke = True - - -except Exception as e: - - try: - - from hydrus.core import HydrusData - - HydrusData.DebugPrint( 'Critical boot error occurred! Details written to crash.log!' ) - HydrusData.PrintException( e ) - - except: - - pass - - - error_trace = traceback.format_exc() - - print( error_trace ) - - if 'db_dir' in locals() and os.path.exists( db_dir ): - - emergency_dir = db_dir - - else: - - emergency_dir = os.path.expanduser( '~' ) - - possible_desktop = os.path.join( emergency_dir, 'Desktop' ) - - if os.path.exists( possible_desktop ) and os.path.isdir( possible_desktop ): - - emergency_dir = possible_desktop - - - - dest_path = os.path.join( emergency_dir, 'hydrus_crash.log' ) - - with open( dest_path, 'w', encoding = 'utf-8' ) as f: - - f.write( error_trace ) - - - print( 'Critical boot error occurred! Details written to hydrus_crash.log in either db dir or user dir!' ) - - import sys - - sys.exit( 1 ) - -controller = None +from hydrus import hydrus_client -with HydrusLogger.HydrusLogger( db_dir, 'client' ) as logger: +if __name__ == '__main__': - try: - - HydrusData.Print( 'hydrus client started' ) - - if not HG.twisted_is_broke: - - import threading - - threading.Thread( target = reactor.run, name = 'twisted', kwargs = { 'installSignalHandlers' : 0 } ).start() - - - from hydrus.client import ClientController - - controller = ClientController.Controller( db_dir ) - - controller.Run() - - except: - - HydrusData.Print( 'hydrus client failed' ) - - HydrusData.Print( traceback.format_exc() ) - - finally: - - HG.view_shutdown = True - HG.model_shutdown = True - - if controller is not None: - - controller.pubimmediate( 'wake_daemons' ) - - - if not HG.twisted_is_broke: - - reactor.callFromThread( reactor.stop ) - - - HydrusData.Print( 'hydrus client shut down' ) - - -HG.shutdown_complete = True - -if HG.restart: - - HydrusData.RestartProcess() + hydrus_client.boot() diff --git a/help/changelog.html b/help/changelog.html index 4351ed5f..0f96706b 100755 --- a/help/changelog.html +++ b/help/changelog.html @@ -8,6 +8,40 @@

changelog