hydrus/hydrus/core/HydrusDB.py

963 lines
31 KiB
Python
Raw Permalink Normal View History

2020-12-02 22:04:38 +00:00
import collections
2020-07-29 20:52:44 +00:00
import os
import queue
import sqlite3
import traceback
import time
2021-08-11 21:14:12 +00:00
from hydrus.core import HydrusDBBase
2020-04-22 21:00:35 +00:00
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusEncryption
2020-04-22 21:00:35 +00:00
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusGlobals as HG
from hydrus.core import HydrusPaths
2023-04-19 20:38:13 +00:00
from hydrus.core import HydrusProfiling
from hydrus.core import HydrusTime
2024-02-14 21:20:24 +00:00
from hydrus.core.interfaces import HydrusControllerInterface
2015-04-22 22:57:25 +00:00
2020-04-08 21:10:11 +00:00
def CheckCanVacuum( db_path, stop_time = None ):
2016-04-14 01:54:29 +00:00
2020-04-08 21:10:11 +00:00
db = sqlite3.connect( db_path, isolation_level = None, detect_types = sqlite3.PARSE_DECLTYPES )
c = db.cursor()
2020-12-09 22:18:48 +00:00
CheckCanVacuumCursor( db_path, c, stop_time = stop_time )
def CheckCanVacuumCursor( db_path, c, stop_time = None ):
2020-04-08 21:10:11 +00:00
( page_size, ) = c.execute( 'PRAGMA page_size;' ).fetchone()
( page_count, ) = c.execute( 'PRAGMA page_count;' ).fetchone()
( freelist_count, ) = c.execute( 'PRAGMA freelist_count;' ).fetchone()
2021-07-28 21:12:00 +00:00
CheckCanVacuumData( db_path, page_size, page_count, freelist_count, stop_time = stop_time )
2020-04-08 21:10:11 +00:00
2021-07-28 21:12:00 +00:00
def CheckCanVacuumData( db_path, page_size, page_count, freelist_count, stop_time = None ):
db_size = ( page_count - freelist_count ) * page_size
2020-04-08 21:10:11 +00:00
if stop_time is not None:
2016-04-14 01:54:29 +00:00
2021-07-28 21:12:00 +00:00
approx_vacuum_duration = GetApproxVacuumDuration( db_size )
2016-04-14 01:54:29 +00:00
2020-04-08 21:10:11 +00:00
time_i_will_have_to_start = stop_time - approx_vacuum_duration
2016-04-14 01:54:29 +00:00
2023-04-19 20:38:13 +00:00
if HydrusTime.TimeHasPassed( time_i_will_have_to_start ):
2016-04-14 01:54:29 +00:00
2023-04-19 20:38:13 +00:00
raise Exception( 'I believe you need about ' + HydrusTime.TimeDeltaToPrettyTimeDelta( approx_vacuum_duration ) + ' to vacuum, but there is not enough time allotted.' )
2016-04-14 01:54:29 +00:00
2020-04-08 21:10:11 +00:00
2021-02-24 22:35:18 +00:00
db_dir = os.path.dirname( db_path )
2020-04-08 21:10:11 +00:00
HydrusDBBase.CheckHasSpaceForDBTransaction( db_dir, db_size )
2021-07-28 21:12:00 +00:00
2023-05-17 20:49:46 +00:00
2021-07-28 21:12:00 +00:00
def GetApproxVacuumDuration( db_size ):
vacuum_estimate = int( db_size * 1.2 )
approx_vacuum_speed_mb_per_s = 1048576 * 1
approx_vacuum_duration = vacuum_estimate // approx_vacuum_speed_mb_per_s
return approx_vacuum_duration
2016-04-14 01:54:29 +00:00
2023-05-17 20:49:46 +00:00
2018-11-28 22:31:04 +00:00
def ReadLargeIdQueryInSeparateChunks( cursor, select_statement, chunk_size ):
2019-02-13 22:26:43 +00:00
table_name = 'tempbigread' + os.urandom( 32 ).hex()
2018-11-28 22:31:04 +00:00
2019-02-13 22:26:43 +00:00
cursor.execute( 'CREATE TEMPORARY TABLE ' + table_name + ' ( job_id INTEGER PRIMARY KEY AUTOINCREMENT, temp_id INTEGER );' )
2018-11-28 22:31:04 +00:00
cursor.execute( 'INSERT INTO ' + table_name + ' ( temp_id ) ' + select_statement ) # given statement should end in semicolon, so we are good
2019-02-13 22:26:43 +00:00
num_to_do = cursor.rowcount
2018-11-28 22:31:04 +00:00
2019-02-13 22:26:43 +00:00
if num_to_do is None or num_to_do == -1:
2018-11-28 22:31:04 +00:00
2019-02-13 22:26:43 +00:00
num_to_do = 0
2018-11-28 22:31:04 +00:00
2019-02-13 22:26:43 +00:00
2024-03-06 21:57:34 +00:00
i = 0
2023-07-26 20:57:00 +00:00
num_done = 0
2019-02-13 22:26:43 +00:00
2023-07-26 20:57:00 +00:00
while num_done < num_to_do:
2019-02-13 22:26:43 +00:00
2024-03-06 21:57:34 +00:00
chunk = [ temp_id for ( temp_id, ) in cursor.execute( 'SELECT temp_id FROM ' + table_name + ' WHERE job_id BETWEEN ? AND ?;', ( i, i + ( chunk_size - 1 ) ) ) ]
2019-02-13 22:26:43 +00:00
2024-03-06 21:57:34 +00:00
i += chunk_size
2023-07-26 20:57:00 +00:00
num_done += len( chunk )
2019-02-13 22:26:43 +00:00
2023-07-26 20:57:00 +00:00
yield ( chunk, num_done, num_to_do )
2018-11-28 22:31:04 +00:00
cursor.execute( 'DROP TABLE ' + table_name + ';' )
2023-07-26 20:57:00 +00:00
2016-04-14 01:54:29 +00:00
def VacuumDB( db_path ):
db = sqlite3.connect( db_path, isolation_level = None, detect_types = sqlite3.PARSE_DECLTYPES )
c = db.cursor()
2023-10-25 21:23:53 +00:00
fast_big_transaction_wal = not sqlite3.sqlite_version_info < ( 3, 11, 0 )
2016-04-14 01:54:29 +00:00
2020-12-09 22:18:48 +00:00
if HG.db_journal_mode == 'WAL' and not fast_big_transaction_wal:
2016-04-14 01:54:29 +00:00
c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
2024-05-08 20:25:53 +00:00
# this used to be 1024 for Linux users, so we do want to check and coerce back to SQLite default, 4096
2016-04-14 01:54:29 +00:00
( page_size, ) = c.execute( 'PRAGMA page_size;' ).fetchone()
2024-05-08 20:25:53 +00:00
ideal_page_size = 4096
2016-04-14 01:54:29 +00:00
if page_size != ideal_page_size:
c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
c.execute( 'PRAGMA page_size = ' + str( ideal_page_size ) + ';' )
2017-07-12 20:03:45 +00:00
c.execute( 'PRAGMA auto_vacuum = 0;' ) # none
2016-04-14 01:54:29 +00:00
c.execute( 'VACUUM;' )
2020-12-09 22:18:48 +00:00
c.execute( 'PRAGMA journal_mode = {};'.format( HG.db_journal_mode ) )
2016-04-14 01:54:29 +00:00
2024-05-08 20:25:53 +00:00
2021-08-11 21:14:12 +00:00
class HydrusDB( HydrusDBBase.DBBase ):
2015-04-22 22:57:25 +00:00
READ_WRITE_ACTIONS = []
2016-03-09 19:37:14 +00:00
UPDATE_WAIT = 2
2015-04-22 22:57:25 +00:00
2024-02-14 21:20:24 +00:00
def __init__( self, controller: HydrusControllerInterface.HydrusControllerInterface, db_dir, db_name ):
2015-08-26 21:18:39 +00:00
2021-08-11 21:14:12 +00:00
HydrusDBBase.DBBase.__init__( self )
2015-08-26 21:18:39 +00:00
self._controller = controller
2016-04-06 19:52:45 +00:00
self._db_dir = db_dir
self._db_name = db_name
2015-04-22 22:57:25 +00:00
2021-01-27 22:14:03 +00:00
self._modules = []
2021-08-11 21:14:12 +00:00
HydrusDBBase.TemporaryIntegerTableNameCache()
2020-12-02 22:04:38 +00:00
self._ssl_cert_filename = '{}.crt'.format( self._db_name )
self._ssl_key_filename = '{}.key'.format( self._db_name )
self._ssl_cert_path = os.path.join( self._db_dir, self._ssl_cert_filename )
self._ssl_key_path = os.path.join( self._db_dir, self._ssl_key_filename )
2016-04-06 19:52:45 +00:00
main_db_filename = db_name
if not main_db_filename.endswith( '.db' ):
main_db_filename += '.db'
self._db_filenames = {}
self._db_filenames[ 'main' ] = main_db_filename
2019-09-05 00:05:32 +00:00
self._durable_temp_db_filename = db_name + '.temp.db'
2022-02-02 22:14:01 +00:00
durable_temp_db_path = os.path.join( self._db_dir, self._durable_temp_db_filename )
if os.path.exists( durable_temp_db_path ):
HydrusPaths.DeletePath( durable_temp_db_path )
wal_lad = durable_temp_db_path + '-wal'
if os.path.exists( wal_lad ):
HydrusPaths.DeletePath( wal_lad )
shm_lad = durable_temp_db_path + '-shm'
if os.path.exists( shm_lad ):
HydrusPaths.DeletePath( shm_lad )
HydrusData.Print( 'Found and deleted the durable temporary database on boot. The last exit was probably not clean.' )
2016-04-20 20:42:21 +00:00
self._InitExternalDatabases()
2016-08-31 19:55:14 +00:00
self._is_first_start = False
self._is_db_updated = False
2015-04-22 22:57:25 +00:00
self._local_shutdown = False
2019-10-09 22:03:03 +00:00
self._pause_and_disconnect = False
2015-04-22 22:57:25 +00:00
self._loop_finished = False
2016-03-16 22:19:14 +00:00
self._ready_to_serve_requests = False
self._could_not_initialise = False
2015-04-22 22:57:25 +00:00
2019-01-09 22:59:03 +00:00
self._jobs = queue.Queue()
2015-04-22 22:57:25 +00:00
self._currently_doing_job = False
2017-05-24 20:28:24 +00:00
self._current_status = ''
self._current_job_name = ''
2015-04-22 22:57:25 +00:00
2016-02-17 22:06:47 +00:00
self._db = None
2021-05-27 00:09:06 +00:00
self._is_connected = False
2016-02-17 22:06:47 +00:00
2021-02-24 22:35:18 +00:00
self._cursor_transaction_wrapper = None
2016-04-06 19:52:45 +00:00
if os.path.exists( os.path.join( self._db_dir, self._db_filenames[ 'main' ] ) ):
2015-04-22 22:57:25 +00:00
# open and close to clean up in case last session didn't close well
self._InitDB()
2021-08-11 21:14:12 +00:00
self._CloseDBConnection()
2015-04-22 22:57:25 +00:00
2023-07-05 20:52:58 +00:00
total_db_size = self.GetApproxTotalFileSize()
size_check = min( int( total_db_size * 0.5 ), 500 * 1048576 )
size_check = max( size_check, 64 * 1048576 )
if HydrusPaths.GetFreeSpace( db_dir ) < size_check:
raise HydrusExceptions.DBAccessException( 'Sorry, it looks like the database drive partition has less than {} free space. It needs this for database transactions, so please free up some space.'.format( HydrusData.ToHumanBytes( size_check ) ) )
2015-04-22 22:57:25 +00:00
self._InitDB()
2021-08-11 21:14:12 +00:00
( version, ) = self._Execute( 'SELECT version FROM version;' ).fetchone()
2015-04-22 22:57:25 +00:00
2018-11-28 22:31:04 +00:00
if version > HC.SOFTWARE_VERSION:
2017-03-08 23:23:12 +00:00
2018-11-28 22:31:04 +00:00
self._ReportOverupdatedDB( version )
2017-03-08 23:23:12 +00:00
2024-02-07 21:22:05 +00:00
if version < HC.SOFTWARE_VERSION - 50:
2019-02-13 22:26:43 +00:00
2024-02-07 21:22:05 +00:00
raise HydrusExceptions.DBVersionException( 'Your current database version of hydrus ' + str( version ) + ' is too old for this software version ' + str( HC.SOFTWARE_VERSION ) + ' to update. Please try updating with version ' + str( version + 45 ) + ' or earlier first.' )
2019-02-13 22:26:43 +00:00
2024-02-07 21:22:05 +00:00
bitrot_rows = [
( 'client', 551, 558, 'millisecond timestamp conversion' )
]
for ( bitrot_db_name, latest_affected_version, safe_update_version, reason ) in bitrot_rows:
if self._db_name == bitrot_db_name and version <= latest_affected_version:
raise HydrusExceptions.DBVersionException( f'Sorry, due to a bitrot issue ({reason}), you cannot update to this software version (v{HC.SOFTWARE_VERSION}) if your database is on v{latest_affected_version} or earlier (you are on v{version}). Please download and update to v{safe_update_version} first!' )
if version < ( HC.SOFTWARE_VERSION - 15 ):
2017-03-08 23:23:12 +00:00
2024-02-07 21:22:05 +00:00
self._ReportUnderupdatedDB( version )
2017-03-08 23:23:12 +00:00
2015-04-22 22:57:25 +00:00
self._RepairDB( version )
2021-01-13 21:48:58 +00:00
2015-04-22 22:57:25 +00:00
while version < HC.SOFTWARE_VERSION:
2016-03-09 19:37:14 +00:00
time.sleep( self.UPDATE_WAIT )
2015-04-22 22:57:25 +00:00
2017-03-29 19:39:34 +00:00
try:
2021-02-24 22:35:18 +00:00
self._cursor_transaction_wrapper.BeginImmediate()
2017-03-29 19:39:34 +00:00
2015-04-22 22:57:25 +00:00
except Exception as e:
2019-01-09 22:59:03 +00:00
raise HydrusExceptions.DBAccessException( str( e ) )
2015-04-22 22:57:25 +00:00
try:
self._UpdateDB( version )
2021-02-24 22:35:18 +00:00
self._cursor_transaction_wrapper.Commit()
2015-04-22 22:57:25 +00:00
2016-08-31 19:55:14 +00:00
self._is_db_updated = True
2015-04-22 22:57:25 +00:00
except:
2024-04-10 20:36:05 +00:00
e = Exception( 'Updating the ' + self._db_name + ' db to version ' + str( version + 1 ) + ' caused this error:' + '\n' + traceback.format_exc() )
2016-04-20 20:42:21 +00:00
try:
2021-02-24 22:35:18 +00:00
self._cursor_transaction_wrapper.Rollback()
2016-04-20 20:42:21 +00:00
except Exception as rollback_e:
HydrusData.Print( 'When the update failed, attempting to rollback the database failed.' )
HydrusData.PrintException( rollback_e )
2015-04-22 22:57:25 +00:00
2016-04-20 20:42:21 +00:00
raise e
2015-04-22 22:57:25 +00:00
2021-08-11 21:14:12 +00:00
( version, ) = self._Execute( 'SELECT version FROM version;' ).fetchone()
2015-04-22 22:57:25 +00:00
2021-08-11 21:14:12 +00:00
self._CloseDBConnection()
2015-04-22 22:57:25 +00:00
2017-08-09 21:33:51 +00:00
self._controller.CallToThreadLongRunning( self.MainLoop )
2016-03-16 22:19:14 +00:00
while not self._ready_to_serve_requests:
time.sleep( 0.1 )
if self._could_not_initialise:
raise Exception( 'Could not initialise the db! Error written to the log!' )
2015-04-22 22:57:25 +00:00
2016-03-30 22:56:50 +00:00
def _AttachExternalDatabases( self ):
2021-05-27 00:09:06 +00:00
for ( name, filename ) in self._db_filenames.items():
2016-04-20 20:42:21 +00:00
if name == 'main':
continue
2019-09-05 00:05:32 +00:00
db_path = os.path.join( self._db_dir, filename )
2016-04-20 20:42:21 +00:00
if os.path.exists( db_path ) and not HydrusPaths.FileisWriteable( db_path ):
raise HydrusExceptions.DBAccessException( '"{}" seems to be read-only!'.format( db_path ) )
2021-08-11 21:14:12 +00:00
self._Execute( 'ATTACH ? AS ' + name + ';', ( db_path, ) )
2016-04-20 20:42:21 +00:00
2016-03-30 22:56:50 +00:00
2019-09-05 00:05:32 +00:00
db_path = os.path.join( self._db_dir, self._durable_temp_db_filename )
2021-08-11 21:14:12 +00:00
self._Execute( 'ATTACH ? AS durable_temp;', ( db_path, ) )
2019-09-05 00:05:32 +00:00
2016-03-30 22:56:50 +00:00
2020-08-19 22:38:20 +00:00
def _CleanAfterJobWork( self ):
2021-11-10 21:53:57 +00:00
self._cursor_transaction_wrapper.CleanPubSubs()
2020-08-19 22:38:20 +00:00
2021-08-11 21:14:12 +00:00
def _CloseDBConnection( self ):
2015-04-22 22:57:25 +00:00
2021-08-11 21:14:12 +00:00
HydrusDBBase.TemporaryIntegerTableNameCache.instance().Clear()
2020-12-02 22:04:38 +00:00
2016-02-17 22:06:47 +00:00
if self._db is not None:
2021-02-24 22:35:18 +00:00
if self._cursor_transaction_wrapper.InTransaction():
2017-03-29 19:39:34 +00:00
2021-02-24 22:35:18 +00:00
self._cursor_transaction_wrapper.Commit()
2017-03-29 19:39:34 +00:00
2021-08-11 21:14:12 +00:00
self._CloseCursor()
2016-02-17 22:06:47 +00:00
self._db.close()
del self._db
self._db = None
2021-05-27 00:09:06 +00:00
self._is_connected = False
2021-02-24 22:35:18 +00:00
self._cursor_transaction_wrapper = None
2017-03-29 19:39:34 +00:00
2021-02-24 22:35:18 +00:00
self._UnloadModules()
2017-03-29 19:39:34 +00:00
2015-04-22 22:57:25 +00:00
def _CreateDB( self ):
raise NotImplementedError()
2018-02-14 21:47:18 +00:00
def _DisplayCatastrophicError( self, text ):
message = 'The db encountered a serious error! This is going to be written to the log as well, but here it is for a screenshot:'
2024-04-10 20:36:05 +00:00
message += '\n' * 2
2018-02-14 21:47:18 +00:00
message += text
HydrusData.DebugPrint( message )
2020-08-19 22:38:20 +00:00
def _DoAfterJobWork( self ):
2021-11-10 21:53:57 +00:00
self._cursor_transaction_wrapper.DoPubSubs()
2020-08-19 22:38:20 +00:00
2020-04-08 21:10:11 +00:00
def _GenerateDBJob( self, job_type, synchronous, action, *args, **kwargs ):
return HydrusData.JobDatabase( job_type, synchronous, action, *args, **kwargs )
def _GetPossibleAdditionalDBFilenames( self ):
return [ self._ssl_cert_filename, self._ssl_key_filename ]
2015-04-22 22:57:25 +00:00
def _InitCaches( self ):
2016-03-16 22:19:14 +00:00
pass
2015-04-22 22:57:25 +00:00
def _InitDB( self ):
2023-12-13 22:29:24 +00:00
main_database_is_missing = False
2015-09-23 21:21:02 +00:00
2023-12-13 22:29:24 +00:00
main_db_path = os.path.join( self._db_dir, self._db_filenames[ 'main' ] )
external_db_paths = [ os.path.join( self._db_dir, self._db_filenames[ db_name ] ) for db_name in self._db_filenames if db_name != 'main' ]
2016-04-06 19:52:45 +00:00
2023-12-13 22:29:24 +00:00
existing_external_db_paths = [ external_db_path for external_db_path in external_db_paths if os.path.exists( external_db_path ) ]
if os.path.exists( main_db_path ):
2016-01-13 22:08:19 +00:00
2023-12-13 22:29:24 +00:00
if len( existing_external_db_paths ) < len( external_db_paths ):
external_paths_summary = '"{}"'.format( '", "'.join( [ path for path in external_db_paths if path not in existing_external_db_paths ] ) )
message = f'While the main database file, "{main_db_path}", exists, the external files {external_paths_summary} do not!\n\nIf this is a surprise to you, you have probably had a hard drive failure. You must close this process immediately and diagnose what has happened. Check the "help my db is broke.txt" document in the install_dir/db directory for additional help.\n\nIf this is not a surprise, then you may continue if you wish, and hydrus will do its best to reconstruct the missing files. You will see more error prompts.'
self._controller.BlockingSafeShowCriticalMessage( 'missing database file!', message )
2016-01-13 22:08:19 +00:00
2023-12-13 22:29:24 +00:00
else:
2019-02-27 23:03:30 +00:00
2023-12-13 22:29:24 +00:00
main_database_is_missing = True
2019-02-27 23:03:30 +00:00
if len( existing_external_db_paths ) > 0:
2023-12-13 22:29:24 +00:00
external_paths_summary = '"{}"'.format( '", "'.join( existing_external_db_paths ) )
2019-02-27 23:03:30 +00:00
2023-12-13 22:29:24 +00:00
message = f'Although the external files, {external_paths_summary} do exist, the main database file, "{main_db_path}", does not!\n\nThis makes for an invalid database, and the program will now quit. Please contact hydrus_dev if you do not know how this happened or need help recovering from hard drive failure.'
2019-02-27 23:03:30 +00:00
2020-09-16 20:46:54 +00:00
raise HydrusExceptions.DBAccessException( message )
2019-02-27 23:03:30 +00:00
2015-04-22 22:57:25 +00:00
2021-08-11 21:14:12 +00:00
self._InitDBConnection()
2015-04-22 22:57:25 +00:00
2023-12-13 22:29:24 +00:00
version_is_missing = False
2021-08-11 21:14:12 +00:00
result = self._Execute( 'SELECT 1 FROM sqlite_master WHERE type = ? AND name = ?;', ( 'table', 'version' ) ).fetchone()
2015-09-23 21:21:02 +00:00
if result is None:
2023-12-13 22:29:24 +00:00
if not main_database_is_missing:
message = f'The "version" table in your {main_db_path} database was missing.\n\nIf you have used this database many times before, then you have probably had a hard drive failure. You must close this process immediately and diagnose what has happened. Check the "help my db is broke.txt" document in the install_dir/db directory for additional help.\n\nIf this database is new, and you recently attempted to boot it for the first time, but it failed, then this is less of a worrying situation, and you can continue.'
self._controller.BlockingSafeShowCriticalMessage( 'missing critical database table!', message )
version_is_missing = True
2015-09-23 21:21:02 +00:00
2023-12-13 22:29:24 +00:00
if main_database_is_missing or version_is_missing:
2015-04-22 22:57:25 +00:00
2016-08-31 19:55:14 +00:00
self._is_first_start = True
2015-04-22 22:57:25 +00:00
self._CreateDB()
2021-02-24 22:35:18 +00:00
self._cursor_transaction_wrapper.CommitAndBegin()
2017-07-12 20:03:45 +00:00
2015-04-22 22:57:25 +00:00
2021-08-11 21:14:12 +00:00
def _InitDBConnection( self ):
2015-04-22 22:57:25 +00:00
2021-08-11 21:14:12 +00:00
self._CloseDBConnection()
2016-02-17 22:06:47 +00:00
2016-04-06 19:52:45 +00:00
db_path = os.path.join( self._db_dir, self._db_filenames[ 'main' ] )
2016-01-20 23:57:33 +00:00
2020-09-16 20:46:54 +00:00
try:
2019-03-06 23:06:22 +00:00
if os.path.exists( db_path ) and not HydrusPaths.FileisWriteable( db_path ):
raise HydrusExceptions.DBAccessException( '"{}" seems to be read-only!'.format( db_path ) )
2020-09-16 20:46:54 +00:00
self._db = sqlite3.connect( db_path, isolation_level = None, detect_types = sqlite3.PARSE_DECLTYPES )
2019-03-06 23:06:22 +00:00
2021-08-11 21:14:12 +00:00
c = self._db.cursor()
self._SetCursor( c )
2020-09-16 20:46:54 +00:00
2021-05-27 00:09:06 +00:00
self._is_connected = True
self._cursor_transaction_wrapper = HydrusDBBase.DBCursorTransactionWrapper( self._c, HG.db_transaction_commit_period )
2021-02-24 22:35:18 +00:00
2020-09-16 20:46:54 +00:00
if HG.no_db_temp_files:
2021-08-11 21:14:12 +00:00
self._Execute( 'PRAGMA temp_store = 2;' ) # use memory for temp store exclusively
2020-09-16 20:46:54 +00:00
2020-10-21 22:22:10 +00:00
2020-09-16 20:46:54 +00:00
self._AttachExternalDatabases()
self._LoadModules()
2021-08-11 21:14:12 +00:00
self._Execute( 'ATTACH ":memory:" AS mem;' )
2020-09-16 20:46:54 +00:00
except HydrusExceptions.DBAccessException as e:
raise
2020-09-16 20:46:54 +00:00
except Exception as e:
2024-04-10 20:36:05 +00:00
raise HydrusExceptions.DBAccessException( 'Could not connect to database! If the answer is not obvious to you, please let hydrus dev know. Error follows:' + '\n' * 2 + str( e ) )
2020-09-16 20:46:54 +00:00
2016-04-14 01:54:29 +00:00
2021-08-11 21:14:12 +00:00
HydrusDBBase.TemporaryIntegerTableNameCache.instance().Clear()
2019-09-11 21:51:09 +00:00
# durable_temp is not excluded here
2021-08-11 21:14:12 +00:00
db_names = [ name for ( index, name, path ) in self._Execute( 'PRAGMA database_list;' ) if name not in ( 'mem', 'temp' ) ]
2016-04-14 01:54:29 +00:00
for db_name in db_names:
2016-01-20 23:57:33 +00:00
2021-01-07 01:10:01 +00:00
# MB -> KB
cache_size = HG.db_cache_size * 1024
2020-12-23 23:07:58 +00:00
2021-08-11 21:14:12 +00:00
self._Execute( 'PRAGMA {}.cache_size = -{};'.format( db_name, cache_size ) )
2016-04-20 20:42:21 +00:00
2021-08-11 21:14:12 +00:00
self._Execute( 'PRAGMA {}.journal_mode = {};'.format( db_name, HG.db_journal_mode ) )
2020-12-09 22:18:48 +00:00
if HG.db_journal_mode in ( 'PERSIST', 'WAL' ):
2016-01-20 23:57:33 +00:00
2022-05-25 21:30:53 +00:00
# We tried 1GB here, but I have reports of larger ones that don't seem to truncate ever?
# Not sure what that is about, but I guess the db sometimes doesn't want to (expensively?) recover pages from the journal and just appends more data
# In any case, this pragma is not a 'don't allow it to grow larger than', it's a 'after commit, truncate back to this', so no need to make it so large
# default is -1, which means no limit
self._Execute( 'PRAGMA {}.journal_size_limit = {};'.format( db_name, HydrusDBBase.JOURNAL_SIZE_LIMIT ) )
2019-09-11 21:51:09 +00:00
2021-08-11 21:14:12 +00:00
self._Execute( 'PRAGMA {}.synchronous = {};'.format( db_name, HG.db_synchronous ) )
2019-09-11 21:51:09 +00:00
try:
2021-08-11 21:14:12 +00:00
self._Execute( 'SELECT * FROM {}.sqlite_master;'.format( db_name ) ).fetchone()
2016-04-14 01:54:29 +00:00
2019-09-11 21:51:09 +00:00
except sqlite3.OperationalError as e:
2016-04-14 01:54:29 +00:00
2020-12-09 22:18:48 +00:00
message = 'The database seemed valid, but hydrus failed to read basic data from it. You may need to run the program in a different journal mode using --db_journal_mode. Full error information:'
2016-01-20 23:57:33 +00:00
2024-04-10 20:36:05 +00:00
message += '\n' * 2
2019-09-11 21:51:09 +00:00
message += str( e )
HydrusData.DebugPrint( message )
raise HydrusExceptions.DBAccessException( message )
2016-01-13 22:08:19 +00:00
2015-04-22 22:57:25 +00:00
2017-07-12 20:03:45 +00:00
try:
2021-02-24 22:35:18 +00:00
self._cursor_transaction_wrapper.BeginImmediate()
2017-07-12 20:03:45 +00:00
except Exception as e:
if 'locked' in str( e ):
raise HydrusExceptions.DBAccessException( 'Database appeared to be locked. Please ensure there is not another client already running on this database, and then try restarting the client.' )
2019-01-09 22:59:03 +00:00
raise HydrusExceptions.DBAccessException( str( e ) )
2017-07-12 20:03:45 +00:00
2015-04-22 22:57:25 +00:00
2016-04-20 20:42:21 +00:00
def _InitExternalDatabases( self ):
pass
2021-01-27 22:14:03 +00:00
def _LoadModules( self ):
pass
2015-04-22 22:57:25 +00:00
def _ManageDBError( self, job, e ):
raise NotImplementedError()
def _ProcessJob( self, job ):
job_type = job.GetType()
2016-03-30 22:56:50 +00:00
( action, args, kwargs ) = job.GetCallableTuple()
2015-04-22 22:57:25 +00:00
try:
2016-04-20 20:42:21 +00:00
if job_type in ( 'read_write', 'write' ):
2017-05-24 20:28:24 +00:00
self._current_status = 'db write locked'
2021-02-24 22:35:18 +00:00
self._cursor_transaction_wrapper.NotifyWriteOccuring()
2016-04-20 20:42:21 +00:00
2017-05-24 20:28:24 +00:00
else:
self._current_status = 'db read locked'
2017-07-05 21:09:28 +00:00
self.publish_status_update()
2015-04-22 22:57:25 +00:00
2017-05-24 20:28:24 +00:00
if job_type in ( 'read', 'read_write' ):
result = self._Read( action, *args, **kwargs )
elif job_type in ( 'write' ):
result = self._Write( action, *args, **kwargs )
2015-04-22 22:57:25 +00:00
2021-06-30 21:27:35 +00:00
if job.IsSynchronous():
job.PutResult( result )
2021-07-14 20:42:19 +00:00
self._cursor_transaction_wrapper.Save()
2021-02-24 22:35:18 +00:00
if self._cursor_transaction_wrapper.TimeToCommit():
2016-04-20 20:42:21 +00:00
2017-05-24 20:28:24 +00:00
self._current_status = 'db committing'
2017-07-05 21:09:28 +00:00
self.publish_status_update()
2017-05-24 20:28:24 +00:00
2021-02-24 22:35:18 +00:00
self._cursor_transaction_wrapper.CommitAndBegin()
2019-02-27 23:03:30 +00:00
2015-04-22 22:57:25 +00:00
2020-08-19 22:38:20 +00:00
self._DoAfterJobWork()
2015-04-22 22:57:25 +00:00
except Exception as e:
2018-02-28 22:30:36 +00:00
self._ManageDBError( job, e )
2017-07-12 20:03:45 +00:00
try:
2016-04-20 20:42:21 +00:00
2021-02-24 22:35:18 +00:00
self._cursor_transaction_wrapper.Rollback()
2017-07-12 20:03:45 +00:00
except Exception as rollback_e:
2018-02-28 22:30:36 +00:00
HydrusData.Print( 'When the transaction failed, attempting to rollback the database failed. Please restart the client as soon as is convenient.' )
2021-08-11 21:14:12 +00:00
self._CloseDBConnection()
2018-02-28 22:30:36 +00:00
2021-08-11 21:14:12 +00:00
self._InitDBConnection()
2017-07-12 20:03:45 +00:00
HydrusData.PrintException( rollback_e )
2016-04-20 20:42:21 +00:00
2015-04-22 22:57:25 +00:00
2017-05-24 20:28:24 +00:00
finally:
2020-08-19 22:38:20 +00:00
self._CleanAfterJobWork()
2017-07-12 20:03:45 +00:00
2017-05-24 20:28:24 +00:00
self._current_status = ''
2017-07-05 21:09:28 +00:00
self.publish_status_update()
2017-05-24 20:28:24 +00:00
2015-04-22 22:57:25 +00:00
def _Read( self, action, *args, **kwargs ):
raise NotImplementedError()
def _RepairDB( self, version ):
2018-01-17 22:52:10 +00:00
for module in self._modules:
module.Repair( version, self._cursor_transaction_wrapper )
2018-01-17 22:52:10 +00:00
2024-05-08 20:25:53 +00:00
if HG.controller.LastShutdownWasBad():
for module in self._modules:
module.DoLastShutdownWasBadWork()
2018-01-17 22:52:10 +00:00
2018-11-28 22:31:04 +00:00
def _ReportOverupdatedDB( self, version ):
pass
2019-02-13 22:26:43 +00:00
def _ReportUnderupdatedDB( self, version ):
pass
2019-02-27 23:03:30 +00:00
def _ShrinkMemory( self ):
2021-08-11 21:14:12 +00:00
self._Execute( 'PRAGMA shrink_memory;' )
2017-03-08 23:23:12 +00:00
2021-01-27 22:14:03 +00:00
def _UnloadModules( self ):
2022-04-20 20:18:56 +00:00
self._modules = []
2021-01-27 22:14:03 +00:00
2015-04-22 22:57:25 +00:00
def _UpdateDB( self, version ):
raise NotImplementedError()
def _Write( self, action, *args, **kwargs ):
raise NotImplementedError()
2017-07-05 21:09:28 +00:00
def publish_status_update( self ):
pass
2015-06-03 21:05:13 +00:00
def CurrentlyDoingJob( self ):
return self._currently_doing_job
2017-07-12 20:03:45 +00:00
def GetApproxTotalFileSize( self ):
total = 0
2021-05-27 00:09:06 +00:00
for filename in self._db_filenames.values():
2017-07-12 20:03:45 +00:00
path = os.path.join( self._db_dir, filename )
2023-07-05 20:52:58 +00:00
if os.path.exists( path ):
total += os.path.getsize( path )
2017-07-12 20:03:45 +00:00
return total
def GetSSLPaths( self ):
# create ssl keys
cert_here = os.path.exists( self._ssl_cert_path )
key_here = os.path.exists( self._ssl_key_path )
if cert_here ^ key_here:
raise Exception( 'While creating the server database, only one of the paths "{}" and "{}" existed. You can create a db with these files already in place, but please either delete the existing file (to have hydrus generate its own pair) or find the other in the pair (to use your own).'.format( self._ssl_cert_path, self._ssl_key_path ) )
elif not ( cert_here or key_here ):
HydrusData.Print( 'Generating new cert/key files.' )
if not HydrusEncryption.CRYPTO_OK:
2021-02-11 01:59:52 +00:00
raise Exception( 'The database was asked for ssl cert and keys to start either the server or the client api in https. The files do not exist yet, so the database wanted to create new ones, but unfortunately "cryptography" library is not available, so this cannot be done. If you are running from source, please install this module using pip. Or drop in your own client.crt/client.key or server.crt/server.key files in the db directory.' )
2021-02-11 01:59:52 +00:00
HydrusEncryption.GenerateOpenSSLCertAndKeyFile( self._ssl_cert_path, self._ssl_key_path )
return ( self._ssl_cert_path, self._ssl_key_path )
2017-05-24 20:28:24 +00:00
def GetStatus( self ):
return ( self._current_status, self._current_job_name )
2021-05-27 00:09:06 +00:00
def IsConnected( self ):
return self._is_connected
2016-08-31 19:55:14 +00:00
def IsDBUpdated( self ):
return self._is_db_updated
def IsFirstStart( self ):
return self._is_first_start
2016-03-30 22:56:50 +00:00
def LoopIsFinished( self ):
return self._loop_finished
2015-04-22 22:57:25 +00:00
2016-04-06 19:52:45 +00:00
def JobsQueueEmpty( self ):
return self._jobs.empty()
2015-04-22 22:57:25 +00:00
def MainLoop( self ):
2016-03-16 22:19:14 +00:00
try:
2021-08-11 21:14:12 +00:00
self._InitDBConnection() # have to reinitialise because the thread id has changed
2016-03-16 22:19:14 +00:00
self._InitCaches()
2016-04-06 19:52:45 +00:00
except:
2016-03-16 22:19:14 +00:00
2018-02-14 21:47:18 +00:00
self._DisplayCatastrophicError( traceback.format_exc() )
2016-03-16 22:19:14 +00:00
self._could_not_initialise = True
2016-04-06 19:52:45 +00:00
return
2015-04-22 22:57:25 +00:00
2016-03-16 22:19:14 +00:00
self._ready_to_serve_requests = True
2015-04-29 19:20:35 +00:00
2015-04-22 22:57:25 +00:00
error_count = 0
2019-07-31 22:01:02 +00:00
while not ( ( self._local_shutdown or HG.model_shutdown ) and self._jobs.empty() ):
2015-04-22 22:57:25 +00:00
try:
2019-01-09 22:59:03 +00:00
job = self._jobs.get( timeout = 1 )
2015-04-22 22:57:25 +00:00
self._currently_doing_job = True
2017-05-24 20:28:24 +00:00
self._current_job_name = job.ToString()
2015-04-22 22:57:25 +00:00
2017-07-05 21:09:28 +00:00
self.publish_status_update()
2015-06-03 21:05:13 +00:00
2015-04-22 22:57:25 +00:00
try:
2018-09-05 20:52:32 +00:00
if HG.db_report_mode:
2021-07-14 20:42:19 +00:00
summary = 'Running db job: ' + job.ToString()
2018-09-05 20:52:32 +00:00
HydrusData.ShowText( summary )
2021-07-14 20:42:19 +00:00
if HG.profile_mode:
2015-04-22 22:57:25 +00:00
2021-07-14 20:42:19 +00:00
summary = 'Profiling db job: ' + job.ToString()
2017-03-08 23:23:12 +00:00
2023-04-19 20:38:13 +00:00
HydrusProfiling.Profile( summary, 'self._ProcessJob( job )', globals(), locals(), min_duration_ms = HG.db_profile_min_job_time_ms )
2015-04-22 22:57:25 +00:00
else:
self._ProcessJob( job )
error_count = 0
except:
error_count += 1
2017-08-02 21:32:54 +00:00
if error_count > 5:
raise
2015-04-22 22:57:25 +00:00
2019-01-09 22:59:03 +00:00
self._jobs.put( job ) # couldn't lock db; put job back on queue
2015-04-22 22:57:25 +00:00
time.sleep( 5 )
self._currently_doing_job = False
2017-05-24 20:28:24 +00:00
self._current_job_name = ''
2015-04-22 22:57:25 +00:00
2017-07-05 21:09:28 +00:00
self.publish_status_update()
2015-06-03 21:05:13 +00:00
2019-01-09 22:59:03 +00:00
except queue.Empty:
2016-03-30 22:56:50 +00:00
2021-02-24 22:35:18 +00:00
if self._cursor_transaction_wrapper.TimeToCommit():
2017-08-02 21:32:54 +00:00
2021-02-24 22:35:18 +00:00
self._cursor_transaction_wrapper.CommitAndBegin()
2019-02-27 23:03:30 +00:00
2016-03-30 22:56:50 +00:00
2015-06-17 20:01:41 +00:00
2019-10-09 22:03:03 +00:00
if self._pause_and_disconnect:
2021-08-11 21:14:12 +00:00
self._CloseDBConnection()
2019-10-09 22:03:03 +00:00
while self._pause_and_disconnect:
if self._local_shutdown or HG.model_shutdown:
break
time.sleep( 1 )
2021-08-11 21:14:12 +00:00
self._InitDBConnection()
2019-10-09 22:03:03 +00:00
2015-06-17 20:01:41 +00:00
2021-08-11 21:14:12 +00:00
self._CloseDBConnection()
2015-04-22 22:57:25 +00:00
2019-09-05 00:05:32 +00:00
temp_path = os.path.join( self._db_dir, self._durable_temp_db_filename )
HydrusPaths.DeletePath( temp_path )
2015-04-22 22:57:25 +00:00
self._loop_finished = True
2019-10-09 22:03:03 +00:00
def PauseAndDisconnect( self, pause_and_disconnect ):
self._pause_and_disconnect = pause_and_disconnect
2019-01-09 22:59:03 +00:00
def Read( self, action, *args, **kwargs ):
2015-04-22 22:57:25 +00:00
2019-01-09 22:59:03 +00:00
if action in self.READ_WRITE_ACTIONS:
job_type = 'read_write'
else:
job_type = 'read'
2015-04-22 22:57:25 +00:00
synchronous = True
2020-04-08 21:10:11 +00:00
job = self._GenerateDBJob( job_type, synchronous, action, *args, **kwargs )
2015-04-22 22:57:25 +00:00
2019-07-31 22:01:02 +00:00
if HG.model_shutdown:
2015-11-04 22:30:28 +00:00
raise HydrusExceptions.ShutdownException( 'Application has shut down!' )
2015-04-22 22:57:25 +00:00
2019-01-09 22:59:03 +00:00
self._jobs.put( job )
2015-04-22 22:57:25 +00:00
2016-03-16 22:19:14 +00:00
return job.GetResult()
def ReadyToServeRequests( self ):
return self._ready_to_serve_requests
2015-04-22 22:57:25 +00:00
2016-03-30 22:56:50 +00:00
def Shutdown( self ):
self._local_shutdown = True
2015-04-22 22:57:25 +00:00
2019-01-09 22:59:03 +00:00
def Write( self, action, synchronous, *args, **kwargs ):
2015-04-22 22:57:25 +00:00
2016-03-30 22:56:50 +00:00
job_type = 'write'
2015-04-22 22:57:25 +00:00
2020-04-08 21:10:11 +00:00
job = self._GenerateDBJob( job_type, synchronous, action, *args, **kwargs )
2015-04-22 22:57:25 +00:00
2019-07-31 22:01:02 +00:00
if HG.model_shutdown:
2015-11-04 22:30:28 +00:00
raise HydrusExceptions.ShutdownException( 'Application has shut down!' )
2015-04-22 22:57:25 +00:00
2019-01-09 22:59:03 +00:00
self._jobs.put( job )
2015-04-22 22:57:25 +00:00
if synchronous: return job.GetResult()
2016-09-28 18:48:01 +00:00