2015-04-01 20:44:54 +00:00
|
|
|
import collections
|
|
|
|
import gc
|
|
|
|
import HydrusConstants as HC
|
2015-08-26 21:18:39 +00:00
|
|
|
import HydrusDaemons
|
2015-04-01 20:44:54 +00:00
|
|
|
import HydrusData
|
2015-04-22 22:57:25 +00:00
|
|
|
import HydrusDB
|
2015-04-01 20:44:54 +00:00
|
|
|
import HydrusExceptions
|
|
|
|
import HydrusGlobals
|
|
|
|
import HydrusPubSub
|
2015-08-26 21:18:39 +00:00
|
|
|
import HydrusThreading
|
2016-03-09 19:37:14 +00:00
|
|
|
import os
|
2015-08-26 21:18:39 +00:00
|
|
|
import random
|
2015-04-01 20:44:54 +00:00
|
|
|
import sys
|
|
|
|
import threading
|
|
|
|
import time
|
|
|
|
import traceback
|
|
|
|
|
2015-09-02 23:16:09 +00:00
|
|
|
class HydrusController( object ):
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
pubsub_binding_errors_to_ignore = []
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2016-10-12 21:52:50 +00:00
|
|
|
def __init__( self, db_dir ):
|
2015-09-02 23:16:09 +00:00
|
|
|
|
|
|
|
HydrusGlobals.controller = self
|
|
|
|
|
2016-10-12 21:52:50 +00:00
|
|
|
self._db_dir = db_dir
|
|
|
|
|
2016-08-31 19:55:14 +00:00
|
|
|
self._db = None
|
|
|
|
|
2016-01-20 23:57:33 +00:00
|
|
|
self._no_daemons = False
|
|
|
|
self._no_wal = False
|
|
|
|
|
|
|
|
self._InitArgsBools()
|
|
|
|
|
2016-10-12 21:52:50 +00:00
|
|
|
self._no_wal_path = os.path.join( self._db_dir, 'no-wal' )
|
2016-03-09 19:37:14 +00:00
|
|
|
|
|
|
|
if os.path.exists( self._no_wal_path ):
|
|
|
|
|
|
|
|
self._no_wal = True
|
|
|
|
|
|
|
|
|
2015-11-04 22:30:28 +00:00
|
|
|
self._model_shutdown = False
|
|
|
|
self._view_shutdown = False
|
|
|
|
|
2015-09-02 23:16:09 +00:00
|
|
|
self._pubsub = HydrusPubSub.HydrusPubSub( self, self.pubsub_binding_errors_to_ignore )
|
|
|
|
|
|
|
|
self._currently_doing_pubsub = False
|
|
|
|
|
|
|
|
self._daemons = []
|
|
|
|
self._caches = {}
|
|
|
|
self._managers = {}
|
|
|
|
|
2016-07-20 19:57:10 +00:00
|
|
|
self._call_to_threads = []
|
2015-09-02 23:16:09 +00:00
|
|
|
|
|
|
|
self._timestamps = collections.defaultdict( lambda: 0 )
|
|
|
|
|
|
|
|
self._timestamps[ 'boot' ] = HydrusData.GetNow()
|
|
|
|
|
|
|
|
self._just_woke_from_sleep = False
|
|
|
|
self._system_busy = False
|
|
|
|
|
|
|
|
|
2016-07-20 19:57:10 +00:00
|
|
|
def _GetCallToThread( self ):
|
|
|
|
|
|
|
|
for call_to_thread in self._call_to_threads:
|
|
|
|
|
|
|
|
if not call_to_thread.CurrentlyWorking():
|
|
|
|
|
|
|
|
return call_to_thread
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if len( self._call_to_threads ) > 100:
|
|
|
|
|
|
|
|
raise Exception( 'Too many call to threads!' )
|
|
|
|
|
|
|
|
|
|
|
|
call_to_thread = HydrusThreading.THREADCallToThread( self )
|
|
|
|
|
|
|
|
self._call_to_threads.append( call_to_thread )
|
|
|
|
|
|
|
|
call_to_thread.start()
|
|
|
|
|
|
|
|
return call_to_thread
|
|
|
|
|
|
|
|
|
2016-01-20 23:57:33 +00:00
|
|
|
def _InitArgsBools( self ):
|
|
|
|
|
|
|
|
args = sys.argv[1:]
|
|
|
|
|
|
|
|
for arg in args:
|
|
|
|
|
|
|
|
while arg.startswith( '-' ):
|
|
|
|
|
|
|
|
arg = arg[ 1: ]
|
|
|
|
|
|
|
|
|
|
|
|
if arg in ( 'no-daemon', 'no-daemons' ):
|
|
|
|
|
|
|
|
self._no_daemons = True
|
|
|
|
|
|
|
|
|
|
|
|
if arg == 'no-wal':
|
|
|
|
|
|
|
|
self._no_wal = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-09-02 23:16:09 +00:00
|
|
|
def _InitDB( self ):
|
|
|
|
|
|
|
|
raise NotImplementedError()
|
|
|
|
|
|
|
|
|
2015-04-01 20:44:54 +00:00
|
|
|
def _Read( self, action, *args, **kwargs ):
|
|
|
|
|
|
|
|
result = self._db.Read( action, HC.HIGH_PRIORITY, *args, **kwargs )
|
|
|
|
|
|
|
|
return result
|
|
|
|
|
|
|
|
|
2015-09-16 18:11:00 +00:00
|
|
|
def _ShutdownDaemons( self ):
|
|
|
|
|
|
|
|
for daemon in self._daemons:
|
|
|
|
|
|
|
|
daemon.shutdown()
|
|
|
|
|
|
|
|
|
|
|
|
while True in ( daemon.is_alive() for daemon in self._daemons ):
|
|
|
|
|
|
|
|
time.sleep( 0.1 )
|
|
|
|
|
|
|
|
|
|
|
|
self._daemons = []
|
|
|
|
|
|
|
|
|
2015-04-01 20:44:54 +00:00
|
|
|
def _Write( self, action, priority, synchronous, *args, **kwargs ):
|
|
|
|
|
|
|
|
result = self._db.Write( action, priority, synchronous, *args, **kwargs )
|
|
|
|
|
|
|
|
return result
|
|
|
|
|
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
def pub( self, topic, *args, **kwargs ):
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
self._pubsub.pub( topic, *args, **kwargs )
|
2015-04-01 20:44:54 +00:00
|
|
|
|
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
def pubimmediate( self, topic, *args, **kwargs ):
|
|
|
|
|
|
|
|
self._pubsub.pubimmediate( topic, *args, **kwargs )
|
|
|
|
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
def sub( self, object, method_name, topic ):
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
self._pubsub.sub( object, method_name, topic )
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
|
|
|
|
def CallToThread( self, callable, *args, **kwargs ):
|
|
|
|
|
2016-07-20 19:57:10 +00:00
|
|
|
call_to_thread = self._GetCallToThread()
|
2015-08-26 21:18:39 +00:00
|
|
|
|
|
|
|
call_to_thread.put( callable, *args, **kwargs )
|
|
|
|
|
|
|
|
|
|
|
|
def ClearCaches( self ):
|
|
|
|
|
|
|
|
for cache in self._caches.values(): cache.Clear()
|
|
|
|
|
|
|
|
|
2016-03-09 19:37:14 +00:00
|
|
|
def CreateNoWALFile( self ):
|
|
|
|
|
|
|
|
with open( self._no_wal_path, 'wb' ) as f:
|
|
|
|
|
|
|
|
f.write( 'This file was created because the database failed to set WAL journalling. It will not reattempt WAL as long as this file exists.' )
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
def CurrentlyIdle( self ): return True
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2016-08-31 19:55:14 +00:00
|
|
|
def DBCurrentlyDoingJob( self ):
|
|
|
|
|
|
|
|
if self._db is None:
|
|
|
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
return self._db.CurrentlyDoingJob()
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-04-01 20:44:54 +00:00
|
|
|
def GetCache( self, name ): return self._caches[ name ]
|
|
|
|
|
|
|
|
def GetManager( self, name ): return self._managers[ name ]
|
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
def GoodTimeToDoBackgroundWork( self ):
|
|
|
|
|
|
|
|
return not ( self.JustWokeFromSleep() or self.SystemBusy() )
|
|
|
|
|
|
|
|
|
2015-04-01 20:44:54 +00:00
|
|
|
def JustWokeFromSleep( self ):
|
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
self.SleepCheck()
|
2015-04-01 20:44:54 +00:00
|
|
|
|
|
|
|
return self._just_woke_from_sleep
|
|
|
|
|
|
|
|
|
2015-09-02 23:16:09 +00:00
|
|
|
def InitModel( self ):
|
2015-08-26 21:18:39 +00:00
|
|
|
|
2015-09-02 23:16:09 +00:00
|
|
|
self._db = self._InitDB()
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
|
2015-09-02 23:16:09 +00:00
|
|
|
def InitView( self ):
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2016-01-20 23:57:33 +00:00
|
|
|
if not self._no_daemons:
|
|
|
|
|
|
|
|
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'SleepCheck', HydrusDaemons.DAEMONSleepCheck, period = 120 ) )
|
|
|
|
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'MaintainMemory', HydrusDaemons.DAEMONMaintainMemory, period = 300 ) )
|
|
|
|
|
2016-02-17 22:06:47 +00:00
|
|
|
self._daemons.append( HydrusThreading.DAEMONBigJobWorker( self, 'MaintainDB', HydrusDaemons.DAEMONMaintainDB, period = 300 ) )
|
|
|
|
|
2015-04-01 20:44:54 +00:00
|
|
|
|
|
|
|
|
2016-08-31 19:55:14 +00:00
|
|
|
def IsFirstStart( self ):
|
|
|
|
|
|
|
|
if self._db is None:
|
|
|
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
return self._db.IsFirstStart()
|
|
|
|
|
|
|
|
|
|
|
|
|
2016-05-11 18:16:39 +00:00
|
|
|
def MaintainDB( self, stop_time = None ):
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2015-09-02 23:16:09 +00:00
|
|
|
pass
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
|
|
|
|
def MaintainMemory( self ):
|
|
|
|
|
|
|
|
sys.stdout.flush()
|
|
|
|
sys.stderr.flush()
|
|
|
|
|
|
|
|
gc.collect()
|
|
|
|
|
|
|
|
|
2015-11-04 22:30:28 +00:00
|
|
|
def ModelIsShutdown( self ):
|
|
|
|
|
|
|
|
return self._model_shutdown
|
|
|
|
|
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
def NotifyPubSubs( self ):
|
|
|
|
|
|
|
|
raise NotImplementedError()
|
|
|
|
|
|
|
|
|
|
|
|
def ProcessPubSub( self ):
|
|
|
|
|
|
|
|
self._currently_doing_pubsub = True
|
|
|
|
|
|
|
|
try: self._pubsub.Process()
|
|
|
|
finally: self._currently_doing_pubsub = False
|
2015-04-01 20:44:54 +00:00
|
|
|
|
|
|
|
|
|
|
|
def Read( self, action, *args, **kwargs ): return self._Read( action, *args, **kwargs )
|
|
|
|
|
2015-09-02 23:16:09 +00:00
|
|
|
def ShutdownModel( self ):
|
2015-08-05 18:42:35 +00:00
|
|
|
|
2015-11-04 22:30:28 +00:00
|
|
|
self._model_shutdown = True
|
2015-09-02 23:16:09 +00:00
|
|
|
HydrusGlobals.model_shutdown = True
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2016-01-13 22:08:19 +00:00
|
|
|
if self._db is not None:
|
|
|
|
|
|
|
|
while not self._db.LoopIsFinished(): time.sleep( 0.1 )
|
|
|
|
|
2015-04-01 20:44:54 +00:00
|
|
|
|
|
|
|
|
2015-09-02 23:16:09 +00:00
|
|
|
def ShutdownView( self ):
|
|
|
|
|
2015-11-04 22:30:28 +00:00
|
|
|
self._view_shutdown = True
|
2015-09-02 23:16:09 +00:00
|
|
|
HydrusGlobals.view_shutdown = True
|
|
|
|
|
2015-09-16 18:11:00 +00:00
|
|
|
self._ShutdownDaemons()
|
2015-09-02 23:16:09 +00:00
|
|
|
|
2015-09-16 18:11:00 +00:00
|
|
|
|
|
|
|
def ShutdownFromServer( self ):
|
|
|
|
|
|
|
|
raise Exception( 'This hydrus application cannot be shut down from the server!' )
|
2015-09-02 23:16:09 +00:00
|
|
|
|
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
def SleepCheck( self ):
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
if HydrusData.TimeHasPassed( self._timestamps[ 'now_awake' ] ):
|
|
|
|
|
|
|
|
last_sleep_check = self._timestamps[ 'last_sleep_check' ]
|
|
|
|
|
|
|
|
if last_sleep_check == 0:
|
|
|
|
|
|
|
|
self._just_woke_from_sleep = False
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
if HydrusData.TimeHasPassed( last_sleep_check + 600 ):
|
|
|
|
|
|
|
|
self._just_woke_from_sleep = True
|
|
|
|
|
|
|
|
self._timestamps[ 'now_awake' ] = HydrusData.GetNow() + 180
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
self._just_woke_from_sleep = False
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
self._timestamps[ 'last_sleep_check' ] = HydrusData.GetNow()
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
|
|
|
|
def SystemBusy( self ):
|
2015-04-01 20:44:54 +00:00
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
return self._system_busy
|
2015-04-01 20:44:54 +00:00
|
|
|
|
|
|
|
|
2015-11-04 22:30:28 +00:00
|
|
|
def ViewIsShutdown( self ):
|
|
|
|
|
|
|
|
return self._view_shutdown
|
|
|
|
|
|
|
|
|
2015-08-26 21:18:39 +00:00
|
|
|
def WaitUntilPubSubsEmpty( self ):
|
2015-04-01 20:44:54 +00:00
|
|
|
|
|
|
|
while True:
|
|
|
|
|
2015-12-30 23:44:09 +00:00
|
|
|
if self._view_shutdown:
|
|
|
|
|
|
|
|
raise HydrusExceptions.ShutdownException( 'Application shutting down!' )
|
|
|
|
|
|
|
|
elif self._pubsub.NoJobsQueued() and not self._currently_doing_pubsub:
|
|
|
|
|
|
|
|
return
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
time.sleep( 0.00001 )
|
|
|
|
|
2015-04-01 20:44:54 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def Write( self, action, *args, **kwargs ):
|
|
|
|
|
|
|
|
return self._Write( action, HC.HIGH_PRIORITY, False, *args, **kwargs )
|
|
|
|
|
|
|
|
|
2016-04-20 20:42:21 +00:00
|
|
|
def WriteInterruptable( self, action, *args, **kwargs ):
|
|
|
|
|
|
|
|
return self._Write( action, HC.INTERRUPTABLE_PRIORITY, True, *args, **kwargs )
|
|
|
|
|
|
|
|
|
2015-04-01 20:44:54 +00:00
|
|
|
def WriteSynchronous( self, action, *args, **kwargs ):
|
|
|
|
|
|
|
|
return self._Write( action, HC.LOW_PRIORITY, True, *args, **kwargs )
|
|
|
|
|
|
|
|
|