963 lines
23 KiB
Python
963 lines
23 KiB
Python
import bisect
|
|
import collections
|
|
import os
|
|
import queue
|
|
import random
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import traceback
|
|
|
|
from hydrus.core import HydrusData
|
|
from hydrus.core import HydrusExceptions
|
|
from hydrus.core import HydrusGlobals as HG
|
|
|
|
NEXT_THREAD_CLEAROUT = 0
|
|
|
|
THREADS_TO_THREAD_INFO = {}
|
|
THREAD_INFO_LOCK = threading.Lock()
|
|
|
|
def CheckIfThreadShuttingDown():
|
|
|
|
if IsThreadShuttingDown():
|
|
|
|
raise HydrusExceptions.ShutdownException( 'Thread is shutting down!' )
|
|
|
|
|
|
def ClearOutDeadThreads():
|
|
|
|
with THREAD_INFO_LOCK:
|
|
|
|
all_threads = list( THREADS_TO_THREAD_INFO.keys() )
|
|
|
|
for thread in all_threads:
|
|
|
|
if not thread.is_alive():
|
|
|
|
del THREADS_TO_THREAD_INFO[ thread ]
|
|
|
|
|
|
|
|
|
|
def GetThreadInfo( thread = None ):
|
|
|
|
global NEXT_THREAD_CLEAROUT
|
|
|
|
if HydrusData.TimeHasPassed( NEXT_THREAD_CLEAROUT ):
|
|
|
|
ClearOutDeadThreads()
|
|
|
|
NEXT_THREAD_CLEAROUT = HydrusData.GetNow() + 600
|
|
|
|
|
|
if thread is None:
|
|
|
|
thread = threading.current_thread()
|
|
|
|
|
|
with THREAD_INFO_LOCK:
|
|
|
|
if thread not in THREADS_TO_THREAD_INFO:
|
|
|
|
thread_info = {}
|
|
|
|
thread_info[ 'shutting_down' ] = False
|
|
|
|
THREADS_TO_THREAD_INFO[ thread ] = thread_info
|
|
|
|
|
|
return THREADS_TO_THREAD_INFO[ thread ]
|
|
|
|
|
|
def IsThreadShuttingDown():
|
|
|
|
if HG.controller.DoingFastExit():
|
|
|
|
return True
|
|
|
|
|
|
me = threading.current_thread()
|
|
|
|
if isinstance( me, DAEMON ):
|
|
|
|
if HG.view_shutdown:
|
|
|
|
return True
|
|
|
|
|
|
else:
|
|
|
|
if HG.model_shutdown:
|
|
|
|
return True
|
|
|
|
|
|
|
|
thread_info = GetThreadInfo()
|
|
|
|
return thread_info[ 'shutting_down' ]
|
|
|
|
def ShutdownThread( thread ):
|
|
|
|
thread_info = GetThreadInfo( thread )
|
|
|
|
thread_info[ 'shutting_down' ] = True
|
|
|
|
def SubprocessCommunicate( process: subprocess.Popen ):
|
|
|
|
def do_test():
|
|
|
|
if HG.model_shutdown:
|
|
|
|
try:
|
|
|
|
process.kill()
|
|
|
|
except:
|
|
|
|
pass
|
|
|
|
|
|
raise HydrusExceptions.ShutdownException( 'Application is shutting down!' )
|
|
|
|
|
|
|
|
do_test()
|
|
|
|
while True:
|
|
|
|
try:
|
|
|
|
return process.communicate( timeout = 10 )
|
|
|
|
except subprocess.TimeoutExpired:
|
|
|
|
do_test()
|
|
|
|
|
|
|
|
class DAEMON( threading.Thread ):
|
|
|
|
def __init__( self, controller, name ):
|
|
|
|
threading.Thread.__init__( self, name = name )
|
|
|
|
self._controller = controller
|
|
self._name = name
|
|
|
|
self._event = threading.Event()
|
|
|
|
self._controller.sub( self, 'wake', 'wake_daemons' )
|
|
self._controller.sub( self, 'shutdown', 'shutdown' )
|
|
|
|
|
|
def _DoPreCall( self ):
|
|
|
|
if HG.daemon_report_mode:
|
|
|
|
HydrusData.ShowText( self._name + ' doing a job.' )
|
|
|
|
|
|
|
|
def GetCurrentJobSummary( self ):
|
|
|
|
return 'unknown job'
|
|
|
|
|
|
def GetName( self ):
|
|
|
|
return self._name
|
|
|
|
|
|
def shutdown( self ):
|
|
|
|
ShutdownThread( self )
|
|
|
|
self.wake()
|
|
|
|
|
|
def wake( self ):
|
|
|
|
self._event.set()
|
|
|
|
|
|
class DAEMONWorker( DAEMON ):
|
|
|
|
def __init__( self, controller, name, callable, topics = None, period = 3600, init_wait = 3, pre_call_wait = 0 ):
|
|
|
|
if topics is None:
|
|
|
|
topics = []
|
|
|
|
|
|
DAEMON.__init__( self, controller, name )
|
|
|
|
self._callable = callable
|
|
self._topics = topics
|
|
self._period = period
|
|
self._init_wait = init_wait
|
|
self._pre_call_wait = pre_call_wait
|
|
|
|
for topic in topics:
|
|
|
|
self._controller.sub( self, 'set', topic )
|
|
|
|
|
|
self.start()
|
|
|
|
|
|
def _CanStart( self ):
|
|
|
|
return self._ControllerIsOKWithIt()
|
|
|
|
|
|
def _ControllerIsOKWithIt( self ):
|
|
|
|
return True
|
|
|
|
|
|
def _DoAWait( self, wait_time, event_can_wake = True ):
|
|
|
|
time_to_start = HydrusData.GetNow() + wait_time
|
|
|
|
while not HydrusData.TimeHasPassed( time_to_start ):
|
|
|
|
if event_can_wake:
|
|
|
|
event_was_set = self._event.wait( 1.0 )
|
|
|
|
if event_was_set:
|
|
|
|
self._event.clear()
|
|
|
|
return
|
|
|
|
|
|
else:
|
|
|
|
time.sleep( 1.0 )
|
|
|
|
|
|
CheckIfThreadShuttingDown()
|
|
|
|
|
|
|
|
def _WaitUntilCanStart( self ):
|
|
|
|
while not self._CanStart():
|
|
|
|
time.sleep( 1.0 )
|
|
|
|
CheckIfThreadShuttingDown()
|
|
|
|
|
|
|
|
def GetCurrentJobSummary( self ):
|
|
|
|
return self._callable
|
|
|
|
|
|
def run( self ):
|
|
|
|
try:
|
|
|
|
self._DoAWait( self._init_wait )
|
|
|
|
while True:
|
|
|
|
CheckIfThreadShuttingDown()
|
|
|
|
self._DoAWait( self._pre_call_wait, event_can_wake = False )
|
|
|
|
CheckIfThreadShuttingDown()
|
|
|
|
self._WaitUntilCanStart()
|
|
|
|
CheckIfThreadShuttingDown()
|
|
|
|
self._DoPreCall()
|
|
|
|
try:
|
|
|
|
self._callable( self._controller )
|
|
|
|
except HydrusExceptions.ShutdownException:
|
|
|
|
return
|
|
|
|
except Exception as e:
|
|
|
|
HydrusData.ShowText( 'Daemon ' + self._name + ' encountered an exception:' )
|
|
|
|
HydrusData.ShowException( e )
|
|
|
|
|
|
self._DoAWait( self._period )
|
|
|
|
|
|
except HydrusExceptions.ShutdownException:
|
|
|
|
return
|
|
|
|
|
|
|
|
def set( self, *args, **kwargs ):
|
|
|
|
self._event.set()
|
|
|
|
|
|
# Big stuff like DB maintenance that we don't want to run while other important stuff is going on, like user interaction or vidya on another process
|
|
class DAEMONBackgroundWorker( DAEMONWorker ):
|
|
|
|
def _ControllerIsOKWithIt( self ):
|
|
|
|
return self._controller.GoodTimeToStartBackgroundWork()
|
|
|
|
|
|
# Big stuff that we want to run when the user sees, but not at the expense of something else, like laggy session load
|
|
class DAEMONForegroundWorker( DAEMONWorker ):
|
|
|
|
def _ControllerIsOKWithIt( self ):
|
|
|
|
return self._controller.GoodTimeToStartForegroundWork()
|
|
|
|
|
|
class THREADCallToThread( DAEMON ):
|
|
|
|
def __init__( self, controller, name ):
|
|
|
|
DAEMON.__init__( self, controller, name )
|
|
|
|
self._callable = None
|
|
|
|
self._queue = queue.Queue()
|
|
|
|
self._currently_working = True # start off true so new threads aren't used twice by two quick successive calls
|
|
|
|
|
|
def CurrentlyWorking( self ):
|
|
|
|
return self._currently_working
|
|
|
|
|
|
def GetCurrentJobSummary( self ):
|
|
|
|
return self._callable
|
|
|
|
|
|
def put( self, callable, *args, **kwargs ):
|
|
|
|
self._currently_working = True
|
|
|
|
self._queue.put( ( callable, args, kwargs ) )
|
|
|
|
self._event.set()
|
|
|
|
|
|
def run( self ):
|
|
|
|
try:
|
|
|
|
while True:
|
|
|
|
while self._queue.empty():
|
|
|
|
CheckIfThreadShuttingDown()
|
|
|
|
self._event.wait( 10.0 )
|
|
|
|
self._event.clear()
|
|
|
|
|
|
CheckIfThreadShuttingDown()
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
( callable, args, kwargs ) = self._queue.get( 1.0 )
|
|
|
|
except queue.Empty:
|
|
|
|
# https://github.com/hydrusnetwork/hydrus/issues/750
|
|
# this shouldn't happen, but...
|
|
# even if we assume we'll never get this, we don't want to make a business of hanging forever on things
|
|
|
|
continue
|
|
|
|
|
|
self._DoPreCall()
|
|
|
|
self._callable = ( callable, args, kwargs )
|
|
|
|
if HG.profile_mode:
|
|
|
|
summary = 'Profiling CallTo Job: {}'.format( callable )
|
|
|
|
HydrusData.Profile( summary, 'callable( *args, **kwargs )', globals(), locals(), min_duration_ms = HG.callto_profile_min_job_time_ms )
|
|
|
|
else:
|
|
|
|
callable( *args, **kwargs )
|
|
|
|
|
|
self._callable = None
|
|
|
|
del callable
|
|
|
|
except HydrusExceptions.ShutdownException:
|
|
|
|
return
|
|
|
|
except Exception as e:
|
|
|
|
HydrusData.Print( traceback.format_exc() )
|
|
|
|
HydrusData.ShowException( e )
|
|
|
|
finally:
|
|
|
|
self._currently_working = False
|
|
|
|
|
|
time.sleep( 0.00001 )
|
|
|
|
|
|
except HydrusExceptions.ShutdownException:
|
|
|
|
return
|
|
|
|
|
|
|
|
class JobScheduler( threading.Thread ):
|
|
|
|
def __init__( self, controller ):
|
|
|
|
threading.Thread.__init__( self, name = 'Job Scheduler' )
|
|
|
|
self._controller = controller
|
|
|
|
self._waiting = []
|
|
|
|
self._waiting_lock = threading.Lock()
|
|
|
|
self._new_job_arrived = threading.Event()
|
|
|
|
self._current_job = None
|
|
|
|
self._cancel_filter_needed = threading.Event()
|
|
self._sort_needed = threading.Event()
|
|
|
|
self._controller.sub( self, 'shutdown', 'shutdown' )
|
|
|
|
|
|
def _FilterCancelled( self ):
|
|
|
|
with self._waiting_lock:
|
|
|
|
self._waiting = [ job for job in self._waiting if not job.IsCancelled() ]
|
|
|
|
|
|
|
|
def _GetLoopWaitTime( self ):
|
|
|
|
with self._waiting_lock:
|
|
|
|
if len( self._waiting ) == 0:
|
|
|
|
return 0.2
|
|
|
|
|
|
next_job = self._waiting[0]
|
|
|
|
|
|
time_delta_until_due = next_job.GetTimeDeltaUntilDue()
|
|
|
|
return min( 1.0, time_delta_until_due )
|
|
|
|
|
|
def _NoWorkToStart( self ):
|
|
|
|
with self._waiting_lock:
|
|
|
|
if len( self._waiting ) == 0:
|
|
|
|
return True
|
|
|
|
|
|
next_job = self._waiting[0]
|
|
|
|
|
|
if next_job.IsDue():
|
|
|
|
return False
|
|
|
|
else:
|
|
|
|
return True
|
|
|
|
|
|
|
|
def _SortWaiting( self ):
|
|
|
|
# sort the waiting jobs in ascending order of expected work time
|
|
|
|
with self._waiting_lock: # this uses __lt__ to sort
|
|
|
|
self._waiting.sort()
|
|
|
|
|
|
|
|
def _StartWork( self ):
|
|
|
|
jobs_started = 0
|
|
|
|
while True:
|
|
|
|
with self._waiting_lock:
|
|
|
|
if len( self._waiting ) == 0:
|
|
|
|
break
|
|
|
|
|
|
if jobs_started >= 10: # try to avoid spikes
|
|
|
|
break
|
|
|
|
|
|
next_job = self._waiting[0]
|
|
|
|
if not next_job.IsDue():
|
|
|
|
# front is not due, so nor is the rest of the list
|
|
break
|
|
|
|
|
|
next_job = self._waiting.pop( 0 )
|
|
|
|
|
|
if next_job.IsCancelled():
|
|
|
|
continue
|
|
|
|
|
|
if next_job.SlotOK():
|
|
|
|
# important this happens outside of the waiting lock lmao!
|
|
next_job.StartWork()
|
|
|
|
jobs_started += 1
|
|
|
|
else:
|
|
|
|
# delay is automatically set by SlotOK
|
|
|
|
with self._waiting_lock:
|
|
|
|
bisect.insort( self._waiting, next_job )
|
|
|
|
|
|
|
|
|
|
|
|
def AddJob( self, job ):
|
|
|
|
with self._waiting_lock:
|
|
|
|
bisect.insort( self._waiting, job )
|
|
|
|
|
|
self._new_job_arrived.set()
|
|
|
|
|
|
def ClearOutDead( self ):
|
|
|
|
with self._waiting_lock:
|
|
|
|
self._waiting = [ job for job in self._waiting if not job.IsDead() ]
|
|
|
|
|
|
|
|
def GetName( self ):
|
|
|
|
return 'Job Scheduler'
|
|
|
|
|
|
def GetCurrentJobSummary( self ):
|
|
|
|
with self._waiting_lock:
|
|
|
|
return HydrusData.ToHumanInt( len( self._waiting ) ) + ' jobs'
|
|
|
|
|
|
|
|
def GetJobs( self ):
|
|
|
|
with self._waiting_lock:
|
|
|
|
return list( self._waiting )
|
|
|
|
|
|
|
|
def GetPrettyJobSummary( self ):
|
|
|
|
with self._waiting_lock:
|
|
|
|
num_jobs = len( self._waiting )
|
|
|
|
job_lines = [ repr( job ) for job in self._waiting ]
|
|
|
|
lines = [ HydrusData.ToHumanInt( num_jobs ) + ' jobs:' ] + job_lines
|
|
|
|
text = os.linesep.join( lines )
|
|
|
|
return text
|
|
|
|
|
|
|
|
def JobCancelled( self ):
|
|
|
|
self._cancel_filter_needed.set()
|
|
|
|
|
|
def shutdown( self ):
|
|
|
|
ShutdownThread( self )
|
|
|
|
self._new_job_arrived.set()
|
|
|
|
|
|
def WorkTimesHaveChanged( self ):
|
|
|
|
self._sort_needed.set()
|
|
|
|
|
|
def run( self ):
|
|
|
|
while True:
|
|
|
|
try:
|
|
|
|
while self._NoWorkToStart():
|
|
|
|
if IsThreadShuttingDown():
|
|
|
|
return
|
|
|
|
|
|
#
|
|
|
|
if self._cancel_filter_needed.is_set():
|
|
|
|
self._FilterCancelled()
|
|
|
|
self._cancel_filter_needed.clear()
|
|
|
|
|
|
if self._sort_needed.is_set():
|
|
|
|
self._SortWaiting()
|
|
|
|
self._sort_needed.clear()
|
|
|
|
continue # if some work is now due, let's do it!
|
|
|
|
|
|
#
|
|
|
|
wait_time = self._GetLoopWaitTime()
|
|
|
|
self._new_job_arrived.wait( wait_time )
|
|
|
|
self._new_job_arrived.clear()
|
|
|
|
|
|
self._StartWork()
|
|
|
|
except HydrusExceptions.ShutdownException:
|
|
|
|
return
|
|
|
|
except Exception as e:
|
|
|
|
HydrusData.Print( traceback.format_exc() )
|
|
|
|
HydrusData.ShowException( e )
|
|
|
|
|
|
time.sleep( 0.00001 )
|
|
|
|
|
|
|
|
class SchedulableJob( object ):
|
|
|
|
PRETTY_CLASS_NAME = 'job base'
|
|
|
|
def __init__( self, controller, scheduler: JobScheduler, initial_delay, work_callable ):
|
|
|
|
self._controller = controller
|
|
self._scheduler = scheduler
|
|
self._work_callable = work_callable
|
|
|
|
self._should_delay_on_wakeup = False
|
|
|
|
self._next_work_time = HydrusData.GetNowFloat() + initial_delay
|
|
|
|
self._thread_slot_type = None
|
|
|
|
self._work_lock = threading.Lock()
|
|
|
|
self._currently_working = threading.Event()
|
|
self._is_cancelled = threading.Event()
|
|
|
|
|
|
def __lt__( self, other ): # for the scheduler to do bisect.insort noice
|
|
|
|
return self._next_work_time < other._next_work_time
|
|
|
|
|
|
def __repr__( self ):
|
|
|
|
return '{}: {} {}'.format( self.PRETTY_CLASS_NAME, self.GetPrettyJob(), self.GetDueString() )
|
|
|
|
|
|
def _BootWorker( self ):
|
|
|
|
self._controller.CallToThread( self.Work )
|
|
|
|
|
|
def Cancel( self ):
|
|
|
|
self._is_cancelled.set()
|
|
|
|
self._scheduler.JobCancelled()
|
|
|
|
|
|
def CurrentlyWorking( self ):
|
|
|
|
return self._currently_working.is_set()
|
|
|
|
|
|
def GetDueString( self ):
|
|
|
|
due_delta = self._next_work_time - HydrusData.GetNowFloat()
|
|
|
|
due_string = HydrusData.TimeDeltaToPrettyTimeDelta( due_delta )
|
|
|
|
if due_delta < 0:
|
|
|
|
due_string = 'was due {} ago'.format( due_string )
|
|
|
|
else:
|
|
|
|
due_string = 'due in {}'.format( due_string )
|
|
|
|
|
|
return due_string
|
|
|
|
|
|
def GetNextWorkTime( self ):
|
|
|
|
return self._next_work_time
|
|
|
|
|
|
def GetPrettyJob( self ):
|
|
|
|
return repr( self._work_callable )
|
|
|
|
|
|
def GetTimeDeltaUntilDue( self ):
|
|
|
|
return HydrusData.GetTimeDeltaUntilTimeFloat( self._next_work_time )
|
|
|
|
|
|
def IsCancelled( self ):
|
|
|
|
return self._is_cancelled.is_set()
|
|
|
|
|
|
def IsDead( self ):
|
|
|
|
return False
|
|
|
|
|
|
def IsDue( self ):
|
|
|
|
return HydrusData.TimeHasPassedFloat( self._next_work_time )
|
|
|
|
|
|
def PubSubWake( self, *args, **kwargs ):
|
|
|
|
self.Wake()
|
|
|
|
|
|
def SetThreadSlotType( self, thread_type ):
|
|
|
|
self._thread_slot_type = thread_type
|
|
|
|
|
|
def ShouldDelayOnWakeup( self, value ):
|
|
|
|
self._should_delay_on_wakeup = value
|
|
|
|
|
|
def SlotOK( self ):
|
|
|
|
if self._thread_slot_type is not None:
|
|
|
|
if HG.controller.AcquireThreadSlot( self._thread_slot_type ):
|
|
|
|
return True
|
|
|
|
else:
|
|
|
|
self._next_work_time = HydrusData.GetNowFloat() + 10 + random.random()
|
|
|
|
return False
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
|
def StartWork( self ):
|
|
|
|
if self._is_cancelled.is_set():
|
|
|
|
return
|
|
|
|
|
|
self._currently_working.set()
|
|
|
|
self._BootWorker()
|
|
|
|
|
|
def Wake( self, next_work_time = None ):
|
|
|
|
if next_work_time is None:
|
|
|
|
next_work_time = HydrusData.GetNowFloat()
|
|
|
|
|
|
self._next_work_time = next_work_time
|
|
|
|
self._scheduler.WorkTimesHaveChanged()
|
|
|
|
|
|
def WakeOnPubSub( self, topic ):
|
|
|
|
HG.controller.sub( self, 'PubSubWake', topic )
|
|
|
|
|
|
def Work( self ):
|
|
|
|
try:
|
|
|
|
if self._should_delay_on_wakeup:
|
|
|
|
while HG.controller.JustWokeFromSleep():
|
|
|
|
if IsThreadShuttingDown():
|
|
|
|
return
|
|
|
|
|
|
time.sleep( 1 )
|
|
|
|
|
|
|
|
with self._work_lock:
|
|
|
|
self._work_callable()
|
|
|
|
|
|
finally:
|
|
|
|
if self._thread_slot_type is not None:
|
|
|
|
HG.controller.ReleaseThreadSlot( self._thread_slot_type )
|
|
|
|
|
|
self._currently_working.clear()
|
|
|
|
|
|
|
|
class SingleJob( SchedulableJob ):
|
|
|
|
PRETTY_CLASS_NAME = 'single job'
|
|
|
|
def __init__( self, controller, scheduler: JobScheduler, initial_delay, work_callable ):
|
|
|
|
SchedulableJob.__init__( self, controller, scheduler, initial_delay, work_callable )
|
|
|
|
self._work_complete = threading.Event()
|
|
|
|
|
|
def IsWorkComplete( self ):
|
|
|
|
return self._work_complete.is_set()
|
|
|
|
|
|
def Work( self ):
|
|
|
|
SchedulableJob.Work( self )
|
|
|
|
self._work_complete.set()
|
|
|
|
|
|
class RepeatingJob( SchedulableJob ):
|
|
|
|
PRETTY_CLASS_NAME = 'repeating job'
|
|
|
|
def __init__( self, controller, scheduler: JobScheduler, initial_delay, period, work_callable ):
|
|
|
|
SchedulableJob.__init__( self, controller, scheduler, initial_delay, work_callable )
|
|
|
|
self._period = period
|
|
|
|
self._stop_repeating = threading.Event()
|
|
|
|
|
|
def Cancel( self ):
|
|
|
|
SchedulableJob.Cancel( self )
|
|
|
|
self._stop_repeating.set()
|
|
|
|
|
|
def Delay( self, delay ):
|
|
|
|
self._next_work_time = HydrusData.GetNowFloat() + delay
|
|
|
|
self._scheduler.WorkTimesHaveChanged()
|
|
|
|
|
|
def IsRepeatingWorkFinished( self ):
|
|
|
|
return self._stop_repeating.is_set()
|
|
|
|
|
|
def StartWork( self ):
|
|
|
|
if self._stop_repeating.is_set():
|
|
|
|
return
|
|
|
|
|
|
SchedulableJob.StartWork( self )
|
|
|
|
|
|
def Work( self ):
|
|
|
|
SchedulableJob.Work( self )
|
|
|
|
if not self._stop_repeating.is_set():
|
|
|
|
self._next_work_time = HydrusData.GetNowFloat() + self._period
|
|
|
|
self._scheduler.AddJob( self )
|
|
|
|
|
|
|