2015-03-25 22:04:19 +00:00
import collections
2015-11-25 22:00:57 +00:00
import cProfile
2021-09-22 21:12:34 +00:00
import decimal
2021-09-08 21:41:52 +00:00
import fractions
2019-01-09 22:59:03 +00:00
import io
2020-07-29 20:52:44 +00:00
import itertools
2015-03-25 22:04:19 +00:00
import os
2015-11-25 22:00:57 +00:00
import pstats
2015-08-19 21:48:21 +00:00
import psutil
2016-11-30 20:24:17 +00:00
import random
2018-07-18 21:07:15 +00:00
import re
2015-03-25 22:04:19 +00:00
import sqlite3
2017-01-25 22:56:55 +00:00
import struct
2015-06-17 20:01:41 +00:00
import subprocess
2015-03-25 22:04:19 +00:00
import sys
import threading
import time
import traceback
2020-06-24 21:25:24 +00:00
import typing
2020-07-29 20:52:44 +00:00
import yaml
2015-03-25 22:04:19 +00:00
2020-07-08 22:00:33 +00:00
from hydrus . core import HydrusBoot
2020-05-20 21:36:02 +00:00
from hydrus . core import HydrusConstants as HC
from hydrus . core import HydrusExceptions
from hydrus . core import HydrusGlobals as HG
from hydrus . core import HydrusText
2015-03-25 22:04:19 +00:00
def default_dict_list ( ) : return collections . defaultdict ( list )
def default_dict_set ( ) : return collections . defaultdict ( set )
def BuildKeyToListDict ( pairs ) :
d = collections . defaultdict ( list )
for ( key , value ) in pairs : d [ key ] . append ( value )
return d
def BuildKeyToSetDict ( pairs ) :
d = collections . defaultdict ( set )
for ( key , value ) in pairs : d [ key ] . add ( value )
return d
2021-06-30 21:27:35 +00:00
def BytesToNoneOrHex ( b : typing . Optional [ bytes ] ) :
if b is None :
return None
else :
return b . hex ( )
2015-03-25 22:04:19 +00:00
def CalculateScoreFromRating ( count , rating ) :
2020-07-08 22:00:33 +00:00
# https://www.evanmiller.org/how-not-to-sort-by-average-rating.html
2015-03-25 22:04:19 +00:00
positive = count * rating
negative = count * ( 1.0 - rating )
# positive + negative = count
# I think I've parsed this correctly from the website! Not sure though!
score = ( ( positive + 1.9208 ) / count - 1.96 * ( ( ( positive * negative ) / count + 0.9604 ) * * 0.5 ) / count ) / ( 1 + 3.8416 / count )
return score
2020-06-17 21:31:54 +00:00
def CheckProgramIsNotShuttingDown ( ) :
if HG . model_shutdown :
raise HydrusExceptions . ShutdownException ( ' Application is shutting down! ' )
2016-06-15 18:59:44 +00:00
def CleanRunningFile ( db_path , instance ) :
2019-10-16 20:47:55 +00:00
# just to be careful
2016-06-15 18:59:44 +00:00
path = os . path . join ( db_path , instance + ' _running ' )
try :
os . remove ( path )
except :
pass
2017-04-05 21:16:40 +00:00
def ConvertFloatToPercentage ( f ) :
2019-01-09 22:59:03 +00:00
return ' {:.1f} % ' . format ( f * 100 )
2015-03-25 22:04:19 +00:00
2015-04-08 18:10:50 +00:00
def ConvertIntToPixels ( i ) :
if i == 1 : return ' pixels '
elif i == 1000 : return ' kilopixels '
elif i == 1000000 : return ' megapixels '
2015-04-25 22:31:50 +00:00
else : return ' megapixels '
2015-04-08 18:10:50 +00:00
2020-05-27 21:27:52 +00:00
def ConvertIndexToPrettyOrdinalString ( index : int ) :
2016-10-19 20:02:56 +00:00
2020-05-27 21:27:52 +00:00
if index > = 0 :
return ConvertIntToPrettyOrdinalString ( index + 1 )
else :
return ConvertIntToPrettyOrdinalString ( index )
def ConvertIntToPrettyOrdinalString ( num : int ) :
if num == 0 :
return ' unknown position '
2021-03-17 21:59:28 +00:00
tens = ( abs ( num ) % 100 ) / / 10
2016-10-19 20:02:56 +00:00
2021-03-10 23:10:11 +00:00
if tens == 1 :
2016-10-19 20:02:56 +00:00
2021-03-10 23:10:11 +00:00
ordinal = ' th '
2016-10-19 20:02:56 +00:00
else :
2021-03-10 23:10:11 +00:00
remainder = abs ( num ) % 10
if remainder == 1 :
ordinal = ' st '
elif remainder == 2 :
ordinal = ' nd '
elif remainder == 3 :
ordinal = ' rd '
else :
ordinal = ' th '
2016-10-19 20:02:56 +00:00
2020-05-27 21:27:52 +00:00
s = ' {} {} ' . format ( ToHumanInt ( abs ( num ) ) , ordinal )
if num < 0 :
2021-03-10 23:10:11 +00:00
if num == - 1 :
s = ' last '
else :
s = ' {} from last ' . format ( s )
2020-05-27 21:27:52 +00:00
return s
2015-11-11 21:20:41 +00:00
2015-04-22 22:57:25 +00:00
def ConvertIntToUnit ( unit ) :
if unit == 1 : return ' B '
elif unit == 1024 : return ' KB '
elif unit == 1048576 : return ' MB '
elif unit == 1073741824 : return ' GB '
2015-03-25 22:04:19 +00:00
def ConvertMillisecondsToPrettyTime ( ms ) :
2019-01-09 22:59:03 +00:00
hours = ms / / 3600000
2015-03-25 22:04:19 +00:00
if hours == 1 : hours_result = ' 1 hour '
2015-11-04 22:30:28 +00:00
else : hours_result = str ( hours ) + ' hours '
2015-03-25 22:04:19 +00:00
ms = ms % 3600000
2019-01-09 22:59:03 +00:00
minutes = ms / / 60000
2015-03-25 22:04:19 +00:00
if minutes == 1 : minutes_result = ' 1 minute '
2015-11-04 22:30:28 +00:00
else : minutes_result = str ( minutes ) + ' minutes '
2015-03-25 22:04:19 +00:00
ms = ms % 60000
2019-01-09 22:59:03 +00:00
seconds = ms / / 1000
2015-03-25 22:04:19 +00:00
if seconds == 1 : seconds_result = ' 1 second '
2015-11-04 22:30:28 +00:00
else : seconds_result = str ( seconds ) + ' seconds '
2015-03-25 22:04:19 +00:00
2019-01-09 22:59:03 +00:00
detailed_seconds = ms / 1000
2015-03-25 22:04:19 +00:00
2019-01-09 22:59:03 +00:00
detailed_seconds_result = ' {:.1f} seconds ' . format ( detailed_seconds )
2015-03-25 22:04:19 +00:00
ms = ms % 1000
if hours > 0 : return hours_result + ' ' + minutes_result
if minutes > 0 : return minutes_result + ' ' + seconds_result
if seconds > 0 : return detailed_seconds_result
2019-01-09 22:59:03 +00:00
ms = int ( ms )
if ms == 1 : milliseconds_result = ' 1 millisecond '
else : milliseconds_result = ' {} milliseconds ' . format ( ms )
2015-03-25 22:04:19 +00:00
return milliseconds_result
def ConvertNumericalRatingToPrettyString ( lower , upper , rating , rounded_result = False , out_of = True ) :
rating_converted = ( rating * ( upper - lower ) ) + lower
2019-01-09 22:59:03 +00:00
if rounded_result :
rating_converted = round ( rating_converted )
2015-03-25 22:04:19 +00:00
2019-01-09 22:59:03 +00:00
s = ' {:.2f} ' . format ( rating_converted )
if out_of and lower in ( 0 , 1 ) :
2015-03-25 22:04:19 +00:00
2019-01-09 22:59:03 +00:00
s + = ' / {:.2f} ' . format ( upper )
2015-03-25 22:04:19 +00:00
return s
2015-04-22 22:57:25 +00:00
def ConvertPixelsToInt ( unit ) :
2015-04-08 18:10:50 +00:00
if unit == ' pixels ' : return 1
elif unit == ' kilopixels ' : return 1000
elif unit == ' megapixels ' : return 1000000
2015-03-25 22:04:19 +00:00
def ConvertPrettyStringsToUglyNamespaces ( pretty_strings ) :
result = { s for s in pretty_strings if s != ' no namespace ' }
if ' no namespace ' in pretty_strings : result . add ( ' ' )
return result
2019-01-09 22:59:03 +00:00
def ConvertResolutionToPrettyString ( resolution ) :
2021-11-17 21:22:27 +00:00
if resolution is None :
return ' no resolution '
if not isinstance ( resolution , tuple ) :
try :
resolution = tuple ( resolution )
except :
2021-11-24 21:59:58 +00:00
return ' broken resolution '
2021-11-17 21:22:27 +00:00
2021-07-14 20:42:19 +00:00
if resolution in HC . NICE_RESOLUTIONS :
return HC . NICE_RESOLUTIONS [ resolution ]
2019-01-09 22:59:03 +00:00
( width , height ) = resolution
2018-02-28 22:30:36 +00:00
2021-07-14 20:42:19 +00:00
return ' {} x {} ' . format ( ToHumanInt ( width ) , ToHumanInt ( height ) )
2018-02-28 22:30:36 +00:00
def ConvertStatusToPrefix ( status ) :
if status == HC . CONTENT_STATUS_CURRENT : return ' '
elif status == HC . CONTENT_STATUS_PENDING : return ' (+) '
elif status == HC . CONTENT_STATUS_PETITIONED : return ' (-) '
elif status == HC . CONTENT_STATUS_DELETED : return ' (X) '
2019-06-26 21:27:18 +00:00
def TimeDeltaToPrettyTimeDelta ( seconds , show_seconds = True ) :
2015-11-18 22:44:07 +00:00
2017-03-02 02:14:56 +00:00
if seconds is None :
2017-06-21 21:15:59 +00:00
return ' per month '
2017-03-02 02:14:56 +00:00
2018-12-05 22:35:30 +00:00
if seconds == 0 :
return ' 0 seconds '
2018-07-04 20:48:28 +00:00
if seconds < 0 :
seconds = abs ( seconds )
2017-09-27 21:52:54 +00:00
if seconds > = 60 :
2015-12-09 23:16:41 +00:00
seconds = int ( seconds )
2018-07-04 20:48:28 +00:00
MINUTE = 60
HOUR = 60 * MINUTE
DAY = 24 * HOUR
MONTH = 30 * DAY
2021-08-18 21:10:01 +00:00
YEAR = 365 * DAY
2018-07-04 20:48:28 +00:00
lines = [ ]
lines . append ( ( ' year ' , YEAR ) )
lines . append ( ( ' month ' , MONTH ) )
lines . append ( ( ' day ' , DAY ) )
lines . append ( ( ' hour ' , HOUR ) )
lines . append ( ( ' minute ' , MINUTE ) )
2019-06-26 21:27:18 +00:00
if show_seconds :
lines . append ( ( ' second ' , 1 ) )
2018-07-04 20:48:28 +00:00
result_components = [ ]
for ( time_string , duration ) in lines :
2018-01-31 22:58:15 +00:00
2018-07-04 20:48:28 +00:00
time_quantity = seconds / / duration
2018-01-31 22:58:15 +00:00
2018-07-04 20:48:28 +00:00
seconds % = duration
2018-01-31 22:58:15 +00:00
2018-07-04 20:48:28 +00:00
if time_quantity > 0 :
2017-06-21 21:15:59 +00:00
2018-07-04 20:48:28 +00:00
s = ToHumanInt ( time_quantity ) + ' ' + time_string
2017-06-21 21:15:59 +00:00
2018-07-04 20:48:28 +00:00
if time_quantity > 1 :
2018-01-31 22:58:15 +00:00
2018-07-04 20:48:28 +00:00
s + = ' s '
2018-01-31 22:58:15 +00:00
2018-07-04 20:48:28 +00:00
result_components . append ( s )
2018-01-31 22:58:15 +00:00
2018-07-04 20:48:28 +00:00
if len ( result_components ) == 2 : # we now have 1 month 2 days
2018-01-31 22:58:15 +00:00
2018-07-04 20:48:28 +00:00
break
2018-01-31 22:58:15 +00:00
2017-06-21 21:15:59 +00:00
2018-07-04 20:48:28 +00:00
else :
2016-12-14 21:19:07 +00:00
2018-07-04 20:48:28 +00:00
if len ( result_components ) > 0 : # something like '1 year' -- in which case we do not care about the days and hours
2018-01-31 22:58:15 +00:00
2018-07-04 20:48:28 +00:00
break
2018-01-31 22:58:15 +00:00
2016-12-14 21:19:07 +00:00
2015-12-09 23:16:41 +00:00
2018-07-04 20:48:28 +00:00
result = ' ' . join ( result_components )
elif seconds > 1 :
if int ( seconds ) == seconds :
2016-12-14 21:19:07 +00:00
2018-07-04 20:48:28 +00:00
result = ToHumanInt ( seconds ) + ' seconds '
2015-12-09 23:16:41 +00:00
else :
2019-01-09 22:59:03 +00:00
result = ' {:.1f} seconds ' . format ( seconds )
2015-12-09 23:16:41 +00:00
2017-06-21 21:15:59 +00:00
elif seconds == 1 :
result = ' 1 second '
2015-11-18 22:44:07 +00:00
elif seconds > 0.1 :
2019-01-09 22:59:03 +00:00
result = ' {} milliseconds ' . format ( int ( seconds * 1000 ) )
2015-11-18 22:44:07 +00:00
elif seconds > 0.01 :
2019-01-09 22:59:03 +00:00
result = ' {:.1f} milliseconds ' . format ( int ( seconds * 1000 ) )
2015-11-18 22:44:07 +00:00
elif seconds > 0.001 :
2019-01-09 22:59:03 +00:00
result = ' {:.2f} milliseconds ' . format ( int ( seconds * 1000 ) )
2015-11-18 22:44:07 +00:00
else :
2019-01-09 22:59:03 +00:00
result = ' {} microseconds ' . format ( int ( seconds * 1000000 ) )
2015-11-18 22:44:07 +00:00
return result
2015-03-25 22:04:19 +00:00
def ConvertTimestampToPrettyExpires ( timestamp ) :
2018-07-04 20:48:28 +00:00
if timestamp is None :
2015-03-25 22:04:19 +00:00
2018-07-04 20:48:28 +00:00
return ' does not expire '
2015-03-25 22:04:19 +00:00
2018-07-04 20:48:28 +00:00
if timestamp == 0 :
2015-03-25 22:04:19 +00:00
2018-07-04 20:48:28 +00:00
return ' unknown expiration '
2015-03-25 22:04:19 +00:00
2019-05-15 20:35:00 +00:00
try :
2018-02-14 21:47:18 +00:00
2019-05-15 20:35:00 +00:00
time_delta_string = TimestampToPrettyTimeDelta ( timestamp )
2018-02-14 21:47:18 +00:00
2019-05-15 20:35:00 +00:00
if TimeHasPassed ( timestamp ) :
return ' expired ' + time_delta_string
else :
return ' expires ' + time_delta_string
except :
return ' unparseable time {} ' . format ( timestamp )
2017-11-29 21:48:23 +00:00
2020-08-05 20:10:36 +00:00
def ConvertTimestampToPrettyTime ( timestamp , in_utc = False , include_24h_time = True ) :
2017-09-20 19:47:31 +00:00
2019-05-15 20:35:00 +00:00
if timestamp is None :
2021-04-28 21:43:16 +00:00
return ' unknown time '
2019-05-15 20:35:00 +00:00
2018-01-31 22:58:15 +00:00
if include_24h_time :
2020-08-05 20:10:36 +00:00
phrase = ' % Y- % m- %d % H: % M: % S '
2018-01-31 22:58:15 +00:00
else :
2020-08-05 20:10:36 +00:00
phrase = ' % Y- % m- %d '
2018-01-31 22:58:15 +00:00
2019-05-15 20:35:00 +00:00
try :
2018-02-07 23:40:33 +00:00
2020-08-05 20:10:36 +00:00
if in_utc :
2019-05-15 20:35:00 +00:00
struct_time = time . gmtime ( timestamp )
2020-08-05 20:10:36 +00:00
phrase = phrase + ' UTC '
2019-05-15 20:35:00 +00:00
else :
struct_time = time . localtime ( timestamp )
2018-02-07 23:40:33 +00:00
2019-05-15 20:35:00 +00:00
return time . strftime ( phrase , struct_time )
2018-02-07 23:40:33 +00:00
2019-05-15 20:35:00 +00:00
except :
2018-02-07 23:40:33 +00:00
2019-05-15 20:35:00 +00:00
return ' unparseable time {} ' . format ( timestamp )
2018-02-07 23:40:33 +00:00
2021-09-22 21:12:34 +00:00
def BaseTimestampToPrettyTimeDelta ( timestamp , just_now_string = ' now ' , just_now_threshold = 3 , history_suffix = ' ago ' , show_seconds = True , no_prefix = False ) :
2015-03-25 22:04:19 +00:00
2018-11-07 23:09:40 +00:00
if timestamp is None :
2021-04-28 21:43:16 +00:00
return ' at an unknown time '
2018-11-07 23:09:40 +00:00
2019-06-26 21:27:18 +00:00
if not show_seconds :
just_now_threshold = max ( just_now_threshold , 60 )
2019-05-15 20:35:00 +00:00
try :
2017-09-20 19:47:31 +00:00
2019-05-15 20:35:00 +00:00
time_delta = abs ( timestamp - GetNow ( ) )
2017-09-20 19:47:31 +00:00
2019-05-15 20:35:00 +00:00
if time_delta < = just_now_threshold :
return just_now_string
2017-09-20 19:47:31 +00:00
2019-06-26 21:27:18 +00:00
time_delta_string = TimeDeltaToPrettyTimeDelta ( time_delta , show_seconds = show_seconds )
2017-09-20 19:47:31 +00:00
2019-05-15 20:35:00 +00:00
if TimeHasPassed ( timestamp ) :
2021-07-14 20:42:19 +00:00
return ' {} {} ' . format ( time_delta_string , history_suffix )
2019-05-15 20:35:00 +00:00
else :
2021-02-17 18:22:44 +00:00
if no_prefix :
return time_delta_string
else :
return ' in ' + time_delta_string
2019-05-15 20:35:00 +00:00
except :
2017-09-20 19:47:31 +00:00
2019-05-15 20:35:00 +00:00
return ' unparseable time {} ' . format ( timestamp )
2017-09-20 19:47:31 +00:00
2015-03-25 22:04:19 +00:00
2021-09-22 21:12:34 +00:00
TimestampToPrettyTimeDelta = BaseTimestampToPrettyTimeDelta
2017-05-31 21:50:53 +00:00
def ConvertUglyNamespaceToPrettyString ( namespace ) :
if namespace is None or namespace == ' ' :
return ' no namespace '
else :
return namespace
2015-03-25 22:04:19 +00:00
def ConvertUglyNamespacesToPrettyStrings ( namespaces ) :
2020-05-13 19:03:16 +00:00
namespaces = sorted ( namespaces )
2015-03-25 22:04:19 +00:00
2017-05-31 21:50:53 +00:00
result = [ ConvertUglyNamespaceToPrettyString ( namespace ) for namespace in namespaces ]
2015-03-25 22:04:19 +00:00
return result
2015-04-22 22:57:25 +00:00
def ConvertUnitToInt ( unit ) :
2015-03-25 22:04:19 +00:00
if unit == ' B ' : return 1
elif unit == ' KB ' : return 1024
elif unit == ' MB ' : return 1048576
elif unit == ' GB ' : return 1073741824
2016-03-23 19:42:56 +00:00
def ConvertValueRangeToBytes ( value , range ) :
2019-01-09 22:59:03 +00:00
return ToHumanBytes ( value ) + ' / ' + ToHumanBytes ( range )
2016-03-23 19:42:56 +00:00
2015-06-10 19:40:25 +00:00
def ConvertValueRangeToPrettyString ( value , range ) :
2018-07-04 20:48:28 +00:00
return ToHumanInt ( value ) + ' / ' + ToHumanInt ( range )
2015-06-10 19:40:25 +00:00
2019-03-20 21:22:10 +00:00
def ConvertValueRangeToScanbarTimestampsMS ( value_ms , range_ms ) :
value_ms = int ( round ( value_ms ) )
range_hours = range_ms / / 3600000
value_hours = value_ms / / 3600000
range_minutes = ( range_ms % 3600000 ) / / 60000
value_minutes = ( value_ms % 3600000 ) / / 60000
range_seconds = ( range_ms % 60000 ) / / 1000
value_seconds = ( value_ms % 60000 ) / / 1000
range_ms = range_ms % 1000
value_ms = value_ms % 1000
if range_hours > 0 :
# 0:01:23.033/1:12:57.067
time_phrase = ' {} : {:0>2} : {:0>2} . {:0>3} '
args = ( value_hours , value_minutes , value_seconds , value_ms , range_hours , range_minutes , range_seconds , range_ms )
elif range_minutes > 0 :
# 01:23.033/12:57.067 or 0:23.033/1:57.067
if range_minutes > 9 :
time_phrase = ' {:0>2} : {:0>2} . {:0>3} '
else :
time_phrase = ' {:0>1} : {:0>2} . {:0>3} '
args = ( value_minutes , value_seconds , value_ms , range_minutes , range_seconds , range_ms )
else :
# 23.033/57.067 or 3.033/7.067 or 0.033/0.067
if range_seconds > 9 :
time_phrase = ' {:0>2} . {:0>3} '
else :
time_phrase = ' {:0>1} . {:0>3} '
args = ( value_seconds , value_ms , range_seconds , range_ms )
full_phrase = ' {} / {} ' . format ( time_phrase , time_phrase )
result = full_phrase . format ( * args )
return result
2015-03-25 22:04:19 +00:00
def DebugPrint ( debug_info ) :
2015-11-18 22:44:07 +00:00
Print ( debug_info )
2015-03-25 22:04:19 +00:00
sys . stdout . flush ( )
sys . stderr . flush ( )
2018-07-04 20:48:28 +00:00
def DedupeList ( xs ) :
xs_seen = set ( )
xs_return = [ ]
for x in xs :
if x in xs_seen :
continue
xs_return . append ( x )
xs_seen . add ( x )
return xs_return
2015-07-01 22:02:07 +00:00
def GenerateKey ( ) :
return os . urandom ( HC . HYDRUS_KEY_LENGTH )
2021-12-15 22:16:22 +00:00
def Get64BitHammingDistance ( perceptual_hash1 , perceptual_hash2 ) :
2015-03-25 22:04:19 +00:00
2017-01-25 22:56:55 +00:00
# old way of doing this was:
#while xor > 0:
#
# distance += 1
# xor &= xor - 1
#
2015-03-25 22:04:19 +00:00
2017-01-25 22:56:55 +00:00
# convert to unsigned long long, then xor
# then through the power of stackexchange magic, we get number of bits in record time
2018-07-04 20:48:28 +00:00
# Here it is: https://stackoverflow.com/questions/9829578/fast-way-of-counting-non-zero-bits-in-positive-integer/9830282#9830282
2015-03-25 22:04:19 +00:00
2021-12-15 22:16:22 +00:00
n = struct . unpack ( ' !Q ' , perceptual_hash1 ) [ 0 ] ^ struct . unpack ( ' !Q ' , perceptual_hash2 ) [ 0 ]
2017-01-25 22:56:55 +00:00
n = ( n & 0x5555555555555555 ) + ( ( n & 0xAAAAAAAAAAAAAAAA ) >> 1 ) # 10101010, 01010101
n = ( n & 0x3333333333333333 ) + ( ( n & 0xCCCCCCCCCCCCCCCC ) >> 2 ) # 11001100, 00110011
n = ( n & 0x0F0F0F0F0F0F0F0F ) + ( ( n & 0xF0F0F0F0F0F0F0F0 ) >> 4 ) # 11110000, 00001111
n = ( n & 0x00FF00FF00FF00FF ) + ( ( n & 0xFF00FF00FF00FF00 ) >> 8 ) # etc...
n = ( n & 0x0000FFFF0000FFFF ) + ( ( n & 0xFFFF0000FFFF0000 ) >> 16 )
2018-07-04 20:48:28 +00:00
n = ( n & 0x00000000FFFFFFFF ) + ( n >> 32 )
# you technically are going n & 0xFFFFFFFF00000000 at the end, but that's a no-op with the >> 32 afterwards, so can be omitted
2015-03-25 22:04:19 +00:00
2017-01-25 22:56:55 +00:00
return n
2015-03-25 22:04:19 +00:00
2021-09-08 21:41:52 +00:00
def GetNicelyDivisibleNumberForZoom ( zoom , no_bigger_than ) :
# it is most convenient to have tiles that line up with the current zoom ratio
# 768 is a convenient size for meaty GPU blitting, but as a number it doesn't make for nice multiplication
# a 'nice' size is one that divides nicely by our zoom, so that integer translations between canvas and native res aren't losing too much in the float remainder
# the trick of going ( 123456 // 16 ) * 16 to give you a nice multiple of 16 does not work with floats like 1.4 lmao.
# what we can do instead is phrase 1.4 as 7/5 and use 7 as our int. any number cleanly divisible by 7 is cleanly divisible by 1.4
base_frac = fractions . Fraction ( zoom )
denominator_limit = 10000
frac = base_frac
while frac . numerator > no_bigger_than :
frac = base_frac . limit_denominator ( denominator_limit )
denominator_limit / / = 2
if denominator_limit < 10 :
return - 1
if frac . numerator == 0 :
return - 1
return frac . numerator
2017-06-07 22:05:15 +00:00
def GetEmptyDataDict ( ) :
data = collections . defaultdict ( default_dict_list )
return data
2020-03-11 21:52:11 +00:00
def GetNonDupeName ( original_name , disallowed_names ) :
i = 1
non_dupe_name = original_name
while non_dupe_name in disallowed_names :
non_dupe_name = original_name + ' ( ' + str ( i ) + ' ) '
i + = 1
return non_dupe_name
2017-06-14 21:19:11 +00:00
def GetNow ( ) :
return int ( time . time ( ) )
2018-02-14 21:47:18 +00:00
def GetNowFloat ( ) :
return time . time ( )
2015-03-25 22:04:19 +00:00
def GetNowPrecise ( ) :
2019-11-20 23:10:46 +00:00
return time . perf_counter ( )
2015-03-25 22:04:19 +00:00
2016-06-15 18:59:44 +00:00
def GetSiblingProcessPorts ( db_path , instance ) :
2015-09-16 18:11:00 +00:00
2016-06-15 18:59:44 +00:00
path = os . path . join ( db_path , instance + ' _running ' )
2015-09-16 18:11:00 +00:00
if os . path . exists ( path ) :
2019-01-16 22:40:53 +00:00
with open ( path , ' r ' , encoding = ' utf-8 ' ) as f :
2015-09-16 18:11:00 +00:00
2019-01-09 22:59:03 +00:00
file_text = f . read ( )
2015-09-16 18:11:00 +00:00
try :
2019-01-09 22:59:03 +00:00
( pid , create_time ) = HydrusText . DeserialiseNewlinedTexts ( file_text )
2015-09-16 18:11:00 +00:00
pid = int ( pid )
create_time = float ( create_time )
except ValueError :
return None
try :
if psutil . pid_exists ( pid ) :
ports = [ ]
p = psutil . Process ( pid )
for conn in p . connections ( ) :
if conn . status == ' LISTEN ' :
ports . append ( int ( conn . laddr [ 1 ] ) )
return ports
except psutil . Error :
return None
return None
2019-01-09 22:59:03 +00:00
def GetSubprocessEnv ( ) :
2019-05-15 20:35:00 +00:00
if HG . subprocess_report_mode :
2019-01-09 22:59:03 +00:00
2019-05-15 20:35:00 +00:00
env = os . environ . copy ( )
ShowText ( ' Your unmodified env is: {} ' . format ( env ) )
2020-06-11 12:01:08 +00:00
env = os . environ . copy ( )
2020-07-08 22:00:33 +00:00
if HydrusBoot . ORIGINAL_PATH is not None :
2020-06-11 12:01:08 +00:00
2020-07-08 22:00:33 +00:00
env [ ' PATH ' ] = HydrusBoot . ORIGINAL_PATH
2020-06-11 12:01:08 +00:00
2019-05-15 20:35:00 +00:00
if HC . RUNNING_FROM_FROZEN_BUILD :
2019-02-13 22:26:43 +00:00
2019-01-09 22:59:03 +00:00
# let's make a proper env for subprocess that doesn't have pyinstaller woo woo in it
changes_made = False
2020-06-11 12:01:08 +00:00
orig_swaperoo_strings = [ ' LD_LIBRARY_PATH ' , ' XDG_DATA_DIRS ' ]
2020-05-06 21:31:41 +00:00
ok_to_remove_absent_orig = [ ' LD_LIBRARY_PATH ' ]
2019-01-09 22:59:03 +00:00
2020-06-11 12:01:08 +00:00
for key in orig_swaperoo_strings :
2019-01-09 22:59:03 +00:00
2020-05-06 21:31:41 +00:00
orig_key = ' {} _ORIG ' . format ( key )
2019-01-09 22:59:03 +00:00
2020-05-06 21:31:41 +00:00
if orig_key in env :
env [ key ] = env [ orig_key ]
changes_made = True
elif key in env and key in ok_to_remove_absent_orig :
del env [ key ]
changes_made = True
2019-06-26 21:27:18 +00:00
2019-01-09 22:59:03 +00:00
2020-06-11 12:01:08 +00:00
remove_if_hydrus_base_dir = [ ' QT_PLUGIN_PATH ' , ' QML2_IMPORT_PATH ' , ' SSL_CERT_FILE ' ]
2020-06-17 21:31:54 +00:00
hydrus_base_dir = HG . controller . GetDBDir ( )
2020-06-11 12:01:08 +00:00
for key in remove_if_hydrus_base_dir :
if key in env and env [ key ] . startswith ( hydrus_base_dir ) :
del env [ key ]
changes_made = True
2020-05-06 21:31:41 +00:00
if ( HC . PLATFORM_LINUX or HC . PLATFORM_MACOS ) :
2019-01-09 22:59:03 +00:00
2020-05-06 21:31:41 +00:00
if ' PATH ' in env :
2019-01-09 22:59:03 +00:00
2020-05-06 21:31:41 +00:00
# fix for pyinstaller, which drops this stuff for some reason and hence breaks ffmpeg
path = env [ ' PATH ' ]
path_locations = set ( path . split ( ' : ' ) )
desired_path_locations = [ ' /usr/bin ' , ' /usr/local/bin ' ]
for desired_path_location in desired_path_locations :
if desired_path_location not in path_locations :
path = desired_path_location + ' : ' + path
env [ ' PATH ' ] = path
changes_made = True
2019-01-23 22:19:16 +00:00
2020-05-06 21:31:41 +00:00
if ' XDG_DATA_DIRS ' in env :
xdg_data_dirs = env [ ' XDG_DATA_DIRS ' ]
# pyinstaller can just replace this nice usually long str with multiple paths with base_dir/share
# absent the _orig above to rescue this, we'll populate with basic
if ' : ' not in xdg_data_dirs and HC . BASE_DIR in xdg_data_dirs :
2019-01-23 22:19:16 +00:00
2020-05-06 21:31:41 +00:00
xdg_data_dirs = ' /usr/local/share:/usr/share '
2019-01-23 22:19:16 +00:00
changes_made = True
2019-01-09 22:59:03 +00:00
if not changes_made :
env = None
else :
env = None
return env
def GetSubprocessHideTerminalStartupInfo ( ) :
if HC . PLATFORM_WINDOWS :
# This suppresses the terminal window that tends to pop up when calling ffmpeg or whatever
startupinfo = subprocess . STARTUPINFO ( )
startupinfo . dwFlags | = subprocess . STARTF_USESHOWWINDOW
else :
startupinfo = None
return startupinfo
2019-01-16 22:40:53 +00:00
def GetSubprocessKWArgs ( hide_terminal = True , text = False ) :
2019-01-09 22:59:03 +00:00
sbp_kwargs = { }
sbp_kwargs [ ' env ' ] = GetSubprocessEnv ( )
2019-01-16 22:40:53 +00:00
if text :
# probably need to override the stdXXX pipes with i/o encoding wrappers in the case of 3.5 here
if sys . version_info . minor > = 6 :
sbp_kwargs [ ' encoding ' ] = ' utf-8 '
if sys . version_info . minor > = 7 :
sbp_kwargs [ ' text ' ] = True
else :
sbp_kwargs [ ' universal_newlines ' ] = True
2019-01-09 22:59:03 +00:00
if hide_terminal :
sbp_kwargs [ ' startupinfo ' ] = GetSubprocessHideTerminalStartupInfo ( )
2019-02-13 22:26:43 +00:00
if HG . subprocess_report_mode :
message = ' KWargs are: {} ' . format ( sbp_kwargs )
2019-05-15 20:35:00 +00:00
ShowText ( message )
2019-02-13 22:26:43 +00:00
2019-01-09 22:59:03 +00:00
return sbp_kwargs
2018-07-04 20:48:28 +00:00
def GetTimeDeltaSinceTime ( timestamp ) :
time_since = timestamp - GetNow ( )
result = min ( time_since , 0 )
return - result
2018-02-14 21:47:18 +00:00
def GetTimeDeltaUntilTime ( timestamp ) :
time_remaining = timestamp - GetNow ( )
return max ( time_remaining , 0 )
def GetTimeDeltaUntilTimeFloat ( timestamp ) :
time_remaining = timestamp - GetNowFloat ( )
return max ( time_remaining , 0.0 )
def GetTimeDeltaUntilTimePrecise ( t ) :
time_remaining = t - GetNowPrecise ( )
return max ( time_remaining , 0.0 )
2018-06-20 20:20:22 +00:00
def GetTypeName ( obj_type ) :
if hasattr ( obj_type , ' __name__ ' ) :
return obj_type . __name__
else :
return repr ( obj_type )
2019-01-09 22:59:03 +00:00
def GenerateHumanTextSortKey ( ) :
2018-07-18 21:07:15 +00:00
""" Solves the 19, 20, 200, 21, 22 issue when sorting ' Page 21.jpg ' type strings.
2019-01-09 22:59:03 +00:00
Breaks the string into groups of text and int ( i . e . ( " Page " , 21 , " .jpg " ) ) . """
int_convert = lambda t : int ( t ) if t . isdecimal ( ) else t
2018-07-18 21:07:15 +00:00
2019-01-09 22:59:03 +00:00
split_alphanum = lambda t : tuple ( ( int_convert ( sub_t ) for sub_t in re . split ( ' ([0-9]+) ' , t . lower ( ) ) ) )
2018-07-18 21:07:15 +00:00
2019-01-09 22:59:03 +00:00
return split_alphanum
2018-07-18 21:07:15 +00:00
2019-01-09 22:59:03 +00:00
HumanTextSortKey = GenerateHumanTextSortKey ( )
def HumanTextSort ( texts ) :
texts . sort ( key = HumanTextSortKey )
2018-07-18 21:07:15 +00:00
2015-03-25 22:04:19 +00:00
def IntelligentMassIntersect ( sets_to_reduce ) :
answer = None
for set_to_reduce in sets_to_reduce :
2018-05-09 20:23:00 +00:00
if len ( set_to_reduce ) == 0 :
return set ( )
2015-03-25 22:04:19 +00:00
2018-05-09 20:23:00 +00:00
if answer is None :
answer = set ( set_to_reduce )
2015-03-25 22:04:19 +00:00
else :
2018-05-09 20:23:00 +00:00
if len ( answer ) == 0 :
return set ( )
else :
answer . intersection_update ( set_to_reduce )
2015-03-25 22:04:19 +00:00
2018-05-09 20:23:00 +00:00
if answer is None :
return set ( )
else :
return answer
2015-03-25 22:04:19 +00:00
2016-06-15 18:59:44 +00:00
def IsAlreadyRunning ( db_path , instance ) :
2015-08-19 21:48:21 +00:00
2016-06-15 18:59:44 +00:00
path = os . path . join ( db_path , instance + ' _running ' )
2015-08-19 21:48:21 +00:00
2015-09-02 23:16:09 +00:00
if os . path . exists ( path ) :
2015-08-19 21:48:21 +00:00
2019-01-16 22:40:53 +00:00
with open ( path , ' r ' , encoding = ' utf-8 ' ) as f :
2015-09-02 23:16:09 +00:00
2019-01-09 22:59:03 +00:00
file_text = f . read ( )
2015-08-19 21:48:21 +00:00
2015-09-02 23:16:09 +00:00
try :
2015-08-26 21:18:39 +00:00
2019-01-09 22:59:03 +00:00
( pid , create_time ) = HydrusText . DeserialiseNewlinedTexts ( file_text )
2015-09-02 23:16:09 +00:00
pid = int ( pid )
create_time = float ( create_time )
except ValueError :
return False
2015-08-26 21:18:39 +00:00
2015-08-19 21:48:21 +00:00
2015-09-02 23:16:09 +00:00
try :
2015-08-26 21:18:39 +00:00
2015-12-23 22:51:04 +00:00
me = psutil . Process ( )
if me . pid == pid and me . create_time ( ) == create_time :
# this is me! there is no conflict, lol!
# this happens when a linux process restarts with os.execl(), for instance (unlike Windows, it keeps its pid)
return False
2015-09-02 23:16:09 +00:00
if psutil . pid_exists ( pid ) :
2015-08-26 21:18:39 +00:00
2015-09-02 23:16:09 +00:00
p = psutil . Process ( pid )
2015-08-26 21:18:39 +00:00
2015-09-02 23:16:09 +00:00
if p . create_time ( ) == create_time and p . is_running ( ) :
return True
2015-08-26 21:18:39 +00:00
2015-09-02 23:16:09 +00:00
except psutil . Error :
2015-08-19 21:48:21 +00:00
2015-09-02 23:16:09 +00:00
return False
2015-08-19 21:48:21 +00:00
return False
2015-12-02 22:32:18 +00:00
def IterateHexPrefixes ( ) :
hex_chars = ' 0123456789abcdef '
for ( one , two ) in itertools . product ( hex_chars , hex_chars ) :
prefix = one + two
yield prefix
2016-08-31 19:55:14 +00:00
def LastShutdownWasBad ( db_path , instance ) :
path = os . path . join ( db_path , instance + ' _running ' )
if os . path . exists ( path ) :
return True
else :
return False
2017-01-04 22:48:23 +00:00
def MassUnion ( lists ) :
return { item for item in itertools . chain . from_iterable ( lists ) }
2016-08-31 19:55:14 +00:00
2016-12-07 22:12:52 +00:00
def MedianPop ( population ) :
# assume it has at least one and comes sorted
2019-01-09 22:59:03 +00:00
median_index = len ( population ) / / 2
2016-12-07 22:12:52 +00:00
row = population . pop ( median_index )
return row
2015-03-25 22:04:19 +00:00
def MergeKeyToListDicts ( key_to_list_dicts ) :
result = collections . defaultdict ( list )
for key_to_list_dict in key_to_list_dicts :
2019-01-09 22:59:03 +00:00
for ( key , value ) in list ( key_to_list_dict . items ( ) ) : result [ key ] . extend ( value )
2015-03-25 22:04:19 +00:00
return result
2020-06-24 21:25:24 +00:00
def PartitionIterator ( pred : typing . Callable [ [ object ] , bool ] , stream : typing . Iterable [ object ] ) :
( t1 , t2 ) = itertools . tee ( stream )
return ( itertools . filterfalse ( pred , t1 ) , filter ( pred , t2 ) )
def PartitionIteratorIntoLists ( pred : typing . Callable [ [ object ] , bool ] , stream : typing . Iterable [ object ] ) :
( a , b ) = PartitionIterator ( pred , stream )
return ( list ( a ) , list ( b ) )
2020-11-25 22:22:47 +00:00
def ParseHashesFromRawHexText ( hash_type , hex_hashes_raw ) :
hash_type_to_hex_length = {
' md5 ' : 32 ,
' sha1 ' : 40 ,
' sha256 ' : 64 ,
' sha512 ' : 128
}
hex_hashes = HydrusText . DeserialiseNewlinedTexts ( hex_hashes_raw )
# convert md5:abcd to abcd
hex_hashes = [ hex_hash . split ( ' : ' ) [ - 1 ] for hex_hash in hex_hashes ]
hex_hashes = [ HydrusText . HexFilter ( hex_hash ) for hex_hash in hex_hashes ]
expected_hex_length = hash_type_to_hex_length [ hash_type ]
bad_hex_hashes = [ hex_hash for hex_hash in hex_hashes if len ( hex_hash ) != expected_hex_length ]
if len ( bad_hex_hashes ) :
m = ' Sorry, {} hashes should have {} hex characters! These did not: ' . format ( hash_type , expected_hex_length )
m + = os . linesep * 2
m + = os . linesep . join ( ( ' {} ( {} characters) ' . format ( bad_hex_hash , len ( bad_hex_hash ) ) for bad_hex_hash in bad_hex_hashes ) )
raise Exception ( m )
hex_hashes = [ hex_hash for hex_hash in hex_hashes if len ( hex_hash ) % 2 == 0 ]
hex_hashes = DedupeList ( hex_hashes )
hashes = tuple ( [ bytes . fromhex ( hex_hash ) for hex_hash in hex_hashes ] )
return hashes
2015-11-18 22:44:07 +00:00
def Print ( text ) :
2017-09-20 19:47:31 +00:00
try :
2019-01-09 22:59:03 +00:00
print ( str ( text ) )
2017-09-20 19:47:31 +00:00
except :
print ( repr ( text ) )
2015-11-18 22:44:07 +00:00
2016-03-23 19:42:56 +00:00
ShowText = Print
2017-01-04 22:48:23 +00:00
def PrintException ( e , do_wait = True ) :
2016-03-23 19:42:56 +00:00
2021-04-07 21:26:45 +00:00
( etype , value , tb ) = sys . exc_info ( )
2016-03-23 19:42:56 +00:00
2021-04-07 21:26:45 +00:00
PrintExceptionTuple ( etype , value , tb , do_wait = do_wait )
2016-03-23 19:42:56 +00:00
2021-04-07 21:26:45 +00:00
def PrintExceptionTuple ( etype , value , tb , do_wait = True ) :
2017-12-13 22:33:07 +00:00
if etype is None :
2021-04-07 21:26:45 +00:00
etype = HydrusExceptions . UnknownException
if etype == HydrusExceptions . ShutdownException :
2017-12-13 22:33:07 +00:00
2021-04-07 21:26:45 +00:00
return
if value is None :
value = ' Unknown error '
if tb is None :
trace = ' No error trace--here is the stack: ' + os . linesep + ' ' . join ( traceback . format_stack ( ) )
2017-12-13 22:33:07 +00:00
else :
trace = ' ' . join ( traceback . format_exception ( etype , value , tb ) )
stack_list = traceback . format_stack ( )
2016-03-23 19:42:56 +00:00
2017-12-13 22:33:07 +00:00
stack = ' ' . join ( stack_list )
2016-03-23 19:42:56 +00:00
2019-01-09 22:59:03 +00:00
message = str ( etype . __name__ ) + ' : ' + str ( value ) + os . linesep + trace + os . linesep + stack
2016-03-23 19:42:56 +00:00
Print ( ' ' )
Print ( ' Exception: ' )
DebugPrint ( message )
2017-01-04 22:48:23 +00:00
if do_wait :
time . sleep ( 1 )
2016-03-23 19:42:56 +00:00
ShowException = PrintException
2021-04-07 21:26:45 +00:00
ShowExceptionTuple = PrintExceptionTuple
2016-03-23 19:42:56 +00:00
2021-01-20 22:22:03 +00:00
def Profile ( summary , code , g , l , min_duration_ms = 20 , show_summary = False ) :
2015-11-25 22:00:57 +00:00
profile = cProfile . Profile ( )
2017-06-14 21:19:11 +00:00
started = GetNowPrecise ( )
2015-11-25 22:00:57 +00:00
2017-06-14 21:19:11 +00:00
profile . runctx ( code , g , l )
2015-11-25 22:00:57 +00:00
2017-06-14 21:19:11 +00:00
finished = GetNowPrecise ( )
2015-11-25 22:00:57 +00:00
2017-07-27 00:47:13 +00:00
time_took = finished - started
time_took_ms = int ( time_took * 1000.0 )
if time_took_ms > min_duration_ms :
2017-06-14 21:19:11 +00:00
2019-01-09 22:59:03 +00:00
output = io . StringIO ( )
2017-06-14 21:19:11 +00:00
stats = pstats . Stats ( profile , stream = output )
stats . strip_dirs ( )
stats . sort_stats ( ' tottime ' )
output . write ( ' Stats ' )
output . write ( os . linesep * 2 )
stats . print_stats ( )
output . write ( ' Callers ' )
output . write ( os . linesep * 2 )
stats . print_callers ( )
output . seek ( 0 )
2021-08-25 21:59:05 +00:00
profile_text = output . read ( )
2017-06-14 21:19:11 +00:00
2021-07-14 20:42:19 +00:00
with HG . profile_counter_lock :
HG . profile_slow_count + = 1
2021-01-20 22:22:03 +00:00
if show_summary :
ShowText ( summary )
2021-08-25 21:59:05 +00:00
HG . controller . PrintProfile ( summary , profile_text = profile_text )
2021-07-14 20:42:19 +00:00
2017-06-14 21:19:11 +00:00
else :
2021-07-14 20:42:19 +00:00
with HG . profile_counter_lock :
HG . profile_fast_count + = 1
2017-07-27 00:47:13 +00:00
2021-07-14 20:42:19 +00:00
if show_summary :
2021-08-25 21:59:05 +00:00
HG . controller . PrintProfile ( summary )
2021-07-14 20:42:19 +00:00
2017-06-14 21:19:11 +00:00
2015-11-25 22:00:57 +00:00
2019-08-21 21:34:01 +00:00
def PullNFromIterator ( iterator , n ) :
chunk = [ ]
for item in iterator :
chunk . append ( item )
if len ( chunk ) == n :
return chunk
return chunk
2016-11-30 20:24:17 +00:00
def RandomPop ( population ) :
random_index = random . randint ( 0 , len ( population ) - 1 )
row = population . pop ( random_index )
return row
2016-06-15 18:59:44 +00:00
def RecordRunningStart ( db_path , instance ) :
2015-09-02 23:16:09 +00:00
2016-06-15 18:59:44 +00:00
path = os . path . join ( db_path , instance + ' _running ' )
2015-09-02 23:16:09 +00:00
record_string = ' '
try :
me = psutil . Process ( )
record_string + = str ( me . pid )
record_string + = os . linesep
record_string + = str ( me . create_time ( ) )
except psutil . Error :
return
2019-01-16 22:40:53 +00:00
with open ( path , ' w ' , encoding = ' utf-8 ' ) as f :
2015-09-02 23:16:09 +00:00
2019-01-09 22:59:03 +00:00
f . write ( record_string )
2015-09-02 23:16:09 +00:00
2015-12-16 22:41:06 +00:00
def RestartProcess ( ) :
time . sleep ( 1 ) # time for ports to unmap
2016-11-30 20:24:17 +00:00
exe = sys . executable
me = sys . argv [ 0 ]
2016-12-07 22:12:52 +00:00
if HC . RUNNING_FROM_SOURCE :
2016-11-30 20:24:17 +00:00
2016-12-07 22:12:52 +00:00
# exe is python's exe, me is the script
2016-11-30 20:24:17 +00:00
args = [ sys . executable ] + sys . argv
else :
# we are running a frozen release--both exe and me are the built exe
2020-05-06 21:31:41 +00:00
2016-11-30 20:24:17 +00:00
# wrap it in quotes because pyinstaller passes it on as raw text, breaking any path with spaces :/
2020-05-06 21:31:41 +00:00
if not me . startswith ( ' " ' ) :
me = ' " {} " ' . format ( me )
2016-11-30 20:24:17 +00:00
2020-05-06 21:31:41 +00:00
args = [ me ] + sys . argv [ 1 : ]
2016-11-30 20:24:17 +00:00
os . execv ( exe , args )
2015-12-16 22:41:06 +00:00
2020-12-02 22:04:38 +00:00
def SampleSetByGettingFirst ( s : set , n ) :
# sampling from a big set can be slow, so if we don't care about super random, let's just rip off the front and let __hash__ be our random
n = min ( len ( s ) , n )
sample = set ( )
if n == 0 :
return sample
for ( i , obj ) in enumerate ( s ) :
sample . add ( obj )
if i > = n - 1 :
break
return sample
2020-10-28 22:20:33 +00:00
def SetsIntersect ( a , b ) :
# not a.isdisjoint( b )
if not isinstance ( a , set ) :
a = set ( a )
if not isinstance ( b , set ) :
b = set ( b )
if len ( a ) > len ( b ) :
( a , b ) = ( b , a )
return True in ( i in b for i in a )
2019-10-02 23:38:59 +00:00
def SmoothOutMappingIterator ( xs , n ) :
# de-spikifies mappings, so if there is ( tag, 20k files ), it breaks that up into manageable chunks
chunk_weight = 0
chunk = [ ]
for ( tag_item , hash_items ) in xs :
for chunk_of_hash_items in SplitIteratorIntoChunks ( hash_items , n ) :
yield ( tag_item , chunk_of_hash_items )
2017-12-13 22:33:07 +00:00
def SplayListForDB ( xs ) :
2015-03-25 22:04:19 +00:00
2017-12-13 22:33:07 +00:00
return ' ( ' + ' , ' . join ( ( str ( x ) for x in xs ) ) + ' ) '
2015-11-04 22:30:28 +00:00
2015-12-30 23:44:09 +00:00
def SplitIteratorIntoChunks ( iterator , n ) :
chunk = [ ]
for item in iterator :
chunk . append ( item )
if len ( chunk ) == n :
yield chunk
chunk = [ ]
if len ( chunk ) > 0 :
yield chunk
2019-08-21 21:34:01 +00:00
def SplitIteratorIntoAutothrottledChunks ( iterator , starting_n , precise_time_to_stop ) :
n = starting_n
chunk = PullNFromIterator ( iterator , n )
while len ( chunk ) > 0 :
time_work_started = GetNowPrecise ( )
yield chunk
work_time = GetNowPrecise ( ) - time_work_started
items_per_second = n / work_time
time_remaining = precise_time_to_stop - GetNowPrecise ( )
if TimeHasPassedPrecise ( precise_time_to_stop ) :
n = 1
else :
expected_items_in_remaining_time = max ( 1 , int ( time_remaining * items_per_second ) )
quad_speed = n * 4
n = min ( quad_speed , expected_items_in_remaining_time )
chunk = PullNFromIterator ( iterator , n )
2015-04-08 18:10:50 +00:00
def SplitListIntoChunks ( xs , n ) :
2016-04-20 20:42:21 +00:00
if isinstance ( xs , set ) :
xs = list ( xs )
2019-01-09 22:59:03 +00:00
for i in range ( 0 , len ( xs ) , n ) :
2017-03-02 02:14:56 +00:00
yield xs [ i : i + n ]
2019-08-21 21:34:01 +00:00
def SplitMappingIteratorIntoAutothrottledChunks ( iterator , starting_n , precise_time_to_stop ) :
n = starting_n
chunk_weight = 0
chunk = [ ]
for ( tag_item , hash_items ) in iterator :
2019-10-02 23:38:59 +00:00
chunk . append ( ( tag_item , hash_items ) )
2019-08-21 21:34:01 +00:00
2019-10-02 23:38:59 +00:00
chunk_weight + = len ( hash_items )
2019-08-21 21:34:01 +00:00
2019-10-02 23:38:59 +00:00
if chunk_weight > = n :
2019-08-21 21:34:01 +00:00
2019-10-02 23:38:59 +00:00
time_work_started = GetNowPrecise ( )
2019-08-21 21:34:01 +00:00
2019-10-02 23:38:59 +00:00
yield chunk
2019-08-21 21:34:01 +00:00
2019-10-02 23:38:59 +00:00
work_time = GetNowPrecise ( ) - time_work_started
2019-08-21 21:34:01 +00:00
2019-10-02 23:38:59 +00:00
chunk_weight = 0
chunk = [ ]
2019-08-15 00:40:48 +00:00
2019-10-02 23:38:59 +00:00
items_per_second = n / work_time
2019-08-15 00:40:48 +00:00
2019-10-02 23:38:59 +00:00
time_remaining = precise_time_to_stop - GetNowPrecise ( )
2019-08-15 00:40:48 +00:00
2019-10-02 23:38:59 +00:00
if TimeHasPassedPrecise ( precise_time_to_stop ) :
2019-08-15 00:40:48 +00:00
2019-10-02 23:38:59 +00:00
n = 1
2019-08-15 00:40:48 +00:00
2019-10-02 23:38:59 +00:00
else :
2019-08-15 00:40:48 +00:00
2019-10-02 23:38:59 +00:00
expected_items_in_remaining_time = max ( 1 , int ( time_remaining * items_per_second ) )
2017-03-02 02:14:56 +00:00
2019-10-02 23:38:59 +00:00
quad_speed = n * 4
2017-03-02 02:14:56 +00:00
2019-10-02 23:38:59 +00:00
n = min ( quad_speed , expected_items_in_remaining_time )
2017-03-02 02:14:56 +00:00
if len ( chunk ) > 0 :
yield chunk
2015-04-08 18:10:50 +00:00
2015-06-24 22:10:14 +00:00
def TimeHasPassed ( timestamp ) :
2016-03-30 22:56:50 +00:00
if timestamp is None :
return False
2015-06-24 22:10:14 +00:00
return GetNow ( ) > timestamp
2018-02-14 21:47:18 +00:00
def TimeHasPassedFloat ( timestamp ) :
return GetNowFloat ( ) > timestamp
2015-07-15 20:28:26 +00:00
def TimeHasPassedPrecise ( precise_timestamp ) :
return GetNowPrecise ( ) > precise_timestamp
2015-08-05 18:42:35 +00:00
def TimeUntil ( timestamp ) :
return timestamp - GetNow ( )
2021-09-22 21:12:34 +00:00
def BaseToHumanBytes ( size , sig_figs = 3 ) :
#
# ░█▓▓▓▓▓▒ ░▒░ ▒ ▒ ░ ░ ░▒ ░░ ░▒ ░ ░▒░ ▒░▒▒▒░▓▓▒▒▓
# ▒▓▒▒▓▒ ░ ░ ▒ ░░ ░░ ░▒▒▒▓▒▒▓▓
# ▓█▓▒░ ▒▒░ ▒░ ▒▓░ ░ ░░░ ░ ░░░▒▒ ░ ░░░ ▒▓▓▓▒▓▓▒
# ▒▒░▒▒░░ ▒░░▒▓░▒░▒░░░░░░░░ ░▒▒▒▓ ░ ░▒▒░ ▒▓▒▓█▒
# ░█▓ ░░░ ▒▒░▒▒ ▒▒▒░░░░▒▒░░▒▒▒░░▒▒ ▒░ ░░░░░░▒█▒
# ░░▒▒ ▒░░▒░▒▒▒░ ░░░▒▒▒░░▒░ ░ ▓▒▒░ ░ ▒█▓░▒░
# ░░░ ░▒ ▓▒░▒░▒ ░▒▒▒▒ ▒▓░ ░ ▒ ▒█▓
# ░░░ ▓░▒▒░░▒ ▒▒▒▒░░ ░░░ ░░░ ▒ ░▒░ ░
# ░▒░ ▓░▒▒░░▒░▒▓▓████▓ ░▒▒▒▒▒ ░▒░░ ▓▒ ▒▒ ░▒░
# ░░░░ ░░ ░░▒░▒░░░ ░░░ ░ ░▒▒▒▒▓█▓ ▒░░░▒▓░▒▒░ ░░░░
# ▒ ░ ░░░░░░ ░▓░▓▒░░░ ░ ░░ ░▒▒░▒▒▒▒░▓░ ░ ░▒░
# ▒░░ ░░░░░░ ▒▒░▒░▒▒░░░░░░ ░░░░░░▒▒░▓▒░▒▒▓░ ░ ░░
# ░░░░▒▒▒░░░ ░░ ▓░▒░░▒▒░ ░░░░░░ ░▒▒▒▒▓▓ ░ ░ ░░ ▒░░
# ▒▒░ ▒▒░ ░░ ▒▒▒▓░░▒ ░ ░░░░░░░ ░░▒░░▒▓ ░░░░ ░░ ░▒░░▒
# ▒░░ ░ ▒▒ ░░ ▒▓▒▓▒ ▒░░▒▓ ▓▒░▒▒░░░▓▒ ░░ ░░ ▒▒░ ░░
# ░ ░▒▒░░▒▒░░ ░▒▒▒▒▒ ▒░░▒▓▒░ ░▒▓█▒░▒ ░░▒▓▒ ▒▒▒ ▒░▒░
# ▒░▒▓▓░░░░░▒▒▒▒░░░▓▒ ▒░ ▒▓░░▒▒░ ░░▒▒░▒▓░ ▒░ ▒░▒▒▒ ░░░░▓ ▒░▒░
# ▒ ▒▒ ▒░░░▒░░ ▒▒ ░░░▒▒░░░░▒▒▒░ ░░▒▒░▒▒░░▒▓▒░▒ ░ ▒░▒▒░░░░░▒░░▒▒▒▒
# ░ ▒▒▒░ ░░▒ ▒░░▒░░░░░▒▒▒▒▒▒▒▒▒▒▓▒░░▒░▒▒░▒▒░ ░ ▒░░░▒░░░░░░░▒▒▒░
# ▒ ▒▓ ░▒▒ ░▒░▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▓▒░▒▒▒▒▒░░▒░ ▒░ ░▒▒░▒░ ▒
# ▒ ▒▒░ ░▒░ ░ ▒░░▒▒░ ░▒▒▒▒ ░▒▒▒░ ▒░▒▒ ▒▒ ▒▒ ▒ ▓░░▒░
# ░░ ░▒ ░▒▒░ ░ ░▒░░░░ ░░ ░ ▒▒▓▓█▒ ░░▒▓▒░ ▒░ ░▒▒░░░
# ▒░ ░░▒ ▒▒▒ ░░▒▒░░░ ░ ▒░ ░░▒▒▒▒▒░██▓▒▓██ ▒▓▒▒░ ░ ▒▒ ▒▓▒ ░░
# ░░▒▒▒▒▒▒░ ░░░ ░ ▒▒░▒ ░▒▒░▓█░ ██▓▓▓▓ ▒ ░▓ ░░▒░ ░▒░ ▒░░░░▒
# ▒▒░ ░▒▒ ▒▓▒ ▓▓▓▓█▒▓▓▓▒▒▓▓▓▓ ▒▓▒ ▒▒ ░░░ ░▒▒░░░▒░
# ░ ░░▒▒▓▒▒ ░░▓▓▓███▒▒█▓██▒░░ ▒▒▒▒░ ▒▓ ░▒▒░ ░ ░▒▒▒░
# ░ ░░ ░▒▒▒▒▒▓▓▒▒░ ▒▒▒█▓▓▓░ ▓▓▒▓▓▒░░░▒▓▒░░ ▒▒ ░▒█▓░ ░░ ░░░
# ▒░░▒▓▒░▒░ ▒▒ ▒▒░ ░░░░▒▓▓▓▒▓▓▓▓ ▒▓▒▒▓▓░▒▒▒▒▒▒▒▓▒ ▒▒ ░▒▒▒▓▓▒░░▒ ░▒▒▒
# ▒▒▒░ ▒░ ░░ ▒ ░▓▓▒▒░░░░░░░ ▓░▒░ ░▓▓▓░ ░▒▓░▒▓▒▓▒▓▓▓
# ▓░ ▒░ ▒▓ ▓░ ░░░░░░░░░░░▒▒▒▒▒▒▓▓▒ ▒▒▓▓▓▓▓▓▓▓▓░ ▒▓▒▓▓▓▒▓▓▓▓
# ░░ ▓▒ ░▒▓▓▒▒▒▓▓▓▒▓▓▓▓▓▓█▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓▒▒▒▒▒▓▓▓▓▒▓▓▒▒▒▒▒▓▓▓▓▒
# ░░░ ▒▒▒▒▓▓░▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░▓▒▒▒▒▒▒▒▓▓▓▓▓▓▓▒▒▒▒▒░ ░
# ░░ ▒▓ ██░▓▓▓▓▓▓▓▓▓▓█▓▓▒▒▒ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░▒▓▒▒▒▒▒▒▒▒▒░▒▒░░▒▒░ ░░░░░
# ░▒░░░▒▓ ▓░▒▓▓▓▓▓▓▓▓▓▒ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓ ▓░ ▒▒▒▒▒▒▒▒░ ░▒▒▒▒░░░░
# ▒▒░ ░▒▓ ▒▓░▓▓▓▓█▓▒ ░▓▓▓▓▓▓▓█▒▓▒ ▒▓▓▓▓▓▒▓░ ▓ ░▒▒░▒░░░░▒▒▒▒▒░░░░░
# ▓▒▒▒▒▒▒▓▓ ▒▓▒▓▓▓▒░ ░▓▓██▓▓▓█▓▒ ░ ▒▓▓▓▓▒▓▒ ▒▓ ▒ ░▓▓▓▓▒ ░░░░▒
# ░░▓▓▓▓▒▓▓ ░░▓▓ ░▓████▓▓▓▒▒▒ ▒██▓▒▓█▒ ▓▒ ▒ ░░▒▓▓▒▓▒▒▒
# ░░▒▒▒▒ ▒▒ ▒▓█▓▒░░ ░▓██▓▒░░░█░ ▓ ▒ ░ ░▓▒▒▓▓▒▓▒
# ░░░░░ ▒▒ ▓▒ ░░ ▒░░▒▒▓▓▓▒░░ ░█▓█▒ ▒▓ ▒░░ ░▓▒▒
# ▒░░░░▒▓▒ ░██ ▒▒▒▒░░▒▓ ▒▒ ░▒░▒███░ ▓▒ ▒░░ ░▓▒░░░░▒
# ░ ░░░▓▒░ ▒█▒▓█░ ░ ▒░ ░░ ▒▓▒ ▓ ░▒ ░▓▓▒░▒░░░
# ░░▒▒▒▓▓▓▓░▓▓ ██▒ ░▒░▒▒▒▒░░▒ ▒▓ ▓ ▒▓ ▒░ ░░▓▓▒░▒▒▒░
# ░░▒▒▓█▓░ █ ░█▒ ░ ░ ░▒▒░░▓░ ▓░ ░▓░ █░ ░░ ▒░ ░ ▓▓▒░ ░▒░
# ░░▒▓░ █ ██ ░ ░▒ ▒ █▓▒▒▒░░▒▒░ ▓▒ ░ ▒▓▒░ ░
#
2015-03-25 22:04:19 +00:00
2019-01-09 22:59:03 +00:00
if size is None :
2015-11-04 22:30:28 +00:00
2019-01-09 22:59:03 +00:00
return ' unknown size '
2015-11-04 22:30:28 +00:00
2019-01-09 22:59:03 +00:00
2021-09-22 21:12:34 +00:00
# my definition of sig figs is nonsense here. basically I mean 'can we show decimal places and not look stupid long?'
2019-01-09 22:59:03 +00:00
if size < 1024 :
2015-11-04 22:30:28 +00:00
2019-01-09 22:59:03 +00:00
return ToHumanInt ( size ) + ' B '
2015-11-04 22:30:28 +00:00
2015-03-25 22:04:19 +00:00
2019-01-09 22:59:03 +00:00
suffixes = ( ' ' , ' K ' , ' M ' , ' G ' , ' T ' , ' P ' )
2018-07-04 20:48:28 +00:00
2019-01-09 22:59:03 +00:00
suffix_index = 0
2018-07-04 20:48:28 +00:00
2021-10-27 21:12:33 +00:00
d = decimal . Decimal ( size )
while d > = 1024 :
2018-07-04 20:48:28 +00:00
2021-10-27 21:12:33 +00:00
d / = 1024
2018-07-04 20:48:28 +00:00
2019-01-09 22:59:03 +00:00
suffix_index + = 1
2018-07-04 20:48:28 +00:00
2019-01-09 22:59:03 +00:00
suffix = suffixes [ suffix_index ]
2015-03-25 22:04:19 +00:00
2021-09-22 21:12:34 +00:00
ctx = decimal . getcontext ( )
# ok, if we have 237KB, we still want all 237, even if user said 2 sf
while d . log10 ( ) > = sig_figs :
2015-11-04 22:30:28 +00:00
2021-09-22 21:12:34 +00:00
sig_figs + = 1
2015-11-04 22:30:28 +00:00
2021-09-22 21:12:34 +00:00
ctx . prec = sig_figs
ctx . rounding = decimal . ROUND_HALF_EVEN
d = d . normalize ( ctx )
try :
2015-11-04 22:30:28 +00:00
2021-09-22 21:12:34 +00:00
# if we have 30, this will be normalised to 3E+1, so we want to quantize it back
2015-11-04 22:30:28 +00:00
2021-09-22 21:12:34 +00:00
( sign , digits , exp ) = d . as_tuple ( )
2015-03-25 22:04:19 +00:00
2021-09-22 21:12:34 +00:00
if exp > 0 :
ctx . prec = 10 # careful to make precising bigger again though, or we get an error
d = d . quantize ( 0 )
except :
# blarg
pass
2015-03-25 22:04:19 +00:00
2021-09-22 21:12:34 +00:00
return ' {} {} B ' . format ( d , suffix )
ToHumanBytes = BaseToHumanBytes
2019-01-09 22:59:03 +00:00
def ToHumanInt ( num ) :
2020-01-29 22:08:37 +00:00
num = int ( num )
2020-01-22 21:04:43 +00:00
# this got stomped on by mpv, which resets locale
#text = locale.format_string( '%d', num, grouping = True )
text = ' {:,} ' . format ( num )
2019-01-09 22:59:03 +00:00
2015-11-04 22:30:28 +00:00
return text
2016-06-22 20:59:24 +00:00
def WaitForProcessToFinish ( p , timeout ) :
started = GetNow ( )
while p . poll ( ) is None :
if TimeHasPassed ( started + timeout ) :
p . kill ( )
2018-07-04 20:48:28 +00:00
raise Exception ( ' Process did not finish within ' + ToHumanInt ( timeout ) + ' seconds! ' )
2016-06-22 20:59:24 +00:00
time . sleep ( 2 )
2015-03-25 22:04:19 +00:00
class HydrusYAMLBase ( yaml . YAMLObject ) :
yaml_loader = yaml . SafeLoader
yaml_dumper = yaml . SafeDumper
2016-01-06 21:17:20 +00:00
class BigJobPauser ( object ) :
def __init__ ( self , period = 10 , wait_time = 0.1 ) :
self . _period = period
self . _wait_time = wait_time
self . _next_pause = GetNow ( ) + self . _period
def Pause ( self ) :
if TimeHasPassed ( self . _next_pause ) :
time . sleep ( self . _wait_time )
self . _next_pause = GetNow ( ) + self . _period
2017-01-25 22:56:55 +00:00
class Call ( object ) :
def __init__ ( self , func , * args , * * kwargs ) :
2021-06-23 21:11:38 +00:00
self . _label = None
2017-01-25 22:56:55 +00:00
self . _func = func
self . _args = args
self . _kwargs = kwargs
def __call__ ( self ) :
self . _func ( * self . _args , * * self . _kwargs )
2018-02-14 21:47:18 +00:00
def __repr__ ( self ) :
2021-06-23 21:11:38 +00:00
label = self . _GetLabel ( )
return ' Call: {} ' . format ( label )
2021-06-09 20:28:09 +00:00
2021-06-23 21:11:38 +00:00
def _GetLabel ( self ) - > str :
2021-06-09 20:28:09 +00:00
2021-06-23 21:11:38 +00:00
if self . _label is None :
2021-06-09 20:28:09 +00:00
2021-06-23 21:11:38 +00:00
# this can actually cause an error with Qt objects that are dead or from the wrong thread, wew!
label = ' {} ( {} , {} ) ' . format ( self . _func , self . _args , self . _kwargs )
else :
2021-06-09 20:28:09 +00:00
2021-06-23 21:11:38 +00:00
label = self . _label
return label
def GetLabel ( self ) - > str :
2021-06-09 20:28:09 +00:00
2021-06-23 21:11:38 +00:00
return self . _GetLabel ( )
2021-06-09 20:28:09 +00:00
def SetLabel ( self , label : str ) :
2021-06-23 21:11:38 +00:00
self . _label = label
2018-02-14 21:47:18 +00:00
2015-03-25 22:04:19 +00:00
class ContentUpdate ( object ) :
2019-04-10 22:50:53 +00:00
def __init__ ( self , data_type , action , row , reason = None ) :
2015-03-25 22:04:19 +00:00
self . _data_type = data_type
self . _action = action
self . _row = row
2019-04-10 22:50:53 +00:00
self . _reason = reason
2015-03-25 22:04:19 +00:00
2017-04-26 21:58:12 +00:00
def __eq__ ( self , other ) :
2020-01-22 21:04:43 +00:00
if isinstance ( other , ContentUpdate ) :
return self . __hash__ ( ) == other . __hash__ ( )
return NotImplemented
2017-04-26 21:58:12 +00:00
2015-03-25 22:04:19 +00:00
2017-04-26 21:58:12 +00:00
def __hash__ ( self ) :
return hash ( ( self . _data_type , self . _action , repr ( self . _row ) ) )
def __repr__ ( self ) :
2019-05-01 21:24:42 +00:00
return ' Content Update: ' + str ( ( self . _data_type , self . _action , self . _row , self . _reason ) )
2017-04-26 21:58:12 +00:00
2015-03-25 22:04:19 +00:00
2017-07-27 00:47:13 +00:00
def GetAction ( self ) :
return self . _action
def GetDataType ( self ) :
return self . _data_type
2015-03-25 22:04:19 +00:00
def GetHashes ( self ) :
2020-05-06 21:31:41 +00:00
hashes = set ( )
2015-10-14 21:02:25 +00:00
if self . _data_type == HC . CONTENT_TYPE_FILES :
2015-03-25 22:04:19 +00:00
2016-02-17 22:06:47 +00:00
if self . _action == HC . CONTENT_UPDATE_ADVANCED :
2021-04-28 21:43:16 +00:00
( sub_action , possible_hashes ) = self . _row
if possible_hashes is None :
hashes = set ( )
else :
hashes = possible_hashes
2016-02-17 22:06:47 +00:00
elif self . _action == HC . CONTENT_UPDATE_ADD :
2015-03-25 22:04:19 +00:00
2017-05-31 21:50:53 +00:00
( file_info_manager , timestamp ) = self . _row
2017-12-06 22:06:56 +00:00
hashes = { file_info_manager . hash }
2015-03-25 22:04:19 +00:00
2019-04-10 22:50:53 +00:00
else :
2016-01-13 22:08:19 +00:00
hashes = self . _row
2015-03-25 22:04:19 +00:00
2016-05-25 21:54:03 +00:00
elif self . _data_type == HC . CONTENT_TYPE_DIRECTORIES :
hashes = set ( )
2017-05-10 21:33:58 +00:00
elif self . _data_type == HC . CONTENT_TYPE_URLS :
2018-05-09 20:23:00 +00:00
( urls , hashes ) = self . _row
2017-05-10 21:33:58 +00:00
2015-10-14 21:02:25 +00:00
elif self . _data_type == HC . CONTENT_TYPE_MAPPINGS :
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if self . _action == HC . CONTENT_UPDATE_ADVANCED :
hashes = set ( )
else :
( tag , hashes ) = self . _row
elif self . _data_type in ( HC . CONTENT_TYPE_TAG_PARENTS , HC . CONTENT_TYPE_TAG_SIBLINGS ) :
hashes = set ( )
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
elif self . _data_type == HC . CONTENT_TYPE_RATINGS :
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if self . _action == HC . CONTENT_UPDATE_ADD :
( rating , hashes ) = self . _row
2015-03-25 22:04:19 +00:00
2018-03-07 22:48:29 +00:00
elif self . _data_type == HC . CONTENT_TYPE_NOTES :
if self . _action == HC . CONTENT_UPDATE_SET :
2020-05-06 21:31:41 +00:00
( hash , name , note ) = self . _row
hashes = { hash }
elif self . _action == HC . CONTENT_UPDATE_DELETE :
( hash , name ) = self . _row
2018-03-07 22:48:29 +00:00
hashes = { hash }
2018-12-05 22:35:30 +00:00
elif self . _data_type == HC . CONTENT_TYPE_FILE_VIEWING_STATS :
2020-05-27 21:27:52 +00:00
if self . _action == HC . CONTENT_UPDATE_ADD :
( hash , preview_views_delta , preview_viewtime_delta , media_views_delta , media_viewtime_delta ) = self . _row
hashes = { hash }
elif self . _action == HC . CONTENT_UPDATE_DELETE :
hashes = self . _row
2018-12-05 22:35:30 +00:00
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if not isinstance ( hashes , set ) :
hashes = set ( hashes )
2015-03-25 22:04:19 +00:00
return hashes
2019-04-10 22:50:53 +00:00
def GetReason ( self ) :
if self . _reason is None :
return ' No reason given. '
else :
return self . _reason
2020-09-16 20:46:54 +00:00
def GetRow ( self ) :
return self . _row
2016-01-06 21:17:20 +00:00
def GetWeight ( self ) :
return len ( self . GetHashes ( ) )
2021-10-13 20:16:57 +00:00
def HasReason ( self ) :
return self . _reason is not None
2015-11-18 22:44:07 +00:00
def IsInboxRelated ( self ) :
return self . _action in ( HC . CONTENT_UPDATE_ARCHIVE , HC . CONTENT_UPDATE_INBOX )
2021-05-19 21:30:28 +00:00
def SetRow ( self , row ) :
self . _row = row
2015-11-18 22:44:07 +00:00
def ToTuple ( self ) :
return ( self . _data_type , self . _action , self . _row )
2015-03-25 22:04:19 +00:00
class JobDatabase ( object ) :
2016-03-30 22:56:50 +00:00
def __init__ ( self , job_type , synchronous , action , * args , * * kwargs ) :
2015-03-25 22:04:19 +00:00
self . _type = job_type
self . _synchronous = synchronous
2016-03-30 22:56:50 +00:00
self . _action = action
2015-03-25 22:04:19 +00:00
self . _args = args
self . _kwargs = kwargs
self . _result_ready = threading . Event ( )
2020-04-08 21:10:11 +00:00
def __str__ ( self ) :
return ' DB Job: {} ' . format ( self . ToString ( ) )
def _DoDelayedResultRelief ( self ) :
pass
2016-03-30 22:56:50 +00:00
def GetCallableTuple ( self ) :
return ( self . _action , self . _args , self . _kwargs )
2015-03-25 22:04:19 +00:00
def GetResult ( self ) :
2018-08-08 20:29:54 +00:00
time . sleep ( 0.00001 ) # this one neat trick can save hassle on superquick jobs as event.wait can be laggy
2015-03-25 22:04:19 +00:00
while True :
2016-09-28 18:48:01 +00:00
if self . _result_ready . wait ( 2 ) == True :
break
2017-05-10 21:33:58 +00:00
elif HG . model_shutdown :
2016-09-28 18:48:01 +00:00
raise HydrusExceptions . ShutdownException ( ' Application quit before db could serve result! ' )
2015-03-25 22:04:19 +00:00
2020-04-08 21:10:11 +00:00
self . _DoDelayedResultRelief ( )
2015-03-25 22:04:19 +00:00
2015-11-11 21:20:41 +00:00
if isinstance ( self . _result , Exception ) :
2015-06-03 21:05:13 +00:00
2016-09-28 18:48:01 +00:00
e = self . _result
raise e
2015-03-25 22:04:19 +00:00
2016-01-20 23:57:33 +00:00
else :
return self . _result
2015-03-25 22:04:19 +00:00
2016-09-28 18:48:01 +00:00
def GetType ( self ) :
return self . _type
2015-03-25 22:04:19 +00:00
2018-08-08 20:29:54 +00:00
def IsSynchronous ( self ) :
return self . _synchronous
2015-03-25 22:04:19 +00:00
def PutResult ( self , result ) :
self . _result = result
self . _result_ready . set ( )
2016-03-30 22:56:50 +00:00
def ToString ( self ) :
2020-04-08 21:10:11 +00:00
return ' {} {} ' . format ( self . _type , self . _action )
2016-03-30 22:56:50 +00:00
2015-03-25 22:04:19 +00:00
class ServiceUpdate ( object ) :
def __init__ ( self , action , row = None ) :
self . _action = action
self . _row = row
2017-03-02 02:14:56 +00:00
def ToTuple ( self ) :
return ( self . _action , self . _row )
2015-03-25 22:04:19 +00:00