2015-03-25 22:04:19 +00:00
import bs4
import collections
2015-11-25 22:00:57 +00:00
import cProfile
import cStringIO
2015-03-25 22:04:19 +00:00
import HydrusConstants as HC
import HydrusExceptions
2017-05-10 21:33:58 +00:00
import HydrusGlobals as HG
2015-06-03 21:05:13 +00:00
import HydrusSerialisable
2015-03-25 22:04:19 +00:00
import locale
import os
2015-11-25 22:00:57 +00:00
import pstats
2015-08-19 21:48:21 +00:00
import psutil
2016-11-30 20:24:17 +00:00
import random
2015-09-16 18:11:00 +00:00
import shutil
2015-03-25 22:04:19 +00:00
import sqlite3
2017-01-25 22:56:55 +00:00
import struct
2015-06-17 20:01:41 +00:00
import subprocess
2015-03-25 22:04:19 +00:00
import sys
import threading
import time
import traceback
import yaml
import itertools
def default_dict_list ( ) : return collections . defaultdict ( list )
def default_dict_set ( ) : return collections . defaultdict ( set )
def BuildKeyToListDict ( pairs ) :
d = collections . defaultdict ( list )
for ( key , value ) in pairs : d [ key ] . append ( value )
return d
def BuildKeyToSetDict ( pairs ) :
d = collections . defaultdict ( set )
for ( key , value ) in pairs : d [ key ] . add ( value )
return d
def CalculateScoreFromRating ( count , rating ) :
# http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
count = float ( count )
positive = count * rating
negative = count * ( 1.0 - rating )
# positive + negative = count
# I think I've parsed this correctly from the website! Not sure though!
score = ( ( positive + 1.9208 ) / count - 1.96 * ( ( ( positive * negative ) / count + 0.9604 ) * * 0.5 ) / count ) / ( 1 + 3.8416 / count )
return score
2016-06-15 18:59:44 +00:00
def CleanRunningFile ( db_path , instance ) :
path = os . path . join ( db_path , instance + ' _running ' )
try :
os . remove ( path )
except :
pass
2017-04-05 21:16:40 +00:00
def ConvertFloatToPercentage ( f ) :
return ' %.1f ' % ( f * 100 ) + ' % '
2015-03-25 22:04:19 +00:00
def ConvertIntToBytes ( size ) :
if size is None : return ' unknown size '
suffixes = ( ' ' , ' K ' , ' M ' , ' G ' , ' T ' , ' P ' )
suffix_index = 0
size = float ( size )
while size > 1024.0 :
size = size / 1024.0
suffix_index + = 1
if size < 10.0 : return ' %.1f ' % size + suffixes [ suffix_index ] + ' B '
return ' %.0f ' % size + suffixes [ suffix_index ] + ' B '
2016-09-07 20:01:05 +00:00
def ConvertIntToFirst ( n ) :
# straight from stack, wew
return " %d %s " % ( n , " tsnrhtdd " [ ( n / 10 % 10 != 1 ) * ( n % 10 < 4 ) * n % 10 : : 4 ] )
2015-04-08 18:10:50 +00:00
def ConvertIntToPixels ( i ) :
if i == 1 : return ' pixels '
elif i == 1000 : return ' kilopixels '
elif i == 1000000 : return ' megapixels '
2015-04-25 22:31:50 +00:00
else : return ' megapixels '
2015-04-08 18:10:50 +00:00
2016-10-19 20:02:56 +00:00
def ConvertIntToPrettyOrdinalString ( num ) :
remainder = num % 10
if remainder == 1 :
ordinal = ' st '
elif remainder == 2 :
ordinal = ' nd '
elif remainder == 3 :
ordinal = ' rd '
else :
ordinal = ' th '
return ConvertIntToPrettyString ( num ) + ordinal
2015-11-11 21:20:41 +00:00
def ConvertIntToPrettyString ( num ) :
2015-11-18 22:44:07 +00:00
# don't feed this a unicode string u'%d'--locale can't handle it
text = locale . format ( ' %d ' , num , grouping = True )
try :
text = text . decode ( locale . getpreferredencoding ( ) )
text = ToUnicode ( text )
except :
text = ToUnicode ( text )
return text
2015-11-11 21:20:41 +00:00
2015-04-22 22:57:25 +00:00
def ConvertIntToUnit ( unit ) :
if unit == 1 : return ' B '
elif unit == 1024 : return ' KB '
elif unit == 1048576 : return ' MB '
elif unit == 1073741824 : return ' GB '
2015-03-25 22:04:19 +00:00
def ConvertMillisecondsToPrettyTime ( ms ) :
hours = ms / 3600000
if hours == 1 : hours_result = ' 1 hour '
2015-11-04 22:30:28 +00:00
else : hours_result = str ( hours ) + ' hours '
2015-03-25 22:04:19 +00:00
ms = ms % 3600000
minutes = ms / 60000
if minutes == 1 : minutes_result = ' 1 minute '
2015-11-04 22:30:28 +00:00
else : minutes_result = str ( minutes ) + ' minutes '
2015-03-25 22:04:19 +00:00
ms = ms % 60000
seconds = ms / 1000
if seconds == 1 : seconds_result = ' 1 second '
2015-11-04 22:30:28 +00:00
else : seconds_result = str ( seconds ) + ' seconds '
2015-03-25 22:04:19 +00:00
detailed_seconds = float ( ms ) / 1000.0
if detailed_seconds == 1.0 : detailed_seconds_result = ' 1.0 seconds '
else : detailed_seconds_result = ' %.1f ' % detailed_seconds + ' seconds '
ms = ms % 1000
if ms == 1 : milliseconds_result = ' 1 millisecond '
2015-11-04 22:30:28 +00:00
else : milliseconds_result = str ( ms ) + ' milliseconds '
2015-03-25 22:04:19 +00:00
if hours > 0 : return hours_result + ' ' + minutes_result
if minutes > 0 : return minutes_result + ' ' + seconds_result
if seconds > 0 : return detailed_seconds_result
return milliseconds_result
def ConvertNumericalRatingToPrettyString ( lower , upper , rating , rounded_result = False , out_of = True ) :
rating_converted = ( rating * ( upper - lower ) ) + lower
2015-11-04 22:30:28 +00:00
if rounded_result : s = ' %.2f ' % round ( rating_converted )
else : s = ' %.2f ' % rating_converted
2015-03-25 22:04:19 +00:00
if out_of :
2015-11-04 22:30:28 +00:00
if lower in ( 0 , 1 ) : s + = ' / %.2f ' % upper
2015-03-25 22:04:19 +00:00
return s
2015-04-22 22:57:25 +00:00
def ConvertPixelsToInt ( unit ) :
2015-04-08 18:10:50 +00:00
if unit == ' pixels ' : return 1
elif unit == ' kilopixels ' : return 1000
elif unit == ' megapixels ' : return 1000000
2015-03-25 22:04:19 +00:00
def ConvertPrettyStringsToUglyNamespaces ( pretty_strings ) :
result = { s for s in pretty_strings if s != ' no namespace ' }
if ' no namespace ' in pretty_strings : result . add ( ' ' )
return result
2015-11-18 22:44:07 +00:00
def ConvertTimeDeltaToPrettyString ( seconds ) :
2017-03-02 02:14:56 +00:00
if seconds is None :
2017-06-21 21:15:59 +00:00
return ' per month '
2017-03-02 02:14:56 +00:00
2015-12-09 23:16:41 +00:00
if seconds > 60 :
seconds = int ( seconds )
2017-06-21 21:15:59 +00:00
if seconds > = 86400 :
2015-12-09 23:16:41 +00:00
days = seconds / 86400
hours = ( seconds % 86400 ) / 3600
2017-06-21 21:15:59 +00:00
if days == 1 :
result = ' 1 day '
else :
result = ' %d ' % days + ' days '
2016-12-14 21:19:07 +00:00
if hours > 0 :
result + = ' %d ' % hours + ' hours '
2015-12-09 23:16:41 +00:00
2017-06-21 21:15:59 +00:00
elif seconds > = 3600 :
2015-12-09 23:16:41 +00:00
hours = seconds / 3600
minutes = ( seconds % 3600 ) / 60
2017-06-21 21:15:59 +00:00
if hours == 1 :
result = ' 1 hour '
else :
result = ' %d ' % hours + ' hours '
2016-12-14 21:19:07 +00:00
if minutes > 0 :
result + = ' %d ' % minutes + ' minutes '
2015-12-09 23:16:41 +00:00
else :
minutes = seconds / 60
seconds = seconds % 60
2016-12-14 21:19:07 +00:00
result = ' %d ' % minutes + ' minutes '
if seconds > 0 :
result + = ' %d ' % seconds + ' seconds '
2015-12-09 23:16:41 +00:00
elif seconds > 1 :
2015-11-18 22:44:07 +00:00
result = ' %.1f ' % seconds + ' seconds '
2017-06-21 21:15:59 +00:00
elif seconds == 1 :
result = ' 1 second '
2015-11-18 22:44:07 +00:00
elif seconds > 0.1 :
result = ' %d ' % ( seconds * 1000 ) + ' milliseconds '
elif seconds > 0.01 :
result = ' %.1f ' % ( seconds * 1000 ) + ' milliseconds '
elif seconds > 0.001 :
result = ' %.2f ' % ( seconds * 1000 ) + ' milliseconds '
else :
result = ' %d ' % ( seconds * 1000000 ) + ' microseconds '
return result
2015-03-25 22:04:19 +00:00
def ConvertStatusToPrefix ( status ) :
2017-03-02 02:14:56 +00:00
if status == HC . CONTENT_STATUS_CURRENT : return ' '
elif status == HC . CONTENT_STATUS_PENDING : return ' (+) '
elif status == HC . CONTENT_STATUS_PETITIONED : return ' (-) '
elif status == HC . CONTENT_STATUS_DELETED : return ' (X) '
2015-03-25 22:04:19 +00:00
def ConvertTimestampToPrettyAge ( timestamp ) :
if timestamp == 0 or timestamp is None : return ' unknown age '
age = GetNow ( ) - timestamp
seconds = age % 60
if seconds == 1 : s = ' 1 second '
2015-11-04 22:30:28 +00:00
else : s = str ( seconds ) + ' seconds '
2015-03-25 22:04:19 +00:00
age = age / 60
minutes = age % 60
if minutes == 1 : m = ' 1 minute '
2015-11-04 22:30:28 +00:00
else : m = str ( minutes ) + ' minutes '
2015-03-25 22:04:19 +00:00
age = age / 60
hours = age % 24
if hours == 1 : h = ' 1 hour '
2015-11-04 22:30:28 +00:00
else : h = str ( hours ) + ' hours '
2015-03-25 22:04:19 +00:00
age = age / 24
days = age % 30
if days == 1 : d = ' 1 day '
2015-11-04 22:30:28 +00:00
else : d = str ( days ) + ' days '
2015-03-25 22:04:19 +00:00
age = age / 30
months = age % 12
if months == 1 : mo = ' 1 month '
2015-11-04 22:30:28 +00:00
else : mo = str ( months ) + ' months '
2015-03-25 22:04:19 +00:00
years = age / 12
if years == 1 : y = ' 1 year '
2015-11-04 22:30:28 +00:00
else : y = str ( years ) + ' years '
2015-03-25 22:04:19 +00:00
if years > 0 : return ' ' . join ( ( y , mo ) ) + ' old '
elif months > 0 : return ' ' . join ( ( mo , d ) ) + ' old '
elif days > 0 : return ' ' . join ( ( d , h ) ) + ' old '
elif hours > 0 : return ' ' . join ( ( h , m ) ) + ' old '
else : return ' ' . join ( ( m , s ) ) + ' old '
def ConvertTimestampToPrettyAgo ( timestamp ) :
2017-09-20 19:47:31 +00:00
if timestamp is None or timestamp == 0 :
return ' unknown time '
2015-03-25 22:04:19 +00:00
age = GetNow ( ) - timestamp
seconds = age % 60
if seconds == 1 : s = ' 1 second '
2015-11-04 22:30:28 +00:00
else : s = str ( seconds ) + ' seconds '
2015-03-25 22:04:19 +00:00
age = age / 60
minutes = age % 60
if minutes == 1 : m = ' 1 minute '
2015-11-04 22:30:28 +00:00
else : m = str ( minutes ) + ' minutes '
2015-03-25 22:04:19 +00:00
age = age / 60
hours = age % 24
if hours == 1 : h = ' 1 hour '
2015-11-04 22:30:28 +00:00
else : h = str ( hours ) + ' hours '
2015-03-25 22:04:19 +00:00
age = age / 24
days = age % 30
if days == 1 : d = ' 1 day '
2015-11-04 22:30:28 +00:00
else : d = str ( days ) + ' days '
2015-03-25 22:04:19 +00:00
age = age / 30
months = age % 12
if months == 1 : mo = ' 1 month '
2015-11-04 22:30:28 +00:00
else : mo = str ( months ) + ' months '
2015-03-25 22:04:19 +00:00
years = age / 12
if years == 1 : y = ' 1 year '
2015-11-04 22:30:28 +00:00
else : y = str ( years ) + ' years '
2015-03-25 22:04:19 +00:00
2017-09-20 19:47:31 +00:00
if years > 0 : return ' ' . join ( ( y , mo ) )
elif months > 0 : return ' ' . join ( ( mo , d ) )
elif days > 0 : return ' ' . join ( ( d , h ) )
elif hours > 0 : return ' ' . join ( ( h , m ) )
else : return ' ' . join ( ( m , s ) )
2015-03-25 22:04:19 +00:00
def ConvertTimestampToPrettyExpires ( timestamp ) :
if timestamp is None : return ' does not expire '
if timestamp == 0 : return ' unknown expiration '
expires = GetNow ( ) - timestamp
if expires > = 0 : already_happend = True
else :
expires * = - 1
already_happend = False
seconds = expires % 60
if seconds == 1 : s = ' 1 second '
2015-11-04 22:30:28 +00:00
else : s = str ( seconds ) + ' seconds '
2015-03-25 22:04:19 +00:00
expires = expires / 60
minutes = expires % 60
if minutes == 1 : m = ' 1 minute '
2015-11-04 22:30:28 +00:00
else : m = str ( minutes ) + ' minutes '
2015-03-25 22:04:19 +00:00
expires = expires / 60
hours = expires % 24
if hours == 1 : h = ' 1 hour '
2015-11-04 22:30:28 +00:00
else : h = str ( hours ) + ' hours '
2015-03-25 22:04:19 +00:00
expires = expires / 24
days = expires % 30
if days == 1 : d = ' 1 day '
2015-11-04 22:30:28 +00:00
else : d = str ( days ) + ' days '
2015-03-25 22:04:19 +00:00
expires = expires / 30
months = expires % 12
if months == 1 : mo = ' 1 month '
2015-11-04 22:30:28 +00:00
else : mo = str ( months ) + ' months '
2015-03-25 22:04:19 +00:00
years = expires / 12
if years == 1 : y = ' 1 year '
2015-11-04 22:30:28 +00:00
else : y = str ( years ) + ' years '
2015-03-25 22:04:19 +00:00
if already_happend :
if years > 0 : return ' expired ' + ' ' . join ( ( y , mo ) ) + ' ago '
elif months > 0 : return ' expired ' + ' ' . join ( ( mo , d ) ) + ' ago '
elif days > 0 : return ' expired ' + ' ' . join ( ( d , h ) ) + ' ago '
elif hours > 0 : return ' expired ' + ' ' . join ( ( h , m ) ) + ' ago '
else : return ' expired ' + ' ' . join ( ( m , s ) ) + ' ago '
else :
if years > 0 : return ' expires in ' + ' ' . join ( ( y , mo ) )
elif months > 0 : return ' expires in ' + ' ' . join ( ( mo , d ) )
elif days > 0 : return ' expires in ' + ' ' . join ( ( d , h ) )
elif hours > 0 : return ' expires in ' + ' ' . join ( ( h , m ) )
else : return ' expires in ' + ' ' . join ( ( m , s ) )
def ConvertTimestampToPrettyPending ( timestamp ) :
if timestamp is None : return ' '
if timestamp == 0 : return ' imminent '
pending = GetNow ( ) - timestamp
if pending > = 0 : return ' imminent '
else : pending * = - 1
seconds = pending % 60
if seconds == 1 : s = ' 1 second '
2015-11-04 22:30:28 +00:00
else : s = str ( seconds ) + ' seconds '
2015-03-25 22:04:19 +00:00
pending = pending / 60
minutes = pending % 60
if minutes == 1 : m = ' 1 minute '
2015-11-04 22:30:28 +00:00
else : m = str ( minutes ) + ' minutes '
2015-03-25 22:04:19 +00:00
pending = pending / 60
hours = pending % 24
if hours == 1 : h = ' 1 hour '
2015-11-04 22:30:28 +00:00
else : h = str ( hours ) + ' hours '
2015-03-25 22:04:19 +00:00
pending = pending / 24
days = pending % 30
if days == 1 : d = ' 1 day '
2015-11-04 22:30:28 +00:00
else : d = str ( days ) + ' days '
2015-03-25 22:04:19 +00:00
pending = pending / 30
months = pending % 12
if months == 1 : mo = ' 1 month '
2015-11-04 22:30:28 +00:00
else : mo = str ( months ) + ' months '
2015-03-25 22:04:19 +00:00
years = pending / 12
if years == 1 : y = ' 1 year '
2015-11-04 22:30:28 +00:00
else : y = str ( years ) + ' years '
2015-03-25 22:04:19 +00:00
if years > 0 : return ' in ' + ' ' . join ( ( y , mo ) )
elif months > 0 : return ' in ' + ' ' . join ( ( mo , d ) )
elif days > 0 : return ' in ' + ' ' . join ( ( d , h ) )
elif hours > 0 : return ' in ' + ' ' . join ( ( h , m ) )
2017-08-09 21:33:51 +00:00
elif minutes > 0 : return ' in ' + ' ' . join ( ( m , s ) )
else : return ' in ' + s
2015-03-25 22:04:19 +00:00
def ConvertTimestampToPrettySync ( timestamp ) :
2015-10-07 21:56:22 +00:00
if timestamp is None or timestamp == 0 : return ' (initial sync has not yet occured) '
2015-03-25 22:04:19 +00:00
age = GetNow ( ) - timestamp
seconds = age % 60
if seconds == 1 : s = ' 1 second '
2015-11-04 22:30:28 +00:00
else : s = str ( seconds ) + ' seconds '
2015-03-25 22:04:19 +00:00
age = age / 60
minutes = age % 60
if minutes == 1 : m = ' 1 minute '
2015-11-04 22:30:28 +00:00
else : m = str ( minutes ) + ' minutes '
2015-03-25 22:04:19 +00:00
age = age / 60
hours = age % 24
if hours == 1 : h = ' 1 hour '
2015-11-04 22:30:28 +00:00
else : h = str ( hours ) + ' hours '
2015-03-25 22:04:19 +00:00
age = age / 24
days = age % 30
if days == 1 : d = ' 1 day '
2015-11-04 22:30:28 +00:00
else : d = str ( days ) + ' days '
2015-03-25 22:04:19 +00:00
age = age / 30
months = age % 12
if months == 1 : mo = ' 1 month '
2015-11-04 22:30:28 +00:00
else : mo = str ( months ) + ' months '
2015-03-25 22:04:19 +00:00
years = age / 12
if years == 1 : y = ' 1 year '
2015-11-04 22:30:28 +00:00
else : y = str ( years ) + ' years '
2015-03-25 22:04:19 +00:00
if years > 0 : return ' ' . join ( ( y , mo ) ) + ' ago '
elif months > 0 : return ' ' . join ( ( mo , d ) ) + ' ago '
elif days > 0 : return ' ' . join ( ( d , h ) ) + ' ago '
elif hours > 0 : return ' ' . join ( ( h , m ) ) + ' ago '
else : return ' ' . join ( ( m , s ) ) + ' ago '
2017-09-20 19:47:31 +00:00
def ConvertTimestampToPrettyTime ( timestamp ) :
return time . strftime ( ' % Y/ % m/ %d % H: % M: % S ' , time . localtime ( timestamp ) )
2015-03-25 22:04:19 +00:00
def ConvertTimestampToHumanPrettyTime ( timestamp ) :
now = GetNow ( )
difference = now - timestamp
2017-09-20 19:47:31 +00:00
if difference < 60 :
return ' just now '
elif difference < 86400 * 7 :
return ConvertTimestampToPrettyAgo ( timestamp ) + ' ago '
else :
return ConvertTimestampToPrettyTime ( timestamp )
2015-03-25 22:04:19 +00:00
def ConvertTimeToPrettyTime ( secs ) :
return time . strftime ( ' % H: % M: % S ' , time . gmtime ( secs ) )
2017-05-31 21:50:53 +00:00
def ConvertUglyNamespaceToPrettyString ( namespace ) :
if namespace is None or namespace == ' ' :
return ' no namespace '
else :
return namespace
2015-03-25 22:04:19 +00:00
def ConvertUglyNamespacesToPrettyStrings ( namespaces ) :
2017-05-31 21:50:53 +00:00
namespaces = list ( namespaces )
2015-03-25 22:04:19 +00:00
2017-05-31 21:50:53 +00:00
namespaces . sort ( )
2015-03-25 22:04:19 +00:00
2017-05-31 21:50:53 +00:00
result = [ ConvertUglyNamespaceToPrettyString ( namespace ) for namespace in namespaces ]
2015-03-25 22:04:19 +00:00
return result
2015-04-22 22:57:25 +00:00
def ConvertUnitToInt ( unit ) :
2015-03-25 22:04:19 +00:00
if unit == ' B ' : return 1
elif unit == ' KB ' : return 1024
elif unit == ' MB ' : return 1048576
elif unit == ' GB ' : return 1073741824
2016-03-23 19:42:56 +00:00
def ConvertValueRangeToBytes ( value , range ) :
return ConvertIntToBytes ( value ) + ' / ' + ConvertIntToBytes ( range )
2015-06-10 19:40:25 +00:00
def ConvertValueRangeToPrettyString ( value , range ) :
return ConvertIntToPrettyString ( value ) + ' / ' + ConvertIntToPrettyString ( range )
2015-03-25 22:04:19 +00:00
def DebugPrint ( debug_info ) :
2015-11-18 22:44:07 +00:00
Print ( debug_info )
2015-03-25 22:04:19 +00:00
sys . stdout . flush ( )
sys . stderr . flush ( )
2015-05-06 20:26:18 +00:00
def DeserialisePrettyTags ( text ) :
text = text . replace ( ' \r ' , ' ' )
tags = text . split ( ' \n ' )
return tags
2016-11-02 21:09:14 +00:00
def EncodeBytes ( encoding , data ) :
data = ToByteString ( data )
if encoding == HC . ENCODING_RAW :
encoded_data = data
elif encoding == HC . ENCODING_HEX :
encoded_data = data . encode ( ' hex ' )
elif encoding == HC . ENCODING_BASE64 :
encoded_data = data . encode ( ' base64 ' )
return encoded_data
2015-07-01 22:02:07 +00:00
def GenerateKey ( ) :
return os . urandom ( HC . HYDRUS_KEY_LENGTH )
2017-01-25 22:56:55 +00:00
def Get64BitHammingDistance ( phash1 , phash2 ) :
2015-03-25 22:04:19 +00:00
2017-01-25 22:56:55 +00:00
# old way of doing this was:
#while xor > 0:
#
# distance += 1
# xor &= xor - 1
#
2015-03-25 22:04:19 +00:00
2017-01-25 22:56:55 +00:00
# convert to unsigned long long, then xor
# then through the power of stackexchange magic, we get number of bits in record time
2015-03-25 22:04:19 +00:00
2017-01-25 22:56:55 +00:00
n = struct . unpack ( ' !Q ' , phash1 ) [ 0 ] ^ struct . unpack ( ' !Q ' , phash2 ) [ 0 ]
n = ( n & 0x5555555555555555 ) + ( ( n & 0xAAAAAAAAAAAAAAAA ) >> 1 ) # 10101010, 01010101
n = ( n & 0x3333333333333333 ) + ( ( n & 0xCCCCCCCCCCCCCCCC ) >> 2 ) # 11001100, 00110011
n = ( n & 0x0F0F0F0F0F0F0F0F ) + ( ( n & 0xF0F0F0F0F0F0F0F0 ) >> 4 ) # 11110000, 00001111
n = ( n & 0x00FF00FF00FF00FF ) + ( ( n & 0xFF00FF00FF00FF00 ) >> 8 ) # etc...
n = ( n & 0x0000FFFF0000FFFF ) + ( ( n & 0xFFFF0000FFFF0000 ) >> 16 )
n = ( n & 0x00000000FFFFFFFF ) + ( ( n & 0xFFFFFFFF00000000 ) >> 32 )
2015-03-25 22:04:19 +00:00
2017-01-25 22:56:55 +00:00
return n
2015-03-25 22:04:19 +00:00
2017-06-07 22:05:15 +00:00
def GetEmptyDataDict ( ) :
data = collections . defaultdict ( default_dict_list )
return data
def GetHideTerminalSubprocessStartupInfo ( ) :
if HC . PLATFORM_WINDOWS :
# This suppresses the terminal window that tends to pop up when calling ffmpeg or whatever
startupinfo = subprocess . STARTUPINFO ( )
startupinfo . dwFlags | = subprocess . STARTF_USESHOWWINDOW
else :
startupinfo = None
return startupinfo
2017-06-14 21:19:11 +00:00
def GetNow ( ) :
return int ( time . time ( ) )
2015-03-25 22:04:19 +00:00
def GetNowPrecise ( ) :
2017-06-14 21:19:11 +00:00
if HC . PLATFORM_WINDOWS :
return time . clock ( )
else :
return time . time ( )
2015-03-25 22:04:19 +00:00
2016-06-15 18:59:44 +00:00
def GetSiblingProcessPorts ( db_path , instance ) :
2015-09-16 18:11:00 +00:00
2016-06-15 18:59:44 +00:00
path = os . path . join ( db_path , instance + ' _running ' )
2015-09-16 18:11:00 +00:00
if os . path . exists ( path ) :
with open ( path , ' rb ' ) as f :
result = f . read ( )
try :
2016-03-23 19:42:56 +00:00
( pid , create_time ) = SplitByLinesep ( result )
2015-09-16 18:11:00 +00:00
pid = int ( pid )
create_time = float ( create_time )
except ValueError :
return None
try :
if psutil . pid_exists ( pid ) :
ports = [ ]
p = psutil . Process ( pid )
for conn in p . connections ( ) :
if conn . status == ' LISTEN ' :
ports . append ( int ( conn . laddr [ 1 ] ) )
return ports
except psutil . Error :
return None
return None
2015-03-25 22:04:19 +00:00
def IntelligentMassIntersect ( sets_to_reduce ) :
answer = None
sets_to_reduce = list ( sets_to_reduce )
2017-03-08 23:23:12 +00:00
def get_len ( item ) :
return len ( item )
sets_to_reduce . sort ( key = get_len )
2015-03-25 22:04:19 +00:00
for set_to_reduce in sets_to_reduce :
if len ( set_to_reduce ) == 0 : return set ( )
if answer is None : answer = set ( set_to_reduce )
else :
# same thing as union; I could go &= here, but I want to be quick, so use the function call
if len ( answer ) == 0 : return set ( )
else : answer . intersection_update ( set_to_reduce )
if answer is None : return set ( )
else : return answer
2016-06-15 18:59:44 +00:00
def IsAlreadyRunning ( db_path , instance ) :
2015-08-19 21:48:21 +00:00
2016-06-15 18:59:44 +00:00
path = os . path . join ( db_path , instance + ' _running ' )
2015-08-19 21:48:21 +00:00
2015-09-02 23:16:09 +00:00
if os . path . exists ( path ) :
2015-08-19 21:48:21 +00:00
2015-09-02 23:16:09 +00:00
with open ( path , ' rb ' ) as f :
result = f . read ( )
2015-08-19 21:48:21 +00:00
2015-09-02 23:16:09 +00:00
try :
2015-08-26 21:18:39 +00:00
2016-03-23 19:42:56 +00:00
( pid , create_time ) = SplitByLinesep ( result )
2015-09-02 23:16:09 +00:00
pid = int ( pid )
create_time = float ( create_time )
except ValueError :
return False
2015-08-26 21:18:39 +00:00
2015-08-19 21:48:21 +00:00
2015-09-02 23:16:09 +00:00
try :
2015-08-26 21:18:39 +00:00
2015-12-23 22:51:04 +00:00
me = psutil . Process ( )
if me . pid == pid and me . create_time ( ) == create_time :
# this is me! there is no conflict, lol!
# this happens when a linux process restarts with os.execl(), for instance (unlike Windows, it keeps its pid)
return False
2015-09-02 23:16:09 +00:00
if psutil . pid_exists ( pid ) :
2015-08-26 21:18:39 +00:00
2015-09-02 23:16:09 +00:00
p = psutil . Process ( pid )
2015-08-26 21:18:39 +00:00
2015-09-02 23:16:09 +00:00
if p . create_time ( ) == create_time and p . is_running ( ) :
return True
2015-08-26 21:18:39 +00:00
2015-09-02 23:16:09 +00:00
except psutil . Error :
2015-08-19 21:48:21 +00:00
2015-09-02 23:16:09 +00:00
return False
2015-08-19 21:48:21 +00:00
return False
2015-12-02 22:32:18 +00:00
def IterateHexPrefixes ( ) :
hex_chars = ' 0123456789abcdef '
for ( one , two ) in itertools . product ( hex_chars , hex_chars ) :
prefix = one + two
yield prefix
2016-08-31 19:55:14 +00:00
def LastShutdownWasBad ( db_path , instance ) :
path = os . path . join ( db_path , instance + ' _running ' )
if os . path . exists ( path ) :
return True
else :
return False
2017-01-04 22:48:23 +00:00
def MassUnion ( lists ) :
return { item for item in itertools . chain . from_iterable ( lists ) }
2016-08-31 19:55:14 +00:00
2016-12-07 22:12:52 +00:00
def MedianPop ( population ) :
# assume it has at least one and comes sorted
median_index = len ( population ) / 2
row = population . pop ( median_index )
return row
2015-03-25 22:04:19 +00:00
def MergeKeyToListDicts ( key_to_list_dicts ) :
result = collections . defaultdict ( list )
for key_to_list_dict in key_to_list_dicts :
for ( key , value ) in key_to_list_dict . items ( ) : result [ key ] . extend ( value )
return result
2015-11-18 22:44:07 +00:00
def Print ( text ) :
2017-09-20 19:47:31 +00:00
try :
print ( ToUnicode ( text ) )
except :
print ( repr ( text ) )
2015-11-18 22:44:07 +00:00
2016-03-23 19:42:56 +00:00
ShowText = Print
2017-01-04 22:48:23 +00:00
def PrintException ( e , do_wait = True ) :
2016-03-23 19:42:56 +00:00
if isinstance ( e , HydrusExceptions . ShutdownException ) :
return
etype = type ( e )
value = ToUnicode ( e )
trace_list = traceback . format_stack ( )
trace = ' ' . join ( trace_list )
message = ToUnicode ( etype . __name__ ) + ' : ' + ToUnicode ( value ) + os . linesep + ToUnicode ( trace )
Print ( ' ' )
Print ( ' Exception: ' )
DebugPrint ( message )
2017-01-04 22:48:23 +00:00
if do_wait :
time . sleep ( 1 )
2016-03-23 19:42:56 +00:00
ShowException = PrintException
2017-06-14 21:19:11 +00:00
def Profile ( summary , code , g , l , min_duration_ms = 20 ) :
2015-11-25 22:00:57 +00:00
profile = cProfile . Profile ( )
2017-06-14 21:19:11 +00:00
started = GetNowPrecise ( )
2015-11-25 22:00:57 +00:00
2017-06-14 21:19:11 +00:00
profile . runctx ( code , g , l )
2015-11-25 22:00:57 +00:00
2017-06-14 21:19:11 +00:00
finished = GetNowPrecise ( )
2015-11-25 22:00:57 +00:00
2017-07-27 00:47:13 +00:00
time_took = finished - started
time_took_ms = int ( time_took * 1000.0 )
if time_took_ms > min_duration_ms :
2017-06-14 21:19:11 +00:00
output = cStringIO . StringIO ( )
stats = pstats . Stats ( profile , stream = output )
stats . strip_dirs ( )
stats . sort_stats ( ' tottime ' )
output . write ( ' Stats ' )
output . write ( os . linesep * 2 )
stats . print_stats ( )
output . write ( ' Callers ' )
output . write ( os . linesep * 2 )
stats . print_callers ( )
output . seek ( 0 )
details = output . read ( )
else :
2017-07-27 00:47:13 +00:00
summary + = ' - It took ' + ConvertIntToPrettyString ( time_took_ms ) + ' ms. '
details = ' '
2017-06-14 21:19:11 +00:00
2015-11-25 22:00:57 +00:00
2017-06-14 21:19:11 +00:00
HG . controller . PrintProfile ( summary , details )
2015-11-25 22:00:57 +00:00
2016-11-30 20:24:17 +00:00
def RandomPop ( population ) :
random_index = random . randint ( 0 , len ( population ) - 1 )
row = population . pop ( random_index )
return row
2016-06-15 18:59:44 +00:00
def RecordRunningStart ( db_path , instance ) :
2015-09-02 23:16:09 +00:00
2016-06-15 18:59:44 +00:00
path = os . path . join ( db_path , instance + ' _running ' )
2015-09-02 23:16:09 +00:00
record_string = ' '
try :
me = psutil . Process ( )
record_string + = str ( me . pid )
record_string + = os . linesep
record_string + = str ( me . create_time ( ) )
except psutil . Error :
return
with open ( path , ' wb ' ) as f :
2015-11-18 22:44:07 +00:00
f . write ( ToByteString ( record_string ) )
2015-09-02 23:16:09 +00:00
2015-12-16 22:41:06 +00:00
def RestartProcess ( ) :
time . sleep ( 1 ) # time for ports to unmap
2016-11-30 20:24:17 +00:00
exe = sys . executable
me = sys . argv [ 0 ]
2016-12-07 22:12:52 +00:00
if HC . RUNNING_FROM_SOURCE :
2016-11-30 20:24:17 +00:00
2016-12-07 22:12:52 +00:00
# exe is python's exe, me is the script
2016-11-30 20:24:17 +00:00
args = [ sys . executable ] + sys . argv
else :
# we are running a frozen release--both exe and me are the built exe
# wrap it in quotes because pyinstaller passes it on as raw text, breaking any path with spaces :/
args = [ ' " ' + me + ' " ' ] + sys . argv [ 1 : ]
os . execv ( exe , args )
2015-12-16 22:41:06 +00:00
2016-03-23 19:42:56 +00:00
def SplayListForDB ( xs ) : return ' ( ' + ' , ' . join ( ( str ( x ) for x in xs ) ) + ' ) '
def SplitByLinesep ( raw_text ) :
2015-03-25 22:04:19 +00:00
2016-03-23 19:42:56 +00:00
if ' \r \n ' in raw_text :
2015-11-04 22:30:28 +00:00
2016-03-23 19:42:56 +00:00
return raw_text . split ( ' \r \n ' )
else :
return raw_text . split ( ' \n ' )
2015-11-04 22:30:28 +00:00
2015-03-25 22:04:19 +00:00
2015-12-30 23:44:09 +00:00
def SplitIteratorIntoChunks ( iterator , n ) :
chunk = [ ]
for item in iterator :
chunk . append ( item )
if len ( chunk ) == n :
yield chunk
chunk = [ ]
if len ( chunk ) > 0 :
yield chunk
2015-04-08 18:10:50 +00:00
def SplitListIntoChunks ( xs , n ) :
2016-04-20 20:42:21 +00:00
if isinstance ( xs , set ) :
xs = list ( xs )
2017-03-02 02:14:56 +00:00
for i in xrange ( 0 , len ( xs ) , n ) :
yield xs [ i : i + n ]
def SplitMappingListIntoChunks ( xs , n ) :
chunk_weight = 0
chunk = [ ]
for ( tag_item , hash_items ) in xs :
for chunk_of_hash_items in SplitListIntoChunks ( hash_items , n ) :
chunk . append ( ( tag_item , chunk_of_hash_items ) )
chunk_weight + = len ( chunk_of_hash_items )
if chunk_weight > n :
yield chunk
chunk_weight = 0
chunk = [ ]
if len ( chunk ) > 0 :
yield chunk
2015-04-08 18:10:50 +00:00
2015-06-24 22:10:14 +00:00
def TimeHasPassed ( timestamp ) :
2016-03-30 22:56:50 +00:00
if timestamp is None :
return False
2015-06-24 22:10:14 +00:00
return GetNow ( ) > timestamp
2015-07-15 20:28:26 +00:00
def TimeHasPassedPrecise ( precise_timestamp ) :
return GetNowPrecise ( ) > precise_timestamp
2015-08-05 18:42:35 +00:00
def TimeUntil ( timestamp ) :
return timestamp - GetNow ( )
2015-11-04 22:30:28 +00:00
def ToByteString ( text_producing_object ) :
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if isinstance ( text_producing_object , unicode ) :
2015-11-04 22:30:28 +00:00
return text_producing_object . encode ( ' utf-8 ' )
2016-01-13 22:08:19 +00:00
elif isinstance ( text_producing_object , str ) :
2015-11-04 22:30:28 +00:00
return text_producing_object
else :
try :
return str ( text_producing_object )
except :
return str ( repr ( text_producing_object ) )
2015-03-25 22:04:19 +00:00
2015-11-04 22:30:28 +00:00
def ToUnicode ( text_producing_object ) :
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if isinstance ( text_producing_object , ( str , unicode , bs4 . element . NavigableString ) ) :
2015-11-04 22:30:28 +00:00
text = text_producing_object
2015-03-25 22:04:19 +00:00
else :
2015-11-04 22:30:28 +00:00
try :
text = str ( text_producing_object ) # dealing with exceptions, etc...
except :
2017-03-29 19:39:34 +00:00
try :
text = unicode ( text_producing_object )
except :
text = repr ( text_producing_object )
2015-11-04 22:30:28 +00:00
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if not isinstance ( text , unicode ) :
2015-03-25 22:04:19 +00:00
2015-11-04 22:30:28 +00:00
try :
text = text . decode ( ' utf-8 ' )
except :
try :
text = text . decode ( locale . getpreferredencoding ( ) )
except :
2017-08-16 21:58:06 +00:00
text = unicode ( repr ( text ) )
2015-11-04 22:30:28 +00:00
2015-03-25 22:04:19 +00:00
2015-11-04 22:30:28 +00:00
return text
2016-06-22 20:59:24 +00:00
def WaitForProcessToFinish ( p , timeout ) :
started = GetNow ( )
while p . poll ( ) is None :
if TimeHasPassed ( started + timeout ) :
p . kill ( )
raise Exception ( ' Process did not finish within ' + ConvertIntToPrettyString ( timeout ) + ' seconds! ' )
time . sleep ( 2 )
2015-03-25 22:04:19 +00:00
class HydrusYAMLBase ( yaml . YAMLObject ) :
yaml_loader = yaml . SafeLoader
yaml_dumper = yaml . SafeDumper
class Account ( HydrusYAMLBase ) :
yaml_tag = u ' !Account '
def __init__ ( self , account_key , account_type , created , expires , used_bytes , used_requests , banned_info = None ) :
HydrusYAMLBase . __init__ ( self )
self . _info = { }
self . _info [ ' account_key ' ] = account_key
self . _info [ ' account_type ' ] = account_type
self . _info [ ' created ' ] = created
self . _info [ ' expires ' ] = expires
self . _info [ ' used_bytes ' ] = used_bytes
self . _info [ ' used_requests ' ] = used_requests
if banned_info is not None : self . _info [ ' banned_info ' ] = banned_info
self . _info [ ' fresh_timestamp ' ] = GetNow ( )
def __repr__ ( self ) : return self . ConvertToString ( )
def __str__ ( self ) : return self . ConvertToString ( )
def _IsBanned ( self ) :
if ' banned_info ' not in self . _info : return False
else :
( reason , created , expires ) = self . _info [ ' banned_info ' ]
if expires is None : return True
2015-06-24 22:10:14 +00:00
else : return not TimeHasPassed ( expires )
2015-03-25 22:04:19 +00:00
def _IsBytesExceeded ( self ) :
account_type = self . _info [ ' account_type ' ]
max_num_bytes = account_type . GetMaxBytes ( )
used_bytes = self . _info [ ' used_bytes ' ]
return max_num_bytes is not None and used_bytes > max_num_bytes
def _IsExpired ( self ) :
if self . _info [ ' expires ' ] is None : return False
2015-06-24 22:10:14 +00:00
else : return TimeHasPassed ( self . _info [ ' expires ' ] )
2015-03-25 22:04:19 +00:00
def _IsRequestsExceeded ( self ) :
account_type = self . _info [ ' account_type ' ]
max_num_requests = account_type . GetMaxRequests ( )
used_requests = self . _info [ ' used_requests ' ]
return max_num_requests is not None and used_requests > max_num_requests
def CheckPermission ( self , permission ) :
if self . _IsBanned ( ) : raise HydrusExceptions . PermissionException ( ' This account is banned! ' )
if self . _IsExpired ( ) : raise HydrusExceptions . PermissionException ( ' This account is expired. ' )
if self . _IsBytesExceeded ( ) : raise HydrusExceptions . PermissionException ( ' You have hit your data transfer limit, and cannot make any more requests for the month. ' )
if self . _IsRequestsExceeded ( ) : raise HydrusExceptions . PermissionException ( ' You have hit your requests limit, and cannot make any more requests for the month. ' )
if not self . _info [ ' account_type ' ] . HasPermission ( permission ) : raise HydrusExceptions . PermissionException ( ' You do not have permission to do that. ' )
def ConvertToString ( self ) : return ConvertTimestampToPrettyAge ( self . _info [ ' created ' ] ) + os . linesep + self . _info [ ' account_type ' ] . ConvertToString ( ) + os . linesep + ' which ' + ConvertTimestampToPrettyExpires ( self . _info [ ' expires ' ] )
def GetAccountKey ( self ) : return self . _info [ ' account_key ' ]
def GetAccountType ( self ) : return self . _info [ ' account_type ' ]
def GetCreated ( self ) : return self . _info [ ' created ' ]
def GetExpires ( self ) : return self . _info [ ' expires ' ]
def GetExpiresString ( self ) :
if self . _IsBanned ( ) :
( reason , created , expires ) = self . _info [ ' banned_info ' ]
return ' banned ' + ConvertTimestampToPrettyAge ( created ) + ' , ' + ConvertTimestampToPrettyExpires ( expires ) + ' because: ' + reason
else : return ConvertTimestampToPrettyAge ( self . _info [ ' created ' ] ) + ' and ' + ConvertTimestampToPrettyExpires ( self . _info [ ' expires ' ] )
def GetUsedBytesString ( self ) :
max_num_bytes = self . _info [ ' account_type ' ] . GetMaxBytes ( )
used_bytes = self . _info [ ' used_bytes ' ]
if max_num_bytes is None : return ConvertIntToBytes ( used_bytes ) + ' used this month '
else : return ConvertIntToBytes ( used_bytes ) + ' / ' + ConvertIntToBytes ( max_num_bytes ) + ' used this month '
def GetUsedRequestsString ( self ) :
max_num_requests = self . _info [ ' account_type ' ] . GetMaxRequests ( )
used_requests = self . _info [ ' used_requests ' ]
if max_num_requests is None : return ConvertIntToPrettyString ( used_requests ) + ' requests used this month '
2015-06-10 19:40:25 +00:00
else : return ConvertValueRangeToPrettyString ( used_requests , max_num_requests ) + ' requests used this month '
2015-03-25 22:04:19 +00:00
def GetUsedBytes ( self ) : return self . _info [ ' used_bytes ' ]
def GetUsedRequests ( self ) : return self . _info [ ' used_bytes ' ]
def HasAccountKey ( self ) :
if ' account_key ' in self . _info and self . _info [ ' account_key ' ] is not None : return True
return False
def HasPermission ( self , permission ) :
if self . _IsBanned ( ) : return False
if self . _IsExpired ( ) : return False
if self . _IsBytesExceeded ( ) : return False
if self . _IsRequestsExceeded ( ) : return False
return self . _info [ ' account_type ' ] . HasPermission ( permission )
def IsBanned ( self ) : return self . _IsBanned ( )
def IsStale ( self ) : return self . _info [ ' fresh_timestamp ' ] + HC . UPDATE_DURATION * 5 < GetNow ( )
def IsUnknownAccount ( self ) : return self . _info [ ' account_type ' ] . IsUnknownAccountType ( )
def MakeFresh ( self ) : self . _info [ ' fresh_timestamp ' ] = GetNow ( )
def MakeStale ( self ) : self . _info [ ' fresh_timestamp ' ] = 0
2017-06-21 21:15:59 +00:00
def ReportDataUsed ( self , num_bytes ) :
2015-03-25 22:04:19 +00:00
self . _info [ ' used_bytes ' ] + = num_bytes
2017-06-21 21:15:59 +00:00
def ReportRequestUsed ( self ) :
2015-03-25 22:04:19 +00:00
self . _info [ ' used_requests ' ] + = 1
sqlite3 . register_adapter ( Account , yaml . safe_dump )
2015-10-14 21:02:25 +00:00
class AccountIdentifier ( HydrusSerialisable . SerialisableBase ) :
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
SERIALISABLE_TYPE = HydrusSerialisable . SERIALISABLE_TYPE_ACCOUNT_IDENTIFIER
SERIALISABLE_VERSION = 1
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
TYPE_ACCOUNT_KEY = 1
TYPE_CONTENT = 2
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
def __init__ ( self , account_key = None , content = None ) :
2015-03-25 22:04:19 +00:00
HydrusYAMLBase . __init__ ( self )
if account_key is not None :
self . _type = self . TYPE_ACCOUNT_KEY
self . _data = account_key
2015-10-14 21:02:25 +00:00
elif content is not None :
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
self . _type = self . TYPE_CONTENT
self . _data = content
2015-03-25 22:04:19 +00:00
def __eq__ ( self , other ) : return self . __hash__ ( ) == other . __hash__ ( )
def __hash__ ( self ) : return ( self . _type , self . _data ) . __hash__ ( )
def __ne__ ( self , other ) : return self . __hash__ ( ) != other . __hash__ ( )
2015-11-04 22:30:28 +00:00
def __repr__ ( self ) : return ' Account Identifier: ' + ToUnicode ( ( self . _type , self . _data ) )
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
def _GetSerialisableInfo ( self ) :
if self . _type == self . TYPE_ACCOUNT_KEY :
serialisable_data = self . _data . encode ( ' hex ' )
elif self . _type == self . TYPE_CONTENT :
serialisable_data = self . _data . GetSerialisableTuple ( )
return ( self . _type , serialisable_data )
def _InitialiseFromSerialisableInfo ( self , serialisable_info ) :
( self . _type , serialisable_data ) = serialisable_info
if self . _type == self . TYPE_ACCOUNT_KEY :
self . _data = serialisable_data . decode ( ' hex ' )
elif self . _type == self . TYPE_CONTENT :
self . _data = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_data )
2015-03-25 22:04:19 +00:00
def GetData ( self ) : return self . _data
def HasAccountKey ( self ) : return self . _type == self . TYPE_ACCOUNT_KEY
2015-10-14 21:02:25 +00:00
def HasContent ( self ) : return self . _type == self . TYPE_CONTENT
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
HydrusSerialisable . SERIALISABLE_TYPES_TO_OBJECT_TYPES [ HydrusSerialisable . SERIALISABLE_TYPE_ACCOUNT_IDENTIFIER ] = AccountIdentifier
2015-03-25 22:04:19 +00:00
class AccountType ( HydrusYAMLBase ) :
yaml_tag = u ' !AccountType '
def __init__ ( self , title , permissions , max_monthly_data ) :
HydrusYAMLBase . __init__ ( self )
self . _title = title
self . _permissions = permissions
self . _max_monthly_data = max_monthly_data
def __repr__ ( self ) : return self . ConvertToString ( )
def GetPermissions ( self ) : return self . _permissions
def GetTitle ( self ) : return self . _title
def GetMaxBytes ( self ) :
( max_num_bytes , max_num_requests ) = self . _max_monthly_data
return max_num_bytes
def GetMaxRequests ( self ) :
( max_num_bytes , max_num_requests ) = self . _max_monthly_data
return max_num_requests
def GetMaxBytesString ( self ) :
( max_num_bytes , max_num_requests ) = self . _max_monthly_data
if max_num_bytes is None : max_num_bytes_string = ' No limit '
else : max_num_bytes_string = ConvertIntToBytes ( max_num_bytes )
return max_num_bytes_string
def GetMaxRequestsString ( self ) :
( max_num_bytes , max_num_requests ) = self . _max_monthly_data
if max_num_requests is None : max_num_requests_string = ' No limit '
else : max_num_requests_string = ConvertIntToPrettyString ( max_num_requests )
return max_num_requests_string
def ConvertToString ( self ) :
result_string = self . _title + ' with '
if self . _permissions == [ HC . UNKNOWN_PERMISSION ] : result_string + = ' no permissions '
else : result_string + = ' , ' . join ( [ HC . permissions_string_lookup [ permission ] for permission in self . _permissions ] ) + ' permissions '
return result_string
def IsUnknownAccountType ( self ) : return self . _permissions == [ HC . UNKNOWN_PERMISSION ]
def HasPermission ( self , permission ) : return permission in self . _permissions
sqlite3 . register_adapter ( AccountType , yaml . safe_dump )
2016-01-06 21:17:20 +00:00
class BigJobPauser ( object ) :
def __init__ ( self , period = 10 , wait_time = 0.1 ) :
self . _period = period
self . _wait_time = wait_time
self . _next_pause = GetNow ( ) + self . _period
def Pause ( self ) :
if TimeHasPassed ( self . _next_pause ) :
time . sleep ( self . _wait_time )
self . _next_pause = GetNow ( ) + self . _period
2017-01-25 22:56:55 +00:00
class Call ( object ) :
def __init__ ( self , func , * args , * * kwargs ) :
self . _func = func
self . _args = args
self . _kwargs = kwargs
def __call__ ( self ) :
self . _func ( * self . _args , * * self . _kwargs )
2015-03-25 22:04:19 +00:00
class ContentUpdate ( object ) :
def __init__ ( self , data_type , action , row ) :
self . _data_type = data_type
self . _action = action
self . _row = row
2017-04-26 21:58:12 +00:00
def __eq__ ( self , other ) :
return hash ( self ) == hash ( other )
2015-03-25 22:04:19 +00:00
def __ne__ ( self , other ) : return not self . __eq__ ( other )
2017-04-26 21:58:12 +00:00
def __hash__ ( self ) :
return hash ( ( self . _data_type , self . _action , repr ( self . _row ) ) )
def __repr__ ( self ) :
return ' Content Update: ' + ToUnicode ( ( self . _data_type , self . _action , self . _row ) )
2015-03-25 22:04:19 +00:00
2017-07-27 00:47:13 +00:00
def GetAction ( self ) :
return self . _action
def GetDataType ( self ) :
return self . _data_type
2015-03-25 22:04:19 +00:00
def GetHashes ( self ) :
2015-10-14 21:02:25 +00:00
if self . _data_type == HC . CONTENT_TYPE_FILES :
2015-03-25 22:04:19 +00:00
2016-02-17 22:06:47 +00:00
if self . _action == HC . CONTENT_UPDATE_ADVANCED :
hashes = set ( )
elif self . _action == HC . CONTENT_UPDATE_ADD :
2015-03-25 22:04:19 +00:00
2017-05-31 21:50:53 +00:00
( file_info_manager , timestamp ) = self . _row
hash = file_info_manager . GetHash ( )
2015-03-25 22:04:19 +00:00
2017-05-10 21:33:58 +00:00
hashes = { hash }
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
elif self . _action in ( HC . CONTENT_UPDATE_ARCHIVE , HC . CONTENT_UPDATE_DELETE , HC . CONTENT_UPDATE_UNDELETE , HC . CONTENT_UPDATE_INBOX , HC . CONTENT_UPDATE_PEND , HC . CONTENT_UPDATE_RESCIND_PEND , HC . CONTENT_UPDATE_RESCIND_PETITION ) :
hashes = self . _row
elif self . _action == HC . CONTENT_UPDATE_PETITION :
( hashes , reason ) = self . _row
2015-03-25 22:04:19 +00:00
2016-05-25 21:54:03 +00:00
elif self . _data_type == HC . CONTENT_TYPE_DIRECTORIES :
hashes = set ( )
2017-05-10 21:33:58 +00:00
elif self . _data_type == HC . CONTENT_TYPE_URLS :
( hash , urls ) = self . _row
hashes = { hash }
2015-10-14 21:02:25 +00:00
elif self . _data_type == HC . CONTENT_TYPE_MAPPINGS :
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if self . _action == HC . CONTENT_UPDATE_ADVANCED :
hashes = set ( )
elif self . _action == HC . CONTENT_UPDATE_PETITION :
( tag , hashes , reason ) = self . _row
else :
( tag , hashes ) = self . _row
elif self . _data_type in ( HC . CONTENT_TYPE_TAG_PARENTS , HC . CONTENT_TYPE_TAG_SIBLINGS ) :
hashes = set ( )
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
elif self . _data_type == HC . CONTENT_TYPE_RATINGS :
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if self . _action == HC . CONTENT_UPDATE_ADD :
( rating , hashes ) = self . _row
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if not isinstance ( hashes , set ) :
hashes = set ( hashes )
2015-03-25 22:04:19 +00:00
return hashes
2016-01-06 21:17:20 +00:00
def GetWeight ( self ) :
return len ( self . GetHashes ( ) )
2015-11-18 22:44:07 +00:00
def IsInboxRelated ( self ) :
return self . _action in ( HC . CONTENT_UPDATE_ARCHIVE , HC . CONTENT_UPDATE_INBOX )
def ToTuple ( self ) :
return ( self . _data_type , self . _action , self . _row )
2015-03-25 22:04:19 +00:00
class EditLogAction ( object ) :
yaml_tag = u ' !EditLogAction '
def __init__ ( self , action ) : self . _action = action
def GetAction ( self ) : return self . _action
class EditLogActionAdd ( EditLogAction ) :
yaml_tag = u ' !EditLogActionAdd '
def __init__ ( self , data ) :
EditLogAction . __init__ ( self , HC . ADD )
self . _data = data
def GetData ( self ) : return self . _data
class EditLogActionDelete ( EditLogAction ) :
yaml_tag = u ' !EditLogActionDelete '
def __init__ ( self , identifier ) :
EditLogAction . __init__ ( self , HC . DELETE )
self . _identifier = identifier
def GetIdentifier ( self ) : return self . _identifier
class EditLogActionEdit ( EditLogAction ) :
yaml_tag = u ' !EditLogActionEdit '
def __init__ ( self , identifier , data ) :
EditLogAction . __init__ ( self , HC . EDIT )
self . _identifier = identifier
self . _data = data
def GetData ( self ) : return self . _data
def GetIdentifier ( self ) : return self . _identifier
class JobDatabase ( object ) :
2016-03-30 22:56:50 +00:00
def __init__ ( self , job_type , synchronous , action , * args , * * kwargs ) :
2015-03-25 22:04:19 +00:00
self . _type = job_type
self . _synchronous = synchronous
2016-03-30 22:56:50 +00:00
self . _action = action
2015-03-25 22:04:19 +00:00
self . _args = args
self . _kwargs = kwargs
self . _result_ready = threading . Event ( )
2016-03-30 22:56:50 +00:00
def GetCallableTuple ( self ) :
return ( self . _action , self . _args , self . _kwargs )
2015-03-25 22:04:19 +00:00
def GetResult ( self ) :
while True :
2016-09-28 18:48:01 +00:00
if self . _result_ready . wait ( 2 ) == True :
break
2017-05-10 21:33:58 +00:00
elif HG . model_shutdown :
2016-09-28 18:48:01 +00:00
raise HydrusExceptions . ShutdownException ( ' Application quit before db could serve result! ' )
2015-03-25 22:04:19 +00:00
2015-11-11 21:20:41 +00:00
if isinstance ( self . _result , Exception ) :
2015-06-03 21:05:13 +00:00
2016-09-28 18:48:01 +00:00
e = self . _result
raise e
2015-03-25 22:04:19 +00:00
2016-01-20 23:57:33 +00:00
else :
return self . _result
2015-03-25 22:04:19 +00:00
2016-09-28 18:48:01 +00:00
def GetType ( self ) :
return self . _type
2015-03-25 22:04:19 +00:00
def IsSynchronous ( self ) : return self . _synchronous
def PutResult ( self , result ) :
self . _result = result
self . _result_ready . set ( )
2016-03-30 22:56:50 +00:00
def ToString ( self ) :
return self . _type + ' ' + self . _action
2015-03-25 22:04:19 +00:00
class ServiceUpdate ( object ) :
def __init__ ( self , action , row = None ) :
self . _action = action
self . _row = row
2017-03-02 02:14:56 +00:00
def ToTuple ( self ) :
return ( self . _action , self . _row )
2015-03-25 22:04:19 +00:00