2015-03-25 22:04:19 +00:00
import bs4
import collections
2015-11-25 22:00:57 +00:00
import cProfile
import cStringIO
2015-03-25 22:04:19 +00:00
import HydrusConstants as HC
import HydrusExceptions
import HydrusGlobals
2015-06-03 21:05:13 +00:00
import HydrusSerialisable
2015-03-25 22:04:19 +00:00
import locale
import os
2015-11-25 22:00:57 +00:00
import pstats
2015-08-19 21:48:21 +00:00
import psutil
2015-09-16 18:11:00 +00:00
import shutil
2015-03-25 22:04:19 +00:00
import sqlite3
2015-06-17 20:01:41 +00:00
import subprocess
2015-03-25 22:04:19 +00:00
import sys
import threading
import time
import traceback
import yaml
import itertools
def default_dict_list ( ) : return collections . defaultdict ( list )
def default_dict_set ( ) : return collections . defaultdict ( set )
def BuildKeyToListDict ( pairs ) :
d = collections . defaultdict ( list )
for ( key , value ) in pairs : d [ key ] . append ( value )
return d
def BuildKeyToSetDict ( pairs ) :
d = collections . defaultdict ( set )
for ( key , value ) in pairs : d [ key ] . add ( value )
return d
def CalculateScoreFromRating ( count , rating ) :
# http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
count = float ( count )
positive = count * rating
negative = count * ( 1.0 - rating )
# positive + negative = count
# I think I've parsed this correctly from the website! Not sure though!
score = ( ( positive + 1.9208 ) / count - 1.96 * ( ( ( positive * negative ) / count + 0.9604 ) * * 0.5 ) / count ) / ( 1 + 3.8416 / count )
return score
2015-10-14 21:02:25 +00:00
def ConvertContentsToClientToServerContentUpdatePackage ( action , contents , reason = None ) :
hashes_to_hash_ids = { }
hash_ids_to_hashes = { }
hash_i = 0
content_data_dict = GetEmptyDataDict ( )
for content in contents :
content_type = content . GetContentType ( )
content_data = content . GetContent ( )
if content_type == HC . CONTENT_TYPE_FILES :
hashes = content_data
elif content_type == HC . CONTENT_TYPE_MAPPINGS :
( tag , hashes ) = content_data
elif content_type in ( HC . CONTENT_TYPE_TAG_PARENTS , HC . CONTENT_TYPE_TAG_SIBLINGS ) :
( old_tag , new_tag ) = content_data
hashes = set ( )
hash_ids = [ ]
for hash in hashes :
if hash not in hashes_to_hash_ids :
hashes_to_hash_ids [ hash ] = hash_i
hash_ids_to_hashes [ hash_i ] = hash
hash_i + = 1
hash_ids . append ( hashes_to_hash_ids [ hash ] )
if action in ( HC . CONTENT_UPDATE_PEND , HC . CONTENT_UPDATE_PETITION ) :
if reason is None :
reason = ' admin '
if content_type == HC . CONTENT_TYPE_FILES :
row = ( hash_ids , reason )
elif content_type == HC . CONTENT_TYPE_MAPPINGS :
row = ( tag , hash_ids , reason )
else :
row = ( ( old_tag , new_tag ) , reason )
elif action in ( HC . CONTENT_UPDATE_DENY_PEND , HC . CONTENT_UPDATE_DENY_PETITION ) :
if content_type == HC . CONTENT_TYPE_FILES :
row = hash_ids
elif content_type == HC . CONTENT_TYPE_MAPPINGS :
row = ( tag , hash_ids )
elif content_type in ( HC . CONTENT_TYPE_TAG_PARENTS , HC . CONTENT_TYPE_TAG_SIBLINGS ) :
row = ( old_tag , new_tag )
content_data_dict [ content_type ] [ action ] . append ( row )
return ClientToServerContentUpdatePackage ( content_data_dict , hash_ids_to_hashes )
2015-03-25 22:04:19 +00:00
def ConvertIntToBytes ( size ) :
if size is None : return ' unknown size '
suffixes = ( ' ' , ' K ' , ' M ' , ' G ' , ' T ' , ' P ' )
suffix_index = 0
size = float ( size )
while size > 1024.0 :
size = size / 1024.0
suffix_index + = 1
if size < 10.0 : return ' %.1f ' % size + suffixes [ suffix_index ] + ' B '
return ' %.0f ' % size + suffixes [ suffix_index ] + ' B '
2015-04-08 18:10:50 +00:00
def ConvertIntToPixels ( i ) :
if i == 1 : return ' pixels '
elif i == 1000 : return ' kilopixels '
elif i == 1000000 : return ' megapixels '
2015-04-25 22:31:50 +00:00
else : return ' megapixels '
2015-04-08 18:10:50 +00:00
2015-11-11 21:20:41 +00:00
def ConvertIntToPrettyString ( num ) :
2015-11-18 22:44:07 +00:00
# don't feed this a unicode string u'%d'--locale can't handle it
text = locale . format ( ' %d ' , num , grouping = True )
try :
text = text . decode ( locale . getpreferredencoding ( ) )
text = ToUnicode ( text )
except :
text = ToUnicode ( text )
return text
2015-11-11 21:20:41 +00:00
2015-04-22 22:57:25 +00:00
def ConvertIntToUnit ( unit ) :
if unit == 1 : return ' B '
elif unit == 1024 : return ' KB '
elif unit == 1048576 : return ' MB '
elif unit == 1073741824 : return ' GB '
2015-03-25 22:04:19 +00:00
def ConvertMillisecondsToPrettyTime ( ms ) :
hours = ms / 3600000
if hours == 1 : hours_result = ' 1 hour '
2015-11-04 22:30:28 +00:00
else : hours_result = str ( hours ) + ' hours '
2015-03-25 22:04:19 +00:00
ms = ms % 3600000
minutes = ms / 60000
if minutes == 1 : minutes_result = ' 1 minute '
2015-11-04 22:30:28 +00:00
else : minutes_result = str ( minutes ) + ' minutes '
2015-03-25 22:04:19 +00:00
ms = ms % 60000
seconds = ms / 1000
if seconds == 1 : seconds_result = ' 1 second '
2015-11-04 22:30:28 +00:00
else : seconds_result = str ( seconds ) + ' seconds '
2015-03-25 22:04:19 +00:00
detailed_seconds = float ( ms ) / 1000.0
if detailed_seconds == 1.0 : detailed_seconds_result = ' 1.0 seconds '
else : detailed_seconds_result = ' %.1f ' % detailed_seconds + ' seconds '
ms = ms % 1000
if ms == 1 : milliseconds_result = ' 1 millisecond '
2015-11-04 22:30:28 +00:00
else : milliseconds_result = str ( ms ) + ' milliseconds '
2015-03-25 22:04:19 +00:00
if hours > 0 : return hours_result + ' ' + minutes_result
if minutes > 0 : return minutes_result + ' ' + seconds_result
if seconds > 0 : return detailed_seconds_result
return milliseconds_result
def ConvertNumericalRatingToPrettyString ( lower , upper , rating , rounded_result = False , out_of = True ) :
rating_converted = ( rating * ( upper - lower ) ) + lower
2015-11-04 22:30:28 +00:00
if rounded_result : s = ' %.2f ' % round ( rating_converted )
else : s = ' %.2f ' % rating_converted
2015-03-25 22:04:19 +00:00
if out_of :
2015-11-04 22:30:28 +00:00
if lower in ( 0 , 1 ) : s + = ' / %.2f ' % upper
2015-03-25 22:04:19 +00:00
return s
2015-04-22 22:57:25 +00:00
def ConvertPixelsToInt ( unit ) :
2015-04-08 18:10:50 +00:00
if unit == ' pixels ' : return 1
elif unit == ' kilopixels ' : return 1000
elif unit == ' megapixels ' : return 1000000
2015-03-25 22:04:19 +00:00
def ConvertPrettyStringsToUglyNamespaces ( pretty_strings ) :
result = { s for s in pretty_strings if s != ' no namespace ' }
if ' no namespace ' in pretty_strings : result . add ( ' ' )
return result
2015-11-18 22:44:07 +00:00
def ConvertTimeDeltaToPrettyString ( seconds ) :
2015-12-09 23:16:41 +00:00
if seconds > 60 :
seconds = int ( seconds )
if seconds > 86400 :
days = seconds / 86400
hours = ( seconds % 86400 ) / 3600
result = ' %d ' % days + ' days ' + ' %d ' % hours + ' hours '
elif seconds > 3600 :
hours = seconds / 3600
minutes = ( seconds % 3600 ) / 60
result = ' %d ' % hours + ' hours ' + ' %d ' % minutes + ' minutes '
else :
minutes = seconds / 60
seconds = seconds % 60
result = ' %d ' % minutes + ' minutes ' + ' %d ' % seconds + ' seconds '
elif seconds > 1 :
2015-11-18 22:44:07 +00:00
result = ' %.1f ' % seconds + ' seconds '
elif seconds > 0.1 :
result = ' %d ' % ( seconds * 1000 ) + ' milliseconds '
elif seconds > 0.01 :
result = ' %.1f ' % ( seconds * 1000 ) + ' milliseconds '
elif seconds > 0.001 :
result = ' %.2f ' % ( seconds * 1000 ) + ' milliseconds '
else :
result = ' %d ' % ( seconds * 1000000 ) + ' microseconds '
return result
2015-03-25 22:04:19 +00:00
def ConvertStatusToPrefix ( status ) :
if status == HC . CURRENT : return ' '
elif status == HC . PENDING : return ' (+) '
elif status == HC . PETITIONED : return ' (-) '
elif status == HC . DELETED : return ' (X) '
elif status == HC . DELETED_PENDING : return ' (X+) '
def ConvertTimestampToPrettyAge ( timestamp ) :
if timestamp == 0 or timestamp is None : return ' unknown age '
age = GetNow ( ) - timestamp
seconds = age % 60
if seconds == 1 : s = ' 1 second '
2015-11-04 22:30:28 +00:00
else : s = str ( seconds ) + ' seconds '
2015-03-25 22:04:19 +00:00
age = age / 60
minutes = age % 60
if minutes == 1 : m = ' 1 minute '
2015-11-04 22:30:28 +00:00
else : m = str ( minutes ) + ' minutes '
2015-03-25 22:04:19 +00:00
age = age / 60
hours = age % 24
if hours == 1 : h = ' 1 hour '
2015-11-04 22:30:28 +00:00
else : h = str ( hours ) + ' hours '
2015-03-25 22:04:19 +00:00
age = age / 24
days = age % 30
if days == 1 : d = ' 1 day '
2015-11-04 22:30:28 +00:00
else : d = str ( days ) + ' days '
2015-03-25 22:04:19 +00:00
age = age / 30
months = age % 12
if months == 1 : mo = ' 1 month '
2015-11-04 22:30:28 +00:00
else : mo = str ( months ) + ' months '
2015-03-25 22:04:19 +00:00
years = age / 12
if years == 1 : y = ' 1 year '
2015-11-04 22:30:28 +00:00
else : y = str ( years ) + ' years '
2015-03-25 22:04:19 +00:00
if years > 0 : return ' ' . join ( ( y , mo ) ) + ' old '
elif months > 0 : return ' ' . join ( ( mo , d ) ) + ' old '
elif days > 0 : return ' ' . join ( ( d , h ) ) + ' old '
elif hours > 0 : return ' ' . join ( ( h , m ) ) + ' old '
else : return ' ' . join ( ( m , s ) ) + ' old '
def ConvertTimestampToPrettyAgo ( timestamp ) :
if timestamp is None or timestamp == 0 : return ' unknown time '
age = GetNow ( ) - timestamp
seconds = age % 60
if seconds == 1 : s = ' 1 second '
2015-11-04 22:30:28 +00:00
else : s = str ( seconds ) + ' seconds '
2015-03-25 22:04:19 +00:00
age = age / 60
minutes = age % 60
if minutes == 1 : m = ' 1 minute '
2015-11-04 22:30:28 +00:00
else : m = str ( minutes ) + ' minutes '
2015-03-25 22:04:19 +00:00
age = age / 60
hours = age % 24
if hours == 1 : h = ' 1 hour '
2015-11-04 22:30:28 +00:00
else : h = str ( hours ) + ' hours '
2015-03-25 22:04:19 +00:00
age = age / 24
days = age % 30
if days == 1 : d = ' 1 day '
2015-11-04 22:30:28 +00:00
else : d = str ( days ) + ' days '
2015-03-25 22:04:19 +00:00
age = age / 30
months = age % 12
if months == 1 : mo = ' 1 month '
2015-11-04 22:30:28 +00:00
else : mo = str ( months ) + ' months '
2015-03-25 22:04:19 +00:00
years = age / 12
if years == 1 : y = ' 1 year '
2015-11-04 22:30:28 +00:00
else : y = str ( years ) + ' years '
2015-03-25 22:04:19 +00:00
if years > 0 : return ' ' . join ( ( y , mo ) ) + ' ago '
elif months > 0 : return ' ' . join ( ( mo , d ) ) + ' ago '
elif days > 0 : return ' ' . join ( ( d , h ) ) + ' ago '
elif hours > 0 : return ' ' . join ( ( h , m ) ) + ' ago '
else : return ' ' . join ( ( m , s ) ) + ' ago '
def ConvertTimestampToPrettyExpires ( timestamp ) :
if timestamp is None : return ' does not expire '
if timestamp == 0 : return ' unknown expiration '
expires = GetNow ( ) - timestamp
if expires > = 0 : already_happend = True
else :
expires * = - 1
already_happend = False
seconds = expires % 60
if seconds == 1 : s = ' 1 second '
2015-11-04 22:30:28 +00:00
else : s = str ( seconds ) + ' seconds '
2015-03-25 22:04:19 +00:00
expires = expires / 60
minutes = expires % 60
if minutes == 1 : m = ' 1 minute '
2015-11-04 22:30:28 +00:00
else : m = str ( minutes ) + ' minutes '
2015-03-25 22:04:19 +00:00
expires = expires / 60
hours = expires % 24
if hours == 1 : h = ' 1 hour '
2015-11-04 22:30:28 +00:00
else : h = str ( hours ) + ' hours '
2015-03-25 22:04:19 +00:00
expires = expires / 24
days = expires % 30
if days == 1 : d = ' 1 day '
2015-11-04 22:30:28 +00:00
else : d = str ( days ) + ' days '
2015-03-25 22:04:19 +00:00
expires = expires / 30
months = expires % 12
if months == 1 : mo = ' 1 month '
2015-11-04 22:30:28 +00:00
else : mo = str ( months ) + ' months '
2015-03-25 22:04:19 +00:00
years = expires / 12
if years == 1 : y = ' 1 year '
2015-11-04 22:30:28 +00:00
else : y = str ( years ) + ' years '
2015-03-25 22:04:19 +00:00
if already_happend :
if years > 0 : return ' expired ' + ' ' . join ( ( y , mo ) ) + ' ago '
elif months > 0 : return ' expired ' + ' ' . join ( ( mo , d ) ) + ' ago '
elif days > 0 : return ' expired ' + ' ' . join ( ( d , h ) ) + ' ago '
elif hours > 0 : return ' expired ' + ' ' . join ( ( h , m ) ) + ' ago '
else : return ' expired ' + ' ' . join ( ( m , s ) ) + ' ago '
else :
if years > 0 : return ' expires in ' + ' ' . join ( ( y , mo ) )
elif months > 0 : return ' expires in ' + ' ' . join ( ( mo , d ) )
elif days > 0 : return ' expires in ' + ' ' . join ( ( d , h ) )
elif hours > 0 : return ' expires in ' + ' ' . join ( ( h , m ) )
else : return ' expires in ' + ' ' . join ( ( m , s ) )
def ConvertTimestampToPrettyPending ( timestamp ) :
if timestamp is None : return ' '
if timestamp == 0 : return ' imminent '
pending = GetNow ( ) - timestamp
if pending > = 0 : return ' imminent '
else : pending * = - 1
seconds = pending % 60
if seconds == 1 : s = ' 1 second '
2015-11-04 22:30:28 +00:00
else : s = str ( seconds ) + ' seconds '
2015-03-25 22:04:19 +00:00
pending = pending / 60
minutes = pending % 60
if minutes == 1 : m = ' 1 minute '
2015-11-04 22:30:28 +00:00
else : m = str ( minutes ) + ' minutes '
2015-03-25 22:04:19 +00:00
pending = pending / 60
hours = pending % 24
if hours == 1 : h = ' 1 hour '
2015-11-04 22:30:28 +00:00
else : h = str ( hours ) + ' hours '
2015-03-25 22:04:19 +00:00
pending = pending / 24
days = pending % 30
if days == 1 : d = ' 1 day '
2015-11-04 22:30:28 +00:00
else : d = str ( days ) + ' days '
2015-03-25 22:04:19 +00:00
pending = pending / 30
months = pending % 12
if months == 1 : mo = ' 1 month '
2015-11-04 22:30:28 +00:00
else : mo = str ( months ) + ' months '
2015-03-25 22:04:19 +00:00
years = pending / 12
if years == 1 : y = ' 1 year '
2015-11-04 22:30:28 +00:00
else : y = str ( years ) + ' years '
2015-03-25 22:04:19 +00:00
if years > 0 : return ' in ' + ' ' . join ( ( y , mo ) )
elif months > 0 : return ' in ' + ' ' . join ( ( mo , d ) )
elif days > 0 : return ' in ' + ' ' . join ( ( d , h ) )
elif hours > 0 : return ' in ' + ' ' . join ( ( h , m ) )
else : return ' in ' + ' ' . join ( ( m , s ) )
def ConvertTimestampToPrettySync ( timestamp ) :
2015-10-07 21:56:22 +00:00
if timestamp is None or timestamp == 0 : return ' (initial sync has not yet occured) '
2015-03-25 22:04:19 +00:00
age = GetNow ( ) - timestamp
seconds = age % 60
if seconds == 1 : s = ' 1 second '
2015-11-04 22:30:28 +00:00
else : s = str ( seconds ) + ' seconds '
2015-03-25 22:04:19 +00:00
age = age / 60
minutes = age % 60
if minutes == 1 : m = ' 1 minute '
2015-11-04 22:30:28 +00:00
else : m = str ( minutes ) + ' minutes '
2015-03-25 22:04:19 +00:00
age = age / 60
hours = age % 24
if hours == 1 : h = ' 1 hour '
2015-11-04 22:30:28 +00:00
else : h = str ( hours ) + ' hours '
2015-03-25 22:04:19 +00:00
age = age / 24
days = age % 30
if days == 1 : d = ' 1 day '
2015-11-04 22:30:28 +00:00
else : d = str ( days ) + ' days '
2015-03-25 22:04:19 +00:00
age = age / 30
months = age % 12
if months == 1 : mo = ' 1 month '
2015-11-04 22:30:28 +00:00
else : mo = str ( months ) + ' months '
2015-03-25 22:04:19 +00:00
years = age / 12
if years == 1 : y = ' 1 year '
2015-11-04 22:30:28 +00:00
else : y = str ( years ) + ' years '
2015-03-25 22:04:19 +00:00
if years > 0 : return ' ' . join ( ( y , mo ) ) + ' ago '
elif months > 0 : return ' ' . join ( ( mo , d ) ) + ' ago '
elif days > 0 : return ' ' . join ( ( d , h ) ) + ' ago '
elif hours > 0 : return ' ' . join ( ( h , m ) ) + ' ago '
else : return ' ' . join ( ( m , s ) ) + ' ago '
def ConvertTimestampToPrettyTime ( timestamp ) : return time . strftime ( ' % Y/ % m/ %d % H: % M: % S ' , time . localtime ( timestamp ) )
def ConvertTimestampToHumanPrettyTime ( timestamp ) :
now = GetNow ( )
difference = now - timestamp
if difference < 60 : return ' just now '
elif difference < 86400 * 7 : return ConvertTimestampToPrettyAgo ( timestamp )
else : return ConvertTimestampToPrettyTime ( timestamp )
def ConvertTimeToPrettyTime ( secs ) :
return time . strftime ( ' % H: % M: % S ' , time . gmtime ( secs ) )
def ConvertUglyNamespacesToPrettyStrings ( namespaces ) :
result = [ namespace for namespace in namespaces if namespace != ' ' and namespace is not None ]
result . sort ( )
if ' ' in namespaces or None in namespaces : result . insert ( 0 , ' no namespace ' )
return result
2015-04-22 22:57:25 +00:00
def ConvertUnitToInt ( unit ) :
2015-03-25 22:04:19 +00:00
if unit == ' B ' : return 1
elif unit == ' KB ' : return 1024
elif unit == ' MB ' : return 1048576
elif unit == ' GB ' : return 1073741824
2016-03-23 19:42:56 +00:00
def ConvertValueRangeToBytes ( value , range ) :
return ConvertIntToBytes ( value ) + ' / ' + ConvertIntToBytes ( range )
2015-06-10 19:40:25 +00:00
def ConvertValueRangeToPrettyString ( value , range ) :
return ConvertIntToPrettyString ( value ) + ' / ' + ConvertIntToPrettyString ( range )
2015-03-25 22:04:19 +00:00
def DebugPrint ( debug_info ) :
2015-11-18 22:44:07 +00:00
Print ( debug_info )
2015-03-25 22:04:19 +00:00
sys . stdout . flush ( )
sys . stderr . flush ( )
2015-05-06 20:26:18 +00:00
def DeserialisePrettyTags ( text ) :
text = text . replace ( ' \r ' , ' ' )
tags = text . split ( ' \n ' )
return tags
2015-07-01 22:02:07 +00:00
def GenerateKey ( ) :
return os . urandom ( HC . HYDRUS_KEY_LENGTH )
2015-03-25 22:04:19 +00:00
def GetEmptyDataDict ( ) :
data = collections . defaultdict ( default_dict_list )
return data
def GetHammingDistance ( phash1 , phash2 ) :
distance = 0
phash1 = bytearray ( phash1 )
phash2 = bytearray ( phash2 )
for i in range ( len ( phash1 ) ) :
xor = phash1 [ i ] ^ phash2 [ i ]
while xor > 0 :
distance + = 1
xor & = xor - 1
return distance
def GetNow ( ) : return int ( time . time ( ) )
def GetNowPrecise ( ) :
if HC . PLATFORM_WINDOWS : return time . clock ( )
else : return time . time ( )
2015-09-16 18:11:00 +00:00
def GetSiblingProcessPorts ( instance ) :
2015-11-04 22:30:28 +00:00
path = os . path . join ( HC . BASE_DIR , instance + ' _running ' )
2015-09-16 18:11:00 +00:00
if os . path . exists ( path ) :
with open ( path , ' rb ' ) as f :
result = f . read ( )
try :
2016-03-23 19:42:56 +00:00
( pid , create_time ) = SplitByLinesep ( result )
2015-09-16 18:11:00 +00:00
pid = int ( pid )
create_time = float ( create_time )
except ValueError :
return None
try :
if psutil . pid_exists ( pid ) :
ports = [ ]
p = psutil . Process ( pid )
for conn in p . connections ( ) :
if conn . status == ' LISTEN ' :
ports . append ( int ( conn . laddr [ 1 ] ) )
return ports
except psutil . Error :
return None
return None
2015-06-17 20:01:41 +00:00
def GetSubprocessStartupInfo ( ) :
if HC . PLATFORM_WINDOWS :
# This suppresses the terminal window that tends to pop up when calling ffmpeg or whatever
startupinfo = subprocess . STARTUPINFO ( )
startupinfo . dwFlags | = subprocess . STARTF_USESHOWWINDOW
else :
startupinfo = None
return startupinfo
2015-03-25 22:04:19 +00:00
def IntelligentMassIntersect ( sets_to_reduce ) :
answer = None
sets_to_reduce = list ( sets_to_reduce )
sets_to_reduce . sort ( cmp = lambda x , y : cmp ( len ( x ) , len ( y ) ) )
for set_to_reduce in sets_to_reduce :
if len ( set_to_reduce ) == 0 : return set ( )
if answer is None : answer = set ( set_to_reduce )
else :
# same thing as union; I could go &= here, but I want to be quick, so use the function call
if len ( answer ) == 0 : return set ( )
else : answer . intersection_update ( set_to_reduce )
if answer is None : return set ( )
else : return answer
2015-09-02 23:16:09 +00:00
def IsAlreadyRunning ( instance ) :
2015-08-19 21:48:21 +00:00
2015-11-04 22:30:28 +00:00
path = os . path . join ( HC . BASE_DIR , instance + ' _running ' )
2015-08-19 21:48:21 +00:00
2015-09-02 23:16:09 +00:00
if os . path . exists ( path ) :
2015-08-19 21:48:21 +00:00
2015-09-02 23:16:09 +00:00
with open ( path , ' rb ' ) as f :
result = f . read ( )
2015-08-19 21:48:21 +00:00
2015-09-02 23:16:09 +00:00
try :
2015-08-26 21:18:39 +00:00
2016-03-23 19:42:56 +00:00
( pid , create_time ) = SplitByLinesep ( result )
2015-09-02 23:16:09 +00:00
pid = int ( pid )
create_time = float ( create_time )
except ValueError :
return False
2015-08-26 21:18:39 +00:00
2015-08-19 21:48:21 +00:00
2015-09-02 23:16:09 +00:00
try :
2015-08-26 21:18:39 +00:00
2015-12-23 22:51:04 +00:00
me = psutil . Process ( )
if me . pid == pid and me . create_time ( ) == create_time :
# this is me! there is no conflict, lol!
# this happens when a linux process restarts with os.execl(), for instance (unlike Windows, it keeps its pid)
return False
2015-09-02 23:16:09 +00:00
if psutil . pid_exists ( pid ) :
2015-08-26 21:18:39 +00:00
2015-09-02 23:16:09 +00:00
p = psutil . Process ( pid )
2015-08-26 21:18:39 +00:00
2015-09-02 23:16:09 +00:00
if p . create_time ( ) == create_time and p . is_running ( ) :
return True
2015-08-26 21:18:39 +00:00
2015-09-02 23:16:09 +00:00
except psutil . Error :
2015-08-19 21:48:21 +00:00
2015-09-02 23:16:09 +00:00
return False
2015-08-19 21:48:21 +00:00
return False
2015-12-02 22:32:18 +00:00
def IterateHexPrefixes ( ) :
hex_chars = ' 0123456789abcdef '
for ( one , two ) in itertools . product ( hex_chars , hex_chars ) :
prefix = one + two
yield prefix
2015-03-25 22:04:19 +00:00
def MergeKeyToListDicts ( key_to_list_dicts ) :
result = collections . defaultdict ( list )
for key_to_list_dict in key_to_list_dicts :
for ( key , value ) in key_to_list_dict . items ( ) : result [ key ] . extend ( value )
return result
2015-11-18 22:44:07 +00:00
def Print ( text ) :
print ( ToByteString ( text ) )
2016-03-23 19:42:56 +00:00
ShowText = Print
def PrintException ( e ) :
if isinstance ( e , HydrusExceptions . ShutdownException ) :
return
etype = type ( e )
value = ToUnicode ( e )
trace_list = traceback . format_stack ( )
trace = ' ' . join ( trace_list )
message = ToUnicode ( etype . __name__ ) + ' : ' + ToUnicode ( value ) + os . linesep + ToUnicode ( trace )
Print ( ' ' )
Print ( ' Exception: ' )
DebugPrint ( message )
time . sleep ( 1 )
ShowException = PrintException
2015-11-25 22:00:57 +00:00
def Profile ( code , g , l ) :
profile = cProfile . Profile ( )
profile . runctx ( code , g , l )
output = cStringIO . StringIO ( )
stats = pstats . Stats ( profile , stream = output )
stats . strip_dirs ( )
stats . sort_stats ( ' tottime ' )
output . seek ( 0 )
output . write ( ' Stats ' )
output . write ( os . linesep )
stats . print_stats ( )
output . seek ( 0 )
2015-12-30 23:44:09 +00:00
DebugPrint ( output . read ( ) )
2015-11-25 22:00:57 +00:00
output . seek ( 0 )
output . write ( ' Callers ' )
output . write ( os . linesep )
stats . print_callers ( )
output . seek ( 0 )
2015-12-30 23:44:09 +00:00
DebugPrint ( output . read ( ) )
2015-11-25 22:00:57 +00:00
2015-09-02 23:16:09 +00:00
def RecordRunningStart ( instance ) :
2015-11-04 22:30:28 +00:00
path = os . path . join ( HC . BASE_DIR , instance + ' _running ' )
2015-09-02 23:16:09 +00:00
record_string = ' '
try :
me = psutil . Process ( )
record_string + = str ( me . pid )
record_string + = os . linesep
record_string + = str ( me . create_time ( ) )
except psutil . Error :
return
with open ( path , ' wb ' ) as f :
2015-11-18 22:44:07 +00:00
f . write ( ToByteString ( record_string ) )
2015-09-02 23:16:09 +00:00
2015-12-16 22:41:06 +00:00
def RestartProcess ( ) :
time . sleep ( 1 ) # time for ports to unmap
os . execl ( sys . executable , sys . executable , * sys . argv )
2016-03-23 19:42:56 +00:00
def SplayListForDB ( xs ) : return ' ( ' + ' , ' . join ( ( str ( x ) for x in xs ) ) + ' ) '
def SplitByLinesep ( raw_text ) :
2015-03-25 22:04:19 +00:00
2016-03-23 19:42:56 +00:00
if ' \r \n ' in raw_text :
2015-11-04 22:30:28 +00:00
2016-03-23 19:42:56 +00:00
return raw_text . split ( ' \r \n ' )
else :
return raw_text . split ( ' \n ' )
2015-11-04 22:30:28 +00:00
2015-03-25 22:04:19 +00:00
2015-12-30 23:44:09 +00:00
def SplitIteratorIntoChunks ( iterator , n ) :
chunk = [ ]
for item in iterator :
chunk . append ( item )
if len ( chunk ) == n :
yield chunk
chunk = [ ]
if len ( chunk ) > 0 :
yield chunk
2015-04-08 18:10:50 +00:00
def SplitListIntoChunks ( xs , n ) :
for i in xrange ( 0 , len ( xs ) , n ) : yield xs [ i : i + n ]
2015-06-24 22:10:14 +00:00
def TimeHasPassed ( timestamp ) :
2016-03-30 22:56:50 +00:00
if timestamp is None :
return False
2015-06-24 22:10:14 +00:00
return GetNow ( ) > timestamp
2015-07-15 20:28:26 +00:00
def TimeHasPassedPrecise ( precise_timestamp ) :
return GetNowPrecise ( ) > precise_timestamp
2015-08-05 18:42:35 +00:00
def TimeUntil ( timestamp ) :
return timestamp - GetNow ( )
2015-11-04 22:30:28 +00:00
def ToByteString ( text_producing_object ) :
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if isinstance ( text_producing_object , unicode ) :
2015-11-04 22:30:28 +00:00
return text_producing_object . encode ( ' utf-8 ' )
2016-01-13 22:08:19 +00:00
elif isinstance ( text_producing_object , str ) :
2015-11-04 22:30:28 +00:00
return text_producing_object
else :
try :
return str ( text_producing_object )
except :
return str ( repr ( text_producing_object ) )
2015-03-25 22:04:19 +00:00
2015-11-04 22:30:28 +00:00
def ToUnicode ( text_producing_object ) :
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if isinstance ( text_producing_object , ( str , unicode , bs4 . element . NavigableString ) ) :
2015-11-04 22:30:28 +00:00
text = text_producing_object
2015-03-25 22:04:19 +00:00
else :
2015-11-04 22:30:28 +00:00
try :
text = str ( text_producing_object ) # dealing with exceptions, etc...
except :
text = repr ( text_producing_object )
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if not isinstance ( text , unicode ) :
2015-03-25 22:04:19 +00:00
2015-11-04 22:30:28 +00:00
try :
text = text . decode ( ' utf-8 ' )
except :
try :
text = text . decode ( locale . getpreferredencoding ( ) )
except :
2015-11-18 22:44:07 +00:00
text = repr ( text ) . decode ( ' utf-8 ' )
2015-11-04 22:30:28 +00:00
2015-03-25 22:04:19 +00:00
2015-11-04 22:30:28 +00:00
return text
2015-03-25 22:04:19 +00:00
class HydrusYAMLBase ( yaml . YAMLObject ) :
yaml_loader = yaml . SafeLoader
yaml_dumper = yaml . SafeDumper
class Account ( HydrusYAMLBase ) :
yaml_tag = u ' !Account '
def __init__ ( self , account_key , account_type , created , expires , used_bytes , used_requests , banned_info = None ) :
HydrusYAMLBase . __init__ ( self )
self . _info = { }
self . _info [ ' account_key ' ] = account_key
self . _info [ ' account_type ' ] = account_type
self . _info [ ' created ' ] = created
self . _info [ ' expires ' ] = expires
self . _info [ ' used_bytes ' ] = used_bytes
self . _info [ ' used_requests ' ] = used_requests
if banned_info is not None : self . _info [ ' banned_info ' ] = banned_info
self . _info [ ' fresh_timestamp ' ] = GetNow ( )
def __repr__ ( self ) : return self . ConvertToString ( )
def __str__ ( self ) : return self . ConvertToString ( )
def _IsBanned ( self ) :
if ' banned_info ' not in self . _info : return False
else :
( reason , created , expires ) = self . _info [ ' banned_info ' ]
if expires is None : return True
2015-06-24 22:10:14 +00:00
else : return not TimeHasPassed ( expires )
2015-03-25 22:04:19 +00:00
def _IsBytesExceeded ( self ) :
account_type = self . _info [ ' account_type ' ]
max_num_bytes = account_type . GetMaxBytes ( )
used_bytes = self . _info [ ' used_bytes ' ]
return max_num_bytes is not None and used_bytes > max_num_bytes
def _IsExpired ( self ) :
if self . _info [ ' expires ' ] is None : return False
2015-06-24 22:10:14 +00:00
else : return TimeHasPassed ( self . _info [ ' expires ' ] )
2015-03-25 22:04:19 +00:00
def _IsRequestsExceeded ( self ) :
account_type = self . _info [ ' account_type ' ]
max_num_requests = account_type . GetMaxRequests ( )
used_requests = self . _info [ ' used_requests ' ]
return max_num_requests is not None and used_requests > max_num_requests
def CheckPermission ( self , permission ) :
if self . _IsBanned ( ) : raise HydrusExceptions . PermissionException ( ' This account is banned! ' )
if self . _IsExpired ( ) : raise HydrusExceptions . PermissionException ( ' This account is expired. ' )
if self . _IsBytesExceeded ( ) : raise HydrusExceptions . PermissionException ( ' You have hit your data transfer limit, and cannot make any more requests for the month. ' )
if self . _IsRequestsExceeded ( ) : raise HydrusExceptions . PermissionException ( ' You have hit your requests limit, and cannot make any more requests for the month. ' )
if not self . _info [ ' account_type ' ] . HasPermission ( permission ) : raise HydrusExceptions . PermissionException ( ' You do not have permission to do that. ' )
def ConvertToString ( self ) : return ConvertTimestampToPrettyAge ( self . _info [ ' created ' ] ) + os . linesep + self . _info [ ' account_type ' ] . ConvertToString ( ) + os . linesep + ' which ' + ConvertTimestampToPrettyExpires ( self . _info [ ' expires ' ] )
def GetAccountKey ( self ) : return self . _info [ ' account_key ' ]
def GetAccountType ( self ) : return self . _info [ ' account_type ' ]
def GetCreated ( self ) : return self . _info [ ' created ' ]
def GetExpires ( self ) : return self . _info [ ' expires ' ]
def GetExpiresString ( self ) :
if self . _IsBanned ( ) :
( reason , created , expires ) = self . _info [ ' banned_info ' ]
return ' banned ' + ConvertTimestampToPrettyAge ( created ) + ' , ' + ConvertTimestampToPrettyExpires ( expires ) + ' because: ' + reason
else : return ConvertTimestampToPrettyAge ( self . _info [ ' created ' ] ) + ' and ' + ConvertTimestampToPrettyExpires ( self . _info [ ' expires ' ] )
def GetUsedBytesString ( self ) :
max_num_bytes = self . _info [ ' account_type ' ] . GetMaxBytes ( )
used_bytes = self . _info [ ' used_bytes ' ]
if max_num_bytes is None : return ConvertIntToBytes ( used_bytes ) + ' used this month '
else : return ConvertIntToBytes ( used_bytes ) + ' / ' + ConvertIntToBytes ( max_num_bytes ) + ' used this month '
def GetUsedRequestsString ( self ) :
max_num_requests = self . _info [ ' account_type ' ] . GetMaxRequests ( )
used_requests = self . _info [ ' used_requests ' ]
if max_num_requests is None : return ConvertIntToPrettyString ( used_requests ) + ' requests used this month '
2015-06-10 19:40:25 +00:00
else : return ConvertValueRangeToPrettyString ( used_requests , max_num_requests ) + ' requests used this month '
2015-03-25 22:04:19 +00:00
def GetUsedBytes ( self ) : return self . _info [ ' used_bytes ' ]
def GetUsedRequests ( self ) : return self . _info [ ' used_bytes ' ]
def HasAccountKey ( self ) :
if ' account_key ' in self . _info and self . _info [ ' account_key ' ] is not None : return True
return False
def HasPermission ( self , permission ) :
if self . _IsBanned ( ) : return False
if self . _IsExpired ( ) : return False
if self . _IsBytesExceeded ( ) : return False
if self . _IsRequestsExceeded ( ) : return False
return self . _info [ ' account_type ' ] . HasPermission ( permission )
def IsAdmin ( self ) : return True in [ self . HasPermission ( permission ) for permission in HC . ADMIN_PERMISSIONS ]
def IsBanned ( self ) : return self . _IsBanned ( )
def IsStale ( self ) : return self . _info [ ' fresh_timestamp ' ] + HC . UPDATE_DURATION * 5 < GetNow ( )
def IsUnknownAccount ( self ) : return self . _info [ ' account_type ' ] . IsUnknownAccountType ( )
def MakeFresh ( self ) : self . _info [ ' fresh_timestamp ' ] = GetNow ( )
def MakeStale ( self ) : self . _info [ ' fresh_timestamp ' ] = 0
def RequestMade ( self , num_bytes ) :
self . _info [ ' used_bytes ' ] + = num_bytes
self . _info [ ' used_requests ' ] + = 1
sqlite3 . register_adapter ( Account , yaml . safe_dump )
2015-10-14 21:02:25 +00:00
class AccountIdentifier ( HydrusSerialisable . SerialisableBase ) :
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
SERIALISABLE_TYPE = HydrusSerialisable . SERIALISABLE_TYPE_ACCOUNT_IDENTIFIER
SERIALISABLE_VERSION = 1
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
TYPE_ACCOUNT_KEY = 1
TYPE_CONTENT = 2
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
def __init__ ( self , account_key = None , content = None ) :
2015-03-25 22:04:19 +00:00
HydrusYAMLBase . __init__ ( self )
if account_key is not None :
self . _type = self . TYPE_ACCOUNT_KEY
self . _data = account_key
2015-10-14 21:02:25 +00:00
elif content is not None :
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
self . _type = self . TYPE_CONTENT
self . _data = content
2015-03-25 22:04:19 +00:00
def __eq__ ( self , other ) : return self . __hash__ ( ) == other . __hash__ ( )
def __hash__ ( self ) : return ( self . _type , self . _data ) . __hash__ ( )
def __ne__ ( self , other ) : return self . __hash__ ( ) != other . __hash__ ( )
2015-11-04 22:30:28 +00:00
def __repr__ ( self ) : return ' Account Identifier: ' + ToUnicode ( ( self . _type , self . _data ) )
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
def _GetSerialisableInfo ( self ) :
if self . _type == self . TYPE_ACCOUNT_KEY :
serialisable_data = self . _data . encode ( ' hex ' )
elif self . _type == self . TYPE_CONTENT :
serialisable_data = self . _data . GetSerialisableTuple ( )
return ( self . _type , serialisable_data )
def _InitialiseFromSerialisableInfo ( self , serialisable_info ) :
( self . _type , serialisable_data ) = serialisable_info
if self . _type == self . TYPE_ACCOUNT_KEY :
self . _data = serialisable_data . decode ( ' hex ' )
elif self . _type == self . TYPE_CONTENT :
self . _data = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_data )
2015-03-25 22:04:19 +00:00
def GetData ( self ) : return self . _data
def HasAccountKey ( self ) : return self . _type == self . TYPE_ACCOUNT_KEY
2015-10-14 21:02:25 +00:00
def HasContent ( self ) : return self . _type == self . TYPE_CONTENT
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
HydrusSerialisable . SERIALISABLE_TYPES_TO_OBJECT_TYPES [ HydrusSerialisable . SERIALISABLE_TYPE_ACCOUNT_IDENTIFIER ] = AccountIdentifier
2015-03-25 22:04:19 +00:00
class AccountType ( HydrusYAMLBase ) :
yaml_tag = u ' !AccountType '
def __init__ ( self , title , permissions , max_monthly_data ) :
HydrusYAMLBase . __init__ ( self )
self . _title = title
self . _permissions = permissions
self . _max_monthly_data = max_monthly_data
def __repr__ ( self ) : return self . ConvertToString ( )
def GetPermissions ( self ) : return self . _permissions
def GetTitle ( self ) : return self . _title
def GetMaxBytes ( self ) :
( max_num_bytes , max_num_requests ) = self . _max_monthly_data
return max_num_bytes
def GetMaxRequests ( self ) :
( max_num_bytes , max_num_requests ) = self . _max_monthly_data
return max_num_requests
def GetMaxBytesString ( self ) :
( max_num_bytes , max_num_requests ) = self . _max_monthly_data
if max_num_bytes is None : max_num_bytes_string = ' No limit '
else : max_num_bytes_string = ConvertIntToBytes ( max_num_bytes )
return max_num_bytes_string
def GetMaxRequestsString ( self ) :
( max_num_bytes , max_num_requests ) = self . _max_monthly_data
if max_num_requests is None : max_num_requests_string = ' No limit '
else : max_num_requests_string = ConvertIntToPrettyString ( max_num_requests )
return max_num_requests_string
def ConvertToString ( self ) :
result_string = self . _title + ' with '
if self . _permissions == [ HC . UNKNOWN_PERMISSION ] : result_string + = ' no permissions '
else : result_string + = ' , ' . join ( [ HC . permissions_string_lookup [ permission ] for permission in self . _permissions ] ) + ' permissions '
return result_string
def IsUnknownAccountType ( self ) : return self . _permissions == [ HC . UNKNOWN_PERMISSION ]
def HasPermission ( self , permission ) : return permission in self . _permissions
sqlite3 . register_adapter ( AccountType , yaml . safe_dump )
UNKNOWN_ACCOUNT_TYPE = AccountType ( ' unknown account ' , [ HC . UNKNOWN_PERMISSION ] , ( None , None ) )
def GetUnknownAccount ( account_key = None ) : return Account ( account_key , UNKNOWN_ACCOUNT_TYPE , 0 , None , 0 , 0 )
2016-01-06 21:17:20 +00:00
class BigJobPauser ( object ) :
def __init__ ( self , period = 10 , wait_time = 0.1 ) :
self . _period = period
self . _wait_time = wait_time
self . _next_pause = GetNow ( ) + self . _period
def Pause ( self ) :
if TimeHasPassed ( self . _next_pause ) :
time . sleep ( self . _wait_time )
self . _next_pause = GetNow ( ) + self . _period
2015-06-03 21:05:13 +00:00
class ClientToServerContentUpdatePackage ( HydrusYAMLBase ) :
2015-03-25 22:04:19 +00:00
2015-06-03 21:05:13 +00:00
yaml_tag = u ' !ClientToServerContentUpdatePackage '
2015-03-25 22:04:19 +00:00
def __init__ ( self , content_data , hash_ids_to_hashes ) :
HydrusYAMLBase . __init__ ( self )
# we need to flatten it from a collections.defaultdict to just a normal dict so it can be YAMLised
self . _content_data = { }
for data_type in content_data :
self . _content_data [ data_type ] = { }
for action in content_data [ data_type ] : self . _content_data [ data_type ] [ action ] = content_data [ data_type ] [ action ]
self . _hash_ids_to_hashes = hash_ids_to_hashes
def GetContentUpdates ( self , for_client = False ) :
2015-10-14 21:02:25 +00:00
data_types = [ HC . CONTENT_TYPE_FILES , HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_TYPE_TAG_SIBLINGS , HC . CONTENT_TYPE_TAG_PARENTS ]
2015-09-23 21:21:02 +00:00
actions = [ HC . CONTENT_UPDATE_PEND , HC . CONTENT_UPDATE_PETITION , HC . CONTENT_UPDATE_DENY_PEND , HC . CONTENT_UPDATE_DENY_PETITION ]
2015-03-25 22:04:19 +00:00
content_updates = [ ]
for ( data_type , action ) in itertools . product ( data_types , actions ) :
munge_row = lambda row : row
if for_client :
2016-01-20 23:57:33 +00:00
if action == HC . CONTENT_UPDATE_PEND :
new_action = HC . CONTENT_UPDATE_ADD
elif action == HC . CONTENT_UPDATE_PETITION :
new_action = HC . CONTENT_UPDATE_DELETE
2015-03-25 22:04:19 +00:00
else : continue
2015-10-14 21:02:25 +00:00
if data_type in ( HC . CONTENT_TYPE_TAG_SIBLINGS , HC . CONTENT_TYPE_TAG_PARENTS ) and action in ( HC . CONTENT_UPDATE_PEND , HC . CONTENT_UPDATE_PETITION ) :
2015-03-25 22:04:19 +00:00
munge_row = lambda ( pair , reason ) : pair
2015-10-14 21:02:25 +00:00
elif data_type == HC . CONTENT_TYPE_FILES and action == HC . CONTENT_UPDATE_PETITION :
2015-03-25 22:04:19 +00:00
munge_row = lambda ( hashes , reason ) : hashes
2015-10-14 21:02:25 +00:00
elif data_type == HC . CONTENT_TYPE_MAPPINGS and action == HC . CONTENT_UPDATE_PETITION :
2015-03-25 22:04:19 +00:00
munge_row = lambda ( tag , hashes , reason ) : ( tag , hashes )
2016-01-20 23:57:33 +00:00
else :
new_action = action
2015-03-25 22:04:19 +00:00
content_updates . extend ( [ ContentUpdate ( data_type , new_action , munge_row ( row ) ) for row in self . GetContentDataIterator ( data_type , action ) ] )
return content_updates
def GetHashes ( self ) : return self . _hash_ids_to_hashes . values ( )
def GetContentDataIterator ( self , data_type , action ) :
if data_type not in self . _content_data or action not in self . _content_data [ data_type ] : return ( )
data = self . _content_data [ data_type ] [ action ]
2015-10-14 21:02:25 +00:00
if data_type == HC . CONTENT_TYPE_FILES :
2015-03-25 22:04:19 +00:00
if action == HC . CONTENT_UPDATE_PETITION : return ( ( { self . _hash_ids_to_hashes [ hash_id ] for hash_id in hash_ids } , reason ) for ( hash_ids , reason ) in data )
else : return ( { self . _hash_ids_to_hashes [ hash_id ] for hash_id in hash_ids } for hash_ids in ( data , ) )
2015-10-14 21:02:25 +00:00
if data_type == HC . CONTENT_TYPE_MAPPINGS :
2015-03-25 22:04:19 +00:00
if action == HC . CONTENT_UPDATE_PETITION : return ( ( tag , [ self . _hash_ids_to_hashes [ hash_id ] for hash_id in hash_ids ] , reason ) for ( tag , hash_ids , reason ) in data )
else : return ( ( tag , [ self . _hash_ids_to_hashes [ hash_id ] for hash_id in hash_ids ] ) for ( tag , hash_ids ) in data )
else : return data . __iter__ ( )
def GetTags ( self ) :
tags = set ( )
2015-10-14 21:02:25 +00:00
try : tags . update ( ( tag for ( tag , hash_ids ) in self . _content_data [ HC . CONTENT_TYPE_MAPPINGS ] [ HC . CONTENT_UPDATE_PEND ] ) )
2015-03-25 22:04:19 +00:00
except : pass
2015-10-14 21:02:25 +00:00
try : tags . update ( ( tag for ( tag , hash_ids , reason ) in self . _content_data [ HC . CONTENT_TYPE_MAPPINGS ] [ HC . CONTENT_UPDATE_PETITION ] ) )
2015-03-25 22:04:19 +00:00
except : pass
return tags
def IsEmpty ( self ) :
num_total = 0
for data_type in self . _content_data :
for action in self . _content_data [ data_type ] : num_total + = len ( self . _content_data [ data_type ] [ action ] )
return num_total == 0
2015-10-14 21:02:25 +00:00
class Content ( HydrusSerialisable . SerialisableBase ) :
SERIALISABLE_TYPE = HydrusSerialisable . SERIALISABLE_TYPE_CONTENT
SERIALISABLE_VERSION = 1
def __init__ ( self , content_type = None , content = None ) :
HydrusSerialisable . SerialisableBase . __init__ ( self )
self . _content_type = content_type
self . _content = content
def __eq__ ( self , other ) : return self . __hash__ ( ) == other . __hash__ ( )
def __hash__ ( self ) : return ( self . _content_type , self . _content ) . __hash__ ( )
def __ne__ ( self , other ) : return self . __hash__ ( ) != other . __hash__ ( )
def __repr__ ( self ) : return ' Content: ' + self . ToString ( )
def _GetSerialisableInfo ( self ) :
def EncodeHashes ( hs ) :
return [ h . encode ( ' hex ' ) for h in hs ]
if self . _content_type == HC . CONTENT_TYPE_FILES :
hashes = self . _content
serialisable_content = EncodeHashes ( hashes )
elif self . _content_type == HC . CONTENT_TYPE_MAPPING :
( tag , hash ) = self . _content
serialisable_content = ( tag , hash . encode ( ' hex ' ) )
elif self . _content_type == HC . CONTENT_TYPE_MAPPINGS :
( tag , hashes ) = self . _content
serialisable_content = ( tag , EncodeHashes ( hashes ) )
elif self . _content_type in ( HC . CONTENT_TYPE_TAG_PARENTS , HC . CONTENT_TYPE_TAG_SIBLINGS ) :
( old_tag , new_tag ) = self . _content
serialisable_content = ( old_tag , new_tag )
return ( self . _content_type , serialisable_content )
def _InitialiseFromSerialisableInfo ( self , serialisable_info ) :
def DecodeHashes ( hs ) :
return [ h . decode ( ' hex ' ) for h in hs ]
( self . _content_type , serialisable_content ) = serialisable_info
if self . _content_type == HC . CONTENT_TYPE_FILES :
serialisable_hashes = serialisable_content
self . _content = DecodeHashes ( serialisable_hashes )
elif self . _content_type == HC . CONTENT_TYPE_MAPPING :
( tag , serialisable_hash ) = serialisable_content
self . _content = ( tag , serialisable_hash . decode ( ' hex ' ) )
elif self . _content_type == HC . CONTENT_TYPE_MAPPINGS :
( tag , serialisable_hashes ) = serialisable_content
self . _content = ( tag , DecodeHashes ( serialisable_hashes ) )
elif self . _content_type in ( HC . CONTENT_TYPE_TAG_PARENTS , HC . CONTENT_TYPE_TAG_SIBLINGS ) :
( old_tag , new_tag ) = serialisable_content
self . _content = ( old_tag , new_tag )
def GetContent ( self ) :
return self . _content
def GetContentType ( self ) :
return self . _content_type
def GetHashes ( self ) :
if self . _content_type == HC . CONTENT_TYPE_FILES :
hashes = self . _content
elif self . _content_type == HC . CONTENT_TYPE_MAPPING :
( tag , hash ) = self . _content
return [ hash ]
elif self . _content_type == HC . CONTENT_TYPE_MAPPINGS :
( tag , hashes ) = self . _content
else :
hashes = [ ]
return hashes
def HasHashes ( self ) :
return self . _content_type in ( HC . CONTENT_TYPE_FILES , HC . CONTENT_TYPE_MAPPING , HC . CONTENT_TYPE_MAPPINGS )
def ToString ( self ) :
if self . _content_type == HC . CONTENT_TYPE_FILES :
hashes = self . _content
text = ' FILES: ' + ConvertIntToPrettyString ( len ( hashes ) ) + ' files '
elif self . _content_type == HC . CONTENT_TYPE_MAPPING :
( tag , hash ) = self . _content
text = ' MAPPING: ' + tag + ' for ' + hash . encode ( ' hex ' )
elif self . _content_type == HC . CONTENT_TYPE_MAPPINGS :
( tag , hashes ) = self . _content
text = ' MAPPINGS: ' + tag + ' for ' + ConvertIntToPrettyString ( len ( hashes ) ) + ' files '
elif self . _content_type == HC . CONTENT_TYPE_TAG_PARENTS :
( child , parent ) = self . _content
text = ' PARENT: ' ' " ' + child + ' " -> " ' + parent + ' " '
elif self . _content_type == HC . CONTENT_TYPE_TAG_SIBLINGS :
( old_tag , new_tag ) = self . _content
text = ' SIBLING: ' + ' " ' + old_tag + ' " -> " ' + new_tag + ' " '
return text
HydrusSerialisable . SERIALISABLE_TYPES_TO_OBJECT_TYPES [ HydrusSerialisable . SERIALISABLE_TYPE_CONTENT ] = Content
2015-03-25 22:04:19 +00:00
class ContentUpdate ( object ) :
def __init__ ( self , data_type , action , row ) :
self . _data_type = data_type
self . _action = action
self . _row = row
def __eq__ ( self , other ) : return self . _data_type == other . _data_type and self . _action == other . _action and self . _row == other . _row
def __ne__ ( self , other ) : return not self . __eq__ ( other )
2015-11-04 22:30:28 +00:00
def __repr__ ( self ) : return ' Content Update: ' + ToUnicode ( ( self . _data_type , self . _action , self . _row ) )
2015-03-25 22:04:19 +00:00
def GetHashes ( self ) :
2015-10-14 21:02:25 +00:00
if self . _data_type == HC . CONTENT_TYPE_FILES :
2015-03-25 22:04:19 +00:00
2016-02-17 22:06:47 +00:00
if self . _action == HC . CONTENT_UPDATE_ADVANCED :
hashes = set ( )
elif self . _action == HC . CONTENT_UPDATE_ADD :
2015-03-25 22:04:19 +00:00
hash = self . _row [ 0 ]
hashes = set ( ( hash , ) )
2016-01-13 22:08:19 +00:00
elif self . _action in ( HC . CONTENT_UPDATE_ARCHIVE , HC . CONTENT_UPDATE_DELETE , HC . CONTENT_UPDATE_UNDELETE , HC . CONTENT_UPDATE_INBOX , HC . CONTENT_UPDATE_PEND , HC . CONTENT_UPDATE_RESCIND_PEND , HC . CONTENT_UPDATE_RESCIND_PETITION ) :
hashes = self . _row
elif self . _action == HC . CONTENT_UPDATE_PETITION :
( hashes , reason ) = self . _row
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
elif self . _data_type == HC . CONTENT_TYPE_MAPPINGS :
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if self . _action == HC . CONTENT_UPDATE_ADVANCED :
hashes = set ( )
elif self . _action == HC . CONTENT_UPDATE_PETITION :
( tag , hashes , reason ) = self . _row
else :
( tag , hashes ) = self . _row
elif self . _data_type in ( HC . CONTENT_TYPE_TAG_PARENTS , HC . CONTENT_TYPE_TAG_SIBLINGS ) :
hashes = set ( )
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
elif self . _data_type == HC . CONTENT_TYPE_RATINGS :
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if self . _action == HC . CONTENT_UPDATE_ADD :
( rating , hashes ) = self . _row
2015-03-25 22:04:19 +00:00
2016-01-13 22:08:19 +00:00
if not isinstance ( hashes , set ) :
hashes = set ( hashes )
2015-03-25 22:04:19 +00:00
return hashes
2016-01-06 21:17:20 +00:00
def GetWeight ( self ) :
return len ( self . GetHashes ( ) )
2015-11-18 22:44:07 +00:00
def IsInboxRelated ( self ) :
return self . _action in ( HC . CONTENT_UPDATE_ARCHIVE , HC . CONTENT_UPDATE_INBOX )
def ToTuple ( self ) :
return ( self . _data_type , self . _action , self . _row )
2015-03-25 22:04:19 +00:00
class EditLogAction ( object ) :
yaml_tag = u ' !EditLogAction '
def __init__ ( self , action ) : self . _action = action
def GetAction ( self ) : return self . _action
class EditLogActionAdd ( EditLogAction ) :
yaml_tag = u ' !EditLogActionAdd '
def __init__ ( self , data ) :
EditLogAction . __init__ ( self , HC . ADD )
self . _data = data
def GetData ( self ) : return self . _data
class EditLogActionDelete ( EditLogAction ) :
yaml_tag = u ' !EditLogActionDelete '
def __init__ ( self , identifier ) :
EditLogAction . __init__ ( self , HC . DELETE )
self . _identifier = identifier
def GetIdentifier ( self ) : return self . _identifier
class EditLogActionEdit ( EditLogAction ) :
yaml_tag = u ' !EditLogActionEdit '
def __init__ ( self , identifier , data ) :
EditLogAction . __init__ ( self , HC . EDIT )
self . _identifier = identifier
self . _data = data
def GetData ( self ) : return self . _data
def GetIdentifier ( self ) : return self . _identifier
class JobDatabase ( object ) :
2016-03-30 22:56:50 +00:00
def __init__ ( self , job_type , synchronous , action , * args , * * kwargs ) :
2015-03-25 22:04:19 +00:00
self . _type = job_type
self . _synchronous = synchronous
2016-03-30 22:56:50 +00:00
self . _action = action
2015-03-25 22:04:19 +00:00
self . _args = args
self . _kwargs = kwargs
self . _result = None
self . _result_ready = threading . Event ( )
2016-03-30 22:56:50 +00:00
def GetCallableTuple ( self ) :
return ( self . _action , self . _args , self . _kwargs )
2015-03-25 22:04:19 +00:00
def GetResult ( self ) :
while True :
if self . _result_ready . wait ( 5 ) == True : break
2015-09-02 23:16:09 +00:00
elif HydrusGlobals . model_shutdown : raise HydrusExceptions . ShutdownException ( ' Application quit before db could serve result! ' )
2015-03-25 22:04:19 +00:00
2015-11-11 21:20:41 +00:00
if isinstance ( self . _result , Exception ) :
2015-06-03 21:05:13 +00:00
2015-11-11 21:20:41 +00:00
if isinstance ( self . _result , HydrusExceptions . DBException ) :
( text , gumpf , db_traceback ) = self . _result . args
trace_list = traceback . format_stack ( )
caller_traceback = ' Stack Trace (most recent call last): ' + os . linesep * 2 + os . linesep . join ( trace_list )
raise HydrusExceptions . DBException ( text , caller_traceback , db_traceback )
else :
raise self . _result
2015-03-25 22:04:19 +00:00
2016-01-20 23:57:33 +00:00
else :
return self . _result
2015-03-25 22:04:19 +00:00
def GetType ( self ) : return self . _type
def IsSynchronous ( self ) : return self . _synchronous
def PutResult ( self , result ) :
self . _result = result
self . _result_ready . set ( )
2016-03-30 22:56:50 +00:00
def ToString ( self ) :
return self . _type + ' ' + self . _action
2015-06-03 21:05:13 +00:00
class ServerToClientContentUpdatePackage ( HydrusSerialisable . SerialisableBase ) :
2015-03-25 22:04:19 +00:00
2015-06-03 21:05:13 +00:00
SERIALISABLE_TYPE = HydrusSerialisable . SERIALISABLE_TYPE_SERVER_TO_CLIENT_CONTENT_UPDATE_PACKAGE
SERIALISABLE_VERSION = 1
def __init__ ( self ) :
2015-03-25 22:04:19 +00:00
2015-06-03 21:05:13 +00:00
HydrusSerialisable . SerialisableBase . __init__ ( self )
self . _content_data = { }
self . _hash_ids_to_hashes = { }
def _GetSerialisableInfo ( self ) :
serialisable_content_data = [ ]
for ( data_type , actions_dict ) in self . _content_data . items ( ) :
serialisable_actions_dict = [ ]
for ( action , rows ) in actions_dict . items ( ) :
serialisable_actions_dict . append ( ( action , rows ) )
serialisable_content_data . append ( ( data_type , serialisable_actions_dict ) )
serialisable_hashes = [ ( hash_id , hash . encode ( ' hex ' ) ) for ( hash_id , hash ) in self . _hash_ids_to_hashes . items ( ) ]
2015-03-25 22:04:19 +00:00
2015-06-03 21:05:13 +00:00
return ( serialisable_content_data , serialisable_hashes )
def _InitialiseFromSerialisableInfo ( self , serialisable_info ) :
( serialisable_content_data , serialisable_hashes ) = serialisable_info
2015-03-25 22:04:19 +00:00
self . _content_data = { }
2015-06-03 21:05:13 +00:00
for ( data_type , serialisable_actions_dict ) in serialisable_content_data :
2015-03-25 22:04:19 +00:00
2015-06-03 21:05:13 +00:00
actions_dict = { }
2015-03-25 22:04:19 +00:00
2015-06-03 21:05:13 +00:00
for ( action , rows ) in serialisable_actions_dict :
actions_dict [ action ] = rows
self . _content_data [ data_type ] = actions_dict
2015-03-25 22:04:19 +00:00
2015-06-03 21:05:13 +00:00
self . _hash_ids_to_hashes = { hash_id : hash . decode ( ' hex ' ) for ( hash_id , hash ) in serialisable_hashes }
2015-03-25 22:04:19 +00:00
2015-06-03 21:05:13 +00:00
def AddContentData ( self , data_type , action , rows , hash_ids_to_hashes ) :
2015-03-25 22:04:19 +00:00
2015-06-03 21:05:13 +00:00
if data_type not in self . _content_data :
self . _content_data [ data_type ] = { }
if action not in self . _content_data [ data_type ] :
self . _content_data [ data_type ] [ action ] = [ ]
2015-03-25 22:04:19 +00:00
2015-06-03 21:05:13 +00:00
self . _content_data [ data_type ] [ action ] . extend ( rows )
self . _hash_ids_to_hashes . update ( hash_ids_to_hashes )
2015-03-25 22:04:19 +00:00
def GetContentDataIterator ( self , data_type , action ) :
if data_type not in self . _content_data or action not in self . _content_data [ data_type ] : return ( )
data = self . _content_data [ data_type ] [ action ]
2015-10-14 21:02:25 +00:00
if data_type == HC . CONTENT_TYPE_FILES :
2015-03-25 22:04:19 +00:00
if action == HC . CONTENT_UPDATE_ADD : return ( ( self . _hash_ids_to_hashes [ hash_id ] , size , mime , timestamp , width , height , duration , num_frames , num_words ) for ( hash_id , size , mime , timestamp , width , height , duration , num_frames , num_words ) in data )
2015-06-03 21:05:13 +00:00
else : return ( ( self . _hash_ids_to_hashes [ hash_id ] , ) for hash_id in data )
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
elif data_type == HC . CONTENT_TYPE_MAPPINGS : return ( ( tag , [ self . _hash_ids_to_hashes [ hash_id ] for hash_id in hash_ids ] ) for ( tag , hash_ids ) in data )
2015-03-25 22:04:19 +00:00
else : return data . __iter__ ( )
def GetNumContentUpdates ( self ) :
num = 0
2015-06-03 21:05:13 +00:00
for data_type in self . _content_data :
2015-03-25 22:04:19 +00:00
2015-06-03 21:05:13 +00:00
for action in self . _content_data [ data_type ] :
num + = len ( self . _content_data [ data_type ] [ action ] )
2015-03-25 22:04:19 +00:00
return num
2015-06-10 19:40:25 +00:00
def GetNumRows ( self ) :
num = 0
for data_type in self . _content_data :
for action in self . _content_data [ data_type ] :
data = self . _content_data [ data_type ] [ action ]
2015-10-14 21:02:25 +00:00
if data_type == HC . CONTENT_TYPE_MAPPINGS :
2015-06-10 19:40:25 +00:00
for ( tag , hash_ids ) in data :
num + = len ( hash_ids )
else : num + = len ( data )
return num
2016-03-23 19:42:56 +00:00
def IterateContentUpdateChunks ( self , chunk_weight = 1250 ) :
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
data_types = [ HC . CONTENT_TYPE_FILES , HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_TYPE_TAG_SIBLINGS , HC . CONTENT_TYPE_TAG_PARENTS ]
2015-03-25 22:04:19 +00:00
actions = [ HC . CONTENT_UPDATE_ADD , HC . CONTENT_UPDATE_DELETE ]
2016-01-20 23:57:33 +00:00
content_updates = [ ]
weight = 0
2015-03-25 22:04:19 +00:00
for ( data_type , action ) in itertools . product ( data_types , actions ) :
for row in self . GetContentDataIterator ( data_type , action ) :
2016-01-20 23:57:33 +00:00
content_update = ContentUpdate ( data_type , action , row )
2015-03-25 22:04:19 +00:00
2016-01-20 23:57:33 +00:00
weight + = content_update . GetWeight ( )
content_updates . append ( content_update )
if weight > chunk_weight :
2015-03-25 22:04:19 +00:00
2016-01-20 23:57:33 +00:00
yield ( content_updates , weight )
content_updates = [ ]
weight = 0
2015-03-25 22:04:19 +00:00
2016-01-20 23:57:33 +00:00
if len ( content_updates ) > 0 :
yield ( content_updates , weight )
2015-03-25 22:04:19 +00:00
def GetHashes ( self ) : return set ( self . _hash_ids_to_hashes . values ( ) )
def GetTags ( self ) :
tags = set ( )
2015-10-14 21:02:25 +00:00
tags . update ( ( tag for ( tag , hash_ids ) in self . GetContentDataIterator ( HC . CONTENT_TYPE_MAPPINGS , HC . CURRENT ) ) )
tags . update ( ( tag for ( tag , hash_ids ) in self . GetContentDataIterator ( HC . CONTENT_TYPE_MAPPINGS , HC . DELETED ) ) )
2015-03-25 22:04:19 +00:00
2015-10-14 21:02:25 +00:00
tags . update ( ( old_tag for ( old_tag , new_tag ) in self . GetContentDataIterator ( HC . CONTENT_TYPE_TAG_PARENTS , HC . CURRENT ) ) )
tags . update ( ( new_tag for ( old_tag , new_tag ) in self . GetContentDataIterator ( HC . CONTENT_TYPE_TAG_SIBLINGS , HC . CURRENT ) ) )
tags . update ( ( old_tag for ( old_tag , new_tag ) in self . GetContentDataIterator ( HC . CONTENT_TYPE_TAG_PARENTS , HC . DELETED ) ) )
tags . update ( ( new_tag for ( old_tag , new_tag ) in self . GetContentDataIterator ( HC . CONTENT_TYPE_TAG_SIBLINGS , HC . DELETED ) ) )
2015-03-25 22:04:19 +00:00
return tags
2015-06-03 21:05:13 +00:00
HydrusSerialisable . SERIALISABLE_TYPES_TO_OBJECT_TYPES [ HydrusSerialisable . SERIALISABLE_TYPE_SERVER_TO_CLIENT_CONTENT_UPDATE_PACKAGE ] = ServerToClientContentUpdatePackage
2015-10-14 21:02:25 +00:00
class ServerToClientPetition ( HydrusSerialisable . SerialisableBase ) :
SERIALISABLE_TYPE = HydrusSerialisable . SERIALISABLE_TYPE_SERVER_TO_CLIENT_PETITION
SERIALISABLE_VERSION = 1
def __init__ ( self , action = None , petitioner_account_identifier = None , reason = None , contents = None ) :
HydrusSerialisable . SerialisableBase . __init__ ( self )
self . _action = action
self . _petitioner_account_identifier = petitioner_account_identifier
self . _reason = reason
self . _contents = contents
def _GetSerialisableInfo ( self ) :
serialisable_petitioner_account_identifier = self . _petitioner_account_identifier . GetSerialisableTuple ( )
serialisable_contents = [ content . GetSerialisableTuple ( ) for content in self . _contents ]
return ( self . _action , serialisable_petitioner_account_identifier , self . _reason , serialisable_contents )
def _InitialiseFromSerialisableInfo ( self , serialisable_info ) :
( self . _action , serialisable_petitioner_account_identifier , self . _reason , serialisable_contents ) = serialisable_info
self . _petitioner_account_identifier = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_petitioner_account_identifier )
self . _contents = [ HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_content ) for serialisable_content in serialisable_contents ]
def GetActionTextAndColour ( self ) :
action_text = ' '
if self . _action == HC . CONTENT_UPDATE_PEND :
action_text + = ' ADD '
action_colour = ( 0 , 128 , 0 )
elif self . _action == HC . CONTENT_UPDATE_PETITION :
action_text + = ' DELETE '
action_colour = ( 128 , 0 , 0 )
return ( action_text , action_colour )
def GetApproval ( self , filtered_contents ) :
return ConvertContentsToClientToServerContentUpdatePackage ( self . _action , filtered_contents , reason = self . _reason )
def GetContents ( self ) :
return self . _contents
def GetDenial ( self , filtered_contents ) :
if self . _action == HC . CONTENT_UPDATE_PEND :
denial_action = HC . CONTENT_UPDATE_DENY_PEND
elif self . _action == HC . CONTENT_UPDATE_PETITION :
denial_action = HC . CONTENT_UPDATE_DENY_PETITION
return ConvertContentsToClientToServerContentUpdatePackage ( denial_action , filtered_contents )
def GetPetitionerIdentifier ( self ) :
return self . _petitioner_account_identifier
def GetReason ( self ) :
return self . _reason
HydrusSerialisable . SERIALISABLE_TYPES_TO_OBJECT_TYPES [ HydrusSerialisable . SERIALISABLE_TYPE_SERVER_TO_CLIENT_PETITION ] = ServerToClientPetition
2015-06-03 21:05:13 +00:00
class ServerToClientServiceUpdatePackage ( HydrusSerialisable . SerialisableBase ) :
SERIALISABLE_TYPE = HydrusSerialisable . SERIALISABLE_TYPE_SERVER_TO_CLIENT_SERVICE_UPDATE_PACKAGE
SERIALISABLE_VERSION = 1
def __init__ ( self ) :
HydrusSerialisable . SerialisableBase . __init__ ( self )
self . _service_data = { }
def _GetSerialisableInfo ( self ) :
return self . _service_data . items ( )
def _InitialiseFromSerialisableInfo ( self , serialisable_info ) :
self . _service_data = dict ( serialisable_info )
def GetBegin ( self ) :
( begin , end ) = self . _service_data [ HC . SERVICE_UPDATE_BEGIN_END ]
return begin
def GetNextBegin ( self ) :
( begin , end ) = self . _service_data [ HC . SERVICE_UPDATE_BEGIN_END ]
return end + 1
def GetSubindexCount ( self ) :
return self . _service_data [ HC . SERVICE_UPDATE_SUBINDEX_COUNT ]
def IterateServiceUpdates ( self ) :
if HC . SERVICE_UPDATE_NEWS in self . _service_data :
news_rows = self . _service_data [ HC . SERVICE_UPDATE_NEWS ]
yield ServiceUpdate ( HC . SERVICE_UPDATE_NEWS , news_rows )
def SetBeginEnd ( self , begin , end ) :
self . _service_data [ HC . SERVICE_UPDATE_BEGIN_END ] = ( begin , end )
2015-03-25 22:04:19 +00:00
2015-06-03 21:05:13 +00:00
def SetNews ( self , news_rows ) :
self . _service_data [ HC . SERVICE_UPDATE_NEWS ] = news_rows
def SetSubindexCount ( self , count ) :
self . _service_data [ HC . SERVICE_UPDATE_SUBINDEX_COUNT ] = count
HydrusSerialisable . SERIALISABLE_TYPES_TO_OBJECT_TYPES [ HydrusSerialisable . SERIALISABLE_TYPE_SERVER_TO_CLIENT_SERVICE_UPDATE_PACKAGE ] = ServerToClientServiceUpdatePackage
2015-03-25 22:04:19 +00:00
class ServiceUpdate ( object ) :
def __init__ ( self , action , row = None ) :
self . _action = action
self . _row = row
def ToTuple ( self ) : return ( self . _action , self . _row )