461 lines
14 KiB
Python
461 lines
14 KiB
Python
import json
|
|
import zlib
|
|
|
|
LZ4_OK = False
|
|
|
|
try:
|
|
|
|
import lz4
|
|
import lz4.block
|
|
|
|
LZ4_OK = True
|
|
|
|
except: # ImportError wasn't enough here as Linux went up the shoot with a __version__ doesn't exist bs
|
|
|
|
print( 'Could not import lz4--nbd.' )
|
|
|
|
SERIALISABLE_TYPE_BASE = 0
|
|
SERIALISABLE_TYPE_BASE_NAMED = 1
|
|
SERIALISABLE_TYPE_SHORTCUT_SET = 2
|
|
SERIALISABLE_TYPE_SUBSCRIPTION = 3
|
|
SERIALISABLE_TYPE_PERIODIC = 4
|
|
SERIALISABLE_TYPE_GALLERY_IDENTIFIER = 5
|
|
SERIALISABLE_TYPE_TAG_IMPORT_OPTIONS = 6
|
|
SERIALISABLE_TYPE_FILE_IMPORT_OPTIONS = 7
|
|
SERIALISABLE_TYPE_FILE_SEED_CACHE = 8
|
|
SERIALISABLE_TYPE_HDD_IMPORT = 9
|
|
SERIALISABLE_TYPE_SERVER_TO_CLIENT_CONTENT_UPDATE_PACKAGE = 10
|
|
SERIALISABLE_TYPE_SERVER_TO_CLIENT_SERVICE_UPDATE_PACKAGE = 11
|
|
SERIALISABLE_TYPE_MANAGEMENT_CONTROLLER = 12
|
|
SERIALISABLE_TYPE_GUI_SESSION = 13
|
|
SERIALISABLE_TYPE_PREDICATE = 14
|
|
SERIALISABLE_TYPE_FILE_SEARCH_CONTEXT = 15
|
|
SERIALISABLE_TYPE_EXPORT_FOLDER = 16
|
|
SERIALISABLE_TYPE_WATCHER_IMPORT = 17
|
|
SERIALISABLE_TYPE_SIMPLE_DOWNLOADER_IMPORT = 18
|
|
SERIALISABLE_TYPE_IMPORT_FOLDER = 19
|
|
SERIALISABLE_TYPE_MULTIPLE_GALLERY_IMPORT = 20
|
|
SERIALISABLE_TYPE_DICTIONARY = 21
|
|
SERIALISABLE_TYPE_CLIENT_OPTIONS = 22
|
|
SERIALISABLE_TYPE_CONTENT = 23
|
|
SERIALISABLE_TYPE_PETITION = 24
|
|
SERIALISABLE_TYPE_ACCOUNT_IDENTIFIER = 25
|
|
SERIALISABLE_TYPE_LIST = 26
|
|
SERIALISABLE_TYPE_PARSE_FORMULA_HTML = 27
|
|
SERIALISABLE_TYPE_URLS_IMPORT = 28
|
|
SERIALISABLE_TYPE_PARSE_NODE_CONTENT_LINK = 29
|
|
SERIALISABLE_TYPE_CONTENT_PARSER = 30
|
|
SERIALISABLE_TYPE_PARSE_FORMULA_JSON = 31
|
|
SERIALISABLE_TYPE_PARSE_ROOT_FILE_LOOKUP = 32
|
|
SERIALISABLE_TYPE_BYTES_DICT = 33
|
|
SERIALISABLE_TYPE_CONTENT_UPDATE = 34
|
|
SERIALISABLE_TYPE_CREDENTIALS = 35
|
|
SERIALISABLE_TYPE_DEFINITIONS_UPDATE = 36
|
|
SERIALISABLE_TYPE_METADATA = 37
|
|
SERIALISABLE_TYPE_BANDWIDTH_RULES = 38
|
|
SERIALISABLE_TYPE_BANDWIDTH_TRACKER = 39
|
|
SERIALISABLE_TYPE_CLIENT_TO_SERVER_UPDATE = 40
|
|
SERIALISABLE_TYPE_SHORTCUT = 41
|
|
SERIALISABLE_TYPE_APPLICATION_COMMAND = 42
|
|
SERIALISABLE_TYPE_DUPLICATE_ACTION_OPTIONS = 43
|
|
SERIALISABLE_TYPE_TAG_FILTER = 44
|
|
SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER = 45
|
|
SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER = 46
|
|
SERIALISABLE_TYPE_NETWORK_CONTEXT = 47
|
|
SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER = 48
|
|
SERIALISABLE_TYPE_MEDIA_SORT = 49
|
|
SERIALISABLE_TYPE_URL_CLASS = 50
|
|
SERIALISABLE_TYPE_STRING_MATCH = 51
|
|
SERIALISABLE_TYPE_CHECKER_OPTIONS = 52
|
|
SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER = 53
|
|
SERIALISABLE_TYPE_SUBSCRIPTION_QUERY = 54
|
|
SERIALISABLE_TYPE_STRING_CONVERTER = 55
|
|
SERIALISABLE_TYPE_FILENAME_TAGGING_OPTIONS = 56
|
|
SERIALISABLE_TYPE_FILE_SEED = 57
|
|
SERIALISABLE_TYPE_PAGE_PARSER = 58
|
|
SERIALISABLE_TYPE_PARSE_FORMULA_COMPOUND = 59
|
|
SERIALISABLE_TYPE_PARSE_FORMULA_CONTEXT_VARIABLE = 60
|
|
SERIALISABLE_TYPE_TAG_SUMMARY_GENERATOR = 61
|
|
SERIALISABLE_TYPE_PARSE_RULE_HTML = 62
|
|
SERIALISABLE_TYPE_SIMPLE_DOWNLOADER_PARSE_FORMULA = 63
|
|
SERIALISABLE_TYPE_MULTIPLE_WATCHER_IMPORT = 64
|
|
SERIALISABLE_TYPE_SERVICE_TAG_IMPORT_OPTIONS = 65
|
|
SERIALISABLE_TYPE_GALLERY_SEED = 66
|
|
SERIALISABLE_TYPE_GALLERY_SEED_LOG = 67
|
|
SERIALISABLE_TYPE_GALLERY_IMPORT = 68
|
|
SERIALISABLE_TYPE_GALLERY_URL_GENERATOR = 69
|
|
SERIALISABLE_TYPE_NESTED_GALLERY_URL_GENERATOR = 70
|
|
SERIALISABLE_TYPE_DOMAIN_METADATA_PACKAGE = 71
|
|
SERIALISABLE_TYPE_LOGIN_CREDENTIAL_DEFINITION = 72
|
|
SERIALISABLE_TYPE_LOGIN_SCRIPT_DOMAIN = 73
|
|
SERIALISABLE_TYPE_LOGIN_STEP = 74
|
|
SERIALISABLE_TYPE_CLIENT_API_MANAGER = 75
|
|
SERIALISABLE_TYPE_CLIENT_API_PERMISSIONS = 76
|
|
SERIALISABLE_TYPE_SERVICE_KEYS_TO_TAGS = 77
|
|
SERIALISABLE_TYPE_MEDIA_COLLECT = 78
|
|
|
|
SERIALISABLE_TYPES_TO_OBJECT_TYPES = {}
|
|
|
|
def CreateFromNetworkBytes( network_string ):
|
|
|
|
try:
|
|
|
|
obj_bytes = zlib.decompress( network_string )
|
|
|
|
except zlib.error:
|
|
|
|
if LZ4_OK:
|
|
|
|
obj_bytes = lz4.block.decompress( network_string )
|
|
|
|
else:
|
|
|
|
raise
|
|
|
|
|
|
|
|
obj_string = str( obj_bytes, 'utf-8' )
|
|
|
|
return CreateFromString( obj_string )
|
|
|
|
def CreateFromString( obj_string ):
|
|
|
|
obj_tuple = json.loads( obj_string )
|
|
|
|
return CreateFromSerialisableTuple( obj_tuple )
|
|
|
|
def CreateFromSerialisableTuple( obj_tuple ):
|
|
|
|
if len( obj_tuple ) == 3:
|
|
|
|
( serialisable_type, version, serialisable_info ) = obj_tuple
|
|
|
|
obj = SERIALISABLE_TYPES_TO_OBJECT_TYPES[ serialisable_type ]()
|
|
|
|
else:
|
|
|
|
( serialisable_type, name, version, serialisable_info ) = obj_tuple
|
|
|
|
obj = SERIALISABLE_TYPES_TO_OBJECT_TYPES[ serialisable_type ]( name )
|
|
|
|
|
|
obj.InitialiseFromSerialisableInfo( version, serialisable_info )
|
|
|
|
return obj
|
|
|
|
def GetNonDupeName( original_name, disallowed_names ):
|
|
|
|
i = 1
|
|
|
|
non_dupe_name = original_name
|
|
|
|
while non_dupe_name in disallowed_names:
|
|
|
|
non_dupe_name = original_name + ' (' + str( i ) + ')'
|
|
|
|
i += 1
|
|
|
|
|
|
return non_dupe_name
|
|
|
|
def SetNonDupeName( obj, disallowed_names ):
|
|
|
|
non_dupe_name = GetNonDupeName( obj.GetName(), disallowed_names )
|
|
|
|
obj.SetName( non_dupe_name )
|
|
|
|
class SerialisableBase( object ):
|
|
|
|
SERIALISABLE_TYPE = SERIALISABLE_TYPE_BASE
|
|
SERIALISABLE_NAME = 'Base Serialisable Object'
|
|
SERIALISABLE_VERSION = 1
|
|
|
|
def _GetSerialisableInfo( self ):
|
|
|
|
raise NotImplementedError()
|
|
|
|
|
|
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
|
|
|
|
raise NotImplementedError()
|
|
|
|
|
|
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
|
|
|
|
return old_serialisable_info
|
|
|
|
|
|
def DumpToNetworkBytes( self ):
|
|
|
|
obj_string = self.DumpToString()
|
|
|
|
obj_bytes = bytes( obj_string, 'utf-8' )
|
|
|
|
return zlib.compress( obj_bytes, 9 )
|
|
|
|
|
|
def DumpToString( self ):
|
|
|
|
obj_tuple = self.GetSerialisableTuple()
|
|
|
|
return json.dumps( obj_tuple )
|
|
|
|
|
|
def Duplicate( self ):
|
|
|
|
return CreateFromString( self.DumpToString() )
|
|
|
|
|
|
def GetSerialisableTuple( self ):
|
|
|
|
if hasattr( self, '_lock' ):
|
|
|
|
with getattr( self, '_lock' ):
|
|
|
|
serialisable_info = self._GetSerialisableInfo()
|
|
|
|
|
|
else:
|
|
|
|
serialisable_info = self._GetSerialisableInfo()
|
|
|
|
|
|
return ( self.SERIALISABLE_TYPE, self.SERIALISABLE_VERSION, serialisable_info )
|
|
|
|
|
|
def InitialiseFromSerialisableInfo( self, version, serialisable_info ):
|
|
|
|
while version < self.SERIALISABLE_VERSION:
|
|
|
|
( version, serialisable_info ) = self._UpdateSerialisableInfo( version, serialisable_info )
|
|
|
|
|
|
self._InitialiseFromSerialisableInfo( serialisable_info )
|
|
|
|
|
|
class SerialisableBaseNamed( SerialisableBase ):
|
|
|
|
SERIALISABLE_TYPE = SERIALISABLE_TYPE_BASE_NAMED
|
|
SERIALISABLE_NAME = 'Named Base Serialisable Object'
|
|
|
|
def __init__( self, name ):
|
|
|
|
SerialisableBase.__init__( self )
|
|
|
|
self._name = name
|
|
|
|
|
|
def GetSerialisableTuple( self ):
|
|
|
|
return ( self.SERIALISABLE_TYPE, self._name, self.SERIALISABLE_VERSION, self._GetSerialisableInfo() )
|
|
|
|
|
|
def GetName( self ): return self._name
|
|
|
|
def SetName( self, name ): self._name = name
|
|
|
|
def SetNonDupeName( self, disallowed_names ):
|
|
|
|
self._name = GetNonDupeName( self._name, disallowed_names )
|
|
|
|
|
|
class SerialisableDictionary( SerialisableBase, dict ):
|
|
|
|
SERIALISABLE_TYPE = SERIALISABLE_TYPE_DICTIONARY
|
|
SERIALISABLE_NAME = 'Serialisable Dictionary'
|
|
SERIALISABLE_VERSION = 1
|
|
|
|
def __init__( self, *args, **kwargs ):
|
|
|
|
dict.__init__( self, *args, **kwargs )
|
|
SerialisableBase.__init__( self )
|
|
|
|
|
|
def _GetSerialisableInfo( self ):
|
|
|
|
simple_key_simple_value_pairs = []
|
|
simple_key_serialisable_value_pairs = []
|
|
serialisable_key_simple_value_pairs = []
|
|
serialisable_key_serialisable_value_pairs = []
|
|
|
|
for ( key, value ) in list(self.items()):
|
|
|
|
if isinstance( key, SerialisableBase ):
|
|
|
|
serialisable_key = key.GetSerialisableTuple()
|
|
|
|
if isinstance( value, SerialisableBase ):
|
|
|
|
serialisable_value = value.GetSerialisableTuple()
|
|
|
|
serialisable_key_serialisable_value_pairs.append( ( serialisable_key, serialisable_value ) )
|
|
|
|
else:
|
|
|
|
serialisable_value = value
|
|
|
|
serialisable_key_simple_value_pairs.append( ( serialisable_key, serialisable_value ) )
|
|
|
|
|
|
else:
|
|
|
|
serialisable_key = key
|
|
|
|
if isinstance( value, SerialisableBase ):
|
|
|
|
serialisable_value = value.GetSerialisableTuple()
|
|
|
|
simple_key_serialisable_value_pairs.append( ( serialisable_key, serialisable_value ) )
|
|
|
|
else:
|
|
|
|
serialisable_value = value
|
|
|
|
simple_key_simple_value_pairs.append( ( serialisable_key, serialisable_value ) )
|
|
|
|
|
|
|
|
|
|
return ( simple_key_simple_value_pairs, simple_key_serialisable_value_pairs, serialisable_key_simple_value_pairs, serialisable_key_serialisable_value_pairs )
|
|
|
|
|
|
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
|
|
|
|
( simple_key_simple_value_pairs, simple_key_serialisable_value_pairs, serialisable_key_simple_value_pairs, serialisable_key_serialisable_value_pairs ) = serialisable_info
|
|
|
|
for ( key, value ) in simple_key_simple_value_pairs:
|
|
|
|
self[ key ] = value
|
|
|
|
|
|
for ( key, serialisable_value ) in simple_key_serialisable_value_pairs:
|
|
|
|
value = CreateFromSerialisableTuple( serialisable_value )
|
|
|
|
self[ key ] = value
|
|
|
|
|
|
for ( serialisable_key, value ) in serialisable_key_simple_value_pairs:
|
|
|
|
key = CreateFromSerialisableTuple( serialisable_key )
|
|
|
|
self[ key ] = value
|
|
|
|
|
|
for ( serialisable_key, serialisable_value ) in serialisable_key_serialisable_value_pairs:
|
|
|
|
key = CreateFromSerialisableTuple( serialisable_key )
|
|
|
|
value = CreateFromSerialisableTuple( serialisable_value )
|
|
|
|
self[ key ] = value
|
|
|
|
|
|
|
|
SERIALISABLE_TYPES_TO_OBJECT_TYPES[ SERIALISABLE_TYPE_DICTIONARY ] = SerialisableDictionary
|
|
|
|
class SerialisableBytesDictionary( SerialisableBase, dict ):
|
|
|
|
SERIALISABLE_TYPE = SERIALISABLE_TYPE_BYTES_DICT
|
|
SERIALISABLE_NAME = 'Serialisable Dictionary With Bytestring Key/Value Support'
|
|
SERIALISABLE_VERSION = 1
|
|
|
|
def __init__( self, *args, **kwargs ):
|
|
|
|
dict.__init__( self, *args, **kwargs )
|
|
SerialisableBase.__init__( self )
|
|
|
|
|
|
def _GetSerialisableInfo( self ):
|
|
|
|
pairs = []
|
|
|
|
for ( key, value ) in list(self.items()):
|
|
|
|
if isinstance( key, int ):
|
|
|
|
encoded_key = key
|
|
|
|
else:
|
|
|
|
encoded_key = key.hex()
|
|
|
|
|
|
if isinstance( value, ( list, tuple, set ) ):
|
|
|
|
encoded_value = [ item.hex() for item in value ]
|
|
|
|
elif value is None:
|
|
|
|
encoded_value = value
|
|
|
|
else:
|
|
|
|
encoded_value = value.hex()
|
|
|
|
|
|
pairs.append( ( encoded_key, encoded_value ) )
|
|
|
|
|
|
return pairs
|
|
|
|
|
|
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
|
|
|
|
for ( encoded_key, encoded_value ) in serialisable_info:
|
|
|
|
if isinstance( encoded_key, int ):
|
|
|
|
key = encoded_key
|
|
|
|
else:
|
|
|
|
key = bytes.fromhex( encoded_key )
|
|
|
|
|
|
if isinstance( encoded_value, ( list, tuple, set ) ):
|
|
|
|
value = [ bytes.fromhex( encoded_item ) for encoded_item in encoded_value ]
|
|
|
|
elif encoded_value is None:
|
|
|
|
value = encoded_value
|
|
|
|
else:
|
|
|
|
value = bytes.fromhex( encoded_value )
|
|
|
|
|
|
self[ key ] = value
|
|
|
|
|
|
|
|
SERIALISABLE_TYPES_TO_OBJECT_TYPES[ SERIALISABLE_TYPE_BYTES_DICT ] = SerialisableBytesDictionary
|
|
|
|
class SerialisableList( SerialisableBase, list ):
|
|
|
|
SERIALISABLE_TYPE = SERIALISABLE_TYPE_LIST
|
|
SERIALISABLE_NAME = 'Serialisable List'
|
|
SERIALISABLE_VERSION = 1
|
|
|
|
def __init__( self, *args, **kwargs ):
|
|
|
|
list.__init__( self, *args, **kwargs )
|
|
SerialisableBase.__init__( self )
|
|
|
|
|
|
def _GetSerialisableInfo( self ):
|
|
|
|
return [ obj.GetSerialisableTuple() for obj in self ]
|
|
|
|
|
|
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
|
|
|
|
for obj_tuple in serialisable_info:
|
|
|
|
self.append( CreateFromSerialisableTuple( obj_tuple ) )
|
|
|
|
|
|
|
|
SERIALISABLE_TYPES_TO_OBJECT_TYPES[ SERIALISABLE_TYPE_LIST ] = SerialisableList
|