hydrus/include/HydrusSerialisable.py

import json
import zlib

LZ4_OK = False

try:
    
    import lz4
    import lz4.block
    
    LZ4_OK = True
    
except: # ImportError wasn't enough here as Linux went up the shoot with a __version__ doesn't exist bs
    
    print( 'Could not import lz4--nbd.' )
    
SERIALISABLE_TYPE_BASE = 0
SERIALISABLE_TYPE_BASE_NAMED = 1
SERIALISABLE_TYPE_SHORTCUT_SET = 2
SERIALISABLE_TYPE_SUBSCRIPTION = 3
SERIALISABLE_TYPE_PERIODIC = 4
SERIALISABLE_TYPE_GALLERY_IDENTIFIER = 5
SERIALISABLE_TYPE_TAG_IMPORT_OPTIONS = 6
SERIALISABLE_TYPE_FILE_IMPORT_OPTIONS = 7
SERIALISABLE_TYPE_FILE_SEED_CACHE = 8
SERIALISABLE_TYPE_HDD_IMPORT = 9
SERIALISABLE_TYPE_SERVER_TO_CLIENT_CONTENT_UPDATE_PACKAGE = 10
SERIALISABLE_TYPE_SERVER_TO_CLIENT_SERVICE_UPDATE_PACKAGE = 11
SERIALISABLE_TYPE_MANAGEMENT_CONTROLLER = 12
SERIALISABLE_TYPE_GUI_SESSION = 13
SERIALISABLE_TYPE_PREDICATE = 14
SERIALISABLE_TYPE_FILE_SEARCH_CONTEXT = 15
SERIALISABLE_TYPE_EXPORT_FOLDER = 16
SERIALISABLE_TYPE_WATCHER_IMPORT = 17
SERIALISABLE_TYPE_SIMPLE_DOWNLOADER_IMPORT = 18
SERIALISABLE_TYPE_IMPORT_FOLDER = 19
SERIALISABLE_TYPE_MULTIPLE_GALLERY_IMPORT = 20
SERIALISABLE_TYPE_DICTIONARY = 21
SERIALISABLE_TYPE_CLIENT_OPTIONS = 22
SERIALISABLE_TYPE_CONTENT = 23
SERIALISABLE_TYPE_PETITION = 24
SERIALISABLE_TYPE_ACCOUNT_IDENTIFIER = 25
SERIALISABLE_TYPE_LIST = 26
SERIALISABLE_TYPE_PARSE_FORMULA_HTML = 27
SERIALISABLE_TYPE_URLS_IMPORT = 28
SERIALISABLE_TYPE_PARSE_NODE_CONTENT_LINK = 29
SERIALISABLE_TYPE_CONTENT_PARSER = 30
SERIALISABLE_TYPE_PARSE_FORMULA_JSON = 31
SERIALISABLE_TYPE_PARSE_ROOT_FILE_LOOKUP = 32
SERIALISABLE_TYPE_BYTES_DICT = 33
SERIALISABLE_TYPE_CONTENT_UPDATE = 34
SERIALISABLE_TYPE_CREDENTIALS = 35
SERIALISABLE_TYPE_DEFINITIONS_UPDATE = 36
SERIALISABLE_TYPE_METADATA = 37
SERIALISABLE_TYPE_BANDWIDTH_RULES = 38
SERIALISABLE_TYPE_BANDWIDTH_TRACKER = 39
SERIALISABLE_TYPE_CLIENT_TO_SERVER_UPDATE = 40
SERIALISABLE_TYPE_SHORTCUT = 41
SERIALISABLE_TYPE_APPLICATION_COMMAND = 42
SERIALISABLE_TYPE_DUPLICATE_ACTION_OPTIONS = 43
SERIALISABLE_TYPE_TAG_FILTER = 44
SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER = 45
SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER = 46
SERIALISABLE_TYPE_NETWORK_CONTEXT = 47
SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER = 48
SERIALISABLE_TYPE_MEDIA_SORT = 49
SERIALISABLE_TYPE_URL_CLASS = 50
SERIALISABLE_TYPE_STRING_MATCH = 51
SERIALISABLE_TYPE_CHECKER_OPTIONS = 52
SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER = 53
SERIALISABLE_TYPE_SUBSCRIPTION_QUERY = 54
SERIALISABLE_TYPE_STRING_CONVERTER = 55
SERIALISABLE_TYPE_FILENAME_TAGGING_OPTIONS = 56
SERIALISABLE_TYPE_FILE_SEED = 57
SERIALISABLE_TYPE_PAGE_PARSER = 58
SERIALISABLE_TYPE_PARSE_FORMULA_COMPOUND = 59
SERIALISABLE_TYPE_PARSE_FORMULA_CONTEXT_VARIABLE = 60
SERIALISABLE_TYPE_TAG_SUMMARY_GENERATOR = 61
SERIALISABLE_TYPE_PARSE_RULE_HTML = 62
SERIALISABLE_TYPE_SIMPLE_DOWNLOADER_PARSE_FORMULA = 63
SERIALISABLE_TYPE_MULTIPLE_WATCHER_IMPORT = 64
SERIALISABLE_TYPE_SERVICE_TAG_IMPORT_OPTIONS = 65
SERIALISABLE_TYPE_GALLERY_SEED = 66
SERIALISABLE_TYPE_GALLERY_SEED_LOG = 67
SERIALISABLE_TYPE_GALLERY_IMPORT = 68
SERIALISABLE_TYPE_GALLERY_URL_GENERATOR = 69
SERIALISABLE_TYPE_NESTED_GALLERY_URL_GENERATOR = 70
SERIALISABLE_TYPE_DOMAIN_METADATA_PACKAGE = 71
SERIALISABLE_TYPE_LOGIN_CREDENTIAL_DEFINITION = 72
SERIALISABLE_TYPE_LOGIN_SCRIPT_DOMAIN = 73
SERIALISABLE_TYPE_LOGIN_STEP = 74
SERIALISABLE_TYPE_CLIENT_API_MANAGER = 75
SERIALISABLE_TYPE_CLIENT_API_PERMISSIONS = 76
SERIALISABLE_TYPE_SERVICE_KEYS_TO_TAGS = 77
SERIALISABLE_TYPE_MEDIA_COLLECT = 78

SERIALISABLE_TYPES_TO_OBJECT_TYPES = {}

def CreateFromNetworkBytes( network_string ):
    
    try:
        
        obj_bytes = zlib.decompress( network_string )
        
    except zlib.error:
        
        if LZ4_OK:
            
            obj_bytes = lz4.block.decompress( network_string )
            
        else:
            
            raise
            
        
    
    obj_string = str( obj_bytes, 'utf-8' )
    
    return CreateFromString( obj_string )
    
def CreateFromString( obj_string ):
    
    obj_tuple = json.loads( obj_string )
    
    return CreateFromSerialisableTuple( obj_tuple )
    
def CreateFromSerialisableTuple( obj_tuple ):
    
    if len( obj_tuple ) == 3:
        
        ( serialisable_type, version, serialisable_info ) = obj_tuple
        
        obj = SERIALISABLE_TYPES_TO_OBJECT_TYPES[ serialisable_type ]()
        
    else:
        
        ( serialisable_type, name, version, serialisable_info ) = obj_tuple
        
        obj = SERIALISABLE_TYPES_TO_OBJECT_TYPES[ serialisable_type ]( name )
        
    
    obj.InitialiseFromSerialisableInfo( version, serialisable_info )
    
    return obj
    
def GetNonDupeName( original_name, disallowed_names ):
    
    i = 1
    
    non_dupe_name = original_name
    
    while non_dupe_name in disallowed_names:
        
        non_dupe_name = original_name + ' (' + str( i ) + ')'
        
        i += 1
        
    
    return non_dupe_name
    
def SetNonDupeName( obj, disallowed_names ):
    
    non_dupe_name = GetNonDupeName( obj.GetName(), disallowed_names )
    
    obj.SetName( non_dupe_name )

class SerialisableBase( object ):
    
    SERIALISABLE_TYPE = SERIALISABLE_TYPE_BASE
    SERIALISABLE_NAME = 'Base Serialisable Object'
    SERIALISABLE_VERSION = 1
    
    def _GetSerialisableInfo( self ):
        
        raise NotImplementedError()
        
    
    def _InitialiseFromSerialisableInfo( self, serialisable_info ):
        
        raise NotImplementedError()
        
    
    def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
        
        return old_serialisable_info
        
    
    def DumpToNetworkBytes( self ):
        
        obj_string = self.DumpToString()
        
        obj_bytes = bytes( obj_string, 'utf-8' )
        
        return zlib.compress( obj_bytes, 9 )
        
    
    def DumpToString( self ):
        
        obj_tuple = self.GetSerialisableTuple()
        
        return json.dumps( obj_tuple )
        
    
    def Duplicate( self ):
        
        return CreateFromString( self.DumpToString() )
        
    
    def GetSerialisableTuple( self ):
        
        if hasattr( self, '_lock' ):
            
            with getattr( self, '_lock' ):
                
                serialisable_info = self._GetSerialisableInfo()
                
            
        else:
            
            serialisable_info = self._GetSerialisableInfo()
            
        
        return ( self.SERIALISABLE_TYPE, self.SERIALISABLE_VERSION, serialisable_info )
        
    
    def InitialiseFromSerialisableInfo( self, version, serialisable_info ):
        
        while version < self.SERIALISABLE_VERSION:
            
            ( version, serialisable_info ) = self._UpdateSerialisableInfo( version, serialisable_info )
            
        
        self._InitialiseFromSerialisableInfo( serialisable_info )
        
    
class SerialisableBaseNamed( SerialisableBase ):
    
    SERIALISABLE_TYPE = SERIALISABLE_TYPE_BASE_NAMED
    SERIALISABLE_NAME = 'Named Base Serialisable Object'
    
    def __init__( self, name ):
        
        SerialisableBase.__init__( self )
        
        self._name = name
        
    
    def GetSerialisableTuple( self ):
        
        return ( self.SERIALISABLE_TYPE, self._name, self.SERIALISABLE_VERSION, self._GetSerialisableInfo() )
        
    
    def GetName( self ): return self._name
    
    def SetName( self, name ): self._name = name
    
    def SetNonDupeName( self, disallowed_names ):
        
        self._name = GetNonDupeName( self._name, disallowed_names )
        
    
class SerialisableDictionary( SerialisableBase, dict ):
    
    SERIALISABLE_TYPE = SERIALISABLE_TYPE_DICTIONARY
    SERIALISABLE_NAME = 'Serialisable Dictionary'
    SERIALISABLE_VERSION = 1
    
    def __init__( self, *args, **kwargs ):
        
        dict.__init__( self, *args, **kwargs )
        SerialisableBase.__init__( self )
        
    
    def _GetSerialisableInfo( self ):
        
        simple_key_simple_value_pairs = []
        simple_key_serialisable_value_pairs = []
        serialisable_key_simple_value_pairs = []
        serialisable_key_serialisable_value_pairs = []
        
        for ( key, value ) in list(self.items()):
            
            if isinstance( key, SerialisableBase ):
                
                serialisable_key = key.GetSerialisableTuple()
                
                if isinstance( value, SerialisableBase ):
                    
                    serialisable_value = value.GetSerialisableTuple()
                    
                    serialisable_key_serialisable_value_pairs.append( ( serialisable_key, serialisable_value ) )
                    
                else:
                    
                    serialisable_value = value
                    
                    serialisable_key_simple_value_pairs.append( ( serialisable_key, serialisable_value ) )
                    
                
            else:
                
                serialisable_key = key
                
                if isinstance( value, SerialisableBase ):
                    
                    serialisable_value = value.GetSerialisableTuple()
                    
                    simple_key_serialisable_value_pairs.append( ( serialisable_key, serialisable_value ) )
                    
                else:
                    
                    serialisable_value = value
                    
                    simple_key_simple_value_pairs.append( ( serialisable_key, serialisable_value ) )
                    
                
            
        
        return ( simple_key_simple_value_pairs, simple_key_serialisable_value_pairs, serialisable_key_simple_value_pairs, serialisable_key_serialisable_value_pairs )
        
    
    def _InitialiseFromSerialisableInfo( self, serialisable_info ):
        
        ( simple_key_simple_value_pairs, simple_key_serialisable_value_pairs, serialisable_key_simple_value_pairs, serialisable_key_serialisable_value_pairs ) = serialisable_info
        
        for ( key, value ) in simple_key_simple_value_pairs:
            
            self[ key ] = value
            
        
        for ( key, serialisable_value ) in simple_key_serialisable_value_pairs:
            
            value = CreateFromSerialisableTuple( serialisable_value )
            
            self[ key ] = value
            
        
        for ( serialisable_key, value ) in serialisable_key_simple_value_pairs:
            
            key = CreateFromSerialisableTuple( serialisable_key )
            
            self[ key ] = value
            
        
        for ( serialisable_key, serialisable_value ) in serialisable_key_serialisable_value_pairs:
            
            key = CreateFromSerialisableTuple( serialisable_key )
            
            value = CreateFromSerialisableTuple( serialisable_value )
            
            self[ key ] = value
            
        
    
SERIALISABLE_TYPES_TO_OBJECT_TYPES[ SERIALISABLE_TYPE_DICTIONARY ] = SerialisableDictionary

class SerialisableBytesDictionary( SerialisableBase, dict ):
    
    SERIALISABLE_TYPE = SERIALISABLE_TYPE_BYTES_DICT
    SERIALISABLE_NAME = 'Serialisable Dictionary With Bytestring Key/Value Support'
    SERIALISABLE_VERSION = 1
    
    def __init__( self, *args, **kwargs ):
        
        dict.__init__( self, *args, **kwargs )
        SerialisableBase.__init__( self )
        
    
    def _GetSerialisableInfo( self ):
        
        pairs = []
        
        for ( key, value ) in list(self.items()):
            
            if isinstance( key, int ):
                
                encoded_key = key
                
            else:
                
                encoded_key = key.hex()
                
            
            if isinstance( value, ( list, tuple, set ) ):
                
                encoded_value = [ item.hex() for item in value ]
                
            elif value is None:
                
                encoded_value = value
                
            else:
                
                encoded_value = value.hex()
                
            
            pairs.append( ( encoded_key, encoded_value ) )
            
        
        return pairs
        
    
    def _InitialiseFromSerialisableInfo( self, serialisable_info ):
        
        for ( encoded_key, encoded_value ) in serialisable_info:
            
            if isinstance( encoded_key, int ):
                
                key = encoded_key
                
            else:
                
                key = bytes.fromhex( encoded_key )
                
            
            if isinstance( encoded_value, ( list, tuple, set ) ):
                
                value = [ bytes.fromhex( encoded_item ) for encoded_item in encoded_value ]
                
            elif encoded_value is None:
                
                value = encoded_value
                
            else:
                
                value = bytes.fromhex( encoded_value )
                
            
            self[ key ] = value
            
        
    
SERIALISABLE_TYPES_TO_OBJECT_TYPES[ SERIALISABLE_TYPE_BYTES_DICT ] = SerialisableBytesDictionary

class SerialisableList( SerialisableBase, list ):
    
    SERIALISABLE_TYPE = SERIALISABLE_TYPE_LIST
    SERIALISABLE_NAME = 'Serialisable List'
    SERIALISABLE_VERSION = 1
    
    def __init__( self, *args, **kwargs ):
        
        list.__init__( self, *args, **kwargs )
        SerialisableBase.__init__( self )
        
    
    def _GetSerialisableInfo( self ):
        
        return [ obj.GetSerialisableTuple() for obj in self ]
        
    
    def _InitialiseFromSerialisableInfo( self, serialisable_info ):
        
        for obj_tuple in serialisable_info:
            
            self.append( CreateFromSerialisableTuple( obj_tuple ) )
            
        
    
SERIALISABLE_TYPES_TO_OBJECT_TYPES[ SERIALISABLE_TYPE_LIST ] = SerialisableList