hydrus/hydrus/core/HydrusSerialisable.py

import json
import os
import zlib

LZ4_OK = False

try:
    
    import lz4
    import lz4.block
    
    LZ4_OK = True
    
except: # ImportError wasn't enough here as Linux went up the shoot with a __version__ doesn't exist bs
    
    print( 'Could not import lz4--nbd.' )
    

from hydrus.core import HydrusData
from hydrus.core import HydrusExceptions

SERIALISABLE_TYPE_BASE = 0
SERIALISABLE_TYPE_BASE_NAMED = 1
SERIALISABLE_TYPE_SHORTCUT_SET = 2
SERIALISABLE_TYPE_SUBSCRIPTION_LEGACY = 3
SERIALISABLE_TYPE_PERIODIC = 4
SERIALISABLE_TYPE_GALLERY_IDENTIFIER = 5
SERIALISABLE_TYPE_TAG_IMPORT_OPTIONS = 6
SERIALISABLE_TYPE_FILE_IMPORT_OPTIONS = 7
SERIALISABLE_TYPE_FILE_SEED_CACHE = 8
SERIALISABLE_TYPE_HDD_IMPORT = 9
SERIALISABLE_TYPE_SERVER_TO_CLIENT_CONTENT_UPDATE_PACKAGE = 10
SERIALISABLE_TYPE_SERVER_TO_CLIENT_SERVICE_UPDATE_PACKAGE = 11
SERIALISABLE_TYPE_MANAGEMENT_CONTROLLER = 12
SERIALISABLE_TYPE_GUI_SESSION = 13
SERIALISABLE_TYPE_PREDICATE = 14
SERIALISABLE_TYPE_FILE_SEARCH_CONTEXT = 15
SERIALISABLE_TYPE_EXPORT_FOLDER = 16
SERIALISABLE_TYPE_WATCHER_IMPORT = 17
SERIALISABLE_TYPE_SIMPLE_DOWNLOADER_IMPORT = 18
SERIALISABLE_TYPE_IMPORT_FOLDER = 19
SERIALISABLE_TYPE_MULTIPLE_GALLERY_IMPORT = 20
SERIALISABLE_TYPE_DICTIONARY = 21
SERIALISABLE_TYPE_CLIENT_OPTIONS = 22
SERIALISABLE_TYPE_CONTENT = 23
SERIALISABLE_TYPE_PETITION = 24
SERIALISABLE_TYPE_ACCOUNT_IDENTIFIER = 25
SERIALISABLE_TYPE_LIST = 26
SERIALISABLE_TYPE_PARSE_FORMULA_HTML = 27
SERIALISABLE_TYPE_URLS_IMPORT = 28
SERIALISABLE_TYPE_PARSE_NODE_CONTENT_LINK = 29
SERIALISABLE_TYPE_CONTENT_PARSER = 30
SERIALISABLE_TYPE_PARSE_FORMULA_JSON = 31
SERIALISABLE_TYPE_PARSE_ROOT_FILE_LOOKUP = 32
SERIALISABLE_TYPE_BYTES_DICT = 33
SERIALISABLE_TYPE_CONTENT_UPDATE = 34
SERIALISABLE_TYPE_CREDENTIALS = 35
SERIALISABLE_TYPE_DEFINITIONS_UPDATE = 36
SERIALISABLE_TYPE_METADATA = 37
SERIALISABLE_TYPE_BANDWIDTH_RULES = 38
SERIALISABLE_TYPE_BANDWIDTH_TRACKER = 39
SERIALISABLE_TYPE_CLIENT_TO_SERVER_UPDATE = 40
SERIALISABLE_TYPE_SHORTCUT = 41
SERIALISABLE_TYPE_APPLICATION_COMMAND = 42
SERIALISABLE_TYPE_DUPLICATE_ACTION_OPTIONS = 43
SERIALISABLE_TYPE_TAG_FILTER = 44
SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER_LEGACY = 45
SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER_LEGACY = 46
SERIALISABLE_TYPE_NETWORK_CONTEXT = 47
SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER = 48
SERIALISABLE_TYPE_MEDIA_SORT = 49
SERIALISABLE_TYPE_URL_CLASS = 50
SERIALISABLE_TYPE_STRING_MATCH = 51
SERIALISABLE_TYPE_CHECKER_OPTIONS = 52
SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER = 53
SERIALISABLE_TYPE_SUBSCRIPTION_QUERY_LEGACY = 54
SERIALISABLE_TYPE_STRING_CONVERTER = 55
SERIALISABLE_TYPE_FILENAME_TAGGING_OPTIONS = 56
SERIALISABLE_TYPE_FILE_SEED = 57
SERIALISABLE_TYPE_PAGE_PARSER = 58
SERIALISABLE_TYPE_PARSE_FORMULA_COMPOUND = 59
SERIALISABLE_TYPE_PARSE_FORMULA_CONTEXT_VARIABLE = 60
SERIALISABLE_TYPE_TAG_SUMMARY_GENERATOR = 61
SERIALISABLE_TYPE_PARSE_RULE_HTML = 62
SERIALISABLE_TYPE_SIMPLE_DOWNLOADER_PARSE_FORMULA = 63
SERIALISABLE_TYPE_MULTIPLE_WATCHER_IMPORT = 64
SERIALISABLE_TYPE_SERVICE_TAG_IMPORT_OPTIONS = 65
SERIALISABLE_TYPE_GALLERY_SEED = 66
SERIALISABLE_TYPE_GALLERY_SEED_LOG = 67
SERIALISABLE_TYPE_GALLERY_IMPORT = 68
SERIALISABLE_TYPE_GALLERY_URL_GENERATOR = 69
SERIALISABLE_TYPE_NESTED_GALLERY_URL_GENERATOR = 70
SERIALISABLE_TYPE_DOMAIN_METADATA_PACKAGE = 71
SERIALISABLE_TYPE_LOGIN_CREDENTIAL_DEFINITION = 72
SERIALISABLE_TYPE_LOGIN_SCRIPT_DOMAIN = 73
SERIALISABLE_TYPE_LOGIN_STEP = 74
SERIALISABLE_TYPE_CLIENT_API_MANAGER = 75
SERIALISABLE_TYPE_CLIENT_API_PERMISSIONS = 76
SERIALISABLE_TYPE_SERVICE_KEYS_TO_TAGS = 77
SERIALISABLE_TYPE_MEDIA_COLLECT = 78
SERIALISABLE_TYPE_TAG_DISPLAY_MANAGER = 79
SERIALISABLE_TYPE_TAG_SEARCH_CONTEXT = 80
SERIALISABLE_TYPE_FAVOURITE_SEARCH_MANAGER = 81
SERIALISABLE_TYPE_NOTE_IMPORT_OPTIONS = 82
SERIALISABLE_TYPE_STRING_SPLITTER = 83
SERIALISABLE_TYPE_STRING_PROCESSOR = 84
SERIALISABLE_TYPE_TAG_AUTOCOMPLETE_OPTIONS = 85
SERIALISABLE_TYPE_SUBSCRIPTION_QUERY_LOG_CONTAINER = 86
SERIALISABLE_TYPE_SUBSCRIPTION_QUERY_HEADER = 87
SERIALISABLE_TYPE_SUBSCRIPTION = 88
SERIALISABLE_TYPE_FILE_SEED_CACHE_STATUS = 89
SERIALISABLE_TYPE_SUBSCRIPTION_CONTAINER = 90
SERIALISABLE_TYPE_COLUMN_LIST_STATUS = 91
SERIALISABLE_TYPE_COLUMN_LIST_MANAGER = 92
SERIALISABLE_TYPE_NUMBER_TEST = 93
SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER = 94
SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER = 95
SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER_SESSION_CONTAINER = 96
SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER_TRACKER_CONTAINER = 97
SERIALISABLE_TYPE_SIDECAR_EXPORTER = 98
SERIALISABLE_TYPE_STRING_SORTER = 99
SERIALISABLE_TYPE_STRING_SLICER = 100
SERIALISABLE_TYPE_TAG_SORT = 101

SERIALISABLE_TYPES_TO_OBJECT_TYPES = {}

def CreateFromNetworkBytes( network_string, raise_error_on_future_version = False ):
    
    try:
        
        obj_bytes = zlib.decompress( network_string )
        
    except zlib.error:
        
        if LZ4_OK:
            
            obj_bytes = lz4.block.decompress( network_string )
            
        else:
            
            raise
            
        
    
    obj_string = str( obj_bytes, 'utf-8' )
    
    return CreateFromString( obj_string, raise_error_on_future_version = raise_error_on_future_version )
    
def CreateFromNoneableSerialisableTuple( obj_tuple_or_none, raise_error_on_future_version = False ):
    
    if obj_tuple_or_none is None:
        
        return None
        
    else:
        
        return CreateFromSerialisableTuple( obj_tuple_or_none, raise_error_on_future_version = raise_error_on_future_version )
        
    
def CreateFromString( obj_string, raise_error_on_future_version = False ):
    
    obj_tuple = json.loads( obj_string )
    
    return CreateFromSerialisableTuple( obj_tuple, raise_error_on_future_version = raise_error_on_future_version )
    
def CreateFromSerialisableTuple( obj_tuple, raise_error_on_future_version = False ):
    
    if len( obj_tuple ) == 3:
        
        ( serialisable_type, version, serialisable_info ) = obj_tuple
        
        obj = SERIALISABLE_TYPES_TO_OBJECT_TYPES[ serialisable_type ]()
        
    else:
        
        ( serialisable_type, name, version, serialisable_info ) = obj_tuple
        
        obj = SERIALISABLE_TYPES_TO_OBJECT_TYPES[ serialisable_type ]( name )
        
    
    obj.InitialiseFromSerialisableInfo( version, serialisable_info, raise_error_on_future_version = raise_error_on_future_version )
    
    return obj
    
def GetNoneableSerialisableTuple( obj_or_none ):
    
    if obj_or_none is None:
        
        return None
        
    else:
        
        return obj_or_none.GetSerialisableTuple()
        
    
def SetNonDupeName( obj, disallowed_names ):
    
    non_dupe_name = HydrusData.GetNonDupeName( obj.GetName(), disallowed_names )
    
    obj.SetName( non_dupe_name )
    
def ObjectVersionIsFromTheFuture( obj_tuple ):
    
    if len( obj_tuple ) == 3:
        
        ( serialisable_type, version, serialisable_info ) = obj_tuple
        
    else:
        
        ( serialisable_type, name, version, serialisable_info ) = obj_tuple
        
    
    return SERIALISABLE_TYPES_TO_OBJECT_TYPES[ serialisable_type ].SERIALISABLE_VERSION > version
    
class SerialisableBase( object ):
    
    SERIALISABLE_TYPE = SERIALISABLE_TYPE_BASE
    SERIALISABLE_NAME = 'Base Serialisable Object'
    SERIALISABLE_VERSION = 1
    
    def _GetSerialisableInfo( self ):
        
        raise NotImplementedError()
        
    
    def _InitialiseFromSerialisableInfo( self, serialisable_info ):
        
        raise NotImplementedError()
        
    
    def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
        
        return old_serialisable_info
        
    
    def DumpToNetworkBytes( self ):
        
        obj_string = self.DumpToString()
        
        obj_bytes = bytes( obj_string, 'utf-8' )
        
        return zlib.compress( obj_bytes, 9 )
        
    
    def DumpToString( self ):
        
        obj_tuple = self.GetSerialisableTuple()
        
        return json.dumps( obj_tuple )
        
    
    def Duplicate( self ):
        
        return CreateFromString( self.DumpToString() )
        
    
    def GetSerialisableTuple( self ):
        
        if hasattr( self, '_lock' ):
            
            with getattr( self, '_lock' ):
                
                serialisable_info = self._GetSerialisableInfo()
                
            
        else:
            
            serialisable_info = self._GetSerialisableInfo()
            
        
        return ( self.SERIALISABLE_TYPE, self.SERIALISABLE_VERSION, serialisable_info )
        
    
    def InitialiseFromSerialisableInfo( self, version, serialisable_info, raise_error_on_future_version = False ):
        
        if version > self.SERIALISABLE_VERSION:
            
            if raise_error_on_future_version:
                
                message = 'Unfortunately, an object of type {} could not be loaded because it was created in a client that uses an updated version of that object! This client supports versions up to {}, but the object was version {}.'.format( self.SERIALISABLE_NAME, self.SERIALISABLE_VERSION, version )
                message += os.linesep * 2
                message += 'Please update your client to import this object.'
                
                raise HydrusExceptions.SerialisationException( message )
                
            else:
                
                message = 'An object of type {} was created in a client that uses an updated version of that object! This client supports versions up to {}, but the object was version {}. For now, the client will try to continue work, but things may break. If you know why this has occured, please correct it. If you do not, please let hydrus dev know.'.format( self.SERIALISABLE_NAME, self.SERIALISABLE_VERSION, version )
                
                HydrusData.ShowText( message )
                
            
        
        while version < self.SERIALISABLE_VERSION:
            
            ( version, serialisable_info ) = self._UpdateSerialisableInfo( version, serialisable_info )
            
        
        self._InitialiseFromSerialisableInfo( serialisable_info )
        
    
class SerialisableBaseNamed( SerialisableBase ):
    
    SERIALISABLE_TYPE = SERIALISABLE_TYPE_BASE_NAMED
    SERIALISABLE_NAME = 'Named Base Serialisable Object'
    
    def __init__( self, name ):
        
        SerialisableBase.__init__( self )
        
        self._name = name
        
    
    def GetSerialisableTuple( self ):
        
        return ( self.SERIALISABLE_TYPE, self._name, self.SERIALISABLE_VERSION, self._GetSerialisableInfo() )
        
    
    def GetName( self ): return self._name
    
    def SetName( self, name ): self._name = name
    
    def SetNonDupeName( self, disallowed_names ):
        
        self._name = HydrusData.GetNonDupeName( self._name, disallowed_names )
        
    
class SerialisableDictionary( SerialisableBase, dict ):
    
    SERIALISABLE_TYPE = SERIALISABLE_TYPE_DICTIONARY
    SERIALISABLE_NAME = 'Serialisable Dictionary'
    SERIALISABLE_VERSION = 1
    
    def __init__( self, *args, **kwargs ):
        
        dict.__init__( self, *args, **kwargs )
        SerialisableBase.__init__( self )
        
    
    def _GetSerialisableInfo( self ):
        
        simple_key_simple_value_pairs = []
        simple_key_serialisable_value_pairs = []
        serialisable_key_simple_value_pairs = []
        serialisable_key_serialisable_value_pairs = []
        
        for ( key, value ) in self.items():
            
            if isinstance( key, SerialisableBase ):
                
                serialisable_key = key.GetSerialisableTuple()
                
                if isinstance( value, SerialisableBase ):
                    
                    serialisable_value = value.GetSerialisableTuple()
                    
                    serialisable_key_serialisable_value_pairs.append( ( serialisable_key, serialisable_value ) )
                    
                else:
                    
                    serialisable_value = value
                    
                    serialisable_key_simple_value_pairs.append( ( serialisable_key, serialisable_value ) )
                    
                
            else:
                
                serialisable_key = key
                
                if isinstance( value, SerialisableBase ):
                    
                    serialisable_value = value.GetSerialisableTuple()
                    
                    simple_key_serialisable_value_pairs.append( ( serialisable_key, serialisable_value ) )
                    
                else:
                    
                    serialisable_value = value
                    
                    simple_key_simple_value_pairs.append( ( serialisable_key, serialisable_value ) )
                    
                
            
        
        return ( simple_key_simple_value_pairs, simple_key_serialisable_value_pairs, serialisable_key_simple_value_pairs, serialisable_key_serialisable_value_pairs )
        
    
    def _InitialiseFromSerialisableInfo( self, serialisable_info ):
        
        have_shown_load_error = False
        
        ( simple_key_simple_value_pairs, simple_key_serialisable_value_pairs, serialisable_key_simple_value_pairs, serialisable_key_serialisable_value_pairs ) = serialisable_info
        
        for ( key, value ) in simple_key_simple_value_pairs:
            
            self[ key ] = value
            
        
        for ( key, serialisable_value ) in simple_key_serialisable_value_pairs:
            
            try:
                
                value = CreateFromSerialisableTuple( serialisable_value )
                
            except HydrusExceptions.SerialisationException as e:
                
                if not have_shown_load_error:
                    
                    HydrusData.ShowText( 'An object in a dictionary could not load. It has been discarded from the dictionary. More may also have failed to load, but to stop error spam, they will go silently. Your client may be running on code versions behind its database. Depending on the severity of this error, you may need to rollback to a previous backup. If you have no backup, you may want to kill your hydrus process now to stop the cleansed dictionary being saved back to the db.' )
                    HydrusData.ShowException( e )
                    
                    have_shown_load_error = True
                    
                
                continue
                
            
            self[ key ] = value
            
        
        for ( serialisable_key, value ) in serialisable_key_simple_value_pairs:
            
            try:
                
                key = CreateFromSerialisableTuple( serialisable_key )
                
            except HydrusExceptions.SerialisationException as e:
                
                if not have_shown_load_error:
                    
                    HydrusData.ShowText( 'An object in a dictionary could not load. It has been discarded from the dictionary. More may also have failed to load, but to stop error spam, they will go silently. Your client may be running on code versions behind its database. Depending on the severity of this error, you may need to rollback to a previous backup. If you have no backup, you may want to kill your hydrus process now to stop the cleansed dictionary being saved back to the db.' )
                    HydrusData.ShowException( e )
                    
                    have_shown_load_error = True
                    
                
                continue
                
            
            self[ key ] = value
            
        
        for ( serialisable_key, serialisable_value ) in serialisable_key_serialisable_value_pairs:
            
            try:
                
                key = CreateFromSerialisableTuple( serialisable_key )
                
                value = CreateFromSerialisableTuple( serialisable_value )
                
            except HydrusExceptions.SerialisationException as e:
                
                if not have_shown_load_error:
                    
                    HydrusData.ShowText( 'An object in a dictionary could not load. It has been discarded from the dictionary. More may also have failed to load, but to stop error spam, they will go silently. Your client may be running on code versions behind its database. Depending on the severity of this error, you may need to rollback to a previous backup. If you have no backup, you may want to kill your hydrus process now to stop the cleansed dictionary being saved back to the db.' )
                    HydrusData.ShowException( e )
                    
                    have_shown_load_error = True
                    
                
                continue
                
            
            self[ key ] = value
            
        
    
SERIALISABLE_TYPES_TO_OBJECT_TYPES[ SERIALISABLE_TYPE_DICTIONARY ] = SerialisableDictionary

class SerialisableBytesDictionary( SerialisableBase, dict ):
    
    SERIALISABLE_TYPE = SERIALISABLE_TYPE_BYTES_DICT
    SERIALISABLE_NAME = 'Serialisable Dictionary With Bytestring Key/Value Support'
    SERIALISABLE_VERSION = 1
    
    def __init__( self, *args, **kwargs ):
        
        dict.__init__( self, *args, **kwargs )
        SerialisableBase.__init__( self )
        
    
    def _GetSerialisableInfo( self ):
        
        pairs = []
        
        for ( key, value ) in list(self.items()):
            
            if isinstance( key, int ):
                
                encoded_key = key
                
            else:
                
                encoded_key = key.hex()
                
            
            if isinstance( value, ( list, tuple, set ) ):
                
                encoded_value = [ item.hex() for item in value ]
                
            elif value is None:
                
                encoded_value = value
                
            else:
                
                encoded_value = value.hex()
                
            
            pairs.append( ( encoded_key, encoded_value ) )
            
        
        return pairs
        
    
    def _InitialiseFromSerialisableInfo( self, serialisable_info ):
        
        for ( encoded_key, encoded_value ) in serialisable_info:
            
            if isinstance( encoded_key, int ):
                
                key = encoded_key
                
            else:
                
                key = bytes.fromhex( encoded_key )
                
            
            if isinstance( encoded_value, ( list, tuple, set ) ):
                
                value = [ bytes.fromhex( encoded_item ) for encoded_item in encoded_value ]
                
            elif encoded_value is None:
                
                value = encoded_value
                
            else:
                
                value = bytes.fromhex( encoded_value )
                
            
            self[ key ] = value
            
        
    
SERIALISABLE_TYPES_TO_OBJECT_TYPES[ SERIALISABLE_TYPE_BYTES_DICT ] = SerialisableBytesDictionary

class SerialisableList( SerialisableBase, list ):
    
    SERIALISABLE_TYPE = SERIALISABLE_TYPE_LIST
    SERIALISABLE_NAME = 'Serialisable List'
    SERIALISABLE_VERSION = 1
    
    def __init__( self, *args, **kwargs ):
        
        list.__init__( self, *args, **kwargs )
        SerialisableBase.__init__( self )
        
    
    def _GetSerialisableInfo( self ):
        
        return [ obj.GetSerialisableTuple() for obj in self ]
        
    
    def _InitialiseFromSerialisableInfo( self, serialisable_info ):
        
        have_shown_load_error = False
        
        for obj_tuple in serialisable_info:
            
            try:
                
                obj = CreateFromSerialisableTuple( obj_tuple )
                
            except HydrusExceptions.SerialisationException as e:
                
                if not have_shown_load_error:
                    
                    HydrusData.ShowText( 'An object in a list could not load. It has been discarded from the list. More may also have failed to load, but to stop error spam, they will go silently. Your client may be running on code versions behind its database. Depending on the severity of this error, you may need to rollback to a previous backup. If you have no backup, you may want to kill your hydrus process now to stop the cleansed list being saved back to the db.' )
                    HydrusData.ShowException( e )
                    
                    have_shown_load_error = True
                    
                
                continue
                
            
            self.append( obj )
            
        
    
SERIALISABLE_TYPES_TO_OBJECT_TYPES[ SERIALISABLE_TYPE_LIST ] = SerialisableList