import itertools
import os
import re
import threading
import time
import urllib.parse

from hydrus.core import HydrusGlobals as HG
from hydrus.core import HydrusData
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusSerialisable

from hydrus.client import ClientConstants as CC
from hydrus.client import ClientParsing
from hydrus.client import ClientStrings
from hydrus.client import ClientThreading
from hydrus.client.networking import ClientNetworkingContexts
from hydrus.client.networking import ClientNetworkingDomain
from hydrus.client.networking import ClientNetworkingJobs

VALIDITY_VALID = 0
VALIDITY_UNTESTED = 1
VALIDITY_INVALID = 2

validity_str_lookup = {}

validity_str_lookup[ VALIDITY_VALID ] = 'valid'
validity_str_lookup[ VALIDITY_UNTESTED ] = 'untested'
validity_str_lookup[ VALIDITY_INVALID ] = 'invalid'

LOGIN_ACCESS_TYPE_EVERYTHING = 0
LOGIN_ACCESS_TYPE_NSFW = 1
LOGIN_ACCESS_TYPE_SPECIAL = 2
LOGIN_ACCESS_TYPE_USER_PREFS_ONLY = 3

login_access_type_str_lookup = {}

login_access_type_str_lookup[ LOGIN_ACCESS_TYPE_EVERYTHING ] = 'Everything'
login_access_type_str_lookup[ LOGIN_ACCESS_TYPE_NSFW ] = 'NSFW'
login_access_type_str_lookup[ LOGIN_ACCESS_TYPE_SPECIAL ] = 'Special'
login_access_type_str_lookup[ LOGIN_ACCESS_TYPE_USER_PREFS_ONLY ] = 'User prefs'

login_access_type_default_description_lookup = {}

login_access_type_default_description_lookup[ LOGIN_ACCESS_TYPE_EVERYTHING ] = 'Login required to access any content.'
login_access_type_default_description_lookup[ LOGIN_ACCESS_TYPE_NSFW ] = 'Login required to access NSFW content.'
login_access_type_default_description_lookup[ LOGIN_ACCESS_TYPE_SPECIAL ] = 'Login required to access special content.'
login_access_type_default_description_lookup[ LOGIN_ACCESS_TYPE_USER_PREFS_ONLY ] = 'Login only required to access user preferences.'

PIXIV_NETWORK_CONTEXT = ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'pixiv.net' )
HENTAI_FOUNDRY_NETWORK_CONTEXT = ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'hentai-foundry.com' )

class NetworkLoginManager( HydrusSerialisable.SerialisableBase ):
    
    SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER
    SERIALISABLE_NAME = 'Login Manager'
    SERIALISABLE_VERSION = 1
    
    SESSION_TIMEOUT = 60 * 45
    
    def __init__( self ):
        
        HydrusSerialisable.SerialisableBase.__init__( self )
        
        # needs _dirty and setdirty and be on that serialisation check and so on
        
        self.engine = None
        
        self._dirty = False
        
        self._lock = threading.Lock()
        
        self._login_scripts = HydrusSerialisable.SerialisableList()
        self._domains_to_login_info = {}
        
        self._login_script_keys_to_login_scripts = {}
        self._login_script_names_to_login_scripts = {}
        
        self._hydrus_login_script = LoginScriptHydrus()
        
        self._error_names = set()
        
    
    def _GetBestLoginScript( self, login_domain ):
        
        self._login_scripts.sort( key = lambda ls: len( ls.GetCredentialDefinitions() ) )
        
        for login_script in self._login_scripts:
            
            if login_domain in login_script.GetExampleDomains():
                
                return login_script
                
            
        
        return None
        
    
    def _GetLoginDomainStatus( self, network_context ):
        
        login_domain = None
        login_expected = False
        login_possible = True
        login_error_text = ''
        
        domain = network_context.context_data
        
        potential_login_domains = ClientNetworkingDomain.ConvertDomainIntoAllApplicableDomains( domain, discard_www = False )
        
        for potential_login_domain in potential_login_domains:
            
            if potential_login_domain in self._domains_to_login_info:
                
                login_domain = potential_login_domain
                
                ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]
                
                if active or login_access_type == LOGIN_ACCESS_TYPE_EVERYTHING:
                    
                    login_expected = True
                    
                
                if not active:
                    
                    login_possible = False
                    login_error_text = 'Not active - ' + login_access_text
                    
                elif validity == VALIDITY_INVALID:
                    
                    login_possible = False
                    login_error_text = validity_error_text
                    
                elif not HydrusData.TimeHasPassed( no_work_until ):
                    
                    login_possible = False
                    login_error_text = no_work_until_reason
                    
                
                break
                
            
        
        return ( login_domain, login_expected, login_possible, login_error_text )
        
    
    def _GetLoginScriptAndCredentials( self, login_domain ):
        
        if login_domain in self._domains_to_login_info:
            
            ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]
            
            ( login_script_key, login_script_name ) = login_script_key_and_name
            
            if login_script_key in self._login_script_keys_to_login_scripts:
                
                login_script = self._login_script_keys_to_login_scripts[ login_script_key ]
                
            elif login_script_name in self._login_script_names_to_login_scripts:
                
                login_script = self._login_script_names_to_login_scripts[ login_script_name ]
                
                login_script_key_and_name = login_script.GetLoginScriptKeyAndName()
                
                self._SetDirty()
                
                self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )
                
            else:
                
                validity = VALIDITY_INVALID
                validity_error_text = 'Could not find the login script for "' + login_domain + '"!'
                
                self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )
                
                self._SetDirty()
                
                raise HydrusExceptions.ValidationException( validity_error_text )
                
            
            try:
                
                login_script.CheckCanLogin( credentials )
                
            except HydrusExceptions.ValidationException as e:
                
                validity = VALIDITY_INVALID
                validity_error_text = str( e )
                
                self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )
                
                self._SetDirty()
                
                raise
                
            
            if validity == VALIDITY_UNTESTED and validity_error_text != '':
                
                # cleaning up the 'restart dialog to test validity in cases where it is valid
                
                validity_error_text = ''
                
                self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )
                
            
            return ( login_script, credentials )
            
        else:
            
            raise HydrusExceptions.ValidationException( 'Could not find any login entry for "' + login_domain + '"!' )
            
        
    
    def _GetSerialisableInfo( self ):
        
        serialisable_login_scripts = self._login_scripts.GetSerialisableTuple()
        
        serialisable_domains_to_login_info = {}
        
        for ( login_domain, ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) ) in list(self._domains_to_login_info.items()):
            
            ( login_script_key, login_script_name ) = login_script_key_and_name
            
            serialisable_login_script_key_and_name = ( login_script_key.hex(), login_script_name )
            
            serialisable_domains_to_login_info[ login_domain ] = ( serialisable_login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )
            
        
        return ( serialisable_login_scripts, serialisable_domains_to_login_info )
        
    
    def _InitialiseFromSerialisableInfo( self, serialisable_info ):
        
        ( serialisable_login_scripts, serialisable_domains_to_login_info ) = serialisable_info
        
        self._login_scripts = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_login_scripts )
        
        self._domains_to_login_info = {}
        
        for ( login_domain, ( serialisable_login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) ) in list(serialisable_domains_to_login_info.items()):
            
            ( serialisable_login_script_key, login_script_name ) = serialisable_login_script_key_and_name
            
            login_script_key_and_name = ( bytes.fromhex( serialisable_login_script_key ), login_script_name )
            
            self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )
            
        
    
    def _RecalcCache( self ):
        
        self._login_script_keys_to_login_scripts = { login_script.GetLoginScriptKey() : login_script for login_script in self._login_scripts }
        self._login_script_names_to_login_scripts = { login_script.GetName() : login_script for login_script in self._login_scripts }
        
        self._RevalidateCache()
        
    
    def _RevalidateCache( self ):
        
        for login_domain in list(self._domains_to_login_info.keys()):
            
            try:
                
                self._GetLoginScriptAndCredentials( login_domain )
                
            except HydrusExceptions.ValidationException:
                
                pass
                
            
        
    
    def _SetDirty( self ):
        
        self._dirty = True
        
    
    def AlreadyHaveExactlyThisLoginScript( self, new_login_script ):
        
        with self._lock:
            
            # absent irrelevant variables, do we have the exact same object already in?
            
            login_script_key_and_name = new_login_script.GetLoginScriptKeyAndName()
            
            dupe_login_scripts = [ login_script.Duplicate() for login_script in self._login_scripts ]
            
            for dupe_login_script in dupe_login_scripts:
                
                dupe_login_script.SetLoginScriptKeyAndName( login_script_key_and_name )
                
                if dupe_login_script.DumpToString() == new_login_script.DumpToString():
                    
                    return True
                    
                
            
        
        return False
        
    
    def AutoAddLoginScripts( self, login_scripts ):
        
        with self._lock:
            
            next_login_scripts = list( self._login_scripts )
            
            for login_script in login_scripts:
                
                login_script.RegenerateLoginScriptKey()
                
            
            next_login_scripts.extend( login_scripts )
            
        
        self.SetLoginScripts( next_login_scripts )
        
    
    def CheckCanLogin( self, network_context ):
        
        with self._lock:
            
            if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
                
                ( login_domain, login_expected, login_possible, login_error_text ) = self._GetLoginDomainStatus( network_context )
                
                if login_domain is None or not login_expected:
                    
                    raise HydrusExceptions.ValidationException( 'The domain ' + login_domain + ' has no active login script--has it just been turned off?' )
                    
                elif not login_possible:
                    
                    raise HydrusExceptions.ValidationException( 'The domain ' + login_domain + ' cannot log in: ' + login_error_text )
                    
                
            elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
                
                service_key = network_context.context_data
                
                services_manager = self.engine.controller.services_manager
                
                if not services_manager.ServiceExists( service_key ):
                    
                    raise HydrusExceptions.ValidationException( 'Service does not exist!' )
                    
                
                service = services_manager.GetService( service_key )
                
                try:
                    
                    service.CheckFunctional( including_bandwidth = False, including_account = False )
                    
                except Exception as e:
                    
                    message = 'Service has had a recent error or is otherwise not functional! You might like to try refreshing its account in \'review services\'. Specific error was: {}'.format( e )
                    
                    raise HydrusExceptions.ValidationException( message )
                    
                
            
        
    
    def DelayLoginScript( self, login_domain, login_script_key, reason ):
        
        with self._lock:
            
            if login_domain not in self._domains_to_login_info:
                
                return
                
            
            ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]
            
            if login_script_key != login_script_key_and_name[0]:
                
                return
                
            
            no_work_until = HydrusData.GetNow() + 3600 * 4
            no_work_until_reason = reason
            
            self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )
            
            self._SetDirty()
            
        
    
    def DeleteLoginDomain( self, login_domain ):
        
        with self._lock:
            
            if login_domain in self._domains_to_login_info:
                
                del self._domains_to_login_info[ login_domain ]
                
                self._RecalcCache()
                
                self._SetDirty()
                
            
        
    
    def DeleteLoginScripts( self, login_script_names ):
        
        with self._lock:
            
            login_scripts = [ login_script for login_script in self._login_scripts if login_script.GetName() not in login_script_names ]
            
        
        self.SetLoginScripts( login_scripts )
        
    
    def DomainHasALoginScript( self, login_domain ):
        
        with self._lock:
            
            if login_domain in self._domains_to_login_info:
                
                ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]
                
                ( login_script_key, login_script_name ) = login_script_key_and_name
                
                if login_script_key in self._login_script_keys_to_login_scripts or login_script_name in self._login_script_names_to_login_scripts:
                    
                    return True
                    
                
            
            return False
            
        
    
    def GenerateLoginProcess( self, network_context ):
        
        with self._lock:
            
            if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
                
                ( login_domain, login_expected, login_possible, login_error_text ) = self._GetLoginDomainStatus( network_context )
                
                if login_domain is None or not login_expected:
                    
                    raise HydrusExceptions.ValidationException( 'The domain ' + login_domain + ' has no active login script--has it just been turned off?' )
                    
                elif not login_possible:
                    
                    raise HydrusExceptions.ValidationException( 'The domain ' + login_domain + ' cannot log in: ' + login_error_text )
                    
                else:
                    
                    login_network_context = ClientNetworkingContexts.NetworkContext( context_type = CC.NETWORK_CONTEXT_DOMAIN, context_data = login_domain )
                    
                    ( login_script, credentials ) = self._GetLoginScriptAndCredentials( login_domain )
                    
                    login_process = LoginProcessDomain( self.engine, login_network_context, login_script, credentials )
                    
                    return login_process
                    
                
            elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
                
                login_process = LoginProcessHydrus( self.engine, network_context, self._hydrus_login_script )
                
                return login_process
                
            
        
    
    def GenerateLoginProcessForDomain( self, login_domain ):
        
        network_context = ClientNetworkingContexts.NetworkContext.STATICGenerateForDomain( login_domain )
        
        return self.GenerateLoginProcess( network_context )
        
    
    def GetDomainsToLoginInfo( self ):
        
        with self._lock:
            
            self._RevalidateCache()
            
            return dict( self._domains_to_login_info )
            
        
    
    def GetLoginScripts( self ):
        
        with self._lock:
            
            return list( self._login_scripts )
            
        
    
    def Initialise( self ):
        
        self._RecalcCache()
        
    
    def InvalidateLoginScript( self, login_domain, login_script_key, reason ):
        
        with self._lock:
            
            if login_domain not in self._domains_to_login_info:
                
                return
                
            
            ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]
            
            if login_script_key != login_script_key_and_name[0]:
                
                return
                
            
            validity = VALIDITY_INVALID
            validity_error_text = reason
            
            self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )
            
            HydrusData.ShowText( 'The login for "' + login_domain + '" failed! It will not be reattempted until the problem is fixed. The failure reason was:' + os.linesep * 2 + validity_error_text )
            
            self._SetDirty()
            
        
    
    def IsDirty( self ):
        
        with self._lock:
            
            return self._dirty
            
        
    
    def NeedsLogin( self, network_context ):
        
        with self._lock:
            
            if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
                
                ( login_domain, login_expected, login_possible, login_error_text ) = self._GetLoginDomainStatus( network_context )
                
                if login_domain is None or not login_expected:
                    
                    return False # no login required, no problem
                    
                else:
                    
                    try:
                        
                        ( login_script, credentials ) = self._GetLoginScriptAndCredentials( login_domain )
                        
                    except HydrusExceptions.ValidationException:
                        
                        # couldn't find the script or something. assume we need a login to move errors forward to checkcanlogin trigger phase
                        
                        return True
                        
                    
                    login_network_context = ClientNetworkingContexts.NetworkContext( context_type = CC.NETWORK_CONTEXT_DOMAIN, context_data = login_domain )
                    
                    return not login_script.IsLoggedIn( self.engine, login_network_context )
                    
                
            elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
                
                return not self._hydrus_login_script.IsLoggedIn( self.engine, network_context )
                
            
        
    
    def OverwriteDefaultLoginScripts( self, login_script_names ):
        
        with self._lock:
            
            from hydrus.client import ClientDefaults
            
            default_login_scripts = ClientDefaults.GetDefaultLoginScripts()
            
            for login_script in default_login_scripts:
                
                login_script.RegenerateLoginScriptKey()
                
            
            existing_login_scripts = list( self._login_scripts )
            
            new_login_scripts = [ login_script for login_script in existing_login_scripts if login_script.GetName() not in login_script_names ]
            new_login_scripts.extend( [ login_script for login_script in default_login_scripts if login_script.GetName() in login_script_names ] )
            
        
        self.SetLoginScripts( new_login_scripts )
        
    
    def SetClean( self ):
        
        with self._lock:
            
            self._dirty = False
            
        
    
    def SetCredentialsAndActivate( self, login_domain, new_credentials ):
        
        with self._lock:
            
            if login_domain not in self._domains_to_login_info:
                
                return
                
            
            ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]
            
            credentials = new_credentials
            active = True
            
            validity = VALIDITY_UNTESTED
            validity_error_text = ''
            
            self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )
            
            self._SetDirty()
            
        
    
    def SetDomainsToLoginInfo( self, domains_to_login_info ):
        
        with self._lock:
            
            self._domains_to_login_info = dict( domains_to_login_info )
            
            self._RecalcCache()
            
            self._SetDirty()
            
        
    
    def SetLoginScripts( self, login_scripts ):
        
        with self._lock:
            
            self._login_scripts = HydrusSerialisable.SerialisableList( login_scripts )
            
            # start with simple stuff first
            self._login_scripts.sort( key = lambda ls: len( ls.GetCredentialDefinitions() ) )
            
            for login_script in self._login_scripts:
                
                login_script_key_and_name = login_script.GetLoginScriptKeyAndName()
                
                example_domains_info = login_script.GetExampleDomainsInfo()
                
                for ( login_domain, login_access_type, login_access_text ) in example_domains_info:
                    
                    if '.' in login_domain:
                        
                        # looks good, so let's see if we can update/add some info
                        
                        if login_domain in self._domains_to_login_info:
                            
                            ( old_login_script_key_and_name, credentials, old_login_access_type, old_login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]
                            
                            if old_login_script_key_and_name[1] == login_script_key_and_name[1]:
                                
                                self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )
                                
                            
                        else:
                            
                            credentials = {}
                            
                            # if there is nothing to enter, turn it on by default, like HF click-through
                            active = len( login_script.GetCredentialDefinitions() ) == 0
                            
                            validity = VALIDITY_UNTESTED
                            validity_error_text = ''
                            
                            no_work_until = 0
                            no_work_until_reason = ''
                            
                            self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )
                            
                        
                    
                
            
            self._RecalcCache()
            
            self._SetDirty()
            
        
    
    def ValidateLoginScript( self, login_domain, login_script_key ):
        
        with self._lock:
            
            if login_domain not in self._domains_to_login_info:
                
                return
                
            
            ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]
            
            if login_script_key != login_script_key_and_name[0]:
                
                return
                
            
            validity = VALIDITY_VALID
            validity_error_text = ''
            
            self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )
            
            self._SetDirty()
            
        
    
    def TryToLinkMissingLoginScripts( self, login_domains ):
        
        with self._lock:
            
            for login_domain in login_domains:
                
                try:
                    
                    ( existing_login_script, existing_credentials ) = self._GetLoginScriptAndCredentials( login_domain )
                    
                    continue # already seems to have a good login script, so nothing to fix
                    
                except HydrusExceptions.ValidationException:
                    
                    pass
                    
                
                ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]
                
                login_script = self._GetBestLoginScript( login_domain )
                
                if login_script is None:
                    
                    continue
                    
                
                validity = VALIDITY_UNTESTED
                validity_error_text = ''
                
                login_script_key_and_name = login_script.GetLoginScriptKeyAndName()
                
                self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )
                
            
            self._SetDirty()
            
        
    
    def LoginTumblrGDPR( self ):
        
        # t-thanks, EU
        # this is cribbed from poking around here https://github.com/johanneszab/TumblThree/commit/3563d6cebf1a467151d6b8d6eee9806ddd6e6364
        
        network_job = ClientNetworkingJobs.NetworkJob( 'GET', 'https://www.tumblr.com/' )
        
        network_job.SetForLogin( True )
        
        self.engine.AddJob( network_job )
        
        network_job.WaitUntilDone()
        
        html = network_job.GetContentText()
        
        formula = ClientParsing.ParseFormulaHTML( tag_rules = [ ClientParsing.ParseRuleHTML( rule_type = ClientParsing.HTML_RULE_TYPE_DESCENDING, tag_name = 'meta', tag_attributes = { 'id' : 'tumblr_form_key' } ) ], content_to_fetch = ClientParsing.HTML_CONTENT_ATTRIBUTE, attribute_to_fetch = "content" )
        
        results = formula.Parse( {}, html )
        
        if len( results ) != 1:
            
            raise HydrusExceptions.ParseException( 'Could not figure out the tumblr form key for the GDPR click-through.' )
            
        
        tumblr_form_key = results[0]
        
        #
        
        body = '{\"eu_resident\":true,\"gdpr_is_acceptable_age\":true,\"gdpr_consent_core\":true,\"gdpr_consent_first_party_ads\":true,\"gdpr_consent_third_party_ads\":true,\"gdpr_consent_search_history\":true,\"redirect_to\":\"\"}'
        referral_url = 'https://www.tumblr.com/privacy/consent?redirect='
        
        network_job = ClientNetworkingJobs.NetworkJob( 'POST', 'https://www.tumblr.com/svc/privacy/consent', body = body, referral_url = referral_url )
        
        network_job.SetForLogin( True )
        
        network_job.AddAdditionalHeader( 'Accept', 'application/json, text/javascript, */*; q=0.01')
        network_job.AddAdditionalHeader( 'Content-Type', 'application/json' )
        network_job.AddAdditionalHeader( 'X-Requested-With', 'XMLHttpRequest' )
        network_job.AddAdditionalHeader( 'X-tumblr-form-key', tumblr_form_key )
        
        self.engine.AddJob( network_job )
        
        network_job.WaitUntilDone()
        
        # test cookies here or something
        
        HydrusData.ShowText( 'Looks like tumblr GDPR click-through worked! You should be good for a year, at which point we should have an automatic solution for this!' )
        
    
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER ] = NetworkLoginManager

CREDENTIAL_TYPE_TEXT = 0
CREDENTIAL_TYPE_PASS = 1

credential_type_str_lookup = {}

credential_type_str_lookup[ CREDENTIAL_TYPE_TEXT ] = 'normal'
credential_type_str_lookup[ CREDENTIAL_TYPE_PASS ] = 'hidden (password)'

class LoginCredentialDefinition( HydrusSerialisable.SerialisableBaseNamed ):
    
    SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_LOGIN_CREDENTIAL_DEFINITION
    SERIALISABLE_NAME = 'Login Credential Definition'
    SERIALISABLE_VERSION = 1
    
    def __init__( self, name = 'username', credential_type = CREDENTIAL_TYPE_TEXT, string_match = None ):
        
        if string_match is None:
            
            string_match = ClientStrings.StringMatch()
            
        
        HydrusSerialisable.SerialisableBaseNamed.__init__( self, name )
        
        self._credential_type = credential_type
        self._string_match = string_match
        
    
    def _GetSerialisableInfo( self ):
        
        serialisable_string_match = self._string_match.GetSerialisableTuple()
        
        return ( self._credential_type, serialisable_string_match )
        
    
    def _InitialiseFromSerialisableInfo( self, serialisable_info ):
        
        ( self._credential_type, serialisable_string_match ) = serialisable_info
        
        self._string_match = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_string_match )
        
    
    def GetStringMatch( self ):
        
        return self._string_match
        
    
    def GetType( self ):
        
        return self._credential_type
        
    
    def SetStringMatch( self, string_match ):
        
        self._string_match = string_match
        
    
    def SetType( self, credential_type ):
        
        self._credential_type = credential_type
        
    
    def ShouldHide( self ):
        
        return self._credential_type == CREDENTIAL_TYPE_PASS
        
    
    def Test( self, text ):
        
        if self._string_match is not None:
            
            try:
                
                self._string_match.Test( text )
                
            except HydrusExceptions.StringMatchException as e:
                
                raise HydrusExceptions.ValidationException( 'Could not validate "' + self._name + '" credential: ' + str( e ) )
                
            
        
    
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_LOGIN_CREDENTIAL_DEFINITION ] = LoginCredentialDefinition

class LoginProcess( object ):
    
    def __init__( self, engine, network_context, login_script ):
        
        self.engine = engine
        self.network_context = network_context
        self.login_script = login_script
        
        self._done = False
        
    
    def _Start( self ):
        
        raise NotImplementedError()
        
    
    def IsDone( self ):
        
        return self._done
        
    
    def Start( self ):
        
        try:
            
            self._Start()
            
        finally:
            
            self._done = True
            
        
    
class LoginProcessDomain( LoginProcess ):
    
    def __init__( self, engine, network_context, login_script, credentials ):
        
        LoginProcess.__init__( self, engine, network_context, login_script )
        
        self.credentials = credentials
        
    
    def _Start( self ):
        
        login_domain = self.network_context.context_data
        
        job_key = ClientThreading.JobKey( cancellable = True )
        
        job_key.SetStatusTitle( 'Logging in ' + login_domain )
        
        HG.client_controller.pub( 'message', job_key )
        
        HydrusData.Print( 'Starting login for ' + login_domain )
        
        result = self.login_script.Start( self.engine, self.network_context, self.credentials, job_key = job_key )
        
        HydrusData.Print( 'Finished login for ' + self.network_context.context_data + '. Result was: ' + result )
        
        job_key.SetVariable( 'popup_text_1', result )
        
        job_key.Finish()
        
        job_key.Delete( 4 )
        
    
class LoginProcessHydrus( LoginProcess ):
    
    def _Start( self ):
        
        self.login_script.Start( self.engine, self.network_context )
        
    
class LoginScriptHydrus( object ):
    
    def _IsLoggedIn( self, engine, network_context ):
        
        session = engine.session_manager.GetSession( network_context )
        
        cookies = session.cookies
        
        cookies.clear_expired_cookies()
        
        return 'session_key' in cookies
        
    
    def IsLoggedIn( self, engine, network_context ):
        
        return self._IsLoggedIn( engine, network_context )
        
    
    def Start( self, engine, network_context ):
        
        service_key = network_context.context_data
        
        try:
            
            service = engine.controller.services_manager.GetService( service_key )
            
        except HydrusExceptions.DataMissing:
            
            return
            
        
        base_url = service.GetBaseURL()
        
        url = base_url + 'session_key'
        
        access_key = service.GetCredentials().GetAccessKey()
        
        network_job = ClientNetworkingJobs.NetworkJobHydrus( service_key, 'GET', url )
        
        network_job.SetForLogin( True )
        
        network_job.AddAdditionalHeader( 'Hydrus-Key', access_key.hex() )
        
        engine.AddJob( network_job )
        
        try:
            
            network_job.WaitUntilDone()
            
            if self._IsLoggedIn( engine, network_context ):
                
                HydrusData.Print( 'Successfully logged into ' + service.GetName() + '.' )
                
            elif service.IsFunctional():
                
                ( is_ok, status_string ) = service.GetStatusInfo()
                
                service.DelayFutureRequests( 'Could not log in for unknown reason. Current service status: {}'.format( status_string ) )
                
            
        except Exception as e:
            
            e_string = str( e )
            
            service.DelayFutureRequests( e_string )
            
        
    
class LoginScriptDomain( HydrusSerialisable.SerialisableBaseNamed ):
    
    SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_LOGIN_SCRIPT_DOMAIN
    SERIALISABLE_NAME = 'Login Script - Domain'
    SERIALISABLE_VERSION = 2
    
    def __init__( self, name = 'login script', login_script_key = None, required_cookies_info = None, credential_definitions = None, login_steps = None, example_domains_info = None ):
        
        if required_cookies_info is None:
            
            required_cookies_info = {}
            
        
        required_cookies_info = HydrusSerialisable.SerialisableDictionary( required_cookies_info )
        
        if credential_definitions is None:
            
            credential_definitions = []
            
        
        credential_definitions = HydrusSerialisable.SerialisableList( credential_definitions )
        
        if login_steps is None:
            
            login_steps = []
            
        
        login_steps = HydrusSerialisable.SerialisableList( login_steps )
        
        if example_domains_info is None:
            
            example_domains_info = []
            
        
        HydrusSerialisable.SerialisableBaseNamed.__init__( self, name )
        
        self._login_script_key = HydrusData.GenerateKey()
        self._required_cookies_info = required_cookies_info # string match : string match
        self._credential_definitions = credential_definitions
        self._login_steps = login_steps
        self._example_domains_info = example_domains_info # domain | login_access_type | login_access_text
        
    
    def _GetSerialisableInfo( self ):
        
        serialisable_login_script_key = self._login_script_key.hex()
        serialisable_required_cookies = self._required_cookies_info.GetSerialisableTuple()
        serialisable_credential_definitions = self._credential_definitions.GetSerialisableTuple()
        serialisable_login_steps = self._login_steps.GetSerialisableTuple()
        
        return ( serialisable_login_script_key, serialisable_required_cookies, serialisable_credential_definitions, serialisable_login_steps, self._example_domains_info )
        
    
    def _InitialiseFromSerialisableInfo( self, serialisable_info ):
        
        ( serialisable_login_script_key, serialisable_required_cookies, serialisable_credential_definitions, serialisable_login_steps, self._example_domains_info ) = serialisable_info
        
        self._login_script_key = bytes.fromhex( serialisable_login_script_key )
        self._required_cookies_info = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_required_cookies )
        self._credential_definitions = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_credential_definitions )
        self._login_steps = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_login_steps )
        
        # convert lists to tups for listctrl data hashing
        self._example_domains_info = [ tuple( l ) for l in self._example_domains_info ]
        
    
    def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
        
        if version == 1:
            
            ( serialisable_login_script_key, serialisable_required_cookies, serialisable_credential_definitions, serialisable_login_steps, example_domains_info ) = old_serialisable_info
            
            old_required_cookies_info = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_required_cookies )
            new_required_cookies_info = HydrusSerialisable.SerialisableDictionary()
            
            for ( name, value_string_match ) in list(old_required_cookies_info.items()):
                
                key_string_match = ClientStrings.StringMatch( match_type = ClientStrings.STRING_MATCH_FIXED, match_value = name, example_string = name )
                
                new_required_cookies_info[ key_string_match ] = value_string_match
                
            
            serialisable_required_cookies = new_required_cookies_info.GetSerialisableTuple()
            
            new_serialisable_info = ( serialisable_login_script_key, serialisable_required_cookies, serialisable_credential_definitions, serialisable_login_steps, example_domains_info )
            
            return ( 2, new_serialisable_info )
            
        
    
    def _IsLoggedIn( self, engine, network_context, validation_check = False ):
        
        session = engine.session_manager.GetSession( network_context )
        
        cookies = session.cookies
        
        cookies.clear_expired_cookies()
        
        search_domain = network_context.context_data
        
        for ( cookie_name_string_match, value_string_match ) in list(self._required_cookies_info.items()):
            
            try:
                
                cookie = ClientNetworkingDomain.GetCookie( cookies, search_domain, cookie_name_string_match )
                
            except HydrusExceptions.DataMissing as e:
                
                if validation_check:
                    
                    raise HydrusExceptions.ValidationException( 'Missing cookie "' + cookie_name_string_match.ToString() + '"!' )
                    
                
                return False
                
            
            cookie_text = cookie.value
            
            try:
                
                value_string_match.Test( cookie_text )
                
            except HydrusExceptions.StringMatchException as e:
                
                if validation_check:
                    
                    raise HydrusExceptions.ValidationException( 'Cookie "' + cookie_name_string_match.ToString() + '" failed: ' + str( e ) + '!' )
                    
                
                return False
                
            
        
        return True
        
    
    def CheckCanLogin( self, given_credentials ):
        
        self.CheckIsValid()
        
        given_cred_names = set( given_credentials.keys() )
        required_cred_names = { name for name in itertools.chain.from_iterable( ( step.GetRequiredCredentials() for step in self._login_steps ) ) }
        
        missing_givens = required_cred_names.difference( given_cred_names )
        
        if len( missing_givens ) > 0:
            
            missing_givens = sorted( missing_givens )
            
            raise HydrusExceptions.ValidationException( 'Missing required credentials: ' + ', '.join( missing_givens ) )
            
        
        #
        
        cred_names_to_definitions = { credential_definition.GetName() : credential_definition for credential_definition in self._credential_definitions }
        
        for ( pretty_name, text ) in given_credentials.items():
            
            if pretty_name not in cred_names_to_definitions:
                
                continue
                
            
            credential_definition = cred_names_to_definitions[ pretty_name ]
            
            credential_definition.Test( text )
            
        
    
    def CheckIsValid( self ):
        
        defined_cred_names = { credential_definition.GetName() for credential_definition in self._credential_definitions }
        required_cred_names = { name for name in itertools.chain.from_iterable( ( step.GetRequiredCredentials() for step in self._login_steps ) ) }
        
        missing_definitions = required_cred_names.difference( defined_cred_names )
        
        if len( missing_definitions ) > 0:
            
            missing_definitions = sorted( missing_definitions )
            
            raise HydrusExceptions.ValidationException( 'Missing required credential definitions: ' + ', '.join( missing_definitions ) )
            
        
        #
        
        temp_vars = set()
        
        for login_step in self._login_steps:
            
            ( required_vars, set_vars ) = login_step.GetRequiredAndSetTempVariables()
            
            missing_vars = required_vars.difference( temp_vars )
            
            if len( missing_vars ) > 0:
                
                missing_vars = sorted( missing_vars )
                
                raise HydrusExceptions.ValidationException( 'Missing temp variables for login step "' + login_step.GetName() + '": ' + ', '.join( missing_vars ) )
                
            
            temp_vars.update( set_vars )
            
        
    
    def GetCredentialDefinitions( self ):
        
        return self._credential_definitions
        
    
    def GetExampleDomains( self ):
        
        return [ domain for ( domain, login_access_type, login_access_text ) in self._example_domains_info ]
        
    
    def GetExampleDomainsInfo( self ):
        
        return self._example_domains_info
        
    
    def GetExampleDomainInfo( self, given_domain ):
        
        for ( domain, login_access_type, login_access_text ) in self._example_domains_info:
            
            if domain == given_domain:
                
                return ( login_access_type, login_access_text )
                
            
        
        raise HydrusExceptions.DataMissing( 'Could not find that domain!' )
        
    
    def GetRequiredCookiesInfo( self ):
        
        return self._required_cookies_info
        
    
    def GetLoginExpiry( self, engine, network_context ):
        
        session = engine.session_manager.GetSession( network_context )
        
        cookies = session.cookies
        
        cookies.clear_expired_cookies()
        
        search_domain = network_context.context_data
        
        session_cookies = False
        expiry_timestamps = []
        
        for cookie_name_string_match in list(self._required_cookies_info.keys()):
            
            try:
                
                cookie = ClientNetworkingDomain.GetCookie( cookies, search_domain, cookie_name_string_match )
                
            except HydrusExceptions.DataMissing as e:
                
                return None
                
            
            expiry = cookie.expires
            
            if expiry is None:
                
                session_cookies = True
                
            else:
                
                expiry_timestamps.append( expiry )
                
            
        
        if session_cookies or len( expiry_timestamps ) == 0:
            
            return None
            
        else:
            
            return min( expiry_timestamps )
            
        
    
    def GetLoginScriptKey( self ):
        
        return self._login_script_key
        
    
    def GetLoginScriptKeyAndName( self ):
        
        return ( self._login_script_key, self._name )
        
    
    def GetLoginSteps( self ):
        
        return self._login_steps
        
    
    def GetRequiredCredentials( self ):
        
        required_creds = []
        
        for login_step in self._login_steps:
            
            required_creds.extend( login_step.GetRequiredCredentials() ) # name with an order
            
        
        return required_creds
        
    
    def GetSafeSummary( self ):
        
        return 'Login Script "' + self._name + '" - ' + ', '.join( self.GetExampleDomains() )
        
    
    def IsLoggedIn( self, engine, network_context ):
        
        return self._IsLoggedIn( engine, network_context )
        
    
    def RegenerateLoginScriptKey( self ):
        
        self._login_script_key = HydrusData.GenerateKey()
        
    
    def SetLoginScriptKey( self, login_script_key ):
        
        self._login_script_key = login_script_key
        
    
    def SetLoginScriptKeyAndName( self, login_script_key_and_name ):
        
        ( login_script_key, name ) = login_script_key_and_name
        
        self._login_script_key = login_script_key
        self._name = name
        
    
    def Start( self, engine, network_context, given_credentials, network_job_presentation_context_factory = None, test_result_callable = None, job_key = None ):
        
        # don't mess with the domain--assume that we are given precisely the right domain
        
        login_domain = network_context.context_data
        
        temp_variables = {}
        
        last_url_used = None
        
        for login_step in self._login_steps:
            
            if job_key is not None:
                
                if job_key.IsCancelled():
                    
                    message = 'User cancelled the login process.'
                    
                    engine.login_manager.DelayLoginScript( login_domain, self._login_script_key, message )
                    
                    return message
                    
                
                job_key.SetVariable( 'popup_text_1', login_step.GetName() )
                
            
            try:
                
                last_url_used = login_step.Start( engine, login_domain, given_credentials, temp_variables, referral_url = last_url_used, network_job_presentation_context_factory = network_job_presentation_context_factory, test_result_callable = test_result_callable )
                
            except HydrusExceptions.ValidationException as e:
                
                if test_result_callable is not None:
                    
                    HydrusData.ShowException( e )
                    
                
                message = str( e )
                
                engine.login_manager.InvalidateLoginScript( login_domain, self._login_script_key, message )
                
                return 'Verification error: ' + message
                
            except HydrusExceptions.NetworkException as e:
                
                if test_result_callable is not None:
                    
                    HydrusData.ShowException( e )
                    
                
                message = str( e )
                
                engine.login_manager.DelayLoginScript( login_domain, self._login_script_key, message )
                
                return 'Network error: ' + message
                
            except Exception as e:
                
                if test_result_callable is not None:
                    
                    HydrusData.ShowException( e )
                    
                
                message = str( e )
                
                engine.login_manager.InvalidateLoginScript( login_domain, self._login_script_key, message )
                
                return 'Unusual error: ' + message
                
            
            time.sleep( 2 )
            
        
        try:
            
            self._IsLoggedIn( engine, network_context, validation_check = True )
            
        except Exception as e:
            
            if test_result_callable is not None:
                
                HydrusData.ShowException( e )
                
            
            message = str( e )
            
            engine.login_manager.InvalidateLoginScript( login_domain, self._login_script_key, message )
            
            return 'Final cookie check failed: ' + message
            
        
        engine.login_manager.ValidateLoginScript( login_domain, self._login_script_key )
        
        return 'Login OK!'
        
    
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_LOGIN_SCRIPT_DOMAIN ] = LoginScriptDomain

LOGIN_PARAMETER_TYPE_PARAMETER = 0
LOGIN_PARAMETER_TYPE_COOKIE = 1
LOGIN_PARAMETER_TYPE_HEADER = 2

class LoginStep( HydrusSerialisable.SerialisableBaseNamed ):
    
    SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_LOGIN_STEP
    SERIALISABLE_NAME = 'Login Step'
    SERIALISABLE_VERSION = 2
    
    def __init__( self, name = 'hit home page to establish session', scheme = 'https', method = 'GET', subdomain = None, path = '/' ):
        
        HydrusSerialisable.SerialisableBaseNamed.__init__( self, name )
        
        self._scheme = scheme
        self._method = method
        self._subdomain = subdomain
        self._path = path
        
        self._CleanseSubdomainAndPath()
        
        self._required_credentials = {} # pretty_name : arg name
        
        self._static_args = {} # arg name : string
        
        self._temp_args = {} # temp arg name : arg name
        
        self._required_cookies_info = HydrusSerialisable.SerialisableDictionary() # string match : string match
        
        self._content_parsers = HydrusSerialisable.SerialisableList()
        
    
    def _CleanseSubdomainAndPath( self ):
        
        if self._subdomain is not None:
            
            self._subdomain = re.sub( '[^a-z\\.]+', '', self._subdomain )
            
        
        if not self._path.startswith( '/' ):
            
            self._path = '/' + self._path
            
        
    
    def _GetSerialisableInfo( self ):
        
        serialisable_required_cookies = self._required_cookies_info.GetSerialisableTuple()
        serialisable_content_parsers = self._content_parsers.GetSerialisableTuple()
        
        return ( self._scheme, self._method, self._subdomain, self._path, self._required_credentials, self._static_args, self._temp_args, serialisable_required_cookies, serialisable_content_parsers )
        
    
    def _InitialiseFromSerialisableInfo( self, serialisable_info ):
        
        ( self._scheme, self._method, self._subdomain, self._path, self._required_credentials, self._static_args, self._temp_args, serialisable_required_cookies, serialisable_content_parsers ) = serialisable_info
        
        self._CleanseSubdomainAndPath()
        
        self._required_cookies_info = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_required_cookies )
        self._content_parsers = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_content_parsers )
        
    
    def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
        
        if version == 1:
            
            ( scheme, method, subdomain, path, required_credentials, static_args, temp_args, serialisable_required_cookies, serialisable_content_parsers ) = old_serialisable_info
            
            old_required_cookies_info = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_required_cookies )
            new_required_cookies_info = HydrusSerialisable.SerialisableDictionary()
            
            for ( name, value_string_match ) in list(old_required_cookies_info.items()):
                
                key_string_match = ClientStrings.StringMatch( match_type = ClientStrings.STRING_MATCH_FIXED, match_value = name, example_string = name )
                
                new_required_cookies_info[ key_string_match ] = value_string_match
                
            
            serialisable_required_cookies = new_required_cookies_info.GetSerialisableTuple()
            
            new_serialisable_info = ( scheme, method, subdomain, path, required_credentials, static_args, temp_args, serialisable_required_cookies, serialisable_content_parsers )
            
            return ( 2, new_serialisable_info )
            
        
    
    def GetRequiredCredentials( self ):
        
        return [ pretty_name for ( pretty_name, arg_name ) in list(self._required_credentials.items()) ]
        
    
    def GetRequiredAndSetTempVariables( self ):
        
        required_temp_variables = set( self._temp_args.keys() )
        
        set_temp_variables = { additional_info for [ ( name, content_type, additional_info ) ] in [ content_parser.GetParsableContent() for content_parser in self._content_parsers ] }
        
        return ( required_temp_variables, set_temp_variables )
        
    
    def SetComplicatedVariables( self, required_credentials, static_args, temp_args, required_cookies_info, content_parsers ):
        
        self._required_credentials = required_credentials
        
        self._static_args = static_args
        
        self._temp_args = temp_args
        
        self._required_cookies_info = HydrusSerialisable.SerialisableDictionary( required_cookies_info )
        
        self._content_parsers = HydrusSerialisable.SerialisableList( content_parsers )
        
    
    def Start( self, engine, domain, given_credentials, temp_variables, referral_url = None, network_job_presentation_context_factory = None, test_result_callable = None ):
        
        def session_to_cookie_strings( sess ):
            
            cookie_strings = set()
            
            for cookie in sess.cookies:
                
                s = cookie.name + ': ' + cookie.value + ' | ' + cookie.domain + ' | '
                
                expiry = cookie.expires
                
                if expiry is None:
                    
                    expiry = -1
                    pretty_expiry = 'session'
                    
                else:
                    
                    pretty_expiry = HydrusData.ConvertTimestampToPrettyExpires( expiry )
                    
                
                s += pretty_expiry
                
                cookie_strings.add( s )
                
            
            return cookie_strings
            
        
        url = 'Did not make a url.'
        test_result_body = None
        downloaded_text = 'Did not download data.'
        new_temp_variables = {}
        original_cookie_strings = session_to_cookie_strings( engine.session_manager.GetSessionForDomain( domain ) )
        test_script_result = 'Did not start.'
        
        try:
            
            domain_to_hit = domain
            
            if self._subdomain is not None:
                
                if domain.startswith( 'www.' ):
                    
                    domain = domain[4:]
                    
                
                domain_to_hit = self._subdomain + '.' + domain
                
            
            query_dict = {}
            
            query_dict.update( self._static_args )
            
            for ( pretty_name, arg_name ) in list(self._required_credentials.items()):
                
                query_dict[ arg_name ] = given_credentials[ pretty_name ]
                
            
            for ( temp_name, arg_name ) in list(self._temp_args.items()):
                
                if temp_name not in temp_variables:
                    
                    raise HydrusExceptions.ValidationException( 'The temporary variable \'' + temp_name + '\' was not found!' )
                    
                
                query_dict[ arg_name ] = temp_variables[ temp_name ]
                
            
            scheme = self._scheme
            netloc = domain_to_hit
            path = self._path
            params = ''
            fragment = ''
            
            single_value_parameters = []
            
            if self._method == 'GET':
                
                query = ClientNetworkingDomain.ConvertQueryDictToText( query_dict, single_value_parameters )
                body = None
                test_result_body = ''
                
            elif self._method == 'POST':
                
                query = ''
                body = query_dict
                test_result_body = ClientNetworkingDomain.ConvertQueryDictToText( query_dict, single_value_parameters )
                
            
            r = urllib.parse.ParseResult( scheme, netloc, path, params, query, fragment )
            
            url = r.geturl()
            
            network_job = ClientNetworkingJobs.NetworkJob( self._method, url, body = body, referral_url = referral_url )
            
            if self._method == 'POST' and referral_url is not None:
                
                p = ClientNetworkingDomain.ParseURL( url )
                
                r = urllib.parse.ParseResult( p.scheme, p.netloc, '', '', '', '' )
                
                origin = r.geturl() # https://accounts.pixiv.net
                
                network_job.AddAdditionalHeader( 'origin', origin ) # GET/POST forms are supposed to have this for CSRF. we'll try it just with POST for now
                
            
            network_job.SetForLogin( True )
            
            engine.AddJob( network_job )
            
            if network_job_presentation_context_factory is not None:
                
                with network_job_presentation_context_factory( network_job ) as njpc:
                    
                    network_job.WaitUntilDone()
                    
                
            else:
                
                network_job.WaitUntilDone()
                
            
            session = network_job.GetSession()
            
            cookies = session.cookies
            
            for ( cookie_name_string_match, string_match ) in list(self._required_cookies_info.items()):
                
                try:
                    
                    cookie = ClientNetworkingDomain.GetCookie( cookies, domain, cookie_name_string_match )
                    
                except HydrusExceptions.DataMissing as e:
                    
                    raise HydrusExceptions.ValidationException( 'Missing cookie "' + cookie_name_string_match.ToString() + '" on step "' + self._name + '"!' )
                    
                
                cookie_text = cookie.value
                
                try:
                    
                    string_match.Test( cookie_text )
                    
                except HydrusExceptions.StringMatchException as e:
                    
                    raise HydrusExceptions.ValidationException( 'Cookie "' + cookie_name_string_match.ToString() + '" failed on step "' + self._name + '": ' + str( e ) + '!' )
                    
                
            
            downloaded_text = network_job.GetContentText()
            
            parsing_context = {}
            
            parsing_context[ 'url' ] = url
            
            for content_parser in self._content_parsers:
                
                try:
                    
                    parse_results = content_parser.Parse( parsing_context, downloaded_text )
                    
                except HydrusExceptions.VetoException as e:
                    
                    raise HydrusExceptions.ValidationException( str( e ) )
                    
                
                result = ClientParsing.GetVariableFromParseResults( parse_results )
                
                if result is not None:
                    
                    ( temp_name, value ) = result
                    
                    new_temp_variables[ temp_name ] = value
                    
                
            
            temp_variables.update( new_temp_variables )
            
            test_script_result = 'OK!'
            
            return url
            
        except Exception as e:
            
            test_script_result = str( e )
            
            raise
            
        finally:
            
            if test_result_callable is not None:
                
                current_cookie_strings = session_to_cookie_strings( engine.session_manager.GetSessionForDomain( domain ) )
                
                new_cookie_strings = tuple( current_cookie_strings.difference( original_cookie_strings ) )
                
                new_temp_strings = tuple( ( key + ': ' + value for ( key, value ) in list(new_temp_variables.items()) ) )
                
                test_result = ( self._name, url, test_result_body, downloaded_text, new_temp_strings, new_cookie_strings, test_script_result )
                
                test_result_callable( test_result )
                
            
        
    
    def ToTuple( self ):
        
        return ( self._scheme, self._method, self._subdomain, self._path, self._required_credentials, self._static_args, self._temp_args, self._required_cookies_info, self._content_parsers )
        
    
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_LOGIN_STEP ] = LoginStep

