465 lines
15 KiB
Python
465 lines
15 KiB
Python
import pickle
|
|
import requests
|
|
import threading
|
|
import typing
|
|
|
|
from hydrus.core import HydrusData
|
|
from hydrus.core import HydrusSerialisable
|
|
from hydrus.core import HydrusGlobals as HG
|
|
from hydrus.core import HydrusTime
|
|
|
|
from hydrus.client import ClientConstants as CC
|
|
from hydrus.client import ClientGlobals as CG
|
|
from hydrus.client.networking import ClientNetworkingContexts
|
|
from hydrus.client.networking import ClientNetworkingFunctions
|
|
|
|
try:
|
|
|
|
import socket
|
|
import socks
|
|
|
|
SOCKS_PROXY_OK = True
|
|
|
|
except:
|
|
|
|
SOCKS_PROXY_OK = False
|
|
|
|
class NetworkSessionManagerSessionContainer( HydrusSerialisable.SerialisableBaseNamed ):
|
|
|
|
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER_SESSION_CONTAINER
|
|
SERIALISABLE_NAME = 'Session Manager Session Container'
|
|
SERIALISABLE_VERSION = 2
|
|
|
|
POOL_CONNECTION_TIMEOUT = 5 * 60
|
|
SESSION_TIMEOUT = 45 * 60
|
|
|
|
def __init__( self, name, network_context = None, session = None ):
|
|
|
|
if network_context is None:
|
|
|
|
network_context = ClientNetworkingContexts.GLOBAL_NETWORK_CONTEXT
|
|
|
|
|
|
HydrusSerialisable.SerialisableBaseNamed.__init__( self, name )
|
|
|
|
self.network_context = network_context
|
|
self.session = session
|
|
self.last_touched_time = HydrusTime.GetNow()
|
|
|
|
self.pool_is_cleared = True
|
|
self.printed_connection_pool_error = False
|
|
|
|
|
|
def _InitialiseEmptySession( self ):
|
|
|
|
self.session = requests.Session()
|
|
|
|
if self.network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
|
|
|
|
self.session.verify = False
|
|
|
|
|
|
|
|
def _GetSerialisableInfo( self ):
|
|
|
|
serialisable_network_context = self.network_context.GetSerialisableTuple()
|
|
|
|
self.session.cookies.clear_session_cookies()
|
|
|
|
pickled_cookies_hex = pickle.dumps( self.session.cookies ).hex()
|
|
|
|
return ( serialisable_network_context, pickled_cookies_hex )
|
|
|
|
|
|
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
|
|
|
|
( serialisable_network_context, pickled_cookies_hex ) = serialisable_info
|
|
|
|
self.network_context = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_network_context )
|
|
|
|
self._InitialiseEmptySession()
|
|
|
|
try:
|
|
|
|
cookies = pickle.loads( bytes.fromhex( pickled_cookies_hex ) )
|
|
|
|
self.session.cookies = cookies
|
|
|
|
except:
|
|
|
|
HydrusData.Print( "Could not load and set cookies for session {}".format( self.network_context ) )
|
|
|
|
|
|
self.session.cookies.clear_session_cookies()
|
|
|
|
|
|
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
|
|
|
|
if version == 1:
|
|
|
|
( serialisable_network_context, pickled_session_hex ) = old_serialisable_info
|
|
|
|
try:
|
|
|
|
session = pickle.loads( bytes.fromhex( pickled_session_hex ) )
|
|
|
|
except:
|
|
|
|
session = requests.Session()
|
|
|
|
|
|
pickled_cookies_hex = pickle.dumps( session.cookies ).hex()
|
|
|
|
new_serialisable_info = ( serialisable_network_context, pickled_cookies_hex )
|
|
|
|
return ( 2, new_serialisable_info )
|
|
|
|
|
|
|
|
def MaintainConnectionPool( self ):
|
|
|
|
if not self.pool_is_cleared and HydrusTime.TimeHasPassed( self.last_touched_time + self.POOL_CONNECTION_TIMEOUT ):
|
|
|
|
try:
|
|
|
|
my_session_adapters = list( self.session.adapters.values() )
|
|
|
|
for adapter in my_session_adapters:
|
|
|
|
poolmanager = getattr( adapter, 'poolmanager', None )
|
|
|
|
if poolmanager is not None:
|
|
|
|
poolmanager.clear()
|
|
|
|
|
|
|
|
self.pool_is_cleared = True
|
|
|
|
except Exception as e:
|
|
|
|
if not self.printed_connection_pool_error:
|
|
|
|
self.printed_connection_pool_error = True
|
|
|
|
HydrusData.Print( 'There was a problem clearing the connection pool, this message will not be printed again this boot:' )
|
|
HydrusData.PrintException( e, do_wait = False )
|
|
|
|
|
|
|
|
|
|
|
|
def PrepareForNewWork( self ):
|
|
|
|
self.last_touched_time = HydrusTime.GetNow()
|
|
self.pool_is_cleared = False
|
|
|
|
my_session_cookies = self.session.cookies
|
|
|
|
if HydrusTime.TimeHasPassed( self.last_touched_time + self.SESSION_TIMEOUT ):
|
|
|
|
my_session_cookies.clear_session_cookies()
|
|
|
|
|
|
my_session_cookies.clear_expired_cookies()
|
|
|
|
|
|
|
|
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER_SESSION_CONTAINER ] = NetworkSessionManagerSessionContainer
|
|
|
|
class NetworkSessionManager( HydrusSerialisable.SerialisableBase ):
|
|
|
|
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER
|
|
SERIALISABLE_NAME = 'Session Manager'
|
|
SERIALISABLE_VERSION = 1
|
|
|
|
def __init__( self ):
|
|
|
|
HydrusSerialisable.SerialisableBase.__init__( self )
|
|
|
|
self._dirty = False
|
|
self._dirty_session_container_names = set()
|
|
self._deletee_session_container_names = set()
|
|
|
|
self._lock = threading.Lock()
|
|
|
|
self._session_container_names = set()
|
|
|
|
self._session_container_names_to_session_containers = {}
|
|
self._network_contexts_to_session_containers = {}
|
|
|
|
self._proxies_dict = {}
|
|
|
|
self._ReinitialiseProxies()
|
|
|
|
CG.client_controller.sub( self, 'ReinitialiseProxies', 'notify_new_options' )
|
|
CG.client_controller.sub( self, 'MaintainConnectionPools', 'memory_maintenance_pulse' )
|
|
|
|
|
|
def _GetSerialisableInfo( self ):
|
|
|
|
return sorted( self._session_container_names )
|
|
|
|
|
|
def _GetSessionNetworkContext( self, network_context ):
|
|
|
|
# just in case one of these slips through somehow
|
|
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
|
|
|
|
second_level_domain = ClientNetworkingFunctions.ConvertDomainIntoSecondLevelDomain( network_context.context_data )
|
|
|
|
network_context = ClientNetworkingContexts.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, second_level_domain )
|
|
|
|
|
|
return network_context
|
|
|
|
|
|
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
|
|
|
|
self._session_container_names = set( serialisable_info )
|
|
|
|
|
|
def _InitialiseSessionContainer( self, network_context ):
|
|
|
|
session = requests.Session()
|
|
|
|
if network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
|
|
|
|
session.verify = False
|
|
|
|
|
|
session_container_name = HydrusData.GenerateKey().hex()
|
|
|
|
session_container = NetworkSessionManagerSessionContainer( session_container_name, network_context = network_context, session = session )
|
|
|
|
self._session_container_names_to_session_containers[ session_container_name ] = session_container
|
|
self._network_contexts_to_session_containers[ network_context ] = session_container
|
|
|
|
self._session_container_names.add( session_container_name )
|
|
self._dirty_session_container_names.add( session_container_name )
|
|
|
|
self._SetDirty()
|
|
|
|
|
|
def _ReinitialiseProxies( self ):
|
|
|
|
self._proxies_dict = {}
|
|
|
|
http_proxy = CG.client_controller.new_options.GetNoneableString( 'http_proxy' )
|
|
https_proxy = CG.client_controller.new_options.GetNoneableString( 'https_proxy' )
|
|
no_proxy = CG.client_controller.new_options.GetNoneableString( 'no_proxy' )
|
|
|
|
if http_proxy is not None:
|
|
|
|
self._proxies_dict[ 'http' ] = http_proxy
|
|
|
|
|
|
if https_proxy is not None:
|
|
|
|
self._proxies_dict[ 'https' ] = https_proxy
|
|
|
|
|
|
if ( http_proxy is not None or https_proxy is not None ) and no_proxy is not None:
|
|
|
|
self._proxies_dict[ 'no_proxy' ] = no_proxy
|
|
|
|
|
|
|
|
def _SetDirty( self ):
|
|
|
|
self._dirty = True
|
|
|
|
|
|
def ClearSession( self, network_context ):
|
|
|
|
with self._lock:
|
|
|
|
network_context = self._GetSessionNetworkContext( network_context )
|
|
|
|
if network_context in self._network_contexts_to_session_containers:
|
|
|
|
session_container = self._network_contexts_to_session_containers[ network_context ]
|
|
|
|
del self._network_contexts_to_session_containers[ network_context ]
|
|
|
|
session_container_name = session_container.GetName()
|
|
|
|
if session_container_name in self._session_container_names_to_session_containers:
|
|
|
|
del self._session_container_names_to_session_containers[ session_container_name ]
|
|
|
|
|
|
self._session_container_names.discard( session_container_name )
|
|
self._dirty_session_container_names.discard( session_container_name )
|
|
self._deletee_session_container_names.add( session_container_name )
|
|
|
|
self._SetDirty()
|
|
|
|
|
|
|
|
|
|
def GetDeleteeSessionNames( self ):
|
|
|
|
with self._lock:
|
|
|
|
return set( self._deletee_session_container_names )
|
|
|
|
|
|
|
|
def GetDirtySessionContainers( self ):
|
|
|
|
with self._lock:
|
|
|
|
return [ self._session_container_names_to_session_containers[ session_container_name ] for session_container_name in self._dirty_session_container_names ]
|
|
|
|
|
|
|
|
def GetNetworkContexts( self ):
|
|
|
|
with self._lock:
|
|
|
|
return list( self._network_contexts_to_session_containers.keys() )
|
|
|
|
|
|
|
|
def GetSession( self, network_context ):
|
|
|
|
with self._lock:
|
|
|
|
network_context = self._GetSessionNetworkContext( network_context )
|
|
|
|
if network_context not in self._network_contexts_to_session_containers:
|
|
|
|
self._InitialiseSessionContainer( network_context )
|
|
|
|
|
|
session_container = self._network_contexts_to_session_containers[ network_context ]
|
|
|
|
session_container.PrepareForNewWork()
|
|
|
|
session = session_container.session
|
|
|
|
if session.proxies != self._proxies_dict:
|
|
|
|
session.proxies = dict( self._proxies_dict )
|
|
|
|
|
|
#
|
|
|
|
# tumblr can't into ssl for some reason, and the data subdomain they use has weird cert properties, looking like amazon S3
|
|
# perhaps it is inward-facing somehow? whatever the case, let's just say fuck it for tumblr
|
|
|
|
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN and network_context.context_data == 'tumblr.com':
|
|
|
|
session.verify = False
|
|
|
|
|
|
if not CG.client_controller.new_options.GetBoolean( 'verify_regular_https' ):
|
|
|
|
session.verify = False
|
|
|
|
|
|
return session
|
|
|
|
|
|
|
|
def GetSessionForDomain( self, domain ):
|
|
|
|
network_context = ClientNetworkingContexts.NetworkContext( context_type = CC.NETWORK_CONTEXT_DOMAIN, context_data = domain )
|
|
|
|
return self.GetSession( network_context )
|
|
|
|
|
|
def HasDirtySessionContainers( self ):
|
|
|
|
with self._lock:
|
|
|
|
return len( self._dirty_session_container_names ) > 0 or len( self._deletee_session_container_names ) > 0
|
|
|
|
|
|
|
|
def IsDirty( self ):
|
|
|
|
with self._lock:
|
|
|
|
return self._dirty
|
|
|
|
|
|
|
|
def MaintainConnectionPools( self ):
|
|
|
|
for session_container in self._network_contexts_to_session_containers.values():
|
|
|
|
session_container.MaintainConnectionPool()
|
|
|
|
|
|
|
|
def ReinitialiseProxies( self ):
|
|
|
|
with self._lock:
|
|
|
|
self._ReinitialiseProxies()
|
|
|
|
|
|
|
|
def SetClean( self ):
|
|
|
|
with self._lock:
|
|
|
|
self._dirty = False
|
|
self._dirty_session_container_names = set()
|
|
self._deletee_session_container_names = set()
|
|
|
|
|
|
|
|
def SetDirty( self ):
|
|
|
|
with self._lock:
|
|
|
|
self._SetDirty()
|
|
|
|
|
|
|
|
def SetSessionContainers( self, session_containers: typing.Collection[ NetworkSessionManagerSessionContainer ], set_all_sessions_dirty = False ):
|
|
|
|
with self._lock:
|
|
|
|
self._session_container_names_to_session_containers = {}
|
|
self._network_contexts_to_session_containers = {}
|
|
|
|
self._session_container_names = set()
|
|
self._dirty_session_container_names = set()
|
|
self._deletee_session_container_names = set()
|
|
|
|
for session_container in session_containers:
|
|
|
|
session_container_name = session_container.GetName()
|
|
|
|
self._session_container_names_to_session_containers[ session_container_name ] = session_container
|
|
self._network_contexts_to_session_containers[ session_container.network_context ] = session_container
|
|
|
|
self._session_container_names.add( session_container_name )
|
|
|
|
if set_all_sessions_dirty:
|
|
|
|
self._dirty_session_container_names.add( session_container_name )
|
|
|
|
|
|
|
|
|
|
|
|
def SetSessionDirty( self, network_context: ClientNetworkingContexts.NetworkContext ):
|
|
|
|
with self._lock:
|
|
|
|
network_context = self._GetSessionNetworkContext( network_context )
|
|
|
|
if network_context in self._network_contexts_to_session_containers:
|
|
|
|
self._dirty_session_container_names.add( self._network_contexts_to_session_containers[ network_context ].GetName() )
|
|
|
|
|
|
|
|
|
|
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER ] = NetworkSessionManager
|