2020-05-20 21:36:02 +00:00
import collections
import os
import threading
import time
2022-12-21 22:00:27 +00:00
import typing
2020-05-20 21:36:02 +00:00
import urllib . parse
2020-04-22 21:00:35 +00:00
from hydrus . core import HydrusConstants as HC
from hydrus . core import HydrusGlobals as HG
from hydrus . core import HydrusData
from hydrus . core import HydrusExceptions
from hydrus . core import HydrusSerialisable
2023-04-19 20:38:13 +00:00
from hydrus . core import HydrusTime
2021-04-07 21:26:45 +00:00
from hydrus . core . networking import HydrusNetworking
2017-09-13 20:50:41 +00:00
2020-07-29 20:52:44 +00:00
from hydrus . client import ClientConstants as CC
from hydrus . client import ClientThreading
from hydrus . client . networking import ClientNetworkingContexts
2022-01-05 22:15:56 +00:00
from hydrus . client . networking import ClientNetworkingFunctions
from hydrus . client . networking import ClientNetworkingURLClass
2020-07-29 20:52:44 +00:00
2022-01-05 22:15:56 +00:00
VALID_DENIED = 0
VALID_APPROVED = 1
VALID_UNKNOWN = 2
2022-03-30 20:28:13 +00:00
valid_str_lookup = {
VALID_DENIED : ' denied ' ,
VALID_APPROVED : ' approved ' ,
2023-03-29 20:57:59 +00:00
VALID_UNKNOWN : ' pending '
2022-03-30 20:28:13 +00:00
}
2022-01-05 22:15:56 +00:00
2023-03-29 20:57:59 +00:00
valid_enum_lookup = { value : key for ( key , value ) in valid_str_lookup . items ( ) }
2022-01-05 22:15:56 +00:00
class NetworkDomainManager ( HydrusSerialisable . SerialisableBase ) :
2020-04-16 00:09:42 +00:00
2022-01-05 22:15:56 +00:00
SERIALISABLE_TYPE = HydrusSerialisable . SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER
SERIALISABLE_NAME = ' Domain Manager '
2022-08-17 20:54:59 +00:00
SERIALISABLE_VERSION = 7
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
def __init__ ( self ) :
2017-10-25 21:45:15 +00:00
2022-01-05 22:15:56 +00:00
HydrusSerialisable . SerialisableBase . __init__ ( self )
2017-10-25 21:45:15 +00:00
2022-01-05 22:15:56 +00:00
self . _gugs = HydrusSerialisable . SerialisableList ( )
self . _url_classes = HydrusSerialisable . SerialisableList ( )
self . _parsers = HydrusSerialisable . SerialisableList ( )
self . _network_contexts_to_custom_header_dicts = collections . defaultdict ( dict )
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
self . _parser_namespaces = [ ]
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
self . _gug_keys_to_display = set ( )
2018-10-31 21:41:14 +00:00
2022-01-05 22:15:56 +00:00
self . _url_class_keys_to_display = set ( )
self . _url_class_keys_to_parser_keys = HydrusSerialisable . SerialisableBytesDictionary ( )
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
self . _second_level_domains_to_url_classes = collections . defaultdict ( list )
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
self . _second_level_domains_to_network_infrastructure_errors = collections . defaultdict ( list )
2018-05-09 20:23:00 +00:00
2022-01-05 22:15:56 +00:00
from hydrus . client . importing . options import TagImportOptions
2018-05-09 20:23:00 +00:00
2022-01-05 22:15:56 +00:00
self . _file_post_default_tag_import_options = TagImportOptions . TagImportOptions ( )
self . _watchable_default_tag_import_options = TagImportOptions . TagImportOptions ( )
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
self . _url_class_keys_to_default_tag_import_options = { }
2018-04-25 22:07:52 +00:00
2022-08-17 20:54:59 +00:00
from hydrus . client . importing . options import NoteImportOptions
self . _file_post_default_note_import_options = NoteImportOptions . NoteImportOptions ( )
self . _watchable_default_note_import_options = NoteImportOptions . NoteImportOptions ( )
self . _url_class_keys_to_default_note_import_options = { }
2022-01-05 22:15:56 +00:00
self . _gug_keys_to_gugs = { }
self . _gug_names_to_gugs = { }
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
self . _parser_keys_to_parsers = { }
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
self . _dirty = False
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
self . _lock = threading . Lock ( )
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
self . _RecalcCache ( )
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
def _CleanURLClassKeysToParserKeys ( self ) :
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
api_pairs = ClientNetworkingURLClass . ConvertURLClassesIntoAPIPairs ( self . _url_classes )
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
# anything that goes to an api url will be parsed by that api's parser--it can't have its own
for ( a , b ) in api_pairs :
unparseable_url_class_key = a . GetClassKey ( )
if unparseable_url_class_key in self . _url_class_keys_to_parser_keys :
del self . _url_class_keys_to_parser_keys [ unparseable_url_class_key ]
2018-04-25 22:07:52 +00:00
2022-08-17 20:54:59 +00:00
def _GetDefaultNoteImportOptionsForURL ( self , url ) :
url_class = self . _GetURLClass ( url )
if url_class is None or url_class . GetURLType ( ) not in ( HC . URL_TYPE_POST , HC . URL_TYPE_WATCHABLE ) :
return self . _file_post_default_note_import_options
try :
( url_class , url ) = self . _GetNormalisedAPIURLClassAndURL ( url )
except HydrusExceptions . URLClassException :
return self . _file_post_default_note_import_options
# some lad decided to api convert one url type to another
if url_class . GetURLType ( ) not in ( HC . URL_TYPE_POST , HC . URL_TYPE_WATCHABLE ) :
return self . _file_post_default_note_import_options
url_class_key = url_class . GetClassKey ( )
if url_class_key in self . _url_class_keys_to_default_note_import_options :
return self . _url_class_keys_to_default_note_import_options [ url_class_key ]
else :
url_type = url_class . GetURLType ( )
if url_type == HC . URL_TYPE_POST :
return self . _file_post_default_note_import_options
elif url_type == HC . URL_TYPE_WATCHABLE :
return self . _watchable_default_note_import_options
else :
raise HydrusExceptions . URLClassException ( ' Could not find note import options for that kind of URL Class! ' )
2022-01-05 22:15:56 +00:00
def _GetDefaultTagImportOptionsForURL ( self , url ) :
url_class = self . _GetURLClass ( url )
2019-11-28 01:11:46 +00:00
2022-01-05 22:15:56 +00:00
if url_class is None or url_class . GetURLType ( ) not in ( HC . URL_TYPE_POST , HC . URL_TYPE_WATCHABLE ) :
return self . _file_post_default_tag_import_options
2019-11-28 01:11:46 +00:00
2022-01-05 22:15:56 +00:00
try :
( url_class , url ) = self . _GetNormalisedAPIURLClassAndURL ( url )
except HydrusExceptions . URLClassException :
return self . _file_post_default_tag_import_options
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
# some lad decided to api convert one url type to another
if url_class . GetURLType ( ) not in ( HC . URL_TYPE_POST , HC . URL_TYPE_WATCHABLE ) :
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
return self . _file_post_default_tag_import_options
2021-09-29 21:20:29 +00:00
2019-11-28 01:11:46 +00:00
2022-01-05 22:15:56 +00:00
url_class_key = url_class . GetClassKey ( )
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
if url_class_key in self . _url_class_keys_to_default_tag_import_options :
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
return self . _url_class_keys_to_default_tag_import_options [ url_class_key ]
else :
url_type = url_class . GetURLType ( )
if url_type == HC . URL_TYPE_POST :
return self . _file_post_default_tag_import_options
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
elif url_type == HC . URL_TYPE_WATCHABLE :
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
return self . _watchable_default_tag_import_options
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
else :
raise HydrusExceptions . URLClassException ( ' Could not find tag import options for that kind of URL Class! ' )
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
def _GetGUG ( self , gug_key_and_name ) :
( gug_key , gug_name ) = gug_key_and_name
if gug_key in self . _gug_keys_to_gugs :
return self . _gug_keys_to_gugs [ gug_key ]
elif gug_name in self . _gug_names_to_gugs :
return self . _gug_names_to_gugs [ gug_name ]
2021-09-29 21:20:29 +00:00
else :
2022-01-05 22:15:56 +00:00
return None
2021-09-29 21:20:29 +00:00
2019-11-28 01:11:46 +00:00
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
def _GetNormalisedAPIURLClassAndURL ( self , url ) :
url_class = self . _GetURLClass ( url )
if url_class is None :
raise HydrusExceptions . URLClassException ( ' Could not find a URL Class for ' + url + ' ! ' )
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
seen_url_classes = set ( )
seen_url_classes . add ( url_class )
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
api_url_class = url_class
api_url = url
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
while api_url_class . UsesAPIURL ( ) :
api_url = api_url_class . GetAPIURL ( api_url )
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
api_url_class = self . _GetURLClass ( api_url )
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
if api_url_class is None :
2021-09-29 21:20:29 +00:00
2023-04-26 21:10:03 +00:00
raise HydrusExceptions . URLClassException ( ' Could not find an API/Redirect URL Class for ' + api_url + ' URL, which originally came from ' + url + ' ! ' )
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
if api_url_class in seen_url_classes :
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
loop_size = len ( seen_url_classes )
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
if loop_size == 1 :
2021-09-29 21:20:29 +00:00
2023-04-26 21:10:03 +00:00
message = ' Could not find an API/Redirect URL Class for ' + url + ' as the url class API-linked to itself! '
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
elif loop_size == 2 :
2023-04-26 21:10:03 +00:00
message = ' Could not find an API/Redirect URL Class for ' + url + ' as the url class and its API url class API-linked to each other! '
2022-01-05 22:15:56 +00:00
else :
2023-04-26 21:10:03 +00:00
message = ' Could not find an API/Redirect URL Class for ' + url + ' as it and its API url classes linked in a loop of size ' + HydrusData . ToHumanInt ( loop_size ) + ' ! '
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
raise HydrusExceptions . URLClassException ( message )
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
seen_url_classes . add ( api_url_class )
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
api_url = api_url_class . Normalise ( api_url )
return ( api_url_class , api_url )
def _GetSerialisableInfo ( self ) :
serialisable_gugs = self . _gugs . GetSerialisableTuple ( )
serialisable_gug_keys_to_display = [ gug_key . hex ( ) for gug_key in self . _gug_keys_to_display ]
serialisable_url_classes = self . _url_classes . GetSerialisableTuple ( )
serialisable_url_class_keys_to_display = [ url_class_key . hex ( ) for url_class_key in self . _url_class_keys_to_display ]
serialisable_url_class_keys_to_parser_keys = self . _url_class_keys_to_parser_keys . GetSerialisableTuple ( )
serialisable_file_post_default_tag_import_options = self . _file_post_default_tag_import_options . GetSerialisableTuple ( )
serialisable_watchable_default_tag_import_options = self . _watchable_default_tag_import_options . GetSerialisableTuple ( )
2022-08-17 20:54:59 +00:00
serialisable_url_class_keys_to_default_tag_import_options = [ ( url_class_key . hex ( ) , tag_import_options . GetSerialisableTuple ( ) ) for ( url_class_key , tag_import_options ) in self . _url_class_keys_to_default_tag_import_options . items ( ) ]
2022-01-05 22:15:56 +00:00
serialisable_default_tag_import_options_tuple = ( serialisable_file_post_default_tag_import_options , serialisable_watchable_default_tag_import_options , serialisable_url_class_keys_to_default_tag_import_options )
2022-08-17 20:54:59 +00:00
serialisable_file_post_default_note_import_options = self . _file_post_default_note_import_options . GetSerialisableTuple ( )
serialisable_watchable_default_note_import_options = self . _watchable_default_note_import_options . GetSerialisableTuple ( )
serialisable_url_class_keys_to_default_note_import_options = [ ( url_class_key . hex ( ) , note_import_options . GetSerialisableTuple ( ) ) for ( url_class_key , note_import_options ) in self . _url_class_keys_to_default_note_import_options . items ( ) ]
serialisable_default_note_import_options_tuple = ( serialisable_file_post_default_note_import_options , serialisable_watchable_default_note_import_options , serialisable_url_class_keys_to_default_note_import_options )
2022-01-05 22:15:56 +00:00
serialisable_parsers = self . _parsers . GetSerialisableTuple ( )
2023-03-29 20:57:59 +00:00
serialisable_network_contexts_to_custom_header_dicts = [ ( network_context . GetSerialisableTuple ( ) , list ( custom_header_dict . items ( ) ) ) for ( network_context , custom_header_dict ) in self . _network_contexts_to_custom_header_dicts . items ( ) ]
2022-01-05 22:15:56 +00:00
2022-08-17 20:54:59 +00:00
return ( serialisable_gugs , serialisable_gug_keys_to_display , serialisable_url_classes , serialisable_url_class_keys_to_display , serialisable_url_class_keys_to_parser_keys , serialisable_default_tag_import_options_tuple , serialisable_default_note_import_options_tuple , serialisable_parsers , serialisable_network_contexts_to_custom_header_dicts )
2022-01-05 22:15:56 +00:00
def _GetURLClass ( self , url ) :
domain = ClientNetworkingFunctions . ConvertURLIntoSecondLevelDomain ( url )
if domain in self . _second_level_domains_to_url_classes :
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
url_classes = self . _second_level_domains_to_url_classes [ domain ]
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
for url_class in url_classes :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
try :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
url_class . Test ( url )
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
return url_class
2019-08-15 00:40:48 +00:00
2022-01-05 22:15:56 +00:00
except HydrusExceptions . URLClassException :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
continue
2018-10-03 21:00:15 +00:00
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
return None
2018-08-22 21:10:59 +00:00
2023-04-26 21:10:03 +00:00
def _GetURLToFetch ( self , url : str ) :
url_class = self . _GetURLClass ( url )
if url_class is None :
return url
try :
( url_class , url_to_fetch ) = self . _GetNormalisedAPIURLClassAndURL ( url )
except HydrusExceptions . URLClassException as e :
raise HydrusExceptions . URLClassException ( ' Could not find a URL class for ' + url + ' ! ' + os . linesep * 2 + str ( e ) )
return url_to_fetch
2022-01-05 22:15:56 +00:00
def _GetURLToFetchAndParser ( self , url ) :
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
try :
( parser_url_class , parser_url ) = self . _GetNormalisedAPIURLClassAndURL ( url )
except HydrusExceptions . URLClassException as e :
2018-02-07 23:40:33 +00:00
2023-04-26 21:10:03 +00:00
raise HydrusExceptions . URLClassException ( ' Could not find a URL class for ' + url + ' ! ' + os . linesep * 2 + str ( e ) )
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
url_class_key = parser_url_class . GetClassKey ( )
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
if url_class_key in self . _url_class_keys_to_parser_keys :
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
parser_key = self . _url_class_keys_to_parser_keys [ url_class_key ]
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
if parser_key is not None and parser_key in self . _parser_keys_to_parsers :
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
return ( parser_url , self . _parser_keys_to_parsers [ parser_key ] )
2018-09-19 21:54:51 +00:00
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
raise HydrusExceptions . URLClassException ( ' Could not find a parser for ' + parser_url_class . GetName ( ) + ' URL Class! ' )
2018-03-14 21:01:02 +00:00
2022-01-05 22:15:56 +00:00
def _InitialiseFromSerialisableInfo ( self , serialisable_info ) :
2018-03-14 21:01:02 +00:00
2022-08-17 20:54:59 +00:00
( serialisable_gugs , serialisable_gug_keys_to_display , serialisable_url_classes , serialisable_url_class_keys_to_display , serialisable_url_class_keys_to_parser_keys , serialisable_default_tag_import_options_tuple , serialisable_default_note_import_options_tuple , serialisable_parsers , serialisable_network_contexts_to_custom_header_dicts ) = serialisable_info
2018-03-14 21:01:02 +00:00
2022-01-05 22:15:56 +00:00
self . _gugs = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_gugs )
2017-11-01 20:37:39 +00:00
2022-01-05 22:15:56 +00:00
self . _gug_keys_to_display = { bytes . fromhex ( serialisable_gug_key ) for serialisable_gug_key in serialisable_gug_keys_to_display }
2017-11-01 20:37:39 +00:00
2022-01-05 22:15:56 +00:00
self . _url_classes = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_url_classes )
2019-07-31 22:01:02 +00:00
2022-01-05 22:15:56 +00:00
self . _url_class_keys_to_display = { bytes . fromhex ( serialisable_url_class_key ) for serialisable_url_class_key in serialisable_url_class_keys_to_display }
self . _url_class_keys_to_parser_keys = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_url_class_keys_to_parser_keys )
2019-07-31 22:01:02 +00:00
2022-01-05 22:15:56 +00:00
( serialisable_file_post_default_tag_import_options , serialisable_watchable_default_tag_import_options , serialisable_url_class_keys_to_default_tag_import_options ) = serialisable_default_tag_import_options_tuple
2019-07-31 22:01:02 +00:00
2022-01-05 22:15:56 +00:00
self . _file_post_default_tag_import_options = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_file_post_default_tag_import_options )
self . _watchable_default_tag_import_options = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_watchable_default_tag_import_options )
2019-07-31 22:01:02 +00:00
2022-01-05 22:15:56 +00:00
self . _url_class_keys_to_default_tag_import_options = { bytes . fromhex ( serialisable_url_class_key ) : HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_tag_import_options ) for ( serialisable_url_class_key , serialisable_tag_import_options ) in serialisable_url_class_keys_to_default_tag_import_options }
2019-07-31 22:01:02 +00:00
2022-08-17 20:54:59 +00:00
( serialisable_file_post_default_note_import_options , serialisable_watchable_default_note_import_options , serialisable_url_class_keys_to_default_note_import_options ) = serialisable_default_note_import_options_tuple
self . _file_post_default_note_import_options = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_file_post_default_note_import_options )
self . _watchable_default_note_import_options = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_watchable_default_note_import_options )
self . _url_class_keys_to_default_note_import_options = { bytes . fromhex ( serialisable_url_class_key ) : HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_note_import_options ) for ( serialisable_url_class_key , serialisable_note_import_options ) in serialisable_url_class_keys_to_default_note_import_options }
2022-01-05 22:15:56 +00:00
self . _parsers = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_parsers )
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
self . _network_contexts_to_custom_header_dicts = collections . defaultdict ( dict )
for ( serialisable_network_context , custom_header_dict_items ) in serialisable_network_contexts_to_custom_header_dicts :
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
network_context = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_network_context )
custom_header_dict = dict ( custom_header_dict_items )
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
self . _network_contexts_to_custom_header_dicts [ network_context ] = custom_header_dict
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
def _RecalcCache ( self ) :
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
self . _second_level_domains_to_url_classes = collections . defaultdict ( list )
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
for url_class in self . _url_classes :
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
domain = ClientNetworkingFunctions . ConvertDomainIntoSecondLevelDomain ( url_class . GetDomain ( ) )
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
self . _second_level_domains_to_url_classes [ domain ] . append ( url_class )
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
for url_classes in self . _second_level_domains_to_url_classes . values ( ) :
ClientNetworkingURLClass . SortURLClassesListDescendingComplexity ( url_classes )
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
self . _gug_keys_to_gugs = { gug . GetGUGKey ( ) : gug for gug in self . _gugs }
self . _gug_names_to_gugs = { gug . GetName ( ) : gug for gug in self . _gugs }
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
self . _parser_keys_to_parsers = { parser . GetParserKey ( ) : parser for parser in self . _parsers }
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
namespaces = set ( )
2019-05-22 22:35:06 +00:00
2022-01-05 22:15:56 +00:00
for parser in self . _parsers :
2019-05-22 22:35:06 +00:00
2022-01-05 22:15:56 +00:00
namespaces . update ( parser . GetNamespaces ( ) )
2019-05-22 22:35:06 +00:00
2022-01-05 22:15:56 +00:00
self . _parser_namespaces = sorted ( namespaces )
2019-05-22 22:35:06 +00:00
2022-01-05 22:15:56 +00:00
def _SetDirty ( self ) :
self . _dirty = True
2021-10-13 20:16:57 +00:00
2022-01-05 22:15:56 +00:00
def _UpdateSerialisableInfo ( self , version , old_serialisable_info ) :
2021-10-13 20:16:57 +00:00
2022-01-05 22:15:56 +00:00
if version == 1 :
2021-10-13 20:16:57 +00:00
2022-01-05 22:15:56 +00:00
( serialisable_url_classes , serialisable_network_contexts_to_custom_header_dicts ) = old_serialisable_info
2021-10-13 20:16:57 +00:00
2022-01-05 22:15:56 +00:00
url_classes = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_url_classes )
2021-10-13 20:16:57 +00:00
2022-01-05 22:15:56 +00:00
url_class_names_to_display = { }
url_class_names_to_page_parser_keys = HydrusSerialisable . SerialisableBytesDictionary ( )
url_class_names_to_gallery_parser_keys = HydrusSerialisable . SerialisableBytesDictionary ( )
2021-10-13 20:16:57 +00:00
2022-01-05 22:15:56 +00:00
for url_class in url_classes :
name = url_class . GetName ( )
if url_class . IsPostURL ( ) :
url_class_names_to_display [ name ] = True
url_class_names_to_page_parser_keys [ name ] = None
if url_class . IsGalleryURL ( ) or url_class . IsWatchableURL ( ) :
url_class_names_to_gallery_parser_keys [ name ] = None
2020-05-13 19:03:16 +00:00
2022-01-05 22:15:56 +00:00
serialisable_url_class_names_to_display = list ( url_class_names_to_display . items ( ) )
serialisable_url_class_names_to_page_parser_keys = url_class_names_to_page_parser_keys . GetSerialisableTuple ( )
serialisable_url_class_names_to_gallery_parser_keys = url_class_names_to_gallery_parser_keys . GetSerialisableTuple ( )
2020-05-13 19:03:16 +00:00
2022-01-05 22:15:56 +00:00
new_serialisable_info = ( serialisable_url_classes , serialisable_url_class_names_to_display , serialisable_url_class_names_to_page_parser_keys , serialisable_url_class_names_to_gallery_parser_keys , serialisable_network_contexts_to_custom_header_dicts )
2020-05-13 19:03:16 +00:00
2022-01-05 22:15:56 +00:00
return ( 2 , new_serialisable_info )
2020-05-13 19:03:16 +00:00
2022-01-05 22:15:56 +00:00
if version == 2 :
2020-05-13 19:03:16 +00:00
2022-01-05 22:15:56 +00:00
( serialisable_url_classes , serialisable_url_class_names_to_display , serialisable_url_class_names_to_page_parser_keys , serialisable_url_class_names_to_gallery_parser_keys , serialisable_network_contexts_to_custom_header_dicts ) = old_serialisable_info
2020-05-13 19:03:16 +00:00
2022-01-05 22:15:56 +00:00
parsers = HydrusSerialisable . SerialisableList ( )
2020-05-13 19:03:16 +00:00
2022-01-05 22:15:56 +00:00
serialisable_parsing_parsers = parsers . GetSerialisableTuple ( )
2020-05-13 19:03:16 +00:00
2022-01-05 22:15:56 +00:00
url_class_names_to_display = dict ( serialisable_url_class_names_to_display )
2020-06-17 21:31:54 +00:00
2022-01-05 22:15:56 +00:00
url_class_keys_to_display = [ ]
2020-06-17 21:31:54 +00:00
2022-01-05 22:15:56 +00:00
url_class_names_to_gallery_parser_keys = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_url_class_names_to_gallery_parser_keys )
url_class_names_to_page_parser_keys = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_url_class_names_to_page_parser_keys )
2020-06-17 21:31:54 +00:00
2022-01-05 22:15:56 +00:00
url_class_keys_to_parser_keys = HydrusSerialisable . SerialisableBytesDictionary ( )
2018-07-04 20:48:28 +00:00
2022-01-05 22:15:56 +00:00
url_classes = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_url_classes )
2018-07-04 20:48:28 +00:00
2022-01-05 22:15:56 +00:00
for url_class in url_classes :
url_class_key = url_class . GetClassKey ( )
name = url_class . GetName ( )
if name in url_class_names_to_display and url_class_names_to_display [ name ] :
url_class_keys_to_display . append ( url_class_key )
2018-07-04 20:48:28 +00:00
2022-01-05 22:15:56 +00:00
serialisable_url_classes = url_classes . GetSerialisableTuple ( ) # added random key this week, so save these changes back again!
2018-07-04 20:48:28 +00:00
2022-01-05 22:15:56 +00:00
serialisable_url_class_keys_to_display = [ url_class_key . hex ( ) for url_class_key in url_class_keys_to_display ]
2018-07-04 20:48:28 +00:00
2022-01-05 22:15:56 +00:00
serialisable_url_class_keys_to_parser_keys = url_class_keys_to_parser_keys . GetSerialisableTuple ( )
2018-07-04 20:48:28 +00:00
2022-01-05 22:15:56 +00:00
new_serialisable_info = ( serialisable_url_classes , serialisable_url_class_keys_to_display , serialisable_url_class_keys_to_parser_keys , serialisable_parsing_parsers , serialisable_network_contexts_to_custom_header_dicts )
2020-03-18 21:35:57 +00:00
2022-01-05 22:15:56 +00:00
return ( 3 , new_serialisable_info )
2020-03-18 21:35:57 +00:00
2022-01-05 22:15:56 +00:00
if version == 3 :
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
( serialisable_url_classes , serialisable_url_class_keys_to_display , serialisable_url_class_keys_to_parser_keys , serialisable_parsing_parsers , serialisable_network_contexts_to_custom_header_dicts ) = old_serialisable_info
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
from hydrus . client . importing . options import TagImportOptions
2018-04-18 22:10:15 +00:00
2022-08-17 20:54:59 +00:00
file_post_default_tag_import_options = TagImportOptions . TagImportOptions ( )
watchable_default_tag_import_options = TagImportOptions . TagImportOptions ( )
2022-01-05 22:15:56 +00:00
2022-08-17 20:54:59 +00:00
url_class_keys_to_default_tag_import_options = { }
2022-01-05 22:15:56 +00:00
2022-08-17 20:54:59 +00:00
serialisable_file_post_default_tag_import_options = file_post_default_tag_import_options . GetSerialisableTuple ( )
serialisable_watchable_default_tag_import_options = watchable_default_tag_import_options . GetSerialisableTuple ( )
serialisable_url_class_keys_to_default_tag_import_options = [ ( url_class_key . hex ( ) , tag_import_options . GetSerialisableTuple ( ) ) for ( url_class_key , tag_import_options ) in url_class_keys_to_default_tag_import_options . items ( ) ]
2022-01-05 22:15:56 +00:00
serialisable_default_tag_import_options_tuple = ( serialisable_file_post_default_tag_import_options , serialisable_watchable_default_tag_import_options , serialisable_url_class_keys_to_default_tag_import_options )
new_serialisable_info = ( serialisable_url_classes , serialisable_url_class_keys_to_display , serialisable_url_class_keys_to_parser_keys , serialisable_default_tag_import_options_tuple , serialisable_parsing_parsers , serialisable_network_contexts_to_custom_header_dicts )
return ( 4 , new_serialisable_info )
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
if version == 4 :
( serialisable_url_classes , serialisable_url_class_keys_to_display , serialisable_url_class_keys_to_parser_keys , serialisable_default_tag_import_options_tuple , serialisable_parsing_parsers , serialisable_network_contexts_to_custom_header_dicts ) = old_serialisable_info
gugs = HydrusSerialisable . SerialisableList ( )
serialisable_gugs = gugs . GetSerialisableTuple ( )
new_serialisable_info = ( serialisable_gugs , serialisable_url_classes , serialisable_url_class_keys_to_display , serialisable_url_class_keys_to_parser_keys , serialisable_default_tag_import_options_tuple , serialisable_parsing_parsers , serialisable_network_contexts_to_custom_header_dicts )
return ( 5 , new_serialisable_info )
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
if version == 5 :
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
( serialisable_gugs , serialisable_url_classes , serialisable_url_class_keys_to_display , serialisable_url_class_keys_to_parser_keys , serialisable_default_tag_import_options_tuple , serialisable_parsing_parsers , serialisable_network_contexts_to_custom_header_dicts ) = old_serialisable_info
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
gugs = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_gugs )
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
gug_keys_to_display = [ gug . GetGUGKey ( ) for gug in gugs if ' ugoira ' not in gug . GetName ( ) ]
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
serialisable_gug_keys_to_display = [ gug_key . hex ( ) for gug_key in gug_keys_to_display ]
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
new_serialisable_info = ( serialisable_gugs , serialisable_gug_keys_to_display , serialisable_url_classes , serialisable_url_class_keys_to_display , serialisable_url_class_keys_to_parser_keys , serialisable_default_tag_import_options_tuple , serialisable_parsing_parsers , serialisable_network_contexts_to_custom_header_dicts )
return ( 6 , new_serialisable_info )
2018-09-05 20:52:32 +00:00
2022-08-17 20:54:59 +00:00
if version == 6 :
( serialisable_gugs , serialisable_gug_keys_to_display , serialisable_url_classes , serialisable_url_class_keys_to_display , serialisable_url_class_keys_to_parser_keys , serialisable_default_tag_import_options_tuple , serialisable_parsing_parsers , serialisable_network_contexts_to_custom_header_dicts ) = old_serialisable_info
from hydrus . client . importing . options import NoteImportOptions
file_post_default_note_import_options = NoteImportOptions . NoteImportOptions ( )
watchable_default_note_import_options = NoteImportOptions . NoteImportOptions ( )
url_class_keys_to_default_note_import_options = { }
serialisable_file_post_default_note_import_options = file_post_default_note_import_options . GetSerialisableTuple ( )
serialisable_watchable_default_note_import_options = watchable_default_note_import_options . GetSerialisableTuple ( )
serialisable_url_class_keys_to_default_note_import_options = [ ( url_class_key . hex ( ) , note_import_options . GetSerialisableTuple ( ) ) for ( url_class_key , note_import_options ) in url_class_keys_to_default_note_import_options . items ( ) ]
serialisable_default_note_import_options_tuple = ( serialisable_file_post_default_note_import_options , serialisable_watchable_default_note_import_options , serialisable_url_class_keys_to_default_note_import_options )
new_serialisable_info = ( serialisable_gugs , serialisable_gug_keys_to_display , serialisable_url_classes , serialisable_url_class_keys_to_display , serialisable_url_class_keys_to_parser_keys , serialisable_default_tag_import_options_tuple , serialisable_default_note_import_options_tuple , serialisable_parsing_parsers , serialisable_network_contexts_to_custom_header_dicts )
return ( 7 , new_serialisable_info )
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
def AddGUGs ( self , new_gugs ) :
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-05-30 20:13:21 +00:00
2022-01-05 22:15:56 +00:00
gugs = list ( self . _gugs )
for gug in new_gugs :
gug . SetNonDupeName ( [ g . GetName ( ) for g in gugs ] )
gugs . append ( gug )
2018-05-30 20:13:21 +00:00
2022-01-05 22:15:56 +00:00
self . SetGUGs ( gugs )
2018-06-06 21:27:02 +00:00
2022-01-05 22:15:56 +00:00
def AddParsers ( self , new_parsers ) :
2018-05-30 20:13:21 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-05-30 20:13:21 +00:00
2022-01-05 22:15:56 +00:00
parsers = list ( self . _parsers )
2018-05-30 20:13:21 +00:00
2022-01-05 22:15:56 +00:00
for parser in new_parsers :
parser . SetNonDupeName ( [ p . GetName ( ) for p in parsers ] )
parsers . append ( parser )
2018-05-30 20:13:21 +00:00
2022-01-05 22:15:56 +00:00
self . SetParsers ( parsers )
def AddURLClasses ( self , new_url_classes ) :
with self . _lock :
url_classes = list ( self . _url_classes )
for url_class in new_url_classes :
2018-05-30 20:13:21 +00:00
2022-01-05 22:15:56 +00:00
url_class . SetNonDupeName ( [ u . GetName ( ) for u in url_classes ] )
url_classes . append ( url_class )
2018-05-30 20:13:21 +00:00
2022-01-05 22:15:56 +00:00
self . SetURLClasses ( url_classes )
def AlreadyHaveExactlyTheseHeaders ( self , network_context , headers_list ) :
with self . _lock :
if network_context in self . _network_contexts_to_custom_header_dicts :
2018-06-06 21:27:02 +00:00
2022-01-05 22:15:56 +00:00
custom_headers_dict = self . _network_contexts_to_custom_header_dicts [ network_context ]
2018-06-06 21:27:02 +00:00
2022-01-05 22:15:56 +00:00
if len ( headers_list ) != len ( custom_headers_dict ) :
2018-06-06 21:27:02 +00:00
2022-01-05 22:15:56 +00:00
return False
2018-06-06 21:27:02 +00:00
2022-01-05 22:15:56 +00:00
for ( key , value , reason ) in headers_list :
2018-06-06 21:27:02 +00:00
2022-01-05 22:15:56 +00:00
if key not in custom_headers_dict :
return False
2018-06-06 21:27:02 +00:00
2022-01-05 22:15:56 +00:00
( existing_value , existing_approved , existing_reason ) = custom_headers_dict [ key ]
2018-06-06 21:27:02 +00:00
2022-01-05 22:15:56 +00:00
if existing_value != value :
return False
2018-06-06 21:27:02 +00:00
2022-01-05 22:15:56 +00:00
return True
2018-06-06 21:27:02 +00:00
2018-05-30 20:13:21 +00:00
2022-01-05 22:15:56 +00:00
return False
2018-05-30 20:13:21 +00:00
2022-01-05 22:15:56 +00:00
def AlreadyHaveExactlyThisGUG ( self , new_gug ) :
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
# absent irrelevant variables, do we have the exact same object already in?
gug_key_and_name = new_gug . GetGUGKeyAndName ( )
dupe_gugs = [ gug . Duplicate ( ) for gug in self . _gugs ]
for dupe_gug in dupe_gugs :
dupe_gug . SetGUGKeyAndName ( gug_key_and_name )
if dupe_gug . DumpToString ( ) == new_gug . DumpToString ( ) :
return True
2017-10-11 17:38:14 +00:00
2022-01-05 22:15:56 +00:00
return False
2017-10-11 17:38:14 +00:00
2022-01-05 22:15:56 +00:00
def AlreadyHaveExactlyThisParser ( self , new_parser ) :
2017-09-27 21:52:54 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2017-09-27 21:52:54 +00:00
2022-01-05 22:15:56 +00:00
# absent irrelevant variables, do we have the exact same object already in?
2017-09-27 21:52:54 +00:00
2022-01-05 22:15:56 +00:00
new_name = new_parser . GetName ( )
new_parser_key = new_parser . GetParserKey ( )
new_example_urls = new_parser . GetExampleURLs ( )
new_example_parsing_context = new_parser . GetExampleParsingContext ( )
dupe_parsers = [ ( parser . Duplicate ( ) , parser ) for parser in self . _parsers ]
for ( dupe_parser , parser ) in dupe_parsers :
2017-09-27 21:52:54 +00:00
2022-01-05 22:15:56 +00:00
dupe_parser . SetName ( new_name )
dupe_parser . SetParserKey ( new_parser_key )
dupe_parser . SetExampleURLs ( new_example_urls )
dupe_parser . SetExampleParsingContext ( new_example_parsing_context )
if dupe_parser . DumpToString ( ) == new_parser . DumpToString ( ) :
2017-11-22 21:03:07 +00:00
2022-01-05 22:15:56 +00:00
# since these are the 'same', let's merge example urls
2017-11-22 21:03:07 +00:00
2022-01-05 22:15:56 +00:00
parser_example_urls = set ( parser . GetExampleURLs ( ) )
2017-09-27 21:52:54 +00:00
2022-01-05 22:15:56 +00:00
parser_example_urls . update ( new_example_urls )
2017-11-22 21:03:07 +00:00
2022-01-05 22:15:56 +00:00
parser_example_urls = list ( parser_example_urls )
parser . SetExampleURLs ( parser_example_urls )
self . _SetDirty ( )
return True
2017-09-27 21:52:54 +00:00
2022-01-05 22:15:56 +00:00
return False
2017-09-27 21:52:54 +00:00
2022-01-05 22:15:56 +00:00
def AlreadyHaveExactlyThisURLClass ( self , new_url_class ) :
2019-02-06 22:41:35 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2019-02-06 22:41:35 +00:00
2022-01-05 22:15:56 +00:00
# absent irrelevant variables, do we have the exact same object already in?
2019-02-06 22:41:35 +00:00
2022-01-05 22:15:56 +00:00
name = new_url_class . GetName ( )
match_key = new_url_class . GetClassKey ( )
example_url = new_url_class . GetExampleURL ( )
2019-02-06 22:41:35 +00:00
2022-01-05 22:15:56 +00:00
dupe_url_classes = [ url_class . Duplicate ( ) for url_class in self . _url_classes ]
2019-02-06 22:41:35 +00:00
2022-01-05 22:15:56 +00:00
for dupe_url_class in dupe_url_classes :
2019-02-06 22:41:35 +00:00
2022-01-05 22:15:56 +00:00
dupe_url_class . SetName ( name )
dupe_url_class . SetClassKey ( match_key )
dupe_url_class . SetExampleURL ( example_url )
if dupe_url_class . DumpToString ( ) == new_url_class . DumpToString ( ) :
return True
2019-02-06 22:41:35 +00:00
2022-01-05 22:15:56 +00:00
return False
2019-02-06 22:41:35 +00:00
2023-03-29 20:57:59 +00:00
def AutoAddDomainMetadatas ( self , domain_metadatas , approved = VALID_APPROVED ) :
2017-10-11 17:38:14 +00:00
2022-01-05 22:15:56 +00:00
for domain_metadata in domain_metadatas :
2018-01-24 23:09:42 +00:00
2022-01-05 22:15:56 +00:00
if not domain_metadata . HasHeaders ( ) :
2018-01-24 23:09:42 +00:00
2022-01-05 22:15:56 +00:00
continue
2018-01-24 23:09:42 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-01-24 23:09:42 +00:00
2022-01-05 22:15:56 +00:00
domain = domain_metadata . GetDomain ( )
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
network_context = ClientNetworkingContexts . NetworkContext ( CC . NETWORK_CONTEXT_DOMAIN , domain )
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
headers_list = domain_metadata . GetHeaders ( )
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
custom_headers_dict = { key : ( value , approved , reason ) for ( key , value , reason ) in headers_list }
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
self . _network_contexts_to_custom_header_dicts [ network_context ] = custom_headers_dict
2018-02-07 23:40:33 +00:00
2017-12-06 22:06:56 +00:00
2022-01-05 22:15:56 +00:00
def AutoAddURLClassesAndParsers ( self , new_url_classes , dupe_url_classes , new_parsers ) :
2017-12-06 22:06:56 +00:00
2022-01-05 22:15:56 +00:00
for url_class in new_url_classes :
2020-06-17 21:31:54 +00:00
2022-01-05 22:15:56 +00:00
url_class . RegenerateClassKey ( )
2017-11-29 21:48:23 +00:00
2022-01-05 22:15:56 +00:00
for parser in new_parsers :
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
parser . RegenerateParserKey ( )
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
# any existing url matches that already do the job of the new ones should be hung on to but renamed
2018-04-18 22:10:15 +00:00
with self . _lock :
2022-01-05 22:15:56 +00:00
prefix = ' zzz - renamed due to auto-import - '
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
renamees = [ ]
for existing_url_class in self . _url_classes :
if existing_url_class . GetName ( ) . startswith ( prefix ) :
continue
for new_url_class in new_url_classes :
if new_url_class . Matches ( existing_url_class . GetExampleURL ( ) ) and existing_url_class . Matches ( new_url_class . GetExampleURL ( ) ) :
# the url matches match each other, so they are doing the same job
renamees . append ( existing_url_class )
break
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
for renamee in renamees :
existing_names = [ url_class . GetName ( ) for url_class in self . _url_classes if url_class != renamee ]
renamee . SetName ( prefix + renamee . GetName ( ) )
renamee . SetNonDupeName ( existing_names )
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
self . AddURLClasses ( new_url_classes )
self . AddParsers ( new_parsers )
# we want to match these url matches and parsers together if possible
2018-02-07 23:40:33 +00:00
with self . _lock :
2022-01-05 22:15:56 +00:00
url_classes_to_link = list ( new_url_classes )
# if downloader adds existing url match but updated parser, we want to update the existing link
for dupe_url_class in dupe_url_classes :
# this is to make sure we have the right match keys for the link update in a minute
actual_existing_dupe_url_class = self . _GetURLClass ( dupe_url_class . GetExampleURL ( ) )
if actual_existing_dupe_url_class is not None :
url_classes_to_link . append ( actual_existing_dupe_url_class )
new_url_class_keys_to_parser_keys = NetworkDomainManager . STATICLinkURLClassesAndParsers ( url_classes_to_link , new_parsers , { } )
2018-02-07 23:40:33 +00:00
2019-05-08 21:06:42 +00:00
self . _url_class_keys_to_parser_keys . update ( new_url_class_keys_to_parser_keys )
2018-02-07 23:40:33 +00:00
2020-06-17 21:31:54 +00:00
self . _CleanURLClassKeysToParserKeys ( )
2022-01-05 22:15:56 +00:00
# let's do a trytolink just in case there are loose ends due to some dupe being discarded earlier (e.g. url match is new, but parser was not).
self . TryToLinkURLClassesAndParsers ( )
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
def CanValidateInPopup ( self , network_contexts ) :
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
# we can always do this for headers
return True
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
def ConvertURLsToMediaViewerTuples ( self , urls ) :
show_unmatched_urls_in_media_viewer = HG . client_controller . new_options . GetBoolean ( ' show_unmatched_urls_in_media_viewer ' )
url_tuples = [ ]
unmatched_url_tuples = [ ]
2018-04-18 22:10:15 +00:00
with self . _lock :
2022-01-05 22:15:56 +00:00
for url in urls :
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
try :
url_class = self . _GetURLClass ( url )
except HydrusExceptions . URLClassException :
continue
if url_class is None :
if show_unmatched_urls_in_media_viewer :
try :
domain = ClientNetworkingFunctions . ConvertURLIntoDomain ( url )
except HydrusExceptions . URLClassException :
continue
unmatched_url_tuples . append ( ( domain , url ) )
else :
url_class_key = url_class . GetClassKey ( )
if url_class_key in self . _url_class_keys_to_display :
url_class_name = url_class . GetName ( )
url_tuples . append ( ( url_class_name , url ) )
if len ( url_tuples ) == 10 :
break
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
url_tuples . sort ( )
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
unmatched_url_tuples . sort ( )
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
url_tuples . extend ( unmatched_url_tuples )
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
return url_tuples
2018-05-09 20:23:00 +00:00
2022-01-05 22:15:56 +00:00
2023-03-29 20:57:59 +00:00
def DeleteCustomHeader ( self , network_context : ClientNetworkingContexts . NetworkContext , key : str ) :
with self . _lock :
if network_context in self . _network_contexts_to_custom_header_dicts :
custom_header_dict = self . _network_contexts_to_custom_header_dicts [ network_context ]
if key in custom_header_dict :
del custom_header_dict [ key ]
self . _SetDirty ( )
2022-01-05 22:15:56 +00:00
def DeleteGUGs ( self , deletee_names ) :
2018-05-09 20:23:00 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
gugs = [ gug for gug in self . _gugs if gug . GetName ( ) not in deletee_names ]
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
self . SetGUGs ( gugs )
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
2023-06-21 19:50:13 +00:00
def DeleteParsers ( self , deletee_names ) :
with self . _lock :
parsers = [ parser for parser in self . _parsers if parser . GetName ( ) not in deletee_names ]
self . SetParsers ( parsers )
2022-01-05 22:15:56 +00:00
def DeleteURLClasses ( self , deletee_names ) :
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
url_classes = [ url_class for url_class in self . _url_classes if url_class . GetName ( ) not in deletee_names ]
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
self . SetURLClasses ( url_classes )
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
def DissolveParserLink ( self , url_class_name , parser_name ) :
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
the_url_class = None
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
for url_class in self . _url_classes :
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
if url_class . GetName ( ) == url_class_name :
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
the_url_class = url_class
2020-06-17 21:31:54 +00:00
2022-01-05 22:15:56 +00:00
break
the_parser = None
for parser in self . _parsers :
if parser . GetName ( ) == parser_name :
the_parser = parser
break
if the_url_class is not None and the_parser is not None :
url_class_key = the_url_class . GetClassKey ( )
parser_key = the_parser . GetParserKey ( )
if url_class_key in self . _url_class_keys_to_parser_keys and self . _url_class_keys_to_parser_keys [ url_class_key ] == parser_key :
del self . _url_class_keys_to_parser_keys [ url_class_key ]
2018-08-01 20:44:57 +00:00
2018-02-07 23:40:33 +00:00
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
def DomainOK ( self , url ) :
with self . _lock :
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
try :
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
domain = ClientNetworkingFunctions . ConvertURLIntoSecondLevelDomain ( url )
except :
return True
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
# this will become flexible and customisable when I have domain profiles/status/ui
# also should extend it to 'global', so if multiple domains are having trouble, we maybe assume the whole connection is down? it would really be nicer to have a better sockets-level check there
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
if domain in self . _second_level_domains_to_network_infrastructure_errors :
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
number_of_errors = HG . client_controller . new_options . GetInteger ( ' domain_network_infrastructure_error_number ' )
if number_of_errors == 0 :
return True
error_time_delta = HG . client_controller . new_options . GetInteger ( ' domain_network_infrastructure_error_time_delta ' )
network_infrastructure_errors = self . _second_level_domains_to_network_infrastructure_errors [ domain ]
2018-08-01 20:44:57 +00:00
2023-04-19 20:38:13 +00:00
network_infrastructure_errors = [ timestamp for timestamp in network_infrastructure_errors if not HydrusTime . TimeHasPassed ( timestamp + error_time_delta ) ]
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
self . _second_level_domains_to_network_infrastructure_errors [ domain ] = network_infrastructure_errors
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
if len ( network_infrastructure_errors ) > = number_of_errors :
2018-02-07 23:40:33 +00:00
2022-01-05 22:15:56 +00:00
return False
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
elif len ( network_infrastructure_errors ) == 0 :
del self . _second_level_domains_to_network_infrastructure_errors [ domain ]
2018-02-07 23:40:33 +00:00
2018-08-01 20:44:57 +00:00
2022-01-05 22:15:56 +00:00
return True
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
def GenerateValidationPopupProcess ( self , network_contexts ) :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
header_tuples = [ ]
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
for network_context in network_contexts :
if network_context in self . _network_contexts_to_custom_header_dicts :
custom_header_dict = self . _network_contexts_to_custom_header_dicts [ network_context ]
for ( key , ( value , approved , reason ) ) in list ( custom_header_dict . items ( ) ) :
if approved == VALID_UNKNOWN :
header_tuples . append ( ( network_context , key , value , reason ) )
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
process = DomainValidationPopupProcess ( self , header_tuples )
return process
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
def GetDefaultGUGKeyAndName ( self ) :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
gug_key = HG . client_controller . new_options . GetKey ( ' default_gug_key ' )
gug_name = HG . client_controller . new_options . GetString ( ' default_gug_name ' )
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
return ( gug_key , gug_name )
2018-10-03 21:00:15 +00:00
2022-08-17 20:54:59 +00:00
def GetDefaultNoteImportOptions ( self ) :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-10-03 21:00:15 +00:00
2022-08-17 20:54:59 +00:00
return ( self . _file_post_default_note_import_options , self . _watchable_default_note_import_options , self . _url_class_keys_to_default_note_import_options )
def GetDefaultNoteImportOptionsForURL ( self , url ) :
with self . _lock :
return self . _GetDefaultNoteImportOptionsForURL ( url )
2018-10-03 21:00:15 +00:00
2022-08-17 20:54:59 +00:00
def GetDefaultTagImportOptions ( self ) :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2022-08-17 20:54:59 +00:00
return ( self . _file_post_default_tag_import_options , self . _watchable_default_tag_import_options , self . _url_class_keys_to_default_tag_import_options )
2022-01-05 22:15:56 +00:00
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
def GetDefaultTagImportOptionsForURL ( self , url ) :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
return self . _GetDefaultTagImportOptionsForURL ( url )
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
def GetDownloader ( self , url ) :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
# this might be better as getdownloaderkey, but we'll see how it shakes out
# might also be worth being a getifhasdownloader
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
# match the url to a url_class, then lookup that in a 'this downloader can handle this url_class type' dict that we'll manage
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
pass
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
def GetGUG ( self , gug_key_and_name ) :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
return self . _GetGUG ( gug_key_and_name )
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
def GetGUGs ( self ) :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
return list ( self . _gugs )
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
def GetGUGKeysToDisplay ( self ) :
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
return set ( self . _gug_keys_to_display )
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
def GetHeaders ( self , network_contexts ) :
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
headers = { }
for network_context in network_contexts :
if network_context in self . _network_contexts_to_custom_header_dicts :
custom_header_dict = self . _network_contexts_to_custom_header_dicts [ network_context ]
2023-03-29 20:57:59 +00:00
for ( key , ( value , approved , reason ) ) in list ( custom_header_dict . items ( ) ) :
2022-01-05 22:15:56 +00:00
if approved == VALID_APPROVED :
headers [ key ] = value
return headers
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
def GetInitialSearchText ( self , gug_key_and_name ) :
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
gug = self . _GetGUG ( gug_key_and_name )
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
if gug is None :
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
return ' unknown downloader '
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
else :
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
return gug . GetInitialSearchText ( )
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
def GetNetworkContextsToCustomHeaderDicts ( self ) :
with self . _lock :
return dict ( self . _network_contexts_to_custom_header_dicts )
def GetParser ( self , name ) :
with self . _lock :
for parser in self . _parsers :
if parser . GetName ( ) == name :
2017-10-04 17:51:58 +00:00
2022-01-05 22:15:56 +00:00
return parser
2017-10-04 17:51:58 +00:00
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
return None
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
def GetParsers ( self ) :
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
return list ( self . _parsers )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
def GetParserNamespaces ( self ) :
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
return list ( self . _parser_namespaces )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
def GetReferralURL ( self , url , referral_url ) :
with self . _lock :
url_class = self . _GetURLClass ( url )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
if url_class is None :
return referral_url
else :
return url_class . GetReferralURL ( url , referral_url )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
def GetShareableCustomHeaders ( self , network_context ) :
with self . _lock :
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
headers_list = [ ]
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
if network_context in self . _network_contexts_to_custom_header_dicts :
custom_header_dict = self . _network_contexts_to_custom_header_dicts [ network_context ]
2023-03-29 20:57:59 +00:00
for ( key , ( value , approved , reason ) ) in list ( custom_header_dict . items ( ) ) :
2022-01-05 22:15:56 +00:00
2023-03-29 20:57:59 +00:00
if approved == VALID_APPROVED :
headers_list . append ( ( key , value , reason ) )
2022-01-05 22:15:56 +00:00
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
return headers_list
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
def GetURLClass ( self , url ) :
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
return self . _GetURLClass ( url )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
def GetURLClassFromName ( self , name ) :
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
for url_class in self . _url_classes :
if url_class . GetName ( ) == name :
return url_class
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
raise HydrusExceptions . DataMissing ( ' Did not find URL Class called " {} " ! ' . format ( name ) )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
def GetURLClassHeaders ( self , url ) :
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
url_class = self . _GetURLClass ( url )
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
if url_class is not None :
return url_class . GetHeaderOverrides ( )
else :
return { }
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
def GetURLClasses ( self ) :
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
return list ( self . _url_classes )
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
def GetURLClassKeysToParserKeys ( self ) :
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
return dict ( self . _url_class_keys_to_parser_keys )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
def GetURLClassKeysToDisplay ( self ) :
with self . _lock :
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
return set ( self . _url_class_keys_to_display )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
def GetURLParseCapability ( self , url ) :
2020-05-20 21:36:02 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
url_class = self . _GetURLClass ( url )
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
if url_class is None :
return ( HC . URL_TYPE_UNKNOWN , ' unknown url ' , False , ' unknown url class ' )
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
url_type = url_class . GetURLType ( )
match_name = url_class . GetName ( )
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
try :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
( url_to_fetch , parser ) = self . _GetURLToFetchAndParser ( url )
2019-01-09 22:59:03 +00:00
2022-01-05 22:15:56 +00:00
can_parse = True
cannot_parse_reason = ' '
2019-08-15 00:40:48 +00:00
2022-01-05 22:15:56 +00:00
except HydrusExceptions . URLClassException as e :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
can_parse = False
cannot_parse_reason = str ( e )
2018-10-03 21:00:15 +00:00
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
return ( url_type , match_name , can_parse , cannot_parse_reason )
2023-04-26 21:10:03 +00:00
def GetURLToFetch ( self , url ) :
with self . _lock :
url_to_fetch = self . _GetURLToFetch ( url )
if HG . network_report_mode :
HydrusData . ShowText ( ' request for URL to fetch: {} -> {} ' . format ( url , url_to_fetch ) )
return url_to_fetch
2022-01-05 22:15:56 +00:00
def GetURLToFetchAndParser ( self , url ) :
with self . _lock :
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
result = self . _GetURLToFetchAndParser ( url )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
if HG . network_report_mode :
( url_to_fetch , parser ) = result
url_class = self . _GetURLClass ( url )
url_name = url_class . GetName ( )
url_to_fetch_match = self . _GetURLClass ( url_to_fetch )
url_to_fetch_name = url_to_fetch_match . GetName ( )
HydrusData . ShowText ( ' request for URL to fetch and parser: {} ( {} ) -> {} ( {} ): {} ' . format ( url , url_name , url_to_fetch , url_to_fetch_name , parser . GetName ( ) ) )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
return result
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
def HasCustomHeaders ( self , network_context ) :
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
return network_context in self . _network_contexts_to_custom_header_dicts and len ( self . _network_contexts_to_custom_header_dicts [ network_context ] ) > 0
2018-10-03 21:00:15 +00:00
2022-01-05 22:15:56 +00:00
def Initialise ( self ) :
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
self . _RecalcCache ( )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
def IsDirty ( self ) :
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
return self . _dirty
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
def IsValid ( self , network_contexts ) :
2018-08-22 21:10:59 +00:00
2023-03-29 20:57:59 +00:00
# denied headers are simply not added--they don't invalidate a query
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
for network_context in network_contexts :
if network_context in self . _network_contexts_to_custom_header_dicts :
custom_header_dict = self . _network_contexts_to_custom_header_dicts [ network_context ]
for ( value , approved , reason ) in list ( custom_header_dict . values ( ) ) :
if approved == VALID_UNKNOWN :
return False
2018-09-19 21:54:51 +00:00
2022-01-05 22:15:56 +00:00
return True
2018-09-19 21:54:51 +00:00
2022-01-05 22:15:56 +00:00
def NormaliseURL ( self , url ) :
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
url_class = self . _GetURLClass ( url )
if url_class is None :
p = ClientNetworkingFunctions . ParseURL ( url )
scheme = p . scheme
netloc = p . netloc
path = p . path
params = p . params
( query_dict , single_value_parameters , param_order ) = ClientNetworkingFunctions . ConvertQueryTextToDict ( p . query )
query = ClientNetworkingFunctions . ConvertQueryDictToText ( query_dict , single_value_parameters )
fragment = ' '
r = urllib . parse . ParseResult ( scheme , netloc , path , params , query , fragment )
normalised_url = r . geturl ( )
else :
normalised_url = url_class . Normalise ( url )
return normalised_url
2018-08-22 21:10:59 +00:00
2022-12-21 22:00:27 +00:00
def NormaliseURLs ( self , urls : typing . Collection [ str ] ) - > typing . List [ str ] :
normalised_urls = [ ]
for url in urls :
try :
normalised_url = self . NormaliseURL ( url )
except HydrusExceptions . URLClassException :
continue
normalised_urls . append ( url )
normalised_urls = HydrusData . DedupeList ( normalised_urls )
return normalised_urls
2022-01-05 22:15:56 +00:00
def OverwriteDefaultGUGs ( self , gug_names ) :
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
from hydrus . client import ClientDefaults
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
default_gugs = ClientDefaults . GetDefaultGUGs ( )
existing_gug_names_to_keys = { gug . GetName ( ) : gug . GetGUGKey ( ) for gug in self . _gugs }
for gug in default_gugs :
gug_name = gug . GetName ( )
if gug_name in existing_gug_names_to_keys :
gug . SetGUGKey ( existing_gug_names_to_keys [ gug_name ] )
else :
gug . RegenerateGUGKey ( )
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
existing_gugs = list ( self . _gugs )
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
new_gugs = [ gug for gug in existing_gugs if gug . GetName ( ) not in gug_names ]
new_gugs . extend ( [ gug for gug in default_gugs if gug . GetName ( ) in gug_names ] )
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
self . SetGUGs ( new_gugs )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
def OverwriteDefaultParsers ( self , parser_names ) :
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
from hydrus . client import ClientDefaults
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
default_parsers = ClientDefaults . GetDefaultParsers ( )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
existing_parser_names_to_keys = { parser . GetName ( ) : parser . GetParserKey ( ) for parser in self . _parsers }
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
for parser in default_parsers :
name = parser . GetName ( )
if name in existing_parser_names_to_keys :
parser . SetParserKey ( existing_parser_names_to_keys [ name ] )
else :
parser . RegenerateParserKey ( )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
existing_parsers = list ( self . _parsers )
new_parsers = [ parser for parser in existing_parsers if parser . GetName ( ) not in parser_names ]
new_parsers . extend ( [ parser for parser in default_parsers if parser . GetName ( ) in parser_names ] )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
self . SetParsers ( new_parsers )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
def OverwriteDefaultURLClasses ( self , url_class_names ) :
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
from hydrus . client import ClientDefaults
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
default_url_classes = ClientDefaults . GetDefaultURLClasses ( )
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
existing_class_names_to_keys = { url_class . GetName ( ) : url_class . GetClassKey ( ) for url_class in self . _url_classes }
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
for url_class in default_url_classes :
name = url_class . GetName ( )
if name in existing_class_names_to_keys :
url_class . SetClassKey ( existing_class_names_to_keys [ name ] )
else :
url_class . RegenerateClassKey ( )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
for url_class in default_url_classes :
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
url_class . RegenerateClassKey ( )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
existing_url_classes = list ( self . _url_classes )
new_url_classes = [ url_class for url_class in existing_url_classes if url_class . GetName ( ) not in url_class_names ]
new_url_classes . extend ( [ url_class for url_class in default_url_classes if url_class . GetName ( ) in url_class_names ] )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
self . SetURLClasses ( new_url_classes )
2018-08-22 21:10:59 +00:00
2022-01-05 22:15:56 +00:00
def OverwriteParserLink ( self , url_class_name , parser_name ) :
2018-09-19 21:54:51 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-09-19 21:54:51 +00:00
2022-01-05 22:15:56 +00:00
url_class_to_link = None
2018-09-19 21:54:51 +00:00
2022-01-05 22:15:56 +00:00
for url_class in self . _url_classes :
2018-09-19 21:54:51 +00:00
2022-01-05 22:15:56 +00:00
if url_class . GetName ( ) == url_class_name :
url_class_to_link = url_class
break
2018-09-19 21:54:51 +00:00
2022-01-05 22:15:56 +00:00
if url_class_to_link is None :
return False
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
parser_to_link = None
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
for parser in self . _parsers :
if parser . GetName ( ) == parser_name :
parser_to_link = parser
break
if parser_to_link is None :
return False
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
url_class_key = url_class_to_link . GetClassKey ( )
parser_key = parser_to_link . GetParserKey ( )
2021-04-20 22:01:22 +00:00
2022-01-05 22:15:56 +00:00
self . _url_class_keys_to_parser_keys [ url_class_key ] = parser_key
2018-09-05 20:52:32 +00:00
2022-09-07 21:16:25 +00:00
def RenameGUG ( self , original_name , new_name ) :
with self . _lock :
existing_gug_names_to_gugs = { gug . GetName ( ) : gug for gug in self . _gugs }
if original_name in existing_gug_names_to_gugs :
gug = existing_gug_names_to_gugs [ original_name ]
del existing_gug_names_to_gugs [ original_name ]
gug . SetName ( new_name )
gug . SetNonDupeName ( set ( existing_gug_names_to_gugs . keys ( ) ) )
existing_gug_names_to_gugs [ gug . GetName ( ) ] = gug
new_gugs = list ( existing_gug_names_to_gugs . values ( ) )
self . SetGUGs ( new_gugs )
2022-01-05 22:15:56 +00:00
def ReportNetworkInfrastructureError ( self , url ) :
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
try :
domain = ClientNetworkingFunctions . ConvertURLIntoDomain ( url )
except :
return
2023-04-19 20:38:13 +00:00
self . _second_level_domains_to_network_infrastructure_errors [ domain ] . append ( HydrusTime . GetNow ( ) )
2022-01-05 22:15:56 +00:00
2020-12-16 22:29:51 +00:00
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
def ScrubDomainErrors ( self , url ) :
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
try :
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
domain = ClientNetworkingFunctions . ConvertURLIntoSecondLevelDomain ( url )
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
except :
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
return
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
if domain in self . _second_level_domains_to_network_infrastructure_errors :
2018-09-05 20:52:32 +00:00
2022-01-05 22:15:56 +00:00
del self . _second_level_domains_to_network_infrastructure_errors [ domain ]
2018-09-05 20:52:32 +00:00
2020-07-29 20:52:44 +00:00
2022-01-05 22:15:56 +00:00
def SetClean ( self ) :
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
self . _dirty = False
2018-01-17 22:52:10 +00:00
2017-11-29 21:48:23 +00:00
2022-01-05 22:15:56 +00:00
2023-03-29 20:57:59 +00:00
def SetCustomHeader ( self , network_context : ClientNetworkingContexts . NetworkContext , key , value = None , approved = None , reason = None ) :
with self . _lock :
fallback_value = None
fallback_approved = VALID_APPROVED
fallback_reason = ' Set by Client API '
if network_context not in self . _network_contexts_to_custom_header_dicts :
self . _network_contexts_to_custom_header_dicts [ network_context ] = { }
custom_header_dict = self . _network_contexts_to_custom_header_dicts [ network_context ]
if key in custom_header_dict :
( fallback_value , fallback_approved , fallback_reason ) = custom_header_dict [ key ]
if value is None :
if fallback_value is None :
raise Exception ( ' Sorry, was called to set HTTP Header information for key " {} " on " {} " , but there was no attached value and it did not already exist! ' . format ( key , network_context ) )
else :
value = fallback_value
if approved is None :
approved = fallback_approved
if reason is None :
reason = fallback_reason
custom_header_dict [ key ] = ( value , approved , reason )
self . _SetDirty ( )
2022-08-17 20:54:59 +00:00
def SetDefaultFilePostNoteImportOptions ( self , note_import_options ) :
with self . _lock :
self . _file_post_default_note_import_options = note_import_options
self . _SetDirty ( )
2022-01-05 22:15:56 +00:00
def SetDefaultFilePostTagImportOptions ( self , tag_import_options ) :
2017-09-27 21:52:54 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2017-09-27 21:52:54 +00:00
2022-01-05 22:15:56 +00:00
self . _file_post_default_tag_import_options = tag_import_options
2017-09-27 21:52:54 +00:00
2022-08-17 20:54:59 +00:00
self . _SetDirty ( )
2017-09-27 21:52:54 +00:00
2022-01-05 22:15:56 +00:00
def SetDefaultGUGKeyAndName ( self , gug_key_and_name ) :
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2017-09-27 21:52:54 +00:00
2022-01-05 22:15:56 +00:00
( gug_key , gug_name ) = gug_key_and_name
2020-11-11 22:20:16 +00:00
2022-01-05 22:15:56 +00:00
HG . client_controller . new_options . SetKey ( ' default_gug_key ' , gug_key )
HG . client_controller . new_options . SetString ( ' default_gug_name ' , gug_name )
2020-11-11 22:20:16 +00:00
2022-01-05 22:15:56 +00:00
2022-08-17 20:54:59 +00:00
def SetDefaultNoteImportOptions ( self , file_post_default_note_import_options , watchable_default_note_import_options , url_class_keys_to_note_import_options ) :
with self . _lock :
self . _file_post_default_note_import_options = file_post_default_note_import_options
self . _watchable_default_note_import_options = watchable_default_note_import_options
self . _url_class_keys_to_default_note_import_options = url_class_keys_to_note_import_options
self . _SetDirty ( )
2022-01-05 22:15:56 +00:00
def SetDefaultTagImportOptions ( self , file_post_default_tag_import_options , watchable_default_tag_import_options , url_class_keys_to_tag_import_options ) :
with self . _lock :
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
self . _file_post_default_tag_import_options = file_post_default_tag_import_options
self . _watchable_default_tag_import_options = watchable_default_tag_import_options
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
self . _url_class_keys_to_default_tag_import_options = url_class_keys_to_tag_import_options
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
self . _SetDirty ( )
2019-10-09 22:03:03 +00:00
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
def SetGUGs ( self , gugs ) :
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
# by default, we will show new gugs
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
old_gug_keys = { gug . GetGUGKey ( ) for gug in self . _gugs }
gug_keys = { gug . GetGUGKey ( ) for gug in gugs }
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
added_gug_keys = gug_keys . difference ( old_gug_keys )
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
self . _gug_keys_to_display . update ( added_gug_keys )
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
#
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
self . _gugs = HydrusSerialisable . SerialisableList ( gugs )
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
self . _RecalcCache ( )
self . _SetDirty ( )
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
def SetGUGKeysToDisplay ( self , gug_keys_to_display ) :
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
self . _gug_keys_to_display = set ( )
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
self . _gug_keys_to_display . update ( gug_keys_to_display )
self . _SetDirty ( )
2018-08-29 20:20:41 +00:00
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
def SetHeaderValidation ( self , network_context , key , approved ) :
with self . _lock :
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
if network_context in self . _network_contexts_to_custom_header_dicts :
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
custom_header_dict = self . _network_contexts_to_custom_header_dicts [ network_context ]
if key in custom_header_dict :
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
( value , old_approved , reason ) = custom_header_dict [ key ]
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
custom_header_dict [ key ] = ( value , approved , reason )
2021-09-29 21:20:29 +00:00
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
self . _SetDirty ( )
2021-09-29 21:20:29 +00:00
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
def SetNetworkContextsToCustomHeaderDicts ( self , network_contexts_to_custom_header_dicts ) :
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
self . _network_contexts_to_custom_header_dicts = network_contexts_to_custom_header_dicts
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
self . _SetDirty ( )
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
def SetParsers ( self , parsers ) :
with self . _lock :
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
self . _parsers = HydrusSerialisable . SerialisableList ( )
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
self . _parsers . extend ( parsers )
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
self . _parsers . sort ( key = lambda p : p . GetName ( ) )
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
# delete orphans
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
parser_keys = { parser . GetParserKey ( ) for parser in parsers }
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
deletee_url_class_keys = set ( )
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
for ( url_class_key , parser_key ) in self . _url_class_keys_to_parser_keys . items ( ) :
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
if parser_key not in parser_keys :
deletee_url_class_keys . add ( url_class_key )
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
for deletee_url_class_key in deletee_url_class_keys :
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
del self . _url_class_keys_to_parser_keys [ deletee_url_class_key ]
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
#
2018-05-09 20:23:00 +00:00
2022-01-05 22:15:56 +00:00
self . _RecalcCache ( )
2018-05-09 20:23:00 +00:00
2022-01-05 22:15:56 +00:00
self . _SetDirty ( )
2018-05-09 20:23:00 +00:00
2022-01-05 22:15:56 +00:00
def SetURLClasses ( self , url_classes ) :
with self . _lock :
# by default, we will show post urls
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
old_post_url_class_keys = { url_class . GetClassKey ( ) for url_class in self . _url_classes if url_class . IsPostURL ( ) }
post_url_class_keys = { url_class . GetClassKey ( ) for url_class in url_classes if url_class . IsPostURL ( ) }
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
added_post_url_class_keys = post_url_class_keys . difference ( old_post_url_class_keys )
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
self . _url_class_keys_to_display . update ( added_post_url_class_keys )
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
#
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
self . _url_classes = HydrusSerialisable . SerialisableList ( )
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
self . _url_classes . extend ( url_classes )
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
self . _url_classes . sort ( key = lambda u : u . GetName ( ) )
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
#
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
# delete orphans
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
url_class_keys = { url_class . GetClassKey ( ) for url_class in url_classes }
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
self . _url_class_keys_to_display . intersection_update ( url_class_keys )
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
for deletee_key in set ( self . _url_class_keys_to_parser_keys . keys ( ) ) . difference ( url_class_keys ) :
del self . _url_class_keys_to_parser_keys [ deletee_key ]
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
# any url matches that link to another via the API conversion will not be using parsers
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
url_class_api_pairs = ClientNetworkingURLClass . ConvertURLClassesIntoAPIPairs ( self . _url_classes )
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
for ( url_class_original , url_class_api ) in url_class_api_pairs :
url_class_key = url_class_original . GetClassKey ( )
if url_class_key in self . _url_class_keys_to_parser_keys :
del self . _url_class_keys_to_parser_keys [ url_class_key ]
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
self . _RecalcCache ( )
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
self . _SetDirty ( )
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
def SetURLClassKeysToParserKeys ( self , url_class_keys_to_parser_keys ) :
with self . _lock :
2019-11-28 01:11:46 +00:00
2022-01-05 22:15:56 +00:00
self . _url_class_keys_to_parser_keys = HydrusSerialisable . SerialisableBytesDictionary ( )
2019-11-28 01:11:46 +00:00
2022-01-05 22:15:56 +00:00
self . _url_class_keys_to_parser_keys . update ( url_class_keys_to_parser_keys )
2019-11-28 01:11:46 +00:00
2022-01-05 22:15:56 +00:00
self . _CleanURLClassKeysToParserKeys ( )
2019-11-28 01:11:46 +00:00
2022-01-05 22:15:56 +00:00
self . _SetDirty ( )
2019-11-28 01:11:46 +00:00
2022-01-05 22:15:56 +00:00
def SetURLClassKeysToDisplay ( self , url_class_keys_to_display ) :
with self . _lock :
2020-07-29 20:52:44 +00:00
2022-01-05 22:15:56 +00:00
self . _url_class_keys_to_display = set ( )
2020-07-29 20:52:44 +00:00
2022-01-05 22:15:56 +00:00
self . _url_class_keys_to_display . update ( url_class_keys_to_display )
2020-07-29 20:52:44 +00:00
2022-01-05 22:15:56 +00:00
self . _SetDirty ( )
2020-07-29 20:52:44 +00:00
2022-01-05 22:15:56 +00:00
def ShouldAssociateURLWithFiles ( self , url ) :
with self . _lock :
2020-11-11 22:20:16 +00:00
2022-01-05 22:15:56 +00:00
url_class = self . _GetURLClass ( url )
2020-11-11 22:20:16 +00:00
2022-01-05 22:15:56 +00:00
if url_class is None :
return True
2020-11-11 22:20:16 +00:00
2022-01-05 22:15:56 +00:00
return url_class . ShouldAssociateWithFiles ( )
2020-11-11 22:20:16 +00:00
2022-01-05 22:15:56 +00:00
def TryToLinkURLClassesAndParsers ( self ) :
with self . _lock :
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
new_url_class_keys_to_parser_keys = NetworkDomainManager . STATICLinkURLClassesAndParsers ( self . _url_classes , self . _parsers , self . _url_class_keys_to_parser_keys )
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
self . _url_class_keys_to_parser_keys . update ( new_url_class_keys_to_parser_keys )
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
self . _CleanURLClassKeysToParserKeys ( )
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
self . _SetDirty ( )
2021-09-29 21:20:29 +00:00
2019-11-28 01:11:46 +00:00
2022-01-05 22:15:56 +00:00
def URLCanReferToMultipleFiles ( self , url ) :
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
url_class = self . _GetURLClass ( url )
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
if url_class is None :
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
return False
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
return url_class . CanReferToMultipleFiles ( )
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
def URLDefinitelyRefersToOneFile ( self , url ) :
2018-05-30 20:13:21 +00:00
2022-01-05 22:15:56 +00:00
with self . _lock :
2018-05-30 20:13:21 +00:00
2022-01-05 22:15:56 +00:00
url_class = self . _GetURLClass ( url )
if url_class is None :
return False
return url_class . RefersToOneFile ( )
2018-05-30 20:13:21 +00:00
2017-11-29 21:48:23 +00:00
2022-01-05 22:15:56 +00:00
@staticmethod
def STATICLinkURLClassesAndParsers ( url_classes , parsers , existing_url_class_keys_to_parser_keys ) :
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
url_classes = list ( url_classes )
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
ClientNetworkingURLClass . SortURLClassesListDescendingComplexity ( url_classes )
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
parsers = list ( parsers )
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
parsers . sort ( key = lambda p : p . GetName ( ) )
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
new_url_class_keys_to_parser_keys = { }
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
api_pairs = ClientNetworkingURLClass . ConvertURLClassesIntoAPIPairs ( url_classes )
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
# anything that goes to an api url will be parsed by that api's parser--it can't have its own
api_pair_unparsable_url_classes = set ( )
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
for ( a , b ) in api_pairs :
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
api_pair_unparsable_url_classes . add ( a )
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
#
2018-08-15 20:40:30 +00:00
2022-01-05 22:15:56 +00:00
# I have to do this backwards, going through parsers and then url_classes, so I can do a proper url match lookup like the real domain manager does it
# otherwise, if we iterate through url matches looking for parsers to match them, we have gallery url matches thinking they match parser post urls
# e.g.
# The page parser might say it supports https://danbooru.donmai.us/posts/3198277
# But the gallery url class might think it recognises that as https://danbooru.donmai.us/posts
#
# So we have to do the normal lookup in the proper descending complexity order, not searching any further than the first, correct match
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
for parser in parsers :
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
example_urls = parser . GetExampleURLs ( )
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
for example_url in example_urls :
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
for url_class in url_classes :
if url_class in api_pair_unparsable_url_classes :
continue
if url_class . Matches ( example_url ) :
# we have a match. this is the 'correct' match for this example url, and we should not search any more, so we break below
url_class_key = url_class . GetClassKey ( )
parsable = url_class . IsParsable ( )
linkable = url_class_key not in existing_url_class_keys_to_parser_keys and url_class_key not in new_url_class_keys_to_parser_keys
if parsable and linkable :
new_url_class_keys_to_parser_keys [ url_class_key ] = parser . GetParserKey ( )
break
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
'''
#
for url_class in url_classes :
if not url_class . IsParsable ( ) or url_class in api_pair_unparsable_url_classes :
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
continue
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
url_class_key = url_class . GetClassKey ( )
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
if url_class_key in existing_url_class_keys_to_parser_keys :
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
continue
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
for parser in parsers :
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
example_urls = parser . GetExampleURLs ( )
if True in ( url_class . Matches ( example_url ) for example_url in example_urls ) :
new_url_class_keys_to_parser_keys [ url_class_key ] = parser . GetParserKey ( )
break
2019-10-09 22:03:03 +00:00
2022-01-05 22:15:56 +00:00
'''
return new_url_class_keys_to_parser_keys
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
HydrusSerialisable . SERIALISABLE_TYPES_TO_OBJECT_TYPES [ HydrusSerialisable . SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER ] = NetworkDomainManager
class DomainMetadataPackage ( HydrusSerialisable . SerialisableBase ) :
2021-02-24 22:35:18 +00:00
2022-01-05 22:15:56 +00:00
SERIALISABLE_TYPE = HydrusSerialisable . SERIALISABLE_TYPE_DOMAIN_METADATA_PACKAGE
SERIALISABLE_NAME = ' Domain Metadata '
SERIALISABLE_VERSION = 1
2019-11-28 01:11:46 +00:00
2022-01-05 22:15:56 +00:00
def __init__ ( self , domain = None , headers_list = None , bandwidth_rules = None ) :
2017-11-29 21:48:23 +00:00
2022-01-05 22:15:56 +00:00
HydrusSerialisable . SerialisableBase . __init__ ( self )
2017-11-29 21:48:23 +00:00
2022-01-05 22:15:56 +00:00
if domain is None :
domain = ' example.com '
2017-12-06 22:06:56 +00:00
2022-01-05 22:15:56 +00:00
self . _domain = domain
self . _headers_list = headers_list
self . _bandwidth_rules = bandwidth_rules
2017-12-06 22:06:56 +00:00
2022-01-05 22:15:56 +00:00
def _GetSerialisableInfo ( self ) :
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
if self . _bandwidth_rules is None :
serialisable_bandwidth_rules = self . _bandwidth_rules
else :
serialisable_bandwidth_rules = self . _bandwidth_rules . GetSerialisableTuple ( )
2017-12-06 22:06:56 +00:00
2022-01-05 22:15:56 +00:00
return ( self . _domain , self . _headers_list , serialisable_bandwidth_rules )
2017-12-06 22:06:56 +00:00
2022-01-05 22:15:56 +00:00
def _InitialiseFromSerialisableInfo ( self , serialisable_info ) :
2017-12-13 22:33:07 +00:00
2022-01-05 22:15:56 +00:00
( self . _domain , self . _headers_list , serialisable_bandwidth_rules ) = serialisable_info
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
if serialisable_bandwidth_rules is None :
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
self . _bandwidth_rules = serialisable_bandwidth_rules
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
else :
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
self . _bandwidth_rules = HydrusSerialisable . CreateFromSerialisableTuple ( serialisable_bandwidth_rules )
2018-01-17 22:52:10 +00:00
2019-12-11 23:18:37 +00:00
2022-01-05 22:15:56 +00:00
def GetBandwidthRules ( self ) :
2019-12-11 23:18:37 +00:00
2022-01-05 22:15:56 +00:00
return self . _bandwidth_rules
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
def GetDetailedSafeSummary ( self ) :
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
components = [ ' For domain " ' + self . _domain + ' " : ' ]
2020-07-29 20:52:44 +00:00
2022-01-05 22:15:56 +00:00
if self . HasBandwidthRules ( ) :
2020-07-29 20:52:44 +00:00
2022-01-05 22:15:56 +00:00
m = ' Bandwidth rules: '
m + = os . linesep
m + = os . linesep . join ( [ HydrusNetworking . ConvertBandwidthRuleToString ( rule ) for rule in self . _bandwidth_rules . GetRules ( ) ] )
2020-07-29 20:52:44 +00:00
2022-01-05 22:15:56 +00:00
components . append ( m )
2020-07-29 20:52:44 +00:00
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
if self . HasHeaders ( ) :
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
m = ' Headers: '
m + = os . linesep
m + = os . linesep . join ( [ key + ' : ' + value + ' - ' + reason for ( key , value , reason ) in self . _headers_list ] )
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
components . append ( m )
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
joiner = os . linesep * 2
s = joiner . join ( components )
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
return s
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
def GetDomain ( self ) :
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
return self . _domain
2018-04-25 22:07:52 +00:00
2022-01-05 22:15:56 +00:00
def GetHeaders ( self ) :
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
return self . _headers_list
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
def GetSafeSummary ( self ) :
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
components = [ ]
2018-01-17 22:52:10 +00:00
2022-01-05 22:15:56 +00:00
if self . HasBandwidthRules ( ) :
components . append ( ' bandwidth rules ' )
if self . HasHeaders ( ) :
components . append ( ' headers ' )
2018-09-19 21:54:51 +00:00
2022-01-05 22:15:56 +00:00
return ' and ' . join ( components ) + ' - ' + self . _domain
2018-09-19 21:54:51 +00:00
2022-01-05 22:15:56 +00:00
def HasBandwidthRules ( self ) :
2018-09-19 21:54:51 +00:00
2022-01-05 22:15:56 +00:00
return self . _bandwidth_rules is not None
2018-09-19 21:54:51 +00:00
2022-01-05 22:15:56 +00:00
def HasHeaders ( self ) :
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
return self . _headers_list is not None
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
HydrusSerialisable . SERIALISABLE_TYPES_TO_OBJECT_TYPES [ HydrusSerialisable . SERIALISABLE_TYPE_DOMAIN_METADATA_PACKAGE ] = DomainMetadataPackage
class DomainValidationPopupProcess ( object ) :
def __init__ ( self , domain_manager , header_tuples ) :
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
self . _domain_manager = domain_manager
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
self . _header_tuples = header_tuples
2019-11-28 01:11:46 +00:00
2022-01-05 22:15:56 +00:00
self . _is_done = False
2019-11-28 01:11:46 +00:00
2022-01-05 22:15:56 +00:00
def IsDone ( self ) :
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
return self . _is_done
2018-04-18 22:10:15 +00:00
2022-01-05 22:15:56 +00:00
def Start ( self ) :
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
try :
results = [ ]
2017-11-29 21:48:23 +00:00
2022-01-05 22:15:56 +00:00
for ( network_context , key , value , reason ) in self . _header_tuples :
2017-11-29 21:48:23 +00:00
2022-01-05 22:15:56 +00:00
job_key = ClientThreading . JobKey ( )
2017-11-29 21:48:23 +00:00
2022-01-05 22:15:56 +00:00
# generate question
2017-11-29 21:48:23 +00:00
2022-01-05 22:15:56 +00:00
question = ' For the network context ' + network_context . ToString ( ) + ' , can the client set this header? '
question + = os . linesep * 2
question + = key + ' : ' + value
question + = os . linesep * 2
question + = reason
2017-11-29 21:48:23 +00:00
2022-01-05 22:15:56 +00:00
job_key . SetVariable ( ' popup_yes_no_question ' , question )
HG . client_controller . pub ( ' message ' , job_key )
2017-11-22 21:03:07 +00:00
2022-01-05 22:15:56 +00:00
result = job_key . GetIfHasVariable ( ' popup_yes_no_answer ' )
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
while result is None :
2018-08-29 20:20:41 +00:00
2022-01-19 21:28:59 +00:00
if HG . started_shutdown :
2022-01-05 22:15:56 +00:00
return
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
time . sleep ( 0.25 )
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
result = job_key . GetIfHasVariable ( ' popup_yes_no_answer ' )
2018-08-29 20:20:41 +00:00
2017-11-22 21:03:07 +00:00
2022-01-05 22:15:56 +00:00
if result :
2018-08-29 20:20:41 +00:00
2022-01-05 22:15:56 +00:00
approved = VALID_APPROVED
2018-08-29 20:20:41 +00:00
else :
2022-01-05 22:15:56 +00:00
approved = VALID_DENIED
2018-08-29 20:20:41 +00:00
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
self . _domain_manager . SetHeaderValidation ( network_context , key , approved )
2017-09-13 20:50:41 +00:00
2022-01-05 22:15:56 +00:00
finally :
2021-09-29 21:20:29 +00:00
2022-01-05 22:15:56 +00:00
self . _is_done = True
2021-09-29 21:20:29 +00:00