RepoMirrors
/
hydrus
mirror of https://github.com/hydrusnetwork/hydrus.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Version 413 

closes #296
This commit is contained in:
Hydrus Network Developer 2020-09-23 16:02:02 -05:00
parent ecde393b54
commit 9fbed11bef
22 changed files with 862 additions and 274 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
help/changelog.html
View File

@ -8,6 +8,31 @@
<div class="content">
<h3>changelog</h3>
<ul>
<li><h3>version 413</h3></li>
<ul>
<li>added 'sort by number of files in collection' file sort type. it obviously only does anything interesting if you are collecting by something</li>
<li>when you enter a tag from a manage tags suggested tags column with a double-click, the tag input box is now immediately focused. entering it with a keyboard action does not move the focus</li>
<li>wrote a new routine for the 'check and repair' database menu that scans for and fixes invalid tags. this might be some system:tag that snuck in, superfluous unicode whitespace, or some weird website encoding that results in null characters, or any other old tag that has since become invalid. tag translations are written to the log</li>
<li>added an experimental 'post_index' CONTEXT VARIABLE to subsidiary page parsers--whenever a non-vetoed post has pursuable URLs, this value is incremented by one. this is an attempt to generate a # 0,1,2,3 series. feedback on this would be appreciated, so I can formalise and document it</li>
<li>added 'no_proxy' option for the options->connection page. it uses comma-separated host/domains, just like for curl or the NO_PROXY environment variable. it defaults to 127.0.0.1. in future, options will be added to auto-inherit proxy info from environment variables</li>
<li>fixed an error when subscriptions try to publish to a page name when a 'page of pages' already has that name</li>
<li>activated some old 'clean url' parsing tech I wrote but never plugged in that helps parsing urls from source fields on sites that start with non-url gubbins</li>
<li>fixed the v411->v412 update step to account for a tags table that has duplicate entries (this shouldn't ever happen, but it seems some legacy bug or storage conversion indicent may have caused this for some users). if a unique constraint error is raised, the update step now gives a little message box and does dedupe work</li>
<li>fixed an issue where the 'will display as' tag was rendering without namespace when 'hide namespace in normal views' was on</li>
<li>fixed a recent character encoding routine that was supposed to filter out null characters</li>
<li>fixed some UPnP error reporting</li>
<li>_may_ have fixed an odd and seemingly rare 'paintevent' issue when expanding the popup toaster from collapsed state--it may also have been a qt bug, and fixed in the new qt:</li>
<li>updated qt to 5.15.1 for windows and linux builds. it fixes a couple of odd issues like 'unclicking' to select a menu item (issue #296)</li>
<li>added session save to holistic ui test suite</li>
<li>misc code cleanup</li>
<li>.</li>
<li>client api:</li>
<li>wrote a client test for the help menu so I can test some basic functions holistically, hoping to stop some recent typo bugs from happening again</li>
<li>did a couple of hotfixes for v412 to deal with some client api url pending bugs. the links in the 412 release now point to new fixed builds</li>
<li>fixed an issue setting additional tags via the client api when the respective service's tag import options are not set to get anything</li>
<li>fixed a 500 error with /add_tags/add_tags when a tags parameter is an empty list</li>
<li>fixed the /manage_pages/get_page_info client api help to show the 'page_info' key in the example response</li>
</ul>
<li><h3>version 412</h3></li>
<ul>
<li>client api:</li>

112
help/client_api.html
View File

@ -748,65 +748,67 @@
<ul>
<li>
<pre>{
"name" : "threads",
"page_key" : "aebbf4b594e6986bddf1eeb0b5846a1e6bc4e07088e517aff166f1aeb1c3c9da",
"page_type" : 3,
"management" : {
"multiple_watcher_import" : {
"watcher_imports" : [
{
"url" : "https://someimageboard.net/m/123456",
"watcher_key" = "cf8c3525c57a46b0e5c2625812964364a2e801f8c49841c216b8f8d7a4d06d85",
"created" = 1566164269,
"last_check_time" = 1566164272,
"next_check_time" = 1566174272,
"files_paused" = false,
"checking_paused" = false,
"checking_status" = 0,
"subject" = "gundam pictures",
"imports" : {
"status" : "4 successful (2 already in db)",
"simple_status" : "4",
"total_processed" : 4,
"total_to_process" : 4
"page_info" : {
"name" : "threads",
"page_key" : "aebbf4b594e6986bddf1eeb0b5846a1e6bc4e07088e517aff166f1aeb1c3c9da",
"page_type" : 3,
"management" : {
"multiple_watcher_import" : {
"watcher_imports" : [
{
"url" : "https://someimageboard.net/m/123456",
"watcher_key" = "cf8c3525c57a46b0e5c2625812964364a2e801f8c49841c216b8f8d7a4d06d85",
"created" = 1566164269,
"last_check_time" = 1566164272,
"next_check_time" = 1566174272,
"files_paused" = false,
"checking_paused" = false,
"checking_status" = 0,
"subject" = "gundam pictures",
"imports" : {
"status" : "4 successful (2 already in db)",
"simple_status" : "4",
"total_processed" : 4,
"total_to_process" : 4
},
"gallery_log" : {
"status" = "1 successful",
"simple_status" = "1",
"total_processed" = 1,
"total_to_process" = 1
}
},
"gallery_log" : {
"status" = "1 successful",
"simple_status" = "1",
"total_processed" = 1,
"total_to_process" = 1
{
"url" : "https://someimageboard.net/a/1234",
"watcher_key" = "6bc17555b76da5bde2dcceedc382cf7d23281aee6477c41b643cd144ec168510",
"created" = 1566063125,
"last_check_time" = 1566063133,
"next_check_time" = 1566104272,
"files_paused" = false,
"checking_paused" = true,
"checking_status" = 1,
"subject" = "anime pictures",
"imports" : {
"status" : "124 successful (22 already in db), 2 previously deleted",
"simple_status" : "124",
"total_processed" : 124,
"total_to_process" : 124
},
"gallery_log" : {
"status" = "3 successful",
"simple_status" = "3",
"total_processed" = 3,
"total_to_process" = 3
}
}
]
},
{
"url" : "https://someimageboard.net/a/1234",
"watcher_key" = "6bc17555b76da5bde2dcceedc382cf7d23281aee6477c41b643cd144ec168510",
"created" = 1566063125,
"last_check_time" = 1566063133,
"next_check_time" = 1566104272,
"files_paused" = false,
"checking_paused" = true,
"checking_status" = 1,
"subject" = "anime pictures",
"imports" : {
"status" : "124 successful (22 already in db), 2 previously deleted",
"simple_status" : "124",
"total_processed" : 124,
"total_to_process" : 124
},
"gallery_log" : {
"status" = "3 successful",
"simple_status" = "3",
"total_processed" = 3,
"total_to_process" = 3
}
}
]
},
"highlight" : "cf8c3525c57a46b0e5c2625812964364a2e801f8c49841c216b8f8d7a4d06d85"
"highlight" : "cf8c3525c57a46b0e5c2625812964364a2e801f8c49841c216b8f8d7a4d06d85"
}
},
"media" : {
"num_files" : 4
}
},
"media" : {
"num_files" : 4
}
}</pre>
</li>

4
hydrus/client/ClientConstants.py
View File

@ -290,6 +290,7 @@ SORT_FILES_BY_HAS_AUDIO = 13
SORT_FILES_BY_FILE_MODIFIED_TIMESTAMP = 14
SORT_FILES_BY_FRAMERATE = 15
SORT_FILES_BY_NUM_FRAMES = 16
SORT_FILES_BY_NUM_COLLECTION_FILES = 17
SYSTEM_SORT_TYPES = []
@ -299,6 +300,7 @@ SYSTEM_SORT_TYPES.append( SORT_FILES_BY_RATIO )
SYSTEM_SORT_TYPES.append( SORT_FILES_BY_NUM_PIXELS )
SYSTEM_SORT_TYPES.append( SORT_FILES_BY_DURATION )
SYSTEM_SORT_TYPES.append( SORT_FILES_BY_FRAMERATE )
SYSTEM_SORT_TYPES.append( SORT_FILES_BY_NUM_COLLECTION_FILES )
SYSTEM_SORT_TYPES.append( SORT_FILES_BY_NUM_FRAMES )
SYSTEM_SORT_TYPES.append( SORT_FILES_BY_FILESIZE )
SYSTEM_SORT_TYPES.append( SORT_FILES_BY_IMPORT_TIME )
@ -313,6 +315,7 @@ SYSTEM_SORT_TYPES.append( SORT_FILES_BY_MEDIA_VIEWTIME )
system_sort_type_submetatype_string_lookup = {}
system_sort_type_submetatype_string_lookup[ SORT_FILES_BY_NUM_COLLECTION_FILES ] = 'collections'
system_sort_type_submetatype_string_lookup[ SORT_FILES_BY_HEIGHT ] = 'dimensions'
system_sort_type_submetatype_string_lookup[ SORT_FILES_BY_NUM_PIXELS ] = 'dimensions'
system_sort_type_submetatype_string_lookup[ SORT_FILES_BY_RATIO ] = 'dimensions'
@ -337,6 +340,7 @@ sort_type_basic_string_lookup[ SORT_FILES_BY_DURATION ] = 'duration'
sort_type_basic_string_lookup[ SORT_FILES_BY_FRAMERATE ] = 'framerate'
sort_type_basic_string_lookup[ SORT_FILES_BY_NUM_FRAMES ] = 'number of frames'
sort_type_basic_string_lookup[ SORT_FILES_BY_HEIGHT ] = 'height'
sort_type_basic_string_lookup[ SORT_FILES_BY_NUM_COLLECTION_FILES ] = 'number of files in collection'
sort_type_basic_string_lookup[ SORT_FILES_BY_NUM_PIXELS ] = 'number of pixels'
sort_type_basic_string_lookup[ SORT_FILES_BY_RATIO ] = 'resolution ratio'
sort_type_basic_string_lookup[ SORT_FILES_BY_WIDTH ] = 'width'

173
hydrus/client/ClientDB.py
View File

@ -1749,8 +1749,6 @@ class DB( HydrusDB.HydrusDB ):
with HydrusDB.TemporaryIntegerTable( self._c, other_chain_tag_ids, 'tag_id' ) as temp_tag_ids_table_name:
self._AnalyzeTempTable( temp_tag_ids_table_name )
# although this is mickey-mouse, it does work, and real fast
# temp hashes to mappings to temp tags
other_chain_hash_ids = self._STL( self._c.execute( 'SELECT hash_id FROM {} WHERE EXISTS ( SELECT 1 FROM {} CROSS JOIN {} USING ( tag_id ) WHERE {}.hash_id = {}.hash_id );'.format( temp_hash_ids_table_name, mappings_table_name, temp_tag_ids_table_name, mappings_table_name, temp_hash_ids_table_name ) ) )
@ -1938,11 +1936,11 @@ class DB( HydrusDB.HydrusDB ):
self._c.executemany( 'DELETE FROM ' + cache_display_pending_mappings_table_name + ' WHERE hash_id = ? AND tag_id = ?;', ( ( hash_id, ideal_tag_id ) for hash_id in deletable_hash_ids ) )
num_deleted = self._GetRowCount()
num_rescinded = self._GetRowCount()
if num_deleted > 0:
if num_rescinded > 0:
self._c.execute( 'UPDATE ' + specific_display_ac_cache_table_name + ' SET pending_count = pending_count - ? WHERE tag_id = ?;', ( num_deleted, ideal_tag_id ) )
self._c.execute( 'UPDATE ' + specific_display_ac_cache_table_name + ' SET pending_count = pending_count - ? WHERE tag_id = ?;', ( num_rescinded, ideal_tag_id ) )
self._c.execute( 'DELETE FROM ' + specific_display_ac_cache_table_name + ' WHERE tag_id = ? AND current_count = ? AND pending_count = ?;', ( ideal_tag_id, 0, 0 ) )
@ -2455,12 +2453,14 @@ class DB( HydrusDB.HydrusDB ):
tag_ids_to_ideal_tag_ids = self._CacheTagSiblingsLookupGetIdeals( tag_service_id, set( tag_ids ) )
sibling_tag_ids = set( tag_ids_to_ideal_tag_ids.values() )
ideal_tag_ids = set( tag_ids_to_ideal_tag_ids.values() )
with HydrusDB.TemporaryIntegerTable( self._c, sibling_tag_ids, 'tag_id' ) as temp_table_name:
sibling_tag_ids = set( ideal_tag_ids )
with HydrusDB.TemporaryIntegerTable( self._c, ideal_tag_ids, 'ideal_tag_id' ) as temp_table_name:
# temp tags to lookup
sibling_tag_ids.update( self._STI( self._c.execute( 'SELECT bad_tag_id FROM {} CROSS JOIN {} ON ( ideal_tag_id = tag_id );'.format( temp_table_name, cache_tag_siblings_lookup_table_name ) ) ) )
sibling_tag_ids.update( self._STI( self._c.execute( 'SELECT bad_tag_id FROM {} CROSS JOIN {} USING ( ideal_tag_id );'.format( temp_table_name, cache_tag_siblings_lookup_table_name ) ) ) )
return sibling_tag_ids
@ -2488,9 +2488,7 @@ class DB( HydrusDB.HydrusDB ):
with HydrusDB.TemporaryIntegerTable( self._c, ideal_tag_ids, 'ideal_tag_id' ) as temp_table_name:
# temp tags to lookup
bads_and_ideals = self._c.execute( 'SELECT bad_tag_id, ideal_tag_id FROM {} CROSS JOIN {} USING ( ideal_tag_id );'.format( temp_table_name, cache_tag_siblings_lookup_table_name ) )
ideal_tag_ids_to_chain_members = HydrusData.BuildKeyToSetDict( ( ( ideal_tag_id, bad_tag_id ) for ( bad_tag_id, ideal_tag_id ) in bads_and_ideals ) )
ideal_tag_ids_to_chain_members = HydrusData.BuildKeyToSetDict( self._c.execute( 'SELECT ideal_tag_id, bad_tag_id FROM {} CROSS JOIN {} USING ( ideal_tag_id );'.format( temp_table_name, cache_tag_siblings_lookup_table_name ) ) )
# this returns ideal in the chain, and chains of size 1
@ -13757,6 +13755,118 @@ class DB( HydrusDB.HydrusDB ):
def _RepairInvalidTags( self, job_key: typing.Optional[ ClientThreading.JobKey ] = None ):
invalid_tag_ids_and_tags = set()
BLOCK_SIZE = 1000
select_statement = 'SELECT tag_id FROM tags;'
bad_tag_count = 0
for ( i, group_of_tag_ids ) in enumerate( HydrusDB.ReadLargeIdQueryInSeparateChunks( self._c, select_statement, BLOCK_SIZE ) ):
if job_key is not None:
if job_key.IsCancelled():
break
message = 'Scanning tags: {} - Bad Found: {}'.format( HydrusData.ToHumanInt( i * BLOCK_SIZE ), HydrusData.ToHumanInt( bad_tag_count ) )
job_key.SetVariable( 'popup_text_1', message )
for tag_id in group_of_tag_ids:
tag = self._GetTag( tag_id )
try:
cleaned_tag = HydrusTags.CleanTag( tag )
except:
cleaned_tag = 'unrecoverable invalid tag'
if tag != cleaned_tag:
invalid_tag_ids_and_tags.add( ( tag_id, tag, cleaned_tag ) )
bad_tag_count += 1
for ( i, ( tag_id, tag, cleaned_tag ) ) in enumerate( invalid_tag_ids_and_tags ):
if job_key is not None:
if job_key.IsCancelled():
break
message = 'Fixing bad tags: {}'.format( HydrusData.ConvertValueRangeToPrettyString( i + 1, bad_tag_count ) )
job_key.SetVariable( 'popup_text_1', message )
existing_tags = set()
potential_new_cleaned_tag = cleaned_tag
while self._TagExists( potential_new_cleaned_tag ):
existing_tags.add( potential_new_cleaned_tag )
potential_new_cleaned_tag = HydrusData.GetNonDupeName( cleaned_tag, existing_tags )
cleaned_tag = potential_new_cleaned_tag
( namespace, subtag ) = HydrusTags.SplitTag( cleaned_tag )
namespace_id = self._GetNamespaceId( namespace )
subtag_id = self._GetSubtagId( subtag )
self._c.execute( 'UPDATE tags SET namespace_id = ?, subtag_id = ? WHERE tag_id = ?;', ( namespace_id, subtag_id, tag_id ) )
try:
HydrusData.Print( 'Invalid tag fixing: {} replaced with {}'.format( repr( tag ), repr( cleaned_tag ) ) )
except:
HydrusData.Print( 'Invalid tag fixing: Could not even print the bad tag to the log! It is now known as {}'.format( repr( cleaned_tag ) ) )
if job_key is not None:
if not job_key.IsCancelled():
if bad_tag_count == 0:
message = 'Invalid tag scanning: No bad tags found!'
else:
message = 'Invalid tag scanning: {} bad tags found and fixed! They have been written to the log.'.format( HydrusData.ToHumanInt( bad_tag_count ) )
HydrusData.Print( message )
job_key.SetVariable( 'popup_text_1', message )
job_key.Finish()
def _RepopulateMappingsFromCache( self, job_key = None ):
BLOCK_SIZE = 10000
@ -17081,9 +17191,45 @@ class DB( HydrusDB.HydrusDB ):
self._c.execute( 'DROP INDEX IF EXISTS tags_subtag_id_namespace_id_index;' )
self._controller.frame_splash_status.SetSubtext( 'creating new tag indices' )
self._controller.frame_splash_status.SetSubtext( 'creating first new tag index' )
try:
self._CreateIndex( 'external_master.tags', [ 'namespace_id', 'subtag_id' ], unique = True )
except Exception as e:
if 'unique' in str( e ) or 'constraint' in str( e ):
message = 'Hey, unfortunately, it looks like your master tag definition table had a duplicate entry. This is generally harmless, but it _is_ an invalid state. It probably happened because of a very old bug or other storage conversion incident. The new indices for v412 fix this up right now, but it will need some more work. It could take a while on an HDD. It will start when you ok this message, so kill this program in Task Manager now if you want to cancel out.'
BlockingSafeShowMessage( message )
self._c.execute( 'ALTER TABLE tags RENAME TO tags_old;' )
self._controller.frame_splash_status.SetSubtext( 'running deduplication' )
self._c.execute( 'CREATE TABLE IF NOT EXISTS external_master.tags ( tag_id INTEGER PRIMARY KEY, namespace_id INTEGER, subtag_id INTEGER );' )
self._CreateIndex( 'external_master.tags', [ 'namespace_id', 'subtag_id' ], unique = True )
self._c.execute( 'INSERT OR IGNORE INTO tags SELECT * FROM tags_old;' )
self._controller.frame_splash_status.SetSubtext( 'cleaning up deduplication' )
self._c.execute( 'DROP TABLE tags_old;' )
message = 'Ok, looks like the deduplication worked! There is a small chance you will get a tag definition error notification in the coming days or months. Do not worry too much--hydrus will generally fix itself, but let hydev know if it causes a bigger problem.'
BlockingSafeShowMessage( message )
else:
raise
self._controller.frame_splash_status.SetSubtext( 'creating second new tag index' )
self._CreateIndex( 'external_master.tags', [ 'namespace_id', 'subtag_id' ], unique = True )
self._CreateIndex( 'external_master.tags', [ 'subtag_id' ] )
self._controller.frame_splash_status.SetSubtext( 'optimising new tag indices' )
@ -17772,6 +17918,7 @@ class DB( HydrusDB.HydrusDB ):
elif action == 'remove_duplicates_member': self._DuplicatesRemoveMediaIdMemberFromHashes( *args, **kwargs )
elif action == 'remove_potential_pairs': self._DuplicatesRemovePotentialPairsFromHashes( *args, **kwargs )
elif action == 'repair_client_files': self._RepairClientFiles( *args, **kwargs )
elif action == 'repair_invalid_tags': self._RepairInvalidTags( *args, **kwargs )
elif action == 'reprocess_repository': self._ReprocessRepository( *args, **kwargs )
elif action == 'reset_repository': self._ResetRepository( *args, **kwargs )
elif action == 'reset_potential_search_status': self._PHashesResetSearchFromHashes( *args, **kwargs )

14
hydrus/client/ClientLocalServerResources.py
View File

@ -924,19 +924,19 @@ class HydrusResourceClientAPIRestrictedAddTagsAddTags( HydrusResourceClientAPIRe
for ( content_action, tags ) in actions_to_tags.items():
content_action = int( content_action )
tags = list( tags )
if len( tags ) == 0:
continue
if isinstance( tags[0], str ):
tags = HydrusTags.CleanTags( tags )
if len( tags ) == 0:
continue
content_action = int( content_action )
if service.GetServiceType() == HC.LOCAL_TAG:
@ -962,8 +962,6 @@ class HydrusResourceClientAPIRestrictedAddTagsAddTags( HydrusResourceClientAPIRe
if content_action == HC.CONTENT_UPDATE_PETITION:
tags = list( tags )
if isinstance( tags[0], str ):
tags_and_reasons = [ ( tag, 'Petitioned from API' ) for tag in tags ]

1
hydrus/client/ClientOptions.py
View File

@ -423,6 +423,7 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
self._dictionary[ 'noneable_strings' ][ 'media_background_bmp_path' ] = None
self._dictionary[ 'noneable_strings' ][ 'http_proxy' ] = None
self._dictionary[ 'noneable_strings' ][ 'https_proxy' ] = None
self._dictionary[ 'noneable_strings' ][ 'no_proxy' ] = '127.0.0.1'
self._dictionary[ 'noneable_strings' ][ 'qt_style_name' ] = None
self._dictionary[ 'noneable_strings' ][ 'qt_stylesheet_name' ] = None

47
hydrus/client/ClientParsing.py
View File

@ -470,6 +470,23 @@ def MakeParsedTextPretty( parsed_text ):
return parsed_text
def ParseResultsHavePursuableURLs( results ):
for ( ( name, content_type, additional_info ), parsed_text ) in results:
if content_type == HC.CONTENT_TYPE_URLS:
( url_type, priority ) = additional_info
if url_type == HC.URL_TYPE_DESIRED:
return True
return False
def RenderJSONParseRule( rule ):
( parse_rule_type, parse_rule ) = rule
@ -791,7 +808,7 @@ class ParseFormulaContextVariable( ParseFormula ):
if self._variable_name in parsing_context:
raw_texts.append( parsing_context[ self._variable_name ] )
raw_texts.append( str( parsing_context[ self._variable_name ] ) )
return raw_texts
@ -2021,6 +2038,22 @@ class ContentParser( HydrusSerialisable.SerialisableBase ):
u = re.sub( r'^.*\shttp', 'http', u )
return u
clean_parsed_texts = []
for parsed_text in parsed_texts:
if not parsed_text.startswith( 'http' ) and ( 'http://' in parsed_text or 'https://' in parsed_text ):
parsed_text = clean_url( parsed_text )
clean_parsed_texts.append( parsed_text )
parsed_texts = clean_parsed_texts
parsed_texts = [ urllib.parse.urljoin( base_url, parsed_text ) for parsed_text in parsed_texts ]
@ -2303,11 +2336,21 @@ class PageParser( HydrusSerialisable.SerialisableBaseNamed ):
try:
if 'post_index' not in parsing_context:
parsing_context[ 'post_index' ] = '0'
for content_parser in self._content_parsers:
whole_page_parse_results.extend( content_parser.Parse( parsing_context, converted_parsing_text ) )
if ParseResultsHavePursuableURLs( whole_page_parse_results ):
parsing_context[ 'post_index' ] = str( int( parsing_context[ 'post_index' ] ) + 1 )
except HydrusExceptions.ParseException as e:
prefix = 'Page Parser ' + self._name + ': '
@ -2347,7 +2390,7 @@ class PageParser( HydrusSerialisable.SerialisableBaseNamed ):
posts = formula.Parse( parsing_context, converted_parsing_text )
for post in posts:
for ( i, post ) in enumerate( posts ):
try:

629
hydrus/client/gui/ClientGUI.py
View File

@ -659,175 +659,6 @@ class FrameGUI( ClientGUITopLevelWindows.MainFrameThatResizes ):
def _AutoServerSetup( self ):
def do_it():
host = '127.0.0.1'
port = HC.DEFAULT_SERVER_ADMIN_PORT
if HydrusNetworking.LocalPortInUse( port ):
HydrusData.ShowText( 'The server appears to be already running. Either that, or something else is using port ' + str( HC.DEFAULT_SERVER_ADMIN_PORT ) + '.' )
return
else:
try:
HydrusData.ShowText( 'Starting server\u2026' )
db_param = '-d=' + self._controller.GetDBDir()
if HC.PLATFORM_WINDOWS:
server_frozen_path = os.path.join( HC.BASE_DIR, 'server.exe' )
else:
server_frozen_path = os.path.join( HC.BASE_DIR, 'server' )
if os.path.exists( server_frozen_path ):
cmd = [ server_frozen_path, db_param ]
else:
python_executable = sys.executable
if python_executable.endswith( 'client.exe' ) or python_executable.endswith( 'client' ):
raise Exception( 'Could not automatically set up the server--could not find server executable or python executable.' )
if 'pythonw' in python_executable:
python_executable = python_executable.replace( 'pythonw', 'python' )
server_script_path = os.path.join( HC.BASE_DIR, 'server.py' )
cmd = [ python_executable, server_script_path, db_param ]
sbp_kwargs = HydrusData.GetSubprocessKWArgs( hide_terminal = False )
HydrusData.CheckProgramIsNotShuttingDown()
subprocess.Popen( cmd, **sbp_kwargs )
time_waited = 0
while not HydrusNetworking.LocalPortInUse( port ):
time.sleep( 3 )
time_waited += 3
if time_waited > 30:
raise Exception( 'The server\'s port did not appear!' )
except:
HydrusData.ShowText( 'I tried to start the server, but something failed!' + os.linesep + traceback.format_exc() )
return
time.sleep( 5 )
HydrusData.ShowText( 'Creating admin service\u2026' )
admin_service_key = HydrusData.GenerateKey()
service_type = HC.SERVER_ADMIN
name = 'local server admin'
admin_service = ClientServices.GenerateService( admin_service_key, service_type, name )
all_services = list( self._controller.services_manager.GetServices() )
all_services.append( admin_service )
self._controller.SetServices( all_services )
time.sleep( 1 )
admin_service = self._controller.services_manager.GetService( admin_service_key ) # let's refresh it
credentials = HydrusNetwork.Credentials( host, port )
admin_service.SetCredentials( credentials )
time.sleep( 1 )
response = admin_service.Request( HC.GET, 'access_key', { 'registration_key' : b'init' } )
access_key = response[ 'access_key' ]
credentials = HydrusNetwork.Credentials( host, port, access_key )
admin_service.SetCredentials( credentials )
#
HydrusData.ShowText( 'Admin service initialised.' )
QP.CallAfter( ClientGUIFrames.ShowKeys, 'access', (access_key,) )
#
time.sleep( 5 )
HydrusData.ShowText( 'Creating tag and file services\u2026' )
response = admin_service.Request( HC.GET, 'services' )
serverside_services = response[ 'services' ]
service_key = HydrusData.GenerateKey()
tag_service = HydrusNetwork.GenerateService( service_key, HC.TAG_REPOSITORY, 'tag service', HC.DEFAULT_SERVICE_PORT )
serverside_services.append( tag_service )
service_key = HydrusData.GenerateKey()
file_service = HydrusNetwork.GenerateService( service_key, HC.FILE_REPOSITORY, 'file service', HC.DEFAULT_SERVICE_PORT + 1 )
serverside_services.append( file_service )
response = admin_service.Request( HC.POST, 'services', { 'services' : serverside_services } )
service_keys_to_access_keys = response[ 'service_keys_to_access_keys' ]
deletee_service_keys = []
with HG.dirty_object_lock:
self._controller.WriteSynchronous( 'update_server_services', admin_service_key, serverside_services, service_keys_to_access_keys, deletee_service_keys )
self._controller.RefreshServices()
HydrusData.ShowText( 'Done! Check services->review services to see your new server and its services.' )
text = 'This will attempt to start the server in the same install directory as this client, initialise it, and store the resultant admin accounts in the client.'
result = ClientGUIDialogsQuick.GetYesNo( self, text )
if result == QW.QDialog.Accepted:
self._controller.CallToThread( do_it )
def _BackupDatabase( self ):
path = self._new_options.GetNoneableString( 'backup_path' )
@ -3212,6 +3043,28 @@ class FrameGUI( ClientGUITopLevelWindows.MainFrameThatResizes ):
def _RepairInvalidTags( self ):
message = 'This will scan all your tags and repair any that are invalid. This might mean taking out unrenderable characters or cleaning up improper whitespace. If there is a tag collision once cleaned, it may add a (1)-style number on the end.'
message += os.linesep * 2
message += 'If you have a lot of tags, it can take a long time, during which the gui may hang. If it finds bad tags, you should restart the program once it is complete.'
message += os.linesep * 2
message += 'If you have not had tag rendering problems, there is no reason to run this.'
result = ClientGUIDialogsQuick.GetYesNo( self, message, yes_label = 'do it', no_label = 'forget it' )
if result == QW.QDialog.Accepted:
job_key = ClientThreading.JobKey( cancellable = True )
job_key.SetVariable( 'popup_title', 'repairing invalid tags' )
self._controller.pub( 'message', job_key )
self._controller.Write( 'repair_invalid_tags', job_key = job_key )
def _RepopulateMappingsTables( self ):
message = 'WARNING: Do not run this for no reason!'
@ -3361,6 +3214,261 @@ class FrameGUI( ClientGUITopLevelWindows.MainFrameThatResizes ):
self._controller.pub( 'notify_new_export_folders' )
def _RunClientAPITest( self ):
# this is not to be a comprehensive test of client api functions, but a holistic sanity check to make sure everything is wired up right at UI level, with a live functioning client
from hydrus.client import ClientAPI
def do_it():
# job key
client_api_service = HG.client_controller.services_manager.GetService( CC.CLIENT_API_SERVICE_KEY )
port = client_api_service.GetPort()
was_running_before = port is not None
if not was_running_before:
port = 6666
client_api_service._port = port
HG.client_controller.RestartClientServerServices()
time.sleep( 5 )
#
api_permissions = ClientAPI.APIPermissions( name = 'hydrus test access', basic_permissions = list( ClientAPI.ALLOWED_PERMISSIONS ), search_tag_filter = ClientTags.TagFilter() )
access_key = api_permissions.GetAccessKey()
HG.client_controller.client_api_manager.AddAccess( api_permissions )
#
try:
job_key = ClientThreading.JobKey()
job_key.SetVariable( 'popup_title', 'client api test' )
HG.client_controller.pub( 'message', job_key )
import requests
import json
s = requests.Session()
s.verify = False
s.headers[ 'Hydrus-Client-API-Access-Key' ] = access_key.hex()
s.headers[ 'Content-Type' ] = 'application/json'
if client_api_service.UseHTTPS():
schema = 'https'
else:
schema = 'http'
api_base = '{}://127.0.0.1:{}'.format( schema, port )
#
r = s.get( '{}/api_version'.format( api_base ) )
j = r.json()
if j[ 'version' ] != HC.CLIENT_API_VERSION:
HydrusData.ShowText( 'version incorrect!: {}, {}'.format( j[ 'version' ], HC.CLIENT_API_VERSION ) )
#
job_key.SetVariable( 'popup_text_1', 'add url test' )
local_tag_services = HG.client_controller.services_manager.GetServices( ( HC.LOCAL_TAG, ) )
local_tag_service = random.choice( local_tag_services )
local_tag_service_name = local_tag_service.GetName()
samus_url = 'https://safebooru.org/index.php?page=post&s=view&id=3195917'
samus_hash_hex = '78f92ba4a786225ee2a1236efa6b7dc81dd729faf4af99f96f3e20bad6d8b538'
samus_test_tag = 'client api test tag'
samus_test_tag_filterable = 'client api test tag filterable'
destination_page_name = 'client api test'
request_args = {}
request_args[ 'url' ] = samus_url
request_args[ 'destination_page_name' ] = destination_page_name
request_args[ 'service_names_to_additional_tags' ] = {
local_tag_service_name : [ samus_test_tag ]
}
request_args[ 'filterable_tags' ] = [
samus_test_tag_filterable
]
data = json.dumps( request_args )
r = s.post( '{}/add_urls/add_url'.format( api_base ), data = data )
time.sleep( 0.25 )
#
job_key.SetVariable( 'popup_text_1', 'get session test' )
def get_client_api_page():
r = s.get( '{}/manage_pages/get_pages'.format( api_base ) )
pages_to_process = [ r.json()[ 'pages' ] ]
pages = []
while len( pages_to_process ) > 0:
page_to_process = pages_to_process.pop()
if page_to_process[ 'page_type' ] == ClientGUIManagement.MANAGEMENT_TYPE_PAGE_OF_PAGES:
pages_to_process.extend( page_to_process[ 'pages' ] )
else:
pages.append( page_to_process )
for page in pages:
if page[ 'name' ] == destination_page_name:
return page
client_api_page = get_client_api_page()
if client_api_page is None:
raise Exception( 'Could not find download page!' )
destination_page_key_hex = client_api_page[ 'page_key' ]
def get_hash_ids():
r = s.get( '{}/manage_pages/get_page_info?page_key={}'.format( api_base, destination_page_key_hex ) )
hash_ids = r.json()[ 'page_info' ][ 'media' ][ 'hash_ids' ]
return hash_ids
hash_ids = get_hash_ids()
if len( hash_ids ) == 0:
time.sleep( 3 )
hash_ids = get_hash_ids()
if len( hash_ids ) == 0:
raise Exception( 'The download page had no hashes!' )
#
def get_hash_ids_to_hashes_and_tag_info():
r = s.get( '{}/get_files/file_metadata?file_ids={}'.format( api_base, json.dumps( hash_ids ) ) )
hash_ids_to_hashes_and_tag_info = {}
for item in r.json()[ 'metadata' ]:
hash_ids_to_hashes_and_tag_info[ item[ 'file_id' ] ] = ( item[ 'hash' ], item[ 'service_names_to_statuses_to_tags' ] )
return hash_ids_to_hashes_and_tag_info
hash_ids_to_hashes_and_tag_info = get_hash_ids_to_hashes_and_tag_info()
samus_hash_id = None
for ( hash_id, ( hash_hex, tag_info ) ) in hash_ids_to_hashes_and_tag_info.items():
if hash_hex == samus_hash_hex:
samus_hash_id = hash_id
if samus_hash_id is None:
raise Exception( 'Could not find the samus hash!' )
samus_tag_info = hash_ids_to_hashes_and_tag_info[ samus_hash_id ][1]
if samus_test_tag not in samus_tag_info[ local_tag_service_name ][ str( HC.CONTENT_STATUS_CURRENT ) ]:
raise Exception( 'Did not have the tag!' )
#
def qt_session_gubbins():
self.ProposeSaveGUISession( 'last session' )
page = self._notebook.GetPageFromPageKey( bytes.fromhex( destination_page_key_hex ) )
self._notebook.ShowPage( page )
self._notebook.CloseCurrentPage()
self.ProposeSaveGUISession( 'last session' )
HG.client_controller.CallBlockingToQt( HG.client_controller.gui, qt_session_gubbins )
finally:
#
HG.client_controller.client_api_manager.DeleteAccess( ( access_key, ) )
#
if not was_running_before:
client_api_service._port = None
HG.client_controller.RestartClientServerServices()
job_key.Delete()
HG.client_controller.CallToThread( do_it )
def _RunUITest( self ):
def qt_open_pages():
@ -3399,6 +3507,10 @@ class FrameGUI( ClientGUITopLevelWindows.MainFrameThatResizes ):
HG.client_controller.CallLaterQtSafe(self, t, self.ProcessApplicationCommand, CAC.ApplicationCommand(CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_NEW_WATCHER_DOWNLOADER_PAGE))
t += 0.25
HG.client_controller.CallLaterQtSafe(self, t, self.ProposeSaveGUISession, 'last session' )
return page_of_pages
@ -3547,6 +3659,175 @@ class FrameGUI( ClientGUITopLevelWindows.MainFrameThatResizes ):
HG.client_controller.CallToThread( do_it )
def _RunServerTest( self ):
def do_it():
host = '127.0.0.1'
port = HC.DEFAULT_SERVER_ADMIN_PORT
if HydrusNetworking.LocalPortInUse( port ):
HydrusData.ShowText( 'The server appears to be already running. Either that, or something else is using port ' + str( HC.DEFAULT_SERVER_ADMIN_PORT ) + '.' )
return
else:
try:
HydrusData.ShowText( 'Starting server\u2026' )
db_param = '-d=' + self._controller.GetDBDir()
if HC.PLATFORM_WINDOWS:
server_frozen_path = os.path.join( HC.BASE_DIR, 'server.exe' )
else:
server_frozen_path = os.path.join( HC.BASE_DIR, 'server' )
if os.path.exists( server_frozen_path ):
cmd = [ server_frozen_path, db_param ]
else:
python_executable = sys.executable
if python_executable.endswith( 'client.exe' ) or python_executable.endswith( 'client' ):
raise Exception( 'Could not automatically set up the server--could not find server executable or python executable.' )
if 'pythonw' in python_executable:
python_executable = python_executable.replace( 'pythonw', 'python' )
server_script_path = os.path.join( HC.BASE_DIR, 'server.py' )
cmd = [ python_executable, server_script_path, db_param ]
sbp_kwargs = HydrusData.GetSubprocessKWArgs( hide_terminal = False )
HydrusData.CheckProgramIsNotShuttingDown()
subprocess.Popen( cmd, **sbp_kwargs )
time_waited = 0
while not HydrusNetworking.LocalPortInUse( port ):
time.sleep( 3 )
time_waited += 3
if time_waited > 30:
raise Exception( 'The server\'s port did not appear!' )
except:
HydrusData.ShowText( 'I tried to start the server, but something failed!' + os.linesep + traceback.format_exc() )
return
time.sleep( 5 )
HydrusData.ShowText( 'Creating admin service\u2026' )
admin_service_key = HydrusData.GenerateKey()
service_type = HC.SERVER_ADMIN
name = 'local server admin'
admin_service = ClientServices.GenerateService( admin_service_key, service_type, name )
all_services = list( self._controller.services_manager.GetServices() )
all_services.append( admin_service )
self._controller.SetServices( all_services )
time.sleep( 1 )
admin_service = self._controller.services_manager.GetService( admin_service_key ) # let's refresh it
credentials = HydrusNetwork.Credentials( host, port )
admin_service.SetCredentials( credentials )
time.sleep( 1 )
response = admin_service.Request( HC.GET, 'access_key', { 'registration_key' : b'init' } )
access_key = response[ 'access_key' ]
credentials = HydrusNetwork.Credentials( host, port, access_key )
admin_service.SetCredentials( credentials )
#
HydrusData.ShowText( 'Admin service initialised.' )
QP.CallAfter( ClientGUIFrames.ShowKeys, 'access', (access_key,) )
#
time.sleep( 5 )
HydrusData.ShowText( 'Creating tag and file services\u2026' )
response = admin_service.Request( HC.GET, 'services' )
serverside_services = response[ 'services' ]
service_key = HydrusData.GenerateKey()
tag_service = HydrusNetwork.GenerateService( service_key, HC.TAG_REPOSITORY, 'tag service', HC.DEFAULT_SERVICE_PORT )
serverside_services.append( tag_service )
service_key = HydrusData.GenerateKey()
file_service = HydrusNetwork.GenerateService( service_key, HC.FILE_REPOSITORY, 'file service', HC.DEFAULT_SERVICE_PORT + 1 )
serverside_services.append( file_service )
response = admin_service.Request( HC.POST, 'services', { 'services' : serverside_services } )
service_keys_to_access_keys = response[ 'service_keys_to_access_keys' ]
deletee_service_keys = []
with HG.dirty_object_lock:
self._controller.WriteSynchronous( 'update_server_services', admin_service_key, serverside_services, service_keys_to_access_keys, deletee_service_keys )
self._controller.RefreshServices()
HydrusData.ShowText( 'Done! Check services->review services to see your new server and its services.' )
text = 'This will attempt to start the server in the same install directory as this client, initialise it, and store the resultant admin accounts in the client.'
result = ClientGUIDialogsQuick.GetYesNo( self, text )
if result == QW.QDialog.Accepted:
self._controller.CallToThread( do_it )
def _SaveSplitterPositions( self ):
page = self._notebook.GetCurrentMediaPage()
@ -4533,6 +4814,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
ClientGUIMenus.AppendMenuItem( submenu, 'database integrity', 'Have the database examine all its records for internal consistency.', self._CheckDBIntegrity )
ClientGUIMenus.AppendMenuItem( submenu, 'repopulate truncated mappings tables', 'Use the mappings cache to try to repair a previously damaged mappings file.', self._RepopulateMappingsTables )
ClientGUIMenus.AppendMenuItem( submenu, 'fix invalid tags', 'Scan the database for invalid tags.', self._RepairInvalidTags )
ClientGUIMenus.AppendMenu( menu, submenu, 'check and repair' )
@ -4910,7 +5192,6 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
ClientGUIMenus.AppendMenuItem( gui_actions, 'make a parentless text ctrl dialog', 'Make a parentless text control in a dialog to test some character event catching.', self._DebugMakeParentlessTextCtrl )
ClientGUIMenus.AppendMenuItem( gui_actions, 'force a main gui layout now', 'Tell the gui to relayout--useful to test some gui bootup layout issues.', self.adjustSize )
ClientGUIMenus.AppendMenuItem( gui_actions, 'save \'last session\' gui session', 'Make an immediate save of the \'last session\' gui session. Mostly for testing crashes, where last session is not saved correctly.', self.ProposeSaveGUISession, 'last session' )
ClientGUIMenus.AppendMenuItem( gui_actions, 'run the ui test', 'Run hydrus_dev\'s weekly UI Test. Guaranteed to work and not mess up your session, ha ha.', self._RunUITest )
ClientGUIMenus.AppendMenu( debug, gui_actions, 'gui actions' )
@ -4944,7 +5225,13 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
ClientGUIMenus.AppendMenu( debug, network_actions, 'network actions' )
ClientGUIMenus.AppendMenuItem( debug, 'run and initialise server for testing', 'This will try to boot the server in your install folder and initialise it. This is mostly here for testing purposes.', self._AutoServerSetup )
tests = QW.QMenu( debug )
ClientGUIMenus.AppendMenuItem( tests, 'run the ui test', 'Run hydrus_dev\'s weekly UI Test. Guaranteed to work and not mess up your session, ha ha.', self._RunUITest )
ClientGUIMenus.AppendMenuItem( tests, 'run the client api test', 'Run hydrus_dev\'s weekly Client API Test. Guaranteed to work and not mess up your session, ha ha.', self._RunClientAPITest )
ClientGUIMenus.AppendMenuItem( tests, 'run the server test', 'This will try to boot the server in your install folder and initialise it. This is mostly here for testing purposes.', self._RunServerTest )
ClientGUIMenus.AppendMenu( debug, tests, 'tests, do not touch' )
ClientGUIMenus.AppendMenu( menu, debug, 'debug' )

13
hydrus/client/gui/ClientGUIPages.py
View File

@ -1274,18 +1274,23 @@ class PagesNotebook( QP.TabWidgetWithDnD ):
return [ self.widget( i ) for i in range( self.count() ) ]
def _GetPageFromName( self, page_name ):
def _GetPageFromName( self, page_name, only_media_pages = False ):
for page in self._GetPages():
if page.GetName() == page_name:
return page
do_not_do_it = only_media_pages and isinstance( page, PagesNotebook )
if not do_not_do_it:
return page
if isinstance( page, PagesNotebook ):
result = page._GetPageFromName( page_name )
result = page._GetPageFromName( page_name, only_media_pages = only_media_pages )
if result is not None:
@ -2741,7 +2746,7 @@ class PagesNotebook( QP.TabWidgetWithDnD ):
def PresentImportedFilesToPage( self, hashes, page_name ):
page = self._GetPageFromName( page_name )
page = self._GetPageFromName( page_name, only_media_pages = True )
if page is None:

9
hydrus/client/gui/ClientGUIParsing.py
View File

@ -3183,6 +3183,7 @@ class EditPageParserPanel( ClientGUIScrolledPanels.EditPanel ):
example_parsing_context = self._test_panel.GetExampleParsingContext()
example_parsing_context[ 'url' ] = url
example_parsing_context[ 'post_index' ] = '0'
self._test_panel.SetExampleParsingContext( example_parsing_context )
@ -4327,6 +4328,7 @@ class TestPanel( QW.QWidget ):
example_parsing_context = self._example_parsing_context.GetValue()
example_parsing_context[ 'url' ] = url
example_parsing_context[ 'post_index' ] = '0'
self._example_parsing_context.SetValue( example_parsing_context )
@ -4479,6 +4481,11 @@ class TestPanel( QW.QWidget ):
try:
if 'post_index' in test_data.parsing_context:
del test_data.parsing_context[ 'post_index' ]
results_text = obj.ParsePretty( test_data.parsing_context, test_data.texts[0] )
self._results.setPlainText( results_text )
@ -4773,6 +4780,8 @@ class TestPanelPageParserSubsidiary( TestPanelPageParser ):
test_data = self.GetTestData()
test_data.parsing_context[ 'post_index' ] = 0
if formula is None:
posts = test_data.texts

7
hydrus/client/gui/ClientGUIPopupMessages.py
View File

@ -582,6 +582,8 @@ class PopupMessageManager( QW.QWidget ):
self._update_job = HG.client_controller.CallRepeatingQtSafe( self, 0.25, 0.5, self.REPEATINGUpdate )
self._summary_bar.expandCollapse.connect( self.ExpandCollapse )
HG.client_controller.CallLaterQtSafe(self, 0.5, self.AddMessage, job_key)
HG.client_controller.CallLaterQtSafe(self, 1.0, job_key.Delete)
@ -997,6 +999,7 @@ class PopupMessageManager( QW.QWidget ):
self._message_panel.show()
self.MakeSureEverythingFits()
@ -1260,6 +1263,8 @@ class PopupMessageDialogPanel( QW.QWidget ):
class PopupMessageSummaryBar( PopupWindow ):
expandCollapse = QC.Signal()
def __init__( self, parent, manager ):
PopupWindow.__init__( self, parent, manager )
@ -1282,7 +1287,7 @@ class PopupMessageSummaryBar( PopupWindow ):
def ExpandCollapse( self ):
self._manager.ExpandCollapse()
self.expandCollapse.emit()
current_text = self._expand_collapse.text()

8
hydrus/client/gui/ClientGUIScrolledPanelsManagement.py
View File

@ -315,6 +315,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._http_proxy = ClientGUICommon.NoneableTextCtrl( proxy_panel )
self._https_proxy = ClientGUICommon.NoneableTextCtrl( proxy_panel )
self._no_proxy = ClientGUICommon.NoneableTextCtrl( proxy_panel )
#
@ -322,6 +323,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._http_proxy.SetValue( self._new_options.GetNoneableString( 'http_proxy' ) )
self._https_proxy.SetValue( self._new_options.GetNoneableString( 'https_proxy' ) )
self._no_proxy.SetValue( self._new_options.GetNoneableString( 'no_proxy' ) )
self._network_timeout.setValue( self._new_options.GetInteger( 'network_timeout' ) )
self._connection_error_wait_time.setValue( self._new_options.GetInteger( 'connection_error_wait_time' ) )
@ -363,7 +365,9 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
general.Add( gridbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
text = 'Enter strings such as "http://ip:port" or "http://user:pass@ip:port". It should take affect immediately on dialog ok.'
text = 'Enter strings such as "http://ip:port" or "http://user:pass@ip:port" to use for http and https traffic. It should take effect immediately on dialog ok.'
text += os.linesep * 2
text += 'no_proxy takes the form of comma-separated hosts/domains, just as in curl or the NO_PROXY environment variable. When http and/or https proxies are set, they will not be used for these.'
text += os.linesep * 2
if ClientNetworkingSessions.SOCKS_PROXY_OK:
@ -387,6 +391,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
rows.append( ( 'http: ', self._http_proxy ) )
rows.append( ( 'https: ', self._https_proxy ) )
rows.append( ( 'no_proxy: ', self._no_proxy ) )
gridbox = ClientGUICommon.WrapInGrid( proxy_panel, rows )
@ -409,6 +414,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._new_options.SetNoneableString( 'http_proxy', self._http_proxy.GetValue() )
self._new_options.SetNoneableString( 'https_proxy', self._https_proxy.GetValue() )
self._new_options.SetNoneableString( 'no_proxy', self._no_proxy.GetValue() )
self._new_options.SetInteger( 'network_timeout', self._network_timeout.value() )
self._new_options.SetInteger( 'connection_error_wait_time', self._connection_error_wait_time.value() )

26
hydrus/client/gui/ClientGUITagSuggestions.py
View File

@ -144,6 +144,8 @@ class ListBoxTagsSuggestionsRelated( ClientGUIListBoxes.ListBoxTagsPredicates ):
class FavouritesTagsPanel( QW.QWidget ):
mouseActivationOccurred = QC.Signal()
def __init__( self, parent, service_key, media, activate_callable ):
QW.QWidget.__init__( self, parent )
@ -161,6 +163,8 @@ class FavouritesTagsPanel( QW.QWidget ):
self._UpdateTagDisplay()
self._favourite_tags.mouseActivationOccurred.connect( self.mouseActivationOccurred )
def _UpdateTagDisplay( self ):
@ -192,6 +196,8 @@ class FavouritesTagsPanel( QW.QWidget ):
class RecentTagsPanel( QW.QWidget ):
mouseActivationOccurred = QC.Signal()
def __init__( self, parent, service_key, media, activate_callable ):
QW.QWidget.__init__( self, parent )
@ -217,6 +223,8 @@ class RecentTagsPanel( QW.QWidget ):
self._RefreshRecentTags()
self._recent_tags.mouseActivationOccurred.connect( self.mouseActivationOccurred )
def _RefreshRecentTags( self ):
@ -296,6 +304,8 @@ class RecentTagsPanel( QW.QWidget ):
class RelatedTagsPanel( QW.QWidget ):
mouseActivationOccurred = QC.Signal()
def __init__( self, parent, service_key, media, activate_callable ):
QW.QWidget.__init__( self, parent )
@ -331,6 +341,8 @@ class RelatedTagsPanel( QW.QWidget ):
self.setLayout( vbox )
self._related_tags.mouseActivationOccurred.connect( self.mouseActivationOccurred )
def _FetchRelatedTags( self, max_time_to_take ):
@ -417,6 +429,8 @@ class RelatedTagsPanel( QW.QWidget ):
class FileLookupScriptTagsPanel( QW.QWidget ):
mouseActivationOccurred = QC.Signal()
def __init__( self, parent, service_key, media, activate_callable ):
QW.QWidget.__init__( self, parent )
@ -455,6 +469,8 @@ class FileLookupScriptTagsPanel( QW.QWidget ):
self._FetchScripts()
self._tags.mouseActivationOccurred.connect( self.mouseActivationOccurred )
def _FetchScripts( self ):
@ -605,6 +621,8 @@ class FileLookupScriptTagsPanel( QW.QWidget ):
class SuggestedTagsPanel( QW.QWidget ):
mouseActivationOccurred = QC.Signal()
def __init__( self, parent, service_key, media, activate_callable ):
QW.QWidget.__init__( self, parent )
@ -639,6 +657,8 @@ class SuggestedTagsPanel( QW.QWidget ):
self._favourite_tags = FavouritesTagsPanel( panel_parent, service_key, media, activate_callable )
self._favourite_tags.mouseActivationOccurred.connect( self.mouseActivationOccurred )
panels.append( ( 'favourites', self._favourite_tags ) )
@ -648,6 +668,8 @@ class SuggestedTagsPanel( QW.QWidget ):
self._related_tags = RelatedTagsPanel( panel_parent, service_key, media, activate_callable )
self._related_tags.mouseActivationOccurred.connect( self.mouseActivationOccurred )
panels.append( ( 'related', self._related_tags ) )
@ -657,6 +679,8 @@ class SuggestedTagsPanel( QW.QWidget ):
self._file_lookup_script_tags = FileLookupScriptTagsPanel( panel_parent, service_key, media, activate_callable )
self._file_lookup_script_tags.mouseActivationOccurred.connect( self.mouseActivationOccurred )
panels.append( ( 'file lookup scripts', self._file_lookup_script_tags ) )
@ -666,6 +690,8 @@ class SuggestedTagsPanel( QW.QWidget ):
self._recent_tags = RecentTagsPanel( panel_parent, service_key, media, activate_callable )
self._recent_tags.mouseActivationOccurred.connect( self.mouseActivationOccurred )
panels.append( ( 'recent', self._recent_tags ) )

2
hydrus/client/gui/ClientGUITags.py
View File

@ -2039,6 +2039,8 @@ class ManageTagsPanel( ClientGUIScrolledPanels.ManagePanel ):
HG.client_controller.sub( self, 'CheckboxExpandParents', 'checkbox_manager_inverted' )
self._suggested_tags.mouseActivationOccurred.connect( self.SetTagBoxFocus )
def _EnterTags( self, tags, only_add = False, only_remove = False, forced_reason = None ):

13
hydrus/client/gui/lists/ClientGUIListBoxes.py
View File

@ -297,7 +297,7 @@ class AddEditDeleteListBox( QW.QWidget ):
def _AddData( self, data ):
self._SetNoneDupeName( data )
self._SetNonDupeName( data )
pretty_data = self._data_to_pretty_callable( data )
@ -369,7 +369,7 @@ class AddEditDeleteListBox( QW.QWidget ):
break
self._SetNoneDupeName( new_data )
self._SetNonDupeName( new_data )
pretty_new_data = self._data_to_pretty_callable( new_data )
@ -577,7 +577,7 @@ class AddEditDeleteListBox( QW.QWidget ):
self.listBoxChanged.emit()
def _SetNoneDupeName( self, obj ):
def _SetNonDupeName( self, obj ):
pass
@ -688,7 +688,7 @@ class AddEditDeleteListBox( QW.QWidget ):
class AddEditDeleteListBoxUniqueNamedObjects( AddEditDeleteListBox ):
def _SetNoneDupeName( self, obj ):
def _SetNonDupeName( self, obj ):
disallowed_names = { o.GetName() for o in self.GetData() }
@ -912,6 +912,7 @@ class QueueListBox( QW.QWidget ):
class ListBox( QW.QScrollArea ):
listBoxChanged = QC.Signal()
mouseActivationOccurred = QC.Signal()
TEXT_X_PADDING = 3
@ -1711,6 +1712,8 @@ class ListBox( QW.QScrollArea ):
self._Activate()
self.mouseActivationOccurred.emit()
def EventMouseSelect( self, event ):
@ -3056,7 +3059,7 @@ class ListBoxTagsSiblingCapable( ListBoxTags ):
ideal = result
tag_string = '{} (will display as {})'.format( tag_string, ClientTags.RenderTag( ideal, True ) )
tag_string = '{} (will display as {})'.format( tag_string, ideal )

1
hydrus/client/importing/ClientImportGallerySeeds.py
View File

@ -364,6 +364,7 @@ class GallerySeed( HydrusSerialisable.SerialisableBase ):
parsing_context[ 'gallery_url' ] = self.url
parsing_context[ 'url' ] = url_to_check
parsing_context[ 'post_index' ] = '0'
all_parse_results = parser.Parse( parsing_context, parsing_text )

27
hydrus/client/importing/ClientImportOptions.py
View File

@ -1371,13 +1371,7 @@ class TagImportOptions( HydrusSerialisable.SerialisableBase ):
service_keys_to_tags = ClientTags.ServiceKeysToTags()
for ( service_key, service_tag_import_options ) in self._service_keys_to_service_tag_import_options.items():
service_filterable_tags = set( filterable_tags )
service_filterable_tags.update( external_filterable_tags )
service_filterable_tags = parents_manager.ExpandTags( service_key, service_filterable_tags )
for service_key in HG.client_controller.services_manager.GetServiceKeys( HC.REAL_TAG_SERVICES ):
service_additional_tags = set()
@ -1388,12 +1382,25 @@ class TagImportOptions( HydrusSerialisable.SerialisableBase ):
service_additional_tags = parents_manager.ExpandTags( service_key, service_additional_tags )
service_tags = service_tag_import_options.GetTags( service_key, status, media_result, service_filterable_tags, service_additional_tags )
if service_key in self._service_keys_to_service_tag_import_options:
service_tag_import_options = self._service_keys_to_service_tag_import_options[ service_key ]
service_filterable_tags = set( filterable_tags )
service_filterable_tags.update( external_filterable_tags )
service_filterable_tags = parents_manager.ExpandTags( service_key, service_filterable_tags )
service_tags = service_tag_import_options.GetTags( service_key, status, media_result, service_filterable_tags, service_additional_tags )
else:
service_tags = service_additional_tags
if len( service_tags ) > 0:
service_tags = parents_manager.ExpandTags( service_key, service_tags )
service_keys_to_tags[ service_key ] = service_tags

8
hydrus/client/media/ClientMedia.py
View File

@ -2749,6 +2749,13 @@ class MediaSort( HydrusSerialisable.SerialisableBase ):
return num_frames / duration
elif sort_data == CC.SORT_FILES_BY_NUM_COLLECTION_FILES:
def sort_key( x ):
return ( x.GetNumFiles(), isinstance( x, MediaCollection ) )
elif sort_data == CC.SORT_FILES_BY_NUM_FRAMES:
def sort_key( x ):
@ -2913,6 +2920,7 @@ class MediaSort( HydrusSerialisable.SerialisableBase ):
sort_string_lookup[ CC.SORT_FILES_BY_FILESIZE ] = ( 'smallest first', 'largest first', CC.SORT_DESC )
sort_string_lookup[ CC.SORT_FILES_BY_DURATION ] = ( 'shortest first', 'longest first', CC.SORT_DESC )
sort_string_lookup[ CC.SORT_FILES_BY_FRAMERATE ] = ( 'slowest first', 'fastest first', CC.SORT_DESC )
sort_string_lookup[ CC.SORT_FILES_BY_NUM_COLLECTION_FILES ] = ( 'fewest first', 'most first', CC.SORT_DESC )
sort_string_lookup[ CC.SORT_FILES_BY_NUM_FRAMES ] = ( 'smallest first', 'largest first', CC.SORT_DESC )
sort_string_lookup[ CC.SORT_FILES_BY_HAS_AUDIO ] = ( 'audio first', 'silent first', CC.SORT_ASC )
sort_string_lookup[ CC.SORT_FILES_BY_IMPORT_TIME ] = ( 'oldest first', 'newest first', CC.SORT_DESC )

6
hydrus/client/networking/ClientNetworkingSessions.py
View File

@ -130,6 +130,7 @@ class NetworkSessionManager( HydrusSerialisable.SerialisableBase ):
http_proxy = HG.client_controller.new_options.GetNoneableString( 'http_proxy' )
https_proxy = HG.client_controller.new_options.GetNoneableString( 'https_proxy' )
no_proxy = HG.client_controller.new_options.GetNoneableString( 'no_proxy' )
if http_proxy is not None:
@ -141,6 +142,11 @@ class NetworkSessionManager( HydrusSerialisable.SerialisableBase ):
self._proxies_dict[ 'https' ] = https_proxy
if ( http_proxy is not None or https_proxy is not None ) and no_proxy is not None:
self._proxies_dict[ 'no_proxy' ] = no_proxy
def _SetDirty( self ):

2
hydrus/core/HydrusConstants.py
View File

@ -70,7 +70,7 @@ options = {}
# Misc
NETWORK_VERSION = 18
SOFTWARE_VERSION = 412
SOFTWARE_VERSION = 413
CLIENT_API_VERSION = 14
SERVER_THUMBNAIL_DIMENSIONS = ( 200, 200 )

7
hydrus/core/HydrusNATPunch.py
View File

@ -97,7 +97,7 @@ def AddUPnPMapping( internal_client, internal_port, external_port, protocol, des
if 'x.x.x.x:' + str( external_port ) + ' TCP is redirected to internal ' + internal_client + ':' + str( internal_port ) in stdout:
raise HydrusExceptions.FirewallException( 'The UPnP mapping of ' + internal_client + ':' + internal_port + '->external:' + external_port + ' already exists as a port forward. If this UPnP mapping is automatic, please disable it.' )
raise HydrusExceptions.FirewallException( 'The UPnP mapping of ' + internal_client + ':' + str( internal_port ) + '->external:' + str( external_port ) + ' already exists as a port forward. If this UPnP mapping is automatic, please disable it.' )
if stdout is not None and 'failed with code' in stdout:
@ -229,7 +229,10 @@ def RemoveUPnPMapping( external_port, protocol ):
( stdout, stderr ) = HydrusThreading.SubprocessCommunicate( p )
if stderr is not None and len( stderr ) > 0: raise Exception( 'Problem while trying to remove UPnP mapping:' + os.linesep * 2 + stderr )
if stderr is not None and len( stderr ) > 0:
raise Exception( 'Problem while trying to remove UPnP mapping:' + os.linesep * 2 + stderr )
class ServicesUPnPManager( object ):

2
hydrus/core/HydrusText.py
View File

@ -103,7 +103,7 @@ def NonFailingUnicodeDecode( data, encoding ):
except UnicodeDecodeError:
unicode_replacement_character = u'\ufffd'
null_character = '\0x0'
null_character = '\x00'
text = str( data, encoding, errors = 'replace' )