Version 427

closes #788
This commit is contained in:
Hydrus Network Developer 2021-01-27 16:14:03 -06:00
parent db6a478265
commit 9d95ee79fe
68 changed files with 2827 additions and 1807 deletions

230
client.py
View File

@ -4,233 +4,9 @@
# You just DO WHAT THE FUCK YOU WANT TO.
# https://github.com/sirkris/WTFPL/blob/master/WTFPL.md
try:
import locale
try: locale.setlocale( locale.LC_ALL, '' )
except: pass
import os
import argparse
import traceback
from hydrus.core import HydrusBoot
HydrusBoot.AddBaseDirToEnvPath()
# initialise Qt here, important it is done early
from hydrus.client.gui import QtPorting as QP
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusPaths
from hydrus.core import HydrusGlobals as HG
argparser = argparse.ArgumentParser( description = 'hydrus network client (console)' )
argparser.add_argument( '-d', '--db_dir', help = 'set an external db location' )
argparser.add_argument( '--temp_dir', help = 'override the program\'s temporary directory' )
argparser.add_argument( '--db_journal_mode', default = 'WAL', choices = [ 'WAL', 'TRUNCATE', 'PERSIST', 'MEMORY' ], help = 'change db journal mode (default=WAL)' )
argparser.add_argument( '--db_cache_size', type = int, help = 'override SQLite cache_size per db file, in MB (default=200)' )
argparser.add_argument( '--db_synchronous_override', type = int, choices = range(4), help = 'override SQLite Synchronous PRAGMA (default=2)' )
argparser.add_argument( '--no_db_temp_files', action='store_true', help = 'run db temp operations entirely in memory' )
argparser.add_argument( '--boot_debug', action='store_true', help = 'print additional bootup information to the log' )
argparser.add_argument( '--no_daemons', action='store_true', help = 'run without background daemons' )
argparser.add_argument( '--no_wal', action='store_true', help = 'OBSOLETE: run using TRUNCATE db journaling' )
argparser.add_argument( '--db_memory_journaling', action='store_true', help = 'OBSOLETE: run using MEMORY db journaling (DANGEROUS)' )
result = argparser.parse_args()
if result.db_dir is None:
db_dir = HC.DEFAULT_DB_DIR
if not HydrusPaths.DirectoryIsWritable( db_dir ) or HC.RUNNING_FROM_MACOS_APP:
db_dir = HC.USERPATH_DB_DIR
else:
db_dir = result.db_dir
db_dir = HydrusPaths.ConvertPortablePathToAbsPath( db_dir, HC.BASE_DIR )
try:
HydrusPaths.MakeSureDirectoryExists( db_dir )
except:
raise Exception( 'Could not ensure db path "{}" exists! Check the location is correct and that you have permission to write to it!'.format( db_dir ) )
if not os.path.isdir( db_dir ):
raise Exception( 'The given db path "{}" is not a directory!'.format( db_dir ) )
if not HydrusPaths.DirectoryIsWritable( db_dir ):
raise Exception( 'The given db path "{}" is not a writable-to!'.format( db_dir ) )
HG.no_daemons = result.no_daemons
HG.db_journal_mode = result.db_journal_mode
if result.no_wal:
HG.db_journal_mode = 'TRUNCATE'
if result.db_memory_journaling:
HG.db_journal_mode = 'MEMORY'
if result.db_cache_size is not None:
HG.db_cache_size = result.db_cache_size
else:
HG.db_cache_size = 200
if result.db_synchronous_override is not None:
HG.db_synchronous = int( result.db_synchronous_override )
else:
if HG.db_journal_mode == 'WAL':
HG.db_synchronous = 1
else:
HG.db_synchronous = 2
HG.no_db_temp_files = result.no_db_temp_files
HG.boot_debug = result.boot_debug
if result.temp_dir is not None:
HydrusPaths.SetEnvTempDir( result.temp_dir )
from hydrus.core import HydrusData
from hydrus.core import HydrusLogger
try:
from twisted.internet import reactor
except:
HG.twisted_is_broke = True
except Exception as e:
try:
from hydrus.core import HydrusData
HydrusData.DebugPrint( 'Critical boot error occurred! Details written to crash.log!' )
HydrusData.PrintException( e )
except:
pass
error_trace = traceback.format_exc()
print( error_trace )
if 'db_dir' in locals() and os.path.exists( db_dir ):
emergency_dir = db_dir
else:
emergency_dir = os.path.expanduser( '~' )
possible_desktop = os.path.join( emergency_dir, 'Desktop' )
if os.path.exists( possible_desktop ) and os.path.isdir( possible_desktop ):
emergency_dir = possible_desktop
dest_path = os.path.join( emergency_dir, 'hydrus_crash.log' )
with open( dest_path, 'w', encoding = 'utf-8' ) as f:
f.write( error_trace )
print( 'Critical boot error occurred! Details written to hydrus_crash.log in either db dir or user dir!' )
import sys
sys.exit( 1 )
controller = None
from hydrus import hydrus_client
with HydrusLogger.HydrusLogger( db_dir, 'client' ) as logger:
if __name__ == '__main__':
try:
HydrusData.Print( 'hydrus client started' )
if not HG.twisted_is_broke:
import threading
threading.Thread( target = reactor.run, name = 'twisted', kwargs = { 'installSignalHandlers' : 0 } ).start()
from hydrus.client import ClientController
controller = ClientController.Controller( db_dir )
controller.Run()
except:
HydrusData.Print( 'hydrus client failed' )
HydrusData.Print( traceback.format_exc() )
finally:
HG.view_shutdown = True
HG.model_shutdown = True
if controller is not None:
controller.pubimmediate( 'wake_daemons' )
if not HG.twisted_is_broke:
reactor.callFromThread( reactor.stop )
HydrusData.Print( 'hydrus client shut down' )
hydrus_client.boot()
HG.shutdown_complete = True
if HG.restart:
HydrusData.RestartProcess()

View File

@ -4,232 +4,9 @@
# You just DO WHAT THE FUCK YOU WANT TO.
# https://github.com/sirkris/WTFPL/blob/master/WTFPL.md
try:
import locale
try: locale.setlocale( locale.LC_ALL, '' )
except: pass
import os
import argparse
import traceback
from hydrus.core import HydrusBoot
HydrusBoot.AddBaseDirToEnvPath()
# initialise Qt here, important it is done early
from hydrus.client.gui import QtPorting as QP
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusPaths
from hydrus.core import HydrusGlobals as HG
argparser = argparse.ArgumentParser( description = 'hydrus network client (windowed)' )
argparser.add_argument( '-d', '--db_dir', help = 'set an external db location' )
argparser.add_argument( '--temp_dir', help = 'override the program\'s temporary directory' )
argparser.add_argument( '--db_journal_mode', default = 'WAL', choices = [ 'WAL', 'TRUNCATE', 'PERSIST', 'MEMORY' ], help = 'change db journal mode (default=WAL)' )
argparser.add_argument( '--db_cache_size', type = int, help = 'override SQLite cache_size per db file, in MB (default=200)' )
argparser.add_argument( '--db_synchronous_override', type = int, choices = range(4), help = 'override SQLite Synchronous PRAGMA (default=2)' )
argparser.add_argument( '--no_db_temp_files', action='store_true', help = 'run db temp operations entirely in memory' )
argparser.add_argument( '--boot_debug', action='store_true', help = 'print additional bootup information to the log' )
argparser.add_argument( '--no_daemons', action='store_true', help = 'run without background daemons' )
argparser.add_argument( '--no_wal', action='store_true', help = 'OBSOLETE: run using TRUNCATE db journaling' )
argparser.add_argument( '--db_memory_journaling', action='store_true', help = 'OBSOLETE: run using MEMORY db journaling (DANGEROUS)' )
result = argparser.parse_args()
if result.db_dir is None:
db_dir = HC.DEFAULT_DB_DIR
if not HydrusPaths.DirectoryIsWritable( db_dir ) or HC.RUNNING_FROM_MACOS_APP:
db_dir = HC.USERPATH_DB_DIR
else:
db_dir = result.db_dir
db_dir = HydrusPaths.ConvertPortablePathToAbsPath( db_dir, HC.BASE_DIR )
try:
HydrusPaths.MakeSureDirectoryExists( db_dir )
except:
raise Exception( 'Could not ensure db path "{}" exists! Check the location is correct and that you have permission to write to it!'.format( db_dir ) )
if not os.path.isdir( db_dir ):
raise Exception( 'The given db path "{}" is not a directory!'.format( db_dir ) )
if not HydrusPaths.DirectoryIsWritable( db_dir ):
raise Exception( 'The given db path "{}" is not a writable-to!'.format( db_dir ) )
HG.no_daemons = result.no_daemons
HG.db_journal_mode = result.db_journal_mode
if result.no_wal:
HG.db_journal_mode = 'TRUNCATE'
if result.db_memory_journaling:
HG.db_journal_mode = 'MEMORY'
if result.db_cache_size is not None:
HG.db_cache_size = result.db_cache_size
else:
HG.db_cache_size = 200
if result.db_synchronous_override is not None:
HG.db_synchronous = int( result.db_synchronous_override )
else:
if HG.db_journal_mode == 'WAL':
HG.db_synchronous = 1
else:
HG.db_synchronous = 2
HG.no_db_temp_files = result.no_db_temp_files
HG.boot_debug = result.boot_debug
if result.temp_dir is not None:
HydrusPaths.SetEnvTempDir( result.temp_dir )
from hydrus.core import HydrusData
from hydrus.core import HydrusLogger
try:
from twisted.internet import reactor
except:
HG.twisted_is_broke = True
except Exception as e:
try:
from hydrus.core import HydrusData
HydrusData.DebugPrint( 'Critical boot error occurred! Details written to crash.log!' )
HydrusData.PrintException( e )
except:
pass
error_trace = traceback.format_exc()
print( error_trace )
if 'db_dir' in locals() and os.path.exists( db_dir ):
emergency_dir = db_dir
else:
emergency_dir = os.path.expanduser( '~' )
possible_desktop = os.path.join( emergency_dir, 'Desktop' )
if os.path.exists( possible_desktop ) and os.path.isdir( possible_desktop ):
emergency_dir = possible_desktop
dest_path = os.path.join( emergency_dir, 'hydrus_crash.log' )
with open( dest_path, 'w', encoding = 'utf-8' ) as f:
f.write( error_trace )
print( 'Critical boot error occurred! Details written to hydrus_crash.log in either db dir or user dir!' )
import sys
sys.exit( 1 )
controller = None
from hydrus import hydrus_client
with HydrusLogger.HydrusLogger( db_dir, 'client' ) as logger:
if __name__ == '__main__':
try:
HydrusData.Print( 'hydrus client started' )
if not HG.twisted_is_broke:
import threading
threading.Thread( target = reactor.run, name = 'twisted', kwargs = { 'installSignalHandlers' : 0 } ).start()
from hydrus.client import ClientController
controller = ClientController.Controller( db_dir )
controller.Run()
except:
HydrusData.Print( 'hydrus client failed' )
HydrusData.Print( traceback.format_exc() )
finally:
HG.view_shutdown = True
HG.model_shutdown = True
if controller is not None:
controller.pubimmediate( 'wake_daemons' )
if not HG.twisted_is_broke:
reactor.callFromThread( reactor.stop )
HydrusData.Print( 'hydrus client shut down' )
HG.shutdown_complete = True
if HG.restart:
HydrusData.RestartProcess()
hydrus_client.boot()

View File

@ -8,6 +8,40 @@
<div class="content">
<h3 id="changelog"><a href="#changelog">changelog</a></h3>
<ul>
<li><h3 id="version_427"><a href="#version_427">version 427</a></h3></li>
<ul>
<li>ghost pending tags:</li>
<li>fixed another ghost pending tags bug. this may have been new or there since the display cache started, I am not sure, but it shouldn't happen again. it was occuring when a pending tag was being committed to 'current' and another tag in its sibling group already existed as a current tag for that file. the pending tag and its count would not clear for non-'all known files' domains, causing ghosts to appear in search pages but not typically manage tags. you may have noticed ghost tags hanging around after a pending commit--this was it. the true problem here was in the 'rescind pending' action that occurs just before an add/commit. a new unit test tests for this situation both for two non-ideal tags being pend-merged, and a non-ideal tag being pend-merged into the existing ideal</li>
<li>wrote a routine to regenerate _pending_ tag storage and autocomplete counts from scratch for the combined and specific display tag caches. this job is special in that it regens tags instantly without having to reset sibling/parent sync. you can run the job from the database->regen menu. this is the start of 'fix just this tag' maintenance ability</li>
<li>the pending regen routines will occur on update. it shouldn't take long at all, unless you have five million tags pending, where it could be a couple minutes</li>
<li>.</li>
<li>autocomplete shortcuts:</li>
<li>there is a new shortcut set under _file->shortcuts_ just for tag autocomplete shortcuts. any 'switch searching immediately' shortcut previously on 'main gui' will be migrated over</li>
<li>the tag autocomplete input text box is now plugged into the new shortcut system and uses this set</li>
<li>migrated previously hardcoded autocomplete shortcuts to the shortcut system (defaults):</li>
<li>- force search now, for when you have automatic searching turned off (ctrl+space)</li>
<li>- enable IME-friendly mode (insert)</li>
<li>- if input empty, move left/right a tab (left/right arrow)</li>
<li>- if input empty, move left/right a service page (up/down arrow)</li>
<li>- if input empty and on media viewer, move to previous/next media (page up/down)</li>
<li>misc improvements to my shortcut handler</li>
<li>misc shortcut code cleanup</li>
<li>.</li>
<li>the rest:</li>
<li>I fixed a bad example url in the new gelbooru file page parser that was sometimes leading to a link to the gallery url class. this was an artifact of an old experiment with md5-search parsing, now fixed with newer redirection tech. the updated parser is folded into update, and if you ended up with the incorrect link, it should be detected, dissolved, and re-linked with the file page parser</li>
<li>thanks to a user report, wrote a new url class for 420chan's newer thread url format</li>
<li>sorting a gallery downloader or thread watcher multi-column list by 'status' should now group 'done' and 'paused' items separately</li>
<li>fixed a bug in the /add_tags/add_tags Client API call when checking some petitioned tags data types. cleaned all that code, it was ugly (issue #788)</li>
<li>added unit tests for /add_tags/add_tags to test the service_names_to_actions_to_tags parameter better and repository actions, including petitioning with and without specified reason</li>
<li>.</li>
<li>code refactoring:</li>
<li>finally addressing the near-1MB ClientDB file, I have started a framework to break the db into separate modules with their own creation/repair/work responsibilities. this will make the file easier to work on, maintain, update, and test. this week starts off simple, with the master definitions being peeled off into hashes, tags, urls, and texts submodules</li>
<li>cleaned some misc code around here, including a bunch of related decoupling</li>
<li>ClientDB.py is now in its own 'db' module as well. the db will further fracture and this module will gain more files in future</li>
<li>the boot code in the launch scripts is now migrated to the 'hydrus' directory, with the actual launch scripts now doing nicer __main__ checks to not launch the program if you want to play around with importing hydrus. more work to come here</li>
<li>finished the help's header linking job--all headers across the help are now #fragment links</li>
<li>misc help cleanup</li>
</ul>
<li><h3 id="version_426"><a href="#version_426">version 426</a></h3></li>
<ul>
<li>misc:</li>
@ -36,7 +70,7 @@
<li>the parser 'show what this can parse in nice text' routine now fails gracefully</li>
<li>multi-column lists now handle a situation where either the display or sort data for a row cannot be generated. a single error popup per list will be generated so as not to spam, bad sorts will be put at the top, and 'unable to render' will occupy all display cells</li>
<li>.</li>
<li>network server stuff</li>
<li>network server stuff:</li>
<li>fixed being able to delete an account type in the server admin menu</li>
<li>the way accounts are checked for permissions serverside now works how the client api does it, unified into a neater system that checks before the job starts</li>
<li>did some misc server code cleanup, and clientside, prepped for restoring account modification and future improvements</li>

View File

@ -43,17 +43,17 @@
<p>There is now a simple 'session' system, where you can get a temporary key that gives the same access without having to include the permanent access key in every request. You can fetch a session key with the <a href="#session_key">/session_key</a> command and thereafter use it just as you would an access key, just with <i>Hydrus-Client-API-Session-Key</i> instead.</p>
<p>Session keys will expire if they are not used within 24 hours, or if the client is restarted, or if the underlying access key is deleted. An invalid/expired session key will give a <b>419</b> result with an appropriate error text.</p>
<p>Bear in mind the Client API is still under construction and is http-only for the moment--be careful about transmitting sensitive content outside of localhost. The access key will be unencrypted across any connection, and if it is included as a GET parameter, as simple and convenient as that is, it could be cached in all sorts of places.</p>
<h3 id="contents"><a href="#contents">>Contents</h3>
<h3 id="contents"><a href="#contents">Contents</a></h3>
<ul>
<li>
<h4>Access Management</h4>
<h4><a href="#access_management">Access Management</a></h4>
<ul>
<li><a href="#api_version">GET /api_version</a></li>
<li><a href="#request_new_permissions">GET /request_new_permissions</a></li>
<li><a href="#session_key">GET /session_key</a></li>
<li><a href="#verify_access_key">GET /verify_access_key</a></li>
</ul>
<h4>Adding Files</h4>
<h4><a href="#adding_files">Adding Files</a></h4>
<ul>
<li><a href="#add_files_add_file">POST /add_files/add_file</a></li>
<li><a href="#add_files_delete_files">POST /add_files/delete_files</a></li>
@ -61,31 +61,31 @@
<li><a href="#add_files_archive_files">POST /add_files/archive_files</a></li>
<li><a href="#add_files_unarchive_files">POST /add_files/unarchive_files</a></li>
</ul>
<h4>Adding Tags</h4>
<h4><a href="#adding_tags">Adding Tags</a></h4>
<ul>
<li><a href="#add_tags_clean_tags">GET /add_tags/clean_tags</a></li>
<li><a href="#add_tags_get_tag_services">GET /add_tags/get_tag_services</a></li>
<li><a href="#add_tags_add_tags">POST /add_tags/add_tags</a></li>
</ul>
<h4>Adding URLs</h4>
<h4><a href="#adding_urls">Adding URLs</a></h4>
<ul>
<li><a href="#add_urls_get_url_files">GET /add_urls/get_url_files</a></li>
<li><a href="#add_urls_get_url_info">GET /add_urls/get_url_info</a></li>
<li><a href="#add_urls_add_url">POST /add_urls/add_url</a></li>
<li><a href="#add_urls_associate_url">POST /add_urls/associate_url</a></li>
</ul>
<h4>Managing Cookies</h4>
<h4><a href="#managing_cookies">Managing Cookies</a></h4>
<ul>
<li><a href="#manage_cookies_get_cookies">GET /manage_cookies/get_cookies</a></li>
<li><a href="#manage_cookies_set_cookies">POST /manage_cookies/set_cookies</a></li>
</ul>
<h4>Managing Pages</h4>
<h4><a href="#managing_pages">Managing Pages</a></h4>
<ul>
<li><a href="#manage_pages_get_pages">GET /manage_pages/get_pages</a></li>
<li><a href="#manage_pages_get_page_info">GET /manage_pages/get_page_info</a></li>
<li><a href="#manage_pages_focus_page">POST /manage_pages/focus_page</a></li>
</ul>
<h4>Searching and Fetching Files</h4>
<h4><a href="#searching_files">Searching and Fetching Files</a></h4>
<ul>
<li><a href="#get_files_search_files">GET /get_files/search_files</a></li>
<li><a href="#get_files_file_metadata">GET /get_files/file_metadata</a></li>
@ -93,9 +93,9 @@
<li><a href="#get_files_thumbnail">GET /get_files/thumbnail</a></li>
</ul>
</ul>
<h3>Access Management</h3>
<div class="apiborder" id="api_version">
<h3><b>GET /api_version</b></h3>
<h3 id="access_management"><a href="#access_management">Access Management</a></h3>
<div class="apiborder">
<h3 id="api_version"><a href="#api_version"><b>GET /api_version</b></a></h3>
<p><i>Gets the current API version. I will increment this every time I alter the API.</i></p>
<ul>
<li><p>Restricted access: NO.</p></li>
@ -110,8 +110,8 @@
</li>
</ul>
</div>
<div class="apiborder" id="request_new_permissions">
<h3><b>GET /request_new_permissions</b></h3>
<div class="apiborder">
<h3 id="request_new_permissions"><a href="#request_new_permissions"><b>GET /request_new_permissions</b></a></h3>
<p><i>Register a new external program with the client. This requires the 'add from api request' mini-dialog under </i>services->review services<i> to be open, otherwise it will 403.</i></p>
<ul>
<li><p>Restricted access: NO.</p></li>
@ -149,8 +149,8 @@
</li>
</ul>
</div>
<div class="apiborder" id="session_key">
<h3><b>GET /session_key</b></h3>
<div class="apiborder">
<h3 id="session_key"><a href="#session_key"><b>GET /session_key</b></a></h3>
<p><i>Get a new session key.</i></p>
<ul>
<li><p>Restricted access: YES. No permissions required.</p></li>
@ -171,8 +171,8 @@
</li>
</ul>
</div>
<div class="apiborder" id="verify_access_key">
<h3><b>GET /verify_access_key</b></h3>
<div class="apiborder">
<h3 id="verify_access_key"><a href="#verify_access_key"><b>GET /verify_access_key</b></a></h3>
<p><i>Check your access key is valid.</i></p>
<ul>
<li><p>Restricted access: YES. No permissions required.</p></li>
@ -192,9 +192,9 @@
</li>
</ul>
</div>
<h3>Adding Files</h3>
<div class="apiborder" id="add_files_add_file">
<h3><b>POST /add_files/add_file</b></h3>
<h3 id="adding_files"><a href="#adding_files">Adding Files</a></h3>
<div class="apiborder">
<h3 id="add_files_add_file"><a href="#add_files_add_file"><b>POST /add_files/add_file</b></a></h3>
<p><i>Tell the client to import a file.</i></p>
<ul>
<li><p>Restricted access: YES. Import Files permission needed.</p></li>
@ -232,8 +232,8 @@
</li>
</ul>
</div>
<div class="apiborder" id="add_files_delete_files">
<h3><b>POST /add_files/delete_files</b></h3>
<div class="apiborder">
<h3 id="add_files_delete_files"><a href="#add_files_delete_files"><b>POST /add_files/delete_files</b></a></h3>
<p><i>Tell the client to send files to the trash.</i></p>
<ul>
<li><p>Restricted access: YES. Import Files permission needed.</p></li>
@ -259,8 +259,8 @@
</li>
</ul>
</div>
<div class="apiborder" id="add_files_undelete_files">
<h3><b>POST /add_files/undelete_files</b></h3>
<div class="apiborder">
<h3 id="add_files_undelete_files"><a href="#add_files_undelete_files"><b>POST /add_files/undelete_files</b></a></h3>
<p><i>Tell the client to pull files back out of the trash.</i></p>
<ul>
<li><p>Restricted access: YES. Import Files permission needed.</p></li>
@ -286,8 +286,8 @@
</li>
</ul>
</div>
<div class="apiborder" id="add_files_archive_files">
<h3><b>POST /add_files/archive_files</b></h3>
<div class="apiborder">
<h3 id="add_files_archive_files"><a href="#add_files_archive_files"><b>POST /add_files/archive_files</b></a></h3>
<p><i>Tell the client to archive inboxed files.</i></p>
<ul>
<li><p>Restricted access: YES. Import Files permission needed.</p></li>
@ -313,8 +313,8 @@
</li>
</ul>
</div>
<div class="apiborder" id="add_files_unarchive_files">
<h3><b>POST /add_files/unarchive_files</b></h3>
<div class="apiborder">
<h3 id="add_files_unarchive_files"><a href="#add_files_unarchive_files"><b>POST /add_files/unarchive_files</b></a></h3>
<p><i>Tell the client re-inbox archived files.</i></p>
<ul>
<li><p>Restricted access: YES. Import Files permission needed.</p></li>
@ -340,9 +340,9 @@
</li>
</ul>
</div>
<h3>Adding Tags</h3>
<div class="apiborder" id="add_tags_clean_tags">
<h3><b>GET /add_tags/clean_tags</b></h3>
<h3 id="adding_tags"><a href="#adding_tags">Adding Tags</a></h3>
<div class="apiborder">
<h3 id="add_tags_clean_tags"><a href="#add_tags_clean_tags"><b>GET /add_tags/clean_tags</b></a></h3>
<p><i>Ask the client about how it will see certain tags.</i></p>
<ul>
<li><p>Restricted access: YES. Add Tags permission needed.</p></li>
@ -374,8 +374,8 @@
</li>
</ul>
</div>
<div class="apiborder" id="add_tags_get_tag_services">
<h3><b>GET /add_tags/get_tag_services</b></h3>
<div class="apiborder">
<h3 id="add_tags_get_tag_services"><a href="#add_tags_get_tag_services"><b>GET /add_tags/get_tag_services</b></a></h3>
<p><i>Ask the client about its tag services.</i></p>
<ul>
<li><p>Restricted access: YES. Add Tags permission needed.</p></li>
@ -398,8 +398,8 @@
</li>
</ul>
</div>
<div class="apiborder" id="add_tags_add_tags">
<h3><b>POST /add_tags/add_tags</b></h3>
<div class="apiborder">
<h3 id="add_tags_add_tags"><a href="#add_tags_add_tags"><b>POST /add_tags/add_tags</b></a></h3>
<p><i>Make changes to the tags that files have.</i></p>
<ul>
<li><p>Restricted access: YES. Add Tags permission needed.</p></li>
@ -462,9 +462,9 @@
<p>Note also that hydrus tag actions are safely idempotent. You can pend a tag that is already pended and not worry about an error--it will be discarded. The same for other reasonable logical scenarios: deleting a tag that does not exist will silently make no change, pending a tag that is already 'current' will again be passed over. It is fine to just throw 'process this' tags at every file import you add and not have to worry about checking which files you already added it to.</p>
</ul>
</div>
<h3>Adding URLs</h3>
<div class="apiborder" id="add_urls_get_url_files">
<h3><b>GET /add_urls/get_url_files</b></h3>
<h3 id="adding_urls"><a href="#adding_urls">Adding URLs</a></h3>
<div class="apiborder">
<h3 id="add_urls_get_url_files"><a href="#add_urls_get_url_files"><b>GET /add_urls/get_url_files</b></a></h3>
<p><i>Ask the client about an URL's files.</i></p>
<ul>
<li><p>Restricted access: YES. Import URLs permission needed.</p></li>
@ -511,8 +511,8 @@
</li>
</ul>
</div>
<div class="apiborder" id="add_urls_get_url_info">
<h3><b>GET /add_urls/get_url_info</b></h3>
<div class="apiborder">
<h3 id="add_urls_get_url_info"><a href="#add_urls_get_url_info"><b>GET /add_urls/get_url_info</b></a></h3>
<p><i>Ask the client for information about a URL.</i></p>
<ul>
<li><p>Restricted access: YES. Import URLs permission needed.</p></li>
@ -557,8 +557,8 @@
</li>
</ul>
</div>
<div class="apiborder" id="add_urls_add_url">
<h3><b>POST /add_urls/add_url</b></h3>
<div class="apiborder">
<h3 id="add_urls_add_url"><a href="#add_urls_add_url"><b>POST /add_urls/add_url</b></a></h3>
<p><i>Tell the client to 'import' a URL. This triggers the exact same routine as drag-and-dropping a text URL onto the main client window.</i></p>
<ul>
<li><p>Restricted access: YES. Import URLs permission needed. Add Tags needed to include tags.</p></li>
@ -650,8 +650,8 @@
</li>
</ul>
</div>
<div class="apiborder" id="add_urls_associate_url">
<h3><b>POST /add_urls/associate_url</b></h3>
<div class="apiborder">
<h3 id="add_urls_associate_url"><a href="#add_urls_associate_url"><b>POST /add_urls/associate_url</b></a></h3>
<p><i>Manage which URLs the client considers to be associated with which files.</i></p>
<ul>
<li><p>Restricted access: YES. Import URLs permission needed.</p></li>
@ -686,10 +686,11 @@
</li>
<li><p>Response description: 200 with no content. Like when adding tags, this is safely idempotent--do not worry about re-adding URLs associations that already exist or accidentally trying to delete ones that don't.</p></li>
</ul>
</div><h3>Managing Cookies</h3>
</div>
<h3 id="managing_cookies"><a href="#managing_cookies">Managing Cookies</a></h3>
<p>This refers to the cookies held in the client's session manager, which are sent with network requests to different domains.</p>
<div class="apiborder" id="manage_cookies_get_cookies">
<h3><b>GET /manage_cookies/get_cookies</b></h3>
<div class="apiborder">
<h3 id="manage_cookies_get_cookies"><a href="#manage_cookies_get_cookies"><b>GET /manage_cookies/get_cookies</b></a></h3>
<p><i>Get the cookies for a particular domain.</i></p>
<ul>
<li><p>Restricted access: YES. Manage Cookies permission needed.</p></li>
@ -722,8 +723,8 @@
<p>This request will also return any cookies for subdomains. The session system in hydrus generally stores cookies according to the second-level domain, so if you request for specific.someoverbooru.net, you will still get the cookies for someoverbooru.net and all its subdomains.</p>
</ul>
</div>
<div class="apiborder" id="manage_cookies_set_cookies">
<h3><b>POST /manage_cookies/set_cookies</b></h3>
<div class="apiborder">
<h3 id="manage_cookies_set_cookies"><a href="#manage_cookies_set_cookies"><b>POST /manage_cookies/set_cookies</b></a></h3>
<p>Set some new cookies for the client. This makes it easier to 'copy' a login from a web browser or similar to hydrus if hydrus's login system can't handle the site yet.</p>
<ul>
<li><p>Restricted access: YES. Manage Cookies permission needed.</p></li>
@ -756,10 +757,10 @@
<p>Expires can be null, but session cookies will time-out in hydrus after 60 minutes of non-use.</p>
</ul>
</div>
<h3>Managing Pages</h3>
<h3 id="managing_pages"><a href="#managing_pages">Managing Pages</a></h3>
<p>This refers to the pages of the main client UI.</p>
<div class="apiborder" id="manage_pages_get_pages">
<h3><b>GET /manage_pages/get_pages</b></h3>
<div class="apiborder">
<h3 id="manage_pages_get_pages"><a href="#manage_pages_get_pages"><b>GET /manage_pages/get_pages</b></a></h3>
<p><i>Get the page structure of the current UI session.</i></p>
<ul>
<li><p>Restricted access: YES. Manage Pages permission needed.</p></li>
@ -833,8 +834,8 @@
<p>The 'page_key' is a unique identifier for the page. It will stay the same for a particular page throughout the session, but new ones are generated on a client restart or other session reload.</p>
</ul>
</div>
<div class="apiborder" id="manage_pages_get_page_info">
<h3><b>GET /manage_pages/get_page_info</b></h3>
<div class="apiborder">
<h3 id="manage_pages_get_page_info"><a href="#manage_pages_get_page_info"><b>GET /manage_pages/get_page_info</b></a></h3>
<p><i>Get information about a specific page.</i></p>
<p class="warning">This is under construction. The current call dumps a ton of info for different downloader pages. Please experiment in IRL situations and give feedback for now! I will flesh out this help with more enumeration info and examples as this gets nailed down. POST commands to alter pages (adding, removing, highlighting), will come later.</p>
<ul>
@ -929,8 +930,8 @@
</li>
</ul>
</div>
<div class="apiborder" id="manage_pages_focus_page">
<h3><b>POST /manage_pages/focus_page</b></h3>
<div class="apiborder">
<h3 id="manage_pages_focus_page"><a href="#manage_pages_focus_page"><b>POST /manage_pages/focus_page</b></a></h3>
<p><i>'Show' a page in the main GUI, making it the current page in view. If it is already the current page, no change is made.</i></p>
<ul>
<li><p>Restricted access: YES. Manage Pages permission needed.</p></li>
@ -960,10 +961,10 @@
<li><p>Response description: 200 with no content. If the page key is not found, this will 404.</p></li>
</ul>
</div>
<h3>Searching Files</h3>
<h3 id="searching_files"><a href="#searching_files">Searching Files</a></h3>
<p>File search in hydrus is not paginated like a booru--all searches return all results in one go. In order to keep this fast, search is split into two steps--fetching file identifiers with a search, and then fetching file metadata in batches. You may have noticed that the client itself performs searches like this--thinking a bit about a search and then bundling results in batches of 256 files before eventually throwing all the thumbnails on screen.</p>
<div class="apiborder" id="get_files_search_files">
<h3><b>GET /get_files/search_files</b></h3>
<div class="apiborder">
<h3 id="get_files_search_files"><a href="#get_files_search_files"><b>GET /get_files/search_files</b></a></h3>
<p><i>Search for the client's files.</i></p>
<ul>
<li><p>Restricted access: YES. Search for Files permission needed. Additional search permission limits may apply.</p></li>
@ -999,8 +1000,8 @@
<p>Note that most clients will have an invisible system:limit of 10,000 files on all queries. I expect to add more system predicates to help searching for untagged files, but it is tricky to fetch all files under any circumstance. Large queries may take several seconds to respond.</p>
</ul>
</div>
<div class="apiborder" id="get_files_file_metadata">
<h3><b>GET /get_files/file_metadata</b></h3>
<div class="apiborder">
<h3 id="get_files_file_metadata"><a href="#get_files_file_metadata"><b>GET /get_files/file_metadata</b></a></h3>
<p><i>Get metadata about files in the client.</i></p>
<ul>
<li><p>Restricted access: YES. Search for Files permission needed. Additional search permission limits may apply.</p></li>
@ -1158,8 +1159,8 @@
</ul>
</div>
<div class="apiborder" id="get_files_file">
<h3><b>GET /get_files/file</b></h3>
<div class="apiborder">
<h3 id="get_files_file"><a href="#get_files_file"><b>GET /get_files/file</b></a></h3>
<p><i>Get a file.</i></p>
<ul>
<li><p>Restricted access: YES. Search for Files permission needed. Additional search permission limits may apply.</p></li>
@ -1182,8 +1183,8 @@
<li><p>Response description: The file itself. You should get the correct mime type as the Content-Type header.</p></li>
</ul>
</div>
<div class="apiborder" id="get_files_thumbnail">
<h3><b>GET /get_files/thumbnail</b></h3>
<div class="apiborder">
<h3 id="get_files_thumbnail"><a href="#get_files_thumbnail"><b>GET /get_files/thumbnail</b></a></h3>
<p><i>Get a file's thumbnail.</i></p>
<ul>
<li><p>Restricted access: YES. Search for Files permission needed. Additional search permission limits may apply.</p></li>

View File

@ -6,7 +6,7 @@
</head>
<body>
<div class="content">
<h3>the hydrus database</h3>
<h3 id="intro"><a href="#intro">the hydrus database</a></h3>
<p>A hydrus client consists of three components:</p>
<ol>
<li>
@ -25,10 +25,10 @@
<p>Thumbnails tend to be fetched dozens at a time, so it is, again, ideal if they are stored on an SSD. Your regular media files--which on many clients total hundreds of GB--are usually fetched one at a time for human consumption and do not benefit from the expensive low-latency of an SSD. They are best stored on a cheap HDD, and, if desired, also work well across a network file system.</p>
</li>
</ol>
<h3>these components can be put on different drives</h3>
<h3 id="different_drives"><a href="#different_drives">these components can be put on different drives</a></h3>
<p>Although an initial install will keep these parts together, it is possible to, say, run the database on a fast drive but keep your media in cheap slow storage. This is an excellent arrangement that works for many users. And if you have a very large collection, you can even spread your files across multiple drives. It is not very technically difficult, but I do not recommend it for new users.</p>
<p>Backing such an arrangement up is obviously more complicated, and the internal client backup is not sophisticated enough to capture everything, so I recommend you figure out a broader solution with a third-party backup program like FreeFileSync.</p>
<h3>pulling your media apart</h3>
<h3 id="pulling_media_apart"><a href="#pulling_media_apart">pulling your media apart</a></h3>
<p><b class="warning">As always, I recommend creating a backup before you try any of this, just in case it goes wrong.</b></p>
<p>If you would like to move your files and thumbnails to new locations, I generally recommend you not move their folders around yourself--the database has an internal knowledge of where it thinks its file and thumbnail folders are, and if you move them while it is closed, it will become confused and you will have to manually relocate what is missing on the next boot via a repair dialog. This is not impossible to figure out, but if the program's 'client files' folder confuses you at all, I'd recommend you stay away. Instead, you can simply do it through the gui:</p>
<p>Go <i>database->migrate database</i>, giving you this dialog:</p>
@ -39,7 +39,7 @@
<p><b>Weight</b> means the relative amount of media you would like to store in that location. It only matters if you are spreading your files across multiple locations. If location A has a weight of 1 and B has a weight of 2, A will get approximately one third of your files and B will get approximately two thirds.</p>
<p>The operations on this dialog are simple and atomic--at no point is your db ever invalid. Once you have the locations and ideal usage set how you like, hit the 'move files now' button to actually shuffle your files around. It will take some time to finish, but you can pause and resume it later if the job is large or you want to undo or alter something.</p>
<p>If you decide to move your actual database, the program will have to shut down first. Before you boot up again, you will have to create a new program shortcut:</p>
<h3>informing the software that the database is not in the default location</h3>
<h3 id="launch_parameter"><a href="#launch_parameter">informing the software that the database is not in the default location</a></h3>
<p>A straight call to the client executable will look for a database in <i>install_dir/db</i>. If one is not found, it will create one. So, if you move your database and then try to run the client again, it will try to create a new empty database in the previous location!</p>
<p>So, pass it a -d or --db_dir command line argument, like so:</p>
<ul>
@ -53,9 +53,9 @@
<p>Rather than typing the path out in a terminal every time you want to launch your external database, create a new shortcut with the argument in. Something like this, which is from my main development computer and tests that a fresh default install will run an existing database ok:</p>
<p><img src="db_migration_shortcut.png" /></p>
<p>Note that an install with an 'external' database no longer needs access to write to its own path, so you can store it anywhere you like, including protected read-only locations (e.g. in 'Program Files'). If you do move it, just double-check your shortcuts are still good and you are done.</p>
<h3>finally</h3>
<h3 id="finally"><a href="#finally">finally</a></h3>
<p>If your database now lives in one or more new locations, make sure to update your backup routine to follow them!</p>
<h3 id="ssd_example"><a href="#ssd_example">moving to an SSD</a></h3>
<h3 id="to_an_ssd"><a href="#to_an_ssd">moving to an SSD</a></h3>
<p>As an example, let's say you started using the hydrus client on your HDD, and now you have an SSD available and would like to move your thumbnails and main install to that SSD to speed up the client. Your database will be valid and functional at every stage of this, and it can all be undone. The basic steps are:</p>
<ol>
<li>Move your 'fast' files to the fast location.</li>
@ -83,7 +83,7 @@
</ul>
<p>You should now have <i>something</i> like this:</p>
<p><img src="db_migration_example.png" /></p>
<h3>p.s. running multiple clients</h3>
<h3 id="multiple_clients"><a href="#multiple_clients">p.s. running multiple clients</a></h3>
<p>Since you now know how to tell the software about an external database, you can, if you like, run multiple clients from the same install (and if you previously had multiple install folders, now you can now just use the one). Just make multiple shortcuts to the same client executable but with different database directories. They can run at the same time. You'll save yourself a little memory and update-hassle. I do this on my laptop client to run a regular client for my media and a separate 'admin' client to do PTR petitions and so on.</p>
</div>
</body>

View File

@ -7,7 +7,7 @@
<body>
<div class="content">
<p><a href="downloader_parsers.html"><---- Back to Parsers</a></p>
<h3>putting it all together</h3>
<h3 id="finally"><a href="#finally">putting it all together</a></h3>
<p>Now you know what GUGs, URL Classes, and Parsers are, you should have some ideas of how URL Classes could steer what happens when the downloader is faced with an URL to process. Should a URL be imported as a media file, or should it be parsed? If so, how?</p>
<p>You may have noticed in the Edit GUG ui that it lists if a current URL Class matches the example URL output. If the GUG has no matching URL Class, it won't be listed in the main 'gallery selector' button's list--it'll be relegated to the 'non-functioning' page. Without a URL Class, the client doesn't know what to do with the output of that GUG. But if a URL Class does match, we can then hand the result over to a parser set at <i>network->downloader definitions->manage url class links</i>:</p>
<p><img src="downloader_completion_url_links.png" /></p>
@ -18,4 +18,4 @@
<p class="right"><a href="downloader_sharing.html">Now let's share what we've made ----></a></p>
</div>
</body>
</html>
</html>

View File

@ -7,7 +7,7 @@
<body>
<div class="content">
<p><a href="downloader_intro.html"><---- Back to the introduction</a></p>
<h3>GUGs</h3>
<h3 id="intro"><a href="#intro">GUGs</a></h3>
<p>Gallery URL Generators, or <b>GUGs</b> are simple objects that take a simple string from the user, like:</p>
<ul>
<li>blue_eyes</li>
@ -27,7 +27,7 @@
<li><a href="https://danbooru.donmai.us/posts?page=1&tags=goth*+order:id_asc">https://danbooru.donmai.us/posts?page=1&tags=goth*+order:id_asc</a></li>
</ul>
<p>These are all the 'first page' of the results if you type or click-through to the same location on those sites. We are essentially emulating their own simple search-url generation inside the hydrus client.</p>
<h3>actually doing it</h3>
<h3 id="doing_it"><a href="#doing_it">actually doing it</a></h3>
<p>Although it is usually a fairly simple process of just substituting the inputted tags into a string template, there are a couple of extra things to think about. Let's look at the ui under <i>network->downloader definitions->manage gugs</i>:</p>
<p><img src="downloader_edit_gug_panel.png" /></p>
<p>The client will split whatever the user enters by whitespace, so 'blue_eyes blonde_hair' becomes two <i>search terms</i>, [ 'blue_eyes', 'blonde_hair' ], which are then joined back together with the given 'search terms separator', to make 'blue_eyes+blonde_hair'. Different sites use different separators, although ' ', '+', and ',' are most common. The new string is substituted into the '%tags%' in the template phrase, and the URL is made.</p>
@ -36,9 +36,9 @@
<p>The name of the GUG is important, as this is what will be listed when the user chooses what 'downloader' they want to use. Make sure it has a clear unambiguous name.</p>
<p>The initial search text is also important. Most downloaders just take some text tags, but if your GUG expects a numerical artist id (like pixiv artist search does), you should specify that explicitly to the user. You can even put in a brief '(two tag maximum)' type of instruction if you like.</p>
<p>Notice that the Deviart Art example above is actually the stream of wlop's <i>favourites</i>, not his works, and without an explicit notice of that, a user could easily mistake what they have selected. 'gelbooru' or 'newgrounds' are bad names, 'type here' is a bad initialising text.</p>
<h3>Nested GUGs</h3>
<h3 id="nested_gugs"><a href="#nested_gugs">Nested GUGs</a></h3>
<p>Nested Gallery URL Generators are GUGs that hold other GUGs. Some searches actually use more than one stream (such as a Hentai Foundry artist lookup, where you might want to get both their regular works and their scraps, which are two separate galleries under the site), so NGUGs allow you to generate multiple initialising URLs per input. You can experiment with this ui if you like--it isn't too complicated--but you might want to hold off doing anything for real until you are comfortable with everything and know how producing multiple initialising URLs is going to work in the actual downloader.</p>
<p class="right"><a href="downloader_url_classes.html">Now let's teach the client to recognise that Gallery URL ----></a></p>
</div>
</body>
</html>
</html>

View File

@ -7,23 +7,23 @@
<body>
<div class="content">
<p class="warning">Creating custom downloaders is only for advanced users who understand HTML or JSON. Beware! If you are simply looking for how to add new downloaders, please head over <a href="adding_new_downloaders.html">here</a>.</p>
<h3>this system</h3>
<h3 id="intro"><a href="#intro">this system</a></h3>
<p>The first versions of hydrus's downloaders were all hardcoded and static--I wrote everything into the program itself and nothing was user-creatable or -fixable. After the maintenance burden of the entire messy system proved too large for me to keep up with and a semi-editable booru system proved successful, I decided to overhaul the entire thing to allow user creation and sharing of every component. It is designed to be very simple to the front-end user--they will typically handle a couple of png files and then select a new downloader from a list--but very flexible (and hence potentially complicated) on the back-end. These help pages describe the different compontents with the intention of making an HTML- or JSON- fluent user able to create and share a full new downloader on their own.</p>
<p>As always, this is all under active development. Your feedback on the system would be appreciated, and if something is confusing or you discover something in here that is out of date, please <a href="contact.html">let me know</a>.</p>
<h3>what is a downloader?</h3>
<h3 id="downloader"><a href="#downloader">what is a downloader?</a></h3>
<p>In hydrus, a downloader is one of:</p>
<ul>
<li><h3>Gallery Downloader</h3></li>
<li><h4>Gallery Downloader</h4></li>
<li>This takes a string like 'blue_eyes' to produce a series of thumbnail gallery page URLs that can be parsed for image page URLs which can ultimately be parsed for file URLs and metadata like tags. Boorus fall into this category.</li>
<li><h3>URL Downloader</h3></li>
<li><h4>URL Downloader</h4></li>
<li>This does just the Gallery Downloader's back-end--instead of taking a string query, it takes the gallery or post URLs directly from the user, whether that is one from a drag-and-drop event or hundreds pasted from clipboard. For our purposes here, the URL Downloader is a subset of the Gallery Downloader.</li>
<li><h3>Watcher</h3></li>
<li><h4>Watcher</h4></li>
<li>This takes a URL that it will check in timed intervals, parsing it for new URLs that it then queues up to be downloaded. It typically stops checking after the 'file velocity' (such as '1 new file per day') drops below a certain level. It is mostly for watching imageboard threads.</li>
<li><h3>Simple Downloader</h3></li>
<li><h4>Simple Downloader</h4></li>
<li>This takes a URL one-time and parses it for direct file URLs. This is a miscellaneous system for certain simple gallery types and some testing/'I just need the third &lt;img&gt; tag's <i>src</i> on this one page' jobs.</li>
</ul>
<p>The system currently supports HTML and JSON parsing. XML should be fine under the HTML parser--it isn't strict about checking types and all that.</p>
<h3>what does a downloader do?</h3>
<h3 id="pipeline"><a href="#pipeline">what does a downloader do?</a></h3>
<p>The Gallery Downloader is the most complicated downloader and uses all the possible components. In order for hydrus to convert our example 'blue_eyes' query into a bunch of files with tags, it needs to:</p>
<ul>
<li>Present some user interface named 'safebooru tag search' to the user that will convert their input of 'blue_eyes' into <a href="https://safebooru.org/index.php?page=post&s=list&tags=blue_eyes&pid=0">https://safebooru.org/index.php?page=post&s=list&tags=blue_eyes&pid=0</a>.</li>
@ -42,4 +42,4 @@
<p class="right"><a href="downloader_gugs.html">Let's first learn about GUGs ----></a></p>
</div>
</body>
</html>
</html>

View File

@ -7,9 +7,9 @@
<body>
<div class="content">
<p><a href="downloader_sharing.html"><---- Back to sharing</a></p>
<h3>login</h3>
<p>This is not done yet!</p>
<h3 id="intro"><a href="#intro">login</a></h3>
<p>The system works, but this help was never done! Check the defaults for examples of how it works, sorry!</p>
<p class="right"><a href="index.html">Back to the index ----></a></p>
</div>
</body>
</html>
</html>

View File

@ -7,7 +7,7 @@
<body>
<div class="content">
<p><a href="downloader_url_classes.html"><---- Back to URL Classes</a></p>
<h3>parsers</h3>
<h3 id="intro"><a href="#intro">parsers</a></h3>
<p>In hydrus, a parser is an object that takes a single block of HTML or JSON data and returns many kinds of hydrus-level metadata.</p>
<p>Parsers are flexible and potentially quite complicated. You might like to open <i>network->manage parsers</i> and explore the UI as you read these pages. Check out how the default parsers already in the client work, and if you want to write a new one, see if there is something already in there that is similar--it is usually easier to duplicate an existing parser and then alter it than to create a new one from scratch every time.</p>
<p>There are three main components in the parsing system (click to open each component's help page):</p>
@ -35,4 +35,4 @@
<p class="right"><a href="downloader_completion.html">Taken a break? Now let's put it all together ----></a></p>
</div>
</body>
</html>
</html>

View File

@ -15,7 +15,7 @@
<p>The current content types are:</p>
<ul>
<li>
<h3>urls</h3>
<h3 id="intro"><a href="#intro">urls</a></h3>
<p>This should be applied to relative ('/image/smile.jpg') and absolute ('https://mysite.com/content/image/smile.jpg') URLs. If the URL is relative, the client will generate an absolute URL based on the original URL used to fetch the data being parsed (i.e. it should all just work).</p>
<p>You can set several types of URL:</p>
<ul>
@ -35,28 +35,28 @@
<p>Sites can change suddenly, so it is nice to have a bit of redundancy here if it is easy.</p>
</li>
<li>
<h3>tags</h3>
<h3 id="tags"><a href="#tags">tags</a></h3>
<p>These are simple--they tell the client that the given strings are tags. You set the namespace here as well. I recommend you parse 'splashbrush' and set the namespace 'creator' here rather than trying to mess around with 'append prefix "creator:"' string conversions at the formula level--it is simpler up here and it lets hydrus handle any edge case logic for you.</p>
<p>Leave the namespace field blank for unnamespaced tags.</p>
</li>
<li>
<h3>file hash</h3>
<h3 id="file_hash"><a href="#file_hash">file hash</a></h3>
<p>This says 'this is the hash for the file otherwise referenced in this parser'. So, if you have another content parser finding a File or Post URL, this lets the client know early that that destination happens to have a particular MD5, for instance. The client will look for that hash in its own database, and if it finds a match, it can predetermine if it already has the file (or has previously deleted it) without ever having to download it. When this happens, it will still add tags and associate the file with the URL for it's 'known urls' just as if it <i>had</i> downloaded it!</p>
<p>If you understand this concept, it is great to include. It saves time and bandwidth for everyone. Many site APIs include a hash for this exact reason--they want you to be able to skip a needless download just as much as you do.</p>
<p><img src="edit_content_parser_panel_hash.png" /></p>
<p>The usual suite of hash types are supported: MD5, SHA1, SHA256, and SHA512. An old version of this required some weird string decoding, but this is no longer true. Select 'hex' or 'base64' from the encoding type dropdown, and then just parse the 'e5af57a687f089894f5ecede50049458' or '5a9XpofwiYlPXs7eUASUWA==' text, and hydrus should handle the rest. It will present the parsed hash in hex.</p>
</li>
<li>
<h3>timestamp</h3>
<h3 id="timestamp"><a href="#timestamp">timestamp</a></h3>
<p>This lets you say that a given number refers to a particular time for a file. At the moment, I only support 'source time', which represents a 'post' time for the file and is useful for thread and subscription check time calculations. It takes a Unix time integer, like 1520203484, which many APIs will provide.</p>
<p>If you are feeling very clever, you can decode a 'MM/DD/YYYY hh:mm:ss' style string to a Unix time integer using string converters, which use some hacky and semi-reliable python %d-style values as per <a href="https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior">here</a>. Look at the existing defaults for examples of this, and don't worry about being more accurate than 12/24 hours--trying to figure out timezone is a hell not worth attempting, and doesn't really matter in the long-run for subscriptions and thread watchers that might care.</p>
</li>
<li>
<h3>watcher page title</h3>
<h3 id="page_title"><a href="#page_title">watcher page title</a></h3>
<p>This lets the watcher know a good name/subject for its entries. The subject of a thread is obviously ideal here, but failing that you can try to fetch the first part of the first post's comment. It has precendence, like for URLs, so you can tell the parser which to prefer if you have multiple options. Just for neatness and ease of testing, you probably want to use a string converter here to cut it down to the first 64 characters or so.</p>
</li>
<li>
<h3>veto</h3>
<h3 id="veto"><a href="#veto">veto</a></h3>
<p>This is a special content type--it tells the next highest stage of parsing that this 'post' of parsing is invalid and to cancel and not return any data. For instance, if a thread post's file was deleted, the site might provide a default '404' stock File URL using the same markup structure as it would for normal images. You don't want to give the user the same 404 image ten times over (with fifteen kinds of tag and source time metadata attached), so you can add a little rule here that says "If the image link is 'https://somesite.com/404.png', raise a veto: File 404" or "If the page has 'No results found' in its main content div, raise a veto: No results found" or "If the expected download tag does not have 'download link' as its text, raise a veto: No Download Link found--possibly Ugoira?" and so on.</p>
<p><img src="edit_content_parser_panel_veto.png" /></p>
<p>They will associate their name with the veto being raised, so it is useful to give these a decent descriptive name so you can see what might be going right or wrong during testing. If it is an appropriate and serious enough veto, it may also rise up to the user level and will be useful if they need to report you an error (like "After five pages of parsing, it gives 'veto: no next page link'").</p>

View File

@ -7,7 +7,7 @@
<body>
<div class="content">
<p><a href="downloader_parsers.html"><---- Back to main parsers page</a></p>
<h3>api parsing</h3>
<h3 id="intro"><a href="#intro">api parsing</a></h3>
<p>Some sites offer API calls for their pages. Depending on complexity and quality of content, using these APIs may or may not be a good idea. Artstation has a good one--let's first review our URL Classes:</p>
<p><img src="downloader_api_example_url_class_1.png" /> <img src="downloader_api_example_url_class_2.png" /></p>
<p>We convert the original Post URL, <a href="https://www.artstation.com/artwork/mQLe1">https://www.artstation.com/artwork/mQLe1</a> to <a href="https://www.artstation.com/projects/mQLe1.json">https://www.artstation.com/projects/mQLe1.json</a>. Note that Artstation Post URLs can produce multiple files, and that the API url should not be associated with those final files.</p>
@ -31,8 +31,8 @@
<p>These are all simple. You can take or leave the title and medium tags--some people like them, some don't. This example has no unnamespaced tags, but <a href="https://www.artstation.com/projects/XRm50.json">this one</a> does. Creator-entered tags are sometimes not worth parsing (on tumblr, for instance, you often get run-on tags like #imbored #whatisevengoingon that are irrelevent to the work), but Artstation users are all professionals trying to get their work noticed, so the tags are usually pretty good.</p>
<p><img src="downloader_api_example_source_time.png" /></p>
<p>This again uses python's datetime to decode the date, which Artstation presents with millisecond accuracy, ha ha. I use a (.+:..)\..*->\1 regex (i.e. "get everything before the period") to strip off the timezone and milliseconds and then decode as normal.</p>
<h3>summary</h3>
<h3 id="summary"><a href="#summary">summary</a></h3>
<p>APIs that are stable and free to access (e.g. do not require OAuth or other complicated login headers) can make parsing fantastic. They save bandwidth and CPU time, and they are typically easier to work with than HTML. Unfortunately, the boorus that do provide APIs often list their tags without namespace information, so I recommend you double-check you can get what you want before you get too deep into it. Some APIs also offer incomplete data, such as relative URLs (relative to the original URL!), which can be a pain to figure out in our system.</p>
</div>
</body>
</html>
</html>

View File

@ -7,7 +7,7 @@
<body>
<div class="content">
<p><a href="downloader_parsers.html"><---- Back to main parsers page</a></p>
<h3>post pages</h3>
<h3 id="intro"><a href="#intro">post pages</a></h3>
<p>Let's look at this page: <a href="https://gelbooru.com/index.php?page=post&s=view&id=3837615">https://gelbooru.com/index.php?page=post&s=view&id=3837615</a>.</p>
<p>What sorts of data are we interested in here?</p>
<ul>
@ -17,7 +17,7 @@
<li>The post time.</li>
<li>The Deviant Art source URL.</li>
</ul>
<h3>the file url</h3>
<h3 id="the_file_url"><a href="#the_file_url">the file url</a></h3>
<p>A tempting strategy for pulling the file URL is to just fetch the src of the embedded &lt;img&gt; tag, but:</p>
<ul>
<li>If the booru also supports videos or flash, you'll have to write separate and likely more complicated rules for &lt;video&gt; and &lt;embed&gt; tags.</li>
@ -36,19 +36,19 @@
<p>I think I wrote my gelbooru parser before I added String Matches to individual HTML formulae tag rules, so I went with this, which is a bit more cheeky:</p>
<p><img src="downloader_post_example_cheeky.png" /></p>
<p>But it works. Sometimes, just regexing for links that fit the site's CDN is a good bet for finding difficult stuff.</p>
<h3>tags</h3>
<h3 id="tags"><a href="#tags">tags</a></h3>
<p>Most boorus have a taglist on the left that has a nice id or class you can pull, and then each namespace gets its own class for CSS-colouring:</p>
<p><img src="downloader_post_example_meta_tag.png" /></p>
<p>Make sure you browse around the booru for a bit, so you can find all the different classes they use. character/artist/copyright are common, but some sneak in the odd meta/species/rating.</p>
<p>Skipping ?/-/+ characters can be a pain if you are lacking a nice tag-text class, in which case you can add a regex String Match to the HTML formula (as I do here, since Gelb offers '?' links for tag definitions) like [^\?\-+\s], which means "the text includes something other than just '?' or '-' or '+' or whitespace".</p>
<h3>md5 hash</h3>
<h3 id="md5_hash"><a href="#md5_hash">md5 hash</a></h3>
<p>If you look at the Gelbooru File URL, <a href="https://gelbooru.com/images/38/6e/386e12e33726425dbd637e134c4c09b5.jpeg"><b>https://gelbooru.com/images/38/6e/386e12e33726425dbd637e134c4c09b5.jpeg</b></a>, you may notice the filename is all hexadecimal. It looks like they store their files under a two-deep folder structure, using the first four characters--386e here--as the key. It sure looks like '386e12e33726425dbd637e134c4c09b5' is not random ephemeral garbage!</p>
<p>In fact, Gelbooru use the MD5 of the file as the filename. Many storage systems do something like this (hydrus uses SHA256!), so if they don't offer a &lt;meta&gt; tag that explicitly states the md5 or sha1 or whatever, you can sometimes infer it from one of the file links. This screenshot is from the more recent version of hydrus, which has the more powerful 'string processing' system for string transformations. It has an intimidating number of nested dialogs, but we can stay simple for now, with only the one regex substitution step inside a string 'converter':</p>
<p><img src="downloader_post_example_md5.png" /></p>
<p>Here we are using the same property="og:image" rule to fetch the File URL, and then we are regexing the hex hash with .*([0-9a-f]{32}).* (MD5s are 32 hex characters). We select 'hex' as the encoding type. Hashes require a tiny bit more data handling behind the scenes, but in the Content Parser test page it presents the hash again neatly in English: "md5 hash: 386e12e33726425dbd637e134c4c09b5"), meaning everything parsed correct. It presents the hash in hex even if you select the encoding type as base64.</p>
<p>If you think you have found a hash string, you should obviously test your theory! The site might not be using the actual MD5 of file bytes, as hydrus does, but instead some proprietary scheme. Download the file and run it through a program like HxD (or hydrus!) to figure out its hashes, and then search the View Source for those hex strings--you might be surprised!</p>
<p>Finding the hash is hugely beneficial for a parser--it lets hydrus skip downloading files without ever having seen them before!</p>
<h3>source time</h3>
<h3 id="source_time"><a href="#source_time">source time</a></h3>
<p>Post/source time lets subscriptions and watchers make more accurate guesses at current file velocity. It is neat to have if you can find it, but:</p>
<p><b class="dealwithit">FUCK ALL TIMEZONES FOREVER</b></p>
<p>Gelbooru offers--</p>
@ -56,12 +56,12 @@
<p>--so let's see how we can turn that into a Unix timestamp:</p>
<p><img src="downloader_post_example_source_time.png" /></p>
<p>I find the &lt;li&gt; that starts "Posted: " and then decode the date according to the hackery-dackery-doo format from <a href="https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior">here</a>. %c and %z are unreliable, and attempting timezone adjustments is overall a supervoid that will kill your time for no real benefit--subs and watchers work fine with 12-hour imprecision, so if you have a +0300 or EST in your string, just cut those characters off with another String Transformation. As long as you are getting about the right day, you are fine.</p>
<h3>source url</h3>
<h3 id="source_url"><a href="#source_url">source url</a></h3>
<p>Source URLs are nice to have if they are high quality. Some boorus only ever offer artist profiles, like https://twitter.com/artistname, whereas we want singular Post URLs that point to other places that host this work. For Gelbooru, you could fetch the Source URL as we did source time, searching for "Source: ", but they also offer more easily in an edit form:</p>
<p><pre>&lt;input type="text" name="source" size="40" id="source" value="https://www.deviantart.com/art/Lara-Croft-Artifact-Dive-699335378" /&gt;</pre></p>
<p>This is a bit of a fragile location to parse from--Gelb could change or remove this form at any time, whereas the "Posted: " &lt;li&gt; is probably firmer, but I expect I wrote it before I had String Matches in. It works for now, which in this game is often Good Enough&trade;.</p>
<p>Also--be careful pulling from text or tooltips rather than an href-like attribute, as whatever is presented to the user may be clipped for longer URLs. Make sure you try your rules on a couple of different pages to make sure you aren't pulling "https://www.deviantart.com/art/Lara..." by accident anywhere!</p>
<h3>summary</h3>
<h3 id="summary"><a href="#summary">summary</a></h3>
<p>Phew--all that for a bit of Lara Croft! Thankfully, most sites use similar schemes. Once you are familiar with the basic idea, the only real work is to duplicate an existing parser and edit for differences. Our final parser looks like this:</p>
<p><img src="downloader_post_example_final.png" /></p>
<p>This is overall a decent parser. Some parts of it may fail when Gelbooru update to their next version, but that can be true of even very good parsers with multiple redundancy. For now, hydrus can use this to quickly and efficiently pull content from anything running Gelbooru 0.2.5., and the effort spent now can save millions of combined <i>right-click->save as</i> and manual tag copies in future. If you make something like this and share it about, you'll be doing a good service for those who could never figure it out.</p>

View File

@ -8,26 +8,26 @@
<div class="content">
<p><a href="downloader_parsers.html"><---- Back to main parsers page</a></p>
<p class="warning">These guides should <i>roughly</i> follow what comes with the client by default! You might like to have the actual UI open in front of you so you can play around with the rules and try different test parses yourself.</p>
<h3>gallery pages</h3>
<h3 id="intro"><a href="#intro">gallery pages</a></h3>
<p>Let's look at this page: <a href="https://e621.net/post/index/1/rating:safe pokemon">https://e621.net/post/index/1/rating:safe pokemon</a></p>
<p>We've got 75 thumbnails and a bunch of page URLs at the bottom.</p>
<h3>first, the main page</h3>
<h3 id="main_page"><a href="#main_page">first, the main page</a></h3>
<p>This is easy. It gets a good name and some example URLs. e621 has some different ways of writing out their queries (and as they use some tags with '/', like 'male/female', this can cause character encoding issues depending on whether the tag is in the path or query!), but we'll put that off for now--we just want to parse some stuff.</p>
<p><img src="downloader_gallery_example_main.png" /></p>
<h3>thumbnail links</h3>
<h3 id="thumbnail_urls"><a href="#thumbnail_urls">thumbnail links</a></h3>
<p>Most browsers have some good developer tools to let you Inspect Element and get a better view of the HTML DOM. Be warned that this information isn't always the same as View Source (which is what hydrus will get when it downloads the initial HTML document), as some sites load results dynamically with javascript and maybe an internal JSON API call (when sites move to systems that load more thumbs as you scroll down, it makes our job more difficult--in these cases, you'll need to chase down the embedded JSON or figure out what API calls their JS is making--the browser's developer tools can help you here again). Thankfully, e621 is (and most boorus are) fairly static and simple:</p>
<p><img src="downloader_gallery_example_thumb_html.png" /></p>
<p>Every thumb on e621 is a &lt;span&gt; with class="thumb" wrapping an &lt;a&gt; and an &lt;img&gt;. This is a common pattern, and easy to parse:</p>
<p><img src="downloader_gallery_example_thumb_parsing.png" /></p>
<p>There's no tricky String Matches or String Converters needed--we are just fetching hrefs. Note that the links get relative-matched to example.com for now--I'll probably fix this to apply to one of the example URLs, but rest assured that IRL the parser will 'join' its url up with the appropriate Gallery URL used to fetch the data. Sometimes, you might want to add a rule for 'search descendents for the first &lt;div&gt; tag with id=content' to make sure you are only grabbing thumbs from the main box, whether that is a &lt;div&gt; or a &lt;span&gt;, and whether it has id="content" or class="mainBox", but unless you know that booru likes to embed "popular" or "favourite" 'thumbs' up top that will be accidentally caught by a &lt;span&gt;'s with class="thumb", I recommend you not make your rules overly specific--all it takes is for their dev to change the name of their content box, and your whole parser breaks. I've ditched the &lt;span&gt; requirement in the rule here for exactly that reason--class="thumb" is necessary and sufficient.</p>
<p>Remember that the parsing system allows you to go up ancestors as well as down descendants. If your thumb-box has multiple links--like to see the artist's profile or 'set as favourite'--you can try searching for the &lt;span&gt;s, then down to the &lt;img&gt;, and then <i>up</i> to the nearest &lt;a&gt;. In English, this is saying, "Find me all the image link URLs in the thumb boxes."</p>
<h3>next gallery page link</h3>
<h3 id="next_gallery_url"><a href="#next_gallery_url">next gallery page link</a></h3>
<p>Most boorus have 'next' or '>>' at the bottom, which can be simple enough, but many have a neat &lt;link href="/post/index/2/rating:safe%20pokemon" rel="next" /&gt; in the &lt;head&gt;. The &lt;head&gt; solution is easier, if available, but my default e621 parser happens to pursue the 'paginator':</p>
<p><img src="downloader_gallery_example_paginator_parsing.png" /></p>
<p>As it happens, e621 also apply the rel="next" attribute to their "Next >>" links, which makes it all that easier for us to find. Sometimes there is no "next" id or class, and you'll want to add a String Match to your html formula to test for a string value of '>>' or whatever it is. A good trick is to View Source and then search for the critical "/post/index/2/" phrase you are looking for--you might find what you want in a &lt;link&gt; tag you didn't expect or even buried in a hidden 'share to tumblr' button. &lt;form&gt;s for reporting or commenting on content are another good place to find content ids.</p>
<p>Note that this finds two URLs. e621 apply the rel="next" to both the "2" link and the "Next >>" one. The download engine merges the parser's dupes, so don't worry if you end up parsing both the 'top' and 'bottom' next page links, or if you use multiple rules to parse the same data in different ways.</p>
<h3>summary</h3>
<h3 id="summary"><a href="#summary">summary</a></h3>
<p>With those two rules, we are done. Gallery parsers are nice and simple.</p>
</div>
</body>
</html>
</html>

View File

@ -7,18 +7,18 @@
<body>
<div class="content">
<p><a href="downloader_parsers.html"><---- Back to main parsers page</a></p>
<h3 id="page_parsers"><a href="#page_parsers">page parsers</a></h3>
<h3 id="intro"><a href="#intro">page parsers</a></h3>
<p>We can now produce individual rows of rich metadata. To arrange them all into a useful structure, we will use Page Parsers.</p>
<p>The Page Parser is the top level parsing object. It takes a single document and produces a list--or a list of lists--of metadata. Here's the main UI:</p>
<p><img src="edit_page_parser_panel_e621_main.png" /></p>
<p>Notice that the edit panel has three sub-pages.</p>
<h3>main</h3>
<h3 id="main"><a href="#main">main</a></h3>
<ul>
<li><b>Name</b>: Like for content parsers, I recommend you add good names for your parsers.</li>
<li><b>Pre-parsing conversion</b>: If your API source encodes or wraps the data you want to parse, you can do some string transformations here. You won't need to use this very often, but if your source gives the JSON wrapped in javascript (like the old tumblr API), it can be invaluable.</li>
<li><b>Example URLs</b>: Here you should add a list of example URLs the parser works for. This lets the client automatically link this parser up with URL classes for you and any users you share the parser with.</li>
</ul>
<h3>content parsers</h3>
<h3 id="content_parsers"><a href="#content_parsers">content parsers</a></h3>
<p>This page is just a simple list:</p>
<p><img src="edit_page_parser_panel_e621_content_parsers.png" /></p>
<p>Each content parser here will be applied to the document and returned in this page parser's results list. Like most boorus, e621's File Pages only ever present one file, and they have simple markup, so the solution here was simple. The full contents of that test window are:</p>
@ -50,7 +50,7 @@ tag: species:mammal
*** RESULTS END ***</pre></p>
<p>When the client sees this in a downloader context, it will where to download the file and which tags to associate with it based on what the user has chosen in their 'tag import options'.</p>
<h3>subsidiary page parsers</h3>
<h3 id="subsidiary_page_parsers"><a href="#subsidiary_page_parsers">subsidiary page parsers</a></h3>
<p>Here be dragons. This was an attempt to make parsing more helpful in certain API situations, but it ended up ugly. I do not recommend you use it, as I will likely scratch the whole thing and replace it with something better one day. It basically splits the page up into pieces that can then be parsed by nested page parsers as separate objects, but the UI and workflow is hell. Afaik, the imageboard API parsers use it, but little/nothing else. If you are really interested, check out how those work and maybe duplicate to figure out your own imageboard parser and/or send me your thoughts on how to separate File URL/timestamp combos better.</p>
</div>
</body>

View File

@ -7,7 +7,7 @@
<body>
<div class="content">
<p><a href="downloader_completion.html"><---- Back to putting downloaders together</a></p>
<h3>sharing</h3>
<h3 id="intro"><a href="#intro">sharing</a></h3>
<p>If you are working with users who also understand the downloader system, you can swap your GUGs, URL Classes, and Parsers separately using the import/export buttons on the relevant dialogs, which work in pngs and clipboard text.</p>
<p>But if you want to share conveniently, and with users who are not familiar with the different downloader objects, you can package everything into a single easy-import png as per <a href="adding_new_downloaders.html">here</a>.</p>
<p>The dialog to use is <i>network->downloader definitions->export downloaders</i>:</p>
@ -18,4 +18,4 @@
<p class="right"><a href="downloader_login.html">Onto the login manager ----></a></p>
</div>
</body>
</html>
</html>

View File

@ -7,28 +7,28 @@
<body>
<div class="content">
<p><a href="downloader_gugs.html"><---- Back to GUGs</a></p>
<h3>url classes</h3>
<h3 id="intro"><a href="#intro">url classes</a></h3>
<p>The fundamental connective tissue of the downloader system is the 'URL Class'. This object identifies and normalises URLs and links them to other components. Whenever the client handles a URL, it tries to match it to a URL Class to figure out what to do.</p>
<h3>the types of url</h3>
<h3 id="url_types"><a href="#url_types">the types of url</a></h3>
<p>For hydrus, an URL is useful if it is one of:</p>
<ul>
<li><h3>File URL</h3></li>
<li><h4>File URL</h4></li>
<li>
<p>This returns the full, raw media file with no HTML wrapper. They typically end in a filename like <a href="http://safebooru.org//images/2333/cab1516a7eecf13c462615120ecf781116265f17.jpg">http://safebooru.org//images/2333/cab1516a7eecf13c462615120ecf781116265f17.jpg</a>, but sometimes they have a more complicated fetch command ending like 'file.php?id=123456' or '/post/content/123456'.</p>
<p>These URLs are remembered for the file in the 'known urls' list, so if the client happens to encounter the same URL in future, it can determine whether it can skip the download because the file is already in the database or has previously been deleted.</p>
<p>It is not important that File URLs be matched by a URL Class. File URL is considered the 'default', so if the client finds no match, it will assume the URL is a file and try to download and import the result. You might want to particularly specify them if you want to present them in the media viewer or discover File URLs are being confused for Post URLs or something.</p>
</li>
<li><h3>Post URL</h3></li>
<li><h4>Post URL</h4></li>
<li>
<p>This typically return some HTML that contains a File URL and metadata such as tags and post time. They sometimes present multiple sizes (like 'sample' vs 'full size') of the file or even different formats (like 'ugoira' vs 'webm'). The Post URL for the file above, <a href="http://safebooru.org/index.php?page=post&s=view&id=2429668">http://safebooru.org/index.php?page=post&s=view&id=2429668</a> has this 'sample' presentation. Finding the best File URL in these cases can be tricky!</p>
<p>This URL is also saved to 'known urls' and will usually be similarly skipped if it has previously been downloaded. It will also appear in the media viewer as a clickable link.</p>
</li>
<li><h3>Gallery URL</h3></li>
<li><h4>Gallery URL</h4></li>
<li>This presents a list of Post URLs or File URLs. They often also present a 'next page' URL. It could be a page like <a href="http://safebooru.org/index.php?page=post&s=list&tags=yorha_no._2_type_b&pid=0">http://safebooru.org/index.php?page=post&s=list&tags=yorha_no._2_type_b&pid=0</a> or an API URL like <a href="http://safebooru.org/index.php?page=dapi&s=post&tags=yorha_no._2_type_b&q=index&pid=0">http://safebooru.org/index.php?page=dapi&s=post&tags=yorha_no._2_type_b&q=index&pid=0</a>.</li>
<li><h3>Watchable URL</h3></li>
<li><h4>Watchable URL</h4></li>
<li>This is the same as a Gallery URL but represents an ephemeral page that receives new files much faster than a gallery but will soon 'die' and be deleted. For our purposes, this typically means imageboard threads.</li>
</ul>
<h3>the components of a url</h3>
<h3 id="url_components"><a href="#url_components">the components of a url</a></h3>
<p>As far as we are concerned, a URL string has four parts:</p>
<ul>
<li><b>Scheme:</b> "http" or "https"</li>
@ -40,12 +40,12 @@
<p><img src="downloader_edit_url_class_panel.png" /></p>
<p>A TBIB File Page like <a href="https://tbib.org/index.php?page=post&s=view&id=6391256">https://tbib.org/index.php?page=post&s=view&id=6391256</a> is a Post URL. Let's look at the metadata first:</p>
<ul>
<li><h3>Name and type</h3></li>
<li><h4>Name and type</h4></li>
<li>
<p>Like with GUGs, we should set a good unambiguous name so the client can clearly summarise this url to the user. 'tbib file page' is good.</p>
<p>This is a Post URL, so we set the 'post url' type.</p>
</li>
<li><h3>Association logic</h3></li>
<li><h4>Association logic</h4></li>
<li>
<p>All boorus and most sites only present one file per page, but some sites present multiple files on one page, usually several pages in a series/comic, as with pixiv. Danbooru-style thumbnail links to 'this file has a post parent' do not count here--I mean that a single URL embeds multiple full-size images, either with shared or separate tags. It is <b>very important</b> to the hydrus client's downloader logic (making decisions about whether it has previously visited a URL, so whether to skip checking it again) that if a site can present multiple files on a single page that 'can produce multiple files' is checked.</p>
<p>Related is the idea of whether a 'known url' should be associated. Typically, this should be checked for Post and File URLs, which are fixed, and unchecked for Gallery and Watchable URLs, which are ephemeral and give different results from day to day. There are some unusual exceptions, so give it a brief thought--but if you have no special reason, leave this as the default for the url type.</p>
@ -53,31 +53,31 @@
</ul>
<p>And now, for matching the string itself, let's revisit our four components:</p>
<ul>
<li><h3>Scheme</h3></li>
<li><h4>Scheme</h4></li>
<li>
<p>TBIB supports http and https, so I have set the 'preferred' scheme to https. Any 'http' TBIB URL a user inputs will be automatically converted to https.</p>
</li>
<li><h3>Location/Domain</h3></li>
<li><h4>Location/Domain</h4></li>
<li>
<p>For Post URLs, the domain is always "tbib.org".</p>
<p>The 'allow' and 'keep' subdomains checkboxes let you determine if a URL with "artistname.artsite.com" will match a URL Class with "artsite.com" domain and if that subdomain should be remembered going forward. Most sites do not host content on subdomains, so you can usually leave 'match' unchecked. The 'keep' option (which is only available if 'keep' is checked) is more subtle, only useful for rare cases, and unless you have a special reason, you should leave it checked. (For keep: In cases where a site farms out File URLs to CDN servers on subdomains--like randomly serving a mirror of "https://muhbooru.org/file/123456" on "https://srv2.muhbooru.org/file/123456"--and removing the subdomain still gives a valid URL, you may not wish to keep the subdomain.) Since TBIB does not use subdomains, these options do not matter--we can leave both unchecked.</p>
<p>'www' and 'www2' and similar subdomains are automatically matched. Don't worry about them.</p>
</li>
<li><h3>Path Components</h3></li>
<li><h4>Path Components</h4></li>
<li>
<p>TBIB just uses a single "index.php" on the root directory, so the path is not complicated. Were it longer (like "gallery/cgi/index.php", we would add more ("gallery" and "cgi"), and since the path of a URL has a strict order, we would need to arrange the items in the listbox there so they were sorted correctly.</p>
</li>
<li><h3>Query Parameters</h3></li>
<li><h4>Query Parameters</h4></li>
<li>
<p>TBIB's index.php takes many query parameters to render different page types. Note that the Post URL uses "s=view", while TBIB Gallery URLs use "s=list". In any case, for a Post URL, "id", "page", and "s" are necessary and sufficient.</p>
</li>
</ul>
<h3>string matches</h3>
<h3 id="string_matches"><a href="#string_matches">string matches</a></h3>
<p>As you edit these components, you will be presented with the Edit String Match Panel:</p>
<p><img src="edit_string_match_panel.png" /></p>
<p>This lets you set the type of string that will be valid for that component. If a given path or query component does not match the rules given here, the URL will not match the URL Class. Most of the time you will probably want to set 'fixed characters' of something like "post" or "index.php", but if the component you are editing is more complicated and could have a range of different valid values, you can specify just numbers or letters or even a regex pattern. If you try to do something complicated, experiment with the 'example string' entry to make sure you have it set how you think.</p>
<p>Don't go overboard with this stuff, though--most sites do not have super-fine distinctions between their different URL types, and hydrus users will not be dropping user account or logout pages or whatever on the client, so you can be fairly liberal with the rules.</p>
<h3>how do they match, exactly?</h3>
<h3 id="match_details"><a href="#match_details">how do they match, exactly?</a></h3>
<p>This URL Class will be assigned to any URL that matches the location, path, and query. Missing path compontent or query parameters in the URL will invalidate the match but additonal ones will not!</p>
<p>For instance, given:</p>
<ul>
@ -114,7 +114,7 @@
<p>URL A will match URL Class A but not URL Class B and so will receive A.</p>
<p>URL B will match both and receive URL Class B as it is more complicated.</p>
<p>This situation is not common, but when it does pop up, it can be a pain. It is usually a good idea to match exactly what you need--no more, no less.</p>
<h3>normalising urls</h3>
<h3 id="url_normalisation"><a href="#url_normalisation">normalising urls</a></h3>
<p>Different URLs can give the same content. The http and https versions of a URL are typically the same, and:</p>
<ul>
<li><a href="https://gelbooru.com/index.php?page=post&s=view&id=3767497">https://gelbooru.com/index.php?page=post&s=view&id=3767497</a></li>
@ -133,7 +133,7 @@
<p>Note that in e621's case (and for many other sites!), that text after the id is purely decoration. It can change when the file's tags change, so if we want to compare today's URLs with those we saw a month ago, we'd rather just be without it.</p>
<p>On normalisation, all URLs will get the preferred http/https switch, and their query parameters will be alphabetised. File and Post URLs will also cull out any surplus path or query components. This wouldn't affect our TBIB example above, but it will clip the e621 example down to that 'bare' id URL, and it will take any surplus 'lang=en' or 'browser=netscape_24.11' garbage off the query text as well. URLs that are not associated and saved and compared (i.e. normal Gallery and Watchable URLs) are not culled of unmatched path components or query parameters, which can sometimes be useful if you want to match (and keep intact) gallery URLs that might or might not include an important 'sort=desc' type of parameter.</p>
<p>Since File and Post URLs will do this culling, be careful that you not leave out anything important in your rules. Make sure what you have is both necessary (nothing can be removed and still keep it valid) and sufficient (no more needs to be added to make it valid). It is a good idea to try pasting the 'normalised' version of the example URL into your browser, just to check it still works.</p>
<h3>'default' values</h3>
<h3 id="default_values"><a href="#default_values">'default' values</a></h3>
<p>Some sites present the first page of a search like this:</p>
<p><a href="https://danbooru.donmai.us/posts?tags=skirt">https://danbooru.donmai.us/posts?tags=skirt</a></p>
<p>But the second page is:<p>
@ -144,11 +144,11 @@
<p>What happened to 'page=1' and '/page/1'? Adding those '1' values in works fine! Many sites, when an index is absent, will secretly imply an appropriate 0 or 1. This looks pretty to users looking at a browser address bar, but it can be a pain for us, who want to match both styles to one URL Class. It would be nice if we could recognise the 'bare' initial URL and fill in the '1' values to coerce it to the explicit, automation-friendly format. Defaults to the rescue:</p>
<p><img src="downloader_edit_url_class_panel_default.png" /></p>
<p>After you set a path component or query parameter String Match, you will be asked for an optional 'default' value. You won't want to set one most of the time, but for Gallery URLs, it can be hugely useful--see how the normalisation process automatically fills in the missing path component with the default! There are plenty of examples in the default Gallery URLs of this, so check them out. Most sites use page indices starting at '1', but Gelbooru-style imageboards use 'pid=0' file index (and often move forward 42, so the next pages will be 'pid=42', 'pid=84', and so on, although others use deltas of 20 or 40).</p>
<h3>can we predict the next gallery page?</h3>
<h3 id="next_gallery_page_prediction"><a href="#next_gallery_page_prediction">can we predict the next gallery page?</a></h3>
<p>Now we can harmonise gallery urls to a single format, we can predict the next gallery page! If, say, the third path component or 'page' query parameter is always a number referring to page, you can select this under the 'next gallery page' section and set the delta to change it by. The 'next gallery page url' section will be automatically filled in. This value will be consulted if the parser cannot find a 'next gallery page url' from the page content.</p>
<p>It is neat to set this up, but I only recommend it if you actually cannot reliably parse a next gallery page url from the HTML later in the process. It is neater to have searches stop naturally because the parser said 'no more gallery pages' than to have hydrus always one page beyond and end every single search on an uglier 'No results found' or 404 result.</p>
<p>Unfortunately, some sites will either not produce an easily parsable next page link or randomly just not include it due to some issue on their end (Gelbooru is a funny example of this). Also, APIs will often have a kind of 'start=200&num=50', 'start=250&num=50' progression but not include that state in the XML or JSON they return. These cases require the automatic next gallery page rules (check out Artstation and tumblr api gallery page URL Classes in the defaults for examples of this).</p>
<h3>how do we link to APIs?</h3>
<h3 id="api_links"><a href="#api_links">how do we link to APIs?</a></h3>
<p>If you know that a URL has an API backend, you can tell the client to use that API URL when it fetches data. The API URL needs its own URL Class.</p>
<p>To define the relationship, click the "String Converter" button, which gives you this:</p>
<p><img src="edit_string_converter_panel.png" /></p>
@ -157,4 +157,4 @@
<p class="right"><a href="downloader_parsers.html">Take a deep breath, because we are now about to do Parsers ----></a></p>
</div>
</body>
</html>
</html>

View File

@ -6,7 +6,7 @@
</head>
<body>
<div class="content">
<h3>duplicates</h3>
<h3 id="intro"><a href="#intro">duplicates</a></h3>
<p>As files are shared on the internet, they are often resized, cropped, converted to a different format, altered by the original or a new artist, or turned into a template and reinterpreted over and over and over. Even if you have a very restrictive importing workflow, your client is almost certainly going to get some <b>duplicates</b>. Some will be interesting alternate versions that you want to keep, and others will be thumbnails and other low-quality garbage you accidentally imported and would rather delete. Along the way, it would be nice to merge your ratings and tags to the better files so you don't lose any work.</p>
<p>Finding and processing duplicates within a large collection is impossible to do by hand, so I have written a system to do the heavy lifting for you. It currently works on still images, but an extension for gifs and video is planned.</p>
<p>Hydrus finds <i>potential</i> duplicates using a search algorithm that compares images by their shape. Once these pairs of potentials are found, they are presented to you through a filter like the archive/delete filter to determine their exact relationship and if you want to make a further action, such as deleting the 'worse' file of a pair. All of your decisions build up in the database to form logically consistent groups of duplicates and 'alternate' relationships that can be used to infer future information. For instance, if you say that file A is a duplicate of B and B is a duplicate of C, A and C are automatically recognised as duplicates as well.</p>
@ -60,12 +60,12 @@
<p>By default, these options are fairly empty. You will have to set up what you want based on your services and preferences. Setting a simple 'copy all tags' is generally a good idea, and like/dislike ratings also often make sense. The settings for better and same quality should probably be similar, but it depends on your situation.</p>
<p>If you choose the 'custom action' in the duplicate filter, you will be presented with a fresh 'edit duplicate merge options' panel for the action you select and can customise the merge specifically for that choice. ('favourite' options will come here in the future!)</p>
<p>Once you are all set up here, you can dive into the duplicate filter. Please let me know how you get on with it!</p>
<h3>what now?</h3>
<h3 id="future"><a href="#future">what now?</a></h3>
<p>The duplicate system is still incomplete. Now the db side is solid, the UI needs to catch up. Future versions will show duplicate information on thumbnails and the media viewer and allow quick-navigation to a file's duplicates and alternates.</p>
<p>For now, if you wish to see a file's duplicates, right-click it and select <i>file relationships</i>. You can review all its current duplicates, open them in a new page, appoint the new 'best file' of a duplicate group, and even mass-action selections of thumbnails.</p>
<p>You can also search for files based on the number of file relations they have (including when setting the search domain of the duplicate filter!) using <i>system:file relationships</i>. You can also search for best/not best files of groups, which makes it easy, for instance, to find all the spare duplicate files if you decide you no longer want to keep them.</p>
<p>I expect future versions of the system to also auto-resolve easy duplicate pairs, such as clearing out pixel-for-pixel png versions of jpgs.</p>
<h3>game cgs</h3>
<h3 id="game_cgs"><a href="#game_cgs">game cgs</a></h3>
<p>If you import a lot of game CGs, which frequently have dozens or hundreds of alternates, I recommend you set them as alternates by selecting them all and setting the status through the thumbnail right-click menu. The duplicate filter, being limited to pairs, needs to compare all new members of an alternate group to all other members once to verify they are not duplicates. This is not a big deal for alternates with three or four members, but game CGs provide an overwhelming edge case. Setting a group of thumbnails as alternate 'fixes' their alternate status immediately, discounting the possibility of any internate duplicates, and provides an easy way out of this situation.</p>
<h3 id="duplicates_examples"><a href="#duplicates_examples">more information and examples</a></h3>
<ul>

View File

@ -6,9 +6,9 @@
</head>
<body>
<div class="content">
<a id="repositories"><h3>what is a repository?</h3></a>
<h3 id="repositories"><a id="repositories">what is a repository?</a></h3>
<p>A <i>repository</i> is a service in the hydrus network that stores a certain kind of information--files or tag mappings, for instance--as submitted by users all over the internet. Those users periodically synchronise with the repository so they know everything that it stores. Sometimes, like with tags, this means creating a complete local copy of everything on the repository. Hydrus network clients never send queries to repositories; they perform queries over their local cache of the repository's data, keeping everything confined to the same computer.</p>
<a id="tags"><h3>what is a tag?</h3></a>
<h3 id="tags"><a id="tags">what is a tag?</a></h3>
<p><a href="https://en.wikipedia.org/wiki/Tag_(metadata)">wiki</a></p>
<p>A <i>tag</i> is a small bit of text describing a single property of something. They make searching easy. Good examples are "flower" or "nicolas cage" or "the sopranos" or "2003". By combining several tags together ( e.g. [ 'tiger woods', 'sports illustrated', '2008' ] or [ 'cosplay', 'the legend of zelda' ] ), a huge image collection is reduced to a tiny and easy-to-digest sample.</p>
<p>A good word for the connection of a particular tag to a particular file is <i>mapping</i>.</p>
@ -19,10 +19,10 @@
<li>As 'The Lord of the Rings' and 'the lord of the rings' are semantically identical, it is natural to search in a case insensitive way. When case does not matter, what point is there in recording it?</li>
</ol>
<p>Furthermore, leading and trailing whitespace is removed, and multiple whitespace is collapsed to a single character. <pre>' yellow dress '</pre> becomes <pre>'yellow dress'</pre></p>
<a id="namespaces"><h3>what is a namespace?</h3></a>
<h3 id="namespaces"><a id="namespaces">what is a namespace?</a></h3>
<p>A <i>namespace</i> is a category that in hydrus prefixes a tag. An example is 'person' in the tag 'person:ron paul'--it lets people and software know that 'ron paul' is a name. You can create any namespace you like; just type one or more words and then a colon, and then the next string of text will have that namespace.</p>
<p>The hydrus client gives namespaces different colours so you can pick out important tags more easily in a large list, and you can also search by a particular namespace, even creating complicated predicates like 'give all files that do not have any character tags', for instance.</p>
<a id="filenames"><h3>why not use filenames and folders?</h3></a>
<h3 id="filenames"><a id="filenames">why not use filenames and folders?</a></h3>
<p>As a retrieval method, filenames and folders are less and less useful as the number of files increases. Why?</p>
<ul>
<li>A filename is not unique; did you mean this "04.jpg" or <i>this</i> "04.jpg" in another folder? Perhaps "04 (3).jpg"?</li>
@ -33,7 +33,7 @@
</ul>
<p>So, the client tracks files by their <i>hash</i>. This technical identifier easily eliminates duplicates and permits the database to robustly attach other metadata like tags and ratings and known urls and notes and everything else, even across multiple clients and even if a file is deleted and later imported.</p>
<p>As a general rule, I suggest you not set up hydrus to parse and display all your imported files' filenames as tags. 'image.jpg' is useless as a tag. <a href="https://www.youtube.com/watch?v=_yYS0ZZdsnA">Shed the concept of filenames as you would chains.</a></p>
<a id="external_files"><h3>can the client manage files from their original locations?</h3></a>
<h3 id="external_files"><a id="external_files">can the client manage files from their original locations?</a></h3>
<p>When the client imports a file, it makes a quickly accessible but human-ugly copy in its internal database, by default under <i>install_dir/db/client_files</i>. When it needs to access that file again, it always knows where it is, and it can be confident it is what it expects it to be. It never accesses the original again.</p>
<p>This storage method is not always convenient, particularly for those who are hesitant about converting to using hydrus completely and also do not want to maintain two large copies of their collections. The question comes up--"can hydrus track files from their original locations, without having to copy them into the db?"</p>
<p>The technical answer is, "This support could be added," but I have decided not to, mainly because:</p>
@ -47,24 +47,24 @@
</ul>
<p>It is not unusual for new users who ask for this feature to find their feelings change after getting more experience with the software. If desired, path text can be preserved as tags using regexes during import, and getting into the swing of searching by metadata rather than navigating folders often shows how very effective the former is over the latter. Most users eventually import most or all of their collection into hydrus permanently, deleting their old folder structure as they go.</p>
<p>For this reason, if you are hesitant about doing things the hydrus way, I advise you try running it on a smaller subset of your collection, say 5,000 files, leaving the original copies completely intact. After a month or two, think about how often you used hydrus to look at the files versus navigating through folders. If you barely used the folders, you probably do not need them any more, but if you used them a lot, then hydrus might not be for you, or it might only be for some sorts of files in your collection.</p>
<a id="sqlite"><h3>why use sqlite?</h3></a>
<h3 id="sqlite"><a id="sqlite">why use sqlite?</a></h3>
<p>Hydrus uses SQLite for its database engine. Some users who have experience with other engines such as MySQL or PostgreSQL sometimes suggest them as alternatives. SQLite serves hydrus's needs well, and at the moment, there are no plans to change.</p>
<p>Since this question has come up frequently, a user has written an excellent document talking about the reasons to stick with SQLite. If you are interested in this subject, please check it out here:</p>
<p><a href="https://gitgud.io/prkc/hydrus-why-sqlite/blob/master/README.md">https://gitgud.io/prkc/hydrus-why-sqlite/blob/master/README.md</a></p>
<a id="hashes"><h3>what is a hash?</h3></a>
<h3 id="hashes"><a id="hashes">what is a hash?</a></h3>
<p><a href="https://en.wikipedia.org/wiki/Hash_function">wiki</a></p>
<p>Hashes are a subject you usually have to be a software engineer to find interesting. The simple answer is that they are unique names for things. Hashes make excellent identifiers inside software, as you can safely assume that f099b5823f4e36a4bd6562812582f60e49e818cf445902b504b5533c6a5dad94 refers to one particular file and no other. In the client's normal operation, you will never encounter a file's hash. If you want to see a thumbnail bigger, double-click it; the software handles the mathematics.</p>
<p><i>For those who </i>are<i> interested: hydrus uses SHA-256, which spits out 32-byte (256-bit) hashes. The software stores the hash densely, as 32 bytes, only encoding it to 64 hex characters when the user views it or copies to clipboard. SHA-256 is not perfect, but it is a great compromise candidate; it is secure for now, it is reasonably fast, it is available for most programming languages, and newer CPUs perform it more efficiently all the time.</i></p>
<a id="access_keys"><h3>what is an access key?</h3></a>
<h3 id="access_keys"><a id="access_keys">what is an access key?</a></h3>
<p>The hydrus network's repositories do not use username/password, but instead a single strong identifier-password like this:</p>
<p><i>7ce4dbf18f7af8b420ee942bae42030aab344e91dc0e839260fcd71a4c9879e3</i></p>
<p>These hex numbers give you access to a particular account on a particular repository, and are often combined like so:</p>
<p><i>7ce4dbf18f7af8b420ee942bae42030aab344e91dc0e839260fcd71a4c9879e3@hostname.com:45871</i></p>
<p>They are long enough to be impossible to guess, and also randomly generated, so they reveal nothing personally identifying about you. Many people can use the same access key (and hence the same account) on a repository without consequence, although they will have to share any bandwidth limits, and if one person screws around and gets the account banned, everyone will lose access.</p>
<p>The access key is the account. Do not give it to anyone you do not want to have access to the account. An administrator will never need it; instead they will want your <i>account key</i>.</p>
<a id="account_keys"><h3>what is an account key?</h3></a>
<h3 id="account_keys"><a id="account_keys">what is an account key?</a></h3>
<p>This is another long string of random hexadecimal that <i>identifies</i> your account without giving away access. If you need to identify yourself to a repository administrator (say, to get your account's permissions modified), you will need to tell them your account key. You can copy it to your clipboard in <i>services->review services</i>.</p>
<a id="delays"><h3>why can my friend not see what I just uploaded?</h3></a>
<h3 id="delays"><a id="delays">why can my friend not see what I just uploaded?</a></h3>
<p>The repositories do not work like conventional search engines; it takes a short but predictable while for changes to propagate to other users.</p>
<p>The client's searches only ever happen over its local cache of what is on the repository. Any changes you make will be delayed for others until their next update occurs. At the moment, the update period is 100,000 seconds, which is about 1 day and 4 hours.</p>
</div>

View File

@ -77,13 +77,13 @@
<p>By default, hydrus stores all your user data in one location, so backing up is simple:</p>
<ul>
<li>
<h3>the simple way - inside the client</h3>
<h4>the simple way - inside the client</h4>
<p>Go <i>database->set up a database backup location</i> in the client. This will tell the client where you want your backup to be stored. A fresh, empty directory on a different drive is ideal.</p>
<p>Once you have your location set up, you can thereafter hit <i>database->update database backup</i>. It will lock everything and mirror your files, showing its progress in a popup message. The first time you make this backup, it may take a little while (as it will have to fully copy your database and all its files), but after that, it will only have to copy new or altered files and should only ever take a couple of minutes.</p>
<p>Advanced users who have migrated their database across multiple locations will not have this option--use an external program in this case.</p>
</li>
<li>
<h3>the powerful way - using an external program</h3>
<h4>the powerful way - using an external program</h4>
<p>If you would like to integrate hydrus into a broader backup scheme you already run, or you are an advanced user with a complicated hydrus install that you have migrated across multiple drives, then you need to backup two things: the client*.db files and your client_files directory(ies). By default, they are all stored in install_dir/db. The .db files contain your settings and file metadata like inbox/archive and tags, while the client_files subdirs store your actual media and its thumbnails. If everything is still under install_dir/db, then it is usually easiest to just backup the whole install dir, keeping a functional 'portable' copy of your install that you can restore no prob. Make sure you keep the .db files together--they are not interchangeable and mostly useless on their own!</p>
<p>Shut the client down while you run the backup, obviously.</p>
</li>

View File

@ -1,56 +0,0 @@
<html>
<head>
<title>getting started - messages</title>
<link href="hydrus.ico" rel="shortcut icon" />
<link href="style.css" rel="stylesheet" type="text/css" />
</head>
<body>
<div class="content">
<p><a href="getting_started_tags.html"><--- Back to ratings</a></p>
<h3 class="warning">messaging, you say?</h3>
<p class="warning">Caveat: this is a mess. I'll get round to improving it sometime.</p>
<p class="warning">Secondly: I use encryption to protect your privacy. Although I am confident my code is mostly good, cryptography is <i>really difficult</i> to get 100% right. I use the right random number generator and key lengths and everything, but don't work under the assumption a dedicated and well funded attacker will never be able to break what I have done. I can guarantee your guildmaster will not be able to read your messages, not the NSA. Don't Do Drugs&#8482;.</p>
<h3>ok, let's messaging</h3>
<p>With the aid of a service called a <i>message depot</i>, the clients can send messages to one another.</p>
<p>A message depot is a bit like an IMAP email server. It has a number of contacts registered with it, and any client who knows those contacts can upload messages to it. A client can have many contacts (you can be several different people on the same computer), and a contact many clients (you can be the same person on several different computers). Clients check their appropriate message depots regularly, and download any new messages.</p>
<h3>how is it different from email?</h3>
<p>All hydrus network messages are stored on the message depots in an encrypted format, and <i>only</i> the recipient's client(s) have the key to decrypt them. If someone hacks/steals/whatever a message depot, they cannot read the messages, nor tell who they are from.</p>
<p>Messages are verifiable, meaning the client knows for sure if they came from who they say they did.</p>
<p>A client can send messages anonymously. These messages cannot (right now) be replied to.</p>
<h3>adding contacts</h3>
<p>First, let's discern a couple differences:</p>
<ul>
<li>contact: <b>An address that can send and recieve messages.</b></li>
<li>identity: <b>A contact you control.</b></li>
</ul>
<p>The dialog under <i>services->manage contacts and identities</i> lets you add new contacts.</p>
<p><img src="contacts.png" /></p>
<p><i>add manually</i> lets you enter the information in each field, while <i>add by contact address</i> grabs the public key from the server for you after you put in the contact_key@host:port. If you do add manually, make sure you copy the public key <i>very</i> carefully, and check the resultant contact key is correct; if your OS converts newlines incorrectly, it'll all go wrong!</p>
<p>Your identities are listed here (and you can rename them), but you do not create them here.</p>
<h3>how do i create a new identity?</h3>
<p>If you want to send messages as anything other than Anonymous, you need an access key at a message depot. The access key is not the same as your eventual contact key; the first gives you access, the second is how people's clients will identity you. They are both random numbers.</p>
<p><a href="server.html">Creating</a> or adding a message depot works exactly the same as for any other type of service.</a> When you add it, the client will do some heavy math in the background, which should freeze the interface for a few seconds. This is it generating the contact's <i>private key</i>, which is the secret that lets it decrypt messages.</p>
<p><img src="manage_services_edit_message_depot.png" /></p>
<p class="warning">If you want to use the same identity with several clients, don't try to add the service in the usual way on your extra clients, or they will generate their own private keys, overwriting each other! Instead, export the message depot from <i>services->manage services</i> on the first client, and import (drag and drop the .yaml file onto the dialog) into the second, third, whatever client. This will copy the original private key across without any errors.</p>
<p>When you are done, the 'messages' F9 menu will show your new identity.</p>
<p>If you want to share your identity, you can either send people your contact_key@host:port, or you can just message them, which will add you to their contacts automatically.</p>
<h3>composing messages</h3>
<p>Composing messages is easy. Just hit the button on your messages page. The 'recipients can see each other' checkbox does email's cc vs bcc.</p>
<p>I will add file attachments in future.</p>
<h3>finding messages</h3>
<p>The interface is just like searching for files. Put in a normal word to search message bodies/subjects, or use the system predicates to perform more complicated queries. Conversations that match will appear on the top right, and any selected convo will appear below.</p>
<p><a href="example_convo.png"><img src="example_convo.png" width=604 height=964/></a></p>
<p>You can hit F7 or delete or just right click on a conversation above to archive or delete it. Beside each message, you can see its current status with each recipient. If it says <i>failed</i>, you can click on it to retry, and if your identity is the recipient, you can switch between <i>read</i> and <i>unread</i>.</p>
<p>I will add:</p>
<ul>
<li>rich text</li>
<li>file attachments</li>
<li>custom status</li>
<li>auto-adding of new messages</li>
<li>live times (3 mins 21 seconds ago auto-updating)</li>
</ul>
<p>in future.</p>
<p class="right"><a href="index.html">Go back to the index ---></a></p>
</div>
</body>
</html>

View File

@ -7,7 +7,7 @@
<body>
<div class="content">
<p><a href="getting_started_files.html"><---- Back</a></p>
<h3>exporting and uploading</h3>
<h3 id="intro"><a href="#intro">exporting and uploading</a></h3>
<p>There are many ways to export files from the client:</p>
<ul>
<li>

View File

@ -20,7 +20,7 @@
<p>This is a big and powerful panel! I recommend you open the screenshot up in a new browser tab, or in the actual client, so you can refer to it.</p>
<p>Despite all the controls, the basic idea is simple: Up top, I have selected the 'safebooru tag search' download source, and then I have added two artists--"hong_soon-jae" and "houtengeki". These two queries have their own panels for reviewing what URLs they have worked on and further customising their behaviour, but all they <i>really</i> are is little bits of search text. When the subscription runs, it will put the given search text into the given download source just as if you were running the regular downloader.</p>
<p><b>For the most part, all you need to do to set up a good subscription is give it a name, select the download source, and use the 'paste queries' button to paste what you want to search. Subscriptions have great default options for almost all query types, so you don't have to go any deeper than that to get started.</b></p>
<h3><b class="warning">Do not change the max number of new files options until you know <i>exactly</i> what they do and have a good reason to alter them!</b></h3>
<h4><b class="warning">Do not change the max number of new files options until you know <i>exactly</i> what they do and have a good reason to alter them!</b></h4>
<h3 id="description"><a href="#description">how do subscriptions work?</a></h3>
<p>Once you hit ok on the main subscription dialog, the subscription system should immediately come alive. If any queries are due for a 'check', they will perform their search and look for new files (i.e. URLs it has not seen before). Once that is finished, the file download queue will be worked through as normal. Typically, the sub will make a popup like this while it works:</p>
<p><img src="subscriptions_popup.png" /></p>

View File

@ -16,11 +16,11 @@
<a class="screenshot" href="screenshot_video.png" title="Many file formats are supported."><img src="screenshot_video_thumb.png" /></a>
<a class="screenshot" href="screenshot_booru.png" title="You can run your own (simple!) booru"><img src="screenshot_booru_thumb.png" /></a>
<a class="screenshot" href="screenshot_advanced_autocomplete.png" title="The client can get complicated if you want it to. This screenshot shows a tag sibling, where one tag is immediately swapped with another, and a non-local search, where results that are known but not on the computer are shown."><img src="screenshot_advanced_autocomplete_thumb.png" /></a>
<h3>hydrus help</h3>
<h3 id="top"><a href="#top">hydrus help</a></h3>
<p>Although I try to make hydrus's interface simple, some of the things it does are quite complicated. Please read how to update/backup and skim the first getting started guides at the least. If you like, you can revisit the more complicated topics later, once you are experienced in the basics.</p>
<p>Keeping the help up to date is a constant battle. If you discover something really does not match the program, or is otherwise confusing or not well worded, please <a href="contact.html">let me know</a>.</p>
<ul>
<li><h3>starting out</h3></li>
<li><h3 id="starting_out"><a href="#starting_out">starting out</a></h3></li>
<ul>
<li><a href="introduction.html">introduction and statement of principles</a></li>
<li><a href="getting_started_installing.html">installing, updating and <span class="warning">backing up</span></a></li>
@ -30,7 +30,7 @@
<li><a href="getting_started_ratings.html">getting started with ratings</a></li>
<li><a href="access_keys.html">access keys to the public tag repository</a></li>
</ul>
<li><h3>the next step</h3></li>
<li><h3 id="the_next_step"><a href="#the_next_step">the next step</a></h3></li>
<ul>
<li><a href="getting_started_more_files.html">more getting started with files</a></li>
<li><a href="adding_new_downloaders.html">adding new downloaders</a></li>
@ -39,7 +39,7 @@
<li><a href="duplicates.html">filtering duplicates</a></li>
<li><a href="reducing_lag.html">reducing program lag</a></li>
</ul>
<li><h3>advanced usage</h3></li>
<li><h3 id="advanced"><a href="#advanced">advanced usage</a></h3></li>
<ul>
<li><a href="advanced.html">advanced usage - general</a></li>
<li><a href="advanced_siblings.html">advanced usage - tag siblings</a></li>
@ -53,7 +53,7 @@
<li><a href="wine.html">running a client or server in wine</a></li>
<li><a href="running_from_source.html">running a client or server from source</a></li>
</ul>
<li><h3>making a downloader</h3></li>
<li><h3 id="makin_a_downloader"><a href="#makin_a_downloader">making a downloader</a></h3></li>
<ul>
<li><a href="downloader_intro.html">introduction</a></li>
<li><a href="downloader_gugs.html">gallery url generators</a></li>
@ -63,7 +63,7 @@
<li><a href="downloader_sharing.html">sharing downloaders</a></li>
<li><a href="downloader_login.html">login manager</a></li>
</ul>
<li><h3>misc</h3></li>
<li><h3 id="misc"><a href="#misc">misc</a></h3></li>
<ul>
<li><a href="privacy.html">privacy</a></li>
<li><a href="contact.html">developer contact and links</a></li>

View File

@ -6,7 +6,7 @@
</head>
<body>
<div class="content">
<h3>ipfs</h3>
<h3 id="intro"><a href="#intro">ipfs</a></h3>
<p>IPFS is a p2p protocol that makes it easy to share many sorts of data. The hydrus client can communicate with an IPFS daemon to send and receive files.</p>
<p>You can read more about IPFS from <a href="http://ipfs.io">their homepage</a>, or <a href="https://medium.com/@ConsenSys/an-introduction-to-ipfs-9bba4860abd0">this guide</a> that explains its various rules in more detail.</p>
<p>For our purposes, we only need to know about these concepts:</p>
@ -16,7 +16,7 @@
<li><b>pin</b> -- To tell our IPFS daemon to host a file or group of files.</li>
<li><b>unpin</b> -- To tell our IPFS daemon to stop hosting a file or group of files.</li>
</ul>
<h3>getting ipfs</h3>
<h3 id="getting_ipfs"><a href="#getting_ipfs">getting ipfs</a></h3>
<p>Get the prebuilt executable <a href="https://docs.ipfs.io/guides/guides/install/#installing-from-a-prebuilt-package">here</a>. Inside should be a very simple 'ipfs' executable that does everything. Extract it somewhere and open up a terminal in the same folder, and then type:</p>
<ul>
<li>ipfs init</li>
@ -27,7 +27,7 @@
<p>You can kill it with Ctrl+C and restart it with the 'ipfs daemon' call again (you only have to run 'ipfs init' once).</p>
<p>When it is running, opening <a href="http://127.0.0.1:8080/ipfs/QmfM2r8seH2GiRaC4esTjeraXEachRt8ZsSeGaWTPLyMoG">this page</a> should download and display an example 'Hello World!' file from <span class="dealwithit">~~~across the internet~~~</span>.</p>
<p>Your daemon listens for other instances of ipfs using port 4001, so if you know how to open that port in your firewall and router, make sure you do.</p>
<h3>connecting your client</h3>
<h3 id="connecting"><a href="#connecting">connecting your client</a></h3>
<p>IPFS daemons are treated as services inside hydrus, so go to <i>services->manage services->remote->ipfs daemons</i> and add in your information. Hydrus uses the API port, default 5001, so you will probably want to use credentials of '127.0.0.1:5001'. You can click 'test credentials' to make sure everything is working.</p>
<p><img src="ipfs_services.png" /></p>
<p>Thereafter, you will get the option to 'pin' and 'unpin' from a thumbnail's right-click menu, like so:</p>
@ -45,14 +45,14 @@
<li>View it through a public web gateway, such as the one the IPFS people run, at http://ipfs.io/ipfs/[multihash]</li>
<li>Download it through their ipfs-connected hydrus client by going <i>pages->new download popup->an ipfs multihash</i>.</li>
</ul>
<h3>directories</h3>
<h3 id="directories"><a href="#directories">directories</a></h3>
<p>If you have many files to share, IPFS also supports directories, and now hydrus does as well. IPFS directories use the same sorts of multihash as files, and you can download them into the hydrus client using the same <i>pages->new download popup->an ipfs multihash</i> menu entry. The client will detect the multihash represents a directory and give you a simple selection dialog:</p>
<p><img src="ipfs_dir_download.png" /></p>
<p>You may recognise those hash filenames--this example was created by hydrus, which can create ipfs directories from any selection of files from the same right-click menu:</p>
<p><img src="ipfs_dir_upload.png" /></p>
<p>Hydrus will pin all the files and then wrap them in a directory, showing its progress in a popup. Your current directory shares are summarised on the respective <i>services->review services</i> panel:</p>
<p><img src="ipfs_review_services.png" /></p>
<h3>additional links</h3>
<h3 id="additional_links"><a href="#additional_links">additional links</a></h3>
<p>If you find you use IPFS a lot, here are some add-ons for your web browser, as recommended by /tech/:</p>
<blockquote><i>
<p>This script changes all bare ipfs hashes into clickable links to the ipfs gateway (on page loads):</p>

View File

@ -6,7 +6,7 @@
</head>
<body>
<div class="content">
<h3>launch arguments</h3>
<h3 id="intro"><a href="#intro">launch arguments</a></h3>
<p>You can launch the program with several different arguments to alter core behaviour. If you are not familiar with this, you are essentially putting additional text after the launch command that runs the program. You can run this straight from a terminal console (usually good to test with), or you can bundle it into an easy shortcut that you only have to double-click. An example of a launch command with arguments:</p>
<blockquote>C:\Hydrus Network\client.exe -d="E:\hydrus db" --no_db_temp_files</blockquote>
<p>You can also add --help to your program path, like this:</p>

View File

@ -6,10 +6,11 @@
</head>
<body>
<div class="content">
<h3>local booru</h3>
<p class="warning">This was a fun project, but it never advanced beyond a prototype. The future of this system is other people's nice applications plugging into the <a href="client_api.html">Client API</a>.</p>
<h3 id="intro"><a href="#intro">local booru</a></h3>
<p>The hydrus client has a simple booru to help you share your files with others over the internet.</p>
<p>First of all, this is <b>hosted from your client</b>, which means other people will be connecting to your computer and fetching files you choose to share from your hard drive. If you close your client or shut your computer down, the local booru will no longer work.</p>
<h3>how to do it</h3>
<h3 id="setting_up"><a href="#setting_up">how to do it</a></h3>
<p>First of all, turn the local booru server on by going to <i>services->manage services</i> and giving it a port:</p>
<p><img src="local_booru_services.png" /></p>
<p>It doesn't matter what you pick, but make it something fairly high. When you ok that dialog, the client should start the booru. You may get a firewall warning.</p>
@ -19,23 +20,23 @@
<p>You can also copy either the internal or external link to your clipboard. The internal link (usually starting something like http://127.0.0.1:45866/) works inside your network and is great just for testing, while the external link (starting http://[your external ip address]:[external port]/) will work for anyone around the world, <b>as long as your booru's port is being forwarded correctly</b>.</p>
<p>If you use a dynamic-ip service like <a href="https://www.noip.com/">No-IP</a>, you can replace your external IP with your redirect hostname. You have to do it by hand right now, but I'll add a way to do it automatically in future.</p>
<p class="warning">Note that anyone with the external link will be able to see your share, so make sure you only share links with people you trust.</p>
<h3>forwarding your port</h3>
<h3 id="port_forwarding"><a href="#port_forwarding">forwarding your port</a></h3>
<p>Your home router acts as a barrier between the computers inside the network and the internet. Those inside can see out, but outsiders can only see what you tell the router to permit. Since you want to let people connect to your computer, you need to tell the router to forward all requests of a certain kind to your computer, and thus your client.</p>
<p>If you have never done this before, it can be a headache, especially doing it manually. Luckily, a technology called UPnP makes it a ton easier, and this is how your Skype or Bittorrent clients do it automatically. Not all routers support it, but most do. You can have hydrus try to open a port this way back on <i>services->manage services</i>. Unless you know what you are doing and have a good reason to make them different, you might as well keep the internal and external ports the same.</p>
<p>Once you have it set up, the client will try to make sure your router keeps that port open for your client. If it all works, you should see the new mapping appear in your <i>services->manage local upnp</i> dialog, which lists all your router's current port mappings.</p>
<p>If you want to test that the port forward is set up correctly, going to http://[external ip]:[external port]/ should give a little html just saying hello. Your ISP might not allow you to talk to yourself, though, so ask a friend to try if you are having trouble.</p>
<p>If you still do not understand what is going on here, <a href="http://www.howtogeek.com/66214/how-to-forward-ports-on-your-router/">this</a> is a good article explaining everything.</p>
<p>If you do not like UPnP or your router does not support it, you can set the port forward up manually, but I encourage you to keep the internal and external port the same, because absent a 'upnp port' option, the 'copy external share link' button will use the internal port.</p>
<h3>so, what do you get?</h3>
<h3 id="example"><a href="#example">so, what do you get?</a></h3>
<p>The html layout is very simple:</p>
<hr />
<p><img src="local_booru_html.png" /></p>
<hr />
<p>It uses a very similar stylesheet to these help pages. If you would like to change the style, have a look at the html and then edit install_dir/static/local_booru_style.css. The thumbnails will be the same size as in your client.</p>
<h3>editing an existing share</h3>
<h3 id="editing_shares"><a href="#editing_shares">editing an existing share</a></h3>
<p>You can review all your shares on <i>services->review services</i>, under <i>local->booru</i>. You can copy the links again, change the title/text/expiration, and delete any shares you don't want any more.</p>
<h3>future plans</h3>
<p>I would like to add sorting controls, tag display, make those tags searchable, and generally AJAX-ify the entire thing to reduce bandwidth and offload CPU time to the web browser.</p>
<h3 id="future"><a href="#future">future plans</a></h3>
<p>This was a fun project, but it never advanced beyond a prototype. The future of this system is other people's nice applications plugging into the <a href="client_api.html">Client API</a>.</p>
</div>
</body>
</html>
</html>

View File

@ -6,9 +6,11 @@
</head>
<body>
<div class="content">
<h3>privacy</h3>
<p>Repositories never know what you are searching for. The client synchronises (copies) the repository's entire file or mapping list to its internal database, and does its own searches over those internal caches, all on your hard drive. <span class="warning">It <i>never</i> sends search queries outside your own computer, nor does it log what you do look for</span>. Your searches are your business, and no-one else's.</p>
<p>Repositories know nothing more about your client than they can infer, and the software usually commands them to forget as much as possible as soon as possible. Specifically:</p>
<h3 id="intro"><a href="#intro">privacy</a></h3>
<p>Repositories are designed to respect your privacy. They never know what you are searching for. The client synchronises (copies) the repository's entire file or mapping list to its internal database, and does its own searches over those internal caches, all on your hard drive. <span class="warning">It <i>never</i> sends search queries outside your own computer, nor does it log what you do look for</span>. Your searches are your business, and no-one else's.</p>
<p class="warning">The PTR has a public shared access key. You do not have to contact anyone to get the key, so no one can infer who you are from it, and all regular user uploads are merged together, making it all a big mess. The PTR is more private than this document's worst case scenarios.</p>
<p>The only privacy risk for hydrus's repositories are in what you upload (ultimately by using the pending menu at the top of the program). Even then, it would typically be very difficult even for an admin to figure anything about you, but it is possible.</p>
<p>Repositories know nothing more about your client than they can infer from what you choose upload, and the software usually commands them to forget as much as possible as soon as possible. Specifically:</p>
<table cellpadding="5" cellspacing="2" border="1">
<tr>
<td />
@ -23,7 +25,7 @@
<th>download file</th>
</tr>
<tr>
<th>Account is linked to action</th>
<th>Anonymous account is linked to action</th>
<td>Yes</td>
<td>No</td>
<td>Yes</td>
@ -53,9 +55,10 @@
<li>All accounts are anonymous. Repositories do not <i>know</i> any of their accounts' access keys and cannot produce them on demand; they can determine whether a particular access key refers to a particular account, but the access keys themselves are all irreversibly hashed inside the repository database.</li>
</ul>
</p>
<p>There are of course some clever exceptions. If you tag a file three years before it surfaces on the internet, someone with enough knowledge will be able to infer it was most likely you who created it. If you set up a file repository for just a friend and yourself, it becomes trivial by elimination to guess who uploaded the NarutoXSonichu shota diaper fanon. If you sign up for a file repository that hosts only certain stuff and rack up a huge bandwidth record for the current month, anyone who knows that and also knows the account is yours alone will know basically what you were up to.</p>
<p>Note also that the file repository code is freely available and entirely mutable. If someone wants to put the time in, they can create a file repository that looks from the outside like any other but nonetheless logs the IP and nature of every request. Just make sure you trust the person running the repository. (And make sure they suck at programming python!)</p>
<p><a href="https://en.wikipedia.org/wiki/AOL_search_data_leak">Even anonymised records can reveal personally identifying information.</a> Don't trust anyone who plans to release maps of accounts -> files or accounts -> mappings, even for some benevolent academic purpose.</p>
<p>As always, there are some clever exceptions, mostly in servers between friends that will just have a handful of users, where the admin would be handing out registration keys and, with effort, could pick through the limited user creation records to figure out which access key you were. In that case, if you were to tag a file three years before it surfaced on the internet, and the admin knew you are attached to the account that made that tag, they could infer you most likely created it. If you set up a file repository for just a friend and yourself, it becomes trivial by elimination to guess who uploaded the NarutoXSonichu shota diaper fanon. If you sign up for a file repository that hosts only certain stuff and rack up a huge bandwidth record for the current month, anyone who knows that and also knows the account is yours alone will know basically what you were up to.</p>
<p>The PTR has a shared access key that is already public, so the risks are far smaller. No one can figure out who you are from the access key.</p>
<p>Note that the code is freely available and entirely mutable. If someone wants to put the time in, they could create a file repository that looks from the outside like any other but nonetheless logs the IP and nature of every request. As with any website, protect yourself, and if you do not trust an admin, do not give them or their server any information about you.</p>
<p><a href="https://en.wikipedia.org/wiki/AOL_search_data_leak">Even anonymised records can reveal personally identifying information.</a> Don't trust anyone on any site who plans to release internal maps of 'anonymised' accounts -> content, even for some benevolent academic purpose.</p>
</div>
</body>
</html>
</html>

View File

@ -6,15 +6,15 @@
</head>
<body>
<div class="content">
<h3>hydrus is cpu and hdd hungry</h3>
<h3 id="intro"><a href="#intro">hydrus is cpu and hdd hungry</a></h3>
<p>The hydrus client manages a lot of complicated data and gives you a lot of power over it. To add millions of files and tags to its database, and then to perform difficult searches over that information, it needs to use a lot of CPU time and hard drive time--sometimes in small laggy blips, and occasionally in big 100% CPU chunks. I don't put training wheels or limiters on the software either, so if you search for 300,000 files, the client will try to fetch that many.</p>
<p>In general, the client works best on snappy computers with low-latency hard drives where it does not have to constantly compete with other CPU- or HDD- heavy programs. Running hydrus on your games computer is no problem at all, but if you leave the client on all the time, then make sure under the options it is set not to do idle work while your CPU is busy, so your games can run freely. Similarly, if you run two clients on the same computer, you should have them set to work at different times, because if they both try to process 500,000 tags at once on the same hard drive, they will each slow to a crawl.</p>
<p>If you run on an HDD, keeping it defragged is very important, and good practice for all your programs anyway. Make sure you know what this is and that you do it.</p>
<h3>maintenance and processing</h3>
<h3 id="maintenance_and_processing"><a href="#maintenance_and_processing">maintenance and processing</a></h3>
<p>I have attempted to offload most of the background maintenance of the client (which typically means repository processing and internal database defragging) to time when you are not using the client. This can either be 'idle time' or 'shutdown time'. The calculations for what these exactly mean are customisable in <i>file->options->maintenance and processing</i>.</p>
<p>If you run a quick computer, you likely don't have to change any of these options. Repositories will synchronise and the database will stay fairly optimal without you even noticing the work that is going on. This is especially true if you leave your client on all the time.</p>
<p>If you have an old, slower computer though, or if your hard drive is high latency, make sure these options are set for whatever is best for your situation. Turning off idle time completely is often helpful as some older computers are slow to even recognise--mid task--that you want to use the client again, or take too long to abandon a big task half way through. If you set your client to only do work on shutdown, then you can control exactly when that happens.</p>
<h3>reducing search and general gui lag</h3>
<h3 id="reducing_lag"><a href="#reducing_lag">reducing search and general gui lag</a></h3>
<p>Searching for tags via the autocomplete dropdown and searching for files in general can sometimes take a very long time. It depends on many things. In general, the more predicates (tags and system:something) you have active for a search, and the more specific they are, the faster it will be.</p>
<p>You can also look at <i>file->options->speed and memory</i>, again especially if you have a slow computer. Increasing the autocomplete thresholds is very often helpful. You can even force autocompletes to only fetch results when you manually ask for them.</p>
<p>Having lots of thumbnails open or downloads running can slow many things down. Check the 'pages' menu to see your current session weight. If it is about 50,000, or you have individual pages with more than 10,000 files or download URLs, try cutting down a bit.</p>

View File

@ -6,15 +6,15 @@
</head>
<body>
<div class="content">
<h3>running from source</h3>
<h3 id="intro"><a href="#intro">running from source</a></h3>
<p>I write the client and server entirely in <a href="https://python.org">python</a>, which can run straight from source. It is not simple to get hydrus running this way, but if none of the built packages work for you (for instance you use a non-Ubuntu-compatible flavour of Linux), it may be the only way you can get the program to run. Also, if you have a general interest in exploring the code or wish to otherwise modify the program, you will obviously need to do this stuff.</p>
<h3>a quick note about Linux flavours</h3>
<h3 id="linux_flavours"><a href="#linux_flavours">a quick note about Linux flavours</a></h3>
<p>I often point people here when they are running non-Ubuntu flavours of Linux and cannot run my build. One Debian user mentioned that he had an error like this:</p>
<p><ul>
<li><i>ImportError: /home/user/hydrus/libX11.so.6: undefined symbol: xcb_poll_for_reply64</i></li>
</ul></p>
<p>But that by simply deleting the <i>libX11.so.6</i> file in the hydrus install directory, he was able to boot. I presume this meant my hydrus build was then relying on his local libX11.so, which happened to have better API compatibility. If you receive a similar error, you might like to try the same sort of thing. Let me know if you discover anything!</p>
<h3>what you will need</h3>
<h3 id="what_you_need"><a href="#what_you_need">what you will need</a></h3>
<p>You will need basic python experience, python 3.x and a number of python modules. Most of it you can get through pip.</p>
<p>If you are on Linux or macOS, or if you are on Windows and have an existing python you do not want to stomp all over with new modules, I recommend you create a virtual environment:</p>
<p><i>Note, if you are on Linux, it may be easier to use your package manager instead of messing around with venv. A user has written a great summary with all needed packages <a href="running_from_source_linux_packages.txt">here</a>.</i></p>
@ -63,7 +63,7 @@
<p>If you don't have ffmpeg in your PATH and you want to import videos, you will need to put a static <a href="https://ffmpeg.org/">FFMPEG</a> executable in the install_dir/bin directory. Have a look at how I do it in the extractable compiled releases if you can't figure it out. On Windows, you can copy the exe from one of those releases, or just download the latest static build right from the FFMPEG site.</a>
<p>Once you have everything set up, client.pyw and server.py should look for and run off client.db and server.db just like the executables. They will look in the 'db' directory by default, or anywhere you point them with the "-d" parameter, again just like the executables.</p>
<p>I develop hydrus on and am most experienced with Windows, so the program is more stable and reasonable on that. I do not have as much experience with Linux or macOS, so I would particularly appreciate your Linux/macOS bug reports and any informed suggestions.</p>
<h3>my code</h3>
<h3 id="my_code"><a href="#my_code">my code</a></h3>
<p>Unlike most software people, I am more INFJ than INTP/J. My coding style is unusual and unprofessional, and everything is pretty much hacked together. Please look through the source if you are interested in how things work and ask me if you don't understand something. I'm constantly throwing new code together and then cleaning and overhauling it down the line.</p>
<p>I work strictly alone, so while I am very interested in detailed bug reports or suggestions for good libraries to use, I am not looking for pull requests. Everything I do is <a href="https://github.com/sirkris/WTFPL/blob/master/WTFPL.md">WTFPL</a>, so feel free to fork and play around with things on your end as much as you like.</p>
</div>

View File

@ -8,7 +8,7 @@
<div class="content">
<p class="warning"><b>You do not need the server to do anything with hydrus! It is only for advanced users to do very specific jobs!</b> The server is also hacked-together and quite technical. It requires a fair amount of experience with the client and its concepts, and it does not operate on a timescale that works well on a LAN. Only try running your own server once you have a bit of experience synchronising with something like the PTR and you think, 'Hey, I know exactly what that does, and I would like one!'</p>
<p><b><a href="https://github.com/Zweibach/text/blob/master/Hydrus/youDontWantTheServer.md">Here is a document put together by a user describing whether you want the server.</a></b></p>
<h3>setting up a server</h3>
<h3 id="intro"><a href="#intro">setting up a server</a></h3>
<p>I will use two terms, <i>server</i> and <i>service</i>, to mean two distinct things:</p>
<ul>
<li>A <b>server</b> is an instantiation of the hydrus server executable (e.g. server.exe in Windows). It has a complicated and flexible database that can run many different services in parallel.</li>
@ -25,32 +25,32 @@
<li>Profit</li>
</ul>
<p>Let's look at these steps in more detail:</p>
<h3>start the server</h3>
<p>Since the server and client have so much common code, I package them together. If you have the client, you have the server. If you installed in Windows, you can hit the shortcut in your start menu. Otherwise, go straight to 'server' or 'server.exe' or 'server.pyw' in your installation directory. The program will first try to take port 45870 for its administration interface, so make sure that is free. Open your firewall as appropriate.</p>
<h3>set up the client</h3>
<h3 id="start"><a href="#start">start the server</a></h3>
<p>Since the server and client have so much common code, I package them together. If you have the client, you have the server. If you installed in Windows, you can hit the shortcut in your start menu. Otherwise, go straight to 'server' or 'server.exe' or 'server.pyw' in your installation directory. The program will first try to take port 45870 for its administration interface, so make sure that is free. Open your firewall as appropriate.</p>_client
<h3 id="setting_up_the_client"><a href="#setting_up_the_client">set up the client</a></h3>
<p>In the <i>services->manage services</i> dialog, add a new 'hydrus server administration service' and set up the basic options as appropriate. If you are running the server on the same computer as the client, its hostname is 'localhost'.</p>
<p>In order to set up the first admin account and an access key, use 'init' as a registration key. This special registration key will only work to initialise this first super-account.</p>
<p class="warning">YOU'LL WANT TO SAVE YOUR ACCESS KEY IN A SAFE PLACE</p>
<p>If you lose your admin access key, there is no way to get it back, and if you are not sqlite-proficient, you'll have to restart from the beginning by deleting your server's database files.</p>
<p>If the client can't connect to the server, it is either not running or you have a firewall/port-mapping problem. If you want a quick way to test the server's visibility, just put https://host:port into your browser (make sure it is https! http will not work)--if it is working, your browser will probably complain about its self-signed https certificate. Once you add a certificate exception, the server should return some simple html identifying itself.</p>
<h3>set up the server</h3>
<h3 id="setting_up_the_server"><a href="#setting_up_the_server">set up the server</a></h3>
<p>You should have a new submenu, 'administrate services', under 'services', in the client gui. This is where you control most server and service-wide stuff.</p>
<p><i>admin->your server->manage services</i> lets you add, edit, and delete the services your server runs. Every time you add one, you will also be added as that service's first administrator, and the admin menu will gain a new entry for it.</i>
<h3>making accounts</h3>
<h3 id="making_accounts"><a href="#making_accounts">making accounts</a></h3>
<p>Go <i>admin->your service->create new accounts</i> to create new registration keys. Send the registration keys to the users you want to give these new accounts. A registration key will only work once, so if you want to give several people the same account, they will have to share the access key amongst themselves once one of them has registered the account. (Or you can register the account yourself and send them all the same access key. Do what you like!)</p>
<p>Go <i>admin->manage account types</i> to add, remove, or edit account types. Make sure everyone has at least downloader (get_data) permissions so they can stay synchronised.</p>
<p>You can create as many accounts of whatever kind you like. Depending on your usage scenario, you may want to have all uploaders, one uploader and many downloaders, or just a single administrator. There are many combinations.</p>
<h3>???</h3>
<h3 id="have_fun"><a href="#have_fun">???</a></h3>
<p>The most important part is to have fun! There are no losers on the INFORMATION SUPERHIGHWAY.</p>
<h3>profit</h3>
<h3 id="profit"><a href="#profit">profit</a></h3>
<p>I honestly hope you can get some benefit out of my code, whether just as a backup or as part of a far more complex system. Please mail me your comments as I am always keen to make improvements.</p>
<h3>btw, how to backup a repo's db</h3>
<h3 id="backing_up"><a href="#backing_up">btw, how to backup a repo's db</a></h3>
<p>All of a server's files and options are stored in its accompanying .db file and respective subdirectories, which are created on first startup (just like with the client). To backup or restore, you have two options:</p>
<ul>
<li>Shut down the server, copy the database files and directories, then restart it. This is the only way, currently, to restore a db.</li>
<li>In the client, hit admin->your server->make a backup. This will lock the db server-side while it makes a copy of everything server-related to server_install_dir/db/server_backup. When the operation is complete, you can ftp/batch-copy/whatever the server_backup folder wherever you like.</li>
</ul>
<h3>OMG EVERYTHING WENT WRONG</h3>
<h3 id="hell"><a href="#hell">OMG EVERYTHING WENT WRONG</a></h3>
<p>If you get to a point where you can no longer boot the repository, try running SQLite Studio and opening server.db. If the issue is simple--like manually changing the port number--you may be in luck. Send me an email if it is tricky.</p>
<p>Remember that everything is breaking all the time. Make regular backups, and you'll minimise your problems.</p>
</div>

View File

@ -6,7 +6,7 @@
</head>
<body>
<div class="content">
<h3>can I contribute to hydrus development?</h3>
<h3 id="support"><a href="#support">can I contribute to hydrus development?</a></h3>
<p>I do not expect anything from anyone. I'm amazed and grateful that anyone wants to use my software and share tags with others. I enjoy the feedback and work, and I hope to keep putting completely free weekly releases out as long as there is more to do.</p>
<p>That said, as I have developed the software, several users have kindly offered to contribute money, either as thanks for a specific feature or just in general. I kept putting the thought off, but I eventually got over my hesitance and set something up.</p>
<p>I find the tactics of most internet fundraising very distasteful, especially when they promise something they then fail to deliver. I much prefer the 'if you like me and would like to contribute, then please do, meanwhile I'll keep doing what I do' model. I support several 'put out regular free content' creators on Patreon in this way, and I get a lot out of it, even though I have no direct reward beyond the knowledge that I helped some people do something neat.</p>

View File

@ -8,11 +8,11 @@
<div class="content">
<p class="warning">This document was originally written for when I ran the Public Tag Repository. This is now run by users, so I am no longer an authority for it. I am briefly editing the page and leaving it as a record for some of my thoughts on tagging if you are interested. You can, of course, run your own tag repositories and do your own thing additionally or instead.</p>
<p class="warning">A newer guide and schema for the PTR is <a href="https://github.com/Zweibach/text/blob/master/Hydrus/PTR.md">here</a>.</p>
<h3>seriousness of schema</h3>
<h3 id="intro"><a href="#intro">seriousness of schema</a></h3>
<p>This is not all that important; it just makes searches and cooperation easier if most of us can mostly follow some guidelines.</p>
<p>We will never be able to easily and perfectly categorise every single image to everyone's satisfaction, so there is no point defining every possible rule for every possible situation. If you do something that doesn't fit, fixing mistakes is not difficult.</p>
<p>If you are still not confident, just lurk for a bit. See how other people have tagged the popular images and do more of that.</p>
<h3>you can add pretty much whatever the hell you want, but don't screw around</h3>
<h3 id="take_it_easy"><a href="#take_it_easy">you can add pretty much whatever the hell you want, but don't screw around</a></h3>
<p>The most important thing is: <b>if your tag is your opinion, don't add it</b>. 'beautiful' is an unhelpful tag because no one can agree on what it means. 'lingerie', 'blue eyes', and 'male' or 'female' are better since reasonable people can generally agree on what they mean. If someone thinks blue-eyed women are beautiful, they can search for that to find beautiful things.</p>
<p>You can start your own namespaces, categorisation systems, whatever. Just be aware that everyone else will see what you do.</p>
<p>If you are still unsure about the difference between objective and subjective, here's some more examples:</p>
@ -41,18 +41,18 @@
</li>
</ul>
<p>Of course, if you are tagging a picture of someone holding a sign that says 'beautiful', you can bend the rules. Otherwise, please keep your opinions to yourself!</p>
<h3>numbers</h3>
<h3 id="numbers"><a href="#numbers">numbers</a></h3>
<p>Numbers should be written '22', '1457 ce', and 'page:3', unless as part of an official title like 'ocean's eleven'. When the client parses and sorts numbers, it does so intelligently, so just use '1' where you might before have done '01' or '001'. I know it looks ugly sometimes to have '2 girls' or '1 cup', but the rules for writing numbers out in full are hazy for special cases.</p>
<p>(Numbers written as 123 are also readable by many different language-speakers, while 'tano', 'deux' and 'seven' are not.)</p>
<h3>plurals</h3>
<h3 id="plurals"><a href="#plurals">plurals</a></h3>
<p>Nouns should generally be singular, not plural. 'chair' instead of 'chairs', 'cat' instead of 'cats', even if there are several of the thing in the image. If there really are <i>many</i> of the thing in the image, add a seperate 'multiple' or 'lineup' tag as apppropriate.</p>
<p>Ignore this when the thing is normally said in its plural (usually paired) form. Say 'blue eyes', not 'blue eye'; 'breasts', not 'breast', even if only one is pictured.</p>
<h3>acronyms and synonyms</h3>
<p>I prefer the full 'series:the lord of the rings' rather than 'lotr'. If you are an advanced user, please help out with tag siblings to help induce this.</p>
<h3>character:anna (frozen)</h3>
<h3 id="synonyms"><a href="#synonyms">acronyms and synonyms</a></h3>
<p>I personally prefer the full 'series:the lord of the rings' rather than 'lotr'. If you are an advanced user, please help out with tag siblings to help induce this.</p>
<h3 id="namespace_disambiguation"><a href="#namespace_disambiguation">character:anna (frozen)</a></h3>
<p>I am not fond of putting a series name after a character because it looks unusual and is applied unreliably. It is done to separate same-named characters from each other (particularly when they have no canon surname), which is useful in places that search slowly, have thin tag areas on their web pages, or usually only deal in single-tag searches. For archival purposes, I generally prefer that namespaces are stored as the namespace and nowhere else. 'series:harry potter' and 'character:harry potter', not 'harry potter (harry potter)'. Some sites even say things like 'anna (disney)'. It isn't a big deal, but if you are adding a sibling to collapse these divergent tags into the 'proper' one, I'd prefer it all went to the simple and reliable 'character:anna'. Even better would be migrating towards a canon-ok unique name, like 'character:princess anna of arendelle', which could have the parent 'series:frozen'.</p>
<p>Including nicknames, like 'character:angela "mercy" ziegler' can be useful to establish uniqueness, but are not mandatory. 'character:harleen "harley quinn" frances quinzel' is probably overboard.</p>
<h3>protip: rein in your spergitude</h3>
<h3 id="protip"><a href="#protip">protip: rein in your spergitude</a></h3>
<p>In developing hydrus, I have discovered two rules to happy tagging:</p>
<ol>
<li>Don't try to be perfect.</li>
@ -60,15 +60,15 @@
</ol>
<p>Tagging can be fun, but it can also be complicated, and the problem space is gigantic. There is always works to do, and it is easy to exhaust onesself or get lost in the bushes agonising over whether to use 'smile' or 'smiling' or 'smirk' or one of a million other split hairs. Problems are easy to fix, and this marathon will never finish, so do not try to sprint. The ride never ends.</p>
<p>The sheer number of tags can also be overwhelming. Importing all the many tags from the boorus is totally fine, but if you are typing tags yourself, I suggest you try not to exhaustively tag <a href="http://safebooru.org/index.php?page=post&s=list&tags=card_on_necklace">everything</a> <a href="http://gelbooru.com/index.php?page=post&s=list&tags=collarbone">in</a> <a href="https://e621.net/post/index?tags=cum_on_neck">the</a> <a href="http://danbooru.donmai.us/posts?tags=brown_vest">image</a>. You will save a lot of time and ultimately be much happier with your work. Anyone can see what is in an image just by looking at it--tags are primarily for finding things. Character, series and creator namespaces are a great place to start. After that, add what you are interested in, be that 'blue sky' or 'midriff'.</p>
<h3>newer thoughts on presentation preferences</h3>
<h3 id="presentation"><a href="#presentation">newer thoughts on presentation preferences</a></h3>
<p>Since developing and receiving feedback for the siblings system, and then in dealing with siblings with the PTR, I have come to believe that the most difficult disagreement to resolve in tagging is not in what is in an image, but how those tags should present. It is easy to agree that an image contains a 'bikini', but should that show as 'bikini' or 'clothing:bikini' or 'general:bikini' or 'swimwear:bikini'? Which is better?</p>
<p>This is impossible to answer definitively. There is no perfect dictionary that satisfies everyone, and opinions are fairly fixed. My intentions for future versions of the sibling and tag systems is to allow users to broadly tell the client some display rules such as 'Whenever you have a clothing: tag, display it as unnamespaced' and eventually more sophisticated ones like 'I prefer slang, so show pussy instead of vagina'.</p>
<h3>siblings and parents</h3>
<h3 id="siblings_and_parents"><a href="#siblings_and_parents">siblings and parents</a></h3>
<p>Please do add <a href="advanced_siblings.html">siblings</a> and <a href="advanced_parents.html">parents</a>! If it is something not obvious, please explain the relationship in your submitted reason. If it <i>is</i> something obvious (e.g. 'wings' is a parent of 'angel wings'), don't bother to put a reason in; I'll just approve it.</p>
<p>My general thoughts:</p>
<ul>
<li>
<h3>siblings</h3>
<h3 id="siblings"><a href="#siblings">siblings</a></h3>
<p>In general, the correctness of a thing is in how it would describe itself, or how its creator would describe it.</p>
<p>For shorthand, I will say 'a'->'b' to mean 'a' is replaced by 'b'.</p>
<p>For instance, japanese names are usually written surname first and western forename first, so let's go 'character:rei ayanami'->'character:ayanami rei' but leave 'person:emma watson' and other western names as they are.</p>
@ -82,7 +82,7 @@
<p>In general, swap out slang for proper terms. 'lube'->'lubricant', 'series:zelda'->'series:the legend of zelda'.</p>
</li>
<li>
<h3>parents</h3>
<h3 id="parents"><a href="#parents">parents</a></h3>
<p>Be shy about adding character:blah->series:whatever unless you are certain the character name is unique. 'character:harry potter'->'series:harry potter' seems fairly uncontroversial, for instance, but adding specific sub-series just to be completionist, such as 'character:miranda lawson->series:mass effect: redemption' is asking for trouble.</p>
<p>Remember that parents define a relationship that is always true. Don't add 'blonde hair' to 'character:elsa', even though it is true in most files--add 'animal ears' to 'cat ears', as cat ears are always animal ears, no matter what an artist can think up.</p>
<p>Also, tag parents are only worth something if the parent is useful for searching. Adding 'medium:blue background'->'blue' isn't useful since 'blue' itself is not very valuable, but 'fishnet stockings'->'stockings' is useful as both tags are common and used in searches by plenty of people.</p>

View File

@ -6,7 +6,7 @@
</head>
<body>
<div class="content">
<h3>getting it to work on wine</h3>
<h3 id="intro"><a href="#intro">getting it to work on wine</ap</h3>
<p>Several Linux and macOS users have found success running hydrus with Wine. Here is a post from a Linux dude:</p>
<i>
<p>Some things I picked up on after extended use:</p>

View File

@ -129,6 +129,14 @@ SIMPLE_MOVE_PAGES_SELECTION_RIGHT = 122
SIMPLE_MOVE_PAGES_SELECTION_HOME = 123
SIMPLE_MOVE_PAGES_SELECTION_END = 124
SIMPLE_REFRESH_RELATED_TAGS = 125
SIMPLE_AUTOCOMPLETE_FORCE_FETCH = 126
SIMPLE_AUTOCOMPLETE_IME_MODE = 127
SIMPLE_AUTOCOMPLETE_IF_EMPTY_TAB_LEFT = 128
SIMPLE_AUTOCOMPLETE_IF_EMPTY_TAB_RIGHT = 129
SIMPLE_AUTOCOMPLETE_IF_EMPTY_PAGE_LEFT = 130
SIMPLE_AUTOCOMPLETE_IF_EMPTY_PAGE_RIGHT = 131
SIMPLE_AUTOCOMPLETE_IF_EMPTY_MEDIA_PREVIOUS = 132
SIMPLE_AUTOCOMPLETE_IF_EMPTY_MEDIA_NEXT = 133
simple_enum_to_str_lookup = {
SIMPLE_ARCHIVE_DELETE_FILTER_BACK : 'archive/delete filter: back',
@ -256,7 +264,15 @@ simple_enum_to_str_lookup = {
SIMPLE_DUPLICATE_MEDIA_REMOVE_POTENTIALS : 'file relationships: remote files from potential duplicate pairs',
SIMPLE_DUPLICATE_MEDIA_SET_POTENTIAL : 'file relationships: set files as potential duplicates',
SIMPLE_DUPLICATE_MEDIA_CLEAR_FALSE_POSITIVES : 'file relationships: clear false positives',
SIMPLE_DUPLICATE_MEDIA_CLEAR_FOCUSED_FALSE_POSITIVES : 'file relationships: clear focused file false positives'
SIMPLE_DUPLICATE_MEDIA_CLEAR_FOCUSED_FALSE_POSITIVES : 'file relationships: clear focused file false positives',
SIMPLE_AUTOCOMPLETE_FORCE_FETCH : 'force-fetch tag autocomplete results',
SIMPLE_AUTOCOMPLETE_IME_MODE : 'flip IME-friendly mode on/off (this disables extra shortcut processing so you can do IME popup stuff)',
SIMPLE_AUTOCOMPLETE_IF_EMPTY_TAB_LEFT : 'if input is empty, move left one autocomplete dropdown tab',
SIMPLE_AUTOCOMPLETE_IF_EMPTY_TAB_RIGHT : 'if input is empty, move right one autocomplete dropdown tab',
SIMPLE_AUTOCOMPLETE_IF_EMPTY_PAGE_LEFT : 'if input & results list are empty, move to left one service page',
SIMPLE_AUTOCOMPLETE_IF_EMPTY_PAGE_RIGHT : 'if input & results list are empty, move to right one service page',
SIMPLE_AUTOCOMPLETE_IF_EMPTY_MEDIA_PREVIOUS : 'if input & results list are empty and in media viewer manage tags dialog, move to previous media',
SIMPLE_AUTOCOMPLETE_IF_EMPTY_MEDIA_NEXT : 'if input & results list are empty and in media viewer manage tags dialog, move to previous media'
}
legacy_simple_str_to_enum_lookup = {

View File

@ -26,7 +26,6 @@ from hydrus.core import HydrusVideoHandling
from hydrus.client import ClientAPI
from hydrus.client import ClientCaches
from hydrus.client import ClientConstants as CC
from hydrus.client import ClientDB
from hydrus.client import ClientDaemons
from hydrus.client import ClientDefaults
from hydrus.client import ClientDownloading
@ -36,6 +35,7 @@ from hydrus.client import ClientOptions
from hydrus.client import ClientSearch
from hydrus.client import ClientServices
from hydrus.client import ClientThreading
from hydrus.client.db import ClientDB
from hydrus.client.gui import ClientGUI
from hydrus.client.gui import ClientGUIDialogs
from hydrus.client.gui import ClientGUIScrolledPanelsManagement

View File

@ -364,12 +364,27 @@ def GetDefaultShortcuts():
shortcuts.append( media )
tags_autocomplete = ClientGUIShortcuts.ShortcutSet( 'tags_autocomplete' )
tags_autocomplete.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_SPECIAL, ClientGUIShortcuts.SHORTCUT_KEY_SPECIAL_SPACE, ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [ ClientGUIShortcuts.SHORTCUT_MODIFIER_CTRL ] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_AUTOCOMPLETE_FORCE_FETCH ) )
tags_autocomplete.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_SPECIAL, ClientGUIShortcuts.SHORTCUT_KEY_SPECIAL_INSERT, ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_AUTOCOMPLETE_IME_MODE ) )
tags_autocomplete.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_SPECIAL, ClientGUIShortcuts.SHORTCUT_KEY_SPECIAL_LEFT, ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_AUTOCOMPLETE_IF_EMPTY_TAB_LEFT ) )
tags_autocomplete.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_SPECIAL, ClientGUIShortcuts.SHORTCUT_KEY_SPECIAL_RIGHT, ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_AUTOCOMPLETE_IF_EMPTY_TAB_RIGHT ) )
tags_autocomplete.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_SPECIAL, ClientGUIShortcuts.SHORTCUT_KEY_SPECIAL_UP, ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_AUTOCOMPLETE_IF_EMPTY_PAGE_LEFT ) )
tags_autocomplete.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_SPECIAL, ClientGUIShortcuts.SHORTCUT_KEY_SPECIAL_DOWN, ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_AUTOCOMPLETE_IF_EMPTY_PAGE_RIGHT ) )
tags_autocomplete.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_SPECIAL, ClientGUIShortcuts.SHORTCUT_KEY_SPECIAL_PAGE_UP, ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_AUTOCOMPLETE_IF_EMPTY_MEDIA_PREVIOUS ) )
tags_autocomplete.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_SPECIAL, ClientGUIShortcuts.SHORTCUT_KEY_SPECIAL_PAGE_DOWN, ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_AUTOCOMPLETE_IF_EMPTY_MEDIA_NEXT ) )
tags_autocomplete.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_CHARACTER, ord( 'I' ), ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [ ClientGUIShortcuts.SHORTCUT_MODIFIER_CTRL ] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_SYNCHRONISED_WAIT_SWITCH ) )
shortcuts.append( tags_autocomplete )
main_gui = ClientGUIShortcuts.ShortcutSet( 'main_gui' )
main_gui.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_SPECIAL, ClientGUIShortcuts.SHORTCUT_KEY_SPECIAL_F5, ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_REFRESH ) )
main_gui.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_SPECIAL, ClientGUIShortcuts.SHORTCUT_KEY_SPECIAL_F9, ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_NEW_PAGE ) )
main_gui.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_CHARACTER, ord( 'I' ), ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [ ClientGUIShortcuts.SHORTCUT_MODIFIER_CTRL ] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_SYNCHRONISED_WAIT_SWITCH ) )
main_gui.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_CHARACTER, ord( 'M' ), ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [ ClientGUIShortcuts.SHORTCUT_MODIFIER_CTRL ] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_SET_MEDIA_FOCUS ) )
main_gui.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_CHARACTER, ord( 'R' ), ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [ ClientGUIShortcuts.SHORTCUT_MODIFIER_CTRL, ClientGUIShortcuts.SHORTCUT_MODIFIER_SHIFT ] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_SHOW_HIDE_SPLITTERS ) )
main_gui.SetCommand( ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_CHARACTER, ord( 'S' ), ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [ ClientGUIShortcuts.SHORTCUT_MODIFIER_CTRL ] ), CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_SET_SEARCH_FOCUS ) )

View File

@ -1,4 +1,5 @@
import collections
import collections.abc
import json
import os
import time
@ -1064,12 +1065,46 @@ class HydrusResourceClientAPIRestrictedAddTagsAddTags( HydrusResourceClientAPIRe
continue
if isinstance( tags[0], str ):
content_action = int( content_action )
actual_tags = []
tags_to_reasons = {}
for tag_item in tags:
tags = HydrusTags.CleanTags( tags )
reason = 'Petitioned from API'
if isinstance( tag_item, str ):
tag = tag_item
elif isinstance( tag_item, collections.abc.Collection ) and len( tag_item ) == 2:
( tag, reason ) = tag_item
if not ( isinstance( tag, str ) and isinstance( reason, str ) ):
continue
else:
continue
actual_tags.append( tag )
tags_to_reasons[ tag ] = reason
content_action = int( content_action )
actual_tags = HydrusTags.CleanTags( actual_tags )
if len( actual_tags ) == 0:
continue
tags = actual_tags
if service.GetServiceType() == HC.LOCAL_TAG:
@ -1088,16 +1123,7 @@ class HydrusResourceClientAPIRestrictedAddTagsAddTags( HydrusResourceClientAPIRe
if content_action == HC.CONTENT_UPDATE_PETITION:
if isinstance( tags[0], str ):
tags_and_reasons = [ ( tag, 'Petitioned from API' ) for tag in tags ]
else:
tags_and_reasons = tags
content_updates = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, content_action, ( tag, hashes ), reason = reason ) for ( tag, reason ) in tags_and_reasons ]
content_updates = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, content_action, ( tag, hashes ), reason = tags_to_reasons[ tag ] ) for tag in tags ]
else:

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,679 @@
import os
import sqlite3
import typing
from hydrus.core import HydrusData
from hydrus.core import HydrusDB
from hydrus.core import HydrusDBModule
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusTags
from hydrus.client.networking import ClientNetworkingDomain
class ClientDBMasterHashes( HydrusDBModule.HydrusDBModule ):
def __init__( self, cursor: sqlite3.Cursor ):
HydrusDBModule.HydrusDBModule.__init__( self, 'client hashes master', cursor )
self._hash_ids_to_hashes_cache = {}
def _GetIndexGenerationTuples( self ):
index_generation_tuples = []
index_generation_tuples.append( ( 'external_master.local_hashes', [ 'md5' ], False ) )
index_generation_tuples.append( ( 'external_master.local_hashes', [ 'sha1' ], False ) )
index_generation_tuples.append( ( 'external_master.local_hashes', [ 'sha512' ], False ) )
return index_generation_tuples
def _PopulateHashIdsToHashesCache( self, hash_ids, exception_on_error = False ):
if len( self._hash_ids_to_hashes_cache ) > 100000:
if not isinstance( hash_ids, set ):
hash_ids = set( hash_ids )
self._hash_ids_to_hashes_cache = { hash_id : hash for ( hash_id, hash ) in self._hash_ids_to_hashes_cache.items() if hash_id in hash_ids }
uncached_hash_ids = { hash_id for hash_id in hash_ids if hash_id not in self._hash_ids_to_hashes_cache }
if len( uncached_hash_ids ) > 0:
pubbed_error = False
if len( uncached_hash_ids ) == 1:
( uncached_hash_id, ) = uncached_hash_ids
rows = self._c.execute( 'SELECT hash_id, hash FROM hashes WHERE hash_id = ?;', ( uncached_hash_id, ) ).fetchall()
else:
with HydrusDB.TemporaryIntegerTable( self._c, uncached_hash_ids, 'hash_id' ) as temp_table_name:
# temp hash_ids to actual hashes
rows = self._c.execute( 'SELECT hash_id, hash FROM {} CROSS JOIN hashes USING ( hash_id );'.format( temp_table_name ) ).fetchall()
uncached_hash_ids_to_hashes = dict( rows )
if len( uncached_hash_ids_to_hashes ) < len( uncached_hash_ids ):
for hash_id in uncached_hash_ids:
if hash_id not in uncached_hash_ids_to_hashes:
if exception_on_error:
raise HydrusExceptions.DataMissing( 'Did not find all entries for those hash ids!' )
HydrusData.DebugPrint( 'Database hash error: hash_id ' + str( hash_id ) + ' was missing!' )
HydrusData.PrintException( Exception( 'Missing file identifier stack trace.' ) )
if not pubbed_error:
HydrusData.ShowText( 'A file identifier was missing! This is a serious error that means your client database has an orphan file id! Think about contacting hydrus dev!' )
pubbed_error = True
hash = bytes.fromhex( 'aaaaaaaaaaaaaaaa' ) + os.urandom( 16 )
uncached_hash_ids_to_hashes[ hash_id ] = hash
self._hash_ids_to_hashes_cache.update( uncached_hash_ids_to_hashes )
def CreateTables( self ):
self._c.execute( 'CREATE TABLE IF NOT EXISTS external_master.hashes ( hash_id INTEGER PRIMARY KEY, hash BLOB_BYTES UNIQUE );' )
self._c.execute( 'CREATE TABLE IF NOT EXISTS external_master.local_hashes ( hash_id INTEGER PRIMARY KEY, md5 BLOB_BYTES, sha1 BLOB_BYTES, sha512 BLOB_BYTES );' )
def GetExpectedTableNames( self ) -> typing.Collection[ str ]:
expected_table_names = [
'external_master.hashes',
'external_master.local_hashes'
]
return expected_table_names
def GetExtraHash( self, hash_type, hash_id ):
result = self._c.execute( 'SELECT {} FROM local_hashes WHERE hash_id = ?;'.format( hash_type ), ( hash_id, ) ).fetchone()
if result is None:
raise HydrusExceptions.DataMissing( '{} not available for file {}!'.format( hash_type, hash_id ) )
( hash, ) = result
return hash
def GetFileHashes( self, given_hashes, given_hash_type, desired_hash_type ):
if given_hash_type == 'sha256':
hash_ids = self.GetHashIds( given_hashes )
else:
hash_ids = []
for given_hash in given_hashes:
if given_hash is None:
continue
result = self._c.execute( 'SELECT hash_id FROM local_hashes WHERE {} = ?;'.format( given_hash_type ), ( sqlite3.Binary( given_hash ), ) ).fetchone()
if result is not None:
( hash_id, ) = result
hash_ids.append( hash_id )
if desired_hash_type == 'sha256':
desired_hashes = self.GetHashes( hash_ids )
else:
desired_hashes = [ desired_hash for ( desired_hash, ) in self._c.execute( 'SELECT {} FROM local_hashes WHERE hash_id IN {};'.format( desired_hash_type, HydrusData.SplayListForDB( hash_ids ) ) ) ]
return desired_hashes
def GetHash( self, hash_id ):
self._PopulateHashIdsToHashesCache( ( hash_id, ) )
return self._hash_ids_to_hashes_cache[ hash_id ]
def GetHashes( self, hash_ids ):
self._PopulateHashIdsToHashesCache( hash_ids )
return [ self._hash_ids_to_hashes_cache[ hash_id ] for hash_id in hash_ids ]
def GetHashId( self, hash ) -> int:
result = self._c.execute( 'SELECT hash_id FROM hashes WHERE hash = ?;', ( sqlite3.Binary( hash ), ) ).fetchone()
if result is None:
self._c.execute( 'INSERT INTO hashes ( hash ) VALUES ( ? );', ( sqlite3.Binary( hash ), ) )
hash_id = self._c.lastrowid
else:
( hash_id, ) = result
return hash_id
def GetHashIdFromExtraHash( self, hash_type, hash ):
if hash_type == 'md5':
result = self._c.execute( 'SELECT hash_id FROM local_hashes WHERE md5 = ?;', ( sqlite3.Binary( hash ), ) ).fetchone()
elif hash_type == 'sha1':
result = self._c.execute( 'SELECT hash_id FROM local_hashes WHERE sha1 = ?;', ( sqlite3.Binary( hash ), ) ).fetchone()
elif hash_type == 'sha512':
result = self._c.execute( 'SELECT hash_id FROM local_hashes WHERE sha512 = ?;', ( sqlite3.Binary( hash ), ) ).fetchone()
if result is None:
raise HydrusExceptions.DataMissing( 'Hash Id not found for {} hash {}!'.format( hash_type, hash.hex() ) )
( hash_id, ) = result
return hash_id
def GetHashIds( self, hashes ) -> typing.Set[ int ]:
hash_ids = set()
hashes_not_in_db = set()
for hash in hashes:
if hash is None:
continue
result = self._c.execute( 'SELECT hash_id FROM hashes WHERE hash = ?;', ( sqlite3.Binary( hash ), ) ).fetchone()
if result is None:
hashes_not_in_db.add( hash )
else:
( hash_id, ) = result
hash_ids.add( hash_id )
if len( hashes_not_in_db ) > 0:
self._c.executemany( 'INSERT INTO hashes ( hash ) VALUES ( ? );', ( ( sqlite3.Binary( hash ), ) for hash in hashes_not_in_db ) )
for hash in hashes_not_in_db:
( hash_id, ) = self._c.execute( 'SELECT hash_id FROM hashes WHERE hash = ?;', ( sqlite3.Binary( hash ), ) ).fetchone()
hash_ids.add( hash_id )
return hash_ids
def GetHashIdsToHashes( self, hash_ids = None, hashes = None ):
if hash_ids is not None:
self._PopulateHashIdsToHashesCache( hash_ids, exception_on_error = True )
hash_ids_to_hashes = { hash_id : self._hash_ids_to_hashes_cache[ hash_id ] for hash_id in hash_ids }
elif hashes is not None:
hash_ids_to_hashes = { self.GetHashId( hash ) : hash for hash in hashes }
return hash_ids_to_hashes
def HasExtraHashes( self, hash_id ):
result = self._c.execute( 'SELECT 1 FROM local_hashes WHERE hash_id = ?;', ( hash_id, ) ).fetchone()
return result is not None
def SetExtraHashes( self, hash_id, md5, sha1, sha512 ):
self._c.execute( 'INSERT OR IGNORE INTO local_hashes ( hash_id, md5, sha1, sha512 ) VALUES ( ?, ?, ?, ? );', ( hash_id, sqlite3.Binary( md5 ), sqlite3.Binary( sha1 ), sqlite3.Binary( sha512 ) ) )
class ClientDBMasterTexts( HydrusDBModule.HydrusDBModule ):
def __init__( self, cursor: sqlite3.Cursor ):
HydrusDBModule.HydrusDBModule.__init__( self, 'client texts master', cursor )
def _GetIndexGenerationTuples( self ):
index_generation_tuples = []
return index_generation_tuples
def CreateTables( self ):
self._c.execute( 'CREATE TABLE IF NOT EXISTS external_master.labels ( label_id INTEGER PRIMARY KEY, label TEXT UNIQUE );' )
self._c.execute( 'CREATE TABLE IF NOT EXISTS external_master.notes ( note_id INTEGER PRIMARY KEY, note TEXT UNIQUE );' )
self._c.execute( 'CREATE TABLE IF NOT EXISTS external_master.texts ( text_id INTEGER PRIMARY KEY, text TEXT UNIQUE );' )
def GetExpectedTableNames( self ) -> typing.Collection[ str ]:
expected_table_names = [
'external_master.labels',
'external_master.notes',
'external_master.texts'
]
return expected_table_names
def GetLabelId( self, label ):
result = self._c.execute( 'SELECT label_id FROM labels WHERE label = ?;', ( label, ) ).fetchone()
if result is None:
self._c.execute( 'INSERT INTO labels ( label ) VALUES ( ? );', ( label, ) )
label_id = self._c.lastrowid
else:
( label_id, ) = result
return label_id
def GetText( self, text_id ):
result = self._c.execute( 'SELECT text FROM texts WHERE text_id = ?;', ( text_id, ) ).fetchone()
if result is None:
raise HydrusExceptions.DataMissing( 'Text lookup error in database' )
( text, ) = result
return text
def GetTextId( self, text ):
result = self._c.execute( 'SELECT text_id FROM texts WHERE text = ?;', ( text, ) ).fetchone()
if result is None:
self._c.execute( 'INSERT INTO texts ( text ) VALUES ( ? );', ( text, ) )
text_id = self._c.lastrowid
else:
( text_id, ) = result
return text_id
class ClientDBMasterTags( HydrusDBModule.HydrusDBModule ):
def __init__( self, cursor: sqlite3.Cursor ):
HydrusDBModule.HydrusDBModule.__init__( self, 'client master', cursor )
self.null_namespace_id = None
self._tag_ids_to_tags_cache = {}
def _GetIndexGenerationTuples( self ):
index_generation_tuples = []
index_generation_tuples.append( ( 'external_master.tags', [ 'subtag_id' ], False ) )
index_generation_tuples.append( ( 'external_master.tags', [ 'namespace_id', 'subtag_id' ], True ) )
return index_generation_tuples
def _PopulateTagIdsToTagsCache( self, tag_ids ):
if len( self._tag_ids_to_tags_cache ) > 100000:
if not isinstance( tag_ids, set ):
tag_ids = set( tag_ids )
self._tag_ids_to_tags_cache = { tag_id : tag for ( tag_id, tag ) in self._tag_ids_to_tags_cache.items() if tag_id in tag_ids }
uncached_tag_ids = { tag_id for tag_id in tag_ids if tag_id not in self._tag_ids_to_tags_cache }
if len( uncached_tag_ids ) > 0:
if len( uncached_tag_ids ) == 1:
( uncached_tag_id, ) = uncached_tag_ids
rows = self._c.execute( 'SELECT tag_id, namespace, subtag FROM tags NATURAL JOIN namespaces NATURAL JOIN subtags WHERE tag_id = ?;', ( uncached_tag_id, ) ).fetchall()
else:
with HydrusDB.TemporaryIntegerTable( self._c, uncached_tag_ids, 'tag_id' ) as temp_table_name:
# temp tag_ids to tags to subtags and namespaces
rows = self._c.execute( 'SELECT tag_id, namespace, subtag FROM {} CROSS JOIN tags USING ( tag_id ) CROSS JOIN subtags USING ( subtag_id ) CROSS JOIN namespaces USING ( namespace_id );'.format( temp_table_name ) ).fetchall()
uncached_tag_ids_to_tags = { tag_id : HydrusTags.CombineTag( namespace, subtag ) for ( tag_id, namespace, subtag ) in rows }
if len( uncached_tag_ids_to_tags ) < len( uncached_tag_ids ):
for tag_id in uncached_tag_ids:
if tag_id not in uncached_tag_ids_to_tags:
tag = 'unknown tag:' + HydrusData.GenerateKey().hex()
( namespace, subtag ) = HydrusTags.SplitTag( tag )
namespace_id = self.GetNamespaceId( namespace )
subtag_id = self.GetSubtagId( subtag )
self._c.execute( 'REPLACE INTO tags ( tag_id, namespace_id, subtag_id ) VALUES ( ?, ?, ? );', ( tag_id, namespace_id, subtag_id ) )
uncached_tag_ids_to_tags[ tag_id ] = tag
self._tag_ids_to_tags_cache.update( uncached_tag_ids_to_tags )
def CreateTables( self ):
self._c.execute( 'CREATE TABLE IF NOT EXISTS external_master.namespaces ( namespace_id INTEGER PRIMARY KEY, namespace TEXT UNIQUE );' )
self._c.execute( 'CREATE TABLE IF NOT EXISTS external_master.subtags ( subtag_id INTEGER PRIMARY KEY, subtag TEXT UNIQUE );' )
self._c.execute( 'CREATE TABLE IF NOT EXISTS external_master.tags ( tag_id INTEGER PRIMARY KEY, namespace_id INTEGER, subtag_id INTEGER );' )
def GetExpectedTableNames( self ) -> typing.Collection[ str ]:
expected_table_names = [
'external_master.namespaces',
'external_master.subtags',
'external_master.tags'
]
return expected_table_names
def GetNamespaceId( self, namespace ):
if namespace == '':
if self.null_namespace_id is None:
( self.null_namespace_id, ) = self._c.execute( 'SELECT namespace_id FROM namespaces WHERE namespace = ?;', ( '', ) ).fetchone()
return self.null_namespace_id
result = self._c.execute( 'SELECT namespace_id FROM namespaces WHERE namespace = ?;', ( namespace, ) ).fetchone()
if result is None:
self._c.execute( 'INSERT INTO namespaces ( namespace ) VALUES ( ? );', ( namespace, ) )
namespace_id = self._c.lastrowid
else:
( namespace_id, ) = result
return namespace_id
def GetSubtagId( self, subtag ):
result = self._c.execute( 'SELECT subtag_id FROM subtags WHERE subtag = ?;', ( subtag, ) ).fetchone()
if result is None:
self._c.execute( 'INSERT INTO subtags ( subtag ) VALUES ( ? );', ( subtag, ) )
subtag_id = self._c.lastrowid
else:
( subtag_id, ) = result
return subtag_id
def GetTagId( self, tag ):
clean_tag = HydrusTags.CleanTag( tag )
try:
HydrusTags.CheckTagNotEmpty( clean_tag )
except HydrusExceptions.TagSizeException:
raise HydrusExceptions.TagSizeException( '"{}" tag seems not valid--when cleaned, it ends up with zero size!'.format( tag ) )
( namespace, subtag ) = HydrusTags.SplitTag( clean_tag )
namespace_id = self.GetNamespaceId( namespace )
subtag_id = self.GetSubtagId( subtag )
result = self._c.execute( 'SELECT tag_id FROM tags WHERE namespace_id = ? AND subtag_id = ?;', ( namespace_id, subtag_id ) ).fetchone()
if result is None:
self._c.execute( 'INSERT INTO tags ( namespace_id, subtag_id ) VALUES ( ?, ? );', ( namespace_id, subtag_id ) )
tag_id = self._c.lastrowid
else:
( tag_id, ) = result
return tag_id
def GetTagIdsToTags( self, tag_ids = None, tags = None ):
if tag_ids is not None:
self._PopulateTagIdsToTagsCache( tag_ids )
tag_ids_to_tags = { tag_id : self._tag_ids_to_tags_cache[ tag_id ] for tag_id in tag_ids }
elif tags is not None:
tag_ids_to_tags = { self.GetTagId( tag ) : tag for tag in tags }
return tag_ids_to_tags
class ClientDBMasterURLs( HydrusDBModule.HydrusDBModule ):
def __init__( self, cursor: sqlite3.Cursor ):
HydrusDBModule.HydrusDBModule.__init__( self, 'client master', cursor )
def _GetIndexGenerationTuples( self ):
index_generation_tuples = []
index_generation_tuples.append( ( 'external_master.urls', [ 'domain_id' ], False ) )
return index_generation_tuples
def CreateTables( self ):
self._c.execute( 'CREATE TABLE IF NOT EXISTS external_master.url_domains ( domain_id INTEGER PRIMARY KEY, domain TEXT UNIQUE );' )
self._c.execute( 'CREATE TABLE IF NOT EXISTS external_master.urls ( url_id INTEGER PRIMARY KEY, domain_id INTEGER, url TEXT UNIQUE );' )
def GetExpectedTableNames( self ) -> typing.Collection[ str ]:
expected_table_names = [
'external_master.url_domains',
'external_master.urls'
]
return expected_table_names
def GetURLDomainId( self, domain ):
result = self._c.execute( 'SELECT domain_id FROM url_domains WHERE domain = ?;', ( domain, ) ).fetchone()
if result is None:
self._c.execute( 'INSERT INTO url_domains ( domain ) VALUES ( ? );', ( domain, ) )
domain_id = self._c.lastrowid
else:
( domain_id, ) = result
return domain_id
def GetURLDomainAndSubdomainIds( self, domain, only_www_subdomains = False ):
domain = ClientNetworkingDomain.RemoveWWWFromDomain( domain )
domain_ids = set()
domain_ids.add( self.GetURLDomainId( domain ) )
if only_www_subdomains:
search_phrase = 'www%.{}'.format( domain )
else:
search_phrase = '%.{}'.format( domain )
for ( domain_id, ) in self._c.execute( 'SELECT domain_id FROM url_domains WHERE domain LIKE ?;', ( search_phrase, ) ):
domain_ids.add( domain_id )
return domain_ids
def GetURLId( self, url ):
result = self._c.execute( 'SELECT url_id FROM urls WHERE url = ?;', ( url, ) ).fetchone()
if result is None:
try:
domain = ClientNetworkingDomain.ConvertURLIntoDomain( url )
except HydrusExceptions.URLClassException:
domain = 'unknown.com'
domain_id = self.GetURLDomainId( domain )
self._c.execute( 'INSERT INTO urls ( domain_id, url ) VALUES ( ?, ? );', ( domain_id, url ) )
url_id = self._c.lastrowid
else:
( url_id, ) = result
return url_id

View File

@ -0,0 +1 @@

View File

@ -3077,6 +3077,31 @@ class FrameGUI( ClientGUITopLevelWindows.MainFrameThatResizes ):
def _RegenerateTagDisplayPendingMappingsCache( self ):
message = 'This will delete and then recreate the pending tags on the tag \'display\' mappings cache, which is used for user-presented tag searching, loading, and autocomplete counts. This is useful if you have \'ghost\' pending tags or counts hanging around.'
message += os.linesep * 2
message += 'If you have a millions of pending tags, it can take a long time, during which the gui may hang.'
message += os.linesep * 2
message += 'If you do not have a specific reason to run this, it is pointless.'
result = ClientGUIDialogsQuick.GetYesNo( self, message, yes_label = 'do it--now choose which service', no_label = 'forget it' )
if result == QW.QDialog.Accepted:
try:
tag_service_key = GetTagServiceKeyForMaintenance( self )
except HydrusExceptions.CancelledException:
return
self._controller.Write( 'regenerate_tag_display_pending_mappings_cache', tag_service_key = tag_service_key )
def _RegenerateTagMappingsCache( self ):
message = 'WARNING: Do not run this for no reason! On a large database, this could take hours to finish!'
@ -3996,16 +4021,6 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
def _PausePlaySearch( self ):
page = self._notebook.GetCurrentMediaPage()
if page is not None:
page.PausePlaySearch()
def _SetupBackupPath( self ):
backup_intro = 'Everything in your client is stored in the database, which consists of a handful of .db files and a single subdirectory that contains all your media files. It is a very good idea to maintain a regular backup schedule--to save from hard drive failure, serious software fault, accidental deletion, or any other unexpected problem. It sucks to lose all your work, so make sure it can\'t happen!'
@ -4955,7 +4970,8 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
submenu = QW.QMenu( menu )
ClientGUIMenus.AppendMenuItem( submenu, 'tag storage mappings cache', 'Delete and recreate the tag mappings cache, fixing any miscounts.', self._RegenerateTagMappingsCache )
ClientGUIMenus.AppendMenuItem( submenu, 'tag display mappings cache', 'Delete and recreate the tag display mappings cache, fixing any miscounts.', self._RegenerateTagDisplayMappingsCache )
ClientGUIMenus.AppendMenuItem( submenu, 'tag display mappings cache (deferred siblings & parents calculation)', 'Delete and recreate the tag display mappings cache, fixing any miscounts.', self._RegenerateTagDisplayMappingsCache )
ClientGUIMenus.AppendMenuItem( submenu, 'tag display mappings cache (just pending tags, instant calculation)', 'Delete and recreate the tag display pending mappings cache, fixing any miscounts.', self._RegenerateTagDisplayPendingMappingsCache )
ClientGUIMenus.AppendMenuItem( submenu, 'tag siblings lookup cache', 'Delete and recreate the tag siblings cache.', self._RegenerateTagSiblingsLookupCache )
ClientGUIMenus.AppendMenuItem( submenu, 'tag parents lookup cache', 'Delete and recreate the tag siblings cache.', self._RegenerateTagParentsLookupCache )
ClientGUIMenus.AppendMenuItem( submenu, 'tag text search cache', 'Delete and regenerate the cache hydrus uses for fast tag search.', self._RegenerateTagCache )
@ -6253,10 +6269,6 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
self._ShowHideSplitters()
elif action == CAC.SIMPLE_SYNCHRONISED_WAIT_SWITCH:
self._PausePlaySearch()
elif action == CAC.SIMPLE_SET_MEDIA_FOCUS:
self._SetMediaFocus()

View File

@ -373,7 +373,7 @@ class Canvas( QW.QWidget ):
# once we have catch_mouse full shortcut support for canvases, swap out this out for an option to swallow activating clicks
ignore_activating_mouse_click = catch_mouse and not self.PREVIEW_WINDOW
self._my_shortcuts_handler = ClientGUIShortcuts.ShortcutsHandler( self, initial_shortcuts_names = ( 'media', 'media_viewer' ), catch_mouse = catch_mouse, ignore_activating_mouse_click = ignore_activating_mouse_click )
self._my_shortcuts_handler = ClientGUIShortcuts.ShortcutsHandler( self, [ 'media', 'media_viewer' ], catch_mouse = catch_mouse, ignore_activating_mouse_click = ignore_activating_mouse_click )
self._click_drag_reporting_filter = MediaContainerDragClickReportingFilter( self )

View File

@ -902,11 +902,6 @@ class ManagementPanel( QW.QScrollArea ):
pass
def PausePlaySearch( self ):
pass
def REPEATINGPageUpdate( self ):
pass
@ -1852,9 +1847,9 @@ class ManagementPanelImporterMultipleGallery( ManagementPanelImporter ):
sort_gallery_paused = 1
status = gallery_import.GetSimpleStatus()
pretty_status = gallery_import.GetSimpleStatus()
pretty_status = status
sort_status = ( sort_gallery_paused, pretty_status )
file_seed_cache_status = gallery_import.GetFileSeedCache().GetStatus()
@ -1869,7 +1864,7 @@ class ManagementPanelImporterMultipleGallery( ManagementPanelImporter ):
pretty_added = ClientData.TimestampToPrettyTimeDelta( added, show_seconds = False )
display_tuple = ( pretty_query_text, pretty_source, pretty_files_paused, pretty_gallery_paused, pretty_status, pretty_progress, pretty_added )
sort_tuple = ( query_text, pretty_source, files_paused, sort_gallery_paused, status, progress, added )
sort_tuple = ( query_text, pretty_source, files_paused, sort_gallery_paused, sort_status, progress, added )
return ( display_tuple, sort_tuple )
@ -2659,17 +2654,12 @@ class ManagementPanelImporterMultipleWatcher( ManagementPanelImporter ):
pretty_added = ClientData.TimestampToPrettyTimeDelta( added, show_seconds = False )
watcher_status = self._multiple_watcher_import.GetWatcherSimpleStatus( watcher )
pretty_watcher_status = self._multiple_watcher_import.GetWatcherSimpleStatus( watcher )
pretty_watcher_status = watcher_status
if watcher_status == '':
watcher_status = 'zzz' # to sort _after_ DEAD and other interesting statuses on ascending sort
sort_watcher_status = ( sort_checking_paused, pretty_watcher_status )
display_tuple = ( pretty_subject, pretty_files_paused, pretty_checking_paused, pretty_watcher_status, pretty_progress, pretty_added )
sort_tuple = ( subject, files_paused, sort_checking_paused, watcher_status, progress, added )
sort_tuple = ( subject, files_paused, sort_checking_paused, sort_watcher_status, progress, added )
return ( display_tuple, sort_tuple )
@ -4888,14 +4878,6 @@ class ManagementPanelQuery( ManagementPanel ):
def PausePlaySearch( self ):
if self._search_enabled:
self._tag_autocomplete.PausePlaySearch()
def THREADDoQuery( self, controller, page_key, query_job_key, search_context, sort_by ):
def qt_code():

View File

@ -847,11 +847,6 @@ class Page( QW.QSplitter ):
def PausePlaySearch( self ):
self._management_panel.PausePlaySearch()
def _StartInitialMediaResultsLoad( self ):
def qt_code_status( status ):

View File

@ -455,7 +455,7 @@ class EditShortcutsPanel( ClientGUIScrolledPanels.EditPanel ):
if name in ClientGUIShortcuts.shortcut_names_to_descriptions:
pretty_name = ClientGUIShortcuts.shortcut_names_to_pretty_names[ name ]
sort_name = ClientGUIShortcuts.shortcut_names_to_sort_order[ name ]
sort_name = ClientGUIShortcuts.shortcut_names_sorted.index( name )
else:

View File

@ -180,6 +180,7 @@ shortcut_names_to_pretty_names = {}
shortcut_names_to_pretty_names[ 'global' ] = 'global'
shortcut_names_to_pretty_names[ 'main_gui' ] = 'the main window'
shortcut_names_to_pretty_names[ 'tags_autocomplete' ] = 'tag autocomplete'
shortcut_names_to_pretty_names[ 'media' ] = 'media actions, either thumbnails or the viewer'
shortcut_names_to_pretty_names[ 'media_viewer' ] = 'media viewers - all (zoom and pan)'
shortcut_names_to_pretty_names[ 'media_viewer_browser' ] = 'media viewer - \'normal\' browser'
@ -188,17 +189,18 @@ shortcut_names_to_pretty_names[ 'duplicate_filter' ] = 'media viewer - duplicate
shortcut_names_to_pretty_names[ 'preview_media_window' ] = 'media viewer - the preview window'
shortcut_names_to_pretty_names[ 'media_viewer_media_window' ] = 'the actual media in a media viewer'
shortcut_names_to_sort_order = {}
shortcut_names_to_sort_order[ 'global' ] = 0
shortcut_names_to_sort_order[ 'main_gui' ] = 1
shortcut_names_to_sort_order[ 'media' ] = 2
shortcut_names_to_sort_order[ 'media_viewer' ] = 3
shortcut_names_to_sort_order[ 'media_viewer_browser' ] = 4
shortcut_names_to_sort_order[ 'archive_delete_filter' ] = 5
shortcut_names_to_sort_order[ 'duplicate_filter' ] = 6
shortcut_names_to_sort_order[ 'preview_media_window' ] = 7
shortcut_names_to_sort_order[ 'media_viewer_media_window' ] = 8
shortcut_names_sorted = [
'global',
'main_gui',
'tags_autocomplete',
'media',
'media_viewer',
'media_viewer_browser',
'archive_delete_filter',
'duplicate_filter',
'preview_media_window',
'media_viewer_media_window'
]
shortcut_names_to_descriptions = {}
@ -207,6 +209,7 @@ shortcut_names_to_descriptions[ 'archive_delete_filter' ] = 'Navigation actions
shortcut_names_to_descriptions[ 'duplicate_filter' ] = 'Navigation actions for the media viewer during a duplicate filter. Mouse shortcuts should work.'
shortcut_names_to_descriptions[ 'media' ] = 'Actions to alter metadata for media in the media viewer or the thumbnail grid.'
shortcut_names_to_descriptions[ 'main_gui' ] = 'Actions to control pages in the main window of the program.'
shortcut_names_to_descriptions[ 'tags_autocomplete' ] = 'Actions to control tag autocomplete when its input text box is focused.'
shortcut_names_to_descriptions[ 'media_viewer_browser' ] = 'Navigation actions for the regular browsable media viewer.'
shortcut_names_to_descriptions[ 'media_viewer' ] = 'Zoom and pan and player actions for any media viewer.'
shortcut_names_to_descriptions[ 'media_viewer_media_window' ] = 'Actions for any video or audio player in a media viewer window.'
@ -214,13 +217,14 @@ shortcut_names_to_descriptions[ 'preview_media_window' ] = 'Actions for any vide
# shortcut commands
SHORTCUTS_RESERVED_NAMES = [ 'global', 'archive_delete_filter', 'duplicate_filter', 'media', 'main_gui', 'media_viewer_browser', 'media_viewer', 'media_viewer_media_window', 'preview_media_window' ]
SHORTCUTS_RESERVED_NAMES = [ 'global', 'archive_delete_filter', 'duplicate_filter', 'media', 'tags_autocomplete', 'main_gui', 'media_viewer_browser', 'media_viewer', 'media_viewer_media_window', 'preview_media_window' ]
SHORTCUTS_GLOBAL_ACTIONS = [ CAC.SIMPLE_GLOBAL_AUDIO_MUTE, CAC.SIMPLE_GLOBAL_AUDIO_UNMUTE, CAC.SIMPLE_GLOBAL_AUDIO_MUTE_FLIP, CAC.SIMPLE_EXIT_APPLICATION, CAC.SIMPLE_EXIT_APPLICATION_FORCE_MAINTENANCE, CAC.SIMPLE_RESTART_APPLICATION, CAC.SIMPLE_HIDE_TO_SYSTEM_TRAY ]
SHORTCUTS_MEDIA_ACTIONS = [ CAC.SIMPLE_MANAGE_FILE_TAGS, CAC.SIMPLE_MANAGE_FILE_RATINGS, CAC.SIMPLE_MANAGE_FILE_URLS, CAC.SIMPLE_MANAGE_FILE_NOTES, CAC.SIMPLE_ARCHIVE_FILE, CAC.SIMPLE_INBOX_FILE, CAC.SIMPLE_DELETE_FILE, CAC.SIMPLE_UNDELETE_FILE, CAC.SIMPLE_EXPORT_FILES, CAC.SIMPLE_EXPORT_FILES_QUICK_AUTO_EXPORT, CAC.SIMPLE_REMOVE_FILE_FROM_VIEW, CAC.SIMPLE_OPEN_FILE_IN_EXTERNAL_PROGRAM, CAC.SIMPLE_OPEN_SELECTION_IN_NEW_PAGE, CAC.SIMPLE_LAUNCH_THE_ARCHIVE_DELETE_FILTER, CAC.SIMPLE_COPY_BMP, CAC.SIMPLE_COPY_BMP_OR_FILE_IF_NOT_BMPABLE, CAC.SIMPLE_COPY_FILE, CAC.SIMPLE_COPY_PATH, CAC.SIMPLE_COPY_SHA256_HASH, CAC.SIMPLE_COPY_MD5_HASH, CAC.SIMPLE_COPY_SHA1_HASH, CAC.SIMPLE_COPY_SHA512_HASH, CAC.SIMPLE_GET_SIMILAR_TO_EXACT, CAC.SIMPLE_GET_SIMILAR_TO_VERY_SIMILAR, CAC.SIMPLE_GET_SIMILAR_TO_SIMILAR, CAC.SIMPLE_GET_SIMILAR_TO_SPECULATIVE, CAC.SIMPLE_DUPLICATE_MEDIA_SET_ALTERNATE, CAC.SIMPLE_DUPLICATE_MEDIA_SET_ALTERNATE_COLLECTIONS, CAC.SIMPLE_DUPLICATE_MEDIA_SET_CUSTOM, CAC.SIMPLE_DUPLICATE_MEDIA_SET_FOCUSED_BETTER, CAC.SIMPLE_DUPLICATE_MEDIA_SET_FOCUSED_KING, CAC.SIMPLE_DUPLICATE_MEDIA_SET_SAME_QUALITY, CAC.SIMPLE_OPEN_KNOWN_URL ]
SHORTCUTS_MEDIA_VIEWER_ACTIONS = [ CAC.SIMPLE_PAUSE_MEDIA, CAC.SIMPLE_PAUSE_PLAY_MEDIA, CAC.SIMPLE_MOVE_ANIMATION_TO_PREVIOUS_FRAME, CAC.SIMPLE_MOVE_ANIMATION_TO_NEXT_FRAME, CAC.SIMPLE_SWITCH_BETWEEN_FULLSCREEN_BORDERLESS_AND_REGULAR_FRAMED_WINDOW, CAC.SIMPLE_PAN_UP, CAC.SIMPLE_PAN_DOWN, CAC.SIMPLE_PAN_LEFT, CAC.SIMPLE_PAN_RIGHT, CAC.SIMPLE_PAN_TOP_EDGE, CAC.SIMPLE_PAN_BOTTOM_EDGE, CAC.SIMPLE_PAN_LEFT_EDGE, CAC.SIMPLE_PAN_RIGHT_EDGE, CAC.SIMPLE_PAN_VERTICAL_CENTER, CAC.SIMPLE_PAN_HORIZONTAL_CENTER, CAC.SIMPLE_ZOOM_IN, CAC.SIMPLE_ZOOM_OUT, CAC.SIMPLE_SWITCH_BETWEEN_100_PERCENT_AND_CANVAS_ZOOM, CAC.SIMPLE_FLIP_DARKMODE, CAC.SIMPLE_CLOSE_MEDIA_VIEWER ]
SHORTCUTS_MEDIA_VIEWER_BROWSER_ACTIONS = [ CAC.SIMPLE_VIEW_NEXT, CAC.SIMPLE_VIEW_FIRST, CAC.SIMPLE_VIEW_LAST, CAC.SIMPLE_VIEW_PREVIOUS, CAC.SIMPLE_PAUSE_PLAY_SLIDESHOW, CAC.SIMPLE_SHOW_MENU, CAC.SIMPLE_CLOSE_MEDIA_VIEWER ]
SHORTCUTS_MAIN_GUI_ACTIONS = [ CAC.SIMPLE_REFRESH, CAC.SIMPLE_REFRESH_ALL_PAGES, CAC.SIMPLE_REFRESH_PAGE_OF_PAGES_PAGES, CAC.SIMPLE_NEW_PAGE, CAC.SIMPLE_NEW_PAGE_OF_PAGES, CAC.SIMPLE_NEW_DUPLICATE_FILTER_PAGE, CAC.SIMPLE_NEW_GALLERY_DOWNLOADER_PAGE, CAC.SIMPLE_NEW_URL_DOWNLOADER_PAGE, CAC.SIMPLE_NEW_SIMPLE_DOWNLOADER_PAGE, CAC.SIMPLE_NEW_WATCHER_DOWNLOADER_PAGE, CAC.SIMPLE_SYNCHRONISED_WAIT_SWITCH, CAC.SIMPLE_SET_MEDIA_FOCUS, CAC.SIMPLE_SHOW_HIDE_SPLITTERS, CAC.SIMPLE_SET_SEARCH_FOCUS, CAC.SIMPLE_UNCLOSE_PAGE, CAC.SIMPLE_CLOSE_PAGE, CAC.SIMPLE_REDO, CAC.SIMPLE_UNDO, CAC.SIMPLE_FLIP_DARKMODE, CAC.SIMPLE_RUN_ALL_EXPORT_FOLDERS, CAC.SIMPLE_CHECK_ALL_IMPORT_FOLDERS, CAC.SIMPLE_FLIP_DEBUG_FORCE_IDLE_MODE_DO_NOT_SET_THIS, CAC.SIMPLE_SHOW_AND_FOCUS_MANAGE_TAGS_FAVOURITE_TAGS, CAC.SIMPLE_SHOW_AND_FOCUS_MANAGE_TAGS_RELATED_TAGS, CAC.SIMPLE_REFRESH_RELATED_TAGS, CAC.SIMPLE_SHOW_AND_FOCUS_MANAGE_TAGS_FILE_LOOKUP_SCRIPT_TAGS, CAC.SIMPLE_SHOW_AND_FOCUS_MANAGE_TAGS_RECENT_TAGS, CAC.SIMPLE_FOCUS_MEDIA_VIEWER, CAC.SIMPLE_MOVE_PAGES_SELECTION_LEFT, CAC.SIMPLE_MOVE_PAGES_SELECTION_RIGHT, CAC.SIMPLE_MOVE_PAGES_SELECTION_HOME, CAC.SIMPLE_MOVE_PAGES_SELECTION_END ]
SHORTCUTS_MAIN_GUI_ACTIONS = [ CAC.SIMPLE_REFRESH, CAC.SIMPLE_REFRESH_ALL_PAGES, CAC.SIMPLE_REFRESH_PAGE_OF_PAGES_PAGES, CAC.SIMPLE_NEW_PAGE, CAC.SIMPLE_NEW_PAGE_OF_PAGES, CAC.SIMPLE_NEW_DUPLICATE_FILTER_PAGE, CAC.SIMPLE_NEW_GALLERY_DOWNLOADER_PAGE, CAC.SIMPLE_NEW_URL_DOWNLOADER_PAGE, CAC.SIMPLE_NEW_SIMPLE_DOWNLOADER_PAGE, CAC.SIMPLE_NEW_WATCHER_DOWNLOADER_PAGE, CAC.SIMPLE_SET_MEDIA_FOCUS, CAC.SIMPLE_SHOW_HIDE_SPLITTERS, CAC.SIMPLE_SET_SEARCH_FOCUS, CAC.SIMPLE_UNCLOSE_PAGE, CAC.SIMPLE_CLOSE_PAGE, CAC.SIMPLE_REDO, CAC.SIMPLE_UNDO, CAC.SIMPLE_FLIP_DARKMODE, CAC.SIMPLE_RUN_ALL_EXPORT_FOLDERS, CAC.SIMPLE_CHECK_ALL_IMPORT_FOLDERS, CAC.SIMPLE_FLIP_DEBUG_FORCE_IDLE_MODE_DO_NOT_SET_THIS, CAC.SIMPLE_SHOW_AND_FOCUS_MANAGE_TAGS_FAVOURITE_TAGS, CAC.SIMPLE_SHOW_AND_FOCUS_MANAGE_TAGS_RELATED_TAGS, CAC.SIMPLE_REFRESH_RELATED_TAGS, CAC.SIMPLE_SHOW_AND_FOCUS_MANAGE_TAGS_FILE_LOOKUP_SCRIPT_TAGS, CAC.SIMPLE_SHOW_AND_FOCUS_MANAGE_TAGS_RECENT_TAGS, CAC.SIMPLE_FOCUS_MEDIA_VIEWER, CAC.SIMPLE_MOVE_PAGES_SELECTION_LEFT, CAC.SIMPLE_MOVE_PAGES_SELECTION_RIGHT, CAC.SIMPLE_MOVE_PAGES_SELECTION_HOME, CAC.SIMPLE_MOVE_PAGES_SELECTION_END ]
SHORTCUTS_TAGS_AUTOCOMPLETE_ACTIONS = [ CAC.SIMPLE_SYNCHRONISED_WAIT_SWITCH, CAC.SIMPLE_AUTOCOMPLETE_FORCE_FETCH, CAC.SIMPLE_AUTOCOMPLETE_IME_MODE ]
SHORTCUTS_DUPLICATE_FILTER_ACTIONS = [ CAC.SIMPLE_DUPLICATE_FILTER_THIS_IS_BETTER_AND_DELETE_OTHER, CAC.SIMPLE_DUPLICATE_FILTER_THIS_IS_BETTER_BUT_KEEP_BOTH, CAC.SIMPLE_DUPLICATE_FILTER_EXACTLY_THE_SAME, CAC.SIMPLE_DUPLICATE_FILTER_ALTERNATES, CAC.SIMPLE_DUPLICATE_FILTER_FALSE_POSITIVE, CAC.SIMPLE_DUPLICATE_FILTER_CUSTOM_ACTION, CAC.SIMPLE_DUPLICATE_FILTER_SKIP, CAC.SIMPLE_DUPLICATE_FILTER_BACK, CAC.SIMPLE_CLOSE_MEDIA_VIEWER ]
SHORTCUTS_ARCHIVE_DELETE_FILTER_ACTIONS = [ CAC.SIMPLE_ARCHIVE_DELETE_FILTER_KEEP, CAC.SIMPLE_ARCHIVE_DELETE_FILTER_DELETE, CAC.SIMPLE_ARCHIVE_DELETE_FILTER_SKIP, CAC.SIMPLE_ARCHIVE_DELETE_FILTER_BACK, CAC.SIMPLE_CLOSE_MEDIA_VIEWER ]
SHORTCUTS_MEDIA_VIEWER_VIDEO_AUDIO_PLAYER_ACTIONS = [ CAC.SIMPLE_PAUSE_MEDIA, CAC.SIMPLE_PAUSE_PLAY_MEDIA, CAC.SIMPLE_OPEN_FILE_IN_EXTERNAL_PROGRAM, CAC.SIMPLE_CLOSE_MEDIA_VIEWER ]
@ -233,6 +237,7 @@ simple_shortcut_name_to_action_lookup[ 'media' ] = SHORTCUTS_MEDIA_ACTIONS
simple_shortcut_name_to_action_lookup[ 'media_viewer' ] = SHORTCUTS_MEDIA_VIEWER_ACTIONS
simple_shortcut_name_to_action_lookup[ 'media_viewer_browser' ] = SHORTCUTS_MEDIA_VIEWER_BROWSER_ACTIONS
simple_shortcut_name_to_action_lookup[ 'main_gui' ] = SHORTCUTS_MAIN_GUI_ACTIONS
simple_shortcut_name_to_action_lookup[ 'tags_autocomplete' ] = SHORTCUTS_TAGS_AUTOCOMPLETE_ACTIONS
simple_shortcut_name_to_action_lookup[ 'duplicate_filter' ] = SHORTCUTS_DUPLICATE_FILTER_ACTIONS + SHORTCUTS_MEDIA_ACTIONS + SHORTCUTS_MEDIA_VIEWER_ACTIONS
simple_shortcut_name_to_action_lookup[ 'archive_delete_filter' ] = SHORTCUTS_ARCHIVE_DELETE_FILTER_ACTIONS
simple_shortcut_name_to_action_lookup[ 'media_viewer_media_window' ] = SHORTCUTS_MEDIA_VIEWER_VIDEO_AUDIO_PLAYER_ACTIONS
@ -999,6 +1004,14 @@ class ShortcutSet( HydrusSerialisable.SerialisableBaseNamed ):
def DeleteShortcut( self, shortcut ):
if shortcut in self._shortcuts_to_commands:
del self._shortcuts_to_commands[ shortcut ]
def GetCommand( self, shortcut ):
if shortcut in self._shortcuts_to_commands:
@ -1035,19 +1048,23 @@ HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIAL
class ShortcutsHandler( QC.QObject ):
def __init__( self, parent: QW.QWidget, initial_shortcuts_names = None, catch_mouse = False, ignore_activating_mouse_click = False ):
def __init__( self, parent: QW.QWidget, initial_shortcuts_names: typing.Collection[ str ], alternate_filter_target = None, catch_mouse = False, ignore_activating_mouse_click = False ):
QC.QObject.__init__( self, parent )
self._catch_mouse = catch_mouse
if initial_shortcuts_names is None:
filter_target = parent
if alternate_filter_target is not None:
initial_shortcuts_names = []
filter_target = alternate_filter_target
self._filter_target = filter_target
self._parent = parent
self._parent.installEventFilter( self )
self._filter_target.installEventFilter( self )
self._shortcuts_names = list( initial_shortcuts_names )
self._ignore_activating_mouse_click = ignore_activating_mouse_click
@ -1113,7 +1130,7 @@ class ShortcutsHandler( QC.QObject ):
if HG.shortcut_report_mode:
message = 'Shortcut "' + shortcut.ToString() + '" matched to command "' + command.ToString() + '" on ' + repr( self._parent ) + '.'
message = 'Shortcut "{}" matched to command "{}" on {}.'.format( shortcut.ToString(), command.ToString(), repr( self._parent ) )
if command_processed:
@ -1140,7 +1157,7 @@ class ShortcutsHandler( QC.QObject ):
if event.type() == QC.QEvent.KeyPress:
i_should_catch_shortcut_event = IShouldCatchShortcutEvent( self._parent, watched, event = event )
i_should_catch_shortcut_event = IShouldCatchShortcutEvent( self._filter_target, watched, event = event )
shortcut = ConvertKeyEventToShortcut( event )
@ -1148,7 +1165,7 @@ class ShortcutsHandler( QC.QObject ):
if HG.shortcut_report_mode:
message = 'Key shortcut "' + shortcut.ToString() + '" passing through ' + repr( self._parent ) + '.'
message = 'Key shortcut "{}" passing through {}.'.format( shortcut.ToString(), repr( self._parent ) )
if i_should_catch_shortcut_event:
@ -1189,7 +1206,7 @@ class ShortcutsHandler( QC.QObject ):
return False
i_should_catch_shortcut_event = IShouldCatchShortcutEvent( self._parent, watched, event = event )
i_should_catch_shortcut_event = IShouldCatchShortcutEvent( self._filter_target, watched, event = event )
shortcut = ConvertMouseEventToShortcut( event )

View File

@ -93,7 +93,7 @@ class EditTagAutocompleteOptionsPanel( ClientGUIScrolledPanels.EditPanel ):
self._fetch_all_allowed.setToolTip( 'If on, a search for "*" will return all tags. On large tag services, these searches are extremely slow.' )
self._fetch_results_automatically = QW.QCheckBox( self )
self._fetch_results_automatically.setToolTip( 'If on, results will load as you type. If off, you will have to hit Ctrl+Space to load results.' )
self._fetch_results_automatically.setToolTip( 'If on, results will load as you type. If off, you will have to hit a shortcut (default Ctrl+Space) to load results.' )
self._exact_match_character_threshold = ClientGUICommon.NoneableSpinCtrl( self, none_phrase = 'always autocomplete (only appropriate for small tag services)', min = 1, max = 256, unit = 'characters' )
self._exact_match_character_threshold.setToolTip( 'When the search text has <= this many characters, autocomplete will not occur and you will only get results that exactly match the input. Increasing this value makes autocomplete snappier but reduces the number of results.' )

View File

@ -13,6 +13,7 @@ from hydrus.core import HydrusGlobals as HG
from hydrus.core import HydrusTags
from hydrus.core import HydrusText
from hydrus.client import ClientApplicationCommand as CAC
from hydrus.client import ClientConstants as CC
from hydrus.client import ClientData
from hydrus.client import ClientSearch
@ -671,7 +672,7 @@ class AutoCompleteDropdown( QW.QWidget ):
QW.QWidget.__init__( self, parent )
self._intercept_key_events = True
self._can_intercept_unusual_key_events = True
if self.window() == HG.client_controller.gui:
@ -767,6 +768,8 @@ class AutoCompleteDropdown( QW.QWidget ):
self._schedule_results_refresh_job = None
self._my_shortcut_handler = ClientGUIShortcuts.ShortcutsHandler( self, [ 'tags_autocomplete' ], alternate_filter_target = self._text_ctrl )
if self._float_mode:
self._widget_event_filter = QP.WidgetEventFilter( self )
@ -1005,7 +1008,7 @@ class AutoCompleteDropdown( QW.QWidget ):
colour = HG.client_controller.new_options.GetColour( CC.COLOUR_AUTOCOMPLETE_BACKGROUND )
if not self._intercept_key_events:
if not self._can_intercept_unusual_key_events:
colour = ClientGUIFunctions.GetLighterDarkerColour( colour )
@ -1047,26 +1050,12 @@ class AutoCompleteDropdown( QW.QWidget ):
( modifier, key ) = ClientGUIShortcuts.ConvertKeyEventToSimpleTuple( event )
if key in ( QC.Qt.Key_Insert, ):
self._intercept_key_events = not self._intercept_key_events
self._UpdateBackgroundColour()
elif key == QC.Qt.Key_Space and event.modifiers() & QC.Qt.ControlModifier:
self._ScheduleResultsRefresh( 0.0 )
elif self._intercept_key_events:
if self._can_intercept_unusual_key_events:
send_input_to_current_list = False
current_results_list = self._dropdown_notebook.currentWidget()
current_list_is_empty = len( current_results_list ) == 0
input_is_empty = self._text_ctrl.text() == ''
if key in ( ord( 'A' ), ord( 'a' ) ) and modifier == QC.Qt.ControlModifier:
return True # was: event.ignore()
@ -1086,48 +1075,6 @@ class AutoCompleteDropdown( QW.QWidget ):
send_input_to_current_list = True
elif input_is_empty: # maybe we should be sending a 'move' event to a different place
if key in ( QC.Qt.Key_Up, QC.Qt.Key_Down ) and current_list_is_empty:
if key in ( QC.Qt.Key_Up, ):
self.selectUp.emit()
elif key in ( QC.Qt.Key_Down, ):
self.selectDown.emit()
elif key in ( QC.Qt.Key_PageDown, QC.Qt.Key_PageUp ) and current_list_is_empty:
if key in ( QC.Qt.Key_PageUp, ):
self.showPrevious.emit()
elif key in ( QC.Qt.Key_PageDown, ):
self.showNext.emit()
elif key in ( QC.Qt.Key_Right, QC.Qt.Key_Left ):
if key in ( QC.Qt.Key_Left, ):
direction = -1
elif key in ( QC.Qt.Key_Right, ):
direction = 1
self.MoveNotebookPageFocus( direction = direction )
else:
send_input_to_current_list = True
else:
send_input_to_current_list = True
@ -1286,6 +1233,83 @@ class AutoCompleteDropdown( QW.QWidget ):
def ProcessApplicationCommand( self, command: CAC.ApplicationCommand ):
command_processed = True
data = command.GetData()
if command.IsSimpleCommand():
action = data
if action == CAC.SIMPLE_AUTOCOMPLETE_IME_MODE:
self._can_intercept_unusual_key_events = not self._can_intercept_unusual_key_events
self._UpdateBackgroundColour()
elif self._can_intercept_unusual_key_events:
current_results_list = self._dropdown_notebook.currentWidget()
current_list_is_empty = len( current_results_list ) == 0
input_is_empty = self._text_ctrl.text() == ''
everything_is_empty = input_is_empty and current_list_is_empty
if action == CAC.SIMPLE_AUTOCOMPLETE_FORCE_FETCH:
self._ScheduleResultsRefresh( 0.0 )
elif input_is_empty and action in ( CAC.SIMPLE_AUTOCOMPLETE_IF_EMPTY_TAB_LEFT, CAC.SIMPLE_AUTOCOMPLETE_IF_EMPTY_TAB_RIGHT ):
if action == CAC.SIMPLE_AUTOCOMPLETE_IF_EMPTY_TAB_LEFT:
direction = -1
else:
direction = 1
self.MoveNotebookPageFocus( direction = direction )
elif everything_is_empty and action == CAC.SIMPLE_AUTOCOMPLETE_IF_EMPTY_PAGE_LEFT:
self.selectUp.emit()
elif everything_is_empty and action == CAC.SIMPLE_AUTOCOMPLETE_IF_EMPTY_PAGE_RIGHT:
self.selectDown.emit()
elif everything_is_empty and action == CAC.SIMPLE_AUTOCOMPLETE_IF_EMPTY_MEDIA_PREVIOUS:
self.showPrevious.emit()
elif everything_is_empty and action == CAC.SIMPLE_AUTOCOMPLETE_IF_EMPTY_MEDIA_NEXT:
self.showNext.emit()
else:
command_processed = False
else:
command_processed = False
else:
command_processed = False
return command_processed
def SetFetchedResults( self, job_key: ClientThreading.JobKey, parsed_autocomplete_text: ClientSearch.ParsedAutocompleteText, results_cache: ClientSearch.PredicateResultsCache, results: list ):
if self._current_fetch_job_key is not None and self._current_fetch_job_key.GetKey() == job_key.GetKey():
@ -2056,6 +2080,38 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
self._search_pause_play.SetOnOff( False )
def ProcessApplicationCommand( self, command: CAC.ApplicationCommand ):
command_processed = True
data = command.GetData()
if self._can_intercept_unusual_key_events and command.IsSimpleCommand():
action = data
if action == CAC.SIMPLE_SYNCHRONISED_WAIT_SWITCH:
self.PausePlaySearch()
else:
command_processed = False
else:
command_processed = False
if not command_processed:
command_processed = AutoCompleteDropdownTags.ProcessApplicationCommand( self, command )
return command_processed
def SetFetchedResults( self, job_key: ClientThreading.JobKey, parsed_autocomplete_text: ClientSearch.ParsedAutocompleteText, results_cache: ClientSearch.PredicateResultsCache, results: list ):
if self._current_fetch_job_key is not None and self._current_fetch_job_key.GetKey() == job_key.GetKey():

View File

@ -1285,6 +1285,47 @@ class NetworkDomainManager( HydrusSerialisable.SerialisableBase ):
self.SetURLClasses( url_classes )
def DissolveParserLink( self, url_class_name, parser_name ):
with self._lock:
the_url_class = None
for url_class in self._url_classes:
if url_class.GetName() == url_class_name:
the_url_class = url_class
break
the_parser = None
for parser in self._parsers:
if parser.GetName() == parser_name:
the_parser = parser
break
if the_url_class is not None and the_parser is not None:
url_class_key = the_url_class.GetClassKey()
parser_key = the_parser.GetParserKey()
if url_class_key in self._url_class_keys_to_parser_keys and self._url_class_keys_to_parser_keys[ url_class_key ] == parser_key:
del self._url_class_keys_to_parser_keys[ url_class_key ]
def DomainOK( self, url ):
with self._lock:

View File

@ -70,7 +70,7 @@ options = {}
# Misc
NETWORK_VERSION = 19
SOFTWARE_VERSION = 426
SOFTWARE_VERSION = 427
CLIENT_API_VERSION = 15
SERVER_THUMBNAIL_DIMENSIONS = ( 200, 200 )

View File

@ -163,6 +163,8 @@ class HydrusDB( object ):
self._db_dir = db_dir
self._db_name = db_name
self._modules = []
TemporaryIntegerTableNameCache()
self._transaction_started = 0
@ -366,6 +368,8 @@ class HydrusDB( object ):
self._db = None
self._c = None
self._UnloadModules()
def _Commit( self ):
@ -558,6 +562,8 @@ class HydrusDB( object ):
self._c = self._db.cursor()
self._LoadModules()
if HG.no_db_temp_files:
self._c.execute( 'PRAGMA temp_store = 2;' ) # use memory for temp store exclusively
@ -627,6 +633,11 @@ class HydrusDB( object ):
pass
def _LoadModules( self ):
pass
def _ManageDBError( self, job, e ):
raise NotImplementedError()
@ -811,6 +822,11 @@ class HydrusDB( object ):
return result is None
def _UnloadModules( self ):
pass
def _UpdateDB( self, version ):
raise NotImplementedError()

View File

@ -0,0 +1,79 @@
import sqlite3
import typing
class HydrusDBModule( object ):
def __init__( self, name, cursor: sqlite3.Cursor ):
self.name = name
self._c = cursor
def _CreateIndex( self, table_name, columns, unique = False ):
if '.' in table_name:
table_name_simple = table_name.split( '.' )[1]
else:
table_name_simple = table_name
index_name = self._GenerateIndexName( table_name, columns )
if unique:
create_phrase = 'CREATE UNIQUE INDEX IF NOT EXISTS '
else:
create_phrase = 'CREATE INDEX IF NOT EXISTS '
on_phrase = ' ON ' + table_name_simple + ' (' + ', '.join( columns ) + ');'
statement = create_phrase + index_name + on_phrase
self._c.execute( statement )
def _GetIndexGenerationTuples( self ):
raise NotImplementedError()
def _GenerateIndexName( self, table_name, columns ):
return '{}_{}_index'.format( table_name, '_'.join( columns ) )
def CreateIndices( self ):
index_generation_tuples = self._GetIndexGenerationTuples()
for ( table_name, columns, unique ) in index_generation_tuples:
self._CreateIndex( table_name, columns, unique = unique )
def CreateTables( self ):
raise NotImplementedError()
def GetExpectedIndexNames( self ) -> typing.Collection[ str ]:
index_generation_tuples = self._GetIndexGenerationTuples()
expected_index_names = [ self._GenerateIndexName( table_name, columns ) for ( table_name, columns, unique ) in index_generation_tuples ]
return expected_index_names
def GetExpectedTableNames( self ) -> typing.Collection[ str ]:
raise NotImplementedError()

View File

@ -7,6 +7,8 @@ test_controller = None
view_shutdown = False
model_shutdown = False
server_action = 'start'
no_daemons = False
db_journal_mode = 'WAL'
no_db_temp_files = False

View File

@ -103,7 +103,6 @@ except:
OPENCV_OK = False
def ConvertToPNGIfBMP( path ):
with open( path, 'rb' ) as f:

239
hydrus/hydrus_client.py Normal file
View File

@ -0,0 +1,239 @@
#!/usr/bin/env python3
# Hydrus is released under WTFPL
# You just DO WHAT THE FUCK YOU WANT TO.
# https://github.com/sirkris/WTFPL/blob/master/WTFPL.md
import locale
try: locale.setlocale( locale.LC_ALL, '' )
except: pass
try:
import os
import argparse
import sys
from hydrus.core import HydrusBoot
HydrusBoot.AddBaseDirToEnvPath()
# initialise Qt here, important it is done early
from hydrus.client.gui import QtPorting as QP
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusLogger
from hydrus.core import HydrusPaths
from hydrus.core import HydrusGlobals as HG
argparser = argparse.ArgumentParser( description = 'hydrus network client' )
argparser.add_argument( '-d', '--db_dir', help = 'set an external db location' )
argparser.add_argument( '--temp_dir', help = 'override the program\'s temporary directory' )
argparser.add_argument( '--db_journal_mode', default = 'WAL', choices = [ 'WAL', 'TRUNCATE', 'PERSIST', 'MEMORY' ], help = 'change db journal mode (default=WAL)' )
argparser.add_argument( '--db_cache_size', type = int, help = 'override SQLite cache_size per db file, in MB (default=200)' )
argparser.add_argument( '--db_synchronous_override', type = int, choices = range(4), help = 'override SQLite Synchronous PRAGMA (default=2)' )
argparser.add_argument( '--no_db_temp_files', action='store_true', help = 'run db temp operations entirely in memory' )
argparser.add_argument( '--boot_debug', action='store_true', help = 'print additional bootup information to the log' )
argparser.add_argument( '--no_daemons', action='store_true', help = 'run without background daemons' )
argparser.add_argument( '--no_wal', action='store_true', help = 'OBSOLETE: run using TRUNCATE db journaling' )
argparser.add_argument( '--db_memory_journaling', action='store_true', help = 'OBSOLETE: run using MEMORY db journaling (DANGEROUS)' )
result = argparser.parse_args()
if result.db_dir is None:
db_dir = HC.DEFAULT_DB_DIR
if not HydrusPaths.DirectoryIsWritable( db_dir ) or HC.RUNNING_FROM_MACOS_APP:
db_dir = HC.USERPATH_DB_DIR
else:
db_dir = result.db_dir
db_dir = HydrusPaths.ConvertPortablePathToAbsPath( db_dir, HC.BASE_DIR )
try:
HydrusPaths.MakeSureDirectoryExists( db_dir )
except:
raise Exception( 'Could not ensure db path "{}" exists! Check the location is correct and that you have permission to write to it!'.format( db_dir ) )
if not os.path.isdir( db_dir ):
raise Exception( 'The given db path "{}" is not a directory!'.format( db_dir ) )
if not HydrusPaths.DirectoryIsWritable( db_dir ):
raise Exception( 'The given db path "{}" is not a writable-to!'.format( db_dir ) )
HG.no_daemons = result.no_daemons
HG.db_journal_mode = result.db_journal_mode
if result.no_wal:
HG.db_journal_mode = 'TRUNCATE'
if result.db_memory_journaling:
HG.db_journal_mode = 'MEMORY'
if result.db_cache_size is not None:
HG.db_cache_size = result.db_cache_size
else:
HG.db_cache_size = 200
if result.db_synchronous_override is not None:
HG.db_synchronous = int( result.db_synchronous_override )
else:
if HG.db_journal_mode == 'WAL':
HG.db_synchronous = 1
else:
HG.db_synchronous = 2
HG.no_db_temp_files = result.no_db_temp_files
HG.boot_debug = result.boot_debug
try:
from twisted.internet import reactor
except:
HG.twisted_is_broke = True
except Exception as e:
try:
HydrusData.DebugPrint( 'Critical boot error occurred! Details written to crash.log!' )
HydrusData.PrintException( e )
except:
pass
import traceback
error_trace = traceback.format_exc()
print( error_trace )
if 'db_dir' in locals() and os.path.exists( db_dir ):
emergency_dir = db_dir
else:
emergency_dir = os.path.expanduser( '~' )
possible_desktop = os.path.join( emergency_dir, 'Desktop' )
if os.path.exists( possible_desktop ) and os.path.isdir( possible_desktop ):
emergency_dir = possible_desktop
dest_path = os.path.join( emergency_dir, 'hydrus_crash.log' )
with open( dest_path, 'w', encoding = 'utf-8' ) as f:
f.write( error_trace )
print( 'Critical boot error occurred! Details written to hydrus_crash.log in either db dir or user dir!' )
sys.exit( 1 )
def boot():
if result.temp_dir is not None:
HydrusPaths.SetEnvTempDir( result.temp_dir )
controller = None
with HydrusLogger.HydrusLogger( db_dir, 'client' ) as logger:
try:
HydrusData.Print( 'hydrus client started' )
if not HG.twisted_is_broke:
import threading
threading.Thread( target = reactor.run, name = 'twisted', kwargs = { 'installSignalHandlers' : 0 } ).start()
from hydrus.client import ClientController
controller = ClientController.Controller( db_dir )
controller.Run()
except:
HydrusData.Print( 'hydrus client failed' )
import traceback
HydrusData.Print( traceback.format_exc() )
finally:
HG.view_shutdown = True
HG.model_shutdown = True
if controller is not None:
controller.pubimmediate( 'wake_daemons' )
if not HG.twisted_is_broke:
reactor.callFromThread( reactor.stop )
HydrusData.Print( 'hydrus client shut down' )
HG.shutdown_complete = True
if HG.restart:
HydrusData.RestartProcess()

245
hydrus/hydrus_server.py Normal file
View File

@ -0,0 +1,245 @@
#!/usr/bin/env python3
# Hydrus is released under WTFPL
# You just DO WHAT THE FUCK YOU WANT TO.
# https://github.com/sirkris/WTFPL/blob/master/WTFPL.md
action = 'start'
try:
import locale
try: locale.setlocale( locale.LC_ALL, '' )
except: pass
import os
import sys
import threading
from hydrus.core import HydrusBoot
HydrusBoot.AddBaseDirToEnvPath()
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusGlobals as HG
from hydrus.core import HydrusData
from hydrus.core import HydrusLogger
from hydrus.core import HydrusPaths
from hydrus.server import ServerController
from twisted.internet import reactor
#
import argparse
argparser = argparse.ArgumentParser( description = 'hydrus network server' )
argparser.add_argument( 'action', default = 'start', nargs = '?', choices = [ 'start', 'stop', 'restart' ], help = 'either start this server (default), or stop an existing server, or both' )
argparser.add_argument( '-d', '--db_dir', help = 'set an external db location' )
argparser.add_argument( '--temp_dir', help = 'override the program\'s temporary directory' )
argparser.add_argument( '--db_journal_mode', default = 'WAL', choices = [ 'WAL', 'TRUNCATE', 'PERSIST', 'MEMORY' ], help = 'change db journal mode (default=WAL)' )
argparser.add_argument( '--db_cache_size', type = int, help = 'override SQLite cache_size per db file, in MB (default=200)' )
argparser.add_argument( '--db_synchronous_override', type = int, choices = range(4), help = 'override SQLite Synchronous PRAGMA (default=2)' )
argparser.add_argument( '--no_db_temp_files', action='store_true', help = 'run db temp operations entirely in memory' )
argparser.add_argument( '--boot_debug', action='store_true', help = 'print additional bootup information to the log' )
argparser.add_argument( '--no_daemons', action='store_true', help = 'run without background daemons' )
argparser.add_argument( '--no_wal', action='store_true', help = 'OBSOLETE: run using TRUNCATE db journaling' )
argparser.add_argument( '--db_memory_journaling', action='store_true', help = 'OBSOLETE: run using MEMORY db journaling (DANGEROUS)' )
result = argparser.parse_args()
HG.server_action = result.action
if result.db_dir is None:
db_dir = HC.DEFAULT_DB_DIR
if not HydrusPaths.DirectoryIsWritable( db_dir ) or HC.RUNNING_FROM_MACOS_APP:
db_dir = HC.USERPATH_DB_DIR
else:
db_dir = result.db_dir
db_dir = HydrusPaths.ConvertPortablePathToAbsPath( db_dir, HC.BASE_DIR )
try:
HydrusPaths.MakeSureDirectoryExists( db_dir )
except:
raise Exception( 'Could not ensure db path "{}" exists! Check the location is correct and that you have permission to write to it!'.format( db_dir ) )
if not os.path.isdir( db_dir ):
raise Exception( 'The given db path "{}" is not a directory!'.format( db_dir ) )
if not HydrusPaths.DirectoryIsWritable( db_dir ):
raise Exception( 'The given db path "{}" is not a writable-to!'.format( db_dir ) )
HG.no_daemons = result.no_daemons
HG.db_journal_mode = result.db_journal_mode
if result.no_wal:
HG.db_journal_mode = 'TRUNCATE'
if result.db_memory_journaling:
HG.db_journal_mode = 'MEMORY'
if result.db_cache_size is not None:
HG.db_cache_size = result.db_cache_size
else:
HG.db_cache_size = 200
if result.db_synchronous_override is not None:
HG.db_synchronous = int( result.db_synchronous_override )
else:
if HG.db_journal_mode == 'WAL':
HG.db_synchronous = 1
else:
HG.db_synchronous = 2
HG.no_db_temp_files = result.no_db_temp_files
HG.boot_debug = result.boot_debug
if result.temp_dir is not None:
HydrusPaths.SetEnvTempDir( result.temp_dir )
except Exception as e:
import traceback
error_trace = traceback.format_exc()
print( error_trace )
if 'db_dir' in locals() and os.path.exists( db_dir ):
emergency_dir = db_dir
else:
emergency_dir = os.path.expanduser( '~' )
possible_desktop = os.path.join( emergency_dir, 'Desktop' )
if os.path.exists( possible_desktop ) and os.path.isdir( possible_desktop ):
emergency_dir = possible_desktop
dest_path = os.path.join( emergency_dir, 'hydrus_crash.log' )
with open( dest_path, 'w', encoding = 'utf-8' ) as f:
f.write( error_trace )
print( 'Critical boot error occurred! Details written to hydrus_crash.log in either db dir or user dir!' )
sys.exit( 1 )
def boot():
try:
HG.server_action = ServerController.ProcessStartingAction( db_dir, HG.server_action )
except HydrusExceptions.ShutdownException as e:
HydrusData.Print( e )
HG.server_action = 'exit'
if HG.server_action == 'exit':
sys.exit( 0 )
controller = None
with HydrusLogger.HydrusLogger( db_dir, 'server' ) as logger:
try:
if HG.server_action in ( 'stop', 'restart' ):
ServerController.ShutdownSiblingInstance( db_dir )
if HG.server_action in ( 'start', 'restart' ):
HydrusData.Print( 'Initialising controller\u2026' )
threading.Thread( target = reactor.run, name = 'twisted', kwargs = { 'installSignalHandlers' : 0 } ).start()
controller = ServerController.Controller( db_dir )
controller.Run()
except ( HydrusExceptions.DBCredentialsException, HydrusExceptions.ShutdownException ) as e:
error = str( e )
HydrusData.Print( error )
except:
import traceback
error = traceback.format_exc()
HydrusData.Print( 'Hydrus server failed' )
HydrusData.Print( error )
finally:
HG.view_shutdown = True
HG.model_shutdown = True
if controller is not None:
controller.pubimmediate( 'wake_daemons' )
reactor.callFromThread( reactor.stop )
HydrusData.Print( 'hydrus server shut down' )

93
hydrus/hydrus_test.py Normal file
View File

@ -0,0 +1,93 @@
#!/usr/bin/env python3
from hydrus.client.gui import QtPorting as QP
from qtpy import QtWidgets as QW
from qtpy import QtCore as QC
import locale
try: locale.setlocale( locale.LC_ALL, '' )
except: pass
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusGlobals as HG
from hydrus.test import TestController
import sys
import threading
import traceback
from twisted.internet import reactor
def boot():
args = sys.argv[1:]
if len( args ) > 0:
only_run = args[0]
else:
only_run = None
try:
threading.Thread( target = reactor.run, kwargs = { 'installSignalHandlers' : 0 } ).start()
QP.MonkeyPatchMissingMethods()
app = QW.QApplication( sys.argv )
app.call_after_catcher = QP.CallAfterEventCatcher( app )
try:
# we run the tests on the Qt thread atm
# keep a window alive the whole time so the app doesn't finish its mainloop
win = QW.QWidget( None )
win.setWindowTitle( 'Running tests...' )
controller = TestController.Controller( win, only_run )
def do_it():
controller.Run( win )
QP.CallAfter( do_it )
app.exec_()
except:
HydrusData.DebugPrint( traceback.format_exc() )
finally:
HG.view_shutdown = True
controller.pubimmediate( 'wake_daemons' )
HG.model_shutdown = True
controller.pubimmediate( 'wake_daemons' )
controller.TidyUp()
except:
HydrusData.DebugPrint( traceback.format_exc() )
finally:
reactor.callFromThread( reactor.stop )
print( 'This was version ' + str( HC.SOFTWARE_VERSION ) )
input()

View File

@ -58,8 +58,8 @@ class TestClientAPI( unittest.TestCase ):
expected_content_updates = expected_service_keys_to_content_updates[ service_key ]
c_u_tuples = sorted( ( c_u.ToTuple() for c_u in content_updates ) )
e_c_u_tuples = sorted( ( e_c_u.ToTuple() for e_c_u in expected_content_updates ) )
c_u_tuples = sorted( ( ( c_u.ToTuple(), c_u.GetReason() ) for c_u in content_updates ) )
e_c_u_tuples = sorted( ( ( e_c_u.ToTuple(), e_c_u.GetReason() ) for e_c_u in expected_content_updates ) )
self.assertEqual( c_u_tuples, e_c_u_tuples )
@ -891,6 +891,94 @@ class TestClientAPI( unittest.TestCase ):
self._compare_content_updates( service_keys_to_content_updates, expected_service_keys_to_content_updates )
# add tags to local complex
HG.test_controller.ClearWrites( 'content_updates' )
path = '/add_tags/add_tags'
body_dict = { 'hash' : hash_hex, 'service_names_to_actions_to_tags' : { 'my tags' : { str( HC.CONTENT_UPDATE_ADD ) : [ 'test_add', 'test_add2' ], str( HC.CONTENT_UPDATE_DELETE ) : [ 'test_delete', 'test_delete2' ] } } }
body = json.dumps( body_dict )
connection.request( 'POST', path, body = body, headers = headers )
response = connection.getresponse()
data = response.read()
self.assertEqual( response.status, 200 )
expected_service_keys_to_content_updates = collections.defaultdict( list )
expected_service_keys_to_content_updates[ CC.DEFAULT_LOCAL_TAG_SERVICE_KEY ] = [
HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( 'test_add', set( [ hash ] ) ) ),
HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( 'test_add2', set( [ hash ] ) ) ),
HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_DELETE, ( 'test_delete', set( [ hash ] ) ) ),
HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_DELETE, ( 'test_delete2', set( [ hash ] ) ) )
]
[ ( ( service_keys_to_content_updates, ), kwargs ) ] = HG.test_controller.GetWrite( 'content_updates' )
self._compare_content_updates( service_keys_to_content_updates, expected_service_keys_to_content_updates )
# pend tags to repo
HG.test_controller.ClearWrites( 'content_updates' )
path = '/add_tags/add_tags'
body_dict = { 'hash' : hash_hex, 'service_names_to_tags' : { 'example tag repo' : [ 'test', 'test2' ] } }
body = json.dumps( body_dict )
connection.request( 'POST', path, body = body, headers = headers )
response = connection.getresponse()
data = response.read()
self.assertEqual( response.status, 200 )
expected_service_keys_to_content_updates = collections.defaultdict( list )
expected_service_keys_to_content_updates[ HG.test_controller.example_tag_repo_service_key ] = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ( 'test', set( [ hash ] ) ) ), HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ( 'test2', set( [ hash ] ) ) ) ]
[ ( ( service_keys_to_content_updates, ), kwargs ) ] = HG.test_controller.GetWrite( 'content_updates' )
self._compare_content_updates( service_keys_to_content_updates, expected_service_keys_to_content_updates )
# pend tags to repo complex
HG.test_controller.ClearWrites( 'content_updates' )
path = '/add_tags/add_tags'
body_dict = { 'hash' : hash_hex, 'service_names_to_actions_to_tags' : { 'example tag repo' : { str( HC.CONTENT_UPDATE_PEND ) : [ 'test_add', 'test_add2' ], str( HC.CONTENT_UPDATE_PETITION ) : [ [ 'test_delete', 'muh reason' ], 'test_delete2' ] } } }
body = json.dumps( body_dict )
connection.request( 'POST', path, body = body, headers = headers )
response = connection.getresponse()
data = response.read()
self.assertEqual( response.status, 200 )
expected_service_keys_to_content_updates = collections.defaultdict( list )
expected_service_keys_to_content_updates[ HG.test_controller.example_tag_repo_service_key ] = [
HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ( 'test_add', set( [ hash ] ) ) ),
HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ( 'test_add2', set( [ hash ] ) ) ),
HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PETITION, ( 'test_delete', set( [ hash ] ) ), reason = 'muh reason' ),
HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PETITION, ( 'test_delete2', set( [ hash ] ) ), reason = 'Petitioned from API' )
]
[ ( ( service_keys_to_content_updates, ), kwargs ) ] = HG.test_controller.GetWrite( 'content_updates' )
self._compare_content_updates( service_keys_to_content_updates, expected_service_keys_to_content_updates )
# add to multiple files
HG.test_controller.ClearWrites( 'content_updates' )

View File

@ -9,11 +9,11 @@ from hydrus.core import HydrusNetwork
from hydrus.core import HydrusSerialisable
from hydrus.client import ClientConstants as CC
from hydrus.client import ClientDB
from hydrus.client import ClientDefaults
from hydrus.client import ClientExporting
from hydrus.client import ClientSearch
from hydrus.client import ClientServices
from hydrus.client.db import ClientDB
from hydrus.client.gui import ClientGUIManagement
from hydrus.client.gui import ClientGUIPages
from hydrus.client.importing import ClientImportLocal

View File

@ -7,8 +7,8 @@ from hydrus.core import HydrusData
from hydrus.core import HydrusGlobals as HG
from hydrus.client import ClientConstants as CC
from hydrus.client import ClientDB
from hydrus.client import ClientSearch
from hydrus.client.db import ClientDB
from hydrus.client.importing import ClientImportOptions
from hydrus.client.importing import ClientImportFileSeeds

View File

@ -8,9 +8,9 @@ from hydrus.core import HydrusData
from hydrus.core import HydrusGlobals as HG
from hydrus.client import ClientConstants as CC
from hydrus.client import ClientDB
from hydrus.client import ClientSearch
from hydrus.client import ClientServices
from hydrus.client.db import ClientDB
from hydrus.client.importing import ClientImportFileSeeds
from hydrus.client.metadata import ClientTags
@ -824,7 +824,7 @@ class TestClientDBTags( unittest.TestCase ):
} ) )
def test_display_pending_to_current_bug_both_bad( self ):
def test_display_pending_to_current_bug_both_non_ideal( self ):
# rescinding pending (when you set current on pending) two tags that imply the same thing at once can lead to ghost pending when you don't interleave processing
# so lets test that, both for combined and specific domains!
@ -921,7 +921,7 @@ class TestClientDBTags( unittest.TestCase ):
self._test_ac( 'samu*', self._public_service_key, CC.COMBINED_FILE_SERVICE_KEY, { bad_samus_tag_1 : ( 1, None, 0, None ), bad_samus_tag_2 : ( 1, None, 0, None ) }, { good_samus_tag : ( 1, None, 0, None ) } )
def test_display_pending_to_current_bug_bad_and_ideal( self ):
def test_display_pending_to_current_bug_non_ideal_and_ideal( self ):
# like the test above, this will test 'a' and 'b' being commited at the same time, but when 'a->b', rather than 'a->c' and 'b->c'
@ -1017,6 +1017,302 @@ class TestClientDBTags( unittest.TestCase ):
self._test_ac( 'samu*', self._public_service_key, CC.COMBINED_FILE_SERVICE_KEY, { bad_samus_tag_1 : ( 1, None, 0, None ), good_samus_tag : ( 1, None, 0, None ) }, { good_samus_tag : ( 1, None, 0, None ) } )
def test_display_pending_to_current_merge_bug_both_non_ideal( self ):
# rescinding pending (when you set current on pending) a tag when it is already implied by a tag on current can lead to ghost pending when you screw up your existing tag logic!
# so lets test that, both for combined and specific domains!
self._clear_db()
# add samus
bad_samus_tag_1 = 'samus_aran_(character)'
bad_samus_tag_2 = 'samus aran'
good_samus_tag = 'character:samus aran'
service_keys_to_content_updates = {}
content_updates = []
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_TAG_SIBLINGS, HC.CONTENT_UPDATE_ADD, ( bad_samus_tag_1, good_samus_tag ) ) )
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_TAG_SIBLINGS, HC.CONTENT_UPDATE_ADD, ( bad_samus_tag_2, good_samus_tag ) ) )
service_keys_to_content_updates[ self._public_service_key ] = content_updates
self._write( 'content_updates', service_keys_to_content_updates )
self._sync_display()
# import a file
path = os.path.join( HC.STATIC_DIR, 'testing', 'muh_jpg.jpg' )
file_import_job = ClientImportFileSeeds.FileImportJob( path )
file_import_job.GenerateHashAndStatus()
file_import_job.GenerateInfo()
self._write( 'import_file', file_import_job )
muh_jpg_hash = file_import_job.GetHash()
# pend samus to it in one action
service_keys_to_content_updates = {}
content_updates = []
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( bad_samus_tag_1, ( muh_jpg_hash, ) ) ) )
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ( bad_samus_tag_2, ( muh_jpg_hash, ) ) ) )
service_keys_to_content_updates[ self._public_service_key ] = content_updates
self._write( 'content_updates', service_keys_to_content_updates )
# let's check those tags on the file's media result, which uses specific domain to populate tag data
( media_result, ) = self._read( 'media_results', ( muh_jpg_hash, ) )
tags_manager = media_result.GetTagsManager()
self.assertEqual( tags_manager.GetCurrent( self._public_service_key, ClientTags.TAG_DISPLAY_STORAGE ), { bad_samus_tag_1 } )
self.assertEqual( tags_manager.GetPending( self._public_service_key, ClientTags.TAG_DISPLAY_STORAGE ), { bad_samus_tag_2 } )
self.assertEqual( tags_manager.GetCurrent( self._public_service_key, ClientTags.TAG_DISPLAY_ACTUAL ), { good_samus_tag } )
self.assertEqual( tags_manager.GetPending( self._public_service_key, ClientTags.TAG_DISPLAY_ACTUAL ), { good_samus_tag } )
# and a/c results, both specific and combined
self._test_ac( 'samu*', self._public_service_key, CC.LOCAL_FILE_SERVICE_KEY, { bad_samus_tag_1 : ( 1, None, 0, None ), bad_samus_tag_2 : ( 0, None, 1, None ) }, { good_samus_tag : ( 1, None, 1, None ) } )
self._test_ac( 'samu*', self._public_service_key, CC.COMBINED_FILE_SERVICE_KEY, { bad_samus_tag_1 : ( 1, None, 0, None ), bad_samus_tag_2 : ( 0, None, 1, None ) }, { good_samus_tag : ( 1, None, 1, None ) } )
# now we'll currentify the tags in one action
service_keys_to_content_updates = {}
content_updates = []
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( bad_samus_tag_2, ( muh_jpg_hash, ) ) ) )
service_keys_to_content_updates[ self._public_service_key ] = content_updates
self._write( 'content_updates', service_keys_to_content_updates )
# and magically our tags should now be both current, no ghost pending
( media_result, ) = self._read( 'media_results', ( muh_jpg_hash, ) )
tags_manager = media_result.GetTagsManager()
self.assertEqual( tags_manager.GetPending( self._public_service_key, ClientTags.TAG_DISPLAY_STORAGE ), set() )
self.assertEqual( tags_manager.GetCurrent( self._public_service_key, ClientTags.TAG_DISPLAY_STORAGE ), { bad_samus_tag_1, bad_samus_tag_2 } )
self.assertEqual( tags_manager.GetPending( self._public_service_key, ClientTags.TAG_DISPLAY_ACTUAL ), set() )
self.assertEqual( tags_manager.GetCurrent( self._public_service_key, ClientTags.TAG_DISPLAY_ACTUAL ), { good_samus_tag } )
# and a/c results, both specific and combined
self._test_ac( 'samu*', self._public_service_key, CC.LOCAL_FILE_SERVICE_KEY, { bad_samus_tag_1 : ( 1, None, 0, None ), bad_samus_tag_2 : ( 1, None, 0, None ) }, { good_samus_tag : ( 1, None, 0, None ) } )
self._test_ac( 'samu*', self._public_service_key, CC.COMBINED_FILE_SERVICE_KEY, { bad_samus_tag_1 : ( 1, None, 0, None ), bad_samus_tag_2 : ( 1, None, 0, None ) }, { good_samus_tag : ( 1, None, 0, None ) } )
def test_display_pending_to_current_merge_bug_non_ideal_and_ideal( self ):
# like the test above, this will test 'a' being committed onto an existing 'b', but when 'a->b', rather than 'a->c' and 'b->c'
self._clear_db()
# add samus
bad_samus_tag_1 = 'samus_aran_(character)'
bad_samus_tag_2 = 'samus aran'
good_samus_tag = 'character:samus aran'
service_keys_to_content_updates = {}
content_updates = []
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_TAG_SIBLINGS, HC.CONTENT_UPDATE_ADD, ( bad_samus_tag_1, good_samus_tag ) ) )
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_TAG_SIBLINGS, HC.CONTENT_UPDATE_ADD, ( bad_samus_tag_2, good_samus_tag ) ) )
service_keys_to_content_updates[ self._public_service_key ] = content_updates
self._write( 'content_updates', service_keys_to_content_updates )
self._sync_display()
# import a file
path = os.path.join( HC.STATIC_DIR, 'testing', 'muh_jpg.jpg' )
file_import_job = ClientImportFileSeeds.FileImportJob( path )
file_import_job.GenerateHashAndStatus()
file_import_job.GenerateInfo()
self._write( 'import_file', file_import_job )
muh_jpg_hash = file_import_job.GetHash()
# pend samus to it in one action
service_keys_to_content_updates = {}
content_updates = []
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( good_samus_tag, ( muh_jpg_hash, ) ) ) )
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ( bad_samus_tag_1, ( muh_jpg_hash, ) ) ) )
service_keys_to_content_updates[ self._public_service_key ] = content_updates
self._write( 'content_updates', service_keys_to_content_updates )
# let's check those tags on the file's media result, which uses specific domain to populate tag data
( media_result, ) = self._read( 'media_results', ( muh_jpg_hash, ) )
tags_manager = media_result.GetTagsManager()
self.assertEqual( tags_manager.GetCurrent( self._public_service_key, ClientTags.TAG_DISPLAY_STORAGE ), { good_samus_tag } )
self.assertEqual( tags_manager.GetPending( self._public_service_key, ClientTags.TAG_DISPLAY_STORAGE ), { bad_samus_tag_1 } )
self.assertEqual( tags_manager.GetCurrent( self._public_service_key, ClientTags.TAG_DISPLAY_ACTUAL ), { good_samus_tag } )
self.assertEqual( tags_manager.GetPending( self._public_service_key, ClientTags.TAG_DISPLAY_ACTUAL ), { good_samus_tag } )
# and a/c results, both specific and combined
self._test_ac( 'samu*', self._public_service_key, CC.LOCAL_FILE_SERVICE_KEY, { bad_samus_tag_1 : ( 0, None, 1, None ), good_samus_tag : ( 1, None, 0, None ) }, { good_samus_tag : ( 1, None, 1, None ) } )
self._test_ac( 'samu*', self._public_service_key, CC.COMBINED_FILE_SERVICE_KEY, { bad_samus_tag_1 : ( 0, None, 1, None ), good_samus_tag : ( 1, None, 0, None ) }, { good_samus_tag : ( 1, None, 1, None ) } )
# now we'll currentify the tags in one action
service_keys_to_content_updates = {}
content_updates = []
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( bad_samus_tag_1, ( muh_jpg_hash, ) ) ) )
service_keys_to_content_updates[ self._public_service_key ] = content_updates
self._write( 'content_updates', service_keys_to_content_updates )
# and magically our tags should now be both current, no ghost pending
( media_result, ) = self._read( 'media_results', ( muh_jpg_hash, ) )
tags_manager = media_result.GetTagsManager()
self.assertEqual( tags_manager.GetPending( self._public_service_key, ClientTags.TAG_DISPLAY_STORAGE ), set() )
self.assertEqual( tags_manager.GetCurrent( self._public_service_key, ClientTags.TAG_DISPLAY_STORAGE ), { bad_samus_tag_1, good_samus_tag } )
self.assertEqual( tags_manager.GetPending( self._public_service_key, ClientTags.TAG_DISPLAY_ACTUAL ), set() )
self.assertEqual( tags_manager.GetCurrent( self._public_service_key, ClientTags.TAG_DISPLAY_ACTUAL ), { good_samus_tag } )
# and a/c results, both specific and combined
self._test_ac( 'samu*', self._public_service_key, CC.LOCAL_FILE_SERVICE_KEY, { bad_samus_tag_1 : ( 1, None, 0, None ), good_samus_tag : ( 1, None, 0, None ) }, { good_samus_tag : ( 1, None, 0, None ) } )
self._test_ac( 'samu*', self._public_service_key, CC.COMBINED_FILE_SERVICE_KEY, { bad_samus_tag_1 : ( 1, None, 0, None ), good_samus_tag : ( 1, None, 0, None ) }, { good_samus_tag : ( 1, None, 0, None ) } )
def test_display_pending_regen( self ):
self._clear_db()
# add samus
lara_tag = 'character:lara croft'
bad_samus_tag_1 = 'samus_aran_(character)'
bad_samus_tag_2 = 'samus aran'
good_samus_tag = 'character:samus aran'
service_keys_to_content_updates = {}
content_updates = []
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_TAG_SIBLINGS, HC.CONTENT_UPDATE_ADD, ( bad_samus_tag_1, good_samus_tag ) ) )
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_TAG_SIBLINGS, HC.CONTENT_UPDATE_ADD, ( bad_samus_tag_2, good_samus_tag ) ) )
service_keys_to_content_updates[ self._public_service_key ] = content_updates
self._write( 'content_updates', service_keys_to_content_updates )
self._sync_display()
# import a file
path = os.path.join( HC.STATIC_DIR, 'testing', 'muh_jpg.jpg' )
file_import_job = ClientImportFileSeeds.FileImportJob( path )
file_import_job.GenerateHashAndStatus()
file_import_job.GenerateInfo()
self._write( 'import_file', file_import_job )
muh_jpg_hash = file_import_job.GetHash()
# pend samus to it in one action
service_keys_to_content_updates = {}
content_updates = []
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ( lara_tag, ( muh_jpg_hash, ) ) ) )
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ( bad_samus_tag_1, ( muh_jpg_hash, ) ) ) )
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ( bad_samus_tag_2, ( muh_jpg_hash, ) ) ) )
service_keys_to_content_updates[ self._public_service_key ] = content_updates
self._write( 'content_updates', service_keys_to_content_updates )
# let's check those tags on the file's media result, which uses specific domain to populate tag data
( media_result, ) = self._read( 'media_results', ( muh_jpg_hash, ) )
tags_manager = media_result.GetTagsManager()
self.assertEqual( tags_manager.GetCurrent( self._public_service_key, ClientTags.TAG_DISPLAY_STORAGE ), set() )
self.assertEqual( tags_manager.GetPending( self._public_service_key, ClientTags.TAG_DISPLAY_STORAGE ), { lara_tag, bad_samus_tag_1, bad_samus_tag_2 } )
self.assertEqual( tags_manager.GetCurrent( self._public_service_key, ClientTags.TAG_DISPLAY_ACTUAL ), set() )
self.assertEqual( tags_manager.GetPending( self._public_service_key, ClientTags.TAG_DISPLAY_ACTUAL ), { lara_tag, good_samus_tag } )
# and a/c results, both specific and combined
self._test_ac( 'samu*', self._public_service_key, CC.LOCAL_FILE_SERVICE_KEY, { bad_samus_tag_1 : ( 0, None, 1, None ), bad_samus_tag_2 : ( 0, None, 1, None ) }, { good_samus_tag : ( 0, None, 1, None ) } )
self._test_ac( 'samu*', self._public_service_key, CC.COMBINED_FILE_SERVICE_KEY, { bad_samus_tag_1 : ( 0, None, 1, None ), bad_samus_tag_2 : ( 0, None, 1, None ) }, { good_samus_tag : ( 0, None, 1, None ) } )
self._test_ac( 'lara*', self._public_service_key, CC.LOCAL_FILE_SERVICE_KEY, { lara_tag : ( 0, None, 1, None ) }, { lara_tag : ( 0, None, 1, None ) } )
self._test_ac( 'lara*', self._public_service_key, CC.COMBINED_FILE_SERVICE_KEY, { lara_tag : ( 0, None, 1, None ) }, { lara_tag : ( 0, None, 1, None ) } )
# now we'll currentify the tags in one action
self._write( 'regenerate_tag_display_pending_mappings_cache' )
# let's check again
( media_result, ) = self._read( 'media_results', ( muh_jpg_hash, ) )
tags_manager = media_result.GetTagsManager()
self.assertEqual( tags_manager.GetCurrent( self._public_service_key, ClientTags.TAG_DISPLAY_STORAGE ), set() )
self.assertEqual( tags_manager.GetPending( self._public_service_key, ClientTags.TAG_DISPLAY_STORAGE ), { lara_tag, bad_samus_tag_1, bad_samus_tag_2 } )
self.assertEqual( tags_manager.GetCurrent( self._public_service_key, ClientTags.TAG_DISPLAY_ACTUAL ), set() )
self.assertEqual( tags_manager.GetPending( self._public_service_key, ClientTags.TAG_DISPLAY_ACTUAL ), { lara_tag, good_samus_tag } )
# and a/c results, both specific and combined
self._test_ac( 'samu*', self._public_service_key, CC.LOCAL_FILE_SERVICE_KEY, { bad_samus_tag_1 : ( 0, None, 1, None ), bad_samus_tag_2 : ( 0, None, 1, None ) }, { good_samus_tag : ( 0, None, 1, None ) } )
self._test_ac( 'samu*', self._public_service_key, CC.COMBINED_FILE_SERVICE_KEY, { bad_samus_tag_1 : ( 0, None, 1, None ), bad_samus_tag_2 : ( 0, None, 1, None ) }, { good_samus_tag : ( 0, None, 1, None ) } )
self._test_ac( 'lara*', self._public_service_key, CC.LOCAL_FILE_SERVICE_KEY, { lara_tag : ( 0, None, 1, None ) }, { lara_tag : ( 0, None, 1, None ) } )
self._test_ac( 'lara*', self._public_service_key, CC.COMBINED_FILE_SERVICE_KEY, { lara_tag : ( 0, None, 1, None ) }, { lara_tag : ( 0, None, 1, None ) } )
def test_parents_pairs_lookup( self ):
self._clear_db()

View File

@ -11,10 +11,10 @@ from hydrus.core import HydrusGlobals as HG
from hydrus.test import TestController
from hydrus.client import ClientConstants as CC
from hydrus.client import ClientDB
from hydrus.client import ClientManagers
from hydrus.client import ClientMigration
from hydrus.client import ClientServices
from hydrus.client.db import ClientDB
from hydrus.client.importing import ClientImportFileSeeds
from hydrus.client.importing import ClientImportOptions
from hydrus.client.media import ClientMediaResultCache

View File

@ -793,6 +793,8 @@ class Controller( object ):
runner = unittest.TextTestRunner( verbosity = 2 )
runner.failfast = True
def do_it():
try:

235
server.py
View File

@ -4,238 +4,9 @@
# You just DO WHAT THE FUCK YOU WANT TO.
# https://github.com/sirkris/WTFPL/blob/master/WTFPL.md
try:
import locale
try: locale.setlocale( locale.LC_ALL, '' )
except: pass
import os
import sys
import time
import traceback
import threading
from hydrus.core import HydrusBoot
HydrusBoot.AddBaseDirToEnvPath()
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusPaths
from hydrus.server import ServerController
from twisted.internet import reactor
from hydrus.core import HydrusGlobals as HG
from hydrus.core import HydrusLogger
#
import argparse
argparser = argparse.ArgumentParser( description = 'hydrus network server' )
argparser.add_argument( 'action', default = 'start', nargs = '?', choices = [ 'start', 'stop', 'restart' ], help = 'either start this server (default), or stop an existing server, or both' )
argparser.add_argument( '-d', '--db_dir', help = 'set an external db location' )
argparser.add_argument( '--temp_dir', help = 'override the program\'s temporary directory' )
argparser.add_argument( '--db_journal_mode', default = 'WAL', choices = [ 'WAL', 'TRUNCATE', 'PERSIST', 'MEMORY' ], help = 'change db journal mode (default=WAL)' )
argparser.add_argument( '--db_cache_size', type = int, help = 'override SQLite cache_size per db file, in MB (default=200)' )
argparser.add_argument( '--db_synchronous_override', type = int, choices = range(4), help = 'override SQLite Synchronous PRAGMA (default=2)' )
argparser.add_argument( '--no_db_temp_files', action='store_true', help = 'run db temp operations entirely in memory' )
argparser.add_argument( '--boot_debug', action='store_true', help = 'print additional bootup information to the log' )
argparser.add_argument( '--no_daemons', action='store_true', help = 'run without background daemons' )
argparser.add_argument( '--no_wal', action='store_true', help = 'OBSOLETE: run using TRUNCATE db journaling' )
argparser.add_argument( '--db_memory_journaling', action='store_true', help = 'OBSOLETE: run using MEMORY db journaling (DANGEROUS)' )
result = argparser.parse_args()
action = result.action
if result.db_dir is None:
db_dir = HC.DEFAULT_DB_DIR
if not HydrusPaths.DirectoryIsWritable( db_dir ) or HC.RUNNING_FROM_MACOS_APP:
db_dir = HC.USERPATH_DB_DIR
else:
db_dir = result.db_dir
db_dir = HydrusPaths.ConvertPortablePathToAbsPath( db_dir, HC.BASE_DIR )
try:
HydrusPaths.MakeSureDirectoryExists( db_dir )
except:
raise Exception( 'Could not ensure db path "{}" exists! Check the location is correct and that you have permission to write to it!'.format( db_dir ) )
if not os.path.isdir( db_dir ):
raise Exception( 'The given db path "{}" is not a directory!'.format( db_dir ) )
if not HydrusPaths.DirectoryIsWritable( db_dir ):
raise Exception( 'The given db path "{}" is not a writable-to!'.format( db_dir ) )
HG.no_daemons = result.no_daemons
HG.db_journal_mode = result.db_journal_mode
if result.no_wal:
HG.db_journal_mode = 'TRUNCATE'
if result.db_memory_journaling:
HG.db_journal_mode = 'MEMORY'
if result.db_cache_size is not None:
HG.db_cache_size = result.db_cache_size
else:
HG.db_cache_size = 200
if result.db_synchronous_override is not None:
HG.db_synchronous = int( result.db_synchronous_override )
else:
if HG.db_journal_mode == 'WAL':
HG.db_synchronous = 1
else:
HG.db_synchronous = 2
HG.no_db_temp_files = result.no_db_temp_files
HG.boot_debug = result.boot_debug
if result.temp_dir is not None:
HydrusPaths.SetEnvTempDir( result.temp_dir )
#
try:
action = ServerController.ProcessStartingAction( db_dir, action )
except HydrusExceptions.ShutdownException as e:
HydrusData.Print( e )
action = 'exit'
if action == 'exit':
sys.exit( 0 )
except Exception as e:
error_trace = traceback.format_exc()
print( error_trace )
if 'db_dir' in locals() and os.path.exists( db_dir ):
emergency_dir = db_dir
else:
emergency_dir = os.path.expanduser( '~' )
possible_desktop = os.path.join( emergency_dir, 'Desktop' )
if os.path.exists( possible_desktop ) and os.path.isdir( possible_desktop ):
emergency_dir = possible_desktop
dest_path = os.path.join( emergency_dir, 'hydrus_crash.log' )
with open( dest_path, 'w', encoding = 'utf-8' ) as f:
f.write( error_trace )
print( 'Critical boot error occurred! Details written to hydrus_crash.log in either db dir or user dir!' )
import sys
sys.exit( 1 )
controller = None
from hydrus import hydrus_server
with HydrusLogger.HydrusLogger( db_dir, 'server' ) as logger:
if __name__ == '__main__':
try:
if action in ( 'stop', 'restart' ):
ServerController.ShutdownSiblingInstance( db_dir )
if action in ( 'start', 'restart' ):
HydrusData.Print( 'Initialising controller\u2026' )
threading.Thread( target = reactor.run, name = 'twisted', kwargs = { 'installSignalHandlers' : 0 } ).start()
controller = ServerController.Controller( db_dir )
controller.Run()
except ( HydrusExceptions.DBCredentialsException, HydrusExceptions.ShutdownException ) as e:
error = str( e )
HydrusData.Print( error )
except:
error = traceback.format_exc()
HydrusData.Print( 'Hydrus server failed' )
HydrusData.Print( traceback.format_exc() )
finally:
HG.view_shutdown = True
HG.model_shutdown = True
if controller is not None:
controller.pubimmediate( 'wake_daemons' )
reactor.callFromThread( reactor.stop )
HydrusData.Print( 'hydrus server shut down' )
hydrus_server.boot()

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.0 KiB

87
test.py
View File

@ -1,89 +1,12 @@
#!/usr/bin/env python3
from hydrus.client.gui import QtPorting as QP
from qtpy import QtWidgets as QW
from qtpy import QtCore as QC
import locale
# Hydrus is released under WTFPL
# You just DO WHAT THE FUCK YOU WANT TO.
# https://github.com/sirkris/WTFPL/blob/master/WTFPL.md
try: locale.setlocale( locale.LC_ALL, '' )
except: pass
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusGlobals as HG
from hydrus.test import TestController
import sys
import threading
import traceback
from twisted.internet import reactor
from hydrus import hydrus_test
if __name__ == '__main__':
args = sys.argv[1:]
hydrus_test.boot()
if len( args ) > 0:
only_run = args[0]
else:
only_run = None
try:
threading.Thread( target = reactor.run, kwargs = { 'installSignalHandlers' : 0 } ).start()
QP.MonkeyPatchMissingMethods()
app = QW.QApplication( sys.argv )
app.call_after_catcher = QP.CallAfterEventCatcher( app )
try:
# we run the tests on the Qt thread atm
# keep a window alive the whole time so the app doesn't finish its mainloop
win = QW.QWidget( None )
win.setWindowTitle( 'Running tests...' )
controller = TestController.Controller( win, only_run )
def do_it():
controller.Run( win )
QP.CallAfter( do_it )
app.exec_()
except:
HydrusData.DebugPrint( traceback.format_exc() )
finally:
HG.view_shutdown = True
controller.pubimmediate( 'wake_daemons' )
HG.model_shutdown = True
controller.pubimmediate( 'wake_daemons' )
controller.TidyUp()
except:
HydrusData.DebugPrint( traceback.format_exc() )
finally:
reactor.callFromThread( reactor.stop )
print( 'This was version ' + str( HC.SOFTWARE_VERSION ) )
input()