RepoMirrors
/
hydrus
mirror of https://github.com/hydrusnetwork/hydrus.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Version 344

This commit is contained in:
Hydrus Network Developer 2019-03-20 16:22:10 -05:00
parent 67e39d8f88
commit f7763b38ee
36 changed files with 1113 additions and 806 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
client.py
View File

@ -48,6 +48,7 @@ try:
argparser.add_argument( '--no_daemons', action='store_true', help = 'run without background daemons' )
argparser.add_argument( '--no_wal', action='store_true', help = 'run without WAL db journalling' )
argparser.add_argument( '--no_db_temp_files', action='store_true', help = 'run the db entirely in memory' )
argparser.add_argument( '--temp_dir', help = 'override the program\'s temporary directory' )
result = argparser.parse_args()
@ -76,10 +77,28 @@ try:
raise Exception( 'Could not ensure db path ' + db_dir + ' exists! Check the location is correct and that you have permission to write to it!' )
no_daemons = result.no_daemons
no_wal = result.no_wal
HG.no_daemons = result.no_daemons
HG.no_wal = result.no_wal
HG.no_db_temp_files = result.no_db_temp_files
if result.temp_dir is not None:
if not os.path.exists( result.temp_dir ):
raise Exception( 'The given temp directory, "{}", does not exist!'.format( result.temp_dir ) )
if HC.PLATFORM_WINDOWS:
os.environ[ 'TEMP' ] = result.temp_dir
os.environ[ 'TMP' ] = result.temp_dir
else:
os.environ[ 'TMPDIR' ] = result.temp_dir
#
with HydrusLogger.HydrusLogger( db_dir, 'client' ) as logger:
@ -93,7 +112,7 @@ try:
threading.Thread( target = reactor.run, name = 'twisted', kwargs = { 'installSignalHandlers' : 0 } ).start()
controller = ClientController.Controller( db_dir, no_daemons, no_wal )
controller = ClientController.Controller( db_dir )
controller.Run()

25
client.pyw
View File

@ -48,6 +48,7 @@ try:
argparser.add_argument( '--no_daemons', action='store_true', help = 'run without background daemons' )
argparser.add_argument( '--no_wal', action='store_true', help = 'run without WAL db journalling' )
argparser.add_argument( '--no_db_temp_files', action='store_true', help = 'run the db entirely in memory' )
argparser.add_argument( '--temp_dir', help = 'override the program\'s temporary directory' )
result = argparser.parse_args()
@ -76,10 +77,28 @@ try:
raise Exception( 'Could not ensure db path ' + db_dir + ' exists! Check the location is correct and that you have permission to write to it!' )
no_daemons = result.no_daemons
no_wal = result.no_wal
HG.no_daemons = result.no_daemons
HG.no_wal = result.no_wal
HG.no_db_temp_files = result.no_db_temp_files
if result.temp_dir is not None:
if not os.path.exists( result.temp_dir ):
raise Exception( 'The given temp directory, "{}", does not exist!'.format( result.temp_dir ) )
if HC.PLATFORM_WINDOWS:
os.environ[ 'TEMP' ] = result.temp_dir
os.environ[ 'TMP' ] = result.temp_dir
else:
os.environ[ 'TMPDIR' ] = result.temp_dir
#
with HydrusLogger.HydrusLogger( db_dir, 'client' ) as logger:
@ -93,7 +112,7 @@ try:
threading.Thread( target = reactor.run, name = 'twisted', kwargs = { 'installSignalHandlers' : 0 } ).start()
controller = ClientController.Controller( db_dir, no_daemons, no_wal )
controller = ClientController.Controller( db_dir )
controller.Run()

3
db/to permanently disable wal.txt
View File

@ -1,3 +0,0 @@
If you do not want to use WAL journalling, you can either use the switch '--no_wal' on either the client or server, or you can just put a file in this directory called 'no-wal'. SQLite will try to use TRUNCATE instead.
This can happen automatically if WAL fails on db creation.

BIN
help/6c0ae65894c7a5ffd686f54cc052326b8ea188a691a1895b2f88b7c60a07f13f.jpg
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 33 KiB

11
help/advanced.html
View File

@ -67,17 +67,6 @@
<p>So, some gifs will have a coloured first frame but grey frames thereafter; or they will have odd washy noise all over; or they will just be black. The file isn't broken, the client is just looking at it wrong.</p>
<h3>setting a password</h3>
<p>the client offers a very simple password system, enough to keep out noobs. You can set it at <i>database->set a password</i>. It will thereafter ask for the password every time you start the program, and will not open without it. However none of the database is encrypted, and someone with enough enthusiasm or a tool and access to your computer can still very easily see what files you have. The password is mainly to stop idle snoops checking your images if you are away from your machine.</p>
<!--
<h3>the client's server</h3>
<p>The client can run a very simple http server. It does not run by default, but you can start it by giving it a port at <i>file->options->local server</i>. Once you ok that dialog, the client will try to launch it. You may get a firewall warning.</p>
<p>The server responds to /file and /thumbnail requests just like a file repository, but without needing an access key, and only if the request comes from localhost (127.0.0.1).</p>
<p>For instance, the following image (6c0ae65894c7a5ffd686f54cc052326b8ea188a691a1895b2f88b7c60a07f13f.jpg, in the help dir) is served here from disk:</p>
<p><img src="6c0ae65894c7a5ffd686f54cc052326b8ea188a691a1895b2f88b7c60a07f13f.jpg" /></p>
<p>And here it will attempt to load from the client at port 45865:</p>
<p><img src="http://127.0.0.1:45865/file?hash=6c0ae65894c7a5ffd686f54cc052326b8ea188a691a1895b2f88b7c60a07f13f" /></p>
<p>For more information, check the image's two urls. It will of course only display in the second case if you import the file to your client and have the server running on 45865 when you load this page. You can copy the second image's url and replace the hash with that of any other file in your collection and it should work.</p>
<p>I want to do much more with this in future.</p>
-->
</div>
</body>
</html>

26
help/changelog.html
View File

@ -8,6 +8,32 @@
<div class="content">
<h3>changelog</h3>
<ul>
<li><h3>version 344</h3></li>
<ul>
<li>final v1.0 client api polish:</li>
<li>added optional 'show_destination_page' arg to '/add_urls/add_url', defaulting to False, to control whether an URL-add will select (i.e. jump to) the destination page in the ui. this changes the default behaviour for this command</li>
<li>simplified the routine that finds or creates a watcher or url import page and fixed a bug in the api that was not creating new pages when destination_page_name was specified</li>
<li>some misc cleanup</li>
<li>fixed fetching file_metadata by hashes</li>
<li>fixed the client api help regarding file_metadata response example tags</li>
<li>client api version is now 5</li>
<li>.</li>
<li>the rest:</li>
<li>psd support added! because of this format's potential multi-layer complexity, it will not render natively, but width and height are parsed. it is treated as 'application/x-photoshop'. PSB is also recognised and treated as psd</li>
<li>added a 'open_known_url' shortcut to the 'media' shortcut set that lets you quickly open URLs for files. if there is one recognised known url, it will be launched, and if there are multiple, a list with all known urls will appear to select which one you want</li>
<li>animation scanbars now show an x/y current timestamp! it includes millisecond timings and even works for variable framerate gifs. whether to show a second/shadow caret for timestamp position on variable frame rate is a new discussion to have</li>
<li>fixed an issue where animations would sometimes not resume animation for several seconds after a big scanbar drag</li>
<li>when the thumbnail manager cannot produce a thumbnail due to a storage error (like a missing file), it now only puts up a single, more informative error popup on the first problem. subsequent errors are printed silently to the log. (these errors tend to come in en masse, so this cuts down on spam and error-related ui lag that was making loading a bad session difficult)</li>
<li>improved error reporting when an upload pending command would fail due to service non-functionality--it should now give a popup with error info imediately, rather than obscured through the login system</li>
<li>added temp_dir parameter to the client and server that will override which temporary directory the program will use</li>
<li>cleaned up how no_daemons and no_wal mode are handled internally</li>
<li>no_wal mode now has to be called from the command parameter, the no_wal file hack in the db directory no longer works</li>
<li>missing ffmpeg errors now prompt the user to check if it is installed</li>
<li>searching for numerical ratings should now work for files that were rated when the service had a different number of stars (ratings now searches in 'bands' rather than exact values)</li>
<li>reduced the min height of the new import files frame's list</li>
<li>doubled the decompression bomb test to permit files up to ~179 megapixel, we'll see how it goes</li>
<li>misc cleanup</li>
</ul>
<li><h3>version 343</h3></li>
<ul>
<li>client api:</li>

8
help/client_api.html
View File

@ -423,10 +423,12 @@
<ul>
<li>url : (the url you want to add)</li>
<li>destination_page_name : (optional page name to receive the url)</li>
<li>show_destination_page : (optional, defaulting to false, controls whether the UI will change pages on add)</li>
<li>service_names_to_tags : (optional tags to give to any files imported from this url)</li>
</ul>
</li>
<p>If you specify a destination_page_name and an appropriate importer page already exists with that name, that page will be used. Otherwise, a new page with that name will be recreated (and used by subsequent calls with that name). Make sure it that page name is unique (e.g. '/b/ threads', not 'watcher') in your client, or it may not be found.</p>
<p>show_destination_page defaults to False to reduce flicker when adding many URLs to different pages quickly. If you turn it on, the client will behave like a URL drag and drop and select the final page the URL ends up on.</p>
<p>The service_names_to_tags uses the same system as for /add_tags/add_tags. You will need 'add tags' permission, or this will 403.</p>
<li>
<p>Example request body:</p>
@ -605,6 +607,10 @@
"num_words" : null
"service_names_to_statuses_to_tags" : {
"local tags" : {
"0" : [ "favourites" ]
"2" : [ "process this later" ]
},
"my tag repository" : {
"0" : [ "blonde hair", "blue eyes", "looking at viewer" ]
"1" : [ "bodysuit" ]
}
@ -633,7 +639,7 @@
</ul>
</li>
<p>Size is in bytes. Duration is in milliseconds, and may be an int or a float.</p>
<p>The tags structure is similar to the /add_tags/add_tags scheme, excepting that the status numbers are:</p>
<p>The service_names_to_statuses_to_tags structure is similar to the /add_tags/add_tags scheme, excepting that the status numbers are:</p>
<ul>
<li>0 - current</li>
<li>1 - pending</li>

892
include/ClientCaches.py
View File

@ -2263,438 +2263,6 @@ class RenderedImageCache( object ):
return self._data_cache.HasData( key )
class ThumbnailCache( object ):
def __init__( self, controller ):
self._controller = controller
cache_size = self._controller.options[ 'thumbnail_cache_size' ]
cache_timeout = self._controller.new_options.GetInteger( 'thumbnail_cache_timeout' )
self._data_cache = DataCache( self._controller, cache_size, timeout = cache_timeout )
self._lock = threading.Lock()
self._waterfall_queue_quick = set()
self._waterfall_queue_random = []
self._waterfall_event = threading.Event()
self._special_thumbs = {}
self.Clear()
self._controller.CallToThreadLongRunning( self.DAEMONWaterfall )
self._controller.sub( self, 'Clear', 'thumbnail_resize' )
self._controller.sub( self, 'ClearThumbnails', 'clear_thumbnails' )
def _GetResizedHydrusBitmap( self, display_media ):
if not HG.thumbnail_experiment_mode:
return self._GetResizedHydrusBitmapFromHardDrive( display_media )
else:
return self._GetResizedHydrusBitmapFromFullSize( display_media )
def _GetResizedHydrusBitmapFromFullSize( self, display_media ):
thumbnail_dimensions = self._controller.options[ 'thumbnail_dimensions' ]
hash = display_media.GetHash()
mime = display_media.GetMime()
locations_manager = display_media.GetLocationsManager()
try:
path = self._controller.client_files_manager.GetFullSizeThumbnailPath( hash, mime )
except HydrusExceptions.FileMissingException as e:
if locations_manager.IsLocal():
HydrusData.ShowException( e )
return self._special_thumbs[ 'hydrus' ]
try:
numpy_image = ClientImageHandling.GenerateNumpyImage( path, mime )
except Exception as e:
try:
# file is malformed, let's force a regen
self._controller.client_files_manager.RegenerateResizedThumbnail( hash, mime )
try:
numpy_image = ClientImageHandling.GenerateNumpyImage( path, mime )
except Exception as e:
HydrusData.ShowException( e )
raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.hex() + ' was broken. It was regenerated, but the new file would not render for the above reason. Please inform the hydrus developer what has happened.' )
except Exception as e:
HydrusData.ShowException( e )
return self._special_thumbs[ 'hydrus' ]
( fullsize_x, fullsize_y ) = ClientImageHandling.GetNumPyImageResolution( numpy_image )
( resized_x, resized_y ) = HydrusImageHandling.GetThumbnailResolution( ( fullsize_x, fullsize_y ), thumbnail_dimensions )
already_correct = fullsize_x == resized_x and fullsize_y == resized_y
if not already_correct:
numpy_image = ClientImageHandling.EfficientlyThumbnailNumpyImage( numpy_image, ( resized_x, resized_y ) )
hydrus_bitmap = ClientRendering.GenerateHydrusBitmapFromNumPyImage( numpy_image )
return hydrus_bitmap
def _GetResizedHydrusBitmapFromHardDrive( self, display_media ):
thumbnail_dimensions = self._controller.options[ 'thumbnail_dimensions' ]
if tuple( thumbnail_dimensions ) == HC.UNSCALED_THUMBNAIL_DIMENSIONS:
full_size = True
else:
full_size = False
hash = display_media.GetHash()
mime = display_media.GetMime()
locations_manager = display_media.GetLocationsManager()
try:
if full_size:
path = self._controller.client_files_manager.GetFullSizeThumbnailPath( hash, mime )
else:
path = self._controller.client_files_manager.GetResizedThumbnailPath( hash, mime )
except HydrusExceptions.FileMissingException as e:
if locations_manager.IsLocal():
HydrusData.ShowException( e )
return self._special_thumbs[ 'hydrus' ]
try:
hydrus_bitmap = ClientRendering.GenerateHydrusBitmap( path, mime )
except Exception as e:
try:
# file is malformed, let's force a regen
self._controller.client_files_manager.RegenerateFullSizeThumbnail( hash, mime )
try:
hydrus_bitmap = ClientRendering.GenerateHydrusBitmap( path, mime )
except Exception as e:
HydrusData.ShowException( e )
raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.hex() + ' was broken. It was regenerated, but the new file would not render for the above reason. Please inform the hydrus developer what has happened.' )
except Exception as e:
HydrusData.ShowException( e )
return self._special_thumbs[ 'hydrus' ]
( media_x, media_y ) = display_media.GetResolution()
( actual_x, actual_y ) = hydrus_bitmap.GetSize()
( desired_x, desired_y ) = self._controller.options[ 'thumbnail_dimensions' ]
too_large = actual_x > desired_x or actual_y > desired_y
small_original_image = actual_x == media_x and actual_y == media_y
too_small = actual_x < desired_x and actual_y < desired_y
if too_large or ( too_small and not small_original_image ):
self._controller.client_files_manager.RegenerateResizedThumbnail( hash, mime )
hydrus_bitmap = ClientRendering.GenerateHydrusBitmap( path, mime )
return hydrus_bitmap
def _RecalcWaterfallQueueRandom( self ):
# here we sort by the hash since this is both breddy random and more likely to access faster on a well defragged hard drive!
def sort_by_hash_key( item ):
( page_key, media ) = item
return media.GetDisplayMedia().GetHash()
self._waterfall_queue_random = list( self._waterfall_queue_quick )
self._waterfall_queue_random.sort( key = sort_by_hash_key )
def CancelWaterfall( self, page_key, medias ):
with self._lock:
self._waterfall_queue_quick.difference_update( ( ( page_key, media ) for media in medias ) )
self._RecalcWaterfallQueueRandom()
def Clear( self ):
with self._lock:
self._data_cache.Clear()
self._special_thumbs = {}
names = [ 'hydrus', 'pdf', 'audio', 'video', 'zip' ]
( os_file_handle, temp_path ) = ClientPaths.GetTempPath()
try:
for name in names:
path = os.path.join( HC.STATIC_DIR, name + '.png' )
thumbnail_dimensions = self._controller.options[ 'thumbnail_dimensions' ]
thumbnail_bytes = HydrusFileHandling.GenerateThumbnailFileBytesFromStaticImagePath( path, thumbnail_dimensions, HC.IMAGE_PNG )
with open( temp_path, 'wb' ) as f:
f.write( thumbnail_bytes )
hydrus_bitmap = ClientRendering.GenerateHydrusBitmap( temp_path, HC.IMAGE_PNG )
self._special_thumbs[ name ] = hydrus_bitmap
finally:
HydrusPaths.CleanUpTempPath( os_file_handle, temp_path )
def ClearThumbnails( self, hashes ):
with self._lock:
for hash in hashes:
self._data_cache.DeleteData( hash )
def DoingWork( self ):
with self._lock:
return len( self._waterfall_queue_random ) > 0
def GetThumbnail( self, media ):
try:
display_media = media.GetDisplayMedia()
except:
# sometimes media can get switched around during a collect event, and if this happens during waterfall, we have a problem here
# just return for now, we'll see how it goes
return self._special_thumbs[ 'hydrus' ]
locations_manager = display_media.GetLocationsManager()
if locations_manager.ShouldIdeallyHaveThumbnail():
mime = display_media.GetMime()
if mime in HC.MIMES_WITH_THUMBNAILS:
hash = display_media.GetHash()
result = self._data_cache.GetIfHasData( hash )
if result is None:
try:
hydrus_bitmap = self._GetResizedHydrusBitmap( display_media )
except:
hydrus_bitmap = self._special_thumbs[ 'hydrus' ]
self._data_cache.AddData( hash, hydrus_bitmap )
else:
hydrus_bitmap = result
return hydrus_bitmap
elif mime in HC.AUDIO: return self._special_thumbs[ 'audio' ]
elif mime in HC.VIDEO: return self._special_thumbs[ 'video' ]
elif mime == HC.APPLICATION_PDF: return self._special_thumbs[ 'pdf' ]
elif mime in HC.ARCHIVES: return self._special_thumbs[ 'zip' ]
else: return self._special_thumbs[ 'hydrus' ]
else:
return self._special_thumbs[ 'hydrus' ]
def HasThumbnailCached( self, media ):
display_media = media.GetDisplayMedia()
mime = display_media.GetMime()
if mime in HC.MIMES_WITH_THUMBNAILS:
hash = display_media.GetHash()
return self._data_cache.HasData( hash )
else:
return True
def Waterfall( self, page_key, medias ):
with self._lock:
self._waterfall_queue_quick.update( ( ( page_key, media ) for media in medias ) )
self._RecalcWaterfallQueueRandom()
self._waterfall_event.set()
def DAEMONWaterfall( self ):
last_paused = HydrusData.GetNowPrecise()
while not HydrusThreading.IsThreadShuttingDown():
with self._lock:
do_wait = len( self._waterfall_queue_random ) == 0
if do_wait:
self._waterfall_event.wait( 1 )
self._waterfall_event.clear()
last_paused = HydrusData.GetNowPrecise()
start_time = HydrusData.GetNowPrecise()
stop_time = start_time + 0.005 # a bit of a typical frame
page_keys_to_rendered_medias = collections.defaultdict( list )
while not HydrusData.TimeHasPassedPrecise( stop_time ):
with self._lock:
if len( self._waterfall_queue_random ) == 0:
break
result = self._waterfall_queue_random.pop()
self._waterfall_queue_quick.discard( result )
( page_key, media ) = result
try:
self.GetThumbnail( media ) # to load it
page_keys_to_rendered_medias[ page_key ].append( media )
except Exception as e:
HydrusData.ShowException( e )
for ( page_key, rendered_medias ) in list(page_keys_to_rendered_medias.items()):
self._controller.pub( 'waterfall_thumbnails', page_key, rendered_medias )
time.sleep( 0.00001 )
class ServicesManager( object ):
def __init__( self, controller ):
@ -3540,6 +3108,466 @@ class TagSiblingsManager( object ):
class ThumbnailCache( object ):
def __init__( self, controller ):
self._controller = controller
cache_size = self._controller.options[ 'thumbnail_cache_size' ]
cache_timeout = self._controller.new_options.GetInteger( 'thumbnail_cache_timeout' )
self._data_cache = DataCache( self._controller, cache_size, timeout = cache_timeout )
self._lock = threading.Lock()
self._thumbnail_error_occurred = False
self._waterfall_queue_quick = set()
self._waterfall_queue_random = []
self._waterfall_event = threading.Event()
self._special_thumbs = {}
self.Clear()
self._controller.CallToThreadLongRunning( self.DAEMONWaterfall )
self._controller.sub( self, 'Clear', 'thumbnail_resize' )
self._controller.sub( self, 'ClearThumbnails', 'clear_thumbnails' )
def _GetResizedHydrusBitmap( self, display_media ):
if not HG.thumbnail_experiment_mode:
return self._GetResizedHydrusBitmapFromHardDrive( display_media )
else:
return self._GetResizedHydrusBitmapFromFullSize( display_media )
def _GetResizedHydrusBitmapFromFullSize( self, display_media ):
thumbnail_dimensions = self._controller.options[ 'thumbnail_dimensions' ]
hash = display_media.GetHash()
mime = display_media.GetMime()
locations_manager = display_media.GetLocationsManager()
try:
path = self._controller.client_files_manager.GetFullSizeThumbnailPath( hash, mime )
except HydrusExceptions.FileMissingException as e:
if locations_manager.IsLocal():
summary = 'Unable to get full-size thumbnail for file {}.'.format( hash.hex() )
self._HandleThumbnailException( e, summary )
return self._special_thumbs[ 'hydrus' ]
try:
numpy_image = ClientImageHandling.GenerateNumpyImage( path, mime )
except Exception as e:
try:
# file is malformed, let's force a regen
self._controller.client_files_manager.RegenerateResizedThumbnail( hash, mime )
except Exception as e:
summary = 'The thumbnail for file ' + hash.hex() + ' was not loadable. An attempt to regenerate it failed.'
self._HandleThumbnailException( e, summary )
return self._special_thumbs[ 'hydrus' ]
try:
numpy_image = ClientImageHandling.GenerateNumpyImage( path, mime )
except Exception as e:
summary = 'The thumbnail for file ' + hash.hex() + ' was not loadable. It was regenerated, but that file would not render either. Your image libraries or hard drive connection are unreliable. Please inform the hydrus developer what has happened.'
self._HandleThumbnailException( e, summary )
return self._special_thumbs[ 'hydrus' ]
( fullsize_x, fullsize_y ) = ClientImageHandling.GetNumPyImageResolution( numpy_image )
( resized_x, resized_y ) = HydrusImageHandling.GetThumbnailResolution( ( fullsize_x, fullsize_y ), thumbnail_dimensions )
already_correct = fullsize_x == resized_x and fullsize_y == resized_y
if not already_correct:
numpy_image = ClientImageHandling.EfficientlyThumbnailNumpyImage( numpy_image, ( resized_x, resized_y ) )
hydrus_bitmap = ClientRendering.GenerateHydrusBitmapFromNumPyImage( numpy_image )
return hydrus_bitmap
def _GetResizedHydrusBitmapFromHardDrive( self, display_media ):
thumbnail_dimensions = self._controller.options[ 'thumbnail_dimensions' ]
if tuple( thumbnail_dimensions ) == HC.UNSCALED_THUMBNAIL_DIMENSIONS:
full_size = True
else:
full_size = False
hash = display_media.GetHash()
mime = display_media.GetMime()
locations_manager = display_media.GetLocationsManager()
try:
if full_size:
path = self._controller.client_files_manager.GetFullSizeThumbnailPath( hash, mime )
else:
path = self._controller.client_files_manager.GetResizedThumbnailPath( hash, mime )
except HydrusExceptions.FileMissingException as e:
if locations_manager.IsLocal():
summary = 'Unable to get full-size thumbnail for file {}.'.format( hash.hex() )
self._HandleThumbnailException( e, summary )
return self._special_thumbs[ 'hydrus' ]
try:
hydrus_bitmap = ClientRendering.GenerateHydrusBitmap( path, mime )
except Exception as e:
try:
# file is malformed, let's force a regen
self._controller.client_files_manager.RegenerateFullSizeThumbnail( hash, mime )
except Exception as e:
summary = 'The thumbnail for file ' + hash.hex() + ' was not loadable. An attempt to regenerate it failed.'
self._HandleThumbnailException( e, summary )
return self._special_thumbs[ 'hydrus' ]
try:
hydrus_bitmap = ClientRendering.GenerateHydrusBitmap( path, mime )
except Exception as e:
summary = 'The thumbnail for file ' + hash.hex() + ' was not loadable. It was regenerated, but that file would not render either. Your image libraries or hard drive connection are unreliable. Please inform the hydrus developer what has happened.'
self._HandleThumbnailException( e, summary )
return self._special_thumbs[ 'hydrus' ]
( media_x, media_y ) = display_media.GetResolution()
( actual_x, actual_y ) = hydrus_bitmap.GetSize()
( desired_x, desired_y ) = self._controller.options[ 'thumbnail_dimensions' ]
too_large = actual_x > desired_x or actual_y > desired_y
small_original_image = actual_x == media_x and actual_y == media_y
too_small = actual_x < desired_x and actual_y < desired_y
if too_large or ( too_small and not small_original_image ):
self._controller.client_files_manager.RegenerateResizedThumbnail( hash, mime )
hydrus_bitmap = ClientRendering.GenerateHydrusBitmap( path, mime )
return hydrus_bitmap
def _HandleThumbnailException( self, e, summary ):
if self._thumbnail_error_occurred:
HydrusData.Print( summary )
else:
self._thumbnail_error_occurred = True
message = 'A thumbnail error has occurred. The problem thumbnail will appear with the default \'hydrus\' symbol. You may need to take hard drive recovery actions, and if the error is not obviously fixable, you can contact hydrus dev for additional help. Specific information for this first error follows. Subsequent thumbnail errors in this session will be silently printed to the log.'
message += os.linesep * 2
message += str( e )
message += os.linesep * 2
message += summary
HydrusData.ShowText( message )
def _RecalcWaterfallQueueRandom( self ):
# here we sort by the hash since this is both breddy random and more likely to access faster on a well defragged hard drive!
def sort_by_hash_key( item ):
( page_key, media ) = item
return media.GetDisplayMedia().GetHash()
self._waterfall_queue_random = list( self._waterfall_queue_quick )
self._waterfall_queue_random.sort( key = sort_by_hash_key )
def CancelWaterfall( self, page_key, medias ):
with self._lock:
self._waterfall_queue_quick.difference_update( ( ( page_key, media ) for media in medias ) )
self._RecalcWaterfallQueueRandom()
def Clear( self ):
with self._lock:
self._data_cache.Clear()
self._special_thumbs = {}
names = [ 'hydrus', 'pdf', 'psd', 'audio', 'video', 'zip' ]
( os_file_handle, temp_path ) = ClientPaths.GetTempPath()
try:
for name in names:
path = os.path.join( HC.STATIC_DIR, name + '.png' )
thumbnail_dimensions = self._controller.options[ 'thumbnail_dimensions' ]
thumbnail_bytes = HydrusFileHandling.GenerateThumbnailFileBytesFromStaticImagePath( path, thumbnail_dimensions, HC.IMAGE_PNG )
with open( temp_path, 'wb' ) as f:
f.write( thumbnail_bytes )
hydrus_bitmap = ClientRendering.GenerateHydrusBitmap( temp_path, HC.IMAGE_PNG )
self._special_thumbs[ name ] = hydrus_bitmap
finally:
HydrusPaths.CleanUpTempPath( os_file_handle, temp_path )
def ClearThumbnails( self, hashes ):
with self._lock:
for hash in hashes:
self._data_cache.DeleteData( hash )
def DoingWork( self ):
with self._lock:
return len( self._waterfall_queue_random ) > 0
def GetThumbnail( self, media ):
try:
display_media = media.GetDisplayMedia()
except:
# sometimes media can get switched around during a collect event, and if this happens during waterfall, we have a problem here
# just return for now, we'll see how it goes
return self._special_thumbs[ 'hydrus' ]
locations_manager = display_media.GetLocationsManager()
if locations_manager.ShouldIdeallyHaveThumbnail():
mime = display_media.GetMime()
if mime in HC.MIMES_WITH_THUMBNAILS:
hash = display_media.GetHash()
result = self._data_cache.GetIfHasData( hash )
if result is None:
try:
hydrus_bitmap = self._GetResizedHydrusBitmap( display_media )
except:
hydrus_bitmap = self._special_thumbs[ 'hydrus' ]
self._data_cache.AddData( hash, hydrus_bitmap )
else:
hydrus_bitmap = result
return hydrus_bitmap
elif mime in HC.AUDIO: return self._special_thumbs[ 'audio' ]
elif mime in HC.VIDEO: return self._special_thumbs[ 'video' ]
elif mime == HC.APPLICATION_PDF: return self._special_thumbs[ 'pdf' ]
elif mime == HC.APPLICATION_PSD: return self._special_thumbs[ 'psd' ]
elif mime in HC.ARCHIVES: return self._special_thumbs[ 'zip' ]
else: return self._special_thumbs[ 'hydrus' ]
else:
return self._special_thumbs[ 'hydrus' ]
def HasThumbnailCached( self, media ):
display_media = media.GetDisplayMedia()
mime = display_media.GetMime()
if mime in HC.MIMES_WITH_THUMBNAILS:
hash = display_media.GetHash()
return self._data_cache.HasData( hash )
else:
return True
def Waterfall( self, page_key, medias ):
with self._lock:
self._waterfall_queue_quick.update( ( ( page_key, media ) for media in medias ) )
self._RecalcWaterfallQueueRandom()
self._waterfall_event.set()
def DAEMONWaterfall( self ):
last_paused = HydrusData.GetNowPrecise()
while not HydrusThreading.IsThreadShuttingDown():
with self._lock:
do_wait = len( self._waterfall_queue_random ) == 0
if do_wait:
self._waterfall_event.wait( 1 )
self._waterfall_event.clear()
last_paused = HydrusData.GetNowPrecise()
start_time = HydrusData.GetNowPrecise()
stop_time = start_time + 0.005 # a bit of a typical frame
page_keys_to_rendered_medias = collections.defaultdict( list )
while not HydrusData.TimeHasPassedPrecise( stop_time ):
with self._lock:
if len( self._waterfall_queue_random ) == 0:
break
result = self._waterfall_queue_random.pop()
self._waterfall_queue_quick.discard( result )
( page_key, media ) = result
self.GetThumbnail( media )
page_keys_to_rendered_medias[ page_key ].append( media )
for ( page_key, rendered_medias ) in page_keys_to_rendered_medias.items():
self._controller.pub( 'waterfall_thumbnails', page_key, rendered_medias )
time.sleep( 0.00001 )
class UndoManager( object ):
def __init__( self, controller ):

3
include/ClientConstants.py
View File

@ -234,6 +234,7 @@ else:
media_viewer_capabilities[ HC.APPLICATION_PDF ] = no_support
media_viewer_capabilities[ HC.APPLICATION_PSD ] = no_support
media_viewer_capabilities[ HC.APPLICATION_ZIP ] = no_support
media_viewer_capabilities[ HC.APPLICATION_7Z ] = no_support
media_viewer_capabilities[ HC.APPLICATION_RAR ] = no_support
@ -348,7 +349,7 @@ SHORTCUTS_RESERVED_NAMES = [ 'archive_delete_filter', 'duplicate_filter', 'media
# shortcut commands
SHORTCUTS_MEDIA_ACTIONS = [ 'manage_file_tags', 'manage_file_ratings', 'manage_file_urls', 'manage_file_notes', 'archive_file', 'inbox_file', 'delete_file', 'export_files', 'export_files_quick_auto_export', 'remove_file_from_view', 'open_file_in_external_program', 'open_selection_in_new_page', 'launch_the_archive_delete_filter', 'copy_bmp', 'copy_file', 'copy_path', 'copy_sha256_hash', 'get_similar_to_exact', 'get_similar_to_very_similar', 'get_similar_to_similar', 'get_similar_to_speculative', 'duplicate_media_remove_relationships', 'duplicate_media_reset_to_potential', 'duplicate_media_set_alternate', 'duplicate_media_set_alternate_collections', 'duplicate_media_set_custom', 'duplicate_media_set_focused_better', 'duplicate_media_set_not_duplicate', 'duplicate_media_set_same_quality' ]
SHORTCUTS_MEDIA_ACTIONS = [ 'manage_file_tags', 'manage_file_ratings', 'manage_file_urls', 'manage_file_notes', 'archive_file', 'inbox_file', 'delete_file', 'export_files', 'export_files_quick_auto_export', 'remove_file_from_view', 'open_file_in_external_program', 'open_selection_in_new_page', 'launch_the_archive_delete_filter', 'copy_bmp', 'copy_file', 'copy_path', 'copy_sha256_hash', 'get_similar_to_exact', 'get_similar_to_very_similar', 'get_similar_to_similar', 'get_similar_to_speculative', 'duplicate_media_remove_relationships', 'duplicate_media_reset_to_potential', 'duplicate_media_set_alternate', 'duplicate_media_set_alternate_collections', 'duplicate_media_set_custom', 'duplicate_media_set_focused_better', 'duplicate_media_set_not_duplicate', 'duplicate_media_set_same_quality', 'open_known_url' ]
SHORTCUTS_MEDIA_VIEWER_ACTIONS = [ 'move_animation_to_previous_frame', 'move_animation_to_next_frame', 'switch_between_fullscreen_borderless_and_regular_framed_window', 'pan_up', 'pan_down', 'pan_left', 'pan_right', 'zoom_in', 'zoom_out', 'switch_between_100_percent_and_canvas_zoom', 'flip_darkmode' ]
SHORTCUTS_MEDIA_VIEWER_BROWSER_ACTIONS = [ 'view_next', 'view_first', 'view_last', 'view_previous' ]
SHORTCUTS_MAIN_GUI_ACTIONS = [ 'refresh', 'new_page', 'new_page_of_pages', 'new_duplicate_filter_page', 'new_gallery_downloader_page', 'new_url_downloader_page', 'new_simple_downloader_page', 'new_watcher_downloader_page', 'synchronised_wait_switch', 'set_media_focus', 'show_hide_splitters', 'set_search_focus', 'unclose_page', 'close_page', 'redo', 'undo', 'flip_darkmode', 'check_all_import_folders', 'flip_debug_force_idle_mode_do_not_set_this' ]

8
include/ClientController.py
View File

@ -55,7 +55,7 @@ if not HG.twisted_is_broke:
class Controller( HydrusController.HydrusController ):
def __init__( self, db_dir, no_daemons, no_wal ):
def __init__( self, db_dir ):
self._last_shutdown_was_bad = False
@ -63,7 +63,7 @@ class Controller( HydrusController.HydrusController ):
self._splash = None
HydrusController.HydrusController.__init__( self, db_dir, no_daemons, no_wal )
HydrusController.HydrusController.__init__( self, db_dir )
self._name = 'client'
@ -96,7 +96,7 @@ class Controller( HydrusController.HydrusController ):
def _InitDB( self ):
return ClientDB.DB( self, self.db_dir, 'client', no_wal = self._no_wal )
return ClientDB.DB( self, self.db_dir, 'client' )
def _InitTempDir( self ):
@ -796,7 +796,7 @@ class Controller( HydrusController.HydrusController ):
self.RestartClientServerService( CC.LOCAL_BOORU_SERVICE_KEY )
self.RestartClientServerService( CC.CLIENT_API_SERVICE_KEY )
if not self._no_daemons:
if not HG.no_daemons:
self._daemons.append( HydrusThreading.DAEMONForegroundWorker( self, 'DownloadFiles', ClientDaemons.DAEMONDownloadFiles, ( 'notify_new_downloads', 'notify_new_permissions' ) ) )
self._daemons.append( HydrusThreading.DAEMONForegroundWorker( self, 'SynchroniseSubscriptions', ClientDaemons.DAEMONSynchroniseSubscriptions, ( 'notify_restart_subs_sync_daemon', 'notify_new_subscriptions' ), period = 4 * 3600, init_wait = 60, pre_call_wait = 3 ) )

36
include/ClientDB.py
View File

@ -184,11 +184,11 @@ class DB( HydrusDB.HydrusDB ):
READ_WRITE_ACTIONS = [ 'service_info', 'system_predicates', 'missing_thumbnail_hashes' ]
def __init__( self, controller, db_dir, db_name, no_wal = False ):
def __init__( self, controller, db_dir, db_name ):
self._initial_messages = []
HydrusDB.HydrusDB.__init__( self, controller, db_dir, db_name, no_wal = no_wal )
HydrusDB.HydrusDB.__init__( self, controller, db_dir, db_name )
self._controller.pub( 'splash_set_title_text', 'booting db\u2026' )
@ -4699,9 +4699,9 @@ class DB( HydrusDB.HydrusDB ):
query_hash_ids = update_qhi( query_hash_ids, similar_hash_ids )
for ( operator, value, service_key ) in system_predicates.GetRatingsPredicates():
for ( operator, value, rating_service_key ) in system_predicates.GetRatingsPredicates():
service_id = self._GetServiceId( service_key )
service_id = self._GetServiceId( rating_service_key )
if value == 'not rated':
@ -4716,28 +4716,46 @@ class DB( HydrusDB.HydrusDB ):
else:
service = HG.client_controller.services_manager.GetService( rating_service_key )
if service.GetServiceType() == HC.LOCAL_RATING_LIKE:
half_a_star_value = 0.5
else:
num_stars = service.GetNumStars()
if service.AllowZero():
num_stars += 1
half_a_star_value = 1.0 / ( ( num_stars - 1 ) * 2 )
if isinstance( value, str ):
value = float( value )
# floats are a pain!
# floats are a pain! as is storing rating as 0.0-1.0 and then allowing number of stars to change!
if operator == '\u2248':
predicate = str( value * 0.8 ) + ' < rating AND rating < ' + str( value * 1.2 )
predicate = str( ( value - half_a_star_value ) * 0.8 ) + ' < rating AND rating < ' + str( ( value + half_a_star_value ) * 1.2 )
elif operator == '<':
predicate = 'rating < ' + str( value * 0.995 )
predicate = 'rating <= ' + str( value - half_a_star_value )
elif operator == '>':
predicate = 'rating > ' + str( value * 1.005 )
predicate = 'rating > ' + str( value + half_a_star_value )
elif operator == '=':
predicate = str( value * 0.995 ) + ' <= rating AND rating <= ' + str( value * 1.005 )
predicate = str( value - half_a_star_value ) + ' < rating AND rating <= ' + str( value + half_a_star_value )
rating_hash_ids = self._STI( self._c.execute( 'SELECT hash_id FROM local_ratings WHERE service_id = ? AND ' + predicate + ';', ( service_id, ) ) )

355
include/ClientGUI.py
View File

@ -70,6 +70,166 @@ ID_TIMER_ANIMATION_UPDATE = wx.NewId()
MENU_ORDER = [ 'file', 'undo', 'pages', 'database', 'pending', 'network', 'services', 'help' ]
def THREADUploadPending( service_key ):
service = HG.client_controller.services_manager.GetService( service_key )
service_name = service.GetName()
service_type = service.GetServiceType()
nums_pending = HG.client_controller.Read( 'nums_pending' )
info = nums_pending[ service_key ]
initial_num_pending = sum( info.values() )
result = HG.client_controller.Read( 'pending', service_key )
try:
job_key = ClientThreading.JobKey( pausable = True, cancellable = True )
job_key.SetVariable( 'popup_title', 'uploading pending to ' + service_name )
HG.client_controller.pub( 'message', job_key )
while result is not None:
nums_pending = HG.client_controller.Read( 'nums_pending' )
info = nums_pending[ service_key ]
remaining_num_pending = sum( info.values() )
done_num_pending = initial_num_pending - remaining_num_pending
job_key.SetVariable( 'popup_text_1', 'uploading to ' + service_name + ': ' + HydrusData.ConvertValueRangeToPrettyString( done_num_pending, initial_num_pending ) )
job_key.SetVariable( 'popup_gauge_1', ( done_num_pending, initial_num_pending ) )
while job_key.IsPaused() or job_key.IsCancelled():
time.sleep( 0.1 )
if job_key.IsCancelled():
job_key.DeleteVariable( 'popup_gauge_1' )
job_key.SetVariable( 'popup_text_1', 'cancelled' )
HydrusData.Print( job_key.ToString() )
job_key.Delete( 5 )
return
try:
if service_type in HC.REPOSITORIES:
if isinstance( result, ClientMedia.MediaResult ):
media_result = result
client_files_manager = HG.client_controller.client_files_manager
hash = media_result.GetHash()
mime = media_result.GetMime()
path = client_files_manager.GetFilePath( hash, mime )
with open( path, 'rb' ) as f:
file_bytes = f.read()
service.Request( HC.POST, 'file', { 'file' : file_bytes } )
file_info_manager = media_result.GetFileInfoManager()
timestamp = HydrusData.GetNow()
content_update_row = ( file_info_manager, timestamp )
content_updates = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ADD, content_update_row ) ]
else:
client_to_server_update = result
service.Request( HC.POST, 'update', { 'client_to_server_update' : client_to_server_update } )
content_updates = client_to_server_update.GetClientsideContentUpdates()
HG.client_controller.WriteSynchronous( 'content_updates', { service_key : content_updates } )
elif service_type == HC.IPFS:
if isinstance( result, ClientMedia.MediaResult ):
media_result = result
hash = media_result.GetHash()
mime = media_result.GetMime()
service.PinFile( hash, mime )
else:
( hash, multihash ) = result
service.UnpinFile( hash, multihash )
except HydrusExceptions.ServerBusyException:
job_key.SetVariable( 'popup_text_1', service.GetName() + ' was busy. please try again in a few minutes' )
job_key.Cancel()
return
HG.client_controller.pub( 'notify_new_pending' )
time.sleep( 0.1 )
HG.client_controller.WaitUntilViewFree()
result = HG.client_controller.Read( 'pending', service_key )
except Exception as e:
r = re.search( '[a-fA-F0-9]{64}', str( e ) )
if r is not None:
possible_hash = bytes.fromhex( r.group() )
HydrusData.ShowText( 'Found a possible hash in that error message--trying to show it in a new page.' )
HG.client_controller.pub( 'imported_files_to_page', [ possible_hash ], 'files that did not upload right' )
job_key.SetVariable( 'popup_text_1', service.GetName() + ' error' )
job_key.Cancel()
raise
job_key.DeleteVariable( 'popup_gauge_1' )
job_key.SetVariable( 'popup_text_1', 'upload done!' )
HydrusData.Print( job_key.ToString() )
job_key.Finish()
job_key.Delete( 5 )
class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
def __init__( self, controller ):
@ -2299,7 +2459,7 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
def _ImportURL( self, url, service_keys_to_tags = None, destination_page_name = None ):
def _ImportURL( self, url, service_keys_to_tags = None, destination_page_name = None, show_destination_page = True ):
if service_keys_to_tags is None:
@ -2321,11 +2481,14 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
if url_type in ( HC.URL_TYPE_UNKNOWN, HC.URL_TYPE_FILE, HC.URL_TYPE_POST, HC.URL_TYPE_GALLERY ):
page = self._notebook.GetOrMakeURLImportPage( desired_page_name = destination_page_name )
page = self._notebook.GetOrMakeURLImportPage( desired_page_name = destination_page_name, select_page = show_destination_page )
if page is not None:
self._notebook.ShowPage( page )
if show_destination_page:
self._notebook.ShowPage( page )
management_panel = page.GetManagementPanel()
@ -2336,11 +2499,14 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
elif url_type == HC.URL_TYPE_WATCHABLE:
page = self._notebook.GetOrMakeMultipleWatcherPage( desired_page_name = destination_page_name )
page = self._notebook.GetOrMakeMultipleWatcherPage( desired_page_name = destination_page_name, select_page = show_destination_page )
if page is not None:
self._notebook.ShowPage( page )
if show_destination_page:
self._notebook.ShowPage( page )
management_panel = page.GetManagementPanel()
@ -3904,7 +4070,20 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
def _UploadPending( self, service_key ):
self._controller.CallToThread( self._THREADUploadPending, service_key )
service = self._controller.services_manager.GetService( service_key )
try:
service.CheckFunctional( including_bandwidth = False )
except Exception as e:
wx.MessageBox( 'Unfortunately, there is a problem with starting the upload: ' + str( e ) )
return
self._controller.CallToThread( THREADUploadPending, service_key )
def _VacuumDatabase( self ):
@ -4045,166 +4224,6 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
def _THREADUploadPending( self, service_key ):
service = self._controller.services_manager.GetService( service_key )
service_name = service.GetName()
service_type = service.GetServiceType()
nums_pending = self._controller.Read( 'nums_pending' )
info = nums_pending[ service_key ]
initial_num_pending = sum( info.values() )
result = self._controller.Read( 'pending', service_key )
try:
job_key = ClientThreading.JobKey( pausable = True, cancellable = True )
job_key.SetVariable( 'popup_title', 'uploading pending to ' + service_name )
self._controller.pub( 'message', job_key )
while result is not None:
nums_pending = self._controller.Read( 'nums_pending' )
info = nums_pending[ service_key ]
remaining_num_pending = sum( info.values() )
done_num_pending = initial_num_pending - remaining_num_pending
job_key.SetVariable( 'popup_text_1', 'uploading to ' + service_name + ': ' + HydrusData.ConvertValueRangeToPrettyString( done_num_pending, initial_num_pending ) )
job_key.SetVariable( 'popup_gauge_1', ( done_num_pending, initial_num_pending ) )
while job_key.IsPaused() or job_key.IsCancelled():
time.sleep( 0.1 )
if job_key.IsCancelled():
job_key.DeleteVariable( 'popup_gauge_1' )
job_key.SetVariable( 'popup_text_1', 'cancelled' )
HydrusData.Print( job_key.ToString() )
job_key.Delete( 5 )
return
try:
if service_type in HC.REPOSITORIES:
if isinstance( result, ClientMedia.MediaResult ):
media_result = result
client_files_manager = self._controller.client_files_manager
hash = media_result.GetHash()
mime = media_result.GetMime()
path = client_files_manager.GetFilePath( hash, mime )
with open( path, 'rb' ) as f:
file_bytes = f.read()
service.Request( HC.POST, 'file', { 'file' : file_bytes } )
file_info_manager = media_result.GetFileInfoManager()
timestamp = HydrusData.GetNow()
content_update_row = ( file_info_manager, timestamp )
content_updates = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ADD, content_update_row ) ]
else:
client_to_server_update = result
service.Request( HC.POST, 'update', { 'client_to_server_update' : client_to_server_update } )
content_updates = client_to_server_update.GetClientsideContentUpdates()
self._controller.WriteSynchronous( 'content_updates', { service_key : content_updates } )
elif service_type == HC.IPFS:
if isinstance( result, ClientMedia.MediaResult ):
media_result = result
hash = media_result.GetHash()
mime = media_result.GetMime()
service.PinFile( hash, mime )
else:
( hash, multihash ) = result
service.UnpinFile( hash, multihash )
except HydrusExceptions.ServerBusyException:
job_key.SetVariable( 'popup_text_1', service.GetName() + ' was busy. please try again in a few minutes' )
job_key.Cancel()
return
self._controller.pub( 'notify_new_pending' )
time.sleep( 0.1 )
self._controller.WaitUntilViewFree()
result = self._controller.Read( 'pending', service_key )
except Exception as e:
r = re.search( '[a-fA-F0-9]{64}', str( e ) )
if r is not None:
possible_hash = bytes.fromhex( r.group() )
HydrusData.ShowText( 'Found a possible hash in that error message--trying to show it in a new page.' )
HG.client_controller.pub( 'imported_files_to_page', [ possible_hash ], 'files that did not upload right' )
job_key.SetVariable( 'popup_text_1', service.GetName() + ' error' )
job_key.Cancel()
raise
job_key.DeleteVariable( 'popup_gauge_1' )
job_key.SetVariable( 'popup_text_1', 'upload done!' )
HydrusData.Print( job_key.ToString() )
job_key.Finish()
job_key.Delete( 5 )
def AddModalMessage( self, job_key ):
if job_key.IsCancelled() or job_key.IsDeleted():
@ -4619,11 +4638,11 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
self._ImportFiles( paths )
def ImportURLFromAPI( self, url, service_keys_to_tags, destination_page_name ):
def ImportURLFromAPI( self, url, service_keys_to_tags, destination_page_name, show_destination_page ):
try:
( normalised_url, result_text ) = self._ImportURL( url, service_keys_to_tags = service_keys_to_tags, destination_page_name = destination_page_name )
( normalised_url, result_text ) = self._ImportURL( url, service_keys_to_tags = service_keys_to_tags, destination_page_name = destination_page_name, show_destination_page = show_destination_page )
return ( normalised_url, result_text )

46
include/ClientGUICanvas.py
View File

@ -268,6 +268,7 @@ class Animation( wx.Window ):
self._current_frame_index = 0
self._current_frame_drawn = False
self._current_timestamp_ms = None
self._next_frame_due_at = HydrusData.GetNowPrecise()
self._slow_frame_score = 1.0
@ -525,8 +526,13 @@ class Animation( wx.Window ):
buffer_indices = self._video_container.GetBufferIndices()
if self._current_timestamp_ms is None and self._video_container.IsInitialised():
self._current_timestamp_ms = self._video_container.GetTimestampMS( self._current_frame_index )
return ( self._current_frame_index, self._paused, buffer_indices )
return ( self._current_frame_index, self._current_timestamp_ms, self._paused, buffer_indices )
def GotoFrame( self, frame_index ):
@ -536,6 +542,9 @@ class Animation( wx.Window ):
if frame_index != self._current_frame_index:
self._current_frame_index = frame_index
self._current_timestamp_ms = None
self._next_frame_due_at = HydrusData.GetNowPrecise()
self._video_container.GetReadyForFrame( self._current_frame_index )
@ -592,6 +601,7 @@ class Animation( wx.Window ):
self._current_frame_index = int( ( self._num_frames - 1 ) * HC.options[ 'animation_start_position' ] )
self._current_frame_drawn = False
self._current_timestamp_ms = None
self._next_frame_due_at = HydrusData.GetNowPrecise()
self._slow_frame_score = 1.0
@ -639,8 +649,18 @@ class Animation( wx.Window ):
if self._current_frame_index == 0:
self._current_timestamp_ms = 0
self._has_played_once_through = True
else:
if self._current_timestamp_ms is not None and self._video_container is not None and self._video_container.IsInitialised():
duration_ms = self._video_container.GetDuration( self._current_frame_index - 1 )
self._current_timestamp_ms += duration_ms
self._current_frame_drawn = False
@ -681,6 +701,7 @@ class AnimationBar( wx.Window ):
self.SetCursor( wx.Cursor( wx.CURSOR_ARROW ) )
self._media_window = None
self._duration_ms = 1000
self._num_frames = 1
self._last_drawn_info = None
@ -699,10 +720,11 @@ class AnimationBar( wx.Window ):
if FLASHWIN_OK and isinstance( self._media_window, wx.lib.flashwin.FlashWindow ):
current_frame = self._media_window.CurrentFrame()
current_timestamp_ms = None
paused = False
buffer_indices = None
return ( current_frame, paused, buffer_indices )
return ( current_frame, current_timestamp_ms, paused, buffer_indices )
else:
@ -726,7 +748,7 @@ class AnimationBar( wx.Window ):
self._last_drawn_info = self._GetAnimationBarStatus()
( current_frame_index, paused, buffer_indices ) = self._last_drawn_info
( current_frame_index, current_timestamp_ms, paused, buffer_indices ) = self._last_drawn_info
( my_width, my_height ) = self._canvas_bmp.GetSize()
@ -807,6 +829,11 @@ class AnimationBar( wx.Window ):
s = HydrusData.ConvertValueRangeToPrettyString( current_frame_index + 1, self._num_frames )
if current_timestamp_ms is not None:
s += ' - {}'.format( HydrusData.ConvertValueRangeToScanbarTimestampsMS( current_timestamp_ms, self._duration_ms ) )
( x, y ) = dc.GetTextExtent( s )
dc.DrawText( s, my_width - x - 3, 3 )
@ -938,6 +965,7 @@ class AnimationBar( wx.Window ):
def SetMediaAndWindow( self, media, media_window ):
self._media_window = media_window
self._duration_ms = max( media.GetDuration(), 1 )
self._num_frames = max( media.GetNumFrames(), 1 )
self._last_drawn_info = None
@ -1738,6 +1766,14 @@ class Canvas( wx.Window ):
def _OpenKnownURL( self ):
if self._current_media is not None:
ClientGUIMedia.DoOpenKnownURLFromShortcut( self, self._current_media )
def _PauseCurrentMedia( self ):
if self._current_media is None:
@ -2239,6 +2275,10 @@ class Canvas( wx.Window ):
self._ManageNotes()
elif action == 'open_known_url':
self._OpenKnownURL()
elif action == 'archive_file':
self._Archive()

2
include/ClientGUIDialogs.py
View File

@ -589,7 +589,7 @@ class FrameInputLocalFiles( wx.Frame ):
columns = [ ( '#', 4 ), ( 'path', -1 ), ( 'guessed mime', 16 ), ( 'size', 10 ) ]
self._paths_list = ClientGUIListCtrl.BetterListCtrl( listctrl_panel, 'input_local_files', 28, 36, columns, self._ConvertListCtrlDataToTuple, delete_key_callback = self.RemovePaths )
self._paths_list = ClientGUIListCtrl.BetterListCtrl( listctrl_panel, 'input_local_files', 12, 36, columns, self._ConvertListCtrlDataToTuple, delete_key_callback = self.RemovePaths )
listctrl_panel.SetListCtrl( self._paths_list )

65
include/ClientGUIMedia.py
View File

@ -86,6 +86,59 @@ def CopyMediaURLMatchURLs( medias, url_match ):
HG.client_controller.pub( 'clipboard', 'text', urls_string )
def DoOpenKnownURLFromShortcut( win, media ):
urls = media.GetLocationsManager().GetURLs()
matched_labels_and_urls = []
unmatched_urls = []
if len( urls ) > 0:
for url in urls:
url_match = HG.client_controller.network_engine.domain_manager.GetURLMatch( url )
if url_match is None:
unmatched_urls.append( url )
else:
label = url_match.GetName() + ': ' + url
matched_labels_and_urls.append( ( label, url ) )
matched_labels_and_urls.sort()
unmatched_urls.sort()
if len( matched_labels_and_urls ) == 0:
return
elif len( matched_labels_and_urls ) == 1:
url = matched_labels_and_urls[0][1]
else:
matched_labels_and_urls.extend( ( url, url ) for url in unmatched_urls )
try:
url = ClientGUIDialogsQuick.SelectFromList( win, 'Select which URL', matched_labels_and_urls, sort_tuples = False )
except HydrusExceptions.CancelledException:
return
ClientPaths.LaunchURLInWebBrowser( url )
def OpenURLs( urls ):
urls = list( urls )
@ -1519,6 +1572,14 @@ class MediaPanel( ClientMedia.ListeningMediaList, wx.ScrolledCanvas ):
def _OpenKnownURL( self ):
if self._focussed_media is not None:
DoOpenKnownURLFromShortcut( self, self._focussed_media )
def _PetitionFiles( self, remote_service_key ):
hashes = self._GetSelectedHashes()
@ -2256,6 +2317,10 @@ class MediaPanel( ClientMedia.ListeningMediaList, wx.ScrolledCanvas ):
self._ManageNotes()
elif action == 'open_known_url':
self._OpenKnownURL()
elif action == 'archive_file':
self._Archive()

108
include/ClientGUIPages.py
View File

@ -1905,100 +1905,64 @@ class PagesNotebook( wx.Notebook ):
def GetOrMakeMultipleWatcherPage( self, desired_page_name = None ):
def GetOrMakeMultipleWatcherPage( self, desired_page_name = None, select_page = True ):
current_media_page = self.GetCurrentMediaPage()
if current_media_page is not None and current_media_page.IsMultipleWatcherPage():
has_wrong_name = desired_page_name is not None and current_media_page.GetName() != desired_page_name
if not has_wrong_name:
return current_media_page
# first do a search for the page name, if asked for
potential_watcher_pages = [ page for page in self._GetMediaPages( False ) if page.IsMultipleWatcherPage() ]
if desired_page_name is not None:
page = self._GetPageFromName( desired_page_name )
if page is not None and page.IsMultipleWatcherPage():
return page
potential_watcher_pages = [ page for page in potential_watcher_pages if page.GetName() == desired_page_name ]
# failing that, find/generate one with default name
for page in self._GetPages():
if len( potential_watcher_pages ) > 0:
if isinstance( page, PagesNotebook ):
# ok, we can use an existing one. should we use the current?
current_media_page = self.GetCurrentMediaPage()
if current_media_page is not None and current_media_page in potential_watcher_pages:
if page.HasMultipleWatcherPage():
return page.GetOrMakeMultipleWatcherPage( desired_page_name = desired_page_name )
return current_media_page
elif page.IsMultipleWatcherPage():
else:
return page
return potential_watcher_pages[0]
# import page does not exist
return self.NewPageImportMultipleWatcher( page_name = desired_page_name, on_deepest_notebook = True )
else:
return self.NewPageImportMultipleWatcher( page_name = desired_page_name, on_deepest_notebook = True, select_page = select_page )
def GetOrMakeURLImportPage( self, desired_page_name = None ):
def GetOrMakeURLImportPage( self, desired_page_name = None, select_page = True ):
current_media_page = self.GetCurrentMediaPage()
if current_media_page is not None and current_media_page.IsURLImportPage():
has_wrong_name = desired_page_name is not None and current_media_page.GetName() != desired_page_name
if not has_wrong_name:
return current_media_page
# first do a search for the page name, if asked for
potential_url_import_pages = [ page for page in self._GetMediaPages( False ) if page.IsURLImportPage() ]
if desired_page_name is not None:
page = self._GetPageFromName( desired_page_name )
if page is not None and page.IsURLImportPage():
return page
potential_url_import_pages = [ page for page in potential_url_import_pages if page.GetName() == desired_page_name ]
# failing that, find/generate one with default name
for page in self._GetPages():
if len( potential_url_import_pages ) > 0:
if isinstance( page, PagesNotebook ):
# ok, we can use an existing one. should we use the current?
current_media_page = self.GetCurrentMediaPage()
if current_media_page is not None and current_media_page in potential_url_import_pages:
if page.HasURLImportPage():
return page.GetOrMakeURLImportPage( desired_page_name = desired_page_name )
return current_media_page
elif page.IsURLImportPage():
else:
return page
return potential_url_import_pages[0]
# import page does not exist
return self.NewPageImportURLs( page_name = desired_page_name, on_deepest_notebook = True )
else:
return self.NewPageImportURLs( page_name = desired_page_name, on_deepest_notebook = True, select_page = select_page )
def GetPageKey( self ):
@ -2418,18 +2382,18 @@ class PagesNotebook( wx.Notebook ):
return self.NewPage( management_controller, on_deepest_notebook = on_deepest_notebook )
def NewPageImportMultipleWatcher( self, page_name = None, url = None, on_deepest_notebook = False ):
def NewPageImportMultipleWatcher( self, page_name = None, url = None, on_deepest_notebook = False, select_page = True ):
management_controller = ClientGUIManagement.CreateManagementControllerImportMultipleWatcher( page_name = page_name, url = url )
return self.NewPage( management_controller, on_deepest_notebook = on_deepest_notebook )
return self.NewPage( management_controller, on_deepest_notebook = on_deepest_notebook, select_page = select_page )
def NewPageImportURLs( self, page_name = None, on_deepest_notebook = False ):
def NewPageImportURLs( self, page_name = None, on_deepest_notebook = False, select_page = True ):
management_controller = ClientGUIManagement.CreateManagementControllerImportURLs( page_name = page_name )
return self.NewPage( management_controller, on_deepest_notebook = on_deepest_notebook )
return self.NewPage( management_controller, on_deepest_notebook = on_deepest_notebook, select_page = select_page )
def NewPagePetitions( self, service_key, on_deepest_notebook = False ):

4
include/ClientGUIScrolledPanelsEdit.py
View File

@ -3364,7 +3364,7 @@ But if 2 is--and is also perhaps accompanied by many 'could not parse' errors--t
if query_text in self._GetCurrentQueryTexts():
wx.MessageBox( 'You already have a query for "' + query_text + '"! This duplicate entry you just created will not be added.' )
wx.MessageBox( 'You already have a query for "' + query_text + '", so nothing new has been added.' )
return
@ -3698,7 +3698,7 @@ But if 2 is--and is also perhaps accompanied by many 'could not parse' errors--t
message += os.linesep * 2
message += os.linesep.join( already_existing_query_texts )
message += os.linesep * 2
message += 'Were already in the subscription. They will not be added.'
message += 'Were already in the subscription, so they need not be added.'
if len( new_query_texts ) > 0:

2
include/ClientGUIScrolledPanelsManagement.py
View File

@ -2848,7 +2848,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._media_zooms.SetValue( ','.join( ( str( media_zoom ) for media_zoom in media_zooms ) ) )
mimes_in_correct_order = ( HC.IMAGE_JPEG, HC.IMAGE_PNG, HC.IMAGE_APNG, HC.IMAGE_GIF, HC.IMAGE_WEBP, HC.IMAGE_TIFF, HC.APPLICATION_FLASH, HC.APPLICATION_PDF, HC.APPLICATION_ZIP, HC.APPLICATION_RAR, HC.APPLICATION_7Z, HC.APPLICATION_HYDRUS_UPDATE_CONTENT, HC.APPLICATION_HYDRUS_UPDATE_DEFINITIONS, HC.VIDEO_AVI, HC.VIDEO_FLV, HC.VIDEO_MOV, HC.VIDEO_MP4, HC.VIDEO_MKV, HC.VIDEO_MPEG, HC.VIDEO_WEBM, HC.VIDEO_WMV, HC.AUDIO_MP3, HC.AUDIO_OGG, HC.AUDIO_FLAC, HC.AUDIO_WMA )
mimes_in_correct_order = ( HC.IMAGE_JPEG, HC.IMAGE_PNG, HC.IMAGE_APNG, HC.IMAGE_GIF, HC.IMAGE_WEBP, HC.IMAGE_TIFF, HC.APPLICATION_FLASH, HC.APPLICATION_PDF, HC.APPLICATION_PSD, HC.APPLICATION_ZIP, HC.APPLICATION_RAR, HC.APPLICATION_7Z, HC.APPLICATION_HYDRUS_UPDATE_CONTENT, HC.APPLICATION_HYDRUS_UPDATE_DEFINITIONS, HC.VIDEO_AVI, HC.VIDEO_FLV, HC.VIDEO_MOV, HC.VIDEO_MP4, HC.VIDEO_MKV, HC.VIDEO_MPEG, HC.VIDEO_WEBM, HC.VIDEO_WMV, HC.AUDIO_MP3, HC.AUDIO_OGG, HC.AUDIO_FLAC, HC.AUDIO_WMA )
for mime in mimes_in_correct_order:

22
include/ClientLocalServerResources.py
View File

@ -474,6 +474,10 @@ class HydrusResourceBooruThumbnail( HydrusResourceBooru ):
path = os.path.join( HC.STATIC_DIR, 'pdf.png' )
elif mime == HC.APPLICATION_PSD:
path = os.path.join( HC.STATIC_DIR, 'psd.png' )
else:
path = os.path.join( HC.STATIC_DIR, 'hydrus.png' )
@ -1123,9 +1127,21 @@ class HydrusResourceClientAPIRestrictedAddURLsImportURL( HydrusResourceClientAPI
show_destination_page = False
if 'show_destination_page' in request.parsed_request_args:
show_destination_page = request.parsed_request_args[ 'show_destination_page' ]
if not isinstance( show_destination_page, bool ):
raise HydrusExceptions.BadRequestException( '"show_destination_page" did not seem to be a boolean!' )
gui = HG.client_controller.gui
( normalised_url, result_text ) = HG.client_controller.CallBlockingToWX( gui, gui.ImportURLFromAPI, url, service_keys_to_tags, destination_page_name )
( normalised_url, result_text ) = HG.client_controller.CallBlockingToWX( gui, gui.ImportURLFromAPI, url, service_keys_to_tags, destination_page_name, show_destination_page )
time.sleep( 0.05 ) # yield and give the ui time to catch up with new URL pubsubs in case this is being spammed
@ -1247,7 +1263,9 @@ class HydrusResourceClientAPIRestrictedGetFilesFileMetadata( HydrusResourceClien
request.client_api_permissions.CheckCanSeeAllFiles()
hashes = request.parsed_request_args[ 'hashes' ]
hashes_hex = request.parsed_request_args[ 'hashes' ]
hashes = [ bytes.fromhex( hash ) for hash in hashes_hex ]
if only_return_identifiers:

1
include/ClientOptions.py
View File

@ -455,6 +455,7 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
self._dictionary[ 'media_view' ][ HC.APPLICATION_PDF ] = ( CC.MEDIA_VIEWER_ACTION_SHOW_OPEN_EXTERNALLY_BUTTON, CC.MEDIA_VIEWER_ACTION_SHOW_OPEN_EXTERNALLY_BUTTON, null_zoom_info )
self._dictionary[ 'media_view' ][ HC.APPLICATION_PSD ] = ( CC.MEDIA_VIEWER_ACTION_SHOW_OPEN_EXTERNALLY_BUTTON, CC.MEDIA_VIEWER_ACTION_SHOW_OPEN_EXTERNALLY_BUTTON, null_zoom_info )
self._dictionary[ 'media_view' ][ HC.APPLICATION_ZIP ] = ( CC.MEDIA_VIEWER_ACTION_SHOW_OPEN_EXTERNALLY_BUTTON, CC.MEDIA_VIEWER_ACTION_SHOW_OPEN_EXTERNALLY_BUTTON, null_zoom_info )
self._dictionary[ 'media_view' ][ HC.APPLICATION_7Z ] = ( CC.MEDIA_VIEWER_ACTION_SHOW_OPEN_EXTERNALLY_BUTTON, CC.MEDIA_VIEWER_ACTION_SHOW_OPEN_EXTERNALLY_BUTTON, null_zoom_info )
self._dictionary[ 'media_view' ][ HC.APPLICATION_RAR ] = ( CC.MEDIA_VIEWER_ACTION_SHOW_OPEN_EXTERNALLY_BUTTON, CC.MEDIA_VIEWER_ACTION_SHOW_OPEN_EXTERNALLY_BUTTON, null_zoom_info )

20
include/ClientRendering.py
View File

@ -599,6 +599,18 @@ class RasterContainerVideo( RasterContainer ):
return self._target_resolution
def GetTimestampMS( self, frame_index ):
if self._media.GetMime() == HC.IMAGE_GIF:
return sum( self._durations[ : frame_index ] )
else:
return self._average_frame_duration * frame_index
def GetTotalDuration( self ):
if self._media.GetMime() == HC.IMAGE_GIF:
@ -619,6 +631,14 @@ class RasterContainerVideo( RasterContainer ):
def CanHaveVariableFramerate( self ):
with self._lock:
return self._media.GetMime() == HC.IMAGE_GIF
def IsInitialised( self ):
with self._lock:

15
include/HydrusConstants.py
View File

@ -67,8 +67,8 @@ options = {}
# Misc
NETWORK_VERSION = 18
SOFTWARE_VERSION = 343
CLIENT_API_VERSION = 4
SOFTWARE_VERSION = 344
CLIENT_API_VERSION = 5
UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )
@ -463,11 +463,12 @@ APPLICATION_RAR = 31
APPLICATION_7Z = 32
IMAGE_WEBP = 33
IMAGE_TIFF = 34
APPLICATION_PSD = 35
APPLICATION_OCTET_STREAM = 100
APPLICATION_UNKNOWN = 101
ALLOWED_MIMES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_APNG, IMAGE_GIF, IMAGE_BMP, IMAGE_WEBP, IMAGE_TIFF, APPLICATION_FLASH, VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG, APPLICATION_PDF, APPLICATION_ZIP, APPLICATION_RAR, APPLICATION_7Z, AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA, VIDEO_WMV, APPLICATION_HYDRUS_UPDATE_CONTENT, APPLICATION_HYDRUS_UPDATE_DEFINITIONS )
SEARCHABLE_MIMES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_APNG, IMAGE_GIF, IMAGE_WEBP, IMAGE_TIFF, APPLICATION_FLASH, VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG, APPLICATION_PDF, APPLICATION_ZIP, APPLICATION_RAR, APPLICATION_7Z, AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA, VIDEO_WMV )
ALLOWED_MIMES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_APNG, IMAGE_GIF, IMAGE_BMP, IMAGE_WEBP, IMAGE_TIFF, APPLICATION_FLASH, VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG, APPLICATION_PSD, APPLICATION_PDF, APPLICATION_ZIP, APPLICATION_RAR, APPLICATION_7Z, AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA, VIDEO_WMV, APPLICATION_HYDRUS_UPDATE_CONTENT, APPLICATION_HYDRUS_UPDATE_DEFINITIONS )
SEARCHABLE_MIMES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_APNG, IMAGE_GIF, IMAGE_WEBP, IMAGE_TIFF, APPLICATION_FLASH, VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG, APPLICATION_PSD, APPLICATION_PDF, APPLICATION_ZIP, APPLICATION_RAR, APPLICATION_7Z, AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA, VIDEO_WMV )
DECOMPRESSION_BOMB_IMAGES = ( IMAGE_JPEG, IMAGE_PNG )
@ -479,7 +480,7 @@ VIDEO = ( VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_WMV, VIDEO_MKV, VIDE
NATIVE_VIDEO = ( IMAGE_APNG, VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_WMV, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG )
APPLICATIONS = ( APPLICATION_FLASH, APPLICATION_PDF, APPLICATION_ZIP, APPLICATION_RAR, APPLICATION_7Z )
APPLICATIONS = ( APPLICATION_FLASH, APPLICATION_PSD, APPLICATION_PDF, APPLICATION_ZIP, APPLICATION_RAR, APPLICATION_7Z )
NOISY_MIMES = tuple( [ APPLICATION_FLASH ] + list( AUDIO ) + list( VIDEO ) )
@ -509,6 +510,8 @@ mime_enum_lookup[ 'image/tiff' ] = IMAGE_TIFF
mime_enum_lookup[ 'image' ] = IMAGES
mime_enum_lookup[ 'image/vnd.microsoft.icon' ] = IMAGE_ICON
mime_enum_lookup[ 'application/x-shockwave-flash' ] = APPLICATION_FLASH
mime_enum_lookup[ 'application/x-photoshop' ] = APPLICATION_PSD
mime_enum_lookup[ 'image/vnd.adobe.photoshop' ] = APPLICATION_PSD
mime_enum_lookup[ 'application/octet-stream' ] = APPLICATION_OCTET_STREAM
mime_enum_lookup[ 'application/x-yaml' ] = APPLICATION_YAML
mime_enum_lookup[ 'PDF document' ] = APPLICATION_PDF
@ -555,6 +558,7 @@ mime_string_lookup[ APPLICATION_OCTET_STREAM ] = 'application/octet-stream'
mime_string_lookup[ APPLICATION_YAML ] = 'application/x-yaml'
mime_string_lookup[ APPLICATION_JSON ] = 'application/json'
mime_string_lookup[ APPLICATION_PDF ] = 'application/pdf'
mime_string_lookup[ APPLICATION_PSD ] = 'application/x-photoshop'
mime_string_lookup[ APPLICATION_ZIP ] = 'application/zip'
mime_string_lookup[ APPLICATION_RAR ] = 'application/vnd.rar'
mime_string_lookup[ APPLICATION_7Z ] = 'application/x-7z-compressed'
@ -597,6 +601,7 @@ mime_ext_lookup[ APPLICATION_OCTET_STREAM ] = '.bin'
mime_ext_lookup[ APPLICATION_YAML ] = '.yaml'
mime_ext_lookup[ APPLICATION_JSON ] = '.json'
mime_ext_lookup[ APPLICATION_PDF ] = '.pdf'
mime_ext_lookup[ APPLICATION_PSD ] = '.psd'
mime_ext_lookup[ APPLICATION_ZIP ] = '.zip'
mime_ext_lookup[ APPLICATION_RAR ] = '.rar'
mime_ext_lookup[ APPLICATION_7Z ] = '.7z'

19
include/HydrusController.py
View File

@ -18,22 +18,13 @@ import traceback
class HydrusController( object ):
def __init__( self, db_dir, no_daemons, no_wal ):
def __init__( self, db_dir ):
HG.controller = self
self._name = 'hydrus'
self.db_dir = db_dir
self._no_daemons = no_daemons
self._no_wal = no_wal
self._no_wal_path = os.path.join( self.db_dir, 'no-wal' )
if os.path.exists( self._no_wal_path ):
self._no_wal = True
self.db = None
@ -355,14 +346,6 @@ class HydrusController( object ):
for cache in list(self._caches.values()): cache.Clear()
def CreateNoWALFile( self ):
with open( self._no_wal_path, 'w', encoding = 'utf-8' ) as f:
f.write( 'This file was created because the database failed to set WAL journalling. It will not reattempt WAL as long as this file exists.' )
def CurrentlyIdle( self ):
return True

40
include/HydrusDB.py
View File

@ -132,7 +132,7 @@ class HydrusDB( object ):
TRANSACTION_COMMIT_TIME = 10
def __init__( self, controller, db_dir, db_name, no_wal = False ):
def __init__( self, controller, db_dir, db_name ):
if HydrusPaths.GetFreeSpace( db_dir ) < 500 * 1048576:
@ -142,7 +142,6 @@ class HydrusDB( object ):
self._controller = controller
self._db_dir = db_dir
self._db_name = db_name
self._no_wal = no_wal
self._transaction_started = 0
self._in_transaction = False
@ -473,7 +472,7 @@ class HydrusDB( object ):
self._c.execute( 'PRAGMA ' + db_name + '.cache_size = -10000;' )
if self._no_wal:
if HG.no_wal:
self._c.execute( 'PRAGMA ' + db_name + '.journal_mode = TRUNCATE;' )
@ -494,38 +493,15 @@ class HydrusDB( object ):
self._c.execute( 'SELECT * FROM ' + db_name + '.sqlite_master;' ).fetchone()
except sqlite3.OperationalError:
except sqlite3.OperationalError as e:
HydrusData.DebugPrint( traceback.format_exc() )
message = 'The database failed to read some data. You may need to run the program in no-wal mode using the --no_wal command parameter. Full error information:'
message += os.linesep * 2
message += str( e )
def create_no_wal_file():
HG.controller.CreateNoWALFile()
self._no_wal = True
HydrusData.DebugPrint( message )
if db_just_created:
del self._c
del self._db
os.remove( db_path )
create_no_wal_file()
self._InitDBCursor()
else:
self._c.execute( 'PRAGMA ' + db_name + '.journal_mode = TRUNCATE;' )
self._c.execute( 'PRAGMA ' + db_name + '.synchronous = 2;' )
self._c.execute( 'SELECT * FROM ' + db_name + '.sqlite_master;' ).fetchone()
create_no_wal_file()
raise HydrusExceptions.DBAccessException( message )

58
include/HydrusData.py
View File

@ -419,6 +419,64 @@ def ConvertValueRangeToPrettyString( value, range ):
return ToHumanInt( value ) + '/' + ToHumanInt( range )
def ConvertValueRangeToScanbarTimestampsMS( value_ms, range_ms ):
value_ms = int( round( value_ms ) )
range_hours = range_ms // 3600000
value_hours = value_ms // 3600000
range_minutes = ( range_ms % 3600000 ) // 60000
value_minutes = ( value_ms % 3600000 ) // 60000
range_seconds = ( range_ms % 60000 ) // 1000
value_seconds = ( value_ms % 60000 ) // 1000
range_ms = range_ms % 1000
value_ms = value_ms % 1000
if range_hours > 0:
# 0:01:23.033/1:12:57.067
time_phrase = '{}:{:0>2}:{:0>2}.{:0>3}'
args = ( value_hours, value_minutes, value_seconds, value_ms, range_hours, range_minutes, range_seconds, range_ms )
elif range_minutes > 0:
# 01:23.033/12:57.067 or 0:23.033/1:57.067
if range_minutes > 9:
time_phrase = '{:0>2}:{:0>2}.{:0>3}'
else:
time_phrase = '{:0>1}:{:0>2}.{:0>3}'
args = ( value_minutes, value_seconds, value_ms, range_minutes, range_seconds, range_ms )
else:
# 23.033/57.067 or 3.033/7.067 or 0.033/0.067
if range_seconds > 9:
time_phrase = '{:0>2}.{:0>3}'
else:
time_phrase = '{:0>1}.{:0>3}'
args = ( value_seconds, value_ms, range_seconds, range_ms )
full_phrase = '{}/{}'.format( time_phrase, time_phrase )
result = full_phrase.format( *args )
return result
def DebugPrint( debug_info ):
Print( debug_info )

6
include/HydrusFileHandling.py
View File

@ -31,6 +31,8 @@ header_and_mime = [
( 0, b'ZWS', HC.APPLICATION_FLASH ),
( 0, b'FLV', HC.VIDEO_FLV ),
( 0, b'%PDF', HC.APPLICATION_PDF ),
( 0, b'8BPS\x00\x01', HC.APPLICATION_PSD ),
( 0, b'8BPS\x00\x02', HC.APPLICATION_PSD ), # PSB, which is basically PSD v2 and does giganto resolution
( 0, b'PK\x03\x04', HC.APPLICATION_ZIP ),
( 0, b'PK\x05\x06', HC.APPLICATION_ZIP ),
( 0, b'PK\x07\x08', HC.APPLICATION_ZIP ),
@ -238,6 +240,10 @@ def GetFileInfo( path, mime = None ):
num_words = HydrusDocumentHandling.GetPDFNumWords( path ) # this now give None until a better solution can be found
elif mime == HC.APPLICATION_PSD:
( width, height ) = HydrusImageHandling.GetPSDResolution( path )
elif mime in HC.AUDIO:
ffmpeg_lines = HydrusVideoHandling.GetFFMPEGInfoLines( path )

2
include/HydrusGlobals.py
View File

@ -7,6 +7,8 @@ test_controller = None
view_shutdown = False
model_shutdown = False
no_daemons = False
no_wal = False
no_db_temp_files = False
import_folders_running = False

28
include/HydrusImageHandling.py
View File

@ -234,8 +234,14 @@ def GetGIFFrameDurations( path ):
while True:
try: pil_image.seek( i )
except: break
try:
pil_image.seek( i )
except:
break
if 'duration' not in pil_image.info:
@ -277,6 +283,21 @@ def GetImageProperties( path, mime ):
return ( ( width, height ), duration, num_frames )
def GetPSDResolution( path ):
with open( path, 'rb' ) as f:
f.seek( 14 )
height_bytes = f.read( 4 )
width_bytes = f.read( 4 )
height = struct.unpack( '>L', height_bytes )[0]
width = struct.unpack( '>L', width_bytes )[0]
return ( width, height )
def GetResolutionAndNumFrames( path, mime ):
pil_image = GeneratePILImage( path )
@ -343,7 +364,8 @@ def GetThumbnailResolution( image_resolution, target_resolution ):
def IsDecompressionBomb( path ):
PILImage.MAX_IMAGE_PIXELS = OLD_PIL_MAX_IMAGE_PIXELS
# I boosted this up x2 as a temp test
PILImage.MAX_IMAGE_PIXELS = OLD_PIL_MAX_IMAGE_PIXELS * 2
warnings.simplefilter( 'error', PILImage.DecompressionBombWarning )

9
include/HydrusVideoHandling.py
View File

@ -113,7 +113,14 @@ def GetFFMPEGInfoLines( path, count_frames_manually = False ):
sbp_kwargs = HydrusData.GetSubprocessKWArgs()
proc = subprocess.Popen( cmd, bufsize = 10**5, stdout = subprocess.PIPE, stderr = subprocess.PIPE, **sbp_kwargs )
try:
proc = subprocess.Popen( cmd, bufsize = 10**5, stdout = subprocess.PIPE, stderr = subprocess.PIPE, **sbp_kwargs )
except FileNotFoundError as e:
raise FileNotFoundError( 'FFMPEG not found--are you sure it is installed? Full error: ' + str( e ) )
data_bytes = proc.stderr.read()

6
include/ServerController.py
View File

@ -149,9 +149,9 @@ def ShutdownSiblingInstance( db_dir ):
class Controller( HydrusController.HydrusController ):
def __init__( self, db_dir, no_daemons, no_wal ):
def __init__( self, db_dir ):
HydrusController.HydrusController.__init__( self, db_dir, no_daemons, no_wal )
HydrusController.HydrusController.__init__( self, db_dir )
self._name = 'server'
@ -167,7 +167,7 @@ class Controller( HydrusController.HydrusController ):
def _InitDB( self ):
return ServerDB.DB( self, self.db_dir, 'server', no_wal = self._no_wal )
return ServerDB.DB( self, self.db_dir, 'server' )
def StartService( self, service ):

4
include/ServerDB.py
View File

@ -91,7 +91,7 @@ class DB( HydrusDB.HydrusDB ):
TRANSACTION_COMMIT_TIME = 120
def __init__( self, controller, db_dir, db_name, no_wal = False ):
def __init__( self, controller, db_dir, db_name ):
self._files_dir = os.path.join( db_dir, 'server_files' )
@ -103,7 +103,7 @@ class DB( HydrusDB.HydrusDB ):
self._account_type_cache = {}
HydrusDB.HydrusDB.__init__( self, controller, db_dir, db_name, no_wal = no_wal )
HydrusDB.HydrusDB.__init__( self, controller, db_dir, db_name )
def _AddAccountType( self, service_id, account_type ):

10
include/TestClientAPI.py
View File

@ -650,7 +650,7 @@ class TestClientAPI( unittest.TestCase ):
self.assertEqual( response_json[ 'human_result_text' ], '"https://8ch.net/tv/res/1846574.html" URL added successfully.' )
self.assertEqual( response_json[ 'normalised_url' ], 'https://8ch.net/tv/res/1846574.html' )
self.assertEqual( HG.test_controller.GetWrite( 'import_url_test' ), [ ( ( url, None, None ), {} ) ] )
self.assertEqual( HG.test_controller.GetWrite( 'import_url_test' ), [ ( ( url, None, None, False ), {} ) ] )
# with name
@ -675,13 +675,13 @@ class TestClientAPI( unittest.TestCase ):
self.assertEqual( response_json[ 'human_result_text' ], '"https://8ch.net/tv/res/1846574.html" URL added successfully.' )
self.assertEqual( response_json[ 'normalised_url' ], 'https://8ch.net/tv/res/1846574.html' )
self.assertEqual( HG.test_controller.GetWrite( 'import_url_test' ), [ ( ( url, None, 'muh /tv/' ), {} ) ] )
self.assertEqual( HG.test_controller.GetWrite( 'import_url_test' ), [ ( ( url, None, 'muh /tv/', False ), {} ) ] )
# add tags and name
# add tags and name, and show destination page
HG.test_controller.ClearWrites( 'import_url_test' )
request_dict = { 'url' : url, 'destination_page_name' : 'muh /tv/', 'service_names_to_tags' : { 'local tags' : [ '/tv/ thread' ] } }
request_dict = { 'url' : url, 'destination_page_name' : 'muh /tv/', 'show_destination_page' : True, 'service_names_to_tags' : { 'local tags' : [ '/tv/ thread' ] } }
request_body = json.dumps( request_dict )
@ -702,7 +702,7 @@ class TestClientAPI( unittest.TestCase ):
service_keys_to_tags = ClientTags.ServiceKeysToTags( { CC.LOCAL_TAG_SERVICE_KEY : set( [ '/tv/ thread' ] ) } )
self.assertEqual( HG.test_controller.GetWrite( 'import_url_test' ), [ ( ( url, service_keys_to_tags, 'muh /tv/' ), {} ) ] )
self.assertEqual( HG.test_controller.GetWrite( 'import_url_test' ), [ ( ( url, service_keys_to_tags, 'muh /tv/', True ), {} ) ] )
# associate url

4
include/TestController.py
View File

@ -494,13 +494,13 @@ class Controller( object ):
return write
def ImportURLFromAPI( self, url, service_keys_to_tags, destination_page_name ):
def ImportURLFromAPI( self, url, service_keys_to_tags, destination_page_name, show_destination_page ):
normalised_url = self.network_engine.domain_manager.NormaliseURL( url )
human_result_text = '"{}" URL added successfully.'.format( normalised_url )
self.Write( 'import_url_test', url, service_keys_to_tags, destination_page_name )
self.Write( 'import_url_test', url, service_keys_to_tags, destination_page_name, show_destination_page )
return ( normalised_url, human_result_text )

31
include/TestDB.py
View File

@ -477,13 +477,12 @@ class TestClientDB( unittest.TestCase ):
#
like_rating_service_key = HydrusData.GenerateKey()
numerical_rating_service_key = HydrusData.GenerateKey()
from . import TestController
services = self._read( 'services' )
services.append( ClientServices.GenerateService( like_rating_service_key, HC.LOCAL_RATING_LIKE, 'test like rating service' ) )
services.append( ClientServices.GenerateService( numerical_rating_service_key, HC.LOCAL_RATING_NUMERICAL, 'test numerical rating service' ) )
services.append( ClientServices.GenerateService( TestController.LOCAL_RATING_LIKE_SERVICE_KEY, HC.LOCAL_RATING_LIKE, 'test like rating service' ) )
services.append( ClientServices.GenerateService( TestController.LOCAL_RATING_NUMERICAL_SERVICE_KEY, HC.LOCAL_RATING_NUMERICAL, 'test numerical rating service' ) )
self._write( 'update_services', services )
@ -493,7 +492,7 @@ class TestClientDB( unittest.TestCase ):
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_RATINGS, HC.CONTENT_UPDATE_ADD, ( 1.0, ( hash, ) ) ) )
service_keys_to_content_updates[ like_rating_service_key ] = content_updates
service_keys_to_content_updates[ TestController.LOCAL_RATING_LIKE_SERVICE_KEY ] = content_updates
self._write( 'content_updates', service_keys_to_content_updates )
@ -503,23 +502,23 @@ class TestClientDB( unittest.TestCase ):
content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_RATINGS, HC.CONTENT_UPDATE_ADD, ( 0.6, ( hash, ) ) ) )
service_keys_to_content_updates[ numerical_rating_service_key ] = content_updates
service_keys_to_content_updates[ TestController.LOCAL_RATING_NUMERICAL_SERVICE_KEY ] = content_updates
self._write( 'content_updates', service_keys_to_content_updates )
tests = []
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 1.0, like_rating_service_key ), 1 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 0.0, like_rating_service_key ), 0 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 'rated', like_rating_service_key ), 1 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 'not rated', like_rating_service_key ), 0 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 1.0, TestController.LOCAL_RATING_LIKE_SERVICE_KEY ), 1 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 0.0, TestController.LOCAL_RATING_LIKE_SERVICE_KEY ), 0 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 'rated', TestController.LOCAL_RATING_LIKE_SERVICE_KEY ), 1 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 'not rated', TestController.LOCAL_RATING_LIKE_SERVICE_KEY ), 0 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 0.6, numerical_rating_service_key ), 1 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 1.0, numerical_rating_service_key ), 0 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '>', 0.6, numerical_rating_service_key ), 0 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '>', 0.4, numerical_rating_service_key ), 1 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 'rated', numerical_rating_service_key ), 1 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 'not rated', numerical_rating_service_key ), 0 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 0.6, TestController.LOCAL_RATING_NUMERICAL_SERVICE_KEY ), 1 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 1.0, TestController.LOCAL_RATING_NUMERICAL_SERVICE_KEY ), 0 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '>', 0.6, TestController.LOCAL_RATING_NUMERICAL_SERVICE_KEY ), 0 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '>', 0.4, TestController.LOCAL_RATING_NUMERICAL_SERVICE_KEY ), 1 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 'rated', TestController.LOCAL_RATING_NUMERICAL_SERVICE_KEY ), 1 ) )
tests.append( ( HC.PREDICATE_TYPE_SYSTEM_RATING, ( '=', 'not rated', TestController.LOCAL_RATING_NUMERICAL_SERVICE_KEY ), 0 ) )
run_system_predicate_tests( tests )

25
server.py
View File

@ -44,6 +44,7 @@ try:
argparser.add_argument( '--no_daemons', action='store_true', help = 'run without background daemons' )
argparser.add_argument( '--no_wal', action='store_true', help = 'run without WAL db journalling' )
argparser.add_argument( '--no_db_temp_files', action='store_true', help = 'run the db entirely in memory' )
argparser.add_argument( '--temp_dir', help = 'override the program\'s temporary directory' )
result = argparser.parse_args()
@ -75,10 +76,28 @@ try:
raise Exception( 'Could not ensure db path ' + db_dir + ' exists! Check the location is correct and that you have permission to write to it!' )
no_daemons = result.no_daemons
no_wal = result.no_wal
HG.no_daemons = result.no_daemons
HG.no_wal = result.no_wal
HG.no_db_temp_files = result.no_db_temp_files
if result.temp_dir is not None:
if not os.path.exists( result.temp_dir ):
raise Exception( 'The given temp directory, "{}", does not exist!'.format( result.temp_dir ) )
if HC.PLATFORM_WINDOWS:
os.environ[ 'TEMP' ] = result.temp_dir
os.environ[ 'TMP' ] = result.temp_dir
else:
os.environ[ 'TMPDIR' ] = result.temp_dir
#
action = ServerController.ProcessStartingAction( db_dir, action )
@ -98,7 +117,7 @@ try:
threading.Thread( target = reactor.run, name = 'twisted', kwargs = { 'installSignalHandlers' : 0 } ).start()
controller = ServerController.Controller( db_dir, no_daemons, no_wal )
controller = ServerController.Controller( db_dir )
controller.Run()

BIN
static/psd.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 18 KiB