RepoMirrors
/
hydrus
mirror of https://github.com/hydrusnetwork/hydrus.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Version 216

This commit is contained in:
Hydrus Network Developer 2016-07-27 16:53:34 -05:00
parent 634cc57f02
commit 1698dfdee1
24 changed files with 925 additions and 505 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
help/changelog.html
View File

@ -8,6 +8,30 @@
<div class="content">
<h3>changelog</h3>
<ul>
<li><h3>version 216</h3></li>
<ul>
<li>video rendering pipeline rewritten to be much smoother</li>
<li>canvas video render timing rewritten to be more frame accurate and smoother</li>
<li>video rendering pipeline will deal with 100% CPU rendering bottlenecks much better</li>
<li>videocontainers will rush to render towards clicked cursor and then slow down</li>
<li>videocontainers will stop rendering as soon as they are replaced on zoom</li>
<li>videocontainers will stop rendering as soon as their parent canvas is closed</li>
<li>added a video buffer size option to options->speed and memory</li>
<li>client_files subfolders are now split into f, t, and r groups for files, full size thumbnails, and resized thumbnails</li>
<li>added an option to override resized thumbnail storage location under options->file storage locations</li>
<li>added secondary sort option to options->sort/collect</li>
<li>fixed a bug where automatically pended tags could exist on top of current tags, forming redundant (1) (+1) situations</li>
<li>fixed a bug when sorting by namespace and a respective namespaced tag has a sibling that de-namespaces it</li>
<li>relatedly, namespace sorting will now filter correctly by collapsed siblings when those sibling pairs' namespaces differ</li>
<li>and collecting will now collect correctly with sibling pairs where the pairs' namespaces differ</li>
<li>simplified how the server caches its account sessions</li>
<li>improved how the server refreshes specific account sessions</li>
<li>the server now refreshes cached session accounts immediately when an admin actions an account type</li>
<li>neatened a little server db account fetching code</li>
<li>fixed file repository petition denial</li>
<li>added a better catch for upnp addmapping error when the mapping already exists as a port forward</li>
<li>the media canvas will no longer accept any mouse or key event if a menu is open anywhere in the program (this circumstance is difficult to pull off, but does cause a crash)</li>
</ul>
<li><h3>version 215</h3></li>
<ul>
<li>you can now set different suggested favourite tags for different tag services</li>

493
include/ClientCaches.py
View File

@ -200,55 +200,146 @@ class ClientFilesManager( object ):
self._Reinit()
def _GenerateExpectedFilePath( self, location, hash, mime ):
def _GenerateExpectedFilePath( self, hash, mime ):
hash_encoded = hash.encode( 'hex' )
prefix = hash_encoded[:2]
prefix = 'f' + hash_encoded[:2]
return os.path.join( location, prefix, hash_encoded + HC.mime_ext_lookup[ mime ] )
location = self._prefixes_to_locations[ prefix ]
def _GenerateExpectedThumbnailPath( self, location, hash, full_size ):
hash_encoded = hash.encode( 'hex' )
first_two_chars = hash_encoded[:2]
path = os.path.join( location, first_two_chars, hash_encoded ) + '.thumbnail'
if not full_size:
path += '.resized'
path = os.path.join( location, prefix, hash_encoded + HC.mime_ext_lookup[ mime ] )
return path
def _GetLocation( self, hash ):
def _GenerateExpectedFullSizeThumbnailPath( self, hash ):
hash_encoded = hash.encode( 'hex' )
prefix = hash_encoded[:2]
prefix = 't' + hash_encoded[:2]
location = self._prefixes_to_locations[ prefix ]
return location
path = os.path.join( location, prefix, hash_encoded ) + '.thumbnail'
return path
def _GenerateExpectedResizedThumbnailPath( self, hash ):
hash_encoded = hash.encode( 'hex' )
prefix = 'r' + hash_encoded[:2]
location = self._prefixes_to_locations[ prefix ]
path = os.path.join( location, prefix, hash_encoded ) + '.thumbnail.resized'
return path
def _GenerateFullSizeThumbnail( self, hash ):
try:
file_path = self._LookForFilePath( hash )
except HydrusExceptions.FileMissingException:
raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was missing. It could not be regenerated because the original file was also missing. This event could indicate hard drive corruption or an unplugged external drive. Please check everything is ok.' )
try:
thumbnail = HydrusFileHandling.GenerateThumbnail( file_path )
except Exception as e:
HydrusData.ShowException( e )
raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was missing. It could not be regenerated from the original file for the above reason. This event could indicate hard drive corruption. Please check everything is ok.' )
full_size_path = self._GenerateExpectedFullSizeThumbnailPath( hash )
try:
with open( full_size_path, 'wb' ) as f:
f.write( thumbnail )
except Exception as e:
HydrusData.ShowException( e )
raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was missing. It was regenerated from the original file, but hydrus could not write it to the location ' + full_size_path + ' for the above reason. This event could indicate hard drive corruption, and it also suggests that hydrus does not have permission to write to its thumbnail folder. Please check everything is ok.' )
def _GenerateResizedThumbnail( self, hash ):
full_size_path = self._GenerateExpectedFullSizeThumbnailPath( hash )
options = self._controller.GetOptions()
thumbnail_dimensions = options[ 'thumbnail_dimensions' ]
try:
thumbnail_resized = HydrusFileHandling.GenerateThumbnail( full_size_path, thumbnail_dimensions )
except:
try:
HydrusPaths.DeletePath( full_size_path )
except:
raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was found, but it would not render. An attempt to delete it was made, but that failed as well. This event could indicate hard drive corruption, and it also suggests that hydrus does not have permission to write to its thumbnail folder. Please check everything is ok.' )
self._GenerateFullSizeThumbnail( hash )
thumbnail_resized = HydrusFileHandling.GenerateThumbnail( full_size_path, thumbnail_dimensions )
resized_path = self._GenerateExpectedResizedThumbnailPath( hash )
try:
with open( resized_path, 'wb' ) as f:
f.write( thumbnail_resized )
except Exception as e:
HydrusData.ShowException( e )
raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was found, but the resized version would not save to disk. This event suggests that hydrus does not have permission to write to its thumbnail folder. Please check everything is ok.' )
def _GetRecoverTuple( self ):
paths = { path for path in self._prefixes_to_locations.values() }
all_locations = { location for location in self._prefixes_to_locations.values() }
for path in paths:
all_prefixes = self._prefixes_to_locations.keys()
for possible_location in all_locations:
for prefix in HydrusData.IterateHexPrefixes():
for prefix in all_prefixes:
correct_path = self._prefixes_to_locations[ prefix ]
correct_location = self._prefixes_to_locations[ prefix ]
if path != correct_path and os.path.exists( os.path.join( path, prefix ) ):
if possible_location != correct_location and os.path.exists( os.path.join( possible_location, prefix ) ):
return ( prefix, path, correct_path )
recoverable_location = possible_location
return ( prefix, recoverable_location, correct_location )
@ -258,94 +349,141 @@ class ClientFilesManager( object ):
def _GetRebalanceTuple( self ):
paths_to_ideal_weights = self._controller.GetNewOptions().GetClientFilesLocationsToIdealWeights()
( locations_to_ideal_weights, resized_thumbnail_override ) = self._controller.GetNewOptions().GetClientFilesLocationsToIdealWeights()
total_weight = sum( paths_to_ideal_weights.values() )
total_weight = sum( locations_to_ideal_weights.values() )
paths_to_normalised_ideal_weights = { path : weight / total_weight for ( path, weight ) in paths_to_ideal_weights.items() }
ideal_locations_to_normalised_weights = { location : weight / total_weight for ( location, weight ) in locations_to_ideal_weights.items() }
current_paths_to_normalised_weights = collections.defaultdict( lambda: 0 )
current_locations_to_normalised_weights = collections.defaultdict( lambda: 0 )
for ( prefix, path ) in self._prefixes_to_locations.items():
file_prefixes = [ prefix for prefix in self._prefixes_to_locations if prefix.startswith( 'f' ) ]
for file_prefix in file_prefixes:
current_paths_to_normalised_weights[ path ] += 1.0 / 256
location = self._prefixes_to_locations[ file_prefix ]
current_locations_to_normalised_weights[ location ] += 1.0 / 256
for path in current_paths_to_normalised_weights.keys():
for location in current_locations_to_normalised_weights.keys():
if path not in paths_to_normalised_ideal_weights:
if location not in ideal_locations_to_normalised_weights:
paths_to_normalised_ideal_weights[ path ] = 0.0
ideal_locations_to_normalised_weights[ location ] = 0.0
#
overweight_paths = []
underweight_paths = []
overweight_locations = []
underweight_locations = []
for ( path, ideal_weight ) in paths_to_normalised_ideal_weights.items():
for ( location, ideal_weight ) in ideal_locations_to_normalised_weights.items():
if path in current_paths_to_normalised_weights:
if location in current_locations_to_normalised_weights:
current_weight = current_paths_to_normalised_weights[ path ]
current_weight = current_locations_to_normalised_weights[ location ]
if current_weight < ideal_weight:
underweight_paths.append( path )
underweight_locations.append( location )
elif current_weight >= ideal_weight + 1.0 / 256:
overweight_paths.append( path )
overweight_locations.append( location )
else:
underweight_paths.append( path )
underweight_locations.append( location )
#
if len( underweight_paths ) == 0 or len( overweight_paths ) == 0:
if len( underweight_locations ) > 0 and len( overweight_locations ) > 0:
return None
overweight_location = overweight_locations.pop( 0 )
underweight_location = underweight_locations.pop( 0 )
random.shuffle( file_prefixes )
for file_prefix in file_prefixes:
location = self._prefixes_to_locations[ file_prefix ]
if location == overweight_location:
return ( file_prefix, overweight_location, underweight_location )
else:
overweight_path = overweight_paths.pop( 0 )
underweight_path = underweight_paths.pop( 0 )
prefixes_and_paths = self._prefixes_to_locations.items()
random.shuffle( prefixes_and_paths )
for ( prefix, path ) in prefixes_and_paths:
for hex_prefix in HydrusData.IterateHexPrefixes():
if path == overweight_path:
full_size_prefix = 't' + hex_prefix
file_prefix = 'f' + hex_prefix
full_size_location = self._prefixes_to_locations[ full_size_prefix ]
file_location = self._prefixes_to_locations[ file_prefix ]
if full_size_location != file_location:
return ( prefix, overweight_path, underweight_path )
return ( full_size_prefix, full_size_location, file_location )
if resized_thumbnail_override is None:
for hex_prefix in HydrusData.IterateHexPrefixes():
resized_prefix = 'r' + hex_prefix
file_prefix = 'f' + hex_prefix
resized_location = self._prefixes_to_locations[ resized_prefix ]
file_location = self._prefixes_to_locations[ file_prefix ]
if resized_location != file_location:
return ( resized_prefix, resized_location, file_location )
else:
for hex_prefix in HydrusData.IterateHexPrefixes():
resized_prefix = 'r' + hex_prefix
resized_location = self._prefixes_to_locations[ resized_prefix ]
if resized_location != resized_thumbnail_override:
return ( resized_prefix, resized_location, resized_thumbnail_override )
return None
def _IterateAllFilePaths( self ):
for ( prefix, location ) in self._prefixes_to_locations.items():
dir = os.path.join( location, prefix )
filenames = os.listdir( dir )
for filename in filenames:
if prefix.startswith( 'f' ):
if filename.endswith( '.thumbnail' ) or filename.endswith( '.thumbnail.resized' ):
continue
dir = os.path.join( location, prefix )
yield os.path.join( dir, filename )
filenames = os.listdir( dir )
for filename in filenames:
yield os.path.join( dir, filename )
@ -354,13 +492,13 @@ class ClientFilesManager( object ):
for ( prefix, location ) in self._prefixes_to_locations.items():
dir = os.path.join( location, prefix )
filenames = os.listdir( dir )
for filename in filenames:
if prefix.startswith( 't' ) or prefix.startswith( 'r' ):
if filename.endswith( '.thumbnail' ) or filename.endswith( '.thumbnail.resized' ):
dir = os.path.join( location, prefix )
filenames = os.listdir( dir )
for filename in filenames:
yield os.path.join( dir, filename )
@ -368,11 +506,11 @@ class ClientFilesManager( object ):
def _LookForFilePath( self, location, hash ):
def _LookForFilePath( self, hash ):
for potential_mime in HC.ALLOWED_MIMES:
potential_path = self._GenerateExpectedFilePath( location, hash, potential_mime )
potential_path = self._GenerateExpectedFilePath( hash, potential_mime )
if os.path.exists( potential_path ):
@ -380,7 +518,7 @@ class ClientFilesManager( object ):
raise HydrusExceptions.FileMissingException( 'File for ' + hash.encode( 'hex' ) + ' not found in directory ' + location + '!' )
raise HydrusExceptions.FileMissingException( 'File for ' + hash.encode( 'hex' ) + ' not found!' )
def _Reinit( self ):
@ -427,9 +565,7 @@ class ClientFilesManager( object ):
with self._lock:
location = self._GetLocation( hash )
dest_path = self._GenerateExpectedFilePath( location, hash, mime )
dest_path = self._GenerateExpectedFilePath( hash, mime )
if not os.path.exists( dest_path ):
@ -440,13 +576,11 @@ class ClientFilesManager( object ):
def AddThumbnail( self, hash, thumbnail ):
def AddFullSizeThumbnail( self, hash, thumbnail ):
with self._lock:
location = self._GetLocation( hash )
path = self._GenerateExpectedThumbnailPath( location, hash, True )
path = self._GenerateExpectedFullSizeThumbnailPath( hash )
with open( path, 'wb' ) as f:
@ -454,7 +588,7 @@ class ClientFilesManager( object ):
HydrusGlobals.client_controller.pub( 'new_thumbnails', { hash } )
self._controller.pub( 'new_thumbnails', { hash } )
def ClearOrphans( self, move_location = None ):
@ -662,11 +796,9 @@ class ClientFilesManager( object ):
for hash in hashes:
location = self._GetLocation( hash )
try:
path = self._LookForFilePath( location, hash )
path = self._LookForFilePath( hash )
except HydrusExceptions.FileMissingException:
@ -683,10 +815,8 @@ class ClientFilesManager( object ):
for hash in hashes:
location = self._GetLocation( hash )
path = self._GenerateExpectedThumbnailPath( location, hash, True )
resized_path = self._GenerateExpectedThumbnailPath( location, hash, False )
path = self._GenerateExpectedFullSizeThumbnailPath( hash )
resized_path = self._GenerateExpectedResizedThumbnailPath( hash )
HydrusPaths.DeletePath( path )
HydrusPaths.DeletePath( resized_path )
@ -698,15 +828,13 @@ class ClientFilesManager( object ):
with self._lock:
location = self._GetLocation( hash )
if mime is None:
path = self._LookForFilePath( location, hash )
path = self._LookForFilePath( hash )
else:
path = self._GenerateExpectedFilePath( location, hash, mime )
path = self._GenerateExpectedFilePath( hash, mime )
if not os.path.exists( path ):
@ -718,114 +846,21 @@ class ClientFilesManager( object ):
def _GenerateFullSizeThumbnail( self, location, hash ):
try:
file_path = self._LookForFilePath( location, hash )
except HydrusExceptions.FileMissingException:
raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was missing. It could not be regenerated because the original file was also missing. This event could indicate hard drive corruption or an unplugged external drive. Please check everything is ok.' )
try:
thumbnail = HydrusFileHandling.GenerateThumbnail( file_path )
except Exception as e:
HydrusData.ShowException( e )
raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was missing. It could not be regenerated from the original file for the above reason. This event could indicate hard drive corruption. Please check everything is ok.' )
full_size_path = self._GenerateExpectedThumbnailPath( location, hash, True )
try:
with open( full_size_path, 'wb' ) as f:
f.write( thumbnail )
except Exception as e:
HydrusData.ShowException( e )
raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was missing. It was regenerated from the original file, but hydrus could not write it to the location ' + full_size_path + ' for the above reason. This event could indicate hard drive corruption, and it also suggests that hydrus does not have permission to write to its thumbnail folder. Please check everything is ok.' )
def _GenerateResizedThumbnail( self, location, hash ):
full_size_path = self._GenerateExpectedThumbnailPath( location, hash, True )
options = HydrusGlobals.client_controller.GetOptions()
thumbnail_dimensions = options[ 'thumbnail_dimensions' ]
try:
thumbnail_resized = HydrusFileHandling.GenerateThumbnail( full_size_path, thumbnail_dimensions )
except:
try:
HydrusPaths.DeletePath( full_size_path )
except:
raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was found, but it would not render. An attempt to delete it was made, but that failed as well. This event could indicate hard drive corruption, and it also suggests that hydrus does not have permission to write to its thumbnail folder. Please check everything is ok.' )
self._GenerateFullSizeThumbnail( location, hash )
thumbnail_resized = HydrusFileHandling.GenerateThumbnail( full_size_path, thumbnail_dimensions )
resized_path = self._GenerateExpectedThumbnailPath( location, hash, False )
try:
with open( resized_path, 'wb' ) as f:
f.write( thumbnail_resized )
except Exception as e:
HydrusData.ShowException( e )
raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was found, but the resized version would not save to disk. This event suggests that hydrus does not have permission to write to its thumbnail folder. Please check everything is ok.' )
def GetThumbnailPath( self, hash, full_size ):
def GetFullSizeThumbnailPath( self, hash ):
with self._lock:
location = self._GetLocation( hash )
path = self._GenerateExpectedThumbnailPath( location, hash, full_size )
path = self._GenerateExpectedFullSizeThumbnailPath( hash )
if not os.path.exists( path ):
if full_size:
self._GenerateFullSizeThumbnail( hash )
if not self._bad_error_occured:
self._GenerateFullSizeThumbnail( location, hash )
self._bad_error_occured = True
if not self._bad_error_occured:
self._bad_error_occured = True
HydrusData.ShowText( 'A thumbnail for a file, ' + hash.encode( 'hex' ) + ', was missing. It has been regenerated from the original file, but this event could indicate hard drive corruption. Please check everything is ok. This error may be occuring for many files, but this message will only display once per boot. If you are recovering from a fractured database, you may wish to run \'database->maintenance->regenerate thumbnails\'.' )
else:
self._GenerateResizedThumbnail( location, hash )
HydrusData.ShowText( 'A thumbnail for a file, ' + hash.encode( 'hex' ) + ', was missing. It has been regenerated from the original file, but this event could indicate hard drive corruption. Please check everything is ok. This error may be occuring for many files, but this message will only display once per boot. If you are recovering from a fractured database, you may wish to run \'database->maintenance->regenerate thumbnails\'.' )
@ -833,13 +868,26 @@ class ClientFilesManager( object ):
def HaveThumbnail( self, hash ):
def GetResizedThumbnailPath( self, hash ):
with self._lock:
location = self._GetLocation( hash )
path = self._GenerateExpectedResizedThumbnailPath( hash )
path = self._GenerateExpectedThumbnailPath( location, hash, True )
if not os.path.exists( path ):
self._GenerateResizedThumbnail( hash )
return path
def HaveFullSizeThumbnail( self, hash ):
with self._lock:
path = self._GenerateExpectedFullSizeThumbnailPath( hash )
return os.path.exists( path )
@ -858,9 +906,9 @@ class ClientFilesManager( object ):
while rebalance_tuple is not None:
( prefix, overweight_path, underweight_path ) = rebalance_tuple
( prefix, overweight_location, underweight_location ) = rebalance_tuple
text = 'Moving \'' + prefix + '\' files from ' + overweight_path + ' to ' + underweight_path
text = 'Moving \'' + prefix + '\' from ' + overweight_location + ' to ' + underweight_location
if partial:
@ -872,7 +920,8 @@ class ClientFilesManager( object ):
HydrusData.ShowText( text )
self._controller.Write( 'relocate_client_files', prefix, overweight_path, underweight_path )
# these two lines can cause a deadlock because the db sometimes calls stuff in here.
self._controller.Write( 'relocate_client_files', prefix, overweight_location, underweight_location )
self._Reinit()
@ -893,9 +942,9 @@ class ClientFilesManager( object ):
while recover_tuple is not None:
( prefix, incorrect_path, correct_path ) = recover_tuple
( prefix, recoverable_location, correct_location ) = recover_tuple
text = 'Recovering \'' + prefix + '\' files from ' + incorrect_path + ' to ' + correct_path
text = 'Recovering \'' + prefix + '\' from ' + recoverable_location + ' to ' + correct_location
if partial:
@ -907,18 +956,10 @@ class ClientFilesManager( object ):
HydrusData.ShowText( text )
full_incorrect_path = os.path.join( incorrect_path, prefix )
full_correct_path = os.path.join( correct_path, prefix )
recoverable_path = os.path.join( recoverable_location, prefix )
correct_path = os.path.join( correct_location, prefix )
HydrusPaths.MoveAndMergeTree( full_incorrect_path, full_correct_path )
try: HydrusPaths.RecyclePath( full_incorrect_path )
except:
HydrusData.ShowText( 'After recovering some files, attempting to remove ' + full_incorrect_path + ' failed.' )
return
HydrusPaths.MoveAndMergeTree( recoverable_path, correct_path )
if partial:
@ -944,9 +985,7 @@ class ClientFilesManager( object ):
with self._lock:
location = self._GetLocation( hash )
self._GenerateResizedThumbnail( location, hash )
self._GenerateResizedThumbnail( hash )
@ -989,9 +1028,7 @@ class ClientFilesManager( object ):
hash = hash_encoded.decode( 'hex' )
location = self._GetLocation( hash )
full_size_path = self._GenerateExpectedThumbnailPath( location, hash, True )
full_size_path = self._GenerateExpectedFullSizeThumbnailPath( hash )
if only_do_missing and os.path.exists( full_size_path ):
@ -1002,9 +1039,9 @@ class ClientFilesManager( object ):
if mime in HC.MIMES_WITH_THUMBNAILS:
self._GenerateFullSizeThumbnail( location, hash )
self._GenerateFullSizeThumbnail( hash )
thumbnail_resized_path = self._GenerateExpectedThumbnailPath( location, hash, False )
thumbnail_resized_path = self._GenerateExpectedResizedThumbnailPath( hash )
if os.path.exists( thumbnail_resized_path ):
@ -1568,7 +1605,14 @@ class ThumbnailCache( object ):
try:
path = self._client_files_manager.GetThumbnailPath( hash, full_size )
if full_size:
path = self._client_files_manager.GetFullSizeThumbnailPath( hash )
else:
path = self._client_files_manager.GetResizedThumbnailPath( hash )
except HydrusExceptions.FileMissingException as e:
@ -1581,7 +1625,14 @@ class ThumbnailCache( object ):
try:
path = self._client_files_manager.GetThumbnailPath( hash, full_size )
if full_size:
path = self._client_files_manager.GetFullSizeThumbnailPath( hash )
else:
path = self._client_files_manager.GetResizedThumbnailPath( hash )
except HydrusExceptions.FileMissingException:

74
include/ClientDB.py
View File

@ -2134,7 +2134,9 @@ class DB( HydrusDB.HydrusDB ):
for prefix in HydrusData.IterateHexPrefixes():
self._c.execute( 'INSERT INTO client_files_locations ( prefix, location ) VALUES ( ?, ? );', ( prefix, location ) )
self._c.execute( 'INSERT INTO client_files_locations ( prefix, location ) VALUES ( ?, ? );', ( 'f' + prefix, location ) )
self._c.execute( 'INSERT INTO client_files_locations ( prefix, location ) VALUES ( ?, ? );', ( 't' + prefix, location ) )
self._c.execute( 'INSERT INTO client_files_locations ( prefix, location ) VALUES ( ?, ? );', ( 'r' + prefix, location ) )
init_service_info = []
@ -4906,7 +4908,7 @@ class DB( HydrusDB.HydrusDB ):
thumbnail = HydrusFileHandling.GenerateThumbnail( dest_path )
client_files_manager.AddThumbnail( hash, thumbnail )
client_files_manager.AddFullSizeThumbnail( hash, thumbnail )
if mime in ( HC.IMAGE_JPEG, HC.IMAGE_PNG ):
@ -7863,6 +7865,61 @@ class DB( HydrusDB.HydrusDB ):
self._c.execute( 'REPLACE INTO yaml_dumps VALUES ( ?, ?, ? );', ( YAML_DUMP_ID_REMOTE_BOORU, 'sankaku chan', ClientDefaults.GetDefaultBoorus()[ 'sankaku chan' ] ) )
if version == 215:
info = self._c.execute( 'SELECT prefix, location FROM client_files_locations;' ).fetchall()
self._c.execute( 'DELETE FROM client_files_locations;' )
for ( i, ( prefix, location ) ) in enumerate( info ):
self._c.execute( 'INSERT INTO client_files_locations ( prefix, location ) VALUES ( ?, ? );', ( 'f' + prefix, location ) )
self._c.execute( 'INSERT INTO client_files_locations ( prefix, location ) VALUES ( ?, ? );', ( 't' + prefix, location ) )
self._c.execute( 'INSERT INTO client_files_locations ( prefix, location ) VALUES ( ?, ? );', ( 'r' + prefix, location ) )
location = HydrusPaths.ConvertPortablePathToAbsPath( location )
text_prefix = 'rearranging client files: ' + HydrusData.ConvertValueRangeToPrettyString( i + 1, 256 ) + ', '
source = os.path.join( location, prefix )
file_dest = os.path.join( location, 'f' + prefix )
thumb_dest = os.path.join( location, 't' + prefix )
resized_dest = os.path.join( location, 'r' + prefix )
shutil.move( source, file_dest )
os.makedirs( thumb_dest )
os.makedirs( resized_dest )
filenames = os.listdir( file_dest )
num_to_do = len( filenames )
for ( j, filename ) in enumerate( filenames ):
if j % 100 == 0:
self._controller.pub( 'splash_set_status_text', text_prefix + HydrusData.ConvertValueRangeToPrettyString( j, num_to_do ) )
source_path = os.path.join( file_dest, filename )
if source_path.endswith( 'thumbnail' ):
dest_path = os.path.join( thumb_dest, filename )
shutil.move( source_path, dest_path )
elif source_path.endswith( 'resized' ):
dest_path = os.path.join( resized_dest, filename )
shutil.move( source_path, dest_path )
self._controller.pub( 'splash_set_title_text', 'updated db to v' + str( version + 1 ) )
self._c.execute( 'UPDATE version SET version = ?;', ( version + 1, ) )
@ -8052,6 +8109,19 @@ class DB( HydrusDB.HydrusDB ):
if len( pending_mappings_ids ) > 0:
culled_pending_mappings_ids = []
for ( namespace_id, tag_id, hash_ids ) in pending_mappings_ids:
existing_current_hash_ids = { hash_id for ( hash_id, ) in self._c.execute( 'SELECT hash_id FROM ' + current_mappings_table_name + ' WHERE namespace_id = ? AND tag_id = ?;', ( namespace_id, tag_id ) ) }
valid_hash_ids = set( hash_ids ).difference( existing_current_hash_ids )
culled_pending_mappings_ids.append( ( namespace_id, tag_id, valid_hash_ids ) )
pending_mappings_ids = culled_pending_mappings_ids
for ( namespace_id, tag_id, hash_ids ) in pending_mappings_ids:
self._c.executemany( 'INSERT OR IGNORE INTO ' + pending_mappings_table_name + ' VALUES ( ?, ?, ? );', [ ( namespace_id, tag_id, hash_id ) for hash_id in hash_ids ] )

49
include/ClientData.py
View File

@ -480,9 +480,16 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
#
self._dictionary[ 'integers' ] = {}
self._dictionary[ 'integers' ][ 'video_buffer_size_mb' ] = 96
#
client_files_default = os.path.join( HC.DB_DIR, 'client_files' )
self._dictionary[ 'client_files_locations_ideal_weights' ] = [ ( HydrusPaths.ConvertAbsPathToPortablePath( client_files_default ), 1.0 ) ]
self._dictionary[ 'client_files_locations_resized_thumbnail_override' ] = None
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
@ -522,16 +529,23 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
with self._lock:
result = {}
paths_to_weights = {}
for ( portable_path, weight ) in self._dictionary[ 'client_files_locations_ideal_weights' ]:
abs_path = HydrusPaths.ConvertPortablePathToAbsPath( portable_path )
result[ abs_path ] = weight
paths_to_weights[ abs_path ] = weight
return result
resized_thumbnail_override = self._dictionary[ 'client_files_locations_resized_thumbnail_override' ]
if resized_thumbnail_override is not None:
resized_thumbnail_override = HydrusPaths.ConvertPortablePathToAbsPath( resized_thumbnail_override )
return ( paths_to_weights, resized_thumbnail_override )
@ -621,6 +635,14 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
return self._dictionary[ 'frame_locations' ].items()
def GetInteger( self, name ):
with self._lock:
return self._dictionary[ 'integers' ][ name ]
def GetNoneableInteger( self, name ):
with self._lock:
@ -656,7 +678,7 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
def SetClientFilesLocationsToIdealWeights( self, locations_to_weights ):
def SetClientFilesLocationsToIdealWeights( self, locations_to_weights, resized_thumbnail_override ):
with self._lock:
@ -664,6 +686,13 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
self._dictionary[ 'client_files_locations_ideal_weights' ] = portable_locations_and_weights
if resized_thumbnail_override is not None:
resized_thumbnail_override = HydrusPaths.ConvertAbsPathToPortablePath( resized_thumbnail_override )
self._dictionary[ 'client_files_locations_resized_thumbnail_override' ] = resized_thumbnail_override
def SetDefaultImportTagOptions( self, gallery_identifier, import_tag_options ):
@ -679,6 +708,14 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
self._dictionary[ 'frame_locations' ][ frame_key ] = ( remember_size, remember_position, last_size, last_position, default_gravity, default_position, maximised, fullscreen )
def SetInteger( self, name, value ):
with self._lock:
self._dictionary[ 'integers' ][ name ] = value
def SetNoneableInteger( self, name, value ):
with self._lock:
@ -1901,7 +1938,7 @@ class ServiceRepository( ServiceRestricted ):
for hash in remote_thumbnail_hashes_i_should_have:
if not client_files_manager.HaveThumbnail( hash ):
if not client_files_manager.HaveFullSizeThumbnail( hash ):
thumbnail_hashes_i_need.add( hash )
@ -1932,7 +1969,7 @@ class ServiceRepository( ServiceRestricted ):
thumbnail = self.Request( HC.GET, 'thumbnail', request_args = request_args )
client_files_manager.AddThumbnail( hash, thumbnail )
client_files_manager.AddFullSizeThumbnail( hash, thumbnail )
job_key.DeleteVariable( 'popup_gauge_1' )

3
include/ClientDefaults.py
View File

@ -10,7 +10,8 @@ def GetClientDefaultOptions():
options = {}
options[ 'play_dumper_noises' ] = True
options[ 'default_sort' ] = 0
options[ 'default_sort' ] = 0 # smallest
options[ 'sort_fallback' ] = 4 # newest
options[ 'default_collect' ] = None
options[ 'export_path' ] = None
options[ 'hpos' ] = 400

2
include/ClientGUI.py
View File

@ -1598,7 +1598,7 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
def _RebalanceClientFiles( self ):
text = 'This will move your files around your storage directories until they satisfy the weights you have set in the options. It will also recover and folders that are in the wrong place. Use this if you have recently changed your file storage locations and want to hurry any transfers you have set up, or if you are recovering a complicated backup.'
text = 'This will move your files around your storage directories until they satisfy the weights you have set in the options. It will also recover any folders that are in the wrong place. Use this if you have recently changed your file storage locations and want to hurry any transfers you have set up, or if you are recovering a complicated backup.'
text += os.linesep * 2
text += 'The operation will lock file access and the database. Popup messages will report its progress.'

81
include/ClientGUICanvas.py
View File

@ -120,16 +120,15 @@ def ShouldHaveAnimationBar( media ):
class Animation( wx.Window ):
TIMER_MS = 5
def __init__( self, parent, media, initial_size, initial_position, start_paused ):
wx.Window.__init__( self, parent, size = initial_size, pos = initial_position )
self.SetDoubleBuffered( True )
( initial_width, initial_height ) = initial_size
self._media = media
self._video_container = ClientRendering.RasterContainerVideo( self._media, initial_size )
self._animation_bar = None
@ -143,11 +142,13 @@ class Animation( wx.Window ):
self._current_frame_index = int( ( self._num_frames - 1 ) * HC.options[ 'animation_start_position' ] )
self._current_frame_drawn = False
self._current_frame_drawn_at = 0.0
self._next_frame_due_at = 0.0
self._next_frame_due_at = HydrusData.GetNowPrecise()
self._slow_frame_score = 1.0
self._paused = start_paused
self._video_container = ClientRendering.RasterContainerVideo( self._media, initial_size, init_position = self._current_frame_index )
self._canvas_bmp = wx.EmptyBitmap( initial_width, initial_height, 24 )
self._timer_video = wx.Timer( self, id = ID_TIMER_VIDEO )
@ -159,13 +160,15 @@ class Animation( wx.Window ):
self.Bind( wx.EVT_KEY_UP, self.EventPropagateKey )
self.Bind( wx.EVT_ERASE_BACKGROUND, self.EventEraseBackground )
self._timer_video.Start( 5, wx.TIMER_CONTINUOUS )
self._timer_video.Start( self.TIMER_MS, wx.TIMER_CONTINUOUS )
self.Refresh()
def __del__( self ):
self._video_container.Stop()
wx.CallLater( 500, gc.collect )
@ -211,7 +214,7 @@ class Animation( wx.Window ):
dc.StretchBlit( 0, 0, my_width, my_height, mdc, 0, 0, frame_width, frame_height )
wx.CallAfter( wx_bmp.Destroy )
wx_bmp.Destroy()
if self._animation_bar is not None:
@ -222,21 +225,16 @@ class Animation( wx.Window ):
next_frame_time_s = self._video_container.GetDuration( self._current_frame_index ) / 1000.0
if HydrusData.TimeHasPassedPrecise( self._next_frame_due_at + next_frame_time_s ):
next_frame_ideally_due = self._next_frame_due_at + next_frame_time_s
if HydrusData.TimeHasPassedPrecise( next_frame_ideally_due ):
# we are rendering slower than the animation demands, so we'll slow down
# this also initialises self._next_frame_due_at
self._current_frame_drawn_at = HydrusData.GetNowPrecise()
self._next_frame_due_at = HydrusData.GetNowPrecise() + next_frame_time_s
else:
# to make timings more accurate and keep frame throughput accurate, let's pretend we drew this at the right time
self._next_frame_due_at = next_frame_ideally_due
self._current_frame_drawn_at = self._next_frame_due_at
self._next_frame_due_at = self._current_frame_drawn_at + next_frame_time_s
self._a_frame_has_been_drawn = True
@ -264,11 +262,7 @@ class Animation( wx.Window ):
dc = wx.BufferedPaintDC( self, self._canvas_bmp )
if not self._current_frame_drawn and self._video_container.HasFrame( self._current_frame_index ):
self._DrawFrame( dc )
elif not self._a_frame_has_been_drawn:
if not self._a_frame_has_been_drawn:
self._DrawWhite( dc )
@ -313,6 +307,8 @@ class Animation( wx.Window ):
( my_width, my_height ) = self.GetClientSize()
( media_width, media_height ) = self._media.GetResolution()
( current_bmp_width, current_bmp_height ) = self._canvas_bmp.GetSize()
if my_width != current_bmp_width or my_height != current_bmp_height:
@ -322,21 +318,29 @@ class Animation( wx.Window ):
( renderer_width, renderer_height ) = self._video_container.GetSize()
we_just_zoomed_in = my_width > renderer_width or my_height > renderer_height
we_just_zoomed_out = my_width < renderer_width or my_height < renderer_height
if we_just_zoomed_in:
if self._video_container.IsScaled():
( media_width, media_height ) = self._media.GetResolution()
target_width = min( media_width, my_width )
target_height = min( media_height, my_height )
self._video_container = ClientRendering.RasterContainerVideo( self._media, ( target_width, target_height ) )
self._video_container.Stop()
self._video_container = ClientRendering.RasterContainerVideo( self._media, ( target_width, target_height ), init_position = self._current_frame_index )
elif we_just_zoomed_out:
if my_width < media_width or my_height < media_height: # i.e. new zoom is scaled
self._video_container.Stop()
self._video_container = ClientRendering.RasterContainerVideo( self._media, ( my_width, my_height ), init_position = self._current_frame_index )
self._video_container.SetFramePosition( self._current_frame_index )
self._current_frame_drawn = False
self._a_frame_has_been_drawn = False
@ -356,13 +360,10 @@ class Animation( wx.Window ):
self._current_frame_index = frame_index
self._video_container.SetFramePosition( self._current_frame_index )
self._video_container.GetReadyForFrame( self._current_frame_index )
self._current_frame_drawn_at = 0.0
self._current_frame_drawn = False
self.Refresh()
self._paused = True
@ -420,7 +421,7 @@ class Animation( wx.Window ):
if self._current_frame_drawn:
if not self._paused and HydrusData.TimeHasPassedPrecise( self._next_frame_due_at ):
if not self._paused and HydrusData.TimeHasPassedPrecise( self._next_frame_due_at - self.TIMER_MS / 1000.0 ):
num_frames = self._media.GetNumFrames()
@ -433,13 +434,16 @@ class Animation( wx.Window ):
self._current_frame_drawn = False
self._video_container.SetFramePosition( self._current_frame_index )
if not self._current_frame_drawn and self._video_container.HasFrame( self._current_frame_index ):
if not self._current_frame_drawn:
self.Refresh()
if self._video_container.HasFrame( self._current_frame_index ):
dc = wx.BufferedDC( wx.ClientDC( self ), self._canvas_bmp )
self._DrawFrame( dc )
@ -885,6 +889,11 @@ class Canvas( wx.Window ):
def _HydrusShouldNotProcessInput( self ):
if HydrusGlobals.client_controller.MenuIsOpen():
return True
if HydrusGlobals.do_not_catch_char_hook:
HydrusGlobals.do_not_catch_char_hook = False

24
include/ClientGUICommon.py
View File

@ -1641,22 +1641,25 @@ class CheckboxCollect( wx.combo.ComboCtrl ):
class ChoiceSort( BetterChoice ):
def __init__( self, parent, page_key = None, sort_by = None ):
def __init__( self, parent, page_key = None, add_namespaces_and_ratings = True ):
BetterChoice.__init__( self, parent )
self._page_key = page_key
if sort_by is None: sort_by = HC.options[ 'sort_by' ]
sort_choices = list( CC.SORT_CHOICES )
sort_choices = CC.SORT_CHOICES + sort_by
ratings_services = HydrusGlobals.client_controller.GetServicesManager().GetServices( ( HC.LOCAL_RATING_LIKE, HC.LOCAL_RATING_NUMERICAL ) )
for ratings_service in ratings_services:
if add_namespaces_and_ratings:
sort_choices.append( ( 'rating_descend', ratings_service ) )
sort_choices.append( ( 'rating_ascend', ratings_service ) )
sort_choices.extend( HC.options[ 'sort_by' ] )
ratings_services = HydrusGlobals.client_controller.GetServicesManager().GetServices( ( HC.LOCAL_RATING_LIKE, HC.LOCAL_RATING_NUMERICAL ) )
for ratings_service in ratings_services:
sort_choices.append( ( 'rating_descend', ratings_service ) )
sort_choices.append( ( 'rating_ascend', ratings_service ) )
for ( sort_by_type, sort_by_data ) in sort_choices:
@ -1679,9 +1682,6 @@ class ChoiceSort( BetterChoice ):
self.Append( 'sort by ' + string, ( sort_by_type, sort_by_data ) )
try: self.SetSelection( HC.options[ 'default_sort' ] )
except: pass
self.Bind( wx.EVT_CHOICE, self.EventChoice )
HydrusGlobals.client_controller.sub( self, 'ACollectHappened', 'collect_media' )

37
include/ClientGUIManagement.py
View File

@ -660,6 +660,15 @@ class ManagementPanel( wx.lib.scrolledpanel.ScrolledPanel ):
self._sort_by = ClientGUICommon.ChoiceSort( self, self._page_key )
try:
self._sort_by.SetSelection( HC.options[ 'default_sort' ] )
except:
self._sort_by.SetSelection( 0 )
sizer.AddF( self._sort_by, CC.FLAGS_EXPAND_PERPENDICULAR )
@ -2824,24 +2833,20 @@ class ManagementPanelQuery( ManagementPanel ):
def ShowQuery( self, query_key, media_results ):
try:
if query_key == self._query_key:
if query_key == self._query_key:
current_predicates = self._current_predicates_box.GetPredicates()
file_service_key = self._management_controller.GetKey( 'file_service' )
panel = ClientGUIMedia.MediaPanelThumbnails( self._page, self._page_key, file_service_key, media_results )
panel.Collect( self._page_key, self._collect_by.GetChoice() )
panel.Sort( self._page_key, self._sort_by.GetChoice() )
self._controller.pub( 'swap_media_panel', self._page_key, panel )
current_predicates = self._current_predicates_box.GetPredicates()
file_service_key = self._management_controller.GetKey( 'file_service' )
panel = ClientGUIMedia.MediaPanelThumbnails( self._page, self._page_key, file_service_key, media_results )
panel.Collect( self._page_key, self._collect_by.GetChoice() )
panel.Sort( self._page_key, self._sort_by.GetChoice() )
self._controller.pub( 'swap_media_panel', self._page_key, panel )
except: wx.MessageBox( traceback.format_exc() )
management_panel_types_to_classes[ MANAGEMENT_TYPE_QUERY ] = ManagementPanelQuery

108
include/ClientGUIPanels.py
View File

@ -243,15 +243,24 @@ class ManageOptionsPanel( ManagePanel ):
self._delete = wx.Button( self, label = 'delete' )
self._delete.Bind( wx.EVT_BUTTON, self.EventDelete )
self._resized_thumbnails_override = wx.DirPickerCtrl( self, style = wx.DIRP_USE_TEXTCTRL )
#
self._new_options = HydrusGlobals.client_controller.GetNewOptions()
for ( location, weight ) in self._new_options.GetClientFilesLocationsToIdealWeights().items():
( locations_to_ideal_weights, resized_thumbnail_override ) = self._new_options.GetClientFilesLocationsToIdealWeights()
for ( location, weight ) in locations_to_ideal_weights.items():
self._client_files.Append( ( location, HydrusData.ConvertIntToPrettyString( int( weight ) ) ), ( location, weight ) )
if resized_thumbnail_override is not None:
self._resized_thumbnails_override.SetPath( resized_thumbnail_override )
#
vbox = wx.BoxSizer( wx.VERTICAL )
@ -276,6 +285,23 @@ class ManageOptionsPanel( ManagePanel ):
vbox.AddF( hbox, CC.FLAGS_BUTTON_SIZER )
text = 'If you like, you can force all your resized thumbnails to be stored in a single location, for instance on a low-latency SSD.'
text += os.linesep * 2
text += 'Leave it blank to store resized thumbnails with everything else.'
st = wx.StaticText( self, label = text )
st.Wrap( 400 )
vbox.AddF( st, CC.FLAGS_EXPAND_PERPENDICULAR )
hbox = wx.BoxSizer( wx.HORIZONTAL )
hbox.AddF( wx.StaticText( self, label = 'resized thumbnail override location: ' ), CC.FLAGS_MIXED )
hbox.AddF( self._resized_thumbnails_override, CC.FLAGS_EXPAND_BOTH_WAYS )
vbox.AddF( hbox, CC.FLAGS_EXPAND_PERPENDICULAR )
self.SetSizer( vbox )
@ -349,7 +375,14 @@ class ManageOptionsPanel( ManagePanel ):
locations_to_weights[ location ] = weight
self._new_options.SetClientFilesLocationsToIdealWeights( locations_to_weights )
resized_thumbnails_override = self._resized_thumbnails_override.GetPath()
if resized_thumbnails_override == '':
resized_thumbnails_override = None
self._new_options.SetClientFilesLocationsToIdealWeights( locations_to_weights, resized_thumbnails_override )
@ -1162,7 +1195,10 @@ class ManageOptionsPanel( ManagePanel ):
abs_path = HydrusPaths.ConvertPortablePathToAbsPath( HC.options[ 'export_path' ] )
if abs_path is not None: self._export_location.SetPath( abs_path )
if abs_path is not None:
self._export_location.SetPath( abs_path )
self._delete_to_recycle_bin.SetValue( HC.options[ 'delete_to_recycle_bin' ] )
@ -1665,7 +1701,9 @@ class ManageOptionsPanel( ManagePanel ):
self.SetBackgroundColour( wx.SystemSettings.GetColour( wx.SYS_COLOUR_BTNFACE ) )
self._default_sort = ClientGUICommon.ChoiceSort( self, sort_by = HC.options[ 'sort_by' ] )
self._default_sort = ClientGUICommon.ChoiceSort( self )
self._sort_fallback = ClientGUICommon.ChoiceSort( self, add_namespaces_and_ratings = False )
self._default_collect = ClientGUICommon.CheckboxCollect( self )
@ -1677,7 +1715,21 @@ class ManageOptionsPanel( ManagePanel ):
#
for ( sort_by_type, sort_by ) in HC.options[ 'sort_by' ]: self._sort_by.Append( '-'.join( sort_by ), sort_by )
try:
self._default_sort.SetSelection( HC.options[ 'default_sort' ] )
except:
self._default_sort.SetSelection( 0 )
self._sort_fallback.SetSelection( HC.options[ 'sort_fallback' ] )
for ( sort_by_type, sort_by ) in HC.options[ 'sort_by' ]:
self._sort_by.Append( '-'.join( sort_by ), sort_by )
#
@ -1687,6 +1739,8 @@ class ManageOptionsPanel( ManagePanel ):
gridbox.AddF( wx.StaticText( self, label = 'default sort: ' ), CC.FLAGS_MIXED )
gridbox.AddF( self._default_sort, CC.FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( wx.StaticText( self, label = 'secondary sort (when primary gives two equal values): ' ), CC.FLAGS_MIXED )
gridbox.AddF( self._sort_fallback, CC.FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( wx.StaticText( self, label = 'default collect: ' ), CC.FLAGS_MIXED )
gridbox.AddF( self._default_collect, CC.FLAGS_EXPAND_BOTH_WAYS )
@ -1740,6 +1794,7 @@ class ManageOptionsPanel( ManagePanel ):
def UpdateOptions( self ):
HC.options[ 'default_sort' ] = self._default_sort.GetSelection()
HC.options[ 'sort_fallback' ] = self._sort_fallback.GetSelection()
HC.options[ 'default_collect' ] = self._default_collect.GetChoice()
sort_by_choices = []
@ -1816,6 +1871,11 @@ class ManageOptionsPanel( ManagePanel ):
self._estimated_number_fullscreens = wx.StaticText( self, label = '' )
self._video_buffer_size_mb = wx.SpinCtrl( self, min = 48, max = 16 * 1024 )
self._video_buffer_size_mb.Bind( wx.EVT_SPINCTRL, self.EventVideoBufferUpdate )
self._estimated_number_video_frames = wx.StaticText( self, label = '' )
self._forced_search_limit = ClientGUICommon.NoneableSpinCtrl( self, '', min = 1, max = 100000 )
self._num_autocomplete_chars = wx.SpinCtrl( self, min = 1, max = 100 )
@ -1850,6 +1910,8 @@ class ManageOptionsPanel( ManagePanel ):
self._fullscreen_cache_size.SetValue( int( HC.options[ 'fullscreen_cache_size' ] / 1048576 ) )
self._video_buffer_size_mb.SetValue( self._new_options.GetInteger( 'video_buffer_size_mb' ) )
self._forced_search_limit.SetValue( self._new_options.GetNoneableInteger( 'forced_search_limit' ) )
self._num_autocomplete_chars.SetValue( HC.options[ 'num_autocomplete_chars' ] )
@ -1881,6 +1943,11 @@ class ManageOptionsPanel( ManagePanel ):
fullscreens_sizer.AddF( self._fullscreen_cache_size, CC.FLAGS_MIXED )
fullscreens_sizer.AddF( self._estimated_number_fullscreens, CC.FLAGS_MIXED )
video_buffer_sizer = wx.BoxSizer( wx.HORIZONTAL )
video_buffer_sizer.AddF( self._video_buffer_size_mb, CC.FLAGS_MIXED )
video_buffer_sizer.AddF( self._estimated_number_video_frames, CC.FLAGS_MIXED )
vbox = wx.BoxSizer( wx.VERTICAL )
vbox.AddF( self._disk_cache_init_period, CC.FLAGS_EXPAND_PERPENDICULAR )
@ -1913,6 +1980,23 @@ class ManageOptionsPanel( ManagePanel ):
vbox.AddF( gridbox, CC.FLAGS_EXPAND_PERPENDICULAR )
text = 'Hydrus video rendering is CPU intensive.'
text += os.linesep
text += 'If you have a lot of memory, you can set a generous potential video buffer to compensate.'
text += os.linesep
text += 'If the video buffer can hold an entire video, it only needs to be rendered once and will loop smoothly.'
vbox.AddF( wx.StaticText( self, label = text ), CC.FLAGS_MIXED )
gridbox = wx.FlexGridSizer( 0, 2 )
gridbox.AddGrowableCol( 1, 1 )
gridbox.AddF( wx.StaticText( self, label = 'MB memory for video buffer: ' ), CC.FLAGS_MIXED )
gridbox.AddF( video_buffer_sizer, CC.FLAGS_NONE )
vbox.AddF( gridbox, CC.FLAGS_EXPAND_PERPENDICULAR )
gridbox = wx.FlexGridSizer( 0, 2 )
gridbox.AddGrowableCol( 1, 1 )
@ -1955,6 +2039,7 @@ class ManageOptionsPanel( ManagePanel ):
self.EventFullscreensUpdate( None )
self.EventPreviewsUpdate( None )
self.EventThumbnailsUpdate( None )
self.EventVideoBufferUpdate( None )
wx.CallAfter( self.Layout ) # draws the static texts correctly
@ -1998,6 +2083,13 @@ class ManageOptionsPanel( ManagePanel ):
self._estimated_number_thumbnails.SetLabelText( '(about ' + HydrusData.ConvertIntToPrettyString( ( self._thumbnail_cache_size.GetValue() * 1048576 ) / estimated_bytes_per_thumb ) + ' thumbnails)' )
def EventVideoBufferUpdate( self, event ):
estimated_720p_frames = int( ( self._video_buffer_size_mb.GetValue() * 1024 * 1024 ) / ( 1280 * 720 * 3 ) )
self._estimated_number_video_frames.SetLabelText( '(about ' + HydrusData.ConvertIntToPrettyString( estimated_720p_frames ) + ' frames of 720p video)' )
def UpdateOptions( self ):
self._new_options.SetNoneableInteger( 'disk_cache_init_period', self._disk_cache_init_period.GetValue() )
@ -2011,6 +2103,8 @@ class ManageOptionsPanel( ManagePanel ):
HC.options[ 'preview_cache_size' ] = self._preview_cache_size.GetValue() * 1048576
HC.options[ 'fullscreen_cache_size' ] = self._fullscreen_cache_size.GetValue() * 1048576
self._new_options.SetInteger( 'video_buffer_size_mb', self._video_buffer_size_mb.GetValue() )
self._new_options.SetNoneableInteger( 'forced_search_limit', self._forced_search_limit.GetValue() )
HC.options[ 'num_autocomplete_chars' ] = self._num_autocomplete_chars.GetValue()
@ -2440,10 +2534,10 @@ class ManageTagsPanel( ManagePanel ):
self._modify_mappers = wx.Button( self, label = 'modify mappers' )
self._modify_mappers.Bind( wx.EVT_BUTTON, self.EventModify )
self._copy_tags = wx.Button( self, label = 'copy tags' )
self._copy_tags = wx.Button( self, id = wx.ID_COPY, label = 'copy tags' )
self._copy_tags.Bind( wx.EVT_BUTTON, self.EventCopyTags )
self._paste_tags = wx.Button( self, label = 'paste tags' )
self._paste_tags = wx.Button( self, id = wx.ID_PASTE, label = 'paste tags' )
self._paste_tags.Bind( wx.EVT_BUTTON, self.EventPasteTags )
if self._i_am_local_tag_service:

2
include/ClientGUITopLevelWindows.py
View File

@ -393,7 +393,7 @@ class FrameThatTakesScrollablePanel( FrameThatResizes ):
FrameThatResizes.__init__( self, parent, title, frame_key, float_on_parent )
self._ok = wx.Button( self, label = 'close' )
self._ok = wx.Button( self, id = wx.ID_OK, label = 'close' )
self._ok.Bind( wx.EVT_BUTTON, self.EventCloseButton )
self.Bind( wx.EVT_MENU, self.EventMenu )

4
include/ClientLocalServerResources.py
View File

@ -249,7 +249,7 @@ class HydrusResourceCommandBooruThumbnail( HydrusResourceCommandBooru ):
client_files_manager = HydrusGlobals.client_controller.GetClientFilesManager()
path = client_files_manager.GetThumbnailPath( hash, True )
path = client_files_manager.GetFullSizeThumbnailPath( hash )
elif mime in HC.AUDIO: path = os.path.join( HC.STATIC_DIR, 'audio.png' )
elif mime == HC.APPLICATION_PDF: path = os.path.join( HC.STATIC_DIR, 'pdf.png' )
@ -283,7 +283,7 @@ class HydrusResourceCommandLocalThumbnail( HydrusServerResources.HydrusResourceC
client_files_manager = HydrusGlobals.client_controller.GetClientFilesManager()
path = client_files_manager.GetThumbnailPath( hash, True )
path = client_files_manager.GetFullSizeThumbnailPath( hash )
response_context = HydrusServerResources.ResponseContext( 200, path = path )

136
include/ClientMedia.py
View File

@ -394,6 +394,64 @@ class MediaList( object ):
else: return self._sorted_media[ previous_index ]
def _GetSortFunction( self, sort_by ):
( sort_by_type, sort_by_data ) = sort_by
def deal_with_none( x ):
if x is None: return -1
else: return x
if sort_by_type == 'system':
if sort_by_data == CC.SORT_BY_RANDOM: sort_function = lambda x: random.random()
elif sort_by_data == CC.SORT_BY_SMALLEST: sort_function = lambda x: deal_with_none( x.GetSize() )
elif sort_by_data == CC.SORT_BY_LARGEST: sort_function = lambda x: -deal_with_none( x.GetSize() )
elif sort_by_data == CC.SORT_BY_SHORTEST: sort_function = lambda x: deal_with_none( x.GetDuration() )
elif sort_by_data == CC.SORT_BY_LONGEST: sort_function = lambda x: -deal_with_none( x.GetDuration() )
elif sort_by_data == CC.SORT_BY_OLDEST: sort_function = lambda x: deal_with_none( x.GetTimestamp( self._file_service_key ) )
elif sort_by_data == CC.SORT_BY_NEWEST: sort_function = lambda x: -deal_with_none( x.GetTimestamp( self._file_service_key ) )
elif sort_by_data == CC.SORT_BY_MIME: sort_function = lambda x: x.GetMime()
elif sort_by_type == 'namespaces':
def namespace_sort_function( namespaces, x ):
x_tags_manager = x.GetTagsManager()
return [ x_tags_manager.GetComparableNamespaceSlice( ( namespace, ), collapse_siblings = True ) for namespace in namespaces ]
sort_function = lambda x: namespace_sort_function( sort_by_data, x )
elif sort_by_type in ( 'rating_descend', 'rating_ascend' ):
service_key = sort_by_data
def ratings_sort_function( service_key, reverse, x ):
x_ratings_manager = x.GetRatingsManager()
rating = deal_with_none( x_ratings_manager.GetRating( service_key ) )
if reverse:
rating *= -1
return rating
reverse = sort_by_type == 'rating_descend'
sort_function = lambda x: ratings_sort_function( service_key, reverse, x )
return sort_function
def _RecalcHashes( self ):
self._hashes = set()
@ -721,64 +779,25 @@ class MediaList( object ):
def Sort( self, sort_by = None ):
for media in self._collected_media: media.Sort( sort_by )
for media in self._collected_media:
media.Sort( sort_by )
if sort_by is None: sort_by = self._sort_by
if sort_by is None:
sort_by = self._sort_by
self._sort_by = sort_by
( sort_by_type, sort_by_data ) = sort_by
sort_function = self._GetSortFunction( ( 'system', HC.options[ 'sort_fallback' ] ) )
def deal_with_none( x ):
if x is None: return -1
else: return x
self._sorted_media.sort( sort_function )
if sort_by_type == 'system':
if sort_by_data == CC.SORT_BY_RANDOM: sort_function = lambda x: random.random()
elif sort_by_data == CC.SORT_BY_SMALLEST: sort_function = lambda x: deal_with_none( x.GetSize() )
elif sort_by_data == CC.SORT_BY_LARGEST: sort_function = lambda x: -deal_with_none( x.GetSize() )
elif sort_by_data == CC.SORT_BY_SHORTEST: sort_function = lambda x: deal_with_none( x.GetDuration() )
elif sort_by_data == CC.SORT_BY_LONGEST: sort_function = lambda x: -deal_with_none( x.GetDuration() )
elif sort_by_data == CC.SORT_BY_OLDEST: sort_function = lambda x: deal_with_none( x.GetTimestamp( self._file_service_key ) )
elif sort_by_data == CC.SORT_BY_NEWEST: sort_function = lambda x: -deal_with_none( x.GetTimestamp( self._file_service_key ) )
elif sort_by_data == CC.SORT_BY_MIME: sort_function = lambda x: x.GetMime()
elif sort_by_type == 'namespaces':
def namespace_sort_function( namespaces, x ):
x_tags_manager = x.GetTagsManager()
return [ x_tags_manager.GetComparableNamespaceSlice( ( namespace, ), collapse_siblings = True ) for namespace in namespaces ]
sort_function = lambda x: namespace_sort_function( sort_by_data, x )
elif sort_by_type in ( 'rating_descend', 'rating_ascend' ):
service_key = sort_by_data
def ratings_sort_function( service_key, reverse, x ):
x_ratings_manager = x.GetRatingsManager()
rating = deal_with_none( x_ratings_manager.GetRating( service_key ) )
if reverse:
rating *= -1
return rating
reverse = sort_by_type == 'rating_descend'
sort_function = lambda x: ratings_sort_function( service_key, reverse, x )
# this is a stable sort, so the fallback order above will remain for equal items
sort_function = self._GetSortFunction( self._sort_by )
self._sorted_media.sort( sort_function )
@ -1616,14 +1635,17 @@ class TagsManagerSimple( object ):
siblings_manager = HydrusGlobals.client_controller.GetManager( 'tag_siblings' )
if collapse_siblings:
combined = list( siblings_manager.CollapseTags( combined ) )
slice = []
for namespace in namespaces:
tags = [ tag for tag in combined if tag.startswith( namespace + ':' ) ]
if collapse_siblings: tags = list( siblings_manager.CollapseTags( tags ) )
tags = [ tag.split( ':', 1 )[1] for tag in tags ]
tags = HydrusTags.SortNumericTags( tags )
@ -1643,15 +1665,17 @@ class TagsManagerSimple( object ):
combined_current = combined_statuses_to_tags[ HC.CURRENT ]
combined_pending = combined_statuses_to_tags[ HC.PENDING ]
slice = { tag for tag in combined_current.union( combined_pending ) if True in ( tag.startswith( namespace + ':' ) for namespace in namespaces ) }
combined = combined_current.union( combined_pending )
if collapse_siblings:
siblings_manager = HydrusGlobals.client_controller.GetManager( 'tag_siblings' )
slice = siblings_manager.CollapseTags( slice )
combined = siblings_manager.CollapseTags( combined )
slice = { tag for tag in combined if True in ( tag.startswith( namespace + ':' ) for namespace in namespaces ) }
slice = frozenset( slice )
return slice

250
include/ClientRendering.py
View File

@ -149,8 +149,6 @@ class RasterContainerImage( RasterContainer ):
class RasterContainerVideo( RasterContainer ):
BUFFER_SIZE = 1024 * 1024 * 96
def __init__( self, media, target_resolution = None, init_position = 0 ):
RasterContainer.__init__( self, media, target_resolution )
@ -159,10 +157,29 @@ class RasterContainerVideo( RasterContainer ):
self._last_index_asked_for = -1
self._buffer_start_index = -1
self._buffer_end_index = -1
self._renderer_awake = False
self._stop = False
( x, y ) = self._target_resolution
frame_buffer_length = self.BUFFER_SIZE / ( x * y * 3 )
new_options = HydrusGlobals.client_controller.GetNewOptions()
video_buffer_size_mb = new_options.GetInteger( 'video_buffer_size_mb' )
duration = self._media.GetDuration()
num_frames = self._media.GetNumFrames()
self._average_frame_duration = float( duration ) / num_frames
frame_buffer_length = ( video_buffer_size_mb * 1024 * 1024 ) / ( x * y * 3 )
# if we can't buffer the whole vid, then don't have a clunky massive buffer
if num_frames * 0.1 < frame_buffer_length and frame_buffer_length < num_frames:
frame_buffer_length = int( num_frames * 0.1 )
self._num_frames_backwards = frame_buffer_length * 2 / 3
self._num_frames_forwards = frame_buffer_length / 3
@ -174,9 +191,6 @@ class RasterContainerVideo( RasterContainer ):
path = client_files_manager.GetFilePath( hash, mime )
duration = self._media.GetDuration()
num_frames = self._media.GetNumFrames()
if self._media.GetMime() == HC.IMAGE_GIF:
self._durations = HydrusImageHandling.GetGIFFrameDurations( self._path )
@ -185,77 +199,105 @@ class RasterContainerVideo( RasterContainer ):
else:
self._frame_duration = float( duration ) / num_frames
self._renderer = HydrusVideoHandling.VideoRendererFFMPEG( path, mime, duration, num_frames, target_resolution )
self._render_lock = threading.Lock()
self._buffer_lock = threading.Lock()
self._next_render_index = 0
self._next_render_index = -1
self._render_to_index = -1
self._rendered_first_frame = False
self.SetFramePosition( init_position )
self.GetReadyForFrame( init_position )
def _IndexOutOfRange( self, index, range_start, range_end ):
before_start = index < range_start
after_end = range_end < index
if range_start < range_end:
if before_start or after_end:
return True
else:
if after_end and before_start:
return True
return False
def _MaintainBuffer( self ):
deletees = []
for index in self._frames.keys():
with self._buffer_lock:
if self._buffer_start_index < self._buffer_end_index:
deletees = [ index for index in self._frames.keys() if self._IndexOutOfRange( index, self._buffer_start_index, self._buffer_end_index ) ]
for i in deletees:
if index < self._buffer_start_index or self._buffer_end_index < index: deletees.append( index )
else:
if self._buffer_end_index < index and index < self._buffer_start_index: deletees.append( index )
del self._frames[ i ]
for i in deletees: del self._frames[ i ]
def _RENDERERSetRenderToPosition( self, index ):
def THREADMoveBuffer( self, render_to_index ):
with self._render_lock:
if self._render_to_index != index:
if self._render_to_index != render_to_index:
self._render_to_index = index
self._render_to_index = render_to_index
HydrusGlobals.client_controller.CallToThread( self.THREADRender )
if not self._renderer_awake:
HydrusGlobals.client_controller.CallToThread( self.THREADRender )
def _RENDERERSetFramePosition( self, index ):
def THREADMoveRenderer( self, start_index, rush_to_index, render_to_index ):
with self._render_lock:
if index == self._next_render_index: return
else:
if self._next_render_index != start_index:
self._renderer.set_position( index )
self._renderer.set_position( start_index )
self._next_render_index = index
self._render_to_index = index
self._next_render_index = start_index
self._render_to_index = render_to_index
HydrusGlobals.client_controller.CallToThread( self.THREADRender, rush_to_index )
def THREADRender( self ):
def THREADRender( self, rush_to_index = None ):
num_frames = self._media.GetNumFrames()
while True:
time.sleep( 0.00001 ) # thread yield
if self._stop:
self._renderer_awake = False
return
with self._render_lock:
self._renderer_awake = True
if not self._rendered_first_frame or self._next_render_index != ( self._render_to_index + 1 ) % num_frames:
self._rendered_first_frame = True
@ -270,7 +312,9 @@ class RasterContainerVideo( RasterContainer ):
HydrusData.ShowException( e )
break
self._renderer_awake = False
return
finally:
@ -279,34 +323,48 @@ class RasterContainerVideo( RasterContainer ):
frame = GenerateHydrusBitmapFromNumPyImage( numpy_image, compressed = False )
wx.CallAfter( self.AddFrame, frame_index, frame )
with self._buffer_lock:
self._frames[ frame_index ] = frame
else:
break
self._renderer_awake = False
return
def AddFrame( self, index, frame ):
self._frames[ index ] = frame
if rush_to_index is not None and not self._IndexOutOfRange( rush_to_index, self._next_render_index, self._render_to_index ):
time.sleep( 0.00001 )
else:
half_a_frame = ( self._average_frame_duration / 1000.0 ) * 0.5
time.sleep( half_a_frame ) # just so we don't spam cpu
def GetDuration( self, index ):
if self._media.GetMime() == HC.IMAGE_GIF: return self._durations[ index ]
else: return self._frame_duration
else: return self._average_frame_duration
def GetFrame( self, index ):
frame = self._frames[ index ]
with self._buffer_lock:
frame = self._frames[ index ]
self._last_index_asked_for = index
self._MaintainBuffer()
self.GetReadyForFrame( self._last_index_asked_for + 1 )
return frame
@ -317,63 +375,41 @@ class RasterContainerVideo( RasterContainer ):
def GetNumFrames( self ): return self._media.GetNumFrames()
def GetResolution( self ): return self._media.GetResolution()
def GetSize( self ): return self._target_resolution
def GetTotalDuration( self ):
if self._media.GetMime() == HC.IMAGE_GIF: return sum( self._durations )
else: return self._frame_duration * self.GetNumFrames()
def GetZoom( self ): return self._zoom
def HasFrame( self, index ): return index in self._frames
def IsScaled( self ): return self._zoom != 1.0
def SetFramePosition( self, index ):
def GetReadyForFrame( self, next_index_to_expect ):
num_frames = self.GetNumFrames()
if num_frames > self._num_frames_backwards + 1 + self._num_frames_forwards:
new_buffer_start_index = max( 0, index - self._num_frames_backwards ) % num_frames
index_out_of_buffer = self._IndexOutOfRange( next_index_to_expect, self._buffer_start_index, self._buffer_end_index )
new_buffer_end_index = ( index + self._num_frames_forwards ) % num_frames
ideal_buffer_start_index = max( 0, next_index_to_expect - self._num_frames_backwards )
if index == self._last_index_asked_for: return
elif index < self._last_index_asked_for:
ideal_buffer_end_index = ( next_index_to_expect + self._num_frames_forwards ) % num_frames
if not self._rendered_first_frame or index_out_of_buffer:
if index < self._buffer_start_index:
self._buffer_start_index = ideal_buffer_start_index
self._buffer_end_index = ideal_buffer_end_index
HydrusGlobals.client_controller.CallToThread( self.THREADMoveRenderer, self._buffer_start_index, next_index_to_expect, self._buffer_end_index )
else:
# rendering can't go backwards, so dragging caret back shouldn't rewind either of these!
if self.HasFrame( ideal_buffer_start_index ):
self._buffer_start_index = new_buffer_start_index
self._RENDERERSetFramePosition( self._buffer_start_index )
self._buffer_end_index = new_buffer_end_index
self._RENDERERSetRenderToPosition( self._buffer_end_index )
self._buffer_start_index = ideal_buffer_start_index
else: # index > self._last_index_asked_for
currently_no_wraparound = self._buffer_start_index < self._buffer_end_index
self._buffer_start_index = new_buffer_start_index
if currently_no_wraparound:
if not self._IndexOutOfRange( self._next_render_index + 1, self._buffer_start_index, ideal_buffer_end_index ):
if index > self._buffer_end_index:
self._RENDERERSetFramePosition( self._buffer_start_index )
self._buffer_end_index = ideal_buffer_end_index
self._buffer_end_index = new_buffer_end_index
self._RENDERERSetRenderToPosition( self._buffer_end_index )
HydrusGlobals.client_controller.CallToThread( self.THREADMoveBuffer, self._buffer_end_index )
else:
@ -382,17 +418,51 @@ class RasterContainerVideo( RasterContainer ):
self._buffer_start_index = 0
self._RENDERERSetFramePosition( 0 )
self._buffer_end_index = num_frames - 1
self._RENDERERSetRenderToPosition( self._buffer_end_index )
HydrusGlobals.client_controller.CallToThread( self.THREADMoveRenderer, self._buffer_start_index, next_index_to_expect, self._buffer_end_index )
else:
if not self.HasFrame( next_index_to_expect ):
# this rushes rendering to this point
HydrusGlobals.client_controller.CallToThread( self.THREADRender, next_index_to_expect )
self._MaintainBuffer()
def GetResolution( self ): return self._media.GetResolution()
def GetSize( self ): return self._target_resolution
def GetTotalDuration( self ):
if self._media.GetMime() == HC.IMAGE_GIF: return sum( self._durations )
else: return self._average_frame_duration * self.GetNumFrames()
def GetZoom( self ): return self._zoom
def HasFrame( self, index ):
with self._buffer_lock:
return index in self._frames
def IsScaled( self ): return self._zoom != 1.0
def Stop( self ):
self._stop = True
class HydrusBitmap( object ):
def __init__( self, data, format, size, compressed = True ):

2
include/HydrusConstants.py
View File

@ -48,7 +48,7 @@ options = {}
# Misc
NETWORK_VERSION = 17
SOFTWARE_VERSION = 215
SOFTWARE_VERSION = 216
UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )

10
include/HydrusData.py
View File

@ -1460,8 +1460,14 @@ class ClientToServerContentUpdatePackage( HydrusYAMLBase ):
if data_type == HC.CONTENT_TYPE_FILES:
if action == HC.CONTENT_UPDATE_PETITION: return ( ( { self._hash_ids_to_hashes[ hash_id ] for hash_id in hash_ids }, reason ) for ( hash_ids, reason ) in data )
else: return ( { self._hash_ids_to_hashes[ hash_id ] for hash_id in hash_ids } for hash_ids in ( data, ) )
if action == HC.CONTENT_UPDATE_PETITION:
return ( ( { self._hash_ids_to_hashes[ hash_id ] for hash_id in hash_ids }, reason ) for ( hash_ids, reason ) in data )
else:
return [ { self._hash_ids_to_hashes[ hash_id ] for hash_id in hash_ids } for hash_ids in data ]
if data_type == HC.CONTENT_TYPE_MAPPINGS:

5
include/HydrusNATPunch.py
View File

@ -79,6 +79,11 @@ def AddUPnPMapping( internal_client, internal_port, external_port, protocol, des
( output, error ) = p.communicate()
if 'x.x.x.x:' + str( external_port ) + ' TCP is redirected to internal ' + internal_client + ':' + str( internal_port ) in output:
raise Exception( 'The UPnP mapping of ' + internal_client + ':' + internal_port + '->external:' + external_port + ' already exists as a port forward. If this UPnP mapping is automatic, please disable it.' )
if output is not None and 'failed with code' in output:
raise Exception( 'Problem while trying to add UPnP mapping:' + os.linesep * 2 + HydrusData.ToUnicode( output ) )

71
include/HydrusSessions.py
View File

@ -18,7 +18,7 @@ import HydrusData
import HydrusGlobals
HYDRUS_SESSION_LIFETIME = 30 * 86400
'''
class HydrusMessagingSessionManagerServer( object ):
def __init__( self ):
@ -74,7 +74,7 @@ class HydrusMessagingSessionManagerServer( object ):
return session_key
'''
class HydrusSessionManagerServer( object ):
def __init__( self ):
@ -92,26 +92,24 @@ class HydrusSessionManagerServer( object ):
account_key = HydrusGlobals.controller.Read( 'account_key_from_access_key', service_key, access_key )
if account_key not in self._account_keys_to_accounts:
account_keys_to_accounts = self._service_keys_to_account_keys_to_accounts[ service_key ]
if account_key not in account_keys_to_accounts:
account = HydrusGlobals.controller.Read( 'account', account_key )
self._account_keys_to_accounts[ account_key ] = account
account_keys_to_accounts[ account_key ] = account
account = self._account_keys_to_accounts[ account_key ]
session_key = HydrusData.GenerateKey()
self._account_keys_to_session_keys[ account_key ].add( session_key )
now = HydrusData.GetNow()
expires = now + HYDRUS_SESSION_LIFETIME
HydrusGlobals.controller.Write( 'session', session_key, service_key, account_key, expires )
self._service_keys_to_sessions[ service_key ][ session_key ] = ( account, expires )
self._service_keys_to_session_keys_to_sessions[ service_key ][ session_key ] = ( account_key, expires )
return ( session_key, expires )
@ -121,41 +119,44 @@ class HydrusSessionManagerServer( object ):
with self._lock:
service_sessions = self._service_keys_to_sessions[ service_key ]
session_keys_to_sessions = self._service_keys_to_session_keys_to_sessions[ service_key ]
if session_key in service_sessions:
if session_key in session_keys_to_sessions:
( account, expires ) = service_sessions[ session_key ]
( account_key, expires ) = session_keys_to_sessions[ session_key ]
if HydrusData.TimeHasPassed( expires ): del service_sessions[ session_key ]
else: return account
if HydrusData.TimeHasPassed( expires ):
del session_keys_to_sessions[ session_key ]
else:
account = self._service_keys_to_account_keys_to_accounts[ service_key ][ account_key ]
return account
raise HydrusExceptions.SessionException( 'Did not find that session! Try again!' )
def RefreshAccounts( self, service_key, account_keys ):
def RefreshAccounts( self, service_key, account_keys = None ):
with self._lock:
account_keys_to_accounts = self._service_keys_to_account_keys_to_accounts[ service_key ]
if account_keys is None:
account_keys = account_keys_to_accounts.keys()
for account_key in account_keys:
account = HydrusGlobals.controller.Read( 'account', account_key )
self._account_keys_to_accounts[ account_key ] = account
if account_key in self._account_keys_to_session_keys:
session_keys = self._account_keys_to_session_keys[ account_key ]
for session_key in session_keys:
( old_account, expires ) = self._service_keys_to_sessions[ service_key ][ session_key ]
self._service_keys_to_sessions[ service_key ][ session_key ] = ( account, expires )
account_keys_to_accounts[ account_key ] = account
@ -164,11 +165,9 @@ class HydrusSessionManagerServer( object ):
with self._lock:
self._service_keys_to_sessions = collections.defaultdict( dict )
self._service_keys_to_session_keys_to_sessions = collections.defaultdict( dict )
self._account_keys_to_session_keys = HydrusData.default_dict_set()
self._account_keys_to_accounts = {}
self._service_keys_to_account_keys_to_accounts = collections.defaultdict( dict )
#
@ -178,11 +177,9 @@ class HydrusSessionManagerServer( object ):
account_key = account.GetAccountKey()
self._service_keys_to_sessions[ service_key ][ session_key ] = ( account, expires )
self._service_keys_to_session_keys_to_sessions[ service_key ][ session_key ] = ( account_key, expires )
self._account_keys_to_session_keys[ account_key ].add( session_key )
self._account_keys_to_accounts[ account_key ] = account
self._service_keys_to_account_keys_to_accounts[ service_key ][ account_key ] = account

5
include/HydrusVideoHandling.py
View File

@ -307,7 +307,10 @@ class VideoRendererFFMPEG( object ):
self.process = subprocess.Popen( cmd, bufsize = self.bufsize, stdout=subprocess.PIPE, stderr=subprocess.PIPE, startupinfo = HydrusData.GetSubprocessStartupInfo() )
self.skip_frames( skip_frames )
if skip_frames > 0:
self.skip_frames( skip_frames )
def skip_frames( self, n ):

19
include/ServerDB.py
View File

@ -1071,15 +1071,26 @@ class DB( HydrusDB.HydrusDB ):
def _GetAccountKeyFromAccessKey( self, service_key, access_key ):
try: ( account_key, ) = self._c.execute( 'SELECT account_key FROM accounts WHERE hashed_access_key = ?;', ( sqlite3.Binary( hashlib.sha256( access_key ).digest() ), ) ).fetchone()
service_id = self._GetServiceId( service_key )
try:
( account_key, ) = self._c.execute( 'SELECT account_key FROM accounts WHERE service_id = ? AND hashed_access_key = ?;', ( service_id, sqlite3.Binary( hashlib.sha256( access_key ).digest() ), ) ).fetchone()
except:
# we do not delete the registration_key (and hence the raw unhashed access_key)
# until the first attempt to create a session to make sure the user
# has the access_key saved
try: ( account_type_id, account_key, expires ) = self._c.execute( 'SELECT account_type_id, account_key, expiry FROM registration_keys WHERE access_key = ?;', ( sqlite3.Binary( access_key ), ) ).fetchone()
except: raise HydrusExceptions.ForbiddenException( 'The service could not find that account in its database.' )
try:
( account_type_id, account_key, expires ) = self._c.execute( 'SELECT account_type_id, account_key, expiry FROM registration_keys WHERE access_key = ?;', ( sqlite3.Binary( access_key ), ) ).fetchone()
except:
raise HydrusExceptions.ForbiddenException( 'The service could not find that account in its database.' )
self._c.execute( 'DELETE FROM registration_keys WHERE access_key = ?;', ( sqlite3.Binary( access_key ), ) )
@ -1087,8 +1098,6 @@ class DB( HydrusDB.HydrusDB ):
now = HydrusData.GetNow()
service_id = self._GetServiceId( service_key )
self._c.execute( 'INSERT INTO accounts ( service_id, account_key, hashed_access_key, account_type_id, created, expires, used_bytes, used_requests ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ? );', ( service_id, sqlite3.Binary( account_key ), sqlite3.Binary( hashlib.sha256( access_key ).digest() ), account_type_id, now, expires, 0, 0 ) )

4
include/ServerServerResources.py
View File

@ -267,6 +267,10 @@ class HydrusResourceCommandRestrictedAccountTypes( HydrusResourceCommandRestrict
HydrusGlobals.server_controller.WriteSynchronous( 'account_types', self._service_key, edit_log )
session_manager = HydrusGlobals.server_controller.GetServerSessionManager()
session_manager.RefreshAccounts( self._service_key )
response_context = HydrusServerResources.ResponseContext( 200 )
return response_context

9
include/TestDB.py
View File

@ -725,9 +725,12 @@ class TestClientDB( unittest.TestCase ):
for prefix in HydrusData.IterateHexPrefixes():
dir = os.path.join( client_files_default, prefix )
self.assertTrue( os.path.exists( dir ) )
for c in ( 'f', 't', 'r' ):
dir = os.path.join( client_files_default, c + prefix )
self.assertTrue( os.path.exists( dir ) )

8
include/TestHydrusServer.py
View File

@ -122,7 +122,7 @@ class TestServer( unittest.TestCase ):
prefix = hash_encoded[:2]
path = os.path.join( client_files_default, prefix, hash_encoded + '.jpg' )
path = os.path.join( client_files_default, 'f' + prefix, hash_encoded + '.jpg' )
with open( path, 'wb' ) as f: f.write( 'file' )
@ -139,7 +139,7 @@ class TestServer( unittest.TestCase ):
#
path = os.path.join( client_files_default, prefix, hash_encoded + '.thumbnail' )
path = os.path.join( client_files_default, 't' + prefix, hash_encoded + '.thumbnail' )
with open( path, 'wb' ) as f: f.write( 'thumb' )
@ -250,8 +250,8 @@ class TestServer( unittest.TestCase ):
prefix = hash_encoded[:2]
file_path = os.path.join( client_files_default, prefix, hash_encoded + '.jpg' )
thumbnail_path = os.path.join( client_files_default, prefix, hash_encoded + '.thumbnail' )
file_path = os.path.join( client_files_default, 'f' + prefix, hash_encoded + '.jpg' )
thumbnail_path = os.path.join( client_files_default, 't' + prefix, hash_encoded + '.thumbnail' )
with open( file_path, 'wb' ) as f: f.write( 'file' )
with open( thumbnail_path, 'wb' ) as f: f.write( 'thumbnail' )

10
test.py
View File

@ -92,7 +92,15 @@ class Controller( object ):
services.append( ClientData.GenerateService( CC.LOCAL_TAG_SERVICE_KEY, HC.LOCAL_TAG, CC.LOCAL_TAG_SERVICE_KEY, {} ) )
self._reads[ 'services' ] = services
client_files_locations = { prefix : client_files_default for prefix in HydrusData.IterateHexPrefixes() }
client_files_locations = {}
for prefix in HydrusData.IterateHexPrefixes():
for c in ( 'f', 't', 'r' ):
client_files_locations[ c + prefix ] = client_files_default
self._reads[ 'client_files_locations' ] = client_files_locations