RepoMirrors
/
hydrus
mirror of https://github.com/hydrusnetwork/hydrus.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Version 201

This commit is contained in:
Hydrus Network Developer 2016-04-13 20:54:29 -05:00
parent 835779a783
commit 505db9306e
22 changed files with 1409 additions and 853 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
help/changelog.html
View File

@ -8,6 +8,45 @@
<div class="content">
<h3>changelog</h3>
<ul>
<li><h3>version 201</h3></li>
<ul>
<li>exported hash and tag master tables to external database files for both client and server</li>
<li>the client's 'mappings' table is split into 'current_mappings' and 'pending_mappings', saving about 20% of space and speed for almost all mappings operations</li>
<li>removed superfluous rowid column from client mappings tables, saving about another 20% of space</li>
<li>removed superfluous rowid column from server mappings tables, saving about 20% of space</li>
<li>to save time, service mapping counts now only aggregate current mappings, not current and pending</li>
<li>exporting to tag archive now only exports current_mappings, not pending</li>
<li>'all known files' queries will more cleverly discern their current/pending search domain</li>
<li>tag count queries will more cleverly discern their current/pending search domain</li>
<li>cleaned up a lot of mapping-related query code, and any ugly status leftovers from current/pending union</li>
<li>num_namespaces is no longer tracked for tag services</li>
<li>unified clientside service deletion code, fixing several orphan-generation issues</li>
<li>db update will remove all mappings orphans created by a recent tag service deletion</li>
<li>some db settings init is harmonised into one location</li>
<li>fixed up some external db settings init</li>
<li>db connection settings are now applied to all attached db files</li>
<li>db vacuum code is harmonised into one location</li>
<li>vacuum cleans up after itself better</li>
<li>db vacuum is now preceded by a hard drive free space check</li>
<li>db vacuum can now predict how long it will take</li>
<li>if a vacuum fails for an unexpected reason, the db will now recover better</li>
<li>the db recovers from failed backups better</li>
<li>the server db will vacuum all its databases on a backup, except any that it thinks will take total vacuum time to more than five minutes</li>
<li>server backup will report roughly how long it took, clientside</li>
<li>significantly sped up system-tag-only 'all known files' queries for tag services that have many mappings</li>
<li>significantly sped up system:numtags for =0 or >0 when tag services have many mappings</li>
<li>the status bar has some improved grammar, reporting overall common mime as appropriate</li>
<li>the status bar reports single-thumbnail info text when one file is selected</li>
<li>fixed an image cache key type-matching bug</li>
<li>data cache now tracks objects with dynamic memory usage</li>
<li>the static image rendering-cache relationship has been slightly dejanked</li>
<li>added a couple of checkboxes to the options->gui page to hide/show thumbnail title banner and page indicator</li>
<li>the manage tags censorship dialog now defaults to the 'all known tags' service on init</li>
<li>slideshows will not progress if the current media is either a static image that has yet to render or an unpaused animation that has yet to run through once</li>
<li>service content package status messages now summarise their final commit status better</li>
<li>this nice new line is printed to the log as a record of repo sync rows/s speed</li>
<li>the client will shut down faster if big jobs are running</li>
</ul>
<li><h3>version 200</h3></li>
<ul>
<li>added 'censor tag' to tag right-click menu</li>

73
include/ClientCaches.py
View File

@ -527,16 +527,21 @@ class DataCache( object ):
wx.CallLater( 60 * 1000, self.MaintainCache )
def _DeleteItem( self, index = 0 ):
def _DeleteItem( self ):
( deletee_key, last_access_time ) = self._keys_fifo.pop( index )
( deletee_key, last_access_time ) = self._keys_fifo.pop( 0 )
deletee_data = self._keys_to_data[ deletee_key ]
self._total_estimated_memory_footprint -= deletee_data.GetEstimatedMemoryFootprint()
del self._keys_to_data[ deletee_key ]
self._RecalcMemoryUsage()
def _RecalcMemoryUsage( self ):
self._total_estimated_memory_footprint = sum( ( data.GetEstimatedMemoryFootprint() for data in self._keys_to_data.values() ) )
def Clear( self ):
@ -566,7 +571,7 @@ class DataCache( object ):
self._keys_fifo.append( ( key, HydrusData.GetNow() ) )
self._total_estimated_memory_footprint += data.GetEstimatedMemoryFootprint()
self._RecalcMemoryUsage()
@ -575,7 +580,10 @@ class DataCache( object ):
with self._lock:
if key not in self._keys_to_data: raise Exception( 'Cache error! Looking for ' + HydrusData.ToUnicode( key ) + ', but it was missing.' )
if key not in self._keys_to_data:
raise Exception( 'Cache error! Looking for ' + HydrusData.ToUnicode( key ) + ', but it was missing.' )
for ( i, ( fifo_key, last_access_time ) ) in enumerate( self._keys_fifo ):
@ -595,7 +603,10 @@ class DataCache( object ):
def HasData( self, key ):
with self._lock: return key in self._keys_to_data
with self._lock:
return key in self._keys_to_data
def MaintainCache( self ):
@ -604,18 +615,22 @@ class DataCache( object ):
while True:
if len( self._keys_fifo ) == 0: break
if len( self._keys_fifo ) == 0:
break
else:
oldest_index = 0
( key, last_access_time ) = self._keys_fifo[ oldest_index ]
( key, last_access_time ) = self._keys_fifo[ 0 ]
if HydrusData.TimeHasPassed( last_access_time + 1200 ):
self._DeleteItem( oldest_index )
self._DeleteItem()
else:
break
else: break
@ -933,12 +948,6 @@ class RenderedImageCache( object ):
if self._type == 'fullscreen': self._data_cache = DataCache( self._controller, 'fullscreen_cache_size' )
elif self._type == 'preview': self._data_cache = DataCache( self._controller, 'preview_cache_size' )
self._total_estimated_memory_footprint = 0
self._keys_being_rendered = {}
self._controller.sub( self, 'FinishedRendering', 'finished_rendering' )
def Clear( self ): self._data_cache.Clear()
@ -958,16 +967,22 @@ class RenderedImageCache( object ):
target_resolution = media.GetResolution()
else:
target_resolution = ( target_width, target_height ) # to convert from wx.size or list to tuple for the cache key
key = ( hash, target_resolution )
if self._data_cache.HasData( key ): return self._data_cache.GetData( key )
elif key in self._keys_being_rendered: return self._keys_being_rendered[ key ]
if self._data_cache.HasData( key ):
return self._data_cache.GetData( key )
else:
image_container = ClientRendering.RasterContainerImage( media, target_resolution )
self._keys_being_rendered[ key ] = image_container
self._data_cache.AddData( key, image_container )
return image_container
@ -977,19 +992,7 @@ class RenderedImageCache( object ):
key = ( hash, target_resolution )
return self._data_cache.HasData( key ) or key in self._keys_being_rendered
def FinishedRendering( self, key ):
if key in self._keys_being_rendered:
image_container = self._keys_being_rendered[ key ]
del self._keys_being_rendered[ key ]
self._data_cache.AddData( key, image_container )
return self._data_cache.HasData( key )
class ThumbnailCache( object ):

1027
include/ClientDB.py
View File

File diff suppressed because it is too large Load Diff

86
include/ClientDBACCache.py
View File

@ -125,12 +125,7 @@ class SpecificServicesDB( HydrusDB.HydrusDB ):
def _CreateDB( self ):
self._c.execute( 'PRAGMA auto_vacuum = 0;' ) # none
if HC.PLATFORM_WINDOWS:
self._c.execute( 'PRAGMA page_size = 4096;' )
HydrusDB.SetupDBCreatePragma( self._c, no_wal = self._no_wal )
try: self._c.execute( 'BEGIN IMMEDIATE' )
except Exception as e:
@ -291,41 +286,6 @@ class SpecificServicesDB( HydrusDB.HydrusDB ):
self._c.execute( 'UPDATE version SET version = ?;', ( version + 1, ) )
def _Vacuum( self ):
self._c.execute( 'COMMIT' )
if not self._fast_big_transaction_wal:
self._c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
if HC.PLATFORM_WINDOWS:
ideal_page_size = 4096
else:
ideal_page_size = 1024
( page_size, ) = self._c.execute( 'PRAGMA page_size;' ).fetchone()
if page_size != ideal_page_size:
self._c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
self._c.execute( 'PRAGMA page_size = ' + str( ideal_page_size ) + ';' )
self._c.execute( 'VACUUM' )
self._c.execute( 'REPLACE INTO maintenance_timestamps ( name, timestamp ) VALUES ( ?, ? );', ( 'vacuum', HydrusData.GetNow() ) )
self._InitDBCursor()
self._c.execute( 'BEGIN IMMEDIATE' )
def _Write( self, action, *args, **kwargs ):
if action == 'add_files': result = self._AddFiles( *args, **kwargs )
@ -335,7 +295,6 @@ class SpecificServicesDB( HydrusDB.HydrusDB ):
elif action == 'delete_mappings': result = self._DeleteMappings( *args, **kwargs )
elif action == 'pend_mappings': result = self._PendMappings( *args, **kwargs )
elif action == 'rescind_pending_mappings': result = self._RescindPendingMappings( *args, **kwargs )
elif action == 'vacuum': result = self._Vacuum( *args, **kwargs )
else: raise Exception( 'db received an unknown write command: ' + action )
return result
@ -384,12 +343,7 @@ class CombinedFilesDB( HydrusDB.HydrusDB ):
def _CreateDB( self ):
self._c.execute( 'PRAGMA auto_vacuum = 0;' ) # none
if HC.PLATFORM_WINDOWS:
self._c.execute( 'PRAGMA page_size = 4096;' )
HydrusDB.SetupDBCreatePragma( self._c, no_wal = self._no_wal )
try: self._c.execute( 'BEGIN IMMEDIATE' )
except Exception as e:
@ -451,46 +405,10 @@ class CombinedFilesDB( HydrusDB.HydrusDB ):
self._c.execute( 'UPDATE version SET version = ?;', ( version + 1, ) )
def _Vacuum( self ):
self._c.execute( 'COMMIT' )
if not self._fast_big_transaction_wal:
self._c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
if HC.PLATFORM_WINDOWS:
ideal_page_size = 4096
else:
ideal_page_size = 1024
( page_size, ) = self._c.execute( 'PRAGMA page_size;' ).fetchone()
if page_size != ideal_page_size:
self._c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
self._c.execute( 'PRAGMA page_size = ' + str( ideal_page_size ) + ';' )
self._c.execute( 'VACUUM' )
self._c.execute( 'REPLACE INTO maintenance_timestamps ( name, timestamp ) VALUES ( ?, ? );', ( 'vacuum', HydrusData.GetNow() ) )
self._InitDBCursor()
self._c.execute( 'BEGIN IMMEDIATE' )
def _Write( self, action, *args, **kwargs ):
if action == 'update_counts': result = self._UpdateCounts( *args, **kwargs )
elif action == 'analyze': result = self._Analyze( *args, **kwargs )
elif action == 'vacuum': result = self._Vacuum( *args, **kwargs )
else: raise Exception( 'db received an unknown write command: ' + action )
return result

5
include/ClientData.py
View File

@ -402,6 +402,9 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
self._dictionary[ 'booleans' ][ 'filter_inbox_and_archive_predicates' ] = True
self._dictionary[ 'booleans' ][ 'waiting_politely_text' ] = False
self._dictionary[ 'booleans' ][ 'show_thumbnail_title_banner' ] = True
self._dictionary[ 'booleans' ][ 'show_thumbnail_page' ] = True
self._dictionary[ 'noneable_integers' ] = {}
self._dictionary[ 'noneable_integers' ][ 'forced_search_limit' ] = None
@ -1449,7 +1452,7 @@ class ServiceRepository( ServiceRestricted ):
if os.path.exists( path ):
size = HydrusPaths.GetPathSize( path )
size = os.path.getsize( path )
if size == 0:

2
include/ClientFiles.py
View File

@ -503,7 +503,7 @@ class ExportFolder( HydrusSerialisable.SerialisableBaseNamed ):
elif os.path.exists( dest_path ):
dest_size = HydrusPaths.GetPathSize( dest_path )
dest_size = os.path.getsize( dest_path )
if dest_size == size:

11
include/ClientGUI.py
View File

@ -393,6 +393,8 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
def do_it():
started = HydrusData.GetNow()
service = self._controller.GetServicesManager().GetService( service_key )
service.Request( HC.POST, 'backup' )
@ -415,7 +417,9 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
result = service.Request( HC.GET, 'busy' )
HydrusData.ShowText( 'Server backup done!' )
it_took = HydrusData.GetNow() - started
HydrusData.ShowText( 'Server backup done in ' + HydrusData.ConvertTimeDeltaToPrettyString( it_took ) + '!' )
message = 'This will tell the server to lock and copy its database files. It will probably take a few minutes to complete, during which time it will not be able to serve any requests.'
@ -2130,7 +2134,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
job_key.DeleteVariable( 'popup_gauge_1' )
job_key.SetVariable( 'popup_text_1', 'upload done!' )
job_key.SetVariable( 'popup_text_1', u'upload done!' )
HydrusData.Print( job_key.ToString() )
@ -3272,11 +3276,10 @@ class FrameReviewServices( ClientGUICommon.Frame ):
elif service_type in HC.TAG_SERVICES:
num_files = service_info[ HC.SERVICE_INFO_NUM_FILES ]
num_namespaces = service_info[ HC.SERVICE_INFO_NUM_NAMESPACES ]
num_tags = service_info[ HC.SERVICE_INFO_NUM_TAGS ]
num_mappings = service_info[ HC.SERVICE_INFO_NUM_MAPPINGS ]
self._tags_text.SetLabelText( HydrusData.ConvertIntToPrettyString( num_files ) + ' hashes, ' + HydrusData.ConvertIntToPrettyString( num_namespaces ) + ' namespaces, ' + HydrusData.ConvertIntToPrettyString( num_tags ) + ' tags, totalling ' + HydrusData.ConvertIntToPrettyString( num_mappings ) + ' mappings' )
self._tags_text.SetLabelText( HydrusData.ConvertIntToPrettyString( num_files ) + ' hashes, ' + HydrusData.ConvertIntToPrettyString( num_tags ) + ' tags, totalling ' + HydrusData.ConvertIntToPrettyString( num_mappings ) + ' mappings' )
if service_type == HC.TAG_REPOSITORY:

73
include/ClientGUICanvas.py
View File

@ -135,6 +135,7 @@ class Animation( wx.Window ):
self._left_down_event = None
self._a_frame_has_been_drawn = False
self._has_played_once_through = False
self._num_frames = self._media.GetNumFrames()
@ -337,6 +338,11 @@ class Animation( wx.Window ):
self._TellAnimationBarAboutPausedStatus()
def HasPlayedOnceThrough( self ):
return self._has_played_once_through
def IsPlaying( self ):
return not self._paused
@ -389,6 +395,11 @@ class Animation( wx.Window ):
self._current_frame_index = ( self._current_frame_index + 1 ) % num_frames
if self._current_frame_index == 0:
self._has_played_once_through = True
self._current_frame_drawn = False
self._video_container.SetFramePosition( self._current_frame_index )
@ -2461,6 +2472,7 @@ class CanvasMediaListBrowser( CanvasMediaListNavigable ):
CanvasMediaListNavigable.__init__( self, my_parent, page_key, media_results )
self._timer_slideshow = wx.Timer( self, id = ID_TIMER_SLIDESHOW )
self._timer_slideshow_interval = 0
self.Bind( wx.EVT_TIMER, self.TIMEREventSlideshow, id = ID_TIMER_SLIDESHOW )
@ -2495,8 +2507,14 @@ class CanvasMediaListBrowser( CanvasMediaListNavigable ):
def _PausePlaySlideshow( self ):
if self._timer_slideshow.IsRunning(): self._timer_slideshow.Stop()
elif self._timer_slideshow.GetInterval() > 0: self._timer_slideshow.Start()
if self._timer_slideshow.IsRunning():
self._timer_slideshow.Stop()
elif self._timer_slideshow.GetInterval() > 0:
self._timer_slideshow.Start()
def _StartSlideshow( self, interval = None ):
@ -2515,7 +2533,12 @@ class CanvasMediaListBrowser( CanvasMediaListNavigable ):
if interval > 0: self._timer_slideshow.Start( interval, wx.TIMER_CONTINUOUS )
if interval > 0:
self._timer_slideshow_interval = interval
self._timer_slideshow.Start( self._timer_slideshow_interval, wx.TIMER_CONTINUOUS )
def EventCharHook( self, event ):
@ -2774,7 +2797,19 @@ class CanvasMediaListBrowser( CanvasMediaListNavigable ):
try:
self._ShowNext()
if self._media_container is not None:
if self._media_container.ReadyToSlideshow():
self._ShowNext()
self._timer_slideshow.Start( self._timer_slideshow_interval, wx.TIMER_CONTINUOUS )
else:
self._timer_slideshow.Start( 250, wx.TIMER_CONTINUOUS )
except wx.PyDeadObjectError:
@ -3428,6 +3463,27 @@ class MediaContainer( wx.Window ):
def ReadyToSlideshow( self ):
if isinstance( self._media_window, Animation ):
if self._media_window.IsPlaying() and not self._media_window.HasPlayedOnceThrough():
return False
if isinstance( self._media_window, StaticImage ):
if not self._media_window.IsRendered():
return False
return True
class EmbedButton( wx.Window ):
def __init__( self, parent, size ):
@ -3562,6 +3618,8 @@ class StaticImage( wx.Window ):
self._image_cache = image_cache
self._image_container = self._image_cache.GetImage( self._media, initial_size )
self._is_rendered = False
( initial_width, initial_height ) = initial_size
self._canvas_bmp = wx.EmptyBitmap( initial_width, initial_height, 24 )
@ -3613,6 +3671,8 @@ class StaticImage( wx.Window ):
wx.CallAfter( wx_bitmap.Destroy )
self._is_rendered = True
self._dirty = False
@ -3684,6 +3744,11 @@ class StaticImage( wx.Window ):
def IsRendered( self ):
return self._is_rendered
def TIMEREventRenderWait( self, event ):
try:

27
include/ClientGUICommon.py
View File

@ -4543,6 +4543,11 @@ class PopupMessage( PopupWindow ):
self.GetParent().MakeSureEverythingFits()
def GetJobKey( self ):
return self._job_key
def TryToDismiss( self ):
if self._job_key.IsPausable() or self._job_key.IsCancellable(): return
@ -4831,6 +4836,28 @@ class PopupMessageManager( wx.Frame ):
def CleanBeforeDestroy( self ):
for job_key in self._pending_job_keys:
if job_key.IsCancellable():
job_key.Cancel()
sizer_items = self._message_vbox.GetChildren()
for sizer_item in sizer_items:
message_window = sizer_item.GetWindow()
job_key = message_window.GetJobKey()
if job_key.IsCancellable():
job_key.Cancel()
sys.excepthook = self._old_excepthook
HydrusData.ShowException = self._old_show_exception

2
include/ClientGUIDialogs.py
View File

@ -1705,7 +1705,7 @@ class DialogInputLocalFiles( Dialog ):
break
size = HydrusPaths.GetPathSize( path )
size = os.path.getsize( path )
if size == 0:

24
include/ClientGUIDialogsManage.py
View File

@ -4203,8 +4203,13 @@ class DialogManageOptions( ClientGUIDialogs.Dialog ):
self._rating_dialog_position = wx.CheckBox( self )
self._hide_preview = wx.CheckBox( self )
self._show_thumbnail_title_banner = wx.CheckBox( self )
self._show_thumbnail_page = wx.CheckBox( self )
#
self._new_options = HydrusGlobals.client_controller.GetNewOptions()
gui_session_names = HydrusGlobals.client_controller.Read( 'serialisable_names', HydrusSerialisable.SERIALISABLE_TYPE_GUI_SESSION )
if 'last session' not in gui_session_names: gui_session_names.insert( 0, 'last session' )
@ -4240,6 +4245,10 @@ class DialogManageOptions( ClientGUIDialogs.Dialog ):
self._hide_preview.SetValue( HC.options[ 'hide_preview' ] )
self._show_thumbnail_title_banner.SetValue( self._new_options.GetBoolean( 'show_thumbnail_title_banner' ) )
self._show_thumbnail_page.SetValue( self._new_options.GetBoolean( 'show_thumbnail_page' ) )
#
gridbox = wx.FlexGridSizer( 0, 2 )
@ -4276,6 +4285,12 @@ class DialogManageOptions( ClientGUIDialogs.Dialog ):
gridbox.AddF( wx.StaticText( self, label = 'Hide the preview window: ' ), CC.FLAGS_MIXED )
gridbox.AddF( self._hide_preview, CC.FLAGS_MIXED )
gridbox.AddF( wx.StaticText( self, label = 'Show \'title\' banner on thumbnails: ' ), CC.FLAGS_MIXED )
gridbox.AddF( self._show_thumbnail_title_banner, CC.FLAGS_MIXED )
gridbox.AddF( wx.StaticText( self, label = 'Show volume/chapter/page number on thumbnails: ' ), CC.FLAGS_MIXED )
gridbox.AddF( self._show_thumbnail_page, CC.FLAGS_MIXED )
self.SetSizer( gridbox )
@ -4329,6 +4344,9 @@ class DialogManageOptions( ClientGUIDialogs.Dialog ):
HC.options[ 'hide_preview' ] = self._hide_preview.GetValue()
self._new_options.SetBoolean( 'show_thumbnail_title_banner', self._show_thumbnail_title_banner.GetValue() )
self._new_options.SetBoolean( 'show_thumbnail_page', self._show_thumbnail_page.GetValue() )
class _MediaPanel( wx.Panel ):
@ -7606,21 +7624,17 @@ class DialogManageTagCensorship( ClientGUIDialogs.Dialog ):
services = HydrusGlobals.client_controller.GetServicesManager().GetServices( ( HC.COMBINED_TAG, HC.TAG_REPOSITORY, HC.LOCAL_TAG ) )
default_tag_repository_key = HC.options[ 'default_tag_repository' ]
for service in services:
service_key = service.GetServiceKey()
name = service.GetName()
if service_key == default_tag_repository_key: default_name = name
page = self._Panel( self._tag_services, service_key, initial_value )
self._tag_services.AddPage( name, page )
self._tag_services.Select( default_name )
self._tag_services.Select( 'all known tags' )
#

354
include/ClientGUIMedia.py
View File

@ -470,23 +470,84 @@ class MediaPanel( ClientMedia.ListeningMediaList, wx.ScrolledWindow ):
num_selected = self._GetNumSelected()
pretty_total_size = self._GetPrettyTotalSelectedSize()
( sorted_mime_classes, selected_mime_classes ) = self._GetSortedSelectedMimeClasses()
if num_selected == 0:
if sorted_mime_classes == set( [ 'image' ] ):
if num_files == 1: s = '1 file'
else: s = HydrusData.ConvertIntToPrettyString( num_files ) + ' files'
num_files_descriptor = 'image'
elif sorted_mime_classes == set( [ 'video' ] ):
num_files_descriptor = 'video'
elif num_selected == 1: s = '1 of ' + HydrusData.ConvertIntToPrettyString( num_files ) + ' files selected, ' + pretty_total_size
else:
num_inbox = sum( ( media.GetNumInbox() for media in self._selected_media ) )
num_files_descriptor = 'file'
if num_inbox == num_selected: inbox_phrase = 'all in inbox, '
elif num_inbox == 0: inbox_phrase = 'all archived, '
else: inbox_phrase = HydrusData.ConvertIntToPrettyString( num_inbox ) + ' in inbox and ' + HydrusData.ConvertIntToPrettyString( num_selected - num_inbox ) + ' archived, '
if selected_mime_classes == set( [ 'image' ] ):
s = HydrusData.ConvertIntToPrettyString( num_selected ) + ' of ' + HydrusData.ConvertIntToPrettyString( num_files ) + ' files selected, ' + inbox_phrase + 'totalling ' + pretty_total_size
selected_files_descriptor = 'image'
elif selected_mime_classes == set( [ 'video' ] ):
selected_files_descriptor = 'video'
else:
selected_files_descriptor = 'file'
if num_files == 1:
num_files_string = '1 ' + num_files_descriptor
else:
num_files_string = HydrusData.ConvertIntToPrettyString( num_files ) + ' ' + num_files_descriptor + 's'
if selected_files_descriptor == num_files_descriptor:
selected_files_string = HydrusData.ConvertIntToPrettyString( num_selected )
else:
if num_selected == 1:
selected_files_string = '1 ' + selected_files_descriptor
else:
selected_files_string = HydrusData.ConvertIntToPrettyString( num_selected ) + ' ' + selected_files_descriptor + 's'
s = num_files_string # 23 files
if num_selected > 0:
s += ' - '
if num_selected == 1: # 23 files - 1 video selected, file_info
( selected_media, ) = self._selected_media
s += selected_files_string + ' selected, ' + selected_media.GetPrettyInfo()
else: # 23 files - 5 selected, selection_info
num_inbox = sum( ( media.GetNumInbox() for media in self._selected_media ) )
if num_inbox == num_selected: inbox_phrase = 'all in inbox, '
elif num_inbox == 0: inbox_phrase = 'all archived, '
else: inbox_phrase = HydrusData.ConvertIntToPrettyString( num_inbox ) + ' in inbox and ' + HydrusData.ConvertIntToPrettyString( num_selected - num_inbox ) + ' archived, '
pretty_total_size = self._GetPrettyTotalSelectedSize()
s += selected_files_string + ' selected, ' + inbox_phrase + 'totalling ' + pretty_total_size
return s
@ -544,6 +605,59 @@ class MediaPanel( ClientMedia.ListeningMediaList, wx.ScrolledWindow ):
def _GetSortedSelectedMimeClasses( self ):
sorted_mimes = set()
for media in self._sorted_media:
mime = media.GetMime()
if mime in HC.IMAGES:
sorted_mimes.add( 'image' )
elif mime in HC.VIDEO:
sorted_mimes.add( 'video' )
elif mime in HC.AUDIO:
sorted_mimes.add( 'audio' )
else:
sorted_mimes.add( 'misc' )
selected_mimes = set()
for media in self._selected_media:
mime = media.GetMime()
if mime in HC.IMAGES:
selected_mimes.add( 'image' )
elif mime in HC.VIDEO:
selected_mimes.add( 'video' )
elif mime in HC.AUDIO:
selected_mimes.add( 'audio' )
else:
selected_mimes.add( 'misc' )
return ( sorted_mimes, selected_mimes )
def _HitMedia( self, media, ctrl, shift ):
if media is None:
@ -2853,15 +2967,6 @@ class Thumbnail( Selectable ):
local = self.GetLocationsManager().HasLocal()
namespaces = self.GetTagsManager().GetCombinedNamespaces( ( 'creator', 'series', 'title', 'volume', 'chapter', 'page' ) )
creators = namespaces[ 'creator' ]
series = namespaces[ 'series' ]
titles = namespaces[ 'title' ]
volumes = namespaces[ 'volume' ]
chapters = namespaces[ 'chapter' ]
pages = namespaces[ 'page' ]
thumbnail_hydrus_bmp = HydrusGlobals.client_controller.GetCache( 'thumbnail' ).GetThumbnail( self )
( width, height ) = ClientData.AddPaddingToDimensions( HC.options[ 'thumbnail_dimensions' ], CC.THUMBNAIL_BORDER * 2 )
@ -2897,126 +3002,143 @@ class Thumbnail( Selectable ):
wx.CallAfter( wx_bmp.Destroy )
collections_string = ''
namespaces = self.GetTagsManager().GetCombinedNamespaces( ( 'creator', 'series', 'title', 'volume', 'chapter', 'page' ) )
if len( volumes ) > 0:
creators = namespaces[ 'creator' ]
series = namespaces[ 'series' ]
titles = namespaces[ 'title' ]
volumes = namespaces[ 'volume' ]
chapters = namespaces[ 'chapter' ]
pages = namespaces[ 'page' ]
new_options = HydrusGlobals.client_controller.GetNewOptions()
if new_options.GetBoolean( 'show_thumbnail_page' ):
if len( volumes ) == 1:
collections_string = ''
if len( volumes ) > 0:
( volume, ) = volumes
if len( volumes ) == 1:
( volume, ) = volumes
collections_string = 'v' + str( volume )
else:
volumes_sorted = HydrusTags.SortTags( volumes )
collections_string_append = 'v' + str( volumes_sorted[0] ) + '-' + str( volumes_sorted[-1] )
collections_string = 'v' + str( volume )
if len( chapters ) > 0:
else:
if len( chapters ) == 1:
( chapter, ) = chapters
collections_string_append = 'c' + str( chapter )
else:
chapters_sorted = HydrusTags.SortTags( chapters )
collections_string_append = 'c' + str( chapters_sorted[0] ) + '-' + str( chapters_sorted[-1] )
volumes_sorted = HydrusTags.SortTags( volumes )
if len( collections_string ) > 0: collections_string += '-' + collections_string_append
else: collections_string = collections_string_append
collections_string_append = 'v' + str( volumes_sorted[0] ) + '-' + str( volumes_sorted[-1] )
if len( pages ) > 0:
if len( pages ) == 1:
( page, ) = pages
collections_string_append = 'p' + str( page )
else:
pages_sorted = HydrusTags.SortTags( pages )
collections_string_append = 'p' + str( pages_sorted[0] ) + '-' + str( pages_sorted[-1] )
if len( collections_string ) > 0: collections_string += '-' + collections_string_append
else: collections_string = collections_string_append
if len( collections_string ) > 0:
dc.SetFont( wx.SystemSettings.GetFont( wx.SYS_DEFAULT_GUI_FONT ) )
( text_x, text_y ) = dc.GetTextExtent( collections_string )
top_left_x = width - text_x - CC.THUMBNAIL_BORDER
top_left_y = height - text_y - CC.THUMBNAIL_BORDER
dc.SetBrush( wx.Brush( CC.COLOUR_UNSELECTED ) )
dc.SetTextForeground( CC.COLOUR_SELECTED_DARK )
dc.SetPen( wx.TRANSPARENT_PEN )
dc.DrawRectangle( top_left_x - 1, top_left_y - 1, text_x + 2, text_y + 2 )
dc.DrawText( collections_string, top_left_x, top_left_y )
if len( chapters ) > 0:
if new_options.GetBoolean( 'show_thumbnail_title_banner' ):
if len( chapters ) == 1:
siblings_manager = HydrusGlobals.client_controller.GetManager( 'tag_siblings' )
upper_info_string = ''
if len( creators ) > 0:
( chapter, ) = chapters
creators = siblings_manager.CollapseNamespacedTags( 'creator', creators )
collections_string_append = 'c' + str( chapter )
upper_info_string = ', '.join( creators )
else:
chapters_sorted = HydrusTags.SortTags( chapters )
collections_string_append = 'c' + str( chapters_sorted[0] ) + '-' + str( chapters_sorted[-1] )
if len( series ) > 0 or len( titles ) > 0: upper_info_string += ' - '
if len( collections_string ) > 0: collections_string += '-' + collections_string_append
else: collections_string = collections_string_append
if len( pages ) > 0:
if len( pages ) == 1:
if len( series ) > 0:
( page, ) = pages
series = siblings_manager.CollapseNamespacedTags( 'series', series )
collections_string_append = 'p' + str( page )
upper_info_string += ', '.join( series )
else:
elif len( titles ) > 0:
pages_sorted = HydrusTags.SortTags( pages )
titles = siblings_manager.CollapseNamespacedTags( 'title', titles )
collections_string_append = 'p' + str( pages_sorted[0] ) + '-' + str( pages_sorted[-1] )
upper_info_string += ', '.join( titles )
if len( collections_string ) > 0: collections_string += '-' + collections_string_append
else: collections_string = collections_string_append
if len( collections_string ) > 0:
dc.SetFont( wx.SystemSettings.GetFont( wx.SYS_DEFAULT_GUI_FONT ) )
( text_x, text_y ) = dc.GetTextExtent( collections_string )
top_left_x = width - text_x - CC.THUMBNAIL_BORDER
top_left_y = height - text_y - CC.THUMBNAIL_BORDER
dc.SetBrush( wx.Brush( CC.COLOUR_UNSELECTED ) )
dc.SetTextForeground( CC.COLOUR_SELECTED_DARK )
dc.SetPen( wx.TRANSPARENT_PEN )
dc.DrawRectangle( top_left_x - 1, top_left_y - 1, text_x + 2, text_y + 2 )
dc.DrawText( collections_string, top_left_x, top_left_y )
siblings_manager = HydrusGlobals.client_controller.GetManager( 'tag_siblings' )
upper_info_string = ''
if len( creators ) > 0:
creators = siblings_manager.CollapseNamespacedTags( 'creator', creators )
upper_info_string = ', '.join( creators )
if len( series ) > 0 or len( titles ) > 0: upper_info_string += ' - '
if len( series ) > 0:
series = siblings_manager.CollapseNamespacedTags( 'series', series )
upper_info_string += ', '.join( series )
elif len( titles ) > 0:
titles = siblings_manager.CollapseNamespacedTags( 'title', titles )
upper_info_string += ', '.join( titles )
if len( upper_info_string ) > 0:
dc.SetFont( wx.SystemSettings.GetFont( wx.SYS_DEFAULT_GUI_FONT ) )
( text_x, text_y ) = dc.GetTextExtent( upper_info_string )
top_left_x = int( ( width - text_x ) / 2 )
top_left_y = CC.THUMBNAIL_BORDER
dc.SetBrush( wx.Brush( CC.COLOUR_UNSELECTED ) )
dc.SetTextForeground( CC.COLOUR_SELECTED_DARK )
dc.SetPen( wx.TRANSPARENT_PEN )
dc.DrawRectangle( 0, top_left_y - 1, width, text_y + 2 )
dc.DrawText( upper_info_string, top_left_x, top_left_y )
if len( upper_info_string ) > 0:
dc.SetFont( wx.SystemSettings.GetFont( wx.SYS_DEFAULT_GUI_FONT ) )
( text_x, text_y ) = dc.GetTextExtent( upper_info_string )
top_left_x = int( ( width - text_x ) / 2 )
top_left_y = CC.THUMBNAIL_BORDER
dc.SetBrush( wx.Brush( CC.COLOUR_UNSELECTED ) )
dc.SetTextForeground( CC.COLOUR_SELECTED_DARK )
dc.SetPen( wx.TRANSPARENT_PEN )
dc.DrawRectangle( 0, top_left_y - 1, width, text_y + 2 )
dc.DrawText( upper_info_string, top_left_x, top_left_y )
dc.SetBrush( wx.TRANSPARENT_BRUSH )

4
include/ClientImageHandling.py
View File

@ -16,12 +16,10 @@ def EfficientlyResizeNumpyImage( numpy_image, ( target_x, target_y ) ):
if target_x >= im_x and target_y >= im_y: return numpy_image
result = numpy_image
# this seems to slow things down a lot, at least for cv!
#if im_x > 2 * target_x and im_y > 2 * target_y: result = cv2.resize( numpy_image, ( 2 * target_x, 2 * target_y ), interpolation = cv2.INTER_NEAREST )
return cv2.resize( result, ( target_x, target_y ), interpolation = cv2.INTER_AREA )
return cv2.resize( numpy_image, ( target_x, target_y ), interpolation = cv2.INTER_AREA )
def EfficientlyThumbnailNumpyImage( numpy_image, ( target_x, target_y ) ):

19
include/ClientRendering.py
View File

@ -138,8 +138,6 @@ class RasterContainerImage( RasterContainer ):
self._hydrus_bitmap = hydrus_bitmap
HydrusGlobals.client_controller.pub( 'finished_rendering', self.GetKey() )
def GetEstimatedMemoryFootprint( self ):
@ -149,15 +147,16 @@ class RasterContainerImage( RasterContainer ):
return width * height * 3
else: return self._hydrus_bitmap.GetEstimatedMemoryFootprint()
else:
return self._hydrus_bitmap.GetEstimatedMemoryFootprint()
def GetHash( self ): return self._media.GetHash()
def GetHydrusBitmap( self ): return self._hydrus_bitmap
def GetKey( self ): return ( self._media.GetHash(), self._target_resolution )
def GetNumFrames( self ): return self._media.GetNumFrames()
def GetResolution( self ): return self._media.GetResolution()
@ -447,7 +446,10 @@ class HydrusBitmap( object ):
( width, height ) = self._size
if self._format == wx.BitmapBufferFormat_RGB: return wx.ImageFromBuffer( width, height, self._GetData() )
if self._format == wx.BitmapBufferFormat_RGB:
return wx.ImageFromBuffer( width, height, self._GetData() )
else:
bitmap = wx.BitmapFromBufferRGBA( width, height, self._GetData() )
@ -460,7 +462,10 @@ class HydrusBitmap( object ):
def GetEstimatedMemoryFootprint( self ): return len( self._data )
def GetEstimatedMemoryFootprint( self ):
return len( self._data )
def GetSize( self ): return self._size

2
include/HydrusConstants.py
View File

@ -54,7 +54,7 @@ options = {}
# Misc
NETWORK_VERSION = 17
SOFTWARE_VERSION = 200
SOFTWARE_VERSION = 201
UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )

228
include/HydrusDB.py
View File

@ -5,16 +5,131 @@ import HydrusConstants as HC
import HydrusData
import HydrusExceptions
import HydrusGlobals
import HydrusPaths
import os
import pstats
import psutil
import Queue
import random
import sqlite3
import sys
import tempfile
import threading
import traceback
import time
def CanVacuum( db_path, stop_time = None ):
db = sqlite3.connect( db_path, isolation_level = None, detect_types = sqlite3.PARSE_DECLTYPES )
c = db.cursor()
( page_size, ) = c.execute( 'PRAGMA page_size;' ).fetchone()
( page_count, ) = c.execute( 'PRAGMA page_count;' ).fetchone()
( freelist_count, ) = c.execute( 'PRAGMA freelist_count;' ).fetchone()
db_size = ( page_count - freelist_count ) * page_size
if stop_time is not None:
approx_vacuum_speed_mb_per_s = 1048576 * 3
approx_vacuum_duration = db_size / approx_vacuum_speed_mb_per_s
time_i_will_have_to_start = stop_time - approx_vacuum_duration
if HydrusData.TimeHasPassed( time_i_will_have_to_start ):
return False
temp_dir = tempfile.gettempdir()
( db_dir, db_filename ) = os.path.split( db_path )
temp_disk_usage = psutil.disk_usage( temp_dir )
a = HydrusPaths.GetDevice( temp_dir )
b = HydrusPaths.GetDevice( db_dir )
if HydrusPaths.GetDevice( temp_dir ) == HydrusPaths.GetDevice( db_dir ):
if temp_disk_usage.free < db_size * 2.2:
return False
else:
if temp_disk_usage.free < db_size * 1.1:
return False
db_disk_usage = psutil.disk_usage( db_dir )
if db_disk_usage.free < db_size * 1.1:
return False
return True
def SetupDBCreatePragma( c, no_wal = False ):
c.execute( 'PRAGMA auto_vacuum = 0;' ) # none
if HC.PLATFORM_WINDOWS:
c.execute( 'PRAGMA page_size = 4096;' )
if not no_wal:
c.execute( 'PRAGMA journal_mode = WAL;' )
c.execute( 'PRAGMA synchronous = 1;' )
def VacuumDB( db_path ):
db = sqlite3.connect( db_path, isolation_level = None, detect_types = sqlite3.PARSE_DECLTYPES )
c = db.cursor()
( previous_journal_mode, ) = c.execute( 'PRAGMA journal_mode;' ).fetchone()
fast_big_transaction_wal = not distutils.version.LooseVersion( sqlite3.sqlite_version ) < distutils.version.LooseVersion( '3.11.0' )
if previous_journal_mode == 'wal' and not fast_big_transaction_wal:
c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
if HC.PLATFORM_WINDOWS:
ideal_page_size = 4096
else:
ideal_page_size = 1024
( page_size, ) = c.execute( 'PRAGMA page_size;' ).fetchone()
if page_size != ideal_page_size:
c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
c.execute( 'PRAGMA page_size = ' + str( ideal_page_size ) + ';' )
c.execute( 'VACUUM;' )
if previous_journal_mode == 'wal':
c.execute( 'PRAGMA journal_mode = WAL;' )
class HydrusDB( object ):
READ_WRITE_ACTIONS = []
@ -200,63 +315,68 @@ class HydrusDB( object ):
self._c.execute( 'PRAGMA cache_size = -150000;' )
if self._no_wal:
self._c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
self._c.execute( 'PRAGMA synchronous = 2;' )
self._c.execute( 'SELECT * FROM sqlite_master;' ).fetchone()
else:
self._c.execute( 'PRAGMA journal_mode = WAL;' )
self._c.execute( 'PRAGMA synchronous = 1;' )
try:
self._c.execute( 'SELECT * FROM sqlite_master;' ).fetchone()
except sqlite3.OperationalError:
traceback.print_exc()
def create_no_wal_file():
HydrusGlobals.controller.CreateNoWALFile()
self._no_wal = True
if db_just_created:
del self._c
del self._db
os.remove( db_path )
create_no_wal_file()
self._InitDBCursor()
else:
self._c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
self._c.execute( 'PRAGMA synchronous = 2;' )
self._c.execute( 'SELECT * FROM sqlite_master;' ).fetchone()
create_no_wal_file()
self._c.execute( 'ATTACH ":memory:" AS mem;' )
self._AttachExternalDatabases()
db_names = [ name for ( index, name, path ) in self._c.execute( 'PRAGMA database_list;' ) if name not in ( 'mem', 'temp' ) ]
for db_name in db_names:
if self._no_wal:
self._c.execute( 'PRAGMA ' + db_name + '.journal_mode = TRUNCATE;' )
self._c.execute( 'PRAGMA ' + db_name + '.synchronous = 2;' )
self._c.execute( 'SELECT * FROM ' + db_name + '.sqlite_master;' ).fetchone()
else:
self._c.execute( 'PRAGMA ' + db_name + '.journal_mode = WAL;' )
self._c.execute( 'PRAGMA ' + db_name + '.synchronous = 1;' )
try:
self._c.execute( 'SELECT * FROM ' + db_name + '.sqlite_master;' ).fetchone()
except sqlite3.OperationalError:
traceback.print_exc()
def create_no_wal_file():
HydrusGlobals.controller.CreateNoWALFile()
self._no_wal = True
if db_just_created:
del self._c
del self._db
os.remove( db_path )
create_no_wal_file()
self._InitDBCursor()
else:
self._c.execute( 'PRAGMA ' + db_name + '.journal_mode = TRUNCATE;' )
self._c.execute( 'PRAGMA ' + db_name + '.synchronous = 2;' )
self._c.execute( 'SELECT * FROM ' + db_name + '.sqlite_master;' ).fetchone()
create_no_wal_file()
def _ManageDBError( self, job, e ):

2
include/HydrusFileHandling.py
View File

@ -147,7 +147,7 @@ def GetExtraHashesFromPath( path ):
def GetFileInfo( path ):
size = HydrusPaths.GetPathSize( path )
size = os.path.getsize( path )
if size == 0: raise HydrusExceptions.SizeException( 'File is of zero length!' )

31
include/HydrusPaths.py
View File

@ -2,6 +2,7 @@ import gc
import HydrusConstants as HC
import HydrusData
import os
import psutil
import send2trash
import shutil
import subprocess
@ -135,10 +136,29 @@ def DeletePath( path ):
os.remove( path )
def GetDevice( path ):
def GetPathSize( path ):
path = path.lower()
return os.lstat( path )[6]
partition_infos = psutil.disk_partitions()
def sort_descending_mountpoint( partition_info ): # i.e. put '/home' before '/'
return - len( partition_info.mountpoint )
partition_infos.sort( key = sort_descending_mountpoint )
for partition_info in partition_infos:
if path.startswith( partition_info.mountpoint.lower() ):
return partition_info.device
return None
def GetTempFile(): return tempfile.TemporaryFile()
def GetTempFileQuick(): return tempfile.SpooledTemporaryFile( max_size = 1024 * 1024 * 4 )
@ -262,11 +282,8 @@ def PathsHaveSameSizeAndDate( path1, path2 ):
if os.path.exists( path1 ) and os.path.exists( path2 ):
path1_stat = os.lstat( path1 )
path2_stat = os.lstat( path2 )
same_size = path1_stat[6] == path2_stat[6]
same_modified_time = path1_stat[8] == path2_stat[8]
same_size = os.path.getsize( path1 ) == os.path.getsize( path2 )
same_modified_time = os.path.getmtime( path1 ) == os.path.getmtime( path2 )
if same_size and same_modified_time:

2
include/HydrusServerResources.py
View File

@ -292,7 +292,7 @@ class HydrusResourceCommand( Resource ):
path = response_context.GetPath()
size = HydrusPaths.GetPathSize( path )
size = os.path.getsize( path )
if response_context.IsJSON():

6
include/HydrusTagArchive.py
View File

@ -136,12 +136,12 @@ class HydrusTagArchive( object ):
return tag_id
def BeginBigJob( self ): self._c.execute( 'BEGIN IMMEDIATE' )
def BeginBigJob( self ): self._c.execute( 'BEGIN IMMEDIATE;' )
def CommitBigJob( self ):
self._c.execute( 'COMMIT' )
self._c.execute( 'VACUUM' )
self._c.execute( 'COMMIT;' )
self._c.execute( 'VACUUM;' )
def AddMapping( self, hash, tag ):

240
include/ServerDB.py
View File

@ -460,6 +460,28 @@ class DB( HydrusDB.HydrusDB ):
def _AttachExternalDatabases( self ):
self._db_filenames[ 'master' ] = self._db_name + '.master.db'
master_db_path = os.path.join( self._db_dir, self._db_filenames[ 'master' ] )
if not os.path.exists( master_db_path ):
db = sqlite3.connect( master_db_path, isolation_level = None, detect_types = sqlite3.PARSE_DECLTYPES )
c = db.cursor()
HydrusDB.SetupDBCreatePragma( c, no_wal = self._no_wal )
c.execute( 'CREATE TABLE hashes ( hash_id INTEGER PRIMARY KEY, hash BLOB_BYTES UNIQUE );' )
c.execute( 'CREATE TABLE tags ( tag_id INTEGER PRIMARY KEY, tag TEXT UNIQUE );' )
del c
del db
self._c.execute( 'ATTACH ? AS external_master;', ( master_db_path, ) )
self._db_filenames[ 'mappings' ] = self._db_name + '.mappings.db'
mappings_db_path = os.path.join( self._db_dir, self._db_filenames[ 'mappings' ] )
@ -470,24 +492,81 @@ class DB( HydrusDB.HydrusDB ):
c = db.cursor()
c.execute( 'CREATE TABLE mapping_petitions ( service_id INTEGER REFERENCES services ON DELETE CASCADE, account_id INTEGER, tag_id INTEGER, hash_id INTEGER, reason_id INTEGER, timestamp INTEGER, status INTEGER, PRIMARY KEY( service_id, account_id, tag_id, hash_id, status ) );' )
HydrusDB.SetupDBCreatePragma( c, no_wal = self._no_wal )
c.execute( 'CREATE TABLE mapping_petitions ( service_id INTEGER, account_id INTEGER, tag_id INTEGER, hash_id INTEGER, reason_id INTEGER, timestamp INTEGER, status INTEGER, PRIMARY KEY( service_id, account_id, tag_id, hash_id, status ) ) WITHOUT ROWID;' )
c.execute( 'CREATE INDEX mapping_petitions_service_id_account_id_reason_id_tag_id_index ON mapping_petitions ( service_id, account_id, reason_id, tag_id );' )
c.execute( 'CREATE INDEX mapping_petitions_service_id_tag_id_hash_id_index ON mapping_petitions ( service_id, tag_id, hash_id );' )
c.execute( 'CREATE INDEX mapping_petitions_service_id_status_index ON mapping_petitions ( service_id, status );' )
c.execute( 'CREATE INDEX mapping_petitions_service_id_timestamp_index ON mapping_petitions ( service_id, timestamp );' )
c.execute( 'CREATE TABLE mappings ( service_id INTEGER REFERENCES services ON DELETE CASCADE, tag_id INTEGER, hash_id INTEGER, account_id INTEGER, timestamp INTEGER, PRIMARY KEY( service_id, tag_id, hash_id ) );' )
c.execute( 'CREATE TABLE mappings ( service_id INTEGER, tag_id INTEGER, hash_id INTEGER, account_id INTEGER, timestamp INTEGER, PRIMARY KEY( service_id, tag_id, hash_id ) ) WITHOUT ROWID;' )
c.execute( 'CREATE INDEX mappings_account_id_index ON mappings ( account_id );' )
c.execute( 'CREATE INDEX mappings_timestamp_index ON mappings ( timestamp );' )
c.execute( 'PRAGMA journal_mode = WAL;' )
c.execute( 'PRAGMA synchronous = 1;' )
del c
del db
self._c.execute( 'ATTACH ? AS mappings_external;', ( mappings_db_path, ) )
self._c.execute( 'ATTACH ? AS external_mappings;', ( mappings_db_path, ) )
def _Backup( self ):
self._c.execute( 'COMMIT;' )
self._CloseDBCursor()
try:
stop_time = HydrusData.GetNow() + 300
for filename in self._db_filenames.values():
db_path = os.path.join( self._db_dir, filename )
if HydrusDB.CanVacuum( db_path, stop_time ):
HydrusData.Print( 'backing up: vacuuming ' + filename )
HydrusDB.VacuumDB( db_path )
backup_path = os.path.join( HC.DB_DIR, 'server_backup' )
if not os.path.exists( backup_path ):
os.makedirs( backup_path )
for filename in self._db_filenames.values():
HydrusData.Print( 'backing up: copying ' + filename )
source = os.path.join( self._db_dir, filename )
dest = os.path.join( backup_path, filename )
shutil.copy2( source, dest )
HydrusData.Print( 'backing up: copying files' )
HydrusPaths.MirrorTree( HC.SERVER_FILES_DIR, os.path.join( backup_path, 'server_files' ) )
HydrusData.Print( 'backing up: copying thumbnails' )
HydrusPaths.MirrorTree( HC.SERVER_THUMBNAILS_DIR, os.path.join( backup_path, 'server_thumbnails' ) )
HydrusData.Print( 'backing up: copying updates' )
HydrusPaths.MirrorTree( HC.SERVER_UPDATES_DIR, os.path.join( backup_path, 'server_updates' ) )
finally:
self._InitDBCursor()
self._c.execute( 'BEGIN IMMEDIATE;' )
HydrusData.Print( 'backing up: done!' )
def _Ban( self, service_id, action, admin_account_id, subject_account_ids, reason_id, expires = None, lifetime = None ):
@ -631,9 +710,9 @@ class DB( HydrusDB.HydrusDB ):
self._c.execute( 'BEGIN IMMEDIATE' )
HydrusDB.SetupDBCreatePragma( self._c, no_wal = self._no_wal )
self._c.execute( 'PRAGMA auto_vacuum = 0;' ) # none
self._c.execute( 'BEGIN IMMEDIATE' )
now = HydrusData.GetNow()
@ -670,9 +749,6 @@ class DB( HydrusDB.HydrusDB ):
self._c.execute( 'CREATE INDEX file_petitions_service_id_status_index ON file_petitions ( service_id, status );' )
self._c.execute( 'CREATE INDEX file_petitions_service_id_timestamp_index ON file_petitions ( service_id, timestamp );' )
self._c.execute( 'CREATE TABLE hashes ( hash_id INTEGER PRIMARY KEY, hash BLOB_BYTES );' )
self._c.execute( 'CREATE UNIQUE INDEX hashes_hash_index ON hashes ( hash );' )
self._c.execute( 'CREATE TABLE ip_addresses ( service_id INTEGER REFERENCES services ON DELETE CASCADE, hash_id INTEGER, ip TEXT, timestamp INTEGER, PRIMARY KEY( service_id, hash_id ) );' )
self._c.execute( 'CREATE TABLE messages ( message_key BLOB_BYTES PRIMARY KEY, service_id INTEGER REFERENCES services ON DELETE CASCADE, account_id INTEGER, timestamp INTEGER );' )
@ -706,9 +782,6 @@ class DB( HydrusDB.HydrusDB ):
self._c.execute( 'CREATE INDEX tag_siblings_service_id_timestamp_index ON tag_siblings ( service_id, timestamp );' )
self._c.execute( 'CREATE INDEX tag_siblings_service_id_status_index ON tag_siblings ( service_id, status );' )
self._c.execute( 'CREATE TABLE tags ( tag_id INTEGER PRIMARY KEY, tag TEXT );' )
self._c.execute( 'CREATE UNIQUE INDEX tags_tag_index ON tags ( tag );' )
self._c.execute( 'CREATE TABLE update_cache ( service_id INTEGER REFERENCES services ON DELETE CASCADE, begin INTEGER, end INTEGER, dirty INTEGER_BOOLEAN, PRIMARY KEY( service_id, begin ) );' )
self._c.execute( 'CREATE UNIQUE INDEX update_cache_service_id_end_index ON update_cache ( service_id, end );' )
self._c.execute( 'CREATE INDEX update_cache_service_id_dirty_index ON update_cache ( service_id, dirty );' )
@ -1817,68 +1890,6 @@ class DB( HydrusDB.HydrusDB ):
def _MakeBackup( self ):
self._c.execute( 'COMMIT' )
if not self._fast_big_transaction_wal:
self._c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
if HC.PLATFORM_WINDOWS:
ideal_page_size = 4096
else:
ideal_page_size = 1024
( page_size, ) = self._c.execute( 'PRAGMA page_size;' ).fetchone()
if page_size != ideal_page_size:
self._c.execute( 'PRAGMA page_size = ' + str( ideal_page_size ) + ';' )
HydrusData.Print( 'backing up: vacuum' )
self._c.execute( 'VACUUM' )
self._CloseDBCursor()
backup_path = os.path.join( HC.DB_DIR, 'server_backup' )
if not os.path.exists( backup_path ):
os.makedirs( backup_path )
for filename in self._db_filenames.values():
HydrusData.Print( 'backing up: copying ' + filename )
source = os.path.join( self._db_dir, filename )
dest = os.path.join( backup_path, filename )
HydrusData.Print( 'backing up: copying files' )
HydrusPaths.MirrorTree( HC.SERVER_FILES_DIR, os.path.join( backup_path, 'server_files' ) )
HydrusData.Print( 'backing up: copying thumbnails' )
HydrusPaths.MirrorTree( HC.SERVER_THUMBNAILS_DIR, os.path.join( backup_path, 'server_thumbnails' ) )
HydrusData.Print( 'backing up: copying updates' )
HydrusPaths.MirrorTree( HC.SERVER_UPDATES_DIR, os.path.join( backup_path, 'server_updates' ) )
self._InitDBCursor()
self._c.execute( 'BEGIN IMMEDIATE' )
HydrusData.Print( 'backing up: done!' )
def _ManageDBError( self, job, e ):
( exception_type, value, tb ) = sys.exc_info()
@ -2533,18 +2544,83 @@ class DB( HydrusDB.HydrusDB ):
HydrusData.Print( 'exporting mappings to external db' )
self._c.execute( 'INSERT INTO mappings_external.mappings SELECT * FROM main.mappings;' )
self._c.execute( 'INSERT INTO external_mappings.mappings SELECT * FROM main.mappings;' )
self._c.execute( 'DROP TABLE main.mappings;' )
self._c.execute( 'INSERT INTO mappings_external.mapping_petitions SELECT * FROM main.mapping_petitions;' )
self._c.execute( 'INSERT INTO external_mappings.mapping_petitions SELECT * FROM main.mapping_petitions;' )
self._c.execute( 'DROP TABLE main.mapping_petitions;' )
HydrusData.Print( 'analyzing new external db' )
if version == 200:
self._c.execute( 'ANALYZE mappings;' )
self._c.execute( 'ANALYZE mapping_petitions;' )
HydrusData.Print( 'exporting hashes to external db' )
self._c.execute( 'INSERT INTO external_master.hashes SELECT * FROM main.hashes;' )
self._c.execute( 'DROP TABLE main.hashes;' )
HydrusData.Print( 'exporting tags to external db' )
self._c.execute( 'INSERT INTO external_master.tags SELECT * FROM main.tags;' )
self._c.execute( 'DROP TABLE main.tags;' )
#
HydrusData.Print( 'compacting mappings tables' )
self._c.execute( 'DROP INDEX mapping_petitions_service_id_account_id_reason_id_tag_id_index;' )
self._c.execute( 'DROP INDEX mapping_petitions_service_id_tag_id_hash_id_index;' )
self._c.execute( 'DROP INDEX mapping_petitions_service_id_status_index;' )
self._c.execute( 'DROP INDEX mapping_petitions_service_id_timestamp_index;' )
self._c.execute( 'DROP INDEX mappings_account_id_index;' )
self._c.execute( 'DROP INDEX mappings_timestamp_index;' )
self._c.execute( 'ALTER TABLE mapping_petitions RENAME TO mapping_petitions_old;' )
self._c.execute( 'ALTER TABLE mappings RENAME TO mappings_old;' )
self._c.execute( 'CREATE TABLE external_mappings.mapping_petitions ( service_id INTEGER, account_id INTEGER, tag_id INTEGER, hash_id INTEGER, reason_id INTEGER, timestamp INTEGER, status INTEGER, PRIMARY KEY( service_id, account_id, tag_id, hash_id, status ) ) WITHOUT ROWID;' )
self._c.execute( 'CREATE TABLE external_mappings.mappings ( service_id INTEGER, tag_id INTEGER, hash_id INTEGER, account_id INTEGER, timestamp INTEGER, PRIMARY KEY( service_id, tag_id, hash_id ) ) WITHOUT ROWID;' )
self._c.execute( 'INSERT INTO mapping_petitions SELECT * FROM mapping_petitions_old;' )
self._c.execute( 'INSERT INTO mappings SELECT * FROM mappings_old;' )
self._c.execute( 'DROP TABLE mapping_petitions_old;' )
self._c.execute( 'DROP TABLE mappings_old;' )
self._c.execute( 'CREATE INDEX external_mappings.mapping_petitions_service_id_account_id_reason_id_tag_id_index ON mapping_petitions ( service_id, account_id, reason_id, tag_id );' )
self._c.execute( 'CREATE INDEX external_mappings.mapping_petitions_service_id_tag_id_hash_id_index ON mapping_petitions ( service_id, tag_id, hash_id );' )
self._c.execute( 'CREATE INDEX external_mappings.mapping_petitions_service_id_status_index ON mapping_petitions ( service_id, status );' )
self._c.execute( 'CREATE INDEX external_mappings.mapping_petitions_service_id_timestamp_index ON mapping_petitions ( service_id, timestamp );' )
self._c.execute( 'CREATE INDEX external_mappings.mappings_account_id_index ON mappings ( account_id );' )
self._c.execute( 'CREATE INDEX external_mappings.mappings_timestamp_index ON mappings ( timestamp );' )
#
self._c.execute( 'COMMIT;' )
self._CloseDBCursor()
for filename in self._db_filenames.values():
HydrusData.Print( 'vacuuming ' + filename )
db_path = os.path.join( self._db_dir, filename )
if HydrusDB.CanVacuum( db_path ):
HydrusDB.VacuumDB( db_path )
self._InitDBCursor()
self._c.execute( 'BEGIN IMMEDIATE;' )
HydrusData.Print( 'The server has updated to version ' + str( version + 1 ) )
@ -2575,7 +2651,7 @@ class DB( HydrusDB.HydrusDB ):
if action == 'account': result = self._ModifyAccount( *args, **kwargs )
elif action == 'account_types': result = self._ModifyAccountTypes( *args, **kwargs )
elif action == 'analyze': result = self._Analyze( *args, **kwargs )
elif action == 'backup': result = self._MakeBackup( *args, **kwargs )
elif action == 'backup': result = self._Backup( *args, **kwargs )
elif action == 'check_data_usage': result = self._CheckDataUsage( *args, **kwargs )
elif action == 'check_monthly_data': result = self._CheckMonthlyData( *args, **kwargs )
elif action == 'clean_update': result = self._CleanUpdate( *args, **kwargs )

5
include/TestDB.py
View File

@ -51,7 +51,10 @@ class TestClientDB( unittest.TestCase ):
c = db.cursor()
c.execute( 'DELETE FROM mappings;' )
c.execute( 'DELETE FROM current_mappings;' )
c.execute( 'DELETE FROM pending_mappings;' )
c.execute( 'DELETE FROM deleted_mappings;' )
c.execute( 'DELETE FROM mapping_petitions;' )
def _read( self, action, *args, **kwargs ): return self._db.Read( action, HC.HIGH_PRIORITY, *args, **kwargs )