From 8b151cb58982fc03d885167f852a5d9476b83d1a Mon Sep 17 00:00:00 2001
From: Hydrus Network Developer <hydrus.admin@gmail.com>
Date: Wed, 17 Feb 2016 16:06:47 -0600
Subject: [PATCH] Version 193

---
 help/changelog.html               |  22 +++
 include/ClientCaches.py           |  42 +++++-
 include/ClientController.py       |  51 +++++--
 include/ClientDB.py               | 153 ++++++++++++++------
 include/ClientDaemons.py          |  13 +-
 include/ClientData.py             |  88 +++++++++---
 include/ClientDefaults.py         |   2 +-
 include/ClientDownloading.py      |   8 +-
 include/ClientFiles.py            |  14 +-
 include/ClientGUI.py              |  59 ++++++--
 include/ClientGUICanvas.py        |  24 +++-
 include/ClientGUICommon.py        |  25 ++--
 include/ClientGUIDialogs.py       |  49 +------
 include/ClientGUIDialogsManage.py |  71 +++++++---
 include/ClientGUIManagement.py    |   7 +-
 include/ClientGUIMedia.py         |  25 +++-
 include/ClientImporting.py        |   5 +-
 include/ClientMedia.py            |   2 +-
 include/ClientThreading.py        | 208 ++++++++++++++++++++++++++++
 include/HydrusConstants.py        |   2 +-
 include/HydrusController.py       |   3 +-
 include/HydrusDB.py               |  27 +++-
 include/HydrusData.py             |   7 +-
 include/HydrusExceptions.py       |   5 +-
 include/HydrusNATPunch.py         |  14 +-
 include/HydrusThreading.py        | 223 +++++++-----------------------
 include/ServerDB.py               |  18 ++-
 include/TestDialogs.py            |  21 ---
 test.py                           |   2 +
 29 files changed, 782 insertions(+), 408 deletions(-)
 create mode 100644 include/ClientThreading.py

diff --git a/help/changelog.html b/help/changelog.html
index 08e8883e..2ed48514 100755
--- a/help/changelog.html
+++ b/help/changelog.html
@@ -8,6 +8,28 @@
 		<div class="content">
 			<h3>changelog</h3>
 			<ul>
+				<li><h3>version 193</h3></li>
+				<ul>
+					<li>the client's local server and local booru can be turned off from their respective management panels, and from now on, the client will initialise with them this way.</li>
+					<li>if the local server or the local booru are not running, their copy/share commands won't appear in right-click menus</li>
+					<li>the welcome dialog is now a simpler popup message</li>
+					<li>incidence sorted tag lists are now sub-sorted by a-z lexicographic</li>
+					<li>pasting many tags that have siblings to the manage tags dialog will ask you if you want to always preference the sibling, saving time</li>
+					<li>added a 'clear deleted file records' button to the local file service on the review services window</li>
+					<li>idle mode now cannot naturally engage within the first two minutes since client boot</li>
+					<li>the autocomplete search logic will not count namespace characters in the autocomplete character threshold, so typing 'character:a' will not typically trigger a (very laggy) full search</li>
+					<li>putting a '*' anywhere in an autocomplete search_text will force a full search, ignoring the a/c character threshold</li>
+					<li>moved some specific 'give gui time to catch up' pause code to the generalised pause/cancel code that a lot of stuff uses, so big jobs should generally be a bit more polite</li>
+					<li>split the daemon class into two--one for big jobs that remains polite, and another for small jobs that triggers regardless of what else is going on. this should increase responsivity for a number of scenarios</li>
+					<li>fixed some bad wal failure detection and hence no-wal file creation on some instances of db cursor reinit (usually after service modification). because of now many superfluous no-wal files, existing no-wal files will be deleted on db update</li>
+					<li>some external storage location errors are improved</li>
+					<li>some internal and external storage location init is improved.</li>
+					<li>if an error is detected in the external storage location manager, it will not attempt to rebalance again until the client is rebooted</li>
+					<li>improved some upnp error catching</li>
+					<li>cleaned up some misc shutdown thread-gui interaction error spam</li>
+					<li>did some prep work on a future rewrite of daemon jobs pipeline</li>
+					<li>split up some mixed file/data/404 'stuff was missing' exception code</li>
+				</ul>
 				<li><h3>version 192</h3></li>
 				<ul>
 					<li>added a 'check on ok' button to the manage subscriptions dialog's subscription panel</li>
diff --git a/include/ClientCaches.py b/include/ClientCaches.py
index 87bc7687..7ef7a3cf 100644
--- a/include/ClientCaches.py
+++ b/include/ClientCaches.py
@@ -193,6 +193,8 @@ class ClientFilesManager( object ):
         
         self._prefixes_to_locations = {}
         
+        self._bad_error_occured = False
+        
         self._Reinit()
         
     
@@ -320,6 +322,27 @@ class ClientFilesManager( object ):
         
         self._prefixes_to_locations = self._controller.Read( 'client_files_locations' )
         
+        for ( prefix, location ) in self._prefixes_to_locations.items():
+            
+            if os.path.exists( location ):
+                
+                dir = os.path.join( location, prefix )
+                
+                if not os.path.exists( dir ):
+                    
+                    HydrusData.DebugPrint( 'The location ' + dir + ' was not found, so it was created.' )
+                    
+                    os.makedirs( dir )
+                    
+                
+            else:
+                
+                self._bad_error_occured = True
+                
+                HydrusData.DebugPrint( 'The location ' + location + ' was not found during file manager init. A graphical error should follow.' )
+                
+            
+        
     
     def GetExpectedFilePath( self, hash, mime ):
         
@@ -376,6 +399,11 @@ class ClientFilesManager( object ):
     
     def Rebalance( self, partial = True, stop_time = None ):
         
+        if self._bad_error_occured:
+            
+            return
+            
+        
         with self._lock:
             
             rebalance_tuple = self._GetRebalanceTuple()
@@ -474,7 +502,9 @@ class ClientFilesManager( object ):
                 
                 if not os.path.exists( location ):
                     
-                    HydrusData.ShowText( 'The external location ' + location + ' does not exist! Please check your external storage options.' )
+                    self._bad_error_occured = True
+                    
+                    HydrusData.ShowText( 'The external location ' + location + ' does not exist! Please check your external storage options and restart the client.' )
                     
                 
             
@@ -988,7 +1018,7 @@ class ThumbnailCache( object ):
                 
                 path = ClientFiles.GetThumbnailPath( hash, False )
                 
-            except HydrusExceptions.NotFoundException as e:
+            except HydrusExceptions.FileMissingException as e:
                 
                 HydrusData.ShowException( e )
                 
@@ -1034,7 +1064,7 @@ class ThumbnailCache( object ):
                         
                         HydrusData.ShowException( e )
                         
-                        raise HydrusExceptions.NotFoundException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was found, but it would not render for the above reason. Furthermore, the faulty thumbnail file could not be deleted. This event could indicate hard drive corruption, and it also suggests that hydrus does not have permission to write to its thumbnail folder. Please check everything is ok.' )
+                        raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was found, but it would not render for the above reason. Furthermore, the faulty thumbnail file could not be deleted. This event could indicate hard drive corruption, and it also suggests that hydrus does not have permission to write to its thumbnail folder. Please check everything is ok.' )
                         
                     
                     try:
@@ -1045,7 +1075,7 @@ class ThumbnailCache( object ):
                         
                         HydrusData.ShowException( e )
                         
-                        raise HydrusExceptions.NotFoundException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was found, but it would not render for the above reason. It was deleted, but it could not be regenerated for the other above reason. This event could indicate hard drive corruption. Please check everything is ok.' )
+                        raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was found, but it would not render for the above reason. It was deleted, but it could not be regenerated for the other above reason. This event could indicate hard drive corruption. Please check everything is ok.' )
                         
                     
                     HydrusData.ShowText( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was found, but it would not render for the above reason. It was deleted and regenerated. This event could indicate hard drive corruption. Please check everything is ok.' )
@@ -1248,7 +1278,7 @@ class ServicesManager( object ):
         with self._lock:
             
             try: return self._keys_to_services[ service_key ]
-            except KeyError: raise HydrusExceptions.NotFoundException( 'That service was not found!' )
+            except KeyError: raise Exception( 'That service was not found!' )
             
         
     
@@ -1767,7 +1797,7 @@ class UndoManager( object ):
                 ( data_type, action, row ) = content_update.ToTuple()
                 
                 if data_type == HC.CONTENT_TYPE_FILES:
-                    if action in ( HC.CONTENT_UPDATE_ADD, HC.CONTENT_UPDATE_DELETE, HC.CONTENT_UPDATE_UNDELETE, HC.CONTENT_UPDATE_RESCIND_PETITION ): continue
+                    if action in ( HC.CONTENT_UPDATE_ADD, HC.CONTENT_UPDATE_DELETE, HC.CONTENT_UPDATE_UNDELETE, HC.CONTENT_UPDATE_RESCIND_PETITION, HC.CONTENT_UPDATE_ADVANCED ): continue
                 elif data_type == HC.CONTENT_TYPE_MAPPINGS:
                     
                     if action in ( HC.CONTENT_UPDATE_RESCIND_PETITION, HC.CONTENT_UPDATE_ADVANCED ): continue
diff --git a/include/ClientController.py b/include/ClientController.py
index 676fc1c1..7124ac52 100755
--- a/include/ClientController.py
+++ b/include/ClientController.py
@@ -3,6 +3,7 @@ import ClientData
 import ClientDaemons
 import ClientDefaults
 import ClientNetworking
+import ClientThreading
 import hashlib
 import httplib
 import HydrusConstants as HC
@@ -107,7 +108,7 @@ class Controller( HydrusController.HydrusController ):
             finally: job_key.Finish()
             
         
-        job_key = HydrusThreading.JobKey()
+        job_key = ClientThreading.JobKey()
         
         job_key.Begin()
         
@@ -306,6 +307,11 @@ class Controller( HydrusController.HydrusController ):
             return True
             
         
+        if not HydrusData.TimeHasPassed( self._timestamps[ 'boot' ] + 120 ):
+            
+            return False
+            
+        
         idle_normal = self._options[ 'idle_normal' ]
         idle_period = self._options[ 'idle_period' ]
         idle_mouse_period = self._options[ 'idle_mouse_period' ]
@@ -563,16 +569,17 @@ class Controller( HydrusController.HydrusController ):
         
         if not self._no_daemons:
             
-            self._daemons.append( HydrusThreading.DAEMONWorker( self, 'CheckImportFolders', ClientDaemons.DAEMONCheckImportFolders, ( 'notify_restart_import_folders_daemon', 'notify_new_import_folders' ), period = 180 ) )
             self._daemons.append( HydrusThreading.DAEMONWorker( self, 'CheckMouseIdle', ClientDaemons.DAEMONCheckMouseIdle, period = 10 ) )
-            self._daemons.append( HydrusThreading.DAEMONWorker( self, 'CheckExportFolders', ClientDaemons.DAEMONCheckExportFolders, ( 'notify_restart_export_folders_daemon', 'notify_new_export_folders' ), period = 180 ) )
-            self._daemons.append( HydrusThreading.DAEMONWorker( self, 'DownloadFiles', ClientDaemons.DAEMONDownloadFiles, ( 'notify_new_downloads', 'notify_new_permissions' ), pre_callable_wait = 0 ) )
-            self._daemons.append( HydrusThreading.DAEMONWorker( self, 'MaintainTrash', ClientDaemons.DAEMONMaintainTrash, init_wait = 60 ) )
-            self._daemons.append( HydrusThreading.DAEMONWorker( self, 'RebalanceClientFiles', ClientDaemons.DAEMONRebalanceClientFiles, period = 3600 ) )
+            self._daemons.append( HydrusThreading.DAEMONWorker( self, 'DownloadFiles', ClientDaemons.DAEMONDownloadFiles, ( 'notify_new_downloads', 'notify_new_permissions' ) ) )
             self._daemons.append( HydrusThreading.DAEMONWorker( self, 'SynchroniseAccounts', ClientDaemons.DAEMONSynchroniseAccounts, ( 'permissions_are_stale', ) ) )
-            self._daemons.append( HydrusThreading.DAEMONWorker( self, 'SynchroniseRepositories', ClientDaemons.DAEMONSynchroniseRepositories, ( 'notify_restart_repo_sync_daemon', 'notify_new_permissions' ), period = 4 * 3600 ) )
-            self._daemons.append( HydrusThreading.DAEMONWorker( self, 'SynchroniseSubscriptions', ClientDaemons.DAEMONSynchroniseSubscriptions, ( 'notify_restart_subs_sync_daemon', 'notify_new_subscriptions' ), period = 3600, init_wait = 120 ) )
-            self._daemons.append( HydrusThreading.DAEMONWorker( self, 'UPnP', ClientDaemons.DAEMONUPnP, ( 'notify_new_upnp_mappings', ), init_wait = 120, pre_callable_wait = 6 ) )
+            self._daemons.append( HydrusThreading.DAEMONWorker( self, 'SynchroniseSubscriptions', ClientDaemons.DAEMONSynchroniseSubscriptions, ( 'notify_restart_subs_sync_daemon', 'notify_new_subscriptions' ) ) )
+            
+            self._daemons.append( HydrusThreading.DAEMONBigJobWorker( self, 'CheckImportFolders', ClientDaemons.DAEMONCheckImportFolders, ( 'notify_restart_import_folders_daemon', 'notify_new_import_folders' ), period = 180 ) )
+            self._daemons.append( HydrusThreading.DAEMONBigJobWorker( self, 'CheckExportFolders', ClientDaemons.DAEMONCheckExportFolders, ( 'notify_restart_export_folders_daemon', 'notify_new_export_folders' ), period = 180 ) )
+            self._daemons.append( HydrusThreading.DAEMONBigJobWorker( self, 'MaintainTrash', ClientDaemons.DAEMONMaintainTrash, init_wait = 60 ) )
+            self._daemons.append( HydrusThreading.DAEMONBigJobWorker( self, 'RebalanceClientFiles', ClientDaemons.DAEMONRebalanceClientFiles, period = 3600 ) )
+            self._daemons.append( HydrusThreading.DAEMONBigJobWorker( self, 'SynchroniseRepositories', ClientDaemons.DAEMONSynchroniseRepositories, ( 'notify_restart_repo_sync_daemon', 'notify_new_permissions' ), period = 4 * 3600 ) )
+            self._daemons.append( HydrusThreading.DAEMONBigJobWorker( self, 'UPnP', ClientDaemons.DAEMONUPnP, ( 'notify_new_upnp_mappings', ), init_wait = 120, pre_callable_wait = 6 ) )
             
             self._daemons.append( HydrusThreading.DAEMONQueue( self, 'FlushRepositoryUpdates', ClientDaemons.DAEMONFlushServiceUpdates, 'service_updates_delayed', period = 5 ) )
             
@@ -654,7 +661,14 @@ class Controller( HydrusController.HydrusController ):
     
     def PageDeleted( self, page_key ):
         
-        return self._gui.PageDeleted( page_key )
+        try:
+            
+            return self._gui.PageDeleted( page_key )
+            
+        except wx.PyDeadObjectError:
+            
+            return True
+            
         
     
     def PageHidden( self, page_key ):
@@ -744,12 +758,15 @@ class Controller( HydrusController.HydrusController ):
                     
                 
             
-            if self._booru_service is None: StartServer()
+            if self._booru_service is None and port is not None: StartServer()
             else:
                 
                 deferred = defer.maybeDeferred( self._booru_service.stopListening )
                 
-                deferred.addCallback( StartServer )
+                if port is not None:
+                    
+                    deferred.addCallback( StartServer )
+                    
                 
             
         
@@ -804,12 +821,18 @@ class Controller( HydrusController.HydrusController ):
                     
                 
             
-            if self._local_service is None: StartServer()
+            if self._local_service is None and port is not None:
+                
+                StartServer()
+                
             else:
                 
                 deferred = defer.maybeDeferred( self._local_service.stopListening )
                 
-                deferred.addCallback( StartServer )
+                if port is not None:
+                    
+                    deferred.addCallback( StartServer )
+                    
                 
             
         
diff --git a/include/ClientDB.py b/include/ClientDB.py
index f8dd7ce2..7fd46d1c 100755
--- a/include/ClientDB.py
+++ b/include/ClientDB.py
@@ -4,6 +4,7 @@ import ClientFiles
 import ClientImporting
 import ClientMedia
 import ClientRatings
+import ClientThreading
 import collections
 import hashlib
 import httplib
@@ -1091,7 +1092,7 @@ class DB( HydrusDB.HydrusDB ):
                 if 'max_monthly_data' not in info: info[ 'max_monthly_data' ] = None
                 if 'used_monthly_requests' not in info: info[ 'used_monthly_requests' ] = 0
                 if 'current_data_month' not in info: info[ 'current_data_month' ] = ( current_year, current_month )
-                if 'port' not in info: info[ 'port' ] = HC.DEFAULT_LOCAL_BOORU_PORT
+                if 'port' not in info: info[ 'port' ] = None
                 if 'upnp' not in info: info[ 'upnp' ] = None
                 
             
@@ -1232,7 +1233,7 @@ class DB( HydrusDB.HydrusDB ):
     
     def _Backup( self, path ):
         
-        job_key = HydrusThreading.JobKey( cancellable = True )
+        job_key = ClientThreading.JobKey( cancellable = True )
         
         job_key.SetVariable( 'popup_title', 'backing up db' )
         
@@ -1242,8 +1243,7 @@ class DB( HydrusDB.HydrusDB ):
         
         self._c.execute( 'COMMIT' )
         
-        self._c.close()
-        self._db.close()
+        self._CloseDBCursor()
         
         if not os.path.exists( path ):
             
@@ -1283,7 +1283,7 @@ class DB( HydrusDB.HydrusDB ):
         
         prefix_string = 'checking db integrity: '
         
-        job_key = HydrusThreading.JobKey( cancellable = True )
+        job_key = ClientThreading.JobKey( cancellable = True )
         
         job_key.SetVariable( 'popup_title', prefix_string + 'preparing' )
         
@@ -1333,7 +1333,7 @@ class DB( HydrusDB.HydrusDB ):
         
         prefix_string = 'checking file integrity: '
         
-        job_key = HydrusThreading.JobKey( cancellable = True )
+        job_key = ClientThreading.JobKey( cancellable = True )
         
         job_key.SetVariable( 'popup_text_1', prefix_string + 'preparing' )
         
@@ -1360,8 +1360,11 @@ class DB( HydrusDB.HydrusDB ):
             
             hash = self._GetHash( hash_id )
             
-            try: path = client_files_manager.GetFilePath( hash, mime )
-            except HydrusExceptions.NotFoundException:
+            try:
+                
+                path = client_files_manager.GetFilePath( hash, mime )
+                
+            except HydrusExceptions.FileMissingException:
                 
                 print( 'Could not find the file at ' + client_files_manager.GetExpectedFilePath( hash, mime ) + '!' )
                 
@@ -1467,7 +1470,10 @@ class DB( HydrusDB.HydrusDB ):
             
             self.pub_after_commit( 'clipboard', 'paths', paths )
             
-            if len( error_messages ) > 0: raise Exception( 'Some of the file copies failed with the following error message(s):' + os.linesep + os.linesep.join( error_messages ) )
+            if len( error_messages ) > 0:
+                
+                raise Exception( 'Some of the file copies failed with the following error message(s):' + os.linesep + os.linesep.join( error_messages ) )
+                
             
         
     
@@ -1482,10 +1488,6 @@ class DB( HydrusDB.HydrusDB ):
         
         for prefix in HydrusData.IterateHexPrefixes():
             
-            dir = os.path.join( HC.CLIENT_FILES_DIR, prefix )
-            
-            if not os.path.exists( dir ): os.makedirs( dir )
-            
             dir = os.path.join( HC.CLIENT_THUMBNAILS_DIR, prefix )
             
             if not os.path.exists( dir ): os.makedirs( dir )
@@ -1793,7 +1795,7 @@ class DB( HydrusDB.HydrusDB ):
         
         prefix = 'database maintenance - delete orphans: '
         
-        job_key = HydrusThreading.JobKey( cancellable = True )
+        job_key = ClientThreading.JobKey( cancellable = True )
         
         job_key.SetVariable( 'popup_text_1', prefix + 'gathering file information' )
         
@@ -1826,8 +1828,14 @@ class DB( HydrusDB.HydrusDB ):
             
             if hash in deletee_hashes:
                 
-                try: path = client_files_manager.GetFilePath( hash )
-                except HydrusExceptions.NotFoundException: continue
+                try:
+                    
+                    path = client_files_manager.GetFilePath( hash )
+                    
+                except HydrusExceptions.FileMissingException:
+                    
+                    continue
+                    
                 
                 try:
                     
@@ -1962,8 +1970,14 @@ class DB( HydrusDB.HydrusDB ):
             
             for hash in file_hashes:
                 
-                try: path = client_files_manager.GetFilePath( hash )
-                except HydrusExceptions.NotFoundException: continue
+                try:
+                    
+                    path = client_files_manager.GetFilePath( hash )
+                    
+                except HydrusExceptions.FileMissingException:
+                    
+                    continue
+                    
                 
                 deletee_paths.add( path )
                 
@@ -2034,7 +2048,7 @@ class DB( HydrusDB.HydrusDB ):
         
         prefix_string = 'exporting to tag archive: '
         
-        job_key = HydrusThreading.JobKey( cancellable = True )
+        job_key = ClientThreading.JobKey( cancellable = True )
         
         job_key.SetVariable( 'popup_text_1', prefix_string + 'preparing' )
         
@@ -2046,7 +2060,10 @@ class DB( HydrusDB.HydrusDB ):
         
         hta = HydrusTagArchive.HydrusTagArchive( path )
         
-        if hta_exists and hta.GetHashType() != hash_type: raise Exception( 'This tag archive does not use the expected hash type, so it cannot be exported to!' )
+        if hta_exists and hta.GetHashType() != hash_type:
+            
+            raise Exception( 'This tag archive does not use the expected hash type, so it cannot be exported to!' )
+            
         
         hta.SetHashType( hash_type )
         
@@ -2483,7 +2500,10 @@ class DB( HydrusDB.HydrusDB ):
         
         result = self._c.execute( 'SELECT hash FROM hashes WHERE hash_id = ?;', ( hash_id, ) ).fetchone()
         
-        if result is None: raise Exception( 'File hash error in database' )
+        if result is None:
+            
+            raise HydrusExceptions.DataMissing( 'File hash error in database' )
+            
         
         ( hash, ) = result
         
@@ -3378,7 +3398,10 @@ class DB( HydrusDB.HydrusDB ):
         
         result = self._c.execute( 'SELECT mime FROM files_info WHERE service_id = ? AND hash_id = ?;', ( service_id, hash_id ) ).fetchone()
         
-        if result is None: raise HydrusExceptions.NotFoundException()
+        if result is None:
+            
+            raise HydrusExceptions.FileMissingException( 'Did not have mime information for that file!' )
+            
         
         ( mime, ) = result
         
@@ -3395,7 +3418,10 @@ class DB( HydrusDB.HydrusDB ):
             
             namespace_id = self._c.lastrowid
             
-        else: ( namespace_id, ) = result
+        else:
+            
+            ( namespace_id, ) = result
+            
         
         return namespace_id
         
@@ -3440,7 +3466,10 @@ class DB( HydrusDB.HydrusDB ):
         
         result = self._c.execute( 'SELECT tag FROM tags WHERE tag_id = ?;', ( tag_id, ) ).fetchone()
         
-        if result is None: raise Exception( 'Tag error in database' )
+        if result is None:
+            
+            raise HydrusExceptions.DataMissing( 'Tag error in database' )
+            
         
         ( tag, ) = result
         
@@ -3452,7 +3481,10 @@ class DB( HydrusDB.HydrusDB ):
             
             result = self._c.execute( 'SELECT namespace FROM namespaces WHERE namespace_id = ?;', ( namespace_id, ) ).fetchone()
             
-            if result is None: raise Exception( 'Namespace error in database' )
+            if result is None:
+                
+                raise HydrusExceptions.DataMissing( 'Namespace error in database' )
+                
             
             ( namespace, ) = result
             
@@ -3636,7 +3668,10 @@ class DB( HydrusDB.HydrusDB ):
         
         result = self._c.execute( 'SELECT reason FROM reasons WHERE reason_id = ?;', ( reason_id, ) ).fetchone()
         
-        if result is None: raise Exception( 'Reason error in database' )
+        if result is None:
+            
+            raise HydrusExceptions.DataMissing( 'Reason error in database' )
+            
         
         ( reason, ) = result
         
@@ -3707,7 +3742,10 @@ class DB( HydrusDB.HydrusDB ):
         
         result = self._c.execute( 'SELECT service_id FROM services WHERE service_key = ?;', ( sqlite3.Binary( service_key ), ) ).fetchone()
         
-        if result is None: raise HydrusExceptions.NotFoundException( 'Service id error in database' )
+        if result is None:
+            
+            raise HydrusExceptions.DataMissing( 'Service id error in database' )
+            
         
         ( service_id, ) = result
         
@@ -4099,7 +4137,10 @@ class DB( HydrusDB.HydrusDB ):
                     elif dump_name == 'pixiv_account': result = ( '', '' )
                     
                 
-                if result is None: raise Exception( dump_name + ' was not found!' )
+                if result is None:
+                    
+                    raise HydrusExceptions.DataMissing( dump_name + ' was not found!' )
+                    
                 
             else: ( result, ) = result
             
@@ -4354,8 +4395,6 @@ class DB( HydrusDB.HydrusDB ):
             self._c.execute( 'BEGIN IMMEDIATE;' )
             
         
-        pauser = HydrusData.BigJobPauser()
-        
         self.pub_after_commit( 'notify_new_pending' )
         self.pub_after_commit( 'notify_new_siblings' )
         self.pub_after_commit( 'notify_new_parents' )
@@ -4375,8 +4414,6 @@ class DB( HydrusDB.HydrusDB ):
         
         for ( content_updates, weight ) in content_update_package.IterateContentUpdateChunks():
             
-            pauser.Pause()
-            
             options = self._controller.GetOptions()
             
             if only_when_idle and not self._controller.CurrentlyIdle():
@@ -4459,8 +4496,14 @@ class DB( HydrusDB.HydrusDB ):
         
         for ( service_key, content_updates ) in service_keys_to_content_updates.items():
             
-            try: service_id = self._GetServiceId( service_key )
-            except  HydrusExceptions.NotFoundException: continue
+            try:
+                
+                service_id = self._GetServiceId( service_key )
+                
+            except:
+                
+                continue
+                
             
             service = self._GetService( service_id )
             
@@ -4483,7 +4526,18 @@ class DB( HydrusDB.HydrusDB ):
                     
                     if data_type == HC.CONTENT_TYPE_FILES:
                         
-                        if action == HC.CONTENT_UPDATE_ADD:
+                        if action == HC.CONTENT_UPDATE_ADVANCED:
+                            
+                            ( sub_action, sub_row ) = row
+                            
+                            if sub_action == 'delete_deleted':
+                                
+                                self._c.execute( 'DELETE FROM deleted_files WHERE service_id = ?;', ( service_id, ) )
+                                
+                            
+                            self._c.execute( 'DELETE FROM service_info WHERE service_id = ?;', ( service_id, ) )
+                            
+                        elif action == HC.CONTENT_UPDATE_ADD:
                             
                             ( hash, size, mime, timestamp, width, height, duration, num_frames, num_words ) = row
                             
@@ -4931,8 +4985,14 @@ class DB( HydrusDB.HydrusDB ):
         
         for ( service_key, service_updates ) in service_keys_to_service_updates.items():
             
-            try: service_id = self._GetServiceId( service_key )
-            except HydrusExceptions.NotFoundException: continue
+            try:
+                
+                service_id = self._GetServiceId( service_key )
+                
+            except HydrusExceptions.DataMissing:
+                
+                continue
+                
             
             if service_id in self._service_cache: del self._service_cache[ service_id ]
             
@@ -4981,7 +5041,7 @@ class DB( HydrusDB.HydrusDB ):
                             
                             text = name + ' at ' + time.ctime( timestamp ) + ':' + os.linesep * 2 + post
                             
-                            job_key = HydrusThreading.JobKey()
+                            job_key = ClientThreading.JobKey()
                             
                             job_key.SetVariable( 'popup_text_1', text )
                             
@@ -5139,7 +5199,7 @@ class DB( HydrusDB.HydrusDB ):
         
         prefix = 'resetting ' + name
         
-        job_key = HydrusThreading.JobKey()
+        job_key = ClientThreading.JobKey()
         
         job_key.SetVariable( 'popup_text_1', prefix + ': deleting from main service table' )
         
@@ -5214,7 +5274,7 @@ class DB( HydrusDB.HydrusDB ):
             
             prefix = 'deleting old resized thumbnails: '
             
-            job_key = HydrusThreading.JobKey()
+            job_key = ClientThreading.JobKey()
             
             job_key.SetVariable( 'popup_text_1', prefix + 'initialising' )
             
@@ -6315,6 +6375,14 @@ class DB( HydrusDB.HydrusDB ):
             self._c.execute( 'DROP TABLE files_info_old;' )
             
         
+        if version == 192:
+            
+            if os.path.exists( self._no_wal_path ):
+                
+                os.remove( self._no_wal_path )
+                
+            
+        
         self._controller.pub( 'splash_set_title_text', 'updating db to v' + str( version + 1 ) )
         
         self._c.execute( 'UPDATE version SET version = ?;', ( version + 1, ) )
@@ -6845,7 +6913,7 @@ class DB( HydrusDB.HydrusDB ):
         
         prefix = 'database maintenance - vacuum: '
         
-        job_key = HydrusThreading.JobKey()
+        job_key = ClientThreading.JobKey()
         
         job_key.SetVariable( 'popup_text_1', prefix + 'vacuuming' )
         
@@ -6904,9 +6972,6 @@ class DB( HydrusDB.HydrusDB ):
         
         self._c.execute( 'REPLACE INTO shutdown_timestamps ( shutdown_type, timestamp ) VALUES ( ?, ? );', ( CC.SHUTDOWN_TIMESTAMP_VACUUM, HydrusData.GetNow() ) )
         
-        self._c.close()
-        self._db.close()
-        
         self._InitDBCursor()
         
         job_key.SetVariable( 'popup_text_1', prefix + 'done!' )
diff --git a/include/ClientDaemons.py b/include/ClientDaemons.py
index 34c77ab7..c94557c5 100644
--- a/include/ClientDaemons.py
+++ b/include/ClientDaemons.py
@@ -1,5 +1,6 @@
 import ClientDownloading
 import ClientFiles
+import ClientThreading
 import collections
 import hashlib
 import httplib
@@ -82,7 +83,7 @@ def DAEMONDownloadFiles( controller ):
         
         successful_hashes = set()
         
-        job_key = HydrusThreading.JobKey()
+        job_key = ClientThreading.JobKey()
         
         job_key.SetVariable( 'popup_text_1', 'initialising downloader' )
         
@@ -103,8 +104,14 @@ def DAEMONDownloadFiles( controller ):
                 if service_key == CC.LOCAL_FILE_SERVICE_KEY: break
                 elif service_key == CC.TRASH_SERVICE_KEY: continue
                 
-                try: file_repository = controller.GetServicesManager().GetService( service_key )
-                except HydrusExceptions.NotFoundException: continue
+                try:
+                    
+                    file_repository = controller.GetServicesManager().GetService( service_key )
+                    
+                except:
+                    
+                    continue
+                    
                 
                 if file_repository.CanDownload(): 
                     
diff --git a/include/ClientData.py b/include/ClientData.py
index 7806427f..d6c48869 100644
--- a/include/ClientData.py
+++ b/include/ClientData.py
@@ -3,10 +3,12 @@ import ClientDefaults
 import ClientDownloading
 import ClientFiles
 import ClientNetworking
+import ClientThreading
 import collections
 import datetime
 import HydrusConstants as HC
 import HydrusExceptions
+import HydrusFileHandling
 import HydrusPaths
 import HydrusSerialisable
 import HydrusTags
@@ -14,11 +16,13 @@ import threading
 import traceback
 import os
 import random
+import requests
 import shutil
 import sqlite3
 import stat
 import sys
 import time
+import urllib
 import wx
 import yaml
 import HydrusData
@@ -35,7 +39,7 @@ def CatchExceptionClient( etype, value, tb ):
     
     try:
         
-        job_key = HydrusThreading.JobKey()
+        job_key = ClientThreading.JobKey()
         
         if etype == HydrusExceptions.ShutdownException:
             
@@ -254,7 +258,7 @@ def GetMediasTagCount( pool, tag_service_key = CC.COMBINED_TAG_SERVICE_KEY, coll
     
 def ShowExceptionClient( e ):
     
-    job_key = HydrusThreading.JobKey()
+    job_key = ClientThreading.JobKey()
     
     if isinstance( e, HydrusExceptions.ShutdownException ):
         
@@ -311,7 +315,7 @@ def ShowExceptionClient( e ):
     
 def ShowTextClient( text ):
     
-    job_key = HydrusThreading.JobKey()
+    job_key = ClientThreading.JobKey()
     
     job_key.SetVariable( 'popup_text_1', HydrusData.ToUnicode( text ) )
     
@@ -395,8 +399,8 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
         
         self._dictionary[ 'booleans' ][ 'apply_all_parents_to_all_services' ] = False
         self._dictionary[ 'booleans' ][ 'apply_all_siblings_to_all_services' ] = False
-        self._dictionary[ 'booleans' ][ 'waiting_politely_text' ] = False
         self._dictionary[ 'booleans' ][ 'filter_inbox_and_archive_predicates' ] = True
+        self._dictionary[ 'booleans' ][ 'waiting_politely_text' ] = False
         
         self._dictionary[ 'noneable_integers' ] = {}
         
@@ -1363,7 +1367,7 @@ class ServiceRepository( ServiceRestricted ):
     
     def Sync( self, only_when_idle = False, stop_time = None ):
         
-        job_key = HydrusThreading.JobKey( pausable = False, cancellable = True )
+        job_key = ClientThreading.JobKey( pausable = False, cancellable = True )
         
         ( i_paused, should_quit ) = job_key.WaitIfNeeded()
         
@@ -1833,8 +1837,6 @@ class ServiceIPFS( ServiceRemote ):
         
         url = 'http://' + host + ':' + str( port ) + path
         
-        import requests
-        
         response = requests.get( url )
         
         if response.ok:
@@ -1845,30 +1847,84 @@ class ServiceIPFS( ServiceRemote ):
             
         else:
             
-            raise Exception()
+            raise Exception( response.content )
             
         
     
     def ImportFile( self, multihash ):
         
-        method = HC.GET
+        credentials = self.GetCredentials()
+        
+        ( host, port ) = credentials.GetAddress()
+        
+        url = 'http://' + host + ':' + str( port ) + '/api/v0/cat/' + multihash
+        
+        url_string = multihash
+        
+        job_key = ClientThreading.JobKey( pausable = True, cancellable = True )
+        
+        HydrusGlobals.client_controller.pub( 'message', job_key )
+        
+        HydrusGlobals.client_controller.CallToThread( ClientDownloading.THREADDownloadURL, job_key, url, url_string )
+        
+    
+    def PinFile( self, path ):
+        
+        mime = HydrusFileHandling.GetMime( path )
+        
+        mime_string = HC.mime_string_lookup[ mime ]
         
         credentials = self.GetCredentials()
         
         ( host, port ) = credentials.GetAddress()
         
-        path = '/api/v0/cat'
-        query = 'arg=' + multihash
+        url = 'http://' + host + ':' + str( port ) + '/api/v0/add'
         
-        url = 'http://' + host + ':' + str( port ) + path + '?' + query
+        files = { 'path' : ( path, open( path, 'rb' ), mime_string ) }
         
-        url_string = multihash
+        response = requests.put( url, files = files )
         
-        job_key = HydrusThreading.JobKey( pausable = True, cancellable = True )
+        if response.ok:
+            
+            # responds with some json with name and key (ipfs hash)
+            # parse that multihash, wrap it into the content update
+            
+            pass # spin off a content update
+            
+        else:
+            
+            raise Exception( response.content )
+            
         
-        HydrusGlobals.client_controller.pub( 'message', job_key )
+    
+    def SyncPinned( self ):
         
-        HydrusGlobals.client_controller.CallToThread( ClientDownloading.THREADDownloadURL, job_key, url, url_string )
+        # query pin ls and update private cache with what we have
+        # add a button for this on review services, I think
+        
+        pass
+        
+    
+    def UnpinFile( self, multihash ):
+        
+        # will have to get multihash from db
+        
+        credentials = self.GetCredentials()
+        
+        ( host, port ) = credentials.GetAddress()
+        
+        url = 'http://' + host + ':' + str( port ) + '/api/v0/pin/rm/' + multihash
+        
+        response = requests.get( url )
+        
+        if response.ok:
+            
+            pass # spin off a content update
+            
+        else:
+            
+            raise Exception( response.content )
+            
         
     
 class Shortcuts( HydrusSerialisable.SerialisableBaseNamed ):
diff --git a/include/ClientDefaults.py b/include/ClientDefaults.py
index 4551cb68..670816fd 100644
--- a/include/ClientDefaults.py
+++ b/include/ClientDefaults.py
@@ -221,7 +221,7 @@ def GetClientDefaultOptions():
     options[ 'tag_dialog_position' ] = ( False, None )
     options[ 'rating_dialog_position' ] = ( False, None )
     
-    options[ 'local_port' ] = HC.DEFAULT_LOCAL_FILE_PORT
+    options[ 'local_port' ] = None
     
     return options
     
diff --git a/include/ClientDownloading.py b/include/ClientDownloading.py
index c50a3424..6a739004 100644
--- a/include/ClientDownloading.py
+++ b/include/ClientDownloading.py
@@ -273,7 +273,7 @@ def THREADDownloadURL( job_key, url, url_string ):
             
             job_key.Cancel()
             
-            raise Exception( 'Sorry, http failed. This error will improve.' )
+            raise HydrusExceptions.NetworkException( response.content )
             
         
     finally:
@@ -502,7 +502,7 @@ class GalleryBooru( Gallery ):
             
         except:
             
-            raise HydrusExceptions.NotFoundException( 'Attempted to find booru "' + booru_name + '", but it was missing from the database!' )
+            raise Exception( 'Attempted to find booru "' + booru_name + '", but it was missing from the database!' )
             
         
         self._gallery_advance_num = None
@@ -743,12 +743,12 @@ class GalleryBooru( Gallery ):
             
         except Exception as e:
             
-            raise HydrusExceptions.NotFoundException( 'Could not parse a download link for ' + url_base + '!' + os.linesep + HydrusData.ToUnicode( e ) )
+            raise HydrusExceptions.NetworkException( 'Could not parse a download link for ' + url_base + '!' + os.linesep + HydrusData.ToUnicode( e ) )
             
         
         if image_url is None:
             
-            raise HydrusExceptions.NotFoundException( 'Could not parse a download link for ' + url_base + '!' )
+            raise HydrusExceptions.NetworkException( 'Could not parse a download link for ' + url_base + '!' )
             
         
         image_url = urlparse.urljoin( url_base, image_url )
diff --git a/include/ClientFiles.py b/include/ClientFiles.py
index 53642575..a4d76265 100644
--- a/include/ClientFiles.py
+++ b/include/ClientFiles.py
@@ -187,11 +187,11 @@ def GetFilePath( location, hash, mime = None ):
     
     if path is None:
         
-        raise HydrusExceptions.NotFoundException( 'File not found in directory ' + location + '!' )
+        raise HydrusExceptions.FileMissingException( 'File not found in directory ' + location + '!' )
         
     elif not os.path.exists( path ):
         
-        raise HydrusExceptions.NotFoundException( 'File not found in path + ' + path + '!' )
+        raise HydrusExceptions.FileMissingException( 'File not found in path + ' + path + '!' )
         
     
     return path
@@ -222,9 +222,9 @@ def GetThumbnailPath( hash, full_size = True ):
                 
                 file_path = client_files_manager.GetFilePath( hash )
                 
-            except HydrusExceptions.NotFoundException:
+            except HydrusExceptions.FileMissingException:
                 
-                raise HydrusExceptions.NotFoundException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was missing. It could not be regenerated because the original file was also missing. This event could indicate hard drive corruption or an unplugged external drive. Please check everything is ok.' )
+                raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was missing. It could not be regenerated because the original file was also missing. This event could indicate hard drive corruption or an unplugged external drive. Please check everything is ok.' )
                 
             
             try:
@@ -235,7 +235,7 @@ def GetThumbnailPath( hash, full_size = True ):
                 
                 HydrusData.ShowException( e )
                 
-                raise HydrusExceptions.NotFoundException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was missing. It could not be regenerated from the original file for the above reason. This event could indicate hard drive corruption. Please check everything is ok.' )
+                raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was missing. It could not be regenerated from the original file for the above reason. This event could indicate hard drive corruption. Please check everything is ok.' )
                 
             
             try:
@@ -249,7 +249,7 @@ def GetThumbnailPath( hash, full_size = True ):
                 
                 HydrusData.ShowException( e )
                 
-                raise HydrusExceptions.NotFoundException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was missing. It was regenerated from the original file, but hydrus could not write it to the location ' + path + ' for the above reason. This event could indicate hard drive corruption, and it also suggests that hydrus does not have permission to write to its thumbnail folder. Please check everything is ok.' )
+                raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was missing. It was regenerated from the original file, but hydrus could not write it to the location ' + path + ' for the above reason. This event could indicate hard drive corruption, and it also suggests that hydrus does not have permission to write to its thumbnail folder. Please check everything is ok.' )
                 
             
             HydrusData.ShowText( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was missing. It has been regenerated from the original file, but this event could indicate hard drive corruption. Please check everything is ok.' )
@@ -270,7 +270,7 @@ def GetThumbnailPath( hash, full_size = True ):
                     
                 except:
                     
-                    raise HydrusExceptions.NotFoundException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was found, but it would not render. An attempt to delete it was made, but that failed as well. This event could indicate hard drive corruption, and it also suggests that hydrus does not have permission to write to its thumbnail folder. Please check everything is ok.' )
+                    raise HydrusExceptions.FileMissingException( 'The thumbnail for file ' + hash.encode( 'hex' ) + ' was found, but it would not render. An attempt to delete it was made, but that failed as well. This event could indicate hard drive corruption, and it also suggests that hydrus does not have permission to write to its thumbnail folder. Please check everything is ok.' )
                     
                 
                 full_size_path = GetThumbnailPath( hash, True )
diff --git a/include/ClientGUI.py b/include/ClientGUI.py
index 83ba25d2..95cf08c6 100755
--- a/include/ClientGUI.py
+++ b/include/ClientGUI.py
@@ -13,6 +13,7 @@ import ClientGUIPages
 import ClientDownloading
 import ClientMedia
 import ClientSearch
+import ClientThreading
 import gc
 import HydrusData
 import HydrusExceptions
@@ -1523,7 +1524,7 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
                 
                 def THREADRegenerateThumbnails():
                     
-                    job_key = HydrusThreading.JobKey( pausable = True, cancellable = True )
+                    job_key = ClientThreading.JobKey( pausable = True, cancellable = True )
                     
                     job_key.SetVariable( 'popup_title', 'regenerating thumbnails' )
                     job_key.SetVariable( 'popup_text_1', 'creating directories' )
@@ -1796,7 +1797,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
                 
                 url_string = url
                 
-                job_key = HydrusThreading.JobKey( pausable = True, cancellable = True )
+                job_key = ClientThreading.JobKey( pausable = True, cancellable = True )
                 
                 self._controller.pub( 'message', job_key )
                 
@@ -1866,7 +1867,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
     
     def _THREADSyncToTagArchive( self, hta_path, adding, namespaces, service_key ):
         
-        job_key = HydrusThreading.JobKey( pausable = True, cancellable = True )
+        job_key = ClientThreading.JobKey( pausable = True, cancellable = True )
         
         try:
             
@@ -1981,7 +1982,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
         
         try:
             
-            job_key = HydrusThreading.JobKey( pausable = True, cancellable = True )
+            job_key = ClientThreading.JobKey( pausable = True, cancellable = True )
             
             job_key.SetVariable( 'popup_title', 'uploading pending to ' + service_name )
             
@@ -2119,7 +2120,13 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
     
     def DoFirstStart( self ):
         
-        with ClientGUIDialogs.DialogFirstStart( self ) as dlg: dlg.ShowModal()
+        message = 'Hi, this looks like the first time you have started the hydrus client.'
+        message += os.linesep * 2
+        message += 'Don\'t forget to check out the help if you haven\'t already.'
+        message += os.linesep * 2
+        message += 'You can right-click popup messages like this to dismiss them.'
+        
+        HydrusData.ShowText( message )
         
     
     def EventExit( self, event ):
@@ -2885,6 +2892,12 @@ class FrameReviewServices( ClientGUICommon.Frame ):
                 self._service_wide_update.Bind( wx.EVT_BUTTON, self.EventServiceWideUpdate )
                 
             
+            if self._service_key == CC.LOCAL_FILE_SERVICE_KEY:
+                
+                self._delete_local_deleted = wx.Button( self, label = 'clear deleted file record' )
+                self._delete_local_deleted.Bind( wx.EVT_BUTTON, self.EventDeleteLocalDeleted )
+                
+            
             if service_type == HC.SERVER_ADMIN:
                 
                 self._init = wx.Button( self, label = 'initialise server' )
@@ -2990,10 +3003,15 @@ class FrameReviewServices( ClientGUICommon.Frame ):
                 vbox.AddF( self._booru_shares_panel, CC.FLAGS_EXPAND_BOTH_WAYS )
                 
             
-            if service_type in HC.RESTRICTED_SERVICES + [ HC.LOCAL_TAG ]:
+            if service_type in HC.RESTRICTED_SERVICES + [ HC.LOCAL_TAG ] or self._service_key == CC.LOCAL_FILE_SERVICE_KEY:
                 
                 repo_buttons_hbox = wx.BoxSizer( wx.HORIZONTAL )
-            
+                
+                if self._service_key == CC.LOCAL_FILE_SERVICE_KEY:
+                    
+                    repo_buttons_hbox.AddF( self._delete_local_deleted, CC.FLAGS_MIXED )
+                    
+                
                 if service_type in ( HC.LOCAL_TAG, HC.TAG_REPOSITORY ):
                     
                     repo_buttons_hbox.AddF( self._service_wide_update, CC.FLAGS_MIXED )
@@ -3375,11 +3393,36 @@ class FrameReviewServices( ClientGUICommon.Frame ):
                 
             
         
+        def EventDeleteLocalDeleted( self, event ):
+            
+            message = 'This will clear the client\'s memory of which files it has locally deleted, which affects \'exclude already deleted files\' import tests.'
+            message += os.linesep * 2
+            message += 'It will freeze the gui while it works.'
+            message += os.linesep * 2
+            message += 'If you do not know what this does, click \'forget it\'.'
+            
+            with ClientGUIDialogs.DialogYesNo( self, message, yes_label = 'do it', no_label = 'forget it' ) as dlg_add:
+                
+                result = dlg_add.ShowModal()
+                
+                if result == wx.ID_YES:
+                    
+                    content_update = HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ADVANCED, ( 'delete_deleted', None ) )
+                    
+                    service_keys_to_content_updates = { self._service_key : [ content_update ] }
+                    
+                    HydrusGlobals.client_controller.Write( 'content_updates', service_keys_to_content_updates )
+                    
+                    self._DisplayService()
+                    
+                
+            
+        
         def EventImmediateSync( self, event ):
             
             def do_it():
             
-                job_key = HydrusThreading.JobKey( pausable = True, cancellable = True )
+                job_key = ClientThreading.JobKey( pausable = True, cancellable = True )
                 
                 job_key.SetVariable( 'popup_title', self._service.GetName() + ': immediate sync' )
                 job_key.SetVariable( 'popup_text_1', 'downloading' )
diff --git a/include/ClientGUICanvas.py b/include/ClientGUICanvas.py
index be3c97e7..5cd32f00 100755
--- a/include/ClientGUICanvas.py
+++ b/include/ClientGUICanvas.py
@@ -1597,9 +1597,17 @@ class CanvasPanel( Canvas, wx.Window ):
             
             copy_menu.AppendMenu( CC.ID_NULL, 'hash', copy_hash_menu )
             
-            if self._current_display_media.GetMime() in HC.IMAGES and self._current_display_media.GetDuration() is None: copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_bmp' ), 'image' )
+            if self._current_display_media.GetMime() in HC.IMAGES and self._current_display_media.GetDuration() is None:
+                
+                copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_bmp' ), 'image' )
+                
+            
             copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_path' ), 'path' )
-            copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_local_url' ), 'local url' )
+            
+            if HC.options[ 'local_port' ] is not None:
+                
+                copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_local_url' ), 'local url' )
+                
             
             share_menu.AppendMenu( CC.ID_NULL, 'copy', copy_menu )
             
@@ -2639,9 +2647,17 @@ class CanvasFullscreenMediaListBrowser( CanvasFullscreenMediaListNavigable ):
         
         copy_menu.AppendMenu( CC.ID_NULL, 'hash', copy_hash_menu )
         
-        if self._current_display_media.GetMime() in HC.IMAGES and self._current_display_media.GetDuration() is None: copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_bmp' ), 'image' )
+        if self._current_display_media.GetMime() in HC.IMAGES and self._current_display_media.GetDuration() is None:
+            
+            copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_bmp' ), 'image' )
+            
+        
         copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_path' ), 'path' )
-        copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_local_url' ), 'local url' )
+        
+        if HC.options[ 'local_port' ] is not None:
+            
+            copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_local_url' ), 'local url' )
+            
         
         share_menu.AppendMenu( CC.ID_NULL, 'copy', copy_menu )
         
diff --git a/include/ClientGUICommon.py b/include/ClientGUICommon.py
index b5aabd89..966bcabd 100755
--- a/include/ClientGUICommon.py
+++ b/include/ClientGUICommon.py
@@ -964,7 +964,7 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
                     include_current = self._file_search_context.IncludeCurrentTags()
                     include_pending = self._file_search_context.IncludePendingTags()
                     
-                    if len( search_text ) < num_autocomplete_chars:
+                    if len( half_complete_tag ) < num_autocomplete_chars and '*' not in search_text:
                         
                         predicates = HydrusGlobals.client_controller.Read( 'autocomplete_predicates', file_service_key = self._file_service_key, tag_service_key = self._tag_service_key, tag = search_text, include_current = include_current, include_pending = include_pending, add_namespaceless = True )
                         
@@ -1274,7 +1274,7 @@ class AutoCompleteDropdownTagsWrite( AutoCompleteDropdownTags ):
             
             half_complete_tag = search_text
             
-            if len( search_text ) < num_autocomplete_chars:
+            if len( half_complete_tag ) < num_autocomplete_chars and '*' not in search_text:
                 
                 predicates = HydrusGlobals.client_controller.Read( 'autocomplete_predicates', file_service_key = self._file_service_key, tag_service_key = self._tag_service_key, tag = search_text, add_namespaceless = False )
                 
@@ -3867,18 +3867,23 @@ class ListBoxTagsSelection( ListBoxTags ):
             if self._show_pending: tags_to_count.update( self._pending_tags_to_count )
             if self._show_petitioned: tags_to_count.update( self._petitioned_tags_to_count )
             
-            def key( a ):
-                
-                return tags_to_count[ self._strings_to_terms[ a ] ]
-                
-            
             if self._sort == CC.SORT_BY_INCIDENCE_ASC:
+        
+                def key( a ):
+                    
+                    return ( tags_to_count[ self._strings_to_terms[ a ] ], a )
+                    
                 
                 reverse = False
                 
             elif self._sort == CC.SORT_BY_INCIDENCE_DESC:
                 
-                reverse = True
+                def key( a ):
+                    
+                    return ( - tags_to_count[ self._strings_to_terms[ a ] ], a )
+                    
+                
+                reverse = False
                 
             
         
@@ -5692,7 +5697,7 @@ class SaneListCtrl( wx.ListCtrl, ListCtrlAutoWidthMixin, ColumnSorterMixin ):
             if comparison_data == data: return index
             
         
-        raise HydrusExceptions.NotFoundException( 'Data not found!' )
+        raise HydrusExceptions.DataMissing( 'Data not found!' )
         
     
     def HasClientData( self, data, column_index = None ):
@@ -5703,7 +5708,7 @@ class SaneListCtrl( wx.ListCtrl, ListCtrlAutoWidthMixin, ColumnSorterMixin ):
             
             return True
             
-        except HydrusExceptions.NotFoundException:
+        except HydrusExceptions.DataMissing:
             
             return False
             
diff --git a/include/ClientGUIDialogs.py b/include/ClientGUIDialogs.py
index eacd97ce..b026e679 100755
--- a/include/ClientGUIDialogs.py
+++ b/include/ClientGUIDialogs.py
@@ -18,6 +18,7 @@ import ClientFiles
 import ClientGUICommon
 import ClientGUICollapsible
 import ClientGUIPredicates
+import ClientThreading
 import collections
 import gc
 import itertools
@@ -629,46 +630,6 @@ class DialogFinishFiltering( Dialog ):
         wx.CallAfter( self._commit.SetFocus )
         
     
-class DialogFirstStart( Dialog ):
-    
-    def __init__( self, parent ):
-        
-        Dialog.__init__( self, parent, 'First start', position = 'center' )
-        
-        self._hidden_cancel = wx.Button( self, id = wx.ID_CANCEL, size = ( 0, 0 ) )
-        
-        self._ok = wx.Button( self, id = wx.ID_OK, label = 'ok!' )
-        self._ok.SetForegroundColour( ( 0, 128, 0 ) )
-        
-        message1 = 'Hi, this looks like the first time you have started the hydrus client. Don\'t forget to check out the'
-        link = wx.HyperlinkCtrl( self, id = -1, label = 'help', url = 'file://' + HC.HELP_DIR + '/index.html' )
-        message2 = 'if you haven\'t already.'
-        message3 = 'When you close this dialog, the client will start its local http server. You will probably get a firewall warning.'
-        message4 = 'You can block it if you like, or you can allow it. It doesn\'t phone home, or expose your files to your network; it just provides another way to locally export your files.'
-        
-        hbox = wx.BoxSizer( wx.HORIZONTAL )
-        
-        hbox.AddF( wx.StaticText( self, label = message1 ), CC.FLAGS_MIXED )
-        hbox.AddF( link, CC.FLAGS_MIXED )
-        hbox.AddF( wx.StaticText( self, label = message2 ), CC.FLAGS_MIXED )
-        
-        vbox = wx.BoxSizer( wx.VERTICAL )
-        
-        vbox.AddF( hbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
-        
-        vbox.AddF( wx.StaticText( self, label = message3 ), CC.FLAGS_EXPAND_PERPENDICULAR )
-        vbox.AddF( wx.StaticText( self, label = message4 ), CC.FLAGS_EXPAND_PERPENDICULAR )
-        vbox.AddF( self._ok, CC.FLAGS_LONE_BUTTON )
-        
-        self.SetSizer( vbox )
-        
-        ( x, y ) = self.GetEffectiveMinSize()
-        
-        self.SetInitialSize( ( x, y ) )
-        
-        wx.CallAfter( self._ok.SetFocus )
-        
-    
 class DialogGenerateNewAccounts( Dialog ):
     
     def __init__( self, parent, service_key ):
@@ -1513,7 +1474,7 @@ class DialogInputLocalFiles( Dialog ):
         self._current_paths = []
         self._current_paths_set = set()
         
-        self._job_key = HydrusThreading.JobKey()
+        self._job_key = ClientThreading.JobKey()
         
         if len( paths ) > 0: self._AddPathsToList( paths )
         
@@ -1555,7 +1516,7 @@ class DialogInputLocalFiles( Dialog ):
                 self._add_button.Disable()
                 self._tag_button.Disable()
                 
-                self._job_key = HydrusThreading.JobKey()
+                self._job_key = ClientThreading.JobKey()
                 
                 HydrusGlobals.client_controller.CallToThread( self.THREADParseImportablePaths, paths, self._job_key )
                 
@@ -3991,7 +3952,7 @@ class DialogSelectYoutubeURL( Dialog ):
                 
                 url_string = title + ' ' + resolution + ' ' + extension
                 
-                job_key = HydrusThreading.JobKey( pausable = True, cancellable = True )
+                job_key = ClientThreading.JobKey( pausable = True, cancellable = True )
                 
                 HydrusGlobals.client_controller.CallToThread( ClientDownloading.THREADDownloadURL, job_key, url, url_string )
                 
@@ -4522,7 +4483,7 @@ class DialogShortcuts( Dialog ):
                             
                             pretty_service_key = service.GetName()
                             
-                        except HydrusExceptions.NotFoundException:
+                        except HydrusExceptions.DataMissing:
                             
                             pretty_service_key = 'service not found'
                             
diff --git a/include/ClientGUIDialogsManage.py b/include/ClientGUIDialogsManage.py
index 67c49acd..829a33d9 100644
--- a/include/ClientGUIDialogsManage.py
+++ b/include/ClientGUIDialogsManage.py
@@ -4345,7 +4345,7 @@ class DialogManageOptions( ClientGUIDialogs.Dialog ):
             
             self.SetBackgroundColour( wx.SystemSettings.GetColour( wx.SYS_COLOUR_BTNFACE ) )
             
-            self._local_port = wx.SpinCtrl( self, min = 0, max = 65535 )
+            self._local_port = ClientGUICommon.NoneableSpinCtrl( self, 'local server port', none_phrase = 'do not run local server', min = 1, max = 65535 )
             
             #
             
@@ -4355,12 +4355,7 @@ class DialogManageOptions( ClientGUIDialogs.Dialog ):
             
             vbox = wx.BoxSizer( wx.VERTICAL )
             
-            hbox = wx.BoxSizer( wx.HORIZONTAL )
-            
-            hbox.AddF( wx.StaticText( self, label = 'local server port: ' ), CC.FLAGS_MIXED )
-            hbox.AddF( self._local_port, CC.FLAGS_MIXED )
-            
-            vbox.AddF( hbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
+            vbox.AddF( self._local_port, CC.FLAGS_MIXED )
             
             self.SetSizer( vbox )
             
@@ -6409,7 +6404,7 @@ class DialogManageServices( ClientGUIDialogs.Dialog ):
                 
                 self._booru_options_panel = ClientGUICommon.StaticBox( self, 'options' )
                 
-                self._port = wx.SpinCtrl( self._booru_options_panel, min = 0, max = 65535 )
+                self._port = ClientGUICommon.NoneableSpinCtrl( self._booru_options_panel, 'booru local port', none_phrase = 'do not run local booru service', min = 1, max = 65535 )
                 
                 self._upnp = ClientGUICommon.NoneableSpinCtrl( self._booru_options_panel, 'upnp port', none_phrase = 'do not forward port', max = 65535 )
                 
@@ -6567,12 +6562,7 @@ class DialogManageServices( ClientGUIDialogs.Dialog ):
             
             if service_type == HC.LOCAL_BOORU:
                 
-                hbox = wx.BoxSizer( wx.HORIZONTAL )
-                
-                hbox.AddF( wx.StaticText( self._booru_options_panel, label = 'port' ), CC.FLAGS_MIXED )
-                hbox.AddF( self._port, CC.FLAGS_EXPAND_BOTH_WAYS )
-                
-                self._booru_options_panel.AddF( hbox, CC.FLAGS_EXPAND_SIZER_BOTH_WAYS )
+                self._booru_options_panel.AddF( self._port, CC.FLAGS_EXPAND_BOTH_WAYS )
                 self._booru_options_panel.AddF( self._upnp, CC.FLAGS_EXPAND_BOTH_WAYS )
                 self._booru_options_panel.AddF( self._max_monthly_data, CC.FLAGS_EXPAND_BOTH_WAYS )
                 
@@ -9310,12 +9300,15 @@ class DialogManageTags( ClientGUIDialogs.Dialog ):
                 forced_reason = 'admin'
                 
             
+            force_siblings = False
+            
             tag_managers = [ m.GetTagsManager() for m in self._media ]
             
             num_files = len( self._media )
             
             sets_of_choices = []
             
+            potential_num_sibling_count = 0
             potential_num_reasons_needed = 0
             
             for tag in tags:
@@ -9344,6 +9337,8 @@ class DialogManageTags( ClientGUIDialogs.Dialog ):
                             
                             choices.append( ( 'add ' + sibling_tag + ' (preferred sibling) to ' + HydrusData.ConvertIntToPrettyString( num_files - num_sibling_current ) + ' files', ( HC.CONTENT_UPDATE_ADD, sibling_tag ) ) )
                             
+                            potential_num_sibling_count += 1
+                            
                         
                     
                     if not only_add:
@@ -9371,6 +9366,8 @@ class DialogManageTags( ClientGUIDialogs.Dialog ):
                                 
                                 choices.append( ( 'pend ' + sibling_tag + ' (preferred sibling) to ' + HydrusData.ConvertIntToPrettyString( num_files - ( num_sibling_current + num_sibling_pending ) ) + ' files', ( HC.CONTENT_UPDATE_PEND, sibling_tag ) ) )
                                 
+                                potential_num_sibling_count += 1
+                                
                             
                         
                     
@@ -9422,7 +9419,7 @@ class DialogManageTags( ClientGUIDialogs.Dialog ):
                 message += os.linesep * 2
                 message += 'To save you time, would you like to use the same reason for all the petitions?'
                 
-                with ClientGUIDialogs.DialogYesNo( self, message ) as yn_dlg:
+                with ClientGUIDialogs.DialogYesNo( self, message, title = 'Many petitions found' ) as yn_dlg:
                     
                     if yn_dlg.ShowModal() == wx.ID_YES:
                         
@@ -9439,6 +9436,21 @@ class DialogManageTags( ClientGUIDialogs.Dialog ):
                     
                 
             
+            if potential_num_sibling_count > 1:
+                
+                message = 'You are about to add more than one tag that has siblings.'
+                message += os.linesep * 2
+                message += 'To save you time, would you like to always choose those siblings, when they exist?'
+                
+                with ClientGUIDialogs.DialogYesNo( self, message, title = 'Many siblings found' ) as yn_dlg:
+                    
+                    if yn_dlg.ShowModal() == wx.ID_YES:
+                        
+                        force_siblings = True
+                        
+                    
+                
+            
             for choices in sets_of_choices:
                 
                 if len( choices ) == 1:
@@ -9447,17 +9459,32 @@ class DialogManageTags( ClientGUIDialogs.Dialog ):
                     
                 else:
                     
-                    intro = 'What would you like to do?'
-                    
-                    with ClientGUIDialogs.DialogButtonChoice( self, intro, choices ) as dlg:
+                    if force_siblings and True in ( '(preferred sibling)' in text_gumpf for ( text_gumpf, choice ) in choices ):
                         
-                        if dlg.ShowModal() == wx.ID_OK:
+                        for ( text_gumpf, choice ) in choices:
                             
-                            choice = dlg.GetData()
+                            if '(preferred sibling)' in text_gumpf:
+                                
+                                break
+                                
                             
-                        else:
+                        
+                    else:
+                        
+                        intro = 'What would you like to do?'
+                        
+                        with ClientGUIDialogs.DialogButtonChoice( self, intro, choices ) as dlg:
                             
-                            continue
+                            result = dlg.ShowModal()
+                            
+                            if result == wx.ID_OK:
+                                
+                                choice = dlg.GetData()
+                                
+                            else:
+                                
+                                break
+                                
                             
                         
                     
diff --git a/include/ClientGUIManagement.py b/include/ClientGUIManagement.py
index 239284ac..6ab757f6 100755
--- a/include/ClientGUIManagement.py
+++ b/include/ClientGUIManagement.py
@@ -17,6 +17,7 @@ import ClientGUIMedia
 import ClientImporting
 import ClientMedia
 import ClientRendering
+import ClientThreading
 import json
 import multipart
 import os
@@ -893,7 +894,7 @@ class ManagementPanelDumper( ManagementPanel ):
             tags_manager = media.GetTagsManager()
             
             try: service = self._controller.GetServicesManager().GetService( service_key )
-            except HydrusExceptions.NotFoundException: continue
+            except HydrusExceptions.FileMissingException: continue
             
             service_key = service.GetServiceKey()
             
@@ -2537,7 +2538,7 @@ class ManagementPanelQuery( ManagementPanel ):
         
         self._search_enabled = self._management_controller.GetVariable( 'search_enabled' )
         
-        self._query_key = HydrusThreading.JobKey( cancellable = True )
+        self._query_key = ClientThreading.JobKey( cancellable = True )
         
         initial_predicates = file_search_context.GetPredicates()
         
@@ -2583,7 +2584,7 @@ class ManagementPanelQuery( ManagementPanel ):
         
         self._query_key.Cancel()
         
-        self._query_key = HydrusThreading.JobKey()
+        self._query_key = ClientThreading.JobKey()
         
         if self._management_controller.GetVariable( 'search_enabled' ) and self._management_controller.GetVariable( 'synchronised' ):
             
diff --git a/include/ClientGUIMedia.py b/include/ClientGUIMedia.py
index 92b8d515..a36d7815 100755
--- a/include/ClientGUIMedia.py
+++ b/include/ClientGUIMedia.py
@@ -1217,7 +1217,7 @@ class MediaPanelThumbnails( MediaPanel ):
             
             thumbnail_index = self._sorted_media.index( thumbnail )
             
-        except HydrusExceptions.NotFoundException:
+        except HydrusExceptions.DataMissing:
             
             # probably means a collect happened during an ongoing waterfall or whatever
             
@@ -1585,7 +1585,7 @@ class MediaPanelThumbnails( MediaPanel ):
             
             index = self._sorted_media.index( thumbnail )
             
-        except HydrusExceptions.NotFoundException:
+        except HydrusExceptions.DataMissing:
             
             return False
             
@@ -1967,6 +1967,10 @@ class MediaPanelThumbnails( MediaPanel ):
                 
                 local_ratings_services = [ service for service in services if service.GetServiceType() in ( HC.LOCAL_RATING_LIKE, HC.LOCAL_RATING_NUMERICAL ) ]
                 
+                local_booru_service = [ service for service in services if service.GetServiceType() == HC.LOCAL_BOORU ][0]
+                
+                local_booru_is_running = local_booru_service.GetInfo()[ 'port' ] is not None
+                
                 i_can_post_ratings = len( local_ratings_services ) > 0
                 
                 focussed_is_local = CC.LOCAL_FILE_SERVICE_KEY in self._focussed_media.GetLocationsManager().GetCurrent()
@@ -2299,9 +2303,17 @@ class MediaPanelThumbnails( MediaPanel ):
                 
                 if focussed_is_local:
                     
-                    if self._focussed_media.GetMime() in HC.IMAGES and self._focussed_media.GetDuration() is None: copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_bmp' ) , 'image' )
+                    if self._focussed_media.GetMime() in HC.IMAGES and self._focussed_media.GetDuration() is None:
+                        
+                        copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_bmp' ) , 'image' )
+                        
+                    
                     copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_path' ) , 'path' )
-                    copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_local_url' ) , 'local url' )
+                    
+                    if HC.options[ 'local_port' ] is not None:
+                        
+                        copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_local_url' ) , 'local url' )
+                        
                     
                 
                 share_menu.AppendMenu( CC.ID_NULL, 'copy', copy_menu )
@@ -2321,7 +2333,10 @@ class MediaPanelThumbnails( MediaPanel ):
                 
                 #
                 
-                share_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'share_on_local_booru' ), 'on local booru' )
+                if local_booru_is_running:
+                    
+                    share_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'share_on_local_booru' ), 'on local booru' )
+                    
                 
                 #
                 
diff --git a/include/ClientImporting.py b/include/ClientImporting.py
index 4f72184c..cde69db5 100644
--- a/include/ClientImporting.py
+++ b/include/ClientImporting.py
@@ -4,6 +4,7 @@ import ClientData
 import ClientDefaults
 import ClientDownloading
 import ClientFiles
+import ClientThreading
 import collections
 import HydrusConstants as HC
 import HydrusData
@@ -1077,7 +1078,7 @@ class ImportFolder( HydrusSerialisable.SerialisableBaseNamed ):
                 
                 if self._open_popup and len( successful_hashes ) > 0:
                     
-                    job_key = HydrusThreading.JobKey()
+                    job_key = ClientThreading.JobKey()
                     
                     job_key.SetVariable( 'popup_title', 'import folder - ' + self._name )
                     job_key.SetVariable( 'popup_files', successful_hashes )
@@ -2324,7 +2325,7 @@ class Subscription( HydrusSerialisable.SerialisableBaseNamed ):
         
         if p1 and p2 and p3 and ( p4 or p5 ):
             
-            job_key = HydrusThreading.JobKey( pausable = False, cancellable = True )
+            job_key = ClientThreading.JobKey( pausable = False, cancellable = True )
             
             try:
                 
diff --git a/include/ClientMedia.py b/include/ClientMedia.py
index 64165152..82bd6c3c 100644
--- a/include/ClientMedia.py
+++ b/include/ClientMedia.py
@@ -1368,7 +1368,7 @@ class SortedList( object ):
             
         except KeyError:
             
-            raise HydrusExceptions.NotFoundException()
+            raise HydrusExceptions.DataMissing()
             
         
         return result
diff --git a/include/ClientThreading.py b/include/ClientThreading.py
new file mode 100644
index 00000000..1e82a65b
--- /dev/null
+++ b/include/ClientThreading.py
@@ -0,0 +1,208 @@
+import HydrusExceptions
+import Queue
+import threading
+import time
+import traceback
+import HydrusData
+import HydrusGlobals
+import HydrusThreading
+import os
+
+class JobKey( object ):
+    
+    def __init__( self, pausable = False, cancellable = False, only_when_idle = False, only_start_if_unbusy = False ):
+        
+        self._key = HydrusData.GenerateKey()
+        
+        self._pausable = pausable
+        self._cancellable = cancellable
+        self._only_when_idle = only_when_idle
+        self._only_start_if_unbusy = only_start_if_unbusy
+        
+        self._deleted = threading.Event()
+        self._begun = threading.Event()
+        self._done = threading.Event()
+        self._cancelled = threading.Event()
+        self._paused = threading.Event()
+        
+        self._yield_pause_period = 10
+        self._last_yield_pause = HydrusData.GetNow() + self._yield_pause_period
+        
+        self._variable_lock = threading.Lock()
+        self._variables = dict()
+        
+    
+    def __eq__( self, other ): return self.__hash__() == other.__hash__()
+    
+    def __hash__( self ): return self._key.__hash__()
+    
+    def __ne__( self, other ): return self.__hash__() != other.__hash__()
+    
+    def Begin( self ): self._begun.set()
+    
+    def CanBegin( self ):
+        
+        if self._only_when_idle and not HydrusGlobals.client_controller.CurrentlyIdle():
+            
+            return False
+            
+        
+        if self._only_start_if_unbusy and HydrusGlobals.client_controller.SystemBusy():
+            
+            return False
+            
+        
+        return True
+        
+    
+    def Cancel( self ):
+        
+        self._cancelled.set()
+        
+        self.Finish()
+        
+    
+    def Delete( self ):
+        
+        self.Finish()
+        
+        self._deleted.set()
+        
+    
+    def DeleteVariable( self, name ):
+        
+        with self._variable_lock:
+            
+            if name in self._variables: del self._variables[ name ]
+            
+        
+        time.sleep( 0.00001 )
+        
+    
+    def Finish( self ): self._done.set()
+    
+    def GetKey( self ): return self._key
+    
+    def GetVariable( self, name ):
+        
+        with self._variable_lock: return self._variables[ name ]
+        
+    
+    def HasVariable( self, name ):
+        
+        with self._variable_lock: return name in self._variables
+        
+    
+    def IsBegun( self ):
+        
+        return self._begun.is_set()
+        
+    
+    def IsCancellable( self ):
+        
+        return self._cancellable and not self.IsDone()
+        
+    
+    def IsCancelled( self ):
+        
+        return HydrusThreading.IsThreadShuttingDown() or self._cancelled.is_set()
+        
+    
+    def IsDeleted( self ):
+        
+        return HydrusThreading.IsThreadShuttingDown() or self._deleted.is_set()
+        
+    
+    def IsDone( self ):
+        
+        return HydrusThreading.IsThreadShuttingDown() or self._done.is_set()
+        
+    
+    def IsPausable( self ): return self._pausable and not self.IsDone()
+    
+    def IsPaused( self ): return self._paused.is_set() and not self.IsDone()
+    
+    def IsWorking( self ): return self.IsBegun() and not self.IsDone()
+    
+    def PausePlay( self ):
+        
+        if self._paused.is_set(): self._paused.clear()
+        else: self._paused.set()
+        
+    
+    def SetCancellable( self, value ): self._cancellable = value
+    
+    def SetPausable( self, value ): self._pausable = value
+    
+    def SetVariable( self, name, value ):
+        
+        with self._variable_lock: self._variables[ name ] = value
+        
+        time.sleep( 0.00001 )
+        
+    
+    def ToString( self ):
+        
+        stuff_to_print = []
+        
+        with self._variable_lock:
+            
+            if 'popup_title' in self._variables: stuff_to_print.append( self._variables[ 'popup_title' ] )
+            
+            if 'popup_text_1' in self._variables: stuff_to_print.append( self._variables[ 'popup_text_1' ] )
+            
+            if 'popup_text_2' in self._variables: stuff_to_print.append( self._variables[ 'popup_text_2' ] )
+            
+            if 'popup_traceback' in self._variables: stuff_to_print.append( self._variables[ 'popup_traceback' ] )
+            
+            if 'popup_caller_traceback' in self._variables: stuff_to_print.append( self._variables[ 'popup_caller_traceback' ] )
+            
+            if 'popup_db_traceback' in self._variables: stuff_to_print.append( self._variables[ 'popup_db_traceback' ] )
+            
+        
+        stuff_to_print = [ HydrusData.ToUnicode( s ) for s in stuff_to_print ]
+        
+        try:
+            
+            return os.linesep.join( stuff_to_print )
+            
+        except:
+            
+            return repr( stuff_to_print )
+            
+        
+    
+    def WaitIfNeeded( self ):
+        
+        if HydrusData.TimeHasPassed( self._last_yield_pause ):
+            
+            time.sleep( 0.1 )
+            
+            self._last_yield_pause = HydrusData.GetNow() + self._yield_pause_period
+            
+        
+        i_paused = False
+        should_quit = False
+        
+        while self.IsPaused():
+            
+            i_paused = True
+            
+            time.sleep( 0.1 )
+            
+            if HydrusThreading.IsThreadShuttingDown() or self.IsDone(): break
+            
+        
+        if HydrusThreading.IsThreadShuttingDown() or self.IsCancelled():
+            
+            should_quit = True
+            
+        
+        if self._only_when_idle and not HydrusGlobals.client_controller.CurrentlyIdle():
+            
+            should_quit = True
+            
+        
+        return ( i_paused, should_quit )
+        
+    
\ No newline at end of file
diff --git a/include/HydrusConstants.py b/include/HydrusConstants.py
index d7a5c183..81647098 100755
--- a/include/HydrusConstants.py
+++ b/include/HydrusConstants.py
@@ -53,7 +53,7 @@ options = {}
 # Misc
 
 NETWORK_VERSION = 17
-SOFTWARE_VERSION = 192
+SOFTWARE_VERSION = 193
 
 UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )
 
diff --git a/include/HydrusController.py b/include/HydrusController.py
index 4c1fe546..b63757a7 100644
--- a/include/HydrusController.py
+++ b/include/HydrusController.py
@@ -175,9 +175,10 @@ class HydrusController( object ):
         if not self._no_daemons:
             
             self._daemons.append( HydrusThreading.DAEMONWorker( self, 'SleepCheck', HydrusDaemons.DAEMONSleepCheck, period = 120 ) )
-            self._daemons.append( HydrusThreading.DAEMONWorker( self, 'MaintainDB', HydrusDaemons.DAEMONMaintainDB, period = 300 ) )
             self._daemons.append( HydrusThreading.DAEMONWorker( self, 'MaintainMemory', HydrusDaemons.DAEMONMaintainMemory, period = 300 ) )
             
+            self._daemons.append( HydrusThreading.DAEMONBigJobWorker( self, 'MaintainDB', HydrusDaemons.DAEMONMaintainDB, period = 300 ) )
+            
         
     
     def MaintainDB( self ):
diff --git a/include/HydrusDB.py b/include/HydrusDB.py
index def1f635..11ff94cf 100644
--- a/include/HydrusDB.py
+++ b/include/HydrusDB.py
@@ -35,6 +35,9 @@ class HydrusDB( object ):
         
         self._currently_doing_job = False
         
+        self._db = None
+        self._c = None
+        
         if os.path.exists( self._db_path ):
             
             # open and close to clean up in case last session didn't close well
@@ -85,11 +88,17 @@ class HydrusDB( object ):
     
     def _CloseDBCursor( self ):
         
-        self._c.close()
-        self._db.close()
-        
-        del self._db
-        del self._c
+        if self._db is not None:
+            
+            self._c.close()
+            self._db.close()
+            
+            del self._c
+            del self._db
+            
+            self._db = None
+            self._c = None
+            
         
     
     def _CreateDB( self ):
@@ -151,6 +160,8 @@ class HydrusDB( object ):
     
     def _InitDBCursor( self ):
         
+        self._CloseDBCursor()
+        
         db_just_created = not os.path.exists( self._db_path )
         
         if os.path.exists( self._no_wal_path ):
@@ -164,7 +175,7 @@ class HydrusDB( object ):
         
         self._c = self._db.cursor()
         
-        self._c.execute( 'PRAGMA cache_size = -20000;' )
+        self._c.execute( 'PRAGMA cache_size = -50000;' )
         
         if self._no_wal:
             
@@ -184,7 +195,9 @@ class HydrusDB( object ):
                 
                 self._c.execute( 'SELECT * FROM sqlite_master;' ).fetchone()
                 
-            except sqlite3.OperationalError:
+            except sqlite3.OperationalError as e:
+                
+                traceback.print_exc()
                 
                 def create_no_wal_file():
                     
diff --git a/include/HydrusData.py b/include/HydrusData.py
index 0231dd9d..c02c36b3 100644
--- a/include/HydrusData.py
+++ b/include/HydrusData.py
@@ -140,7 +140,6 @@ def ConvertContentsToClientToServerContentUpdatePackage( action, contents, reaso
     
     return ClientToServerContentUpdatePackage( content_data_dict, hash_ids_to_hashes )
     
-    
 def ConvertIntToBytes( size ):
     
     if size is None: return 'unknown size'
@@ -1629,7 +1628,11 @@ class ContentUpdate( object ):
         
         if self._data_type == HC.CONTENT_TYPE_FILES:
             
-            if self._action == HC.CONTENT_UPDATE_ADD:
+            if self._action == HC.CONTENT_UPDATE_ADVANCED:
+                
+                hashes = set()
+                
+            elif self._action == HC.CONTENT_UPDATE_ADD:
                 
                 hash = self._row[0]
                 
diff --git a/include/HydrusExceptions.py b/include/HydrusExceptions.py
index 48224ddc..d3b09739 100644
--- a/include/HydrusExceptions.py
+++ b/include/HydrusExceptions.py
@@ -1,10 +1,10 @@
 class CantRenderWithCVException( Exception ): pass
+class DataMissing( Exception ): pass
 class DBException( Exception ): pass
 class DBAccessException( Exception ): pass
-class FileException( Exception ): pass
+class FileMissingException( Exception ): pass
 class MimeException( Exception ): pass
 class NameException( Exception ): pass
-class NotFoundException( Exception ): pass
 class PermissionException( Exception ): pass
 class ShutdownException( Exception ): pass
 class SizeException( Exception ): pass
@@ -14,6 +14,7 @@ class FirewallException( NetworkException ): pass
 class ForbiddenException( NetworkException ): pass
 class NetworkVersionException( NetworkException ): pass
 class NoContentException( NetworkException ): pass
+class NotFoundException( NetworkException ): pass
 class NotModifiedException( NetworkException ): pass
 class ServerBusyException( NetworkException ): pass
 class SessionException( NetworkException ): pass
diff --git a/include/HydrusNATPunch.py b/include/HydrusNATPunch.py
index 9af96f65..884c5532 100644
--- a/include/HydrusNATPunch.py
+++ b/include/HydrusNATPunch.py
@@ -72,14 +72,24 @@ def GetLocalIP(): return socket.gethostbyname( socket.gethostname() )
 def AddUPnPMapping( internal_client, internal_port, external_port, protocol, description, duration = 3600 ):
     
     cmd = [ upnpc_path, '-e', description, '-a', internal_client, str( internal_port ), str( external_port ), protocol, str( duration ) ]
-    
+    HydrusData.DebugPrint( cmd )
     p = subprocess.Popen( cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE, startupinfo = HydrusData.GetSubprocessStartupInfo() )
     
     p.wait()
     
     ( output, error ) = p.communicate()
     
-    if error is not None and len( error ) > 0: raise Exception( 'Problem while trying to add UPnP mapping:' + os.linesep * 2 + HydrusData.ToUnicode( error ) )
+    HydrusData.DebugPrint( output )
+    
+    if output is not None and 'failed with code' in output:
+        
+        raise Exception( 'Problem while trying to add UPnP mapping:' + os.linesep * 2 + HydrusData.ToUnicode( output ) )
+        
+    
+    if error is not None and len( error ) > 0:
+        
+        raise Exception( 'Problem while trying to add UPnP mapping:' + os.linesep * 2 + HydrusData.ToUnicode( error ) )
+        
     
 def GetUPnPMappings():
     
diff --git a/include/HydrusThreading.py b/include/HydrusThreading.py
index 2579eddc..09bf08c9 100644
--- a/include/HydrusThreading.py
+++ b/include/HydrusThreading.py
@@ -108,13 +108,6 @@ class DAEMONQueue( DAEMON ):
                 self._event.clear()
                 
             
-            while not self._controller.GoodTimeToDoBackgroundWork():
-                
-                if IsThreadShuttingDown(): return
-                
-                time.sleep( 10 )
-                
-            
             items = []
             
             while not self._queue.empty(): items.append( self._queue.get() )
@@ -136,6 +129,56 @@ class DAEMONQueue( DAEMON ):
     
 class DAEMONWorker( DAEMON ):
     
+    def __init__( self, controller, name, callable, topics = None, period = 3600 ):
+        
+        if topics is None: topics = []
+        
+        DAEMON.__init__( self, controller, name )
+        
+        self._callable = callable
+        self._topics = topics
+        self._period = period
+        
+        for topic in topics: self._controller.sub( self, 'set', topic )
+        
+        self.start()
+        
+    
+    def run( self ):
+        
+        time.sleep( 3 )
+        
+        while True:
+            
+            if IsThreadShuttingDown(): return
+            
+            try:
+                
+                self._callable( self._controller )
+                
+            except HydrusExceptions.ShutdownException:
+                
+                return
+                
+            except Exception as e:
+                
+                HydrusData.ShowText( 'Daemon ' + self._name + ' encountered an exception:' )
+                
+                HydrusData.ShowException( e )
+                
+            
+            if IsThreadShuttingDown(): return
+            
+            self._event.wait( self._period )
+            
+            self._event.clear()
+            
+        
+    
+    def set( self, *args, **kwargs ): self._event.set()
+    
+class DAEMONBigJobWorker( DAEMON ):
+    
     def __init__( self, controller, name, callable, topics = None, period = 3600, init_wait = 3, pre_callable_wait = 3 ):
         
         if topics is None: topics = []
@@ -244,170 +287,4 @@ class THREADCallToThread( DAEMON ):
             time.sleep( 0.00001 )
             
         
-    
-class JobKey( object ):
-    
-    def __init__( self, pausable = False, cancellable = False ):
-        
-        self._key = HydrusData.GenerateKey()
-        
-        self._pausable = pausable
-        self._cancellable = cancellable
-        
-        self._deleted = threading.Event()
-        self._begun = threading.Event()
-        self._done = threading.Event()
-        self._cancelled = threading.Event()
-        self._paused = threading.Event()
-        
-        self._variable_lock = threading.Lock()
-        self._variables = dict()
-        
-    
-    def __eq__( self, other ): return self.__hash__() == other.__hash__()
-    
-    def __hash__( self ): return self._key.__hash__()
-    
-    def __ne__( self, other ): return self.__hash__() != other.__hash__()
-    
-    def Begin( self ): self._begun.set()
-    
-    def Cancel( self ):
-        
-        self._cancelled.set()
-        
-        self.Finish()
-        
-    
-    def Delete( self ):
-        
-        self.Finish()
-        
-        self._deleted.set()
-        
-    
-    def DeleteVariable( self, name ):
-        
-        with self._variable_lock:
-            
-            if name in self._variables: del self._variables[ name ]
-            
-        
-        time.sleep( 0.00001 )
-        
-    
-    def Finish( self ): self._done.set()
-    
-    def GetKey( self ): return self._key
-    
-    def GetVariable( self, name ):
-        
-        with self._variable_lock: return self._variables[ name ]
-        
-    
-    def HasVariable( self, name ):
-        
-        with self._variable_lock: return name in self._variables
-        
-    
-    def IsBegun( self ):
-        
-        return self._begun.is_set()
-        
-    
-    def IsCancellable( self ):
-        
-        return self._cancellable and not self.IsDone()
-        
-    
-    def IsCancelled( self ):
-        
-        return IsThreadShuttingDown() or self._cancelled.is_set()
-        
-    
-    def IsDeleted( self ):
-        
-        return IsThreadShuttingDown() or self._deleted.is_set()
-        
-    
-    def IsDone( self ):
-        
-        return IsThreadShuttingDown() or self._done.is_set()
-        
-    
-    def IsPausable( self ): return self._pausable and not self.IsDone()
-    
-    def IsPaused( self ): return self._paused.is_set() and not self.IsDone()
-    
-    def IsWorking( self ): return self.IsBegun() and not self.IsDone()
-    
-    def PausePlay( self ):
-        
-        if self._paused.is_set(): self._paused.clear()
-        else: self._paused.set()
-        
-    
-    def SetCancellable( self, value ): self._cancellable = value
-    
-    def SetPausable( self, value ): self._pausable = value
-    
-    def SetVariable( self, name, value ):
-        
-        with self._variable_lock: self._variables[ name ] = value
-        
-        time.sleep( 0.00001 )
-        
-    
-    def ToString( self ):
-        
-        stuff_to_print = []
-        
-        with self._variable_lock:
-            
-            if 'popup_title' in self._variables: stuff_to_print.append( self._variables[ 'popup_title' ] )
-            
-            if 'popup_text_1' in self._variables: stuff_to_print.append( self._variables[ 'popup_text_1' ] )
-            
-            if 'popup_text_2' in self._variables: stuff_to_print.append( self._variables[ 'popup_text_2' ] )
-            
-            if 'popup_traceback' in self._variables: stuff_to_print.append( self._variables[ 'popup_traceback' ] )
-            
-            if 'popup_caller_traceback' in self._variables: stuff_to_print.append( self._variables[ 'popup_caller_traceback' ] )
-            
-            if 'popup_db_traceback' in self._variables: stuff_to_print.append( self._variables[ 'popup_db_traceback' ] )
-            
-        
-        stuff_to_print = [ HydrusData.ToUnicode( s ) for s in stuff_to_print ]
-        
-        try:
-            
-            return os.linesep.join( stuff_to_print )
-            
-        except:
-            
-            return repr( stuff_to_print )
-            
-        
-    
-    def WaitIfNeeded( self ):
-        
-        i_paused = False
-        should_quit = False
-        
-        while self.IsPaused():
-            
-            i_paused = True
-            
-            time.sleep( 0.1 )
-            
-            if IsThreadShuttingDown() or self.IsDone(): break
-            
-        
-        if IsThreadShuttingDown() or self.IsCancelled():
-            
-            should_quit = True
-            
-        
-        return ( i_paused, should_quit )
-        
     
\ No newline at end of file
diff --git a/include/ServerDB.py b/include/ServerDB.py
index 9f42478e..97f5c99a 100755
--- a/include/ServerDB.py
+++ b/include/ServerDB.py
@@ -1131,7 +1131,10 @@ class DB( HydrusDB.HydrusDB ):
             
             ( account_id, ) = self._c.execute( 'SELECT account_id FROM contacts WHERE service_id = ? AND contact_key = ?;', ( service_id, sqlite3.Binary( contact_key ) ) ).fetchone()
             
-        except: raise HydrusExceptions.NotFoundException( 'Could not find that contact key!' )
+        except:
+            
+            raise HydrusExceptions.NotFoundException( 'Could not find that contact key!' )
+            
         
         return account_id
         
@@ -1191,7 +1194,10 @@ class DB( HydrusDB.HydrusDB ):
         
         result = self._c.execute( 'SELECT account_type_id FROM account_types WHERE service_id = ? AND title = ?;', ( service_id, title ) ).fetchone()
         
-        if result is None: raise HydrusExceptions.NotFoundException( 'Could not find account title ' + HydrusData.ToUnicode( title ) + ' in db for this service.' )
+        if result is None:
+            
+            raise HydrusExceptions.NotFoundException( 'Could not find account title ' + HydrusData.ToUnicode( title ) + ' in db for this service.' )
+            
         
         ( account_type_id, ) = result
         
@@ -1231,7 +1237,10 @@ class DB( HydrusDB.HydrusDB ):
         
         result = self._c.execute( 'SELECT DISTINCT account_id, reason_id FROM file_petitions WHERE service_id = ? AND status = ? ORDER BY RANDOM() LIMIT 1;', ( service_id, status ) ).fetchone()
         
-        if result is None: raise HydrusExceptions.NotFoundException( 'No petitions!' )
+        if result is None:
+            
+            raise HydrusExceptions.NotFoundException( 'No petitions!' )
+            
         
         ( account_id, reason_id ) = result
         
@@ -1814,8 +1823,7 @@ class DB( HydrusDB.HydrusDB ):
         
         self._c.execute( 'VACUUM' )
         
-        self._c.close()
-        self._db.close()
+        self._CloseDBCursor()
         
         backup_path = os.path.join( HC.DB_DIR, 'server_backup' )
         
diff --git a/include/TestDialogs.py b/include/TestDialogs.py
index 11dc34b8..6703d543 100644
--- a/include/TestDialogs.py
+++ b/include/TestDialogs.py
@@ -109,27 +109,6 @@ class TestNonDBDialogs( unittest.TestCase ):
             
         
     
-    def test_dialog_first_start( self ):
-        
-        with ClientGUIDialogs.DialogFirstStart( None ) as dlg:
-            
-            HitCancelButton( dlg )
-            
-            result = dlg.ShowModal()
-            
-            self.assertEqual( result, wx.ID_CANCEL )
-            
-        
-        with ClientGUIDialogs.DialogFirstStart( None ) as dlg:
-            
-            HitButton( dlg._ok )
-            
-            result = dlg.ShowModal()
-            
-            self.assertEqual( result, wx.ID_OK )
-            
-        
-    
     def test_select_from_list_of_strings( self ):
         
         with ClientGUIDialogs.DialogSelectFromListOfStrings( None, 'select from a list of strings', [ 'a', 'b', 'c' ] ) as dlg:
diff --git a/test.py b/test.py
index bf8e7c79..69d4005f 100644
--- a/test.py
+++ b/test.py
@@ -59,6 +59,8 @@ class Controller( object ):
         HC.SERVER_THUMBNAILS_DIR = os.path.join( HC.DB_DIR, 'server_thumbnails' )
         HC.SERVER_UPDATES_DIR = os.path.join( HC.DB_DIR, 'server_updates' )
         
+        os.makedirs( HC.CLIENT_FILES_DIR ) # for the client files manager which I manually create in a bit
+        
         HydrusGlobals.controller = self
         HydrusGlobals.client_controller = self
         HydrusGlobals.server_controller = self