From 6fbe2c285e4761ec137f0f38e312d6f42e4c5673 Mon Sep 17 00:00:00 2001
From: Hydrus Network Developer <hydrus.admin@gmail.com>
Date: Wed, 12 Jul 2017 15:03:45 -0500
Subject: [PATCH] Version 264

---
 help/changelog.html                          |   33 +
 help/database_migration.html                 |    2 +-
 include/ClientConstants.py                   |   19 +
 include/ClientController.py                  |   76 +-
 include/ClientDB.py                          | 1462 +++++++++---------
 include/ClientData.py                        |    1 +
 include/ClientDownloading.py                 |    9 +-
 include/ClientGUI.py                         |  164 +-
 include/ClientGUICanvas.py                   |   15 +-
 include/ClientGUICommon.py                   |   91 +-
 include/ClientGUIControls.py                 |   78 +-
 include/ClientGUIManagement.py               |  166 +-
 include/ClientGUIParsing.py                  |   11 +-
 include/ClientGUIScrolledPanelsManagement.py |    2 +-
 include/ClientGUIScrolledPanelsReview.py     |  457 +++++-
 include/ClientGUITopLevelWindows.py          |    2 +-
 include/ClientImporting.py                   |  423 +++--
 include/ClientNetworking.py                  |   71 +-
 include/HydrusConstants.py                   |    2 +-
 include/HydrusController.py                  |   30 +-
 include/HydrusDB.py                          |  120 +-
 include/HydrusPaths.py                       |   12 +-
 include/HydrusVideoHandling.py               |    4 +-
 include/ServerController.py                  |   10 +-
 include/ServerDB.py                          |   27 +-
 static/refresh.png                           |  Bin 0 -> 685 bytes
 test.py                                      |   14 +-
 27 files changed, 2108 insertions(+), 1193 deletions(-)
 create mode 100644 static/refresh.png

diff --git a/help/changelog.html b/help/changelog.html
index 9c785f9a..b21f7152 100755
--- a/help/changelog.html
+++ b/help/changelog.html
@@ -8,6 +8,39 @@
 		<div class="content">
 			<h3>changelog</h3>
 			<ul>
+				<li><h3>version 264</h3></li>
+				<ul>
+					<li>converted the page of images downloader to the new network engine</li>
+					<li>converted the thread watcher downloader to the new network engine</li>
+					<li>rejiggered page of images and thread watcher management panels to better separate the two loops they work on</li>
+					<li>rejiggered some more downloader layout stuff in prep for a new control to better display url cache and overall import status--downloader-global pause button is now always at the top, for instance</li>
+					<li>removed 'waiting politely' indicator from download pages that use the new network engine</li>
+					<li>network job controls now have a simple border</li>
+					<li>network job controls will show current bytes progress and speed in a more intelligent and useful way--no longer spamming the initial 0KB/s, for instance, and flashing more useful summary information when downloading many small files</li>
+					<li>network job controls now display their current bytes progress and speed on a separate right-aligned text beside the main status text (to stop the speed text jumping around so much)</li>
+					<li>network job controls will not show the current download speed until some bytes are actually read</li>
+					<li>wrote a first version of 'services->review bandwidth' review frame, which lists all network contexts with data in the past month</li>
+					<li>wrote an independant and live-updating review frame for indivdual network contexts, launched from the review bandwidth frame, which shows current and historical bandwidth use for that context</li>
+					<li>the main gui status bar now reports current bandwidth usage! updated once a second</li>
+					<li>database->backup process now has some nicer gui, better and more reliable workflow, improved file copy notification, and now correctly obeys a popup-cancel command</li>
+					<li>started the new 'migrate database' gui under the database menu. it currently shows live usage and will gain some verb buttons to atomically alter things hopefully next week</li>
+					<li>moved from a strict begin/commit db model to a save/release system that only commits changes to disk at most every ten seconds or so. small 'write' transactions will now often have no disk-sync overhead and so are super fast, particularly when they come in a large batch.</li>
+					<li>simplified database connection, disconnection, begin, commit, and rollback code significantly</li>
+					<li>the url table in the client database is now non-unique (i.e. multiple files can have the same url). this may lead to undesired 'already in db' or 'deleted' statuses in some downloaders as they check for url status before starting their downloads. if this change turns out to a be a big pain in the future for pixiv manga or whatever, it may be revisited.</li>
+					<li>fixed a bug in deviant art parser that sometimes meant nsfw urls could not be found</li>
+					<li>fixed an issue where progress gauges could sometimes get stuck in a pulsing state</li>
+					<li>fixed a bug with the new optimised result building which sometimes occured when all the files in a batch had common ipfs service membership</li>
+					<li>improved support for unusual videos</li>
+					<li>improved accuracy of duration calculation for unusual videos</li>
+					<li>improved database optimisation after initial and ongoing repository processing</li>
+					<li>a 'wake subscriptions' event now occurs even on 'manage subscriptions' cancel (meaning temporarily-dialog-paused subs will always continue after the dialog is closed)</li>
+					<li>added a careful manual commit to stop the possibility of the definition-desync some users noticed after a power-loss during a big repository sync commit</li>
+					<li>removed some redundant old db compatibility code</li>
+					<li>permitted sqlite installations that allow in-memory temporary tables to use them (but not for vacuums, which tend to be a bit big for this stuff)</li>
+					<li>cleaned up some redundant timer id code</li>
+					<li>misc improvements</li>
+					<li>misc refactoring</li>
+				</ul>
 				<li><h3>version 263</h3></li>
 				<ul>
 					<li>greatly improved how gui sessions are loaded--now the page tabs are loaded instantly, but the thumbnails are loaded in the background. session loaded should be significantly less laggy and buggy</li>
diff --git a/help/database_migration.html b/help/database_migration.html
index ad214fc0..266479ca 100644
--- a/help/database_migration.html
+++ b/help/database_migration.html
@@ -37,7 +37,7 @@
 			<p>If you move your database, the program will have to shut down first. Before you boot up again, you will have to create a new program shortcut:</p>
 			<h3>informing the software that the database has moved</h3>
 			<p>A straight call to the client executable will look for a database in <i>install_dir/db</i>. If one is not found, it will create one. So, if you move your database and then try to run the client again, it will try to create a new empty database in the previous location!</p>
-			<p>So, pass it a command line argument, like so:</p>
+			<p>So, pass it a -d or --db_dir command line argument, like so:</p>
 			<ul>
 				<li>client -d="D:\media\my_hydrus_database"</li>
 				<li><i>--or--</i></li>
diff --git a/include/ClientConstants.py b/include/ClientConstants.py
index 5e256da0..9308b4b3 100755
--- a/include/ClientConstants.py
+++ b/include/ClientConstants.py
@@ -252,6 +252,24 @@ NETWORK_CONTEXT_DOWNLOADER = 3
 NETWORK_CONTEXT_DOWNLOADER_QUERY = 4
 NETWORK_CONTEXT_SUBSCRIPTION = 5
 
+network_context_type_string_lookup = {}
+
+network_context_type_string_lookup[ NETWORK_CONTEXT_GLOBAL ] = 'global'
+network_context_type_string_lookup[ NETWORK_CONTEXT_HYDRUS ] = 'hydrus service'
+network_context_type_string_lookup[ NETWORK_CONTEXT_DOMAIN ] = 'web domain'
+network_context_type_string_lookup[ NETWORK_CONTEXT_DOWNLOADER ] = 'downloader'
+network_context_type_string_lookup[ NETWORK_CONTEXT_DOWNLOADER_QUERY ] = 'downloader query instance'
+network_context_type_string_lookup[ NETWORK_CONTEXT_SUBSCRIPTION ] = 'subscription'
+
+network_context_type_description_lookup = {}
+
+network_context_type_description_lookup[ NETWORK_CONTEXT_GLOBAL ] = 'All network traffic, no matter the source or destination.'
+network_context_type_description_lookup[ NETWORK_CONTEXT_HYDRUS ] = 'Network traffic going to or from this hydrus service.'
+network_context_type_description_lookup[ NETWORK_CONTEXT_DOMAIN ] = 'Network traffic going to or from this domain (or a subdomain).'
+network_context_type_description_lookup[ NETWORK_CONTEXT_DOWNLOADER ] = 'Network traffic going through this downloader.'
+network_context_type_description_lookup[ NETWORK_CONTEXT_DOWNLOADER_QUERY ] = 'Network traffic going through this single downloader query (you probably shouldn\'t be able to see this!)'
+network_context_type_description_lookup[ NETWORK_CONTEXT_SUBSCRIPTION ] = 'Network traffic going through this subscription.'
+
 SHORTCUT_MODIFIER_CTRL = 0
 SHORTCUT_MODIFIER_ALT = 1
 SHORTCUT_MODIFIER_SHIFT = 2
@@ -511,6 +529,7 @@ class GlobalBMPs( object ):
         GlobalBMPs.inbox = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'inbox.png' ) )
         GlobalBMPs.trash = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'trash.png' ) )
         
+        GlobalBMPs.refresh = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'refresh.png' ) )
         GlobalBMPs.archive = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'archive.png' ) )
         GlobalBMPs.to_inbox = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'to_inbox.png' ) )
         GlobalBMPs.delete = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'trash.png' ) )
diff --git a/include/ClientController.py b/include/ClientController.py
index 454b2e61..42dd10cd 100755
--- a/include/ClientController.py
+++ b/include/ClientController.py
@@ -12,6 +12,7 @@ import HydrusData
 import HydrusExceptions
 import HydrusGlobals as HG
 import HydrusNetworking
+import HydrusPaths
 import HydrusSerialisable
 import HydrusThreading
 import HydrusVideoHandling
@@ -48,7 +49,7 @@ class Controller( HydrusController.HydrusController ):
         
         # just to set up some defaults, in case some db update expects something for an odd yaml-loading reason
         self._options = ClientDefaults.GetClientDefaultOptions()
-        self._new_options = ClientData.ClientOptions( self._db_dir )
+        self._new_options = ClientData.ClientOptions( self.db_dir )
         
         HC.options = self._options
         
@@ -63,28 +64,47 @@ class Controller( HydrusController.HydrusController ):
     
     def _InitDB( self ):
         
-        return ClientDB.DB( self, self._db_dir, 'client', no_wal = self._no_wal )
+        return ClientDB.DB( self, self.db_dir, 'client', no_wal = self._no_wal )
         
     
     def BackupDatabase( self ):
         
-        with wx.DirDialog( self._gui, 'Select backup location.' ) as dlg:
+        path = self._new_options.GetNoneableString( 'backup_path' )
+        
+        if path is None:
             
-            if dlg.ShowModal() == wx.ID_OK:
+            wx.MessageBox( 'No backup path is set!' )
+            
+            return
+            
+        
+        if not os.path.exists( path ):
+            
+            wx.MessageBox( 'The backup path does not exist--creating it now.' )
+            
+            HydrusPaths.MakeSureDirectoryExists( path )
+            
+        
+        client_db_path = os.path.join( path, 'client.db' )
+        
+        if os.path.exists( client_db_path ):
+            
+            action = 'Update the existing'
+            
+        else:
+            
+            action = 'Create a new'
+            
+        
+        text = action + ' backup at "' + path + '"?'
+        text += os.linesep * 2
+        text += 'The database will be locked while the backup occurs, which may lock up your gui as well.'
+        
+        with ClientGUIDialogs.DialogYesNo( self._gui, text ) as dlg_yn:
+            
+            if dlg_yn.ShowModal() == wx.ID_YES:
                 
-                path = HydrusData.ToUnicode( dlg.GetPath() )
-                
-                text = 'Are you sure "' + path + '" is the correct directory?'
-                text += os.linesep * 2
-                text += 'The database will be locked while the backup occurs, which may lock up your gui as well.'
-                
-                with ClientGUIDialogs.DialogYesNo( self._gui, text ) as dlg_yn:
-                    
-                    if dlg_yn.ShowModal() == wx.ID_YES:
-                        
-                        self.Write( 'backup', path )
-                        
-                    
+                self.Write( 'backup', path )
                 
             
         
@@ -153,7 +173,7 @@ class Controller( HydrusController.HydrusController ):
     
     def CheckAlreadyRunning( self ):
     
-        while HydrusData.IsAlreadyRunning( self._db_dir, 'client' ):
+        while HydrusData.IsAlreadyRunning( self.db_dir, 'client' ):
             
             self.pub( 'splash_set_status_text', 'client already running' )
             
@@ -176,7 +196,7 @@ class Controller( HydrusController.HydrusController ):
             
             for i in range( 10, 0, -1 ):
                 
-                if not HydrusData.IsAlreadyRunning( self._db_dir, 'client' ):
+                if not HydrusData.IsAlreadyRunning( self.db_dir, 'client' ):
                     
                     break
                     
@@ -695,7 +715,7 @@ class Controller( HydrusController.HydrusController ):
             self._daemons.append( HydrusThreading.DAEMONBackgroundWorker( self, 'UPnP', ClientDaemons.DAEMONUPnP, ( 'notify_new_upnp_mappings', ), init_wait = 120, pre_call_wait = 6 ) )
             
         
-        if self._db.IsFirstStart():
+        if self.db.IsFirstStart():
             
             message = 'Hi, this looks like the first time you have started the hydrus client.'
             message += os.linesep * 2
@@ -706,12 +726,12 @@ class Controller( HydrusController.HydrusController ):
             HydrusData.ShowText( message )
             
         
-        if self._db.IsDBUpdated():
+        if self.db.IsDBUpdated():
             
             HydrusData.ShowText( 'The client has updated to version ' + str( HC.SOFTWARE_VERSION ) + '!' )
             
         
-        for message in self._db.GetInitialMessages():
+        for message in self.db.GetInitialMessages():
             
             HydrusData.ShowText( message )
             
@@ -966,6 +986,8 @@ class Controller( HydrusController.HydrusController ):
     
     def RestoreDatabase( self ):
         
+        restore_intro = ''
+        
         with wx.DirDialog( self._gui, 'Select backup location.' ) as dlg:
             
             if dlg.ShowModal() == wx.ID_OK:
@@ -986,12 +1008,12 @@ class Controller( HydrusController.HydrusController ):
                             
                             wx.CallAfter( self._gui.Exit )
                             
-                            while not self._db.LoopIsFinished():
+                            while not self.db.LoopIsFinished():
                                 
                                 time.sleep( 0.1 )
                                 
                             
-                            self._db.RestoreBackup( path )
+                            self.db.RestoreBackup( path )
                             
                             while not HG.shutdown_complete:
                                 
@@ -1200,9 +1222,9 @@ class Controller( HydrusController.HydrusController ):
             
             self.CheckAlreadyRunning()
             
-            self._last_shutdown_was_bad = HydrusData.LastShutdownWasBad( self._db_dir, 'client' )
+            self._last_shutdown_was_bad = HydrusData.LastShutdownWasBad( self.db_dir, 'client' )
             
-            HydrusData.RecordRunningStart( self._db_dir, 'client' )
+            HydrusData.RecordRunningStart( self.db_dir, 'client' )
             
             self.InitModel()
             
@@ -1251,7 +1273,7 @@ class Controller( HydrusController.HydrusController ):
             
             self.ShutdownModel()
             
-            HydrusData.CleanRunningFile( self._db_dir, 'client' )
+            HydrusData.CleanRunningFile( self.db_dir, 'client' )
             
         except HydrusExceptions.PermissionException: pass
         except HydrusExceptions.ShutdownException: pass
diff --git a/include/ClientDB.py b/include/ClientDB.py
index da1408b1..67bb99bc 100755
--- a/include/ClientDB.py
+++ b/include/ClientDB.py
@@ -500,8 +500,6 @@ class DB( HydrusDB.HydrusDB ):
         
         job_key.SetVariable( 'popup_text_1', 'closing db' )
         
-        self._Commit()
-        
         self._CloseDBCursor()
         
         try:
@@ -510,6 +508,11 @@ class DB( HydrusDB.HydrusDB ):
             
             for filename in self._db_filenames.values():
                 
+                if job_key.IsCancelled():
+                    
+                    continue
+                    
+                
                 job_key.SetVariable( 'popup_text_1', 'copying ' + filename )
                 
                 source = os.path.join( self._db_dir, filename )
@@ -518,19 +521,25 @@ class DB( HydrusDB.HydrusDB ):
                 HydrusPaths.MirrorFile( source, dest )
                 
             
-            job_key.SetVariable( 'popup_text_1', 'copying files directory' )
+            def is_cancelled_hook():
+                
+                return job_key.IsCancelled()
+                
+            
+            def text_update_hook( text ):
+                
+                job_key.SetVariable( 'popup_text_1', text )
+                
             
             if os.path.exists( client_files_default ):
                 
-                HydrusPaths.MirrorTree( client_files_default, os.path.join( path, 'client_files' ) )
+                HydrusPaths.MirrorTree( client_files_default, os.path.join( path, 'client_files' ), text_update_hook = text_update_hook, is_cancelled_hook = is_cancelled_hook )
                 
             
         finally:
             
             self._InitDBCursor()
             
-            self._BeginImmediate()
-            
         
         job_key.SetVariable( 'popup_text_1', 'done!' )
         
@@ -2634,17 +2643,6 @@ class DB( HydrusDB.HydrusDB ):
         
         HydrusPaths.MakeSureDirectoryExists( client_files_default )
         
-        HydrusDB.SetupDBCreatePragma( self._c, no_wal = self._no_wal )
-        
-        try:
-            
-            self._BeginImmediate()
-            
-        except Exception as e:
-            
-            raise HydrusExceptions.DBAccessException( HydrusData.ToUnicode( e ) )
-            
-        
         self._c.execute( 'CREATE TABLE services ( service_id INTEGER PRIMARY KEY AUTOINCREMENT, service_key BLOB_BYTES UNIQUE, service_type INTEGER, name TEXT, dictionary_string TEXT );' )
         
         # main
@@ -2713,8 +2711,8 @@ class DB( HydrusDB.HydrusDB ):
         
         self._c.execute( 'CREATE TABLE tag_sibling_petitions ( service_id INTEGER REFERENCES services ON DELETE CASCADE, bad_tag_id INTEGER, good_tag_id INTEGER, status INTEGER, reason_id INTEGER, PRIMARY KEY ( service_id, bad_tag_id, status ) );' )
         
-        self._c.execute( 'CREATE TABLE urls ( url TEXT PRIMARY KEY, hash_id INTEGER );' )
-        self._CreateIndex( 'urls', [ 'hash_id' ] )
+        self._c.execute( 'CREATE TABLE urls ( hash_id INTEGER, url TEXT, PRIMARY KEY ( hash_id, url ) );' )
+        self._CreateIndex( 'urls', [ 'url' ] )
         
         self._c.execute( 'CREATE TABLE vacuum_timestamps ( name TEXT, timestamp INTEGER );' )
         
@@ -2825,8 +2823,6 @@ class DB( HydrusDB.HydrusDB ):
         
         self._c.executemany( 'INSERT INTO json_dumps_named VALUES ( ?, ?, ?, ? );', ClientDefaults.GetDefaultScriptRows() )
         
-        self._Commit()
-        
     
     def _DeleteFiles( self, service_id, hash_ids ):
         
@@ -2922,7 +2918,7 @@ class DB( HydrusDB.HydrusDB ):
             
             self._c.executemany( 'UPDATE service_info SET info = info + ? WHERE service_id = ? AND info_type = ?;', service_info_updates )
             
-            self.pub_after_commit( 'notify_new_pending' )
+            self.pub_after_job( 'notify_new_pending' )
             
         
         return valid_hash_ids
@@ -2984,10 +2980,10 @@ class DB( HydrusDB.HydrusDB ):
             self._c.execute( 'DELETE FROM file_petitions WHERE service_id = ?;', ( service_id, ) )
             
         
-        self.pub_after_commit( 'notify_new_pending' )
-        self.pub_after_commit( 'notify_new_siblings_data' )
-        self.pub_after_commit( 'notify_new_siblings_gui' )
-        self.pub_after_commit( 'notify_new_parents' )
+        self.pub_after_job( 'notify_new_pending' )
+        self.pub_after_job( 'notify_new_siblings_data' )
+        self.pub_after_job( 'notify_new_siblings_gui' )
+        self.pub_after_job( 'notify_new_parents' )
         
         self.pub_service_updates_after_commit( { service_key : [ HydrusData.ServiceUpdate( HC.SERVICE_UPDATE_DELETE_PENDING ) ] } )
         
@@ -3098,7 +3094,7 @@ class DB( HydrusDB.HydrusDB ):
         
         self._c.execute( 'DELETE FROM service_info;' )
         
-        self.pub_after_commit( 'notify_new_pending' )
+        self.pub_after_job( 'notify_new_pending' )
         
     
     def _DeleteTagParents( self, service_id, pairs ):
@@ -4965,9 +4961,14 @@ class DB( HydrusDB.HydrusDB ):
             
             if count == len( hash_ids ): # i.e. every hash has this file service
                 
-                common_file_service_id = file_service_id
+                ( file_service_type, ) = self._c.execute( 'SELECT service_type FROM services WHERE service_id = ?;', ( file_service_id, ) ).fetchone()
                 
-                break
+                if file_service_type in HC.AUTOCOMPLETE_CACHE_SPECIFIC_FILE_SERVICES:
+                    
+                    common_file_service_id = file_service_id
+                    
+                    break
+                    
                 
             
         
@@ -6675,6 +6676,648 @@ class DB( HydrusDB.HydrusDB ):
             
         
     
+    def _ProcessContentUpdates( self, service_keys_to_content_updates, do_pubsubs = True ):
+        
+        notify_new_downloads = False
+        notify_new_pending = False
+        notify_new_parents = False
+        notify_new_siblings = False
+        
+        for ( service_key, content_updates ) in service_keys_to_content_updates.items():
+            
+            try:
+                
+                service_id = self._GetServiceId( service_key )
+                
+            except HydrusExceptions.DataMissing:
+                
+                continue
+                
+            
+            service = self._GetService( service_id )
+            
+            service_type = service.GetServiceType()
+            
+            ultimate_mappings_ids = []
+            ultimate_deleted_mappings_ids = []
+            
+            ultimate_pending_mappings_ids = []
+            ultimate_pending_rescinded_mappings_ids = []
+            
+            ultimate_petitioned_mappings_ids = []
+            ultimate_petitioned_rescinded_mappings_ids = []
+            
+            for content_update in content_updates:
+                
+                ( data_type, action, row ) = content_update.ToTuple()
+                
+                if service_type in HC.FILE_SERVICES:
+                    
+                    if data_type == HC.CONTENT_TYPE_FILES:
+                        
+                        if action == HC.CONTENT_UPDATE_ADVANCED:
+                            
+                            ( sub_action, sub_row ) = row
+                            
+                            if sub_action == 'delete_deleted':
+                                
+                                self._c.execute( 'DELETE FROM deleted_files WHERE service_id = ?;', ( service_id, ) )
+                                
+                            
+                            self._c.execute( 'DELETE FROM service_info WHERE service_id = ?;', ( service_id, ) )
+                            
+                        elif action == HC.CONTENT_UPDATE_ADD:
+                            
+                            if service_type in HC.LOCAL_FILE_SERVICES or service_type == HC.FILE_REPOSITORY:
+                                
+                                ( file_info_manager, timestamp ) = row
+                                
+                                ( hash, size, mime, width, height, duration, num_frames, num_words ) = file_info_manager.ToTuple()
+                                
+                                hash_id = self._GetHashId( hash )
+                                
+                                self._AddFilesInfo( [ ( hash_id, size, mime, width, height, duration, num_frames, num_words ) ] )
+                                
+                            elif service_type == HC.IPFS:
+                                
+                                ( hash, multihash ) = row
+                                
+                                hash_id = self._GetHashId( hash )
+                                
+                                self._SetServiceFilename( service_id, hash_id, multihash )
+                                
+                                timestamp = HydrusData.GetNow()
+                                
+                            
+                            self._AddFiles( service_id, [ ( hash_id, timestamp ) ] )
+                            
+                        elif action == HC.CONTENT_UPDATE_PEND:
+                            
+                            hashes = row
+                            
+                            hash_ids = self._GetHashIds( hashes )
+                            
+                            self._c.executemany( 'INSERT OR IGNORE INTO file_transfers ( service_id, hash_id ) VALUES ( ?, ? );', [ ( service_id, hash_id ) for hash_id in hash_ids ] )
+                            
+                            if service_key == CC.COMBINED_LOCAL_FILE_SERVICE_KEY: notify_new_downloads = True
+                            else: notify_new_pending = True
+                            
+                        elif action == HC.CONTENT_UPDATE_PETITION:
+                            
+                            ( hashes, reason ) = row
+                            
+                            hash_ids = self._GetHashIds( hashes )
+                            
+                            reason_id = self._GetTextId( reason )
+                            
+                            self._c.execute( 'DELETE FROM file_petitions WHERE service_id = ? AND hash_id IN ' + HydrusData.SplayListForDB( hash_ids ) + ';', ( service_id, ) )
+                            
+                            self._c.executemany( 'INSERT OR IGNORE INTO file_petitions ( service_id, hash_id, reason_id ) VALUES ( ?, ?, ? );', [ ( service_id, hash_id, reason_id ) for hash_id in hash_ids ] )
+                            
+                            notify_new_pending = True
+                            
+                        elif action == HC.CONTENT_UPDATE_RESCIND_PEND:
+                            
+                            hashes = row
+                            
+                            hash_ids = self._GetHashIds( hashes )
+                            
+                            self._c.execute( 'DELETE FROM file_transfers WHERE service_id = ? AND hash_id IN ' + HydrusData.SplayListForDB( hash_ids ) + ';', ( service_id, ) )
+                            
+                            notify_new_pending = True
+                            
+                        elif action == HC.CONTENT_UPDATE_RESCIND_PETITION:
+                            
+                            hashes = row
+                            
+                            hash_ids = self._GetHashIds( hashes )
+                            
+                            self._c.execute( 'DELETE FROM file_petitions WHERE service_id = ? AND hash_id IN ' + HydrusData.SplayListForDB( hash_ids ) + ';', ( service_id, ) )
+                            
+                            notify_new_pending = True
+                            
+                        else:
+                            
+                            hashes = row
+                            
+                            hash_ids = self._GetHashIds( hashes )
+                            
+                            if action == HC.CONTENT_UPDATE_ARCHIVE:
+                                
+                                self._ArchiveFiles( hash_ids )
+                                
+                            elif action == HC.CONTENT_UPDATE_INBOX:
+                                
+                                self._InboxFiles( hash_ids )
+                                
+                            elif action == HC.CONTENT_UPDATE_DELETE:
+                                
+                                deleted_hash_ids = self._DeleteFiles( service_id, hash_ids )
+                                
+                                if service_id == self._trash_service_id:
+                                    
+                                    self._DeleteFiles( self._combined_local_file_service_id, deleted_hash_ids )
+                                    
+                                
+                            elif action == HC.CONTENT_UPDATE_UNDELETE:
+                                
+                                splayed_hash_ids = HydrusData.SplayListForDB( hash_ids )
+                                
+                                rows = self._c.execute( 'SELECT hash_id, timestamp FROM current_files WHERE service_id = ? AND hash_id IN ' + splayed_hash_ids + ';', ( self._combined_local_file_service_id, ) ).fetchall()
+                                
+                                self._AddFiles( self._local_file_service_id, rows )
+                                
+                            
+                        
+                    elif data_type == HC.CONTENT_TYPE_DIRECTORIES:
+                        
+                        if action == HC.CONTENT_UPDATE_ADD:
+                            
+                            ( hashes, dirname, note ) = row
+                            
+                            hash_ids = self._GetHashIds( hashes )
+                            
+                            self._SetServiceDirectory( service_id, hash_ids, dirname, note )
+                            
+                        elif action == HC.CONTENT_UPDATE_DELETE:
+                            
+                            dirname = row
+                            
+                            self._DeleteServiceDirectory( service_id, dirname )
+                            
+                        
+                    elif data_type == HC.CONTENT_TYPE_URLS:
+                        
+                        if action == HC.CONTENT_UPDATE_ADD:
+                            
+                            ( hash, urls ) = row
+                            
+                            hash_id = self._GetHashId( hash )
+                            
+                            self._c.executemany( 'INSERT OR IGNORE INTO urls ( hash_id, url ) VALUES ( ?, ? );', ( ( hash_id, url ) for url in urls ) )
+                            
+                        elif action == HC.CONTENT_UPDATE_DELETE:
+                            
+                            ( hash, urls ) = row
+                            
+                            hash_id = self._GetHashId( hash )
+                            
+                            self._c.executemany( 'DELETE FROM urls WHERE hash_id = ? AND url = ?;', ( ( hash_id, url ) for url in urls ) )
+                            
+                        
+                    
+                elif service_type in HC.TAG_SERVICES:
+                    
+                    if data_type == HC.CONTENT_TYPE_MAPPINGS:
+                        
+                        if action == HC.CONTENT_UPDATE_ADVANCED:
+                            
+                            ( sub_action, sub_row ) = row
+                            
+                            if sub_action in ( 'copy', 'delete', 'delete_deleted', 'delete_for_deleted_files' ):
+                                
+                                self._c.execute( 'CREATE TEMPORARY TABLE temp_operation ( job_id INTEGER PRIMARY KEY AUTOINCREMENT, tag_id INTEGER, hash_id INTEGER );' )
+                                
+                                ( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = GenerateMappingsTableNames( service_id )
+                                
+                                predicates = []
+                                
+                                if sub_action == 'copy':
+                                    
+                                    ( tag, hashes, service_key_target ) = sub_row
+                                    
+                                    source_table_name = current_mappings_table_name
+                                    
+                                elif sub_action == 'delete':
+                                    
+                                    ( tag, hashes ) = sub_row
+                                    
+                                    source_table_name = current_mappings_table_name
+                                    
+                                elif sub_action == 'delete_deleted':
+                                    
+                                    ( tag, hashes ) = sub_row
+                                    
+                                    source_table_name = deleted_mappings_table_name
+                                    
+                                elif sub_action == 'delete_for_deleted_files':
+                                    
+                                    ( tag, hashes ) = sub_row
+                                    
+                                    source_table_name = current_mappings_table_name + ' NATURAL JOIN deleted_files'
+                                    
+                                    predicates.append( 'deleted_files.service_id = ' + str( self._combined_local_file_service_id ) )
+                                    
+                                
+                                do_namespace_join = False
+                                
+                                if tag is not None:
+                                    
+                                    ( tag_type, tag ) = tag
+                                    
+                                    if tag_type == 'tag':
+                                        
+                                        tag_id = self._GetTagId( tag )
+                                        
+                                        predicates.append( 'tag_id = ' + str( tag_id ) )
+                                        
+                                    elif tag_type == 'namespace':
+                                        
+                                        do_namespace_join = True
+                                        
+                                        namespace = tag
+                                        
+                                        namespace_id = self._GetNamespaceId( namespace )
+                                        
+                                        predicates.append( 'namespace_id = ' + str( namespace_id ) )
+                                        
+                                    elif tag_type == 'namespaced':
+                                        
+                                        do_namespace_join = True
+                                        
+                                        predicates.append( 'namespace_id != ' + str( self._null_namespace_id ) )
+                                        
+                                    elif tag_type == 'unnamespaced':
+                                        
+                                        do_namespace_join = True
+                                        
+                                        predicates.append( 'namespace_id = ' + str( self._null_namespace_id ) )
+                                        
+                                    
+                                
+                                if do_namespace_join:
+                                    
+                                    source_table_name = source_table_name + ' NATURAL JOIN tags NATURAL JOIN namespaces'
+                                    
+                                
+                                if hashes is not None:
+                                    
+                                    hash_ids = self._GetHashIds( hashes )
+                                    
+                                    predicates.append( 'hash_id IN ' + HydrusData.SplayListForDB( hash_ids ) )
+                                    
+                                
+                                if len( predicates ) == 0:
+                                    
+                                    self._c.execute( 'INSERT INTO temp_operation ( tag_id, hash_id ) SELECT tag_id, hash_id FROM ' + source_table_name + ';' )
+                                    
+                                else:
+                                    
+                                    self._c.execute( 'INSERT INTO temp_operation ( tag_id, hash_id ) SELECT tag_id, hash_id FROM ' + source_table_name + ' WHERE ' + ' AND '.join( predicates ) + ';' )
+                                    
+                                
+                                num_to_do = self._GetRowCount()
+                                
+                                i = 0
+                                
+                                block_size = 1000
+                                
+                                while i < num_to_do:
+                                    
+                                    advanced_mappings_ids_flat = self._c.execute( 'SELECT tag_id, hash_id FROM temp_operation WHERE job_id BETWEEN ? AND ?;', ( i, i + block_size - 1 ) )
+                                    
+                                    advanced_mappings_ids = HydrusData.BuildKeyToListDict( advanced_mappings_ids_flat ).items()
+                                    
+                                    if sub_action == 'copy':
+                                        
+                                        service_id_target = self._GetServiceId( service_key_target )
+                                        
+                                        service_target = self._GetService( service_id_target )
+                                        
+                                        if service_target.GetServiceType() == HC.LOCAL_TAG:
+                                            
+                                            kwarg = 'mappings_ids'
+                                            
+                                        else:
+                                            
+                                            kwarg = 'pending_mappings_ids'
+                                            
+                                        
+                                        kwargs = { kwarg : advanced_mappings_ids }
+                                        
+                                        self._UpdateMappings( service_id_target, **kwargs )
+                                        
+                                    elif sub_action in ( 'delete', 'delete_for_deleted_files' ):
+                                        
+                                        self._UpdateMappings( service_id, deleted_mappings_ids = advanced_mappings_ids )
+                                        
+                                    elif sub_action == 'delete_deleted':
+                                        
+                                        for ( tag_id, hash_ids ) in advanced_mappings_ids:
+                                            
+                                            self._c.execute( 'DELETE FROM ' + deleted_mappings_table_name + ' WHERE tag_id = ? AND hash_id IN ' + HydrusData.SplayListForDB( hash_ids ) + ';', ( tag_id, ) )
+                                            
+                                        
+                                        self._c.execute( 'DELETE FROM service_info WHERE service_id = ?;', ( service_id, ) )
+                                        
+                                    
+                                    i += block_size
+                                    
+                                
+                                self._c.execute( 'DROP TABLE temp_operation;' )
+                                
+                                self.pub_after_job( 'notify_new_pending' )
+                                
+                            
+                        else:
+                            
+                            if action == HC.CONTENT_UPDATE_PETITION:
+                                
+                                ( tag, hashes, reason ) = row
+                                
+                            else:
+                                
+                                ( tag, hashes ) = row
+                                
+                            
+                            try:
+                                
+                                tag_id = self._GetTagId( tag )
+                                
+                            except HydrusExceptions.SizeException:
+                                
+                                continue
+                                
+                            
+                            hash_ids = self._GetHashIds( hashes )
+                            
+                            if action == HC.CONTENT_UPDATE_ADD:
+                                
+                                ultimate_mappings_ids.append( ( tag_id, hash_ids ) )
+                                
+                            elif action == HC.CONTENT_UPDATE_DELETE:
+                                
+                                ultimate_deleted_mappings_ids.append( ( tag_id, hash_ids ) )
+                                
+                            elif action == HC.CONTENT_UPDATE_PEND:
+                                
+                                ultimate_pending_mappings_ids.append( ( tag_id, hash_ids ) )
+                                
+                            elif action == HC.CONTENT_UPDATE_RESCIND_PEND:
+                                
+                                ultimate_pending_rescinded_mappings_ids.append( ( tag_id, hash_ids ) )
+                                
+                            elif action == HC.CONTENT_UPDATE_PETITION:
+                                
+                                reason_id = self._GetTextId( reason )
+                                
+                                ultimate_petitioned_mappings_ids.append( ( tag_id, hash_ids, reason_id ) )
+                                
+                            elif action == HC.CONTENT_UPDATE_RESCIND_PETITION:
+                                
+                                ultimate_petitioned_rescinded_mappings_ids.append( ( tag_id, hash_ids ) )
+                                
+                            
+                        
+                    elif data_type == HC.CONTENT_TYPE_TAG_PARENTS:
+                        
+                        if action in ( HC.CONTENT_UPDATE_ADD, HC.CONTENT_UPDATE_DELETE ):
+                            
+                            ( child_tag, parent_tag ) = row
+                            
+                            try:
+                                
+                                child_tag_id = self._GetTagId( child_tag )
+                                
+                                parent_tag_id = self._GetTagId( parent_tag )
+                                
+                            except HydrusExceptions.SizeException:
+                                
+                                continue
+                                
+                            
+                            pairs = ( ( child_tag_id, parent_tag_id ), )
+                            
+                            if action == HC.CONTENT_UPDATE_ADD:
+                                
+                                special_content_updates = self._AddTagParents( service_id, pairs, make_content_updates = True )
+                                
+                                self.pub_content_updates_after_commit( { service_key : special_content_updates } )
+                                
+                            elif action == HC.CONTENT_UPDATE_DELETE:
+                                
+                                self._DeleteTagParents( service_id, pairs )
+                                
+                            
+                        elif action in ( HC.CONTENT_UPDATE_PEND, HC.CONTENT_UPDATE_PETITION ):
+                            
+                            if action == HC.CONTENT_UPDATE_PEND:
+                                
+                                new_status = HC.CONTENT_STATUS_PENDING
+                                
+                            elif action == HC.CONTENT_UPDATE_PETITION:
+                                
+                                new_status = HC.CONTENT_STATUS_PETITIONED
+                                
+                            
+                            ( ( child_tag, parent_tag ), reason ) = row
+                            
+                            try:
+                                
+                                child_tag_id = self._GetTagId( child_tag )
+                                
+                                parent_tag_id = self._GetTagId( parent_tag )
+                                
+                            except HydrusExceptions.SizeException:
+                                
+                                continue
+                                
+                            
+                            reason_id = self._GetTextId( reason )
+                            
+                            self._c.execute( 'DELETE FROM tag_parent_petitions WHERE service_id = ? AND child_tag_id = ? AND parent_tag_id = ?;', ( service_id, child_tag_id, parent_tag_id ) )
+                            
+                            self._c.execute( 'INSERT OR IGNORE INTO tag_parent_petitions ( service_id, child_tag_id, parent_tag_id, reason_id, status ) VALUES ( ?, ?, ?, ?, ? );', ( service_id, child_tag_id, parent_tag_id, reason_id, new_status ) )
+                            
+                            notify_new_pending = True
+                            
+                        elif action in ( HC.CONTENT_UPDATE_RESCIND_PEND, HC.CONTENT_UPDATE_RESCIND_PETITION ):
+                            
+                            if action == HC.CONTENT_UPDATE_RESCIND_PEND:
+                                
+                                deletee_status = HC.CONTENT_STATUS_PENDING
+                                
+                            elif action == HC.CONTENT_UPDATE_RESCIND_PETITION:
+                                
+                                deletee_status = HC.CONTENT_STATUS_PETITIONED
+                                
+                            
+                            ( child_tag, parent_tag ) = row
+                            
+                            try:
+                                
+                                child_tag_id = self._GetTagId( child_tag )
+                                
+                                parent_tag_id = self._GetTagId( parent_tag )
+                                
+                            except HydrusExceptions.SizeException:
+                                
+                                continue
+                                
+                            
+                            self._c.execute( 'DELETE FROM tag_parent_petitions WHERE service_id = ? AND child_tag_id = ? AND parent_tag_id = ? AND status = ?;', ( service_id, child_tag_id, parent_tag_id, deletee_status ) )
+                            
+                            notify_new_pending = True
+                            
+                        
+                        notify_new_parents = True
+                        
+                    elif data_type == HC.CONTENT_TYPE_TAG_SIBLINGS:
+                        
+                        if action in ( HC.CONTENT_UPDATE_ADD, HC.CONTENT_UPDATE_DELETE ):
+                            
+                            ( bad_tag, good_tag ) = row
+                            
+                            try:
+                                
+                                bad_tag_id = self._GetTagId( bad_tag )
+                                
+                                good_tag_id = self._GetTagId( good_tag )
+                                
+                            except HydrusExceptions.SizeException:
+                                
+                                continue
+                                
+                            
+                            pairs = ( ( bad_tag_id, good_tag_id ), )
+                            
+                            if action == HC.CONTENT_UPDATE_ADD:
+                                
+                                self._AddTagSiblings( service_id, pairs )
+                                
+                            elif action == HC.CONTENT_UPDATE_DELETE:
+                                
+                                self._DeleteTagSiblings( service_id, pairs )
+                                
+                            
+                        elif action in ( HC.CONTENT_UPDATE_PEND, HC.CONTENT_UPDATE_PETITION ):
+                            
+                            if action == HC.CONTENT_UPDATE_PEND:
+                                
+                                new_status = HC.CONTENT_STATUS_PENDING
+                                
+                            elif action == HC.CONTENT_UPDATE_PETITION:
+                                
+                                new_status = HC.CONTENT_STATUS_PETITIONED
+                                
+                            
+                            ( ( bad_tag, good_tag ), reason ) = row
+                            
+                            try:
+                                
+                                bad_tag_id = self._GetTagId( bad_tag )
+                                
+                                good_tag_id = self._GetTagId( good_tag )
+                                
+                            except HydrusExceptions.SizeException:
+                                
+                                continue
+                                
+                            
+                            reason_id = self._GetTextId( reason )
+                            
+                            self._c.execute( 'DELETE FROM tag_sibling_petitions WHERE service_id = ? AND bad_tag_id = ?;', ( service_id, bad_tag_id ) )
+                            
+                            self._c.execute( 'INSERT OR IGNORE INTO tag_sibling_petitions ( service_id, bad_tag_id, good_tag_id, reason_id, status ) VALUES ( ?, ?, ?, ?, ? );', ( service_id, bad_tag_id, good_tag_id, reason_id, new_status ) )
+                            
+                            notify_new_pending = True
+                            
+                        elif action in ( HC.CONTENT_UPDATE_RESCIND_PEND, HC.CONTENT_UPDATE_RESCIND_PETITION ):
+                            
+                            if action == HC.CONTENT_UPDATE_RESCIND_PEND:
+                                
+                                deletee_status = HC.CONTENT_STATUS_PENDING
+                                
+                            elif action == HC.CONTENT_UPDATE_RESCIND_PETITION:
+                                
+                                deletee_status = HC.CONTENT_STATUS_PETITIONED
+                                
+                            
+                            ( bad_tag, good_tag ) = row
+                            
+                            try:
+                                
+                                bad_tag_id = self._GetTagId( bad_tag )
+                                
+                            except HydrusExceptions.SizeException:
+                                
+                                continue
+                                
+                            
+                            self._c.execute( 'DELETE FROM tag_sibling_petitions WHERE service_id = ? AND bad_tag_id = ? AND status = ?;', ( service_id, bad_tag_id, deletee_status ) )
+                            
+                            notify_new_pending = True
+                            
+                        
+                        notify_new_siblings = True
+                        
+                    
+                elif service_type in HC.RATINGS_SERVICES:
+                    
+                    if action == HC.CONTENT_UPDATE_ADD:
+                        
+                        ( rating, hashes ) = row
+                        
+                        hash_ids = self._GetHashIds( hashes )
+                        
+                        splayed_hash_ids = HydrusData.SplayListForDB( hash_ids )
+                        
+                        if service_type in ( HC.LOCAL_RATING_LIKE, HC.LOCAL_RATING_NUMERICAL ):
+                            
+                            ratings_added = 0
+                            
+                            self._c.execute( 'DELETE FROM local_ratings WHERE service_id = ? AND hash_id IN ' + splayed_hash_ids + ';', ( service_id, ) )
+                            
+                            ratings_added -= self._GetRowCount()
+                            
+                            if rating is not None:
+                                
+                                self._c.executemany( 'INSERT INTO local_ratings ( service_id, hash_id, rating ) VALUES ( ?, ?, ? );', [ ( service_id, hash_id, rating ) for hash_id in hash_ids ] )
+                                
+                                ratings_added += self._GetRowCount()
+                                
+                            
+                            self._c.execute( 'UPDATE service_info SET info = info + ? WHERE service_id = ? AND info_type = ?;', ( ratings_added, service_id, HC.SERVICE_INFO_NUM_FILES ) )
+                            
+                            # and then do a thing here where it looks up remote services links and then pends/rescinds pends appropriately
+                            
+                        
+                    
+                
+            
+            if len( ultimate_mappings_ids ) + len( ultimate_deleted_mappings_ids ) + len( ultimate_pending_mappings_ids ) + len( ultimate_pending_rescinded_mappings_ids ) + len( ultimate_petitioned_mappings_ids ) + len( ultimate_petitioned_rescinded_mappings_ids ) > 0:
+                
+                self._UpdateMappings( service_id, mappings_ids = ultimate_mappings_ids, deleted_mappings_ids = ultimate_deleted_mappings_ids, pending_mappings_ids = ultimate_pending_mappings_ids, pending_rescinded_mappings_ids = ultimate_pending_rescinded_mappings_ids, petitioned_mappings_ids = ultimate_petitioned_mappings_ids, petitioned_rescinded_mappings_ids = ultimate_petitioned_rescinded_mappings_ids )
+                
+                notify_new_pending = True
+                
+            
+        
+        if do_pubsubs:
+            
+            if notify_new_downloads:
+                
+                self.pub_after_job( 'notify_new_downloads' )
+                
+            if notify_new_pending:
+                
+                self.pub_after_job( 'notify_new_pending' )
+                
+            if notify_new_siblings:
+                
+                self.pub_after_job( 'notify_new_siblings_data' )
+                self.pub_after_job( 'notify_new_siblings_gui' )
+                self.pub_after_job( 'notify_new_parents' )
+                
+            elif notify_new_parents:
+                
+                self.pub_after_job( 'notify_new_parents' )
+                
+            
+            self.pub_content_updates_after_commit( service_keys_to_content_updates )
+            
+        
+    
     def _ProcessRepositoryContentUpdate( self, job_key, service_id, content_update ):
         
         FILES_CHUNK_SIZE = 20
@@ -6982,642 +7625,6 @@ class DB( HydrusDB.HydrusDB ):
             
         
     
-    def _ProcessContentUpdates( self, service_keys_to_content_updates, do_pubsubs = True ):
-        
-        notify_new_downloads = False
-        notify_new_pending = False
-        notify_new_parents = False
-        notify_new_siblings = False
-        
-        for ( service_key, content_updates ) in service_keys_to_content_updates.items():
-            
-            try:
-                
-                service_id = self._GetServiceId( service_key )
-                
-            except HydrusExceptions.DataMissing:
-                
-                continue
-                
-            
-            service = self._GetService( service_id )
-            
-            service_type = service.GetServiceType()
-            
-            ultimate_mappings_ids = []
-            ultimate_deleted_mappings_ids = []
-            
-            ultimate_pending_mappings_ids = []
-            ultimate_pending_rescinded_mappings_ids = []
-            
-            ultimate_petitioned_mappings_ids = []
-            ultimate_petitioned_rescinded_mappings_ids = []
-            
-            for content_update in content_updates:
-                
-                ( data_type, action, row ) = content_update.ToTuple()
-                
-                if service_type in HC.FILE_SERVICES:
-                    
-                    if data_type == HC.CONTENT_TYPE_FILES:
-                        
-                        if action == HC.CONTENT_UPDATE_ADVANCED:
-                            
-                            ( sub_action, sub_row ) = row
-                            
-                            if sub_action == 'delete_deleted':
-                                
-                                self._c.execute( 'DELETE FROM deleted_files WHERE service_id = ?;', ( service_id, ) )
-                                
-                            
-                            self._c.execute( 'DELETE FROM service_info WHERE service_id = ?;', ( service_id, ) )
-                            
-                        elif action == HC.CONTENT_UPDATE_ADD:
-                            
-                            if service_type in HC.LOCAL_FILE_SERVICES or service_type == HC.FILE_REPOSITORY:
-                                
-                                ( file_info_manager, timestamp ) = row
-                                
-                                ( hash, size, mime, width, height, duration, num_frames, num_words ) = file_info_manager.ToTuple()
-                                
-                                hash_id = self._GetHashId( hash )
-                                
-                                self._AddFilesInfo( [ ( hash_id, size, mime, width, height, duration, num_frames, num_words ) ] )
-                                
-                            elif service_type == HC.IPFS:
-                                
-                                ( hash, multihash ) = row
-                                
-                                hash_id = self._GetHashId( hash )
-                                
-                                self._SetServiceFilename( service_id, hash_id, multihash )
-                                
-                                timestamp = HydrusData.GetNow()
-                                
-                            
-                            self._AddFiles( service_id, [ ( hash_id, timestamp ) ] )
-                            
-                        elif action == HC.CONTENT_UPDATE_PEND:
-                            
-                            hashes = row
-                            
-                            hash_ids = self._GetHashIds( hashes )
-                            
-                            self._c.executemany( 'INSERT OR IGNORE INTO file_transfers ( service_id, hash_id ) VALUES ( ?, ? );', [ ( service_id, hash_id ) for hash_id in hash_ids ] )
-                            
-                            if service_key == CC.COMBINED_LOCAL_FILE_SERVICE_KEY: notify_new_downloads = True
-                            else: notify_new_pending = True
-                            
-                        elif action == HC.CONTENT_UPDATE_PETITION:
-                            
-                            ( hashes, reason ) = row
-                            
-                            hash_ids = self._GetHashIds( hashes )
-                            
-                            reason_id = self._GetTextId( reason )
-                            
-                            self._c.execute( 'DELETE FROM file_petitions WHERE service_id = ? AND hash_id IN ' + HydrusData.SplayListForDB( hash_ids ) + ';', ( service_id, ) )
-                            
-                            self._c.executemany( 'INSERT OR IGNORE INTO file_petitions ( service_id, hash_id, reason_id ) VALUES ( ?, ?, ? );', [ ( service_id, hash_id, reason_id ) for hash_id in hash_ids ] )
-                            
-                            notify_new_pending = True
-                            
-                        elif action == HC.CONTENT_UPDATE_RESCIND_PEND:
-                            
-                            hashes = row
-                            
-                            hash_ids = self._GetHashIds( hashes )
-                            
-                            self._c.execute( 'DELETE FROM file_transfers WHERE service_id = ? AND hash_id IN ' + HydrusData.SplayListForDB( hash_ids ) + ';', ( service_id, ) )
-                            
-                            notify_new_pending = True
-                            
-                        elif action == HC.CONTENT_UPDATE_RESCIND_PETITION:
-                            
-                            hashes = row
-                            
-                            hash_ids = self._GetHashIds( hashes )
-                            
-                            self._c.execute( 'DELETE FROM file_petitions WHERE service_id = ? AND hash_id IN ' + HydrusData.SplayListForDB( hash_ids ) + ';', ( service_id, ) )
-                            
-                            notify_new_pending = True
-                            
-                        else:
-                            
-                            hashes = row
-                            
-                            hash_ids = self._GetHashIds( hashes )
-                            
-                            if action == HC.CONTENT_UPDATE_ARCHIVE: self._ArchiveFiles( hash_ids )
-                            elif action == HC.CONTENT_UPDATE_INBOX: self._InboxFiles( hash_ids )
-                            elif action == HC.CONTENT_UPDATE_DELETE:
-                                
-                                deleted_hash_ids = self._DeleteFiles( service_id, hash_ids )
-                                
-                                if service_id == self._trash_service_id:
-                                    
-                                    self._DeleteFiles( self._combined_local_file_service_id, deleted_hash_ids )
-                                    
-                                
-                            elif action == HC.CONTENT_UPDATE_UNDELETE:
-                                
-                                splayed_hash_ids = HydrusData.SplayListForDB( hash_ids )
-                                
-                                rows = self._c.execute( 'SELECT hash_id, timestamp FROM current_files WHERE service_id = ? AND hash_id IN ' + splayed_hash_ids + ';', ( self._combined_local_file_service_id, ) ).fetchall()
-                                
-                                self._AddFiles( self._local_file_service_id, rows )
-                                
-                            
-                        
-                    elif data_type == HC.CONTENT_TYPE_DIRECTORIES:
-                        
-                        if action == HC.CONTENT_UPDATE_ADD:
-                            
-                            ( hashes, dirname, note ) = row
-                            
-                            hash_ids = self._GetHashIds( hashes )
-                            
-                            self._SetServiceDirectory( service_id, hash_ids, dirname, note )
-                            
-                        elif action == HC.CONTENT_UPDATE_DELETE:
-                            
-                            dirname = row
-                            
-                            self._DeleteServiceDirectory( service_id, dirname )
-                            
-                        
-                    elif data_type == HC.CONTENT_TYPE_URLS:
-                        
-                        if action == HC.CONTENT_UPDATE_ADD:
-                            
-                            ( hash, urls ) = row
-                            
-                            hash_id = self._GetHashId( hash )
-                            
-                            self._c.executemany( 'INSERT OR IGNORE INTO urls ( url, hash_id ) VALUES ( ?, ? );', ( ( url, hash_id ) for url in urls ) )
-                            
-                        elif action == HC.CONTENT_UPDATE_DELETE:
-                            
-                            ( hash, urls ) = row
-                            
-                            hash_id = self._GetHashId( hash )
-                            
-                            self._c.executemany( 'DELETE FROM urls WHERE hash_id = ? AND url = ?;', ( ( hash_id, url ) for url in urls ) )
-                            
-                        
-                    
-                elif service_type in HC.TAG_SERVICES:
-                    
-                    if data_type == HC.CONTENT_TYPE_MAPPINGS:
-                        
-                        if action == HC.CONTENT_UPDATE_ADVANCED:
-                            
-                            ( sub_action, sub_row ) = row
-                            
-                            if sub_action in ( 'copy', 'delete', 'delete_deleted', 'delete_for_deleted_files' ):
-                                
-                                self._c.execute( 'CREATE TEMPORARY TABLE temp_operation ( job_id INTEGER PRIMARY KEY AUTOINCREMENT, tag_id INTEGER, hash_id INTEGER );' )
-                                
-                                ( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = GenerateMappingsTableNames( service_id )
-                                
-                                predicates = []
-                                
-                                if sub_action == 'copy':
-                                    
-                                    ( tag, hashes, service_key_target ) = sub_row
-                                    
-                                    source_table_name = current_mappings_table_name
-                                    
-                                elif sub_action == 'delete':
-                                    
-                                    ( tag, hashes ) = sub_row
-                                    
-                                    source_table_name = current_mappings_table_name
-                                    
-                                elif sub_action == 'delete_deleted':
-                                    
-                                    ( tag, hashes ) = sub_row
-                                    
-                                    source_table_name = deleted_mappings_table_name
-                                    
-                                elif sub_action == 'delete_for_deleted_files':
-                                    
-                                    ( tag, hashes ) = sub_row
-                                    
-                                    source_table_name = current_mappings_table_name + ' NATURAL JOIN deleted_files'
-                                    
-                                    predicates.append( 'deleted_files.service_id = ' + str( self._combined_local_file_service_id ) )
-                                    
-                                
-                                do_namespace_join = False
-                                
-                                if tag is not None:
-                                    
-                                    ( tag_type, tag ) = tag
-                                    
-                                    if tag_type == 'tag':
-                                        
-                                        tag_id = self._GetTagId( tag )
-                                        
-                                        predicates.append( 'tag_id = ' + str( tag_id ) )
-                                        
-                                    elif tag_type == 'namespace':
-                                        
-                                        do_namespace_join = True
-                                        
-                                        namespace = tag
-                                        
-                                        namespace_id = self._GetNamespaceId( namespace )
-                                        
-                                        predicates.append( 'namespace_id = ' + str( namespace_id ) )
-                                        
-                                    elif tag_type == 'namespaced':
-                                        
-                                        do_namespace_join = True
-                                        
-                                        predicates.append( 'namespace_id != ' + str( self._null_namespace_id ) )
-                                        
-                                    elif tag_type == 'unnamespaced':
-                                        
-                                        do_namespace_join = True
-                                        
-                                        predicates.append( 'namespace_id = ' + str( self._null_namespace_id ) )
-                                        
-                                    
-                                
-                                if do_namespace_join:
-                                    
-                                    source_table_name = source_table_name + ' NATURAL JOIN tags NATURAL JOIN namespaces'
-                                    
-                                
-                                if hashes is not None:
-                                    
-                                    hash_ids = self._GetHashIds( hashes )
-                                    
-                                    predicates.append( 'hash_id IN ' + HydrusData.SplayListForDB( hash_ids ) )
-                                    
-                                
-                                if len( predicates ) == 0:
-                                    
-                                    self._c.execute( 'INSERT INTO temp_operation ( tag_id, hash_id ) SELECT tag_id, hash_id FROM ' + source_table_name + ';' )
-                                    
-                                else:
-                                    
-                                    self._c.execute( 'INSERT INTO temp_operation ( tag_id, hash_id ) SELECT tag_id, hash_id FROM ' + source_table_name + ' WHERE ' + ' AND '.join( predicates ) + ';' )
-                                    
-                                
-                                num_to_do = self._GetRowCount()
-                                
-                                i = 0
-                                
-                                block_size = 1000
-                                
-                                while i < num_to_do:
-                                    
-                                    advanced_mappings_ids_flat = self._c.execute( 'SELECT tag_id, hash_id FROM temp_operation WHERE job_id BETWEEN ? AND ?;', ( i, i + block_size - 1 ) )
-                                    
-                                    advanced_mappings_ids = HydrusData.BuildKeyToListDict( advanced_mappings_ids_flat ).items()
-                                    
-                                    if sub_action == 'copy':
-                                        
-                                        service_id_target = self._GetServiceId( service_key_target )
-                                        
-                                        service_target = self._GetService( service_id_target )
-                                        
-                                        if service_target.GetServiceType() == HC.LOCAL_TAG:
-                                            
-                                            kwarg = 'mappings_ids'
-                                            
-                                        else:
-                                            
-                                            kwarg = 'pending_mappings_ids'
-                                            
-                                        
-                                        kwargs = { kwarg : advanced_mappings_ids }
-                                        
-                                        self._UpdateMappings( service_id_target, **kwargs )
-                                        
-                                    elif sub_action in ( 'delete', 'delete_for_deleted_files' ):
-                                        
-                                        self._UpdateMappings( service_id, deleted_mappings_ids = advanced_mappings_ids )
-                                        
-                                    elif sub_action == 'delete_deleted':
-                                        
-                                        for ( tag_id, hash_ids ) in advanced_mappings_ids:
-                                            
-                                            self._c.execute( 'DELETE FROM ' + deleted_mappings_table_name + ' WHERE tag_id = ? AND hash_id IN ' + HydrusData.SplayListForDB( hash_ids ) + ';', ( tag_id, ) )
-                                            
-                                        
-                                        self._c.execute( 'DELETE FROM service_info WHERE service_id = ?;', ( service_id, ) )
-                                        
-                                    
-                                    i += block_size
-                                    
-                                
-                                self._c.execute( 'DROP TABLE temp_operation;' )
-                                
-                                self.pub_after_commit( 'notify_new_pending' )
-                                
-                            
-                        else:
-                            
-                            if action == HC.CONTENT_UPDATE_PETITION:
-                                
-                                ( tag, hashes, reason ) = row
-                                
-                            else:
-                                
-                                ( tag, hashes ) = row
-                                
-                            
-                            try:
-                                
-                                tag_id = self._GetTagId( tag )
-                                
-                            except HydrusExceptions.SizeException:
-                                
-                                continue
-                                
-                            
-                            hash_ids = self._GetHashIds( hashes )
-                            
-                            if action == HC.CONTENT_UPDATE_ADD:
-                                
-                                ultimate_mappings_ids.append( ( tag_id, hash_ids ) )
-                                
-                            elif action == HC.CONTENT_UPDATE_DELETE:
-                                
-                                ultimate_deleted_mappings_ids.append( ( tag_id, hash_ids ) )
-                                
-                            elif action == HC.CONTENT_UPDATE_PEND:
-                                
-                                ultimate_pending_mappings_ids.append( ( tag_id, hash_ids ) )
-                                
-                            elif action == HC.CONTENT_UPDATE_RESCIND_PEND:
-                                
-                                ultimate_pending_rescinded_mappings_ids.append( ( tag_id, hash_ids ) )
-                                
-                            elif action == HC.CONTENT_UPDATE_PETITION:
-                                
-                                reason_id = self._GetTextId( reason )
-                                
-                                ultimate_petitioned_mappings_ids.append( ( tag_id, hash_ids, reason_id ) )
-                                
-                            elif action == HC.CONTENT_UPDATE_RESCIND_PETITION:
-                                
-                                ultimate_petitioned_rescinded_mappings_ids.append( ( tag_id, hash_ids ) )
-                                
-                            
-                        
-                    elif data_type == HC.CONTENT_TYPE_TAG_PARENTS:
-                        
-                        if action in ( HC.CONTENT_UPDATE_ADD, HC.CONTENT_UPDATE_DELETE ):
-                            
-                            ( child_tag, parent_tag ) = row
-                            
-                            try:
-                                
-                                child_tag_id = self._GetTagId( child_tag )
-                                
-                                parent_tag_id = self._GetTagId( parent_tag )
-                                
-                            except HydrusExceptions.SizeException:
-                                
-                                continue
-                                
-                            
-                            pairs = ( ( child_tag_id, parent_tag_id ), )
-                            
-                            if action == HC.CONTENT_UPDATE_ADD:
-                                
-                                special_content_updates = self._AddTagParents( service_id, pairs, make_content_updates = True )
-                                
-                                self.pub_content_updates_after_commit( { service_key : special_content_updates } )
-                                
-                            elif action == HC.CONTENT_UPDATE_DELETE:
-                                
-                                self._DeleteTagParents( service_id, pairs )
-                                
-                            
-                        elif action in ( HC.CONTENT_UPDATE_PEND, HC.CONTENT_UPDATE_PETITION ):
-                            
-                            if action == HC.CONTENT_UPDATE_PEND:
-                                
-                                new_status = HC.CONTENT_STATUS_PENDING
-                                
-                            elif action == HC.CONTENT_UPDATE_PETITION:
-                                
-                                new_status = HC.CONTENT_STATUS_PETITIONED
-                                
-                            
-                            ( ( child_tag, parent_tag ), reason ) = row
-                            
-                            try:
-                                
-                                child_tag_id = self._GetTagId( child_tag )
-                                
-                                parent_tag_id = self._GetTagId( parent_tag )
-                                
-                            except HydrusExceptions.SizeException:
-                                
-                                continue
-                                
-                            
-                            reason_id = self._GetTextId( reason )
-                            
-                            self._c.execute( 'DELETE FROM tag_parent_petitions WHERE service_id = ? AND child_tag_id = ? AND parent_tag_id = ?;', ( service_id, child_tag_id, parent_tag_id ) )
-                            
-                            self._c.execute( 'INSERT OR IGNORE INTO tag_parent_petitions ( service_id, child_tag_id, parent_tag_id, reason_id, status ) VALUES ( ?, ?, ?, ?, ? );', ( service_id, child_tag_id, parent_tag_id, reason_id, new_status ) )
-                            
-                            notify_new_pending = True
-                            
-                        elif action in ( HC.CONTENT_UPDATE_RESCIND_PEND, HC.CONTENT_UPDATE_RESCIND_PETITION ):
-                            
-                            if action == HC.CONTENT_UPDATE_RESCIND_PEND:
-                                
-                                deletee_status = HC.CONTENT_STATUS_PENDING
-                                
-                            elif action == HC.CONTENT_UPDATE_RESCIND_PETITION:
-                                
-                                deletee_status = HC.CONTENT_STATUS_PETITIONED
-                                
-                            
-                            ( child_tag, parent_tag ) = row
-                            
-                            try:
-                                
-                                child_tag_id = self._GetTagId( child_tag )
-                                
-                                parent_tag_id = self._GetTagId( parent_tag )
-                                
-                            except HydrusExceptions.SizeException:
-                                
-                                continue
-                                
-                            
-                            self._c.execute( 'DELETE FROM tag_parent_petitions WHERE service_id = ? AND child_tag_id = ? AND parent_tag_id = ? AND status = ?;', ( service_id, child_tag_id, parent_tag_id, deletee_status ) )
-                            
-                            notify_new_pending = True
-                            
-                        
-                        notify_new_parents = True
-                        
-                    elif data_type == HC.CONTENT_TYPE_TAG_SIBLINGS:
-                        
-                        if action in ( HC.CONTENT_UPDATE_ADD, HC.CONTENT_UPDATE_DELETE ):
-                            
-                            ( bad_tag, good_tag ) = row
-                            
-                            try:
-                                
-                                bad_tag_id = self._GetTagId( bad_tag )
-                                
-                                good_tag_id = self._GetTagId( good_tag )
-                                
-                            except HydrusExceptions.SizeException:
-                                
-                                continue
-                                
-                            
-                            pairs = ( ( bad_tag_id, good_tag_id ), )
-                            
-                            if action == HC.CONTENT_UPDATE_ADD:
-                                
-                                self._AddTagSiblings( service_id, pairs )
-                                
-                            elif action == HC.CONTENT_UPDATE_DELETE:
-                                
-                                self._DeleteTagSiblings( service_id, pairs )
-                                
-                            
-                        elif action in ( HC.CONTENT_UPDATE_PEND, HC.CONTENT_UPDATE_PETITION ):
-                            
-                            if action == HC.CONTENT_UPDATE_PEND:
-                                
-                                new_status = HC.CONTENT_STATUS_PENDING
-                                
-                            elif action == HC.CONTENT_UPDATE_PETITION:
-                                
-                                new_status = HC.CONTENT_STATUS_PETITIONED
-                                
-                            
-                            ( ( bad_tag, good_tag ), reason ) = row
-                            
-                            try:
-                                
-                                bad_tag_id = self._GetTagId( bad_tag )
-                                
-                                good_tag_id = self._GetTagId( good_tag )
-                                
-                            except HydrusExceptions.SizeException:
-                                
-                                continue
-                                
-                            
-                            reason_id = self._GetTextId( reason )
-                            
-                            self._c.execute( 'DELETE FROM tag_sibling_petitions WHERE service_id = ? AND bad_tag_id = ?;', ( service_id, bad_tag_id ) )
-                            
-                            self._c.execute( 'INSERT OR IGNORE INTO tag_sibling_petitions ( service_id, bad_tag_id, good_tag_id, reason_id, status ) VALUES ( ?, ?, ?, ?, ? );', ( service_id, bad_tag_id, good_tag_id, reason_id, new_status ) )
-                            
-                            notify_new_pending = True
-                            
-                        elif action in ( HC.CONTENT_UPDATE_RESCIND_PEND, HC.CONTENT_UPDATE_RESCIND_PETITION ):
-                            
-                            if action == HC.CONTENT_UPDATE_RESCIND_PEND:
-                                
-                                deletee_status = HC.CONTENT_STATUS_PENDING
-                                
-                            elif action == HC.CONTENT_UPDATE_RESCIND_PETITION:
-                                
-                                deletee_status = HC.CONTENT_STATUS_PETITIONED
-                                
-                            
-                            ( bad_tag, good_tag ) = row
-                            
-                            try:
-                                
-                                bad_tag_id = self._GetTagId( bad_tag )
-                                
-                            except HydrusExceptions.SizeException:
-                                
-                                continue
-                                
-                            
-                            self._c.execute( 'DELETE FROM tag_sibling_petitions WHERE service_id = ? AND bad_tag_id = ? AND status = ?;', ( service_id, bad_tag_id, deletee_status ) )
-                            
-                            notify_new_pending = True
-                            
-                        
-                        notify_new_siblings = True
-                        
-                    
-                elif service_type in HC.RATINGS_SERVICES:
-                    
-                    if action == HC.CONTENT_UPDATE_ADD:
-                        
-                        ( rating, hashes ) = row
-                        
-                        hash_ids = self._GetHashIds( hashes )
-                        
-                        splayed_hash_ids = HydrusData.SplayListForDB( hash_ids )
-                        
-                        if service_type in ( HC.LOCAL_RATING_LIKE, HC.LOCAL_RATING_NUMERICAL ):
-                            
-                            ratings_added = 0
-                            
-                            self._c.execute( 'DELETE FROM local_ratings WHERE service_id = ? AND hash_id IN ' + splayed_hash_ids + ';', ( service_id, ) )
-                            
-                            ratings_added -= self._GetRowCount()
-                            
-                            if rating is not None:
-                                
-                                self._c.executemany( 'INSERT INTO local_ratings ( service_id, hash_id, rating ) VALUES ( ?, ?, ? );', [ ( service_id, hash_id, rating ) for hash_id in hash_ids ] )
-                                
-                                ratings_added += self._GetRowCount()
-                                
-                            
-                            self._c.execute( 'UPDATE service_info SET info = info + ? WHERE service_id = ? AND info_type = ?;', ( ratings_added, service_id, HC.SERVICE_INFO_NUM_FILES ) )
-                            
-                            # and then do a thing here where it looks up remote services links and then pends/rescinds pends appropriately
-                            
-                        
-                    
-                
-            
-            if len( ultimate_mappings_ids ) + len( ultimate_deleted_mappings_ids ) + len( ultimate_pending_mappings_ids ) + len( ultimate_pending_rescinded_mappings_ids ) + len( ultimate_petitioned_mappings_ids ) + len( ultimate_petitioned_rescinded_mappings_ids ) > 0:
-                
-                self._UpdateMappings( service_id, mappings_ids = ultimate_mappings_ids, deleted_mappings_ids = ultimate_deleted_mappings_ids, pending_mappings_ids = ultimate_pending_mappings_ids, pending_rescinded_mappings_ids = ultimate_pending_rescinded_mappings_ids, petitioned_mappings_ids = ultimate_petitioned_mappings_ids, petitioned_rescinded_mappings_ids = ultimate_petitioned_rescinded_mappings_ids )
-                
-                notify_new_pending = True
-                
-            
-        
-        if do_pubsubs:
-            
-            if notify_new_downloads:
-                
-                self.pub_after_commit( 'notify_new_downloads' )
-                
-            if notify_new_pending:
-                
-                self.pub_after_commit( 'notify_new_pending' )
-                
-            if notify_new_siblings:
-                
-                self.pub_after_commit( 'notify_new_siblings_data' )
-                self.pub_after_commit( 'notify_new_siblings_gui' )
-                self.pub_after_commit( 'notify_new_parents' )
-                
-            elif notify_new_parents:
-                
-                self.pub_after_commit( 'notify_new_parents' )
-                
-            
-            self.pub_content_updates_after_commit( service_keys_to_content_updates )
-            
-        
-    
     def _ProcessRepositoryUpdates( self, service_key, only_when_idle = False, stop_time = None ):
         
         service_id = self._GetServiceId( service_key )
@@ -7626,6 +7633,17 @@ class DB( HydrusDB.HydrusDB ):
         
         repository_updates_table_name = GenerateRepositoryRepositoryUpdatesTableName( service_id )
         
+        result = self._c.execute( 'SELECT 1 FROM ' + repository_updates_table_name + ' WHERE processed = ?;', ( True, ) ).fetchone()
+        
+        if result is None:
+            
+            this_is_first_sync = True
+            
+        else:
+            
+            this_is_first_sync = False
+            
+        
         update_indices_to_unprocessed_hash_ids = HydrusData.BuildKeyToSetDict( self._c.execute( 'SELECT update_index, hash_id FROM ' + repository_updates_table_name + ' WHERE processed = ?;', ( False, ) ) )
         
         hash_ids_i_can_process = set()
@@ -7675,12 +7693,8 @@ class DB( HydrusDB.HydrusDB ):
                 
                 stop_time = HydrusData.GetNow() + min( 5 + ( num_updates_to_do * 2 ), 30 )
                 
-                self._Commit()
-                
                 self._LoadIntoDiskCache( stop_time = stop_time )
                 
-                self._BeginImmediate()
-                
                 num_updates_done = 0
                 
                 client_files_manager = self._controller.client_files_manager
@@ -7737,6 +7751,17 @@ class DB( HydrusDB.HydrusDB ):
                             total_definitions_rows += num_rows
                             
                         
+                        # let's atomically save our progress here to avoid the desync issue some people had.
+                        # the desync issue was that after a power-cut during big sync commits, the master db would sometimes not commit but the mappings _would_
+                        # hence there would be missing definitions
+                        # so, let's lock-in definitions here, while we can, before we add anything that could rely on them
+                        
+                        # this is an issue with WAL journalling, which isn't currently global-atomic with attached dbs, wew lad
+                        
+                        self._Commit()
+                        
+                        self._BeginImmediate()
+                        
                     finally:
                         
                         report_speed_to_log( larger_precise_timestamp, total_definitions_rows, 'definitions' )
@@ -7804,8 +7829,15 @@ class DB( HydrusDB.HydrusDB ):
                         
                     
                 
+                self._AnalyzeStaleBigTables()
+                
             finally:
                 
+                if this_is_first_sync:
+                    
+                    self._AnalyzeStaleBigTables()
+                    
+                
                 job_key.SetVariable( 'popup_text_1', 'finished' )
                 job_key.DeleteVariable( 'popup_gauge_1' )
                 job_key.DeleteVariable( 'popup_text_2' )
@@ -8007,11 +8039,6 @@ class DB( HydrusDB.HydrusDB ):
         
         self._Commit()
         
-        if not self._fast_big_transaction_wal:
-            
-            self._c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
-            
-        
         self._c.execute( 'PRAGMA foreign_keys = ON;' )
         
         self._BeginImmediate()
@@ -8034,19 +8061,17 @@ class DB( HydrusDB.HydrusDB ):
         
         self._AddService( service_key, service_type, name, dictionary )
         
-        self.pub_after_commit( 'notify_unknown_accounts' )
-        self.pub_after_commit( 'notify_new_pending' )
-        self.pub_after_commit( 'notify_new_services_data' )
-        self.pub_after_commit( 'notify_new_services_gui' )
+        self.pub_after_job( 'notify_unknown_accounts' )
+        self.pub_after_job( 'notify_new_pending' )
+        self.pub_after_job( 'notify_new_services_data' )
+        self.pub_after_job( 'notify_new_services_gui' )
         
         job_key.SetVariable( 'popup_text_1', prefix + ': done!' )
         
-        self._Commit()
+        self._CloseDBCursor()
         
         self._InitDBCursor()
         
-        self._BeginImmediate()
-        
         job_key.Finish()
         
     
@@ -8085,10 +8110,10 @@ class DB( HydrusDB.HydrusDB ):
         
         if resize_thumbs:
             
-            self.pub_after_commit( 'thumbnail_resize' )
+            self.pub_after_job( 'thumbnail_resize' )
             
         
-        self.pub_after_commit( 'notify_new_options' )
+        self.pub_after_job( 'notify_new_options' )
         
     
     def _SetJSONDump( self, obj ):
@@ -8210,7 +8235,7 @@ class DB( HydrusDB.HydrusDB ):
             self._c.execute( 'INSERT OR IGNORE INTO tag_censorship ( service_id, blacklist, tags ) VALUES ( ?, ?, ? );', ( service_id, blacklist, tags ) )
             
         
-        self.pub_after_commit( 'notify_new_tag_censorship' )
+        self.pub_after_job( 'notify_new_tag_censorship' )
         
     
     def _SetYAMLDump( self, dump_type, dump_name, data ):
@@ -8683,8 +8708,6 @@ class DB( HydrusDB.HydrusDB ):
             
             self._controller.pub( 'splash_set_status_text', 'committing to disk' )
             
-            self._Commit()
-            
             self._CloseDBCursor()
             
             try:
@@ -8713,8 +8736,6 @@ class DB( HydrusDB.HydrusDB ):
                 
                 self._InitDBCursor()
                 
-                self._BeginImmediate()
-                
             
             for schema in [ 'main', 'external_caches', 'external_master', 'external_mappings' ]:
                 
@@ -9548,8 +9569,6 @@ class DB( HydrusDB.HydrusDB ):
             
             self._c.execute( 'ANALYZE external_master.local_hashes;' )
             
-            self._Commit()
-            
             self._CloseDBCursor()
             
             self._controller.pub( 'splash_set_status_text', 'vacuuming main db ' )
@@ -9571,8 +9590,6 @@ class DB( HydrusDB.HydrusDB ):
             
             self._InitDBCursor()
             
-            self._BeginImmediate()
-            
             #
             
             bandwidth_manager = ClientDefaults.GetDefaultBandwidthManager()
@@ -9630,6 +9647,22 @@ class DB( HydrusDB.HydrusDB ):
                 
             
         
+        if version == 263:
+            
+            self._controller.pub( 'splash_set_status_text', 'rebuilding urls table' )
+            
+            self._c.execute( 'ALTER TABLE urls RENAME TO urls_old;' )
+            
+            self._c.execute( 'CREATE TABLE urls ( hash_id INTEGER, url TEXT, PRIMARY KEY ( hash_id, url ) );' )
+            self._CreateIndex( 'urls', [ 'url' ] )
+            
+            self._c.execute( 'INSERT INTO urls SELECT hash_id, url FROM urls_old;' )
+            
+            self._c.execute( 'DROP TABLE urls_old;' )
+            
+            self._c.execute( 'ANALYZE urls;' )
+            
+        
         self._controller.pub( 'splash_set_title_text', 'updated db to v' + str( version + 1 ) )
         
         self._c.execute( 'UPDATE version SET version = ?;', ( version + 1, ) )
@@ -9947,10 +9980,10 @@ class DB( HydrusDB.HydrusDB ):
             self._DeleteService( service_id )
             
         
-        self.pub_after_commit( 'notify_unknown_accounts' )
-        self.pub_after_commit( 'notify_new_services_data' )
-        self.pub_after_commit( 'notify_new_services_gui' )
-        self.pub_after_commit( 'notify_new_pending' )
+        self.pub_after_job( 'notify_unknown_accounts' )
+        self.pub_after_job( 'notify_new_services_data' )
+        self.pub_after_job( 'notify_new_services_gui' )
+        self.pub_after_job( 'notify_new_pending' )
         
     
     def _UpdateService( self, service ):
@@ -10001,8 +10034,8 @@ class DB( HydrusDB.HydrusDB ):
             
             if old_port != new_port:
                 
-                self.pub_after_commit( 'restart_booru' )
-                self.pub_after_commit( 'notify_new_upnp_mappings' )
+                self.pub_after_job( 'restart_booru' )
+                self.pub_after_job( 'notify_new_upnp_mappings' )
                 
             else:
                 
@@ -10011,7 +10044,7 @@ class DB( HydrusDB.HydrusDB ):
                 
                 if old_upnp != new_upnp:
                     
-                    self.pub_after_commit( 'notify_new_upnp_mappings' )
+                    self.pub_after_job( 'notify_new_upnp_mappings' )
                     
                 
             
@@ -10031,11 +10064,6 @@ class DB( HydrusDB.HydrusDB ):
         
         self._Commit()
         
-        if not self._fast_big_transaction_wal:
-            
-            self._c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
-            
-        
         self._c.execute( 'PRAGMA foreign_keys = ON;' )
         
         self._BeginImmediate()
@@ -10070,17 +10098,15 @@ class DB( HydrusDB.HydrusDB ):
                 
             
         
-        self.pub_after_commit( 'notify_unknown_accounts' )
-        self.pub_after_commit( 'notify_new_services_data' )
-        self.pub_after_commit( 'notify_new_services_gui' )
-        self.pub_after_commit( 'notify_new_pending' )
+        self.pub_after_job( 'notify_unknown_accounts' )
+        self.pub_after_job( 'notify_new_services_data' )
+        self.pub_after_job( 'notify_new_services_gui' )
+        self.pub_after_job( 'notify_new_pending' )
         
-        self._Commit()
+        self._CloseDBCursor()
         
         self._InitDBCursor()
         
-        self._BeginImmediate()
-        
     
     def _Vacuum( self, stop_time = None, force_vacuum = False ):
         
@@ -10110,8 +10136,6 @@ class DB( HydrusDB.HydrusDB ):
         
         if len( due_names ) > 0:
             
-            self._Commit()
-            
             job_key_pubbed = False
             
             job_key = ClientThreading.JobKey()
@@ -10171,8 +10195,6 @@ class DB( HydrusDB.HydrusDB ):
                         
                         self._InitDBCursor()
                         
-                        self._BeginImmediate()
-                        
                         new_options.SetNoneableInteger( 'maintenance_vacuum_period_days', None )
                         
                         self._SaveOptions( HC.options )
@@ -10187,8 +10209,6 @@ class DB( HydrusDB.HydrusDB ):
                 
                 self._InitDBCursor()
                 
-                self._BeginImmediate()
-                
                 self._c.executemany( 'DELETE FROM vacuum_timestamps WHERE name = ?;', ( ( name, ) for name in names_done ) )
                 
                 self._c.executemany( 'INSERT OR IGNORE INTO vacuum_timestamps ( name, timestamp ) VALUES ( ?, ? );', ( ( name, HydrusData.GetNow() ) for name in names_done ) )
@@ -10254,8 +10274,8 @@ class DB( HydrusDB.HydrusDB ):
     
     def pub_content_updates_after_commit( self, service_keys_to_content_updates ):
         
-        self.pub_after_commit( 'content_updates_data', service_keys_to_content_updates )
-        self.pub_after_commit( 'content_updates_gui', service_keys_to_content_updates )
+        self.pub_after_job( 'content_updates_data', service_keys_to_content_updates )
+        self.pub_after_job( 'content_updates_gui', service_keys_to_content_updates )
         
     
     def pub_initial_message( self, message ):
@@ -10265,8 +10285,8 @@ class DB( HydrusDB.HydrusDB ):
     
     def pub_service_updates_after_commit( self, service_keys_to_service_updates ):
         
-        self.pub_after_commit( 'service_updates_data', service_keys_to_service_updates )
-        self.pub_after_commit( 'service_updates_gui', service_keys_to_service_updates )
+        self.pub_after_job( 'service_updates_data', service_keys_to_service_updates )
+        self.pub_after_job( 'service_updates_gui', service_keys_to_service_updates )
         
     
     def publish_status_update( self ):
diff --git a/include/ClientData.py b/include/ClientData.py
index b7e46e7a..93386172 100644
--- a/include/ClientData.py
+++ b/include/ClientData.py
@@ -896,6 +896,7 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
         
         self._dictionary[ 'noneable_strings' ][ 'favourite_file_lookup_script' ] = 'gelbooru md5'
         self._dictionary[ 'noneable_strings' ][ 'suggested_tags_layout' ] = 'notebook'
+        self._dictionary[ 'noneable_strings' ][ 'backup_path' ] = None
         
         self._dictionary[ 'strings' ] = {}
         
diff --git a/include/ClientDownloading.py b/include/ClientDownloading.py
index 220eee67..f00d8b07 100644
--- a/include/ClientDownloading.py
+++ b/include/ClientDownloading.py
@@ -1175,9 +1175,14 @@ class GalleryDeviantArt( Gallery ):
                 
                 # used to fetch this from a tumblr share url, now we grab from some hidden gubbins behind an age gate
                 
-                a_hidden = soup.find( 'a', class_ = 'ismature' )
+                a_ismatures = soup.find_all( 'a', class_ = 'ismature' )
                 
-                imgs = a_hidden.find_all( 'img' )
+                imgs = []
+                
+                for a_ismature in a_ismatures:
+                    
+                    imgs.extend( a_ismature.find_all( 'img' ) )
+                    
                 
                 for img in imgs:
                     
diff --git a/include/ClientGUI.py b/include/ClientGUI.py
index aa4713ae..b7aaa4d2 100755
--- a/include/ClientGUI.py
+++ b/include/ClientGUI.py
@@ -18,6 +18,7 @@ import ClientGUIShortcuts
 import ClientGUITopLevelWindows
 import ClientDownloading
 import ClientMedia
+import ClientNetworking
 import ClientSearch
 import ClientServices
 import ClientThreading
@@ -48,6 +49,8 @@ import types
 import webbrowser
 import wx
 
+ID_TIMER_GUI_BANDWIDTH = wx.NewId()
+
 # Sizer Flags
 
 MENU_ORDER = [ 'file', 'undo', 'pages', 'database', 'pending', 'services', 'help' ]
@@ -69,13 +72,14 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
         
         self.SetDropTarget( ClientDragDrop.FileDropTarget( self.ImportFiles, self.ImportURL ) )
         
+        bandwidth_width = ClientData.ConvertTextToPixelWidth( self, 7 )
         idle_width = ClientData.ConvertTextToPixelWidth( self, 4 )
         system_busy_width = ClientData.ConvertTextToPixelWidth( self, 11 )
         db_width = ClientData.ConvertTextToPixelWidth( self, 12 )
         
         self._statusbar = self.CreateStatusBar()
-        self._statusbar.SetFieldsCount( 4 )
-        self._statusbar.SetStatusWidths( [ -1, idle_width, system_busy_width, db_width ] )
+        self._statusbar.SetFieldsCount( 5 )
+        self._statusbar.SetStatusWidths( [ -1, bandwidth_width, idle_width, system_busy_width, db_width ] )
         
         self._statusbar_thread_updater = ClientGUICommon.ThreadToGUIUpdater( self._statusbar, self.RefreshStatusBar )
         
@@ -103,6 +107,7 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
         self.Bind( wx.EVT_CLOSE, self.EventClose )
         self.Bind( wx.EVT_SET_FOCUS, self.EventFocus )
         self.Bind( wx.EVT_CHAR_HOOK, self.EventCharHook )
+        self.Bind( wx.EVT_TIMER, self.TIMEREventBandwidth, id = ID_TIMER_GUI_BANDWIDTH )
         
         self._controller.sub( self, 'ClearClosedPages', 'clear_closed_pages' )
         self._controller.sub( self, 'NewCompose', 'new_compose_frame' )
@@ -146,6 +151,10 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
         
         wx.CallAfter( self._InitialiseSession ) # do this in callafter as some pages want to talk to controller.gui, which doesn't exist yet!
         
+        self._move_hide_timer = wx.Timer( self, id = ID_TIMER_GUI_BANDWIDTH )
+        
+        self._move_hide_timer.Start( 1000, wx.TIMER_CONTINUOUS )
+        
     
     def _AboutWindow( self ):
         
@@ -1184,8 +1193,25 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
             
             ClientGUIMenus.AppendSeparator( menu )
             
-            ClientGUIMenus.AppendMenuItem( self, menu, 'create a database backup', 'Back the database up to an external location.', self._controller.BackupDatabase )
-            ClientGUIMenus.AppendMenuItem( self, menu, 'restore a database backup', 'Restore the database from an external location.', self._controller.RestoreDatabase )
+            backup_path = self._new_options.GetNoneableString( 'backup_path' )
+            
+            if backup_path is None:
+                
+                ClientGUIMenus.AppendMenuItem( self, menu, 'set up a database backup location', 'Choose a path to back the database up to.', self._SetupBackupPath )
+                
+            else:
+                
+                ClientGUIMenus.AppendMenuItem( self, menu, 'update database backup', 'Back the database up to an external location.', self._controller.BackupDatabase )
+                ClientGUIMenus.AppendMenuItem( self, menu, 'change database backup location', 'Choose a path to back the database up to.', self._SetupBackupPath )
+                
+            
+            ClientGUIMenus.AppendSeparator( menu )
+            
+            ClientGUIMenus.AppendMenuItem( self, menu, 'restore from a database backup', 'Restore the database from an external location.', self._controller.RestoreDatabase )
+            
+            ClientGUIMenus.AppendSeparator( menu )
+            
+            ClientGUIMenus.AppendMenuItem( self, menu, 'migrate database (under construction!)', 'Review and manage the locations your database is stored.', self._MigrateDatabase )
             
             ClientGUIMenus.AppendSeparator( menu )
             
@@ -1399,6 +1425,10 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
                 ClientGUIMenus.AppendMenu( menu, admin_menu, 'administrate services' )
                 
             
+            ClientGUIMenus.AppendSeparator( menu )
+            
+            ClientGUIMenus.AppendMenuItem( self, menu, 'review bandwidth usage (under construction!)', 'See where you are consuming data.', self._ReviewBandwidth )
+            
             ClientGUIMenus.AppendSeparator( menu )
             
             ClientGUIMenus.AppendMenuItem( self, menu, 'import repository update files', 'Add repository update files to the database.', self._ImportUpdateFiles )
@@ -1890,6 +1920,8 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
             
             HC.options[ 'pause_subs_sync' ] = original_pause_status
             
+            HG.client_controller.pub( 'notify_new_subscriptions' )
+            
         
     
     def _ManageTagCensorship( self ):
@@ -1912,6 +1944,15 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
         with ClientGUIDialogsManage.DialogManageUPnP( self ) as dlg: dlg.ShowModal()
         
     
+    def _MigrateDatabase( self ):
+        
+        frame = ClientGUITopLevelWindows.FrameThatTakesScrollablePanel( self, 'migrate database' )
+        
+        panel = ClientGUIScrolledPanelsReview.MigrateDatabasePanel( frame, self._controller )
+        
+        frame.SetPanel( panel )
+        
+    
     def _ModifyAccount( self, service_key ):
         
         wx.MessageBox( 'this does not work yet!' )
@@ -2300,9 +2341,9 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
         self._statusbar.SetToolTipString( job_name )
         
         self._statusbar.SetStatusText( media_status, number = 0 )
-        self._statusbar.SetStatusText( idle_status, number = 1 )
-        self._statusbar.SetStatusText( busy_status, number = 2 )
-        self._statusbar.SetStatusText( db_status, number = 3 )
+        self._statusbar.SetStatusText( idle_status, number = 2 )
+        self._statusbar.SetStatusText( busy_status, number = 3 )
+        self._statusbar.SetStatusText( db_status, number = 4 )
         
     
     def _RegenerateACCache( self ):
@@ -2390,6 +2431,15 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
             
         
     
+    def _ReviewBandwidth( self ):
+        
+        frame = ClientGUITopLevelWindows.FrameThatTakesScrollablePanel( self, 'review bandwidth' )
+        
+        panel = ClientGUIScrolledPanelsReview.ReviewAllBandwidthPanel( frame, self._controller )
+        
+        frame.SetPanel( panel )
+        
+    
     def _ReviewServices( self ):
         
         frame = ClientGUITopLevelWindows.FrameThatTakesScrollablePanel( self, self._controller.PrepStringForDisplay( 'Review Services' ), 'review_services' )
@@ -2507,6 +2557,90 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
         if page is not None: page.SetSynchronisedWait()
         
     
+    def _SetupBackupPath( self ):
+        
+        backup_intro = 'Everything in your client is stored in the database, which consists of a handful of .db files and a single subdirectory that contains all your media files. It is a very good idea to maintain a regular backup schedule--to save from hard drive failure, serious software fault, accidental deletion, or any other unexpected problem. It sucks to lose all your work, so make sure it can\'t happen!'
+        backup_intro += os.linesep * 2
+        backup_intro += 'If you prefer to create a manual backup with an external program like FreeFileSync, then please cancel out of the dialog after this and set up whatever you like, but if you would rather a simple solution, simply select a directory and the client will remember it as the designated backup location. Creating or updating your backup can be triggered at any time from the database menu.'
+        backup_intro += os.linesep * 2
+        backup_intro += 'An ideal backup location is initially empty and on a different hard drive.'
+        backup_intro += os.linesep * 2
+        backup_intro += 'If you have a large database (100,000+ files) or a slow hard drive, creating the initial backup may take a long time--perhaps an hour or more--but updating an existing backup should only take a couple of minutes (since the client only has to copy new or modified files). Try to update your backup every week!'
+        backup_intro += os.linesep * 2
+        backup_intro += 'If you would like some more info on making or restoring backups, please consult the help\'s \'installing and updating\' page.'
+        
+        wx.MessageBox( backup_intro )
+        
+        with wx.DirDialog( self, 'Select backup location.' ) as dlg:
+            
+            if dlg.ShowModal() == wx.ID_OK:
+                
+                path = HydrusData.ToUnicode( dlg.GetPath() )
+                
+                if path == '':
+                    
+                    path = None
+                    
+                
+                if path == self._controller.GetDBDir():
+                    
+                    wx.MessageBox( 'That directory is your current database directory! You cannot backup to the same location you are backing up from!' )
+                    
+                    return
+                    
+                
+                if os.path.exists( path ):
+                    
+                    filenames = os.listdir( path )
+                    
+                    num_files = len( filenames )
+                    
+                    if num_files == 0:
+                        
+                        extra_info = 'It looks currently empty, which is great--there is no danger of anything being overwritten.'
+                        
+                    elif 'client.db' in filenames:
+                        
+                        extra_info = 'It looks like a client database already exists in the location--be certain that it is ok to overwrite it.'
+                        
+                    else:
+                        
+                        extra_info = 'It seems to have some files already in it--be careful and make sure you chose the correct location.'
+                        
+                    
+                else:
+                    
+                    extra_info = 'The path does not exist yet--it will be created when you make your first backup.'
+                    
+                
+                text = 'You chose "' + path + '". Here is what I understand about it:'
+                text += os.linesep * 2
+                text += extra_info
+                text += os.linesep * 2
+                text += 'Are you sure this is the correct directory?'
+                
+                with ClientGUIDialogs.DialogYesNo( self, text ) as dlg_yn:
+                    
+                    if dlg_yn.ShowModal() == wx.ID_YES:
+                        
+                        self._new_options.SetNoneableString( 'backup_path', path )
+                        
+                        text = 'Would you like to create your backup now?'
+                        
+                        with ClientGUIDialogs.DialogYesNo( self, text ) as dlg_yn_2:
+                            
+                            if dlg_yn_2.ShowModal() == wx.ID_YES:
+                                
+                                self._controller.BackupDatabase()
+                                
+                            
+                        
+                    
+                
+            
+        
+        
+    
     def _ShowHideSplitters( self ):
         
         page = self._notebook.GetCurrentPage()
@@ -3114,6 +3248,22 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
         event.Skip( True )
         
     
+    def TIMEREventBandwidth( self, event ):
+        
+        current_usage = self._controller.network_engine.bandwidth_manager.GetTracker( ClientNetworking.GLOBAL_NETWORK_CONTEXT ).GetUsage( HC.BANDWIDTH_TYPE_DATA, 1 )
+        
+        if current_usage == 0:
+            
+            bandwidth_status = ''
+            
+        else:
+            
+            bandwidth_status = HydrusData.ConvertIntToBytes( current_usage ) + '/s'
+            
+        
+        self._statusbar.SetStatusText( bandwidth_status, number = 1 )
+        
+    
     def Exit( self, restart = False ):
         
         if not HG.emergency_exit:
diff --git a/include/ClientGUICanvas.py b/include/ClientGUICanvas.py
index 47149584..d4c57887 100755
--- a/include/ClientGUICanvas.py
+++ b/include/ClientGUICanvas.py
@@ -32,9 +32,6 @@ if HC.PLATFORM_WINDOWS:
     
     import wx.lib.flashwin
     
-ID_TIMER_VIDEO = wx.NewId()
-ID_TIMER_RENDER_WAIT = wx.NewId()
-ID_TIMER_ANIMATION_BAR_UPDATE = wx.NewId()
 ID_TIMER_SLIDESHOW = wx.NewId()
 ID_TIMER_CURSOR_HIDE = wx.NewId()
 ID_TIMER_HOVER_SHOW = wx.NewId()
@@ -260,11 +257,11 @@ class Animation( wx.Window ):
         self._canvas_bmp = None
         self._frame_bmp = None
         
-        self._timer_video = wx.Timer( self, id = ID_TIMER_VIDEO )
+        self._timer_video = wx.Timer( self )
         
         self.Bind( wx.EVT_PAINT, self.EventPaint )
         self.Bind( wx.EVT_SIZE, self.EventResize )
-        self.Bind( wx.EVT_TIMER, self.TIMEREventVideo, id = ID_TIMER_VIDEO )
+        self.Bind( wx.EVT_TIMER, self.TIMEREventVideo )
         self.Bind( wx.EVT_MOUSE_EVENTS, self.EventPropagateMouse )
         self.Bind( wx.EVT_KEY_UP, self.EventPropagateKey )
         self.Bind( wx.EVT_ERASE_BACKGROUND, self.EventEraseBackground )
@@ -700,12 +697,12 @@ class AnimationBar( wx.Window ):
         self._it_was_playing = False
         
         self.Bind( wx.EVT_MOUSE_EVENTS, self.EventMouse )
-        self.Bind( wx.EVT_TIMER, self.TIMERFlashIndexUpdate, id = ID_TIMER_ANIMATION_BAR_UPDATE )
+        self.Bind( wx.EVT_TIMER, self.TIMERFlashIndexUpdate )
         self.Bind( wx.EVT_PAINT, self.EventPaint )
         self.Bind( wx.EVT_SIZE, self.EventResize )
         self.Bind( wx.EVT_ERASE_BACKGROUND, self.EventEraseBackground )
         
-        self._flash_index_update_timer = wx.Timer( self, id = ID_TIMER_ANIMATION_BAR_UPDATE )
+        self._flash_index_update_timer = wx.Timer( self )
         
     
     def _GetXFromFrameIndex( self, index, width_offset = 0 ):
@@ -5505,11 +5502,11 @@ class StaticImage( wx.Window ):
         
         self._canvas_bmp = None
         
-        self._timer_render_wait = wx.Timer( self, id = ID_TIMER_RENDER_WAIT )
+        self._timer_render_wait = wx.Timer( self )
         
         self.Bind( wx.EVT_PAINT, self.EventPaint )
         self.Bind( wx.EVT_SIZE, self.EventResize )
-        self.Bind( wx.EVT_TIMER, self.TIMEREventRenderWait, id = ID_TIMER_RENDER_WAIT )
+        self.Bind( wx.EVT_TIMER, self.TIMEREventRenderWait )
         self.Bind( wx.EVT_MOUSE_EVENTS, self.EventPropagateMouse )
         self.Bind( wx.EVT_ERASE_BACKGROUND, self.EventEraseBackground )
         
diff --git a/include/ClientGUICommon.py b/include/ClientGUICommon.py
index 99f52031..a62173bc 100755
--- a/include/ClientGUICommon.py
+++ b/include/ClientGUICommon.py
@@ -334,6 +334,14 @@ class BetterStaticText( wx.StaticText ):
         # st.Wrap is a pain to deal with here, seems to sometimes/always not be able to increase after an initial non-zero call
         
     
+    def SetLabelText( self, text ):
+        
+        if text != self.GetLabelText():
+            
+            wx.StaticText.SetLabelText( self, text )
+            
+        
+    
 class BufferedWindow( wx.Window ):
     
     def __init__( self, *args, **kwargs ):
@@ -773,24 +781,35 @@ class Gauge( wx.Gauge ):
         
         wx.Gauge.__init__( self, *args, **kwargs )
         
-        self._actual_max = None
+        self._actual_range = None
+        
+        self._is_pulsing = False
         
     
-    def SetRange( self, max ):
+    def SetRange( self, range ):
         
-        if max is None:
+        if range is None:
             
             self.Pulse()
             
-        elif max > 1000:
-            
-            self._actual_max = max
-            wx.Gauge.SetRange( self, 1000 )
+            self._is_pulsing = True
             
         else:
             
-            self._actual_max = None
-            wx.Gauge.SetRange( self, max )
+            if range > 1000:
+                
+                self._actual_range = range
+                range = 1000
+                
+            else:
+                
+                self._actual_range = None
+                
+            
+            if range != self.GetRange():
+                
+                wx.Gauge.SetRange( self, range )
+                
             
         
     
@@ -800,13 +819,19 @@ class Gauge( wx.Gauge ):
             
             self.Pulse()
             
-        elif self._actual_max is None:
-            
-            wx.Gauge.SetValue( self, value )
+            self._is_pulsing = True
             
         else:
             
-            wx.Gauge.SetValue( self, min( int( 1000 * ( float( value ) / self._actual_max ) ), 1000 ) )
+            if self._actual_range is not None:
+                
+                value = min( int( 1000 * ( float( value ) / self._actual_range ) ), 1000 )
+                
+            
+            if value != self.GetValue() or self._is_pulsing:
+                
+                wx.Gauge.SetValue( self, value )
+                
             
         
     
@@ -1873,15 +1898,8 @@ class PopupMessage( PopupWindow ):
             
             ( gauge_value, gauge_range ) = popup_gauge_1
             
-            if gauge_range is None or gauge_value is None:
-                
-                self._gauge_1.Pulse()
-                
-            else:
-                
-                self._gauge_1.SetRange( gauge_range )
-                self._gauge_1.SetValue( gauge_value )
-                
+            self._gauge_1.SetRange( gauge_range )
+            self._gauge_1.SetValue( gauge_value )
             
             self._gauge_1.Show()
             
@@ -1914,15 +1932,8 @@ class PopupMessage( PopupWindow ):
             
             ( gauge_value, gauge_range ) = popup_gauge_2
             
-            if gauge_range is None or gauge_value is None:
-                
-                self._gauge_2.Pulse()
-                
-            else:
-                
-                self._gauge_2.SetRange( gauge_range )
-                self._gauge_2.SetValue( gauge_value )
-                
+            self._gauge_2.SetRange( gauge_range )
+            self._gauge_2.SetValue( gauge_value )
             
             self._gauge_2.Show()
             
@@ -3776,22 +3787,8 @@ class TextAndGauge( wx.Panel ):
             self._st.SetLabelText( text )
             
         
-        if value is None or range is None:
-            
-            self._gauge.Pulse()
-            
-        else:
-            
-            if range != self._gauge.GetRange():
-                
-                self._gauge.SetRange( range )
-                
-            
-            if value != self._gauge.GetValue():
-                
-                self._gauge.SetValue( value )
-                
-            
+        self._gauge.SetRange( range )
+        self._gauge.SetValue( value )
         
     
 ( DirtyEvent, EVT_DIRTY ) = wx.lib.newevent.NewEvent()
diff --git a/include/ClientGUIControls.py b/include/ClientGUIControls.py
index a7a3d428..ea2e8c4e 100644
--- a/include/ClientGUIControls.py
+++ b/include/ClientGUIControls.py
@@ -1,5 +1,6 @@
 import ClientCaches
 import ClientConstants as CC
+import ClientData
 import ClientGUICommon
 import ClientGUIDialogs
 import ClientGUIMenus
@@ -12,8 +13,6 @@ import HydrusNetworking
 import os
 import wx
 
-ID_TIMER_NETWORK_JOB = wx.NewId()
-
 class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
     
     def __init__( self, parent, bandwidth_rules ):
@@ -351,11 +350,20 @@ class NetworkJobControl( wx.Panel ):
     
     def __init__( self, parent ):
         
-        wx.Panel.__init__( self, parent )
+        wx.Panel.__init__( self, parent, style = wx.BORDER_DOUBLE )
         
         self._network_job = None
+        self._download_started = False
         
-        self._text_and_gauge = ClientGUICommon.TextAndGauge( self )
+        self._left_text = ClientGUICommon.BetterStaticText( self )
+        self._right_text = ClientGUICommon.BetterStaticText( self, style = wx.ALIGN_RIGHT )
+        
+        # 512/768KB - 200KB/s
+        right_width = ClientData.ConvertTextToPixelWidth( self._right_text, 20 )
+        
+        self._right_text.SetMinSize( ( right_width, -1 ) )
+        
+        self._gauge = ClientGUICommon.Gauge( self )
         
         self._cancel_button = ClientGUICommon.BetterBitmapButton( self, CC.GlobalBMPs.stop, self.Cancel )
         
@@ -365,18 +373,28 @@ class NetworkJobControl( wx.Panel ):
         
         #
         
+        st_hbox = wx.BoxSizer( wx.HORIZONTAL )
+        
+        st_hbox.AddF( self._left_text, CC.FLAGS_EXPAND_BOTH_WAYS )
+        st_hbox.AddF( self._right_text, CC.FLAGS_VCENTER )
+        
+        left_vbox = wx.BoxSizer( wx.VERTICAL )
+        
+        left_vbox.AddF( st_hbox, CC.FLAGS_EXPAND_SIZER_BOTH_WAYS )
+        left_vbox.AddF( self._gauge, CC.FLAGS_EXPAND_PERPENDICULAR )
+        
         hbox = wx.BoxSizer( wx.HORIZONTAL )
         
-        hbox.AddF( self._text_and_gauge, CC.FLAGS_EXPAND_BOTH_WAYS )
+        hbox.AddF( left_vbox, CC.FLAGS_EXPAND_SIZER_BOTH_WAYS )
         hbox.AddF( self._cancel_button, CC.FLAGS_VCENTER )
         
         self.SetSizer( hbox )
         
         #
         
-        self.Bind( wx.EVT_TIMER, self.TIMEREventUpdate, id = ID_TIMER_NETWORK_JOB )
+        self.Bind( wx.EVT_TIMER, self.TIMEREventUpdate )
         
-        self._move_hide_timer = wx.Timer( self, id = ID_TIMER_NETWORK_JOB )
+        self._move_hide_timer = wx.Timer( self )
         
         self._move_hide_timer.Start( 250, wx.TIMER_CONTINUOUS )
         
@@ -385,7 +403,11 @@ class NetworkJobControl( wx.Panel ):
         
         if self._network_job is None:
             
-            self._text_and_gauge.SetValue( '', 0, 1 )
+            self._left_text.SetLabelText( '' )
+            self._right_text.SetLabelText( '' )
+            self._gauge.SetRange( 1 )
+            self._gauge.SetValue( 0 )
+            
             can_cancel = False
             
         else:
@@ -401,16 +423,43 @@ class NetworkJobControl( wx.Panel ):
             
             ( status_text, current_speed, bytes_read, bytes_to_read ) = self._network_job.GetStatus()
             
-            if self._network_job.HasError():
+            self._left_text.SetLabelText( status_text )
+            
+            if not self._download_started and current_speed > 0:
                 
-                text = status_text
+                self._download_started = True
+                
+            
+            if self._download_started and not self._network_job.HasError():
+                
+                speed_text = ''
+                
+                if bytes_read is not None:
+                    
+                    if bytes_to_read is not None and bytes_read != bytes_to_read:
+                        
+                        speed_text += HydrusData.ConvertValueRangeToBytes( bytes_read, bytes_to_read )
+                        
+                    else:
+                        
+                        speed_text += HydrusData.ConvertIntToBytes( bytes_read )
+                        
+                    
+                
+                if current_speed != bytes_to_read: # if it is a real quick download, just say its size
+                    
+                    speed_text += ' ' + HydrusData.ConvertIntToBytes( current_speed ) + '/s'
+                    
+                
+                self._right_text.SetLabelText( speed_text )
                 
             else:
                 
-                text = status_text + ' ' + HydrusData.ConvertIntToBytes( current_speed ) + '/s'
+                self._right_text.SetLabelText( '' )
                 
             
-            self._text_and_gauge.SetValue( text, bytes_read, bytes_to_read )
+            self._gauge.SetRange( bytes_to_read )
+            self._gauge.SetValue( bytes_read )
             
         
         if can_cancel:
@@ -439,12 +488,17 @@ class NetworkJobControl( wx.Panel ):
     
     def ClearNetworkJob( self ):
         
+        self._Update()
+        
         self._network_job = None
         
+        self._move_hide_timer.Start( 250, wx.TIMER_CONTINUOUS )
+        
     
     def SetNetworkJob( self, network_job ):
         
         self._network_job = network_job
+        self._download_started = False
         
     
     def TIMEREventUpdate( self, event ):
diff --git a/include/ClientGUIManagement.py b/include/ClientGUIManagement.py
index 0290ad69..419e03dd 100755
--- a/include/ClientGUIManagement.py
+++ b/include/ClientGUIManagement.py
@@ -1885,30 +1885,27 @@ class ManagementPanelImporterPageOfImages( ManagementPanelImporter ):
         
         self._page_of_images_panel = ClientGUICommon.StaticBox( self, 'page of images downloader' )
         
-        self._import_queue_panel = ClientGUICommon.StaticBox( self._page_of_images_panel, 'imports' )
-        
-        self._parser_status = wx.StaticText( self._import_queue_panel )
-        self._overall_status = wx.StaticText( self._import_queue_panel )
-        self._current_action = wx.StaticText( self._import_queue_panel )
-        self._file_gauge = ClientGUICommon.Gauge( self._import_queue_panel )
-        self._overall_gauge = ClientGUICommon.Gauge( self._import_queue_panel )
-        
-        self._pause_button = wx.BitmapButton( self._import_queue_panel, bitmap = CC.GlobalBMPs.pause )
+        self._pause_button = wx.BitmapButton( self._page_of_images_panel, bitmap = CC.GlobalBMPs.pause )
         self._pause_button.Bind( wx.EVT_BUTTON, self.EventPause )
         
-        self._waiting_politely_indicator = ClientGUICommon.GetWaitingPolitelyControl( self._import_queue_panel, self._page_key )
+        #
+        
+        self._import_queue_panel = ClientGUICommon.StaticBox( self._page_of_images_panel, 'imports' )
+        
+        self._overall_status = wx.StaticText( self._import_queue_panel )
+        self._current_action = wx.StaticText( self._import_queue_panel )
+        self._file_download_control = ClientGUIControls.NetworkJobControl( self._import_queue_panel )
+        self._overall_gauge = ClientGUICommon.Gauge( self._import_queue_panel )
         
         self._seed_cache_button = ClientGUICommon.BetterBitmapButton( self._import_queue_panel, CC.GlobalBMPs.seed_cache, self._SeedCache )
         self._seed_cache_button.SetToolTipString( 'open detailed file import status' )
         
-        button_sizer = wx.BoxSizer( wx.HORIZONTAL )
-        
-        button_sizer.AddF( self._waiting_politely_indicator, CC.FLAGS_VCENTER )
-        button_sizer.AddF( self._seed_cache_button, CC.FLAGS_VCENTER )
-        button_sizer.AddF( self._pause_button, CC.FLAGS_VCENTER )
-        
         self._pending_page_urls_panel = ClientGUICommon.StaticBox( self._page_of_images_panel, 'pending page urls' )
         
+        self._parser_status = wx.StaticText( self._pending_page_urls_panel )
+        
+        self._page_download_control = ClientGUIControls.NetworkJobControl( self._pending_page_urls_panel )
+        
         self._pending_page_urls_listbox = wx.ListBox( self._pending_page_urls_panel, size = ( -1, 100 ) )
         
         self._advance_button = wx.Button( self._pending_page_urls_panel, label = u'\u2191' )
@@ -1954,18 +1951,20 @@ class ManagementPanelImporterPageOfImages( ManagementPanelImporter ):
         input_hbox.AddF( self._page_url_input, CC.FLAGS_EXPAND_BOTH_WAYS )
         input_hbox.AddF( self._page_url_paste, CC.FLAGS_VCENTER )
         
+        self._pending_page_urls_panel.AddF( self._parser_status, CC.FLAGS_EXPAND_PERPENDICULAR )
+        self._pending_page_urls_panel.AddF( self._page_download_control, CC.FLAGS_EXPAND_PERPENDICULAR )
         self._pending_page_urls_panel.AddF( queue_hbox, CC.FLAGS_EXPAND_SIZER_BOTH_WAYS )
         self._pending_page_urls_panel.AddF( input_hbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
         
         #
         
-        self._import_queue_panel.AddF( self._parser_status, CC.FLAGS_EXPAND_PERPENDICULAR )
         self._import_queue_panel.AddF( self._overall_status, CC.FLAGS_EXPAND_PERPENDICULAR )
         self._import_queue_panel.AddF( self._current_action, CC.FLAGS_EXPAND_PERPENDICULAR )
-        self._import_queue_panel.AddF( self._file_gauge, CC.FLAGS_EXPAND_PERPENDICULAR )
         self._import_queue_panel.AddF( self._overall_gauge, CC.FLAGS_EXPAND_PERPENDICULAR )
-        self._import_queue_panel.AddF( button_sizer, CC.FLAGS_BUTTON_SIZER )
+        self._import_queue_panel.AddF( self._seed_cache_button, CC.FLAGS_LONE_BUTTON )
+        self._import_queue_panel.AddF( self._file_download_control, CC.FLAGS_EXPAND_PERPENDICULAR )
         
+        self._page_of_images_panel.AddF( self._pause_button, CC.FLAGS_LONE_BUTTON )
         self._page_of_images_panel.AddF( self._import_queue_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
         self._page_of_images_panel.AddF( self._pending_page_urls_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
         self._page_of_images_panel.AddF( self._download_image_links, CC.FLAGS_EXPAND_PERPENDICULAR )
@@ -1992,20 +1991,8 @@ class ManagementPanelImporterPageOfImages( ManagementPanelImporter ):
         
         self._page_of_images_import = self._management_controller.GetVariable( 'page_of_images_import' )
         
-        def file_download_hook( gauge_range, gauge_value ):
-            
-            try:
-                
-                self._file_gauge.SetRange( gauge_range )
-                self._file_gauge.SetValue( gauge_value )
-                
-            except wx.PyDeadObjectError:
-                
-                pass
-                
-            
-        
-        self._page_of_images_import.SetDownloadHook( file_download_hook )
+        self._page_of_images_import.SetDownloadControlFile( self._file_download_control )
+        self._page_of_images_import.SetDownloadControlPage( self._page_download_control )
         
         ( import_file_options, download_image_links, download_unlinked_images ) = self._page_of_images_import.GetOptions()
         
@@ -2256,66 +2243,76 @@ class ManagementPanelImporterThreadWatcher( ManagementPanelImporter ):
         
         self._options_panel = wx.Panel( self._thread_watcher_panel )
         
-        self._watcher_status = wx.StaticText( self._options_panel )
-        self._overall_status = wx.StaticText( self._options_panel )
-        self._current_action = wx.StaticText( self._options_panel )
-        self._file_gauge = ClientGUICommon.Gauge( self._options_panel )
-        self._overall_gauge = ClientGUICommon.Gauge( self._options_panel )
+        self._pause_button = wx.BitmapButton( self._options_panel, bitmap = CC.GlobalBMPs.pause )
+        self._pause_button.Bind( wx.EVT_BUTTON, self.EventPause )
+        
+        #
+        
+        imports_panel = ClientGUICommon.StaticBox( self._options_panel, 'imports' )
+        
+        self._overall_status = wx.StaticText( imports_panel )
+        self._current_action = wx.StaticText( imports_panel )
+        self._overall_gauge = ClientGUICommon.Gauge( imports_panel )
+        self._file_download_control = ClientGUIControls.NetworkJobControl( imports_panel )
+        
+        self._seed_cache_button = ClientGUICommon.BetterBitmapButton( imports_panel, CC.GlobalBMPs.seed_cache, self._SeedCache )
+        self._seed_cache_button.SetToolTipString( 'open detailed file import status' )
+        
+        #
+        
+        checker_panel = ClientGUICommon.StaticBox( self._options_panel, 'checker' )
+        
+        self._watcher_status = wx.StaticText( checker_panel )
+        self._thread_download_control = ClientGUIControls.NetworkJobControl( checker_panel )
         
         ( times_to_check, check_period ) = HC.options[ 'thread_checker_timings' ]
         
-        self._thread_times_to_check = wx.SpinCtrl( self._options_panel, size = ( 80, -1 ), min = 0, max = 65536 )
+        self._thread_times_to_check = wx.SpinCtrl( checker_panel, size = ( 80, -1 ), min = 0, max = 65536 )
         self._thread_times_to_check.SetValue( times_to_check )
         self._thread_times_to_check.Bind( wx.EVT_SPINCTRL, self.EventTimesToCheck )
         
-        self._thread_check_period = ClientGUICommon.TimeDeltaButton( self._options_panel, min = 30, days = True, hours = True, minutes = True, seconds = True )
+        self._thread_check_period = ClientGUICommon.TimeDeltaButton( checker_panel, min = 30, days = True, hours = True, minutes = True, seconds = True )
         self._thread_check_period.SetValue( check_period )
         self._thread_check_period.Bind( ClientGUICommon.EVT_TIME_DELTA, self.EventCheckPeriod )
         
-        self._thread_check_now_button = wx.Button( self._options_panel, label = 'check now' )
+        self._thread_check_now_button = wx.Button( checker_panel, label = 'check now' )
         self._thread_check_now_button.Bind( wx.EVT_BUTTON, self.EventCheckNow )
         
-        self._waiting_politely_indicator = ClientGUICommon.GetWaitingPolitelyControl( self._options_panel, self._page_key )
-        
-        self._seed_cache_button = ClientGUICommon.BetterBitmapButton( self._options_panel, CC.GlobalBMPs.seed_cache, self._SeedCache )
-        self._seed_cache_button.SetToolTipString( 'open detailed file import status' )
-        
-        self._pause_button = wx.BitmapButton( self._options_panel, bitmap = CC.GlobalBMPs.pause )
-        self._pause_button.Bind( wx.EVT_BUTTON, self.EventPause )
+        #
         
         self._import_file_options = ClientGUICollapsible.CollapsibleOptionsImportFiles( self._thread_watcher_panel )
         self._import_tag_options = ClientGUICollapsible.CollapsibleOptionsTags( self._thread_watcher_panel, namespaces = [ 'filename' ] )
         
         #
         
+        imports_panel.AddF( self._overall_status, CC.FLAGS_EXPAND_PERPENDICULAR )
+        imports_panel.AddF( self._current_action, CC.FLAGS_EXPAND_PERPENDICULAR )
+        imports_panel.AddF( self._overall_gauge, CC.FLAGS_EXPAND_PERPENDICULAR )
+        imports_panel.AddF( self._seed_cache_button, CC.FLAGS_LONE_BUTTON )
+        imports_panel.AddF( self._file_download_control, CC.FLAGS_EXPAND_PERPENDICULAR )
+        
         hbox_1 = wx.BoxSizer( wx.HORIZONTAL )
         
-        hbox_1.AddF( wx.StaticText( self._options_panel, label = 'check ' ), CC.FLAGS_VCENTER )
+        hbox_1.AddF( wx.StaticText( checker_panel, label = 'check ' ), CC.FLAGS_VCENTER )
         hbox_1.AddF( self._thread_times_to_check, CC.FLAGS_VCENTER )
-        hbox_1.AddF( wx.StaticText( self._options_panel, label = ' more times' ), CC.FLAGS_VCENTER )
+        hbox_1.AddF( wx.StaticText( checker_panel, label = ' more times' ), CC.FLAGS_VCENTER )
         
         hbox_2 = wx.BoxSizer( wx.HORIZONTAL )
         
-        hbox_2.AddF( wx.StaticText( self._options_panel, label = 'check every ' ), CC.FLAGS_VCENTER )
+        hbox_2.AddF( wx.StaticText( checker_panel, label = 'check every ' ), CC.FLAGS_VCENTER )
         hbox_2.AddF( self._thread_check_period, CC.FLAGS_VCENTER )
         
-        button_sizer = wx.BoxSizer( wx.HORIZONTAL )
-        
-        button_sizer.AddF( self._thread_check_now_button, CC.FLAGS_VCENTER )
-        button_sizer.AddF( self._waiting_politely_indicator, CC.FLAGS_VCENTER )
-        button_sizer.AddF( self._seed_cache_button, CC.FLAGS_VCENTER )
-        button_sizer.AddF( self._pause_button, CC.FLAGS_VCENTER )
+        checker_panel.AddF( self._watcher_status, CC.FLAGS_EXPAND_PERPENDICULAR )
+        checker_panel.AddF( self._thread_download_control, CC.FLAGS_EXPAND_PERPENDICULAR )
+        checker_panel.AddF( hbox_1, CC.FLAGS_LONE_BUTTON )
+        checker_panel.AddF( hbox_2, CC.FLAGS_LONE_BUTTON )
+        checker_panel.AddF( self._thread_check_now_button, CC.FLAGS_LONE_BUTTON )
         
         vbox = wx.BoxSizer( wx.VERTICAL )
         
-        vbox.AddF( self._watcher_status, CC.FLAGS_EXPAND_PERPENDICULAR )
-        vbox.AddF( self._overall_status, CC.FLAGS_EXPAND_PERPENDICULAR )
-        vbox.AddF( self._current_action, CC.FLAGS_EXPAND_PERPENDICULAR )
-        vbox.AddF( self._file_gauge, CC.FLAGS_EXPAND_PERPENDICULAR )
-        vbox.AddF( self._overall_gauge, CC.FLAGS_EXPAND_PERPENDICULAR )
-        vbox.AddF( hbox_1, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
-        vbox.AddF( hbox_2, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
-        vbox.AddF( button_sizer, CC.FLAGS_BUTTON_SIZER )
+        vbox.AddF( self._pause_button, CC.FLAGS_LONE_BUTTON )
+        vbox.AddF( imports_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
+        vbox.AddF( checker_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
         
         self._options_panel.SetSizer( vbox )
         
@@ -2346,20 +2343,8 @@ class ManagementPanelImporterThreadWatcher( ManagementPanelImporter ):
         
         self._thread_watcher_import = self._management_controller.GetVariable( 'thread_watcher_import' )
         
-        def file_download_hook( gauge_range, gauge_value ):
-            
-            try:
-                
-                self._file_gauge.SetRange( gauge_range )
-                self._file_gauge.SetValue( gauge_value )
-                
-            except wx.PyDeadObjectError:
-                
-                pass
-                
-            
-        
-        self._thread_watcher_import.SetDownloadHook( file_download_hook )
+        self._thread_watcher_import.SetDownloadControlFile( self._file_download_control )
+        self._thread_watcher_import.SetDownloadControlThread( self._thread_download_control )
         
         ( thread_url, import_file_options, import_tag_options, times_to_check, check_period ) = self._thread_watcher_import.GetOptions()
         
@@ -2614,18 +2599,18 @@ class ManagementPanelImporterURLs( ManagementPanelImporter ):
         
         ManagementPanelImporter.__init__( self, parent, page, controller, management_controller )
         
+        #
+        
         self._url_panel = ClientGUICommon.StaticBox( self, 'raw url downloader' )
         
+        self._pause_button = wx.BitmapButton( self._url_panel, bitmap = CC.GlobalBMPs.pause )
+        self._pause_button.Bind( wx.EVT_BUTTON, self.EventPause )
+        
         self._overall_status = wx.StaticText( self._url_panel )
         self._current_action = wx.StaticText( self._url_panel )
         self._file_download_control = ClientGUIControls.NetworkJobControl( self._url_panel )
         self._overall_gauge = ClientGUICommon.Gauge( self._url_panel )
         
-        self._pause_button = wx.BitmapButton( self._url_panel, bitmap = CC.GlobalBMPs.pause )
-        self._pause_button.Bind( wx.EVT_BUTTON, self.EventPause )
-        
-        self._waiting_politely_indicator = ClientGUICommon.GetWaitingPolitelyControl( self._url_panel, self._page_key )
-        
         self._seed_cache_button = ClientGUICommon.BetterBitmapButton( self._url_panel, CC.GlobalBMPs.seed_cache, self._SeedCache )
         self._seed_cache_button.SetToolTipString( 'open detailed file import status' )
         
@@ -2639,22 +2624,17 @@ class ManagementPanelImporterURLs( ManagementPanelImporter ):
         
         #
         
-        button_sizer = wx.BoxSizer( wx.HORIZONTAL )
-        
-        button_sizer.AddF( self._waiting_politely_indicator, CC.FLAGS_VCENTER )
-        button_sizer.AddF( self._seed_cache_button, CC.FLAGS_VCENTER )
-        button_sizer.AddF( self._pause_button, CC.FLAGS_VCENTER )
-        
         input_hbox = wx.BoxSizer( wx.HORIZONTAL )
         
         input_hbox.AddF( self._url_input, CC.FLAGS_EXPAND_BOTH_WAYS )
         input_hbox.AddF( self._url_paste, CC.FLAGS_VCENTER )
         
+        self._url_panel.AddF( self._pause_button, CC.FLAGS_LONE_BUTTON )
         self._url_panel.AddF( self._overall_status, CC.FLAGS_EXPAND_PERPENDICULAR )
         self._url_panel.AddF( self._current_action, CC.FLAGS_EXPAND_PERPENDICULAR )
-        self._url_panel.AddF( self._file_download_control, CC.FLAGS_EXPAND_PERPENDICULAR )
         self._url_panel.AddF( self._overall_gauge, CC.FLAGS_EXPAND_PERPENDICULAR )
-        self._url_panel.AddF( button_sizer, CC.FLAGS_BUTTON_SIZER )
+        self._url_panel.AddF( self._seed_cache_button, CC.FLAGS_LONE_BUTTON )
+        self._url_panel.AddF( self._file_download_control, CC.FLAGS_EXPAND_PERPENDICULAR )
         self._url_panel.AddF( input_hbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
         self._url_panel.AddF( self._import_file_options, CC.FLAGS_EXPAND_PERPENDICULAR )
         
@@ -2678,7 +2658,7 @@ class ManagementPanelImporterURLs( ManagementPanelImporter ):
         
         self._urls_import = self._management_controller.GetVariable( 'urls_import' )
         
-        self._urls_import.SetDownloadControl( self._file_download_control )
+        self._urls_import.SetDownloadControlFile( self._file_download_control )
         
         import_file_options = self._urls_import.GetOptions()
         
diff --git a/include/ClientGUIParsing.py b/include/ClientGUIParsing.py
index 78f1028a..f23f05c3 100644
--- a/include/ClientGUIParsing.py
+++ b/include/ClientGUIParsing.py
@@ -1981,15 +1981,8 @@ class ScriptManagementControl( wx.Panel ):
                 ( value, range ) = ( 0, 1 )
                 
             
-            if value is None or range is None:
-                
-                self._gauge.Pulse()
-                
-            else:
-                
-                self._gauge.SetRange( range )
-                self._gauge.SetValue( value )
-                
+            self._gauge.SetRange( range )
+            self._gauge.SetValue( value )
             
             urls = self._job_key.GetURLs()
             
diff --git a/include/ClientGUIScrolledPanelsManagement.py b/include/ClientGUIScrolledPanelsManagement.py
index 6bf4ef44..0d3c808e 100644
--- a/include/ClientGUIScrolledPanelsManagement.py
+++ b/include/ClientGUIScrolledPanelsManagement.py
@@ -4854,7 +4854,7 @@ class ManageSubscriptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
         
         HG.client_controller.Write( 'serialisables_overwrite', [ HydrusSerialisable.SERIALISABLE_TYPE_SUBSCRIPTION ], subscriptions )
         
-        HG.client_controller.pub( 'notify_new_subscriptions' )
+        # we pubsub changes outside, so it happens even on cancel
         
     
     def Delete( self ):
diff --git a/include/ClientGUIScrolledPanelsReview.py b/include/ClientGUIScrolledPanelsReview.py
index 0080ec21..9a95fc71 100644
--- a/include/ClientGUIScrolledPanelsReview.py
+++ b/include/ClientGUIScrolledPanelsReview.py
@@ -1,4 +1,5 @@
 import ClientConstants as CC
+import ClientData
 import ClientGUICommon
 import ClientGUIDialogs
 import ClientGUIFrames
@@ -7,12 +8,15 @@ import ClientGUIPanels
 import ClientGUITopLevelWindows
 import ClientTags
 import ClientThreading
+import collections
 import HydrusConstants as HC
 import HydrusData
 import HydrusGlobals as HG
 import HydrusNATPunch
+import HydrusPaths
 import os
 import traceback
+import webbrowser
 import wx
 
 class AdvancedContentUpdatePanel( ClientGUIScrolledPanels.ReviewPanel ):
@@ -274,6 +278,198 @@ class AdvancedContentUpdatePanel( ClientGUIScrolledPanels.ReviewPanel ):
             
         
     
+class ReviewAllBandwidthPanel( ClientGUIScrolledPanels.ReviewPanel ):
+    
+    def __init__( self, parent, controller ):
+        
+        self._controller = controller
+        
+        ClientGUIScrolledPanels.ReviewPanel.__init__( self, parent )
+        
+        self._bandwidths = ClientGUICommon.SaneListCtrlForSingleObject( self, 360, [ ( 'context', -1 ), ( 'context type', 100 ), ( 'current usage', 100 ), ( 'past 24 hours', 100 ), ( 'this month', 100 ) ], activation_callback = self.ShowNetworkContext )
+        
+        self._bandwidths.SetMinSize( ( 640, 360 ) )
+        
+        # a button/checkbox to say 'show only those with data in the past 30 days'
+        # a button to say 'delete all record of this context'
+        
+        #
+        
+        for ( network_context, bandwidth_tracker ) in self._controller.network_engine.bandwidth_manager.GetNetworkContextsAndBandwidthTrackersForUser():
+            
+            ( display_tuple, sort_tuple ) = self._GetTuples( network_context, bandwidth_tracker )
+            
+            self._bandwidths.Append( display_tuple, sort_tuple, network_context )
+            
+        
+        self._bandwidths.SortListItems( 0 )
+        
+        #
+        
+        vbox = wx.BoxSizer( wx.VERTICAL )
+        
+        vbox.AddF( self._bandwidths, CC.FLAGS_EXPAND_BOTH_WAYS )
+        
+        self.SetSizer( vbox )
+        
+    
+    def _GetTuples( self, network_context, bandwidth_tracker ):
+        
+        sortable_network_context = ( network_context.context_type, network_context.context_data )
+        sortable_context_type = CC.network_context_type_string_lookup[ network_context.context_type ]
+        current_usage = bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 1 )
+        day_usage = bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 86400 )
+        month_usage = bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, None )
+        
+        pretty_network_context = network_context.ToUnicode()
+        pretty_context_type = CC.network_context_type_string_lookup[ network_context.context_type ]
+        
+        if current_usage == 0:
+            
+            pretty_current_usage = ''
+            
+        else:
+            
+            pretty_current_usage = HydrusData.ConvertIntToBytes( current_usage ) + '/s'
+            
+        
+        pretty_day_usage = HydrusData.ConvertIntToBytes( day_usage )
+        pretty_month_usage = HydrusData.ConvertIntToBytes( month_usage )
+        
+        return ( ( pretty_network_context, pretty_context_type, pretty_current_usage, pretty_day_usage, pretty_month_usage ), ( sortable_network_context, sortable_context_type, current_usage, day_usage, month_usage ) )
+        
+    
+    def ShowNetworkContext( self ):
+        
+        for network_context in self._bandwidths.GetObjects( only_selected = True ):
+            
+            frame = ClientGUITopLevelWindows.FrameThatTakesScrollablePanel( self._controller.GetGUI(), 'review bandwidth for ' + network_context.ToUnicode() )
+            
+            panel = ReviewNetworkContextBandwidthPanel( frame, self._controller, network_context )
+            
+            frame.SetPanel( panel )
+            
+        
+    
+class ReviewNetworkContextBandwidthPanel( ClientGUIScrolledPanels.ReviewPanel ):
+    
+    def __init__( self, parent, controller, network_context ):
+        
+        self._controller = controller
+        
+        ClientGUIScrolledPanels.ReviewPanel.__init__( self, parent )
+        
+        self._network_context = network_context
+        
+        self._bandwidth_tracker = self._controller.network_engine.bandwidth_manager.GetTracker( self._network_context )
+        
+        #
+        
+        info_panel = ClientGUICommon.StaticBox( self, 'description' )
+        
+        self._name = ClientGUICommon.BetterStaticText( info_panel, label = self._network_context.ToUnicode() )
+        self._description = ClientGUICommon.BetterStaticText( info_panel, label = CC.network_context_type_description_lookup[ self._network_context.context_type ] )
+        
+        #
+        
+        usage_panel = ClientGUICommon.StaticBox( self, 'usage' )
+        
+        self._current_usage_st = ClientGUICommon.BetterStaticText( usage_panel )
+        
+        self._time_delta_usage_bandwidth_type = ClientGUICommon.BetterChoice( usage_panel )
+        self._time_delta_usage_time_delta = ClientGUICommon.TimeDeltaButton( usage_panel, days = True, hours = True, minutes = True, seconds = True )
+        self._time_delta_usage_st = ClientGUICommon.BetterStaticText( usage_panel )
+        
+        #
+        
+        self._time_delta_usage_time_delta.SetValue( 86400 )
+        
+        for bandwidth_type in ( HC.BANDWIDTH_TYPE_DATA, HC.BANDWIDTH_TYPE_REQUESTS ):
+            
+            self._time_delta_usage_bandwidth_type.Append( HC.bandwidth_type_string_lookup[ bandwidth_type ], bandwidth_type )
+            
+        
+        self._time_delta_usage_bandwidth_type.SelectClientData( HC.BANDWIDTH_TYPE_DATA )
+        
+        # usage this month (with dropdown to select previous months for all months on record)
+        
+        # rules panel
+        # a way to show how much the current rules are used up--see review services for how this is already done
+        # button to edit rules for this domain
+        
+        #
+        
+        info_panel.AddF( self._name, CC.FLAGS_EXPAND_PERPENDICULAR )
+        info_panel.AddF( self._description, CC.FLAGS_EXPAND_PERPENDICULAR )
+        
+        #
+        
+        hbox = wx.BoxSizer( wx.HORIZONTAL )
+        
+        hbox.AddF( self._time_delta_usage_bandwidth_type, CC.FLAGS_VCENTER )
+        hbox.AddF( ClientGUICommon.BetterStaticText( usage_panel, ' in the past ' ), CC.FLAGS_VCENTER )
+        hbox.AddF( self._time_delta_usage_time_delta, CC.FLAGS_VCENTER )
+        hbox.AddF( self._time_delta_usage_st, CC.FLAGS_EXPAND_BOTH_WAYS )
+        
+        usage_panel.AddF( self._current_usage_st, CC.FLAGS_EXPAND_PERPENDICULAR )
+        usage_panel.AddF( hbox, CC.FLAGS_EXPAND_PERPENDICULAR )
+        
+        #
+        
+        vbox = wx.BoxSizer( wx.VERTICAL )
+        
+        vbox.AddF( info_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
+        vbox.AddF( usage_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
+        
+        self.SetSizer( vbox )
+        
+        min_width = ClientData.ConvertTextToPixelWidth( self, 60 )
+        
+        self.SetMinSize( ( min_width, -1 ) )
+        
+        #
+        
+        self.Bind( wx.EVT_TIMER, self.TIMEREventUpdate )
+        
+        self._move_hide_timer = wx.Timer( self )
+        
+        self._move_hide_timer.Start( 250, wx.TIMER_CONTINUOUS )
+        
+    
+    def _Update( self ):
+        
+        current_usage = self._bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 1 )
+        
+        pretty_current_usage = 'current usage: ' + HydrusData.ConvertIntToBytes( current_usage ) + '/s'
+        
+        self._current_usage_st.SetLabelText( pretty_current_usage )
+        
+        #
+        
+        bandwidth_type = self._time_delta_usage_bandwidth_type.GetChoice()
+        time_delta = self._time_delta_usage_time_delta.GetValue()
+        
+        time_delta_usage = self._bandwidth_tracker.GetUsage( bandwidth_type, time_delta )
+        
+        if bandwidth_type == HC.BANDWIDTH_TYPE_DATA:
+            
+            converter = HydrusData.ConvertIntToBytes
+            
+        elif bandwidth_type == HC.BANDWIDTH_TYPE_REQUESTS:
+            
+            converter = HydrusData.ConvertIntToPrettyString
+            
+        
+        pretty_time_delta_usage = ': ' + converter( time_delta_usage )
+        
+        self._time_delta_usage_st.SetLabelText( pretty_time_delta_usage )
+        
+    
+    def TIMEREventUpdate( self, event ):
+        
+        self._Update()
+        
+    
 class ReviewServicesPanel( ClientGUIScrolledPanels.ReviewPanel ):
     
     def __init__( self, parent, controller ):
@@ -444,13 +640,33 @@ class MigrateDatabasePanel( ClientGUIScrolledPanels.ReviewPanel ):
         
         ClientGUIScrolledPanels.ReviewPanel.__init__( self, parent )
         
-        # a help button, well labelled, that points to the help page
+        menu_items = []
         
-        # current install dir
-        # current db dir
+        page_func = HydrusData.Call( webbrowser.open, 'file://' + HC.HELP_DIR + '/database_migration.html' )
         
-        # current file dir stuff, listctrl
-        # location | portable | current percents: ftr | ideal percents: ftr
+        menu_items.append( ( 'normal', 'open the html migration help', 'Open the help page for database migration in your web browesr.', page_func ) )
+        
+        help_button = ClientGUICommon.MenuBitmapButton( self, CC.GlobalBMPs.help, menu_items )
+        
+        #
+        
+        info_panel = ClientGUICommon.StaticBox( self, 'current paths' )
+        
+        self._refresh_button = ClientGUICommon.BetterBitmapButton( info_panel, CC.GlobalBMPs.refresh, self._Update )
+        
+        self._current_install_path_st = ClientGUICommon.BetterStaticText( info_panel )
+        self._current_db_path_st = ClientGUICommon.BetterStaticText( info_panel )
+        self._current_media_paths_st = ClientGUICommon.BetterStaticText( info_panel )
+        
+        self._current_media_locations_listctrl = ClientGUICommon.SaneListCtrl( info_panel, 120, [ ( 'location', -1 ), ( 'portable?', 70 ), ( 'weight', 60 ), ( 'ideal usage', 160 ), ( 'current usage', 160 ) ] )
+        
+        # ways to:
+        # increase/decrease ideal weight
+        # force rebalance now
+        # add new path
+        # remove existing path
+        # set/clear thumb locations
+        # move whole db and portable paths (requires shutdown and user shortcut command line yes/no warning)
         
         # move the db and all portable client_files locations (provides warning about the shortcut and lets you copy the new location)
             # this will require a shutdown
@@ -465,3 +681,234 @@ class MigrateDatabasePanel( ClientGUIScrolledPanels.ReviewPanel ):
         # do rebalance now button, only enabled if there is work to do
             # should report to a stoppable job_key panel or something. text, gauge, stop button
         
+        #
+        
+        help_hbox = wx.BoxSizer( wx.HORIZONTAL )
+        
+        st = ClientGUICommon.BetterStaticText( self, 'help for this panel -->' )
+        
+        st.SetForegroundColour( wx.Colour( 0, 0, 255 ) )
+        
+        help_hbox.AddF( st, CC.FLAGS_VCENTER )
+        help_hbox.AddF( help_button, CC.FLAGS_VCENTER )
+        
+        #
+        
+        info_panel.AddF( self._refresh_button, CC.FLAGS_LONE_BUTTON )
+        info_panel.AddF( self._current_install_path_st, CC.FLAGS_EXPAND_PERPENDICULAR )
+        info_panel.AddF( self._current_db_path_st, CC.FLAGS_EXPAND_PERPENDICULAR )
+        info_panel.AddF( self._current_media_paths_st, CC.FLAGS_EXPAND_PERPENDICULAR )
+        info_panel.AddF( self._current_media_locations_listctrl, CC.FLAGS_EXPAND_PERPENDICULAR )
+        
+        #
+        
+        vbox = wx.BoxSizer( wx.VERTICAL )
+        
+        vbox.AddF( help_hbox, CC.FLAGS_BUTTON_SIZER )
+        vbox.AddF( info_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
+        
+        self.SetSizer( vbox )
+        
+        min_width = ClientData.ConvertTextToPixelWidth( self, 90 )
+        
+        self.SetMinSize( ( min_width, -1 ) )
+        
+        self._Update()
+        
+    
+    def _GenerateCurrentMediaTuples( self ):
+        
+        # ideal
+        
+        ( locations_to_ideal_weights, resized_thumbnail_override, full_size_thumbnail_override ) = self._controller.GetNewOptions().GetClientFilesLocationsToIdealWeights()
+        
+        # current
+        
+        prefixes_to_locations = HG.client_controller.Read( 'client_files_locations' )
+        
+        locations_to_file_weights = collections.Counter()
+        locations_to_fs_thumb_weights = collections.Counter()
+        locations_to_r_thumb_weights = collections.Counter()
+        
+        for ( prefix, location ) in prefixes_to_locations.items():
+            
+            if prefix.startswith( 'f' ):
+                
+                locations_to_file_weights[ location ] += 1
+                
+            
+            if prefix.startswith( 't' ):
+                
+                locations_to_fs_thumb_weights[ location ] += 1
+                
+            
+            if prefix.startswith( 'r' ):
+                
+                locations_to_r_thumb_weights[ location ] += 1
+                
+            
+        
+        #
+        
+        all_locations = set()
+        
+        all_locations.update( locations_to_ideal_weights.keys() )
+        
+        if resized_thumbnail_override is not None:
+            
+            all_locations.add( resized_thumbnail_override )
+            
+        
+        if full_size_thumbnail_override is not None:
+            
+            all_locations.add( full_size_thumbnail_override )
+            
+        
+        all_locations.update( locations_to_file_weights.keys() )
+        all_locations.update( locations_to_fs_thumb_weights.keys() )
+        all_locations.update( locations_to_r_thumb_weights.keys() )
+        
+        all_locations = list( all_locations )
+        
+        all_locations.sort()
+        
+        tuples = []
+        
+        total_ideal_weight = sum( locations_to_ideal_weights.values() )
+        
+        for location in all_locations:
+            
+            pretty_location = location
+            
+            portable_location = HydrusPaths.ConvertAbsPathToPortablePath( location )
+            portable = not os.path.isabs( portable_location )
+            
+            if portable:
+                
+                pretty_portable = 'yes'
+                
+            else:
+                
+                pretty_portable = 'no'
+                
+            
+            if location in locations_to_ideal_weights:
+                
+                ideal_weight = locations_to_ideal_weights[ location ]
+                
+                pretty_ideal_weight = str( ideal_weight )
+                
+            else:
+                
+                ideal_weight = 0
+                
+                pretty_ideal_weight = 'n/a'
+                
+            
+            fp = locations_to_file_weights[ location ] / 256.0
+            tp = locations_to_fs_thumb_weights[ location ] / 256.0
+            rp = locations_to_r_thumb_weights[ location ] / 256.0
+            
+            p = HydrusData.ConvertFloatToPercentage
+            
+            current_usage = ( fp, tp, rp )
+            
+            usages = []
+            
+            if fp > 0:
+                
+                usages.append( p( fp ) + ' files' )
+                
+            
+            if tp > 0 and tp != fp:
+                
+                usages.append( p( tp ) + ' full-size thumbnails' )
+                
+            
+            if rp > 0 and rp != fp:
+                
+                usages.append( p( rp ) + ' resized thumbnails' )
+                
+            
+            pretty_current_usage = ','.join( usages )
+            
+            if location in locations_to_ideal_weights:
+                
+                ideal_fp = locations_to_ideal_weights[ location ] / float( total_ideal_weight )
+                
+            else:
+                
+                ideal_fp = 0.0
+                
+            
+            if full_size_thumbnail_override is not None and location == full_size_thumbnail_override:
+                
+                ideal_tp = 1.0
+                
+            else:
+                
+                ideal_tp = 0.0
+                
+            
+            if resized_thumbnail_override is not None and location == resized_thumbnail_override:
+                
+                ideal_rp = 1.0
+                
+            else:
+                
+                ideal_rp = 0.0
+                
+            
+            ideal_usage = ( ideal_fp, ideal_tp, ideal_rp )
+            
+            usages = []
+            
+            if ideal_fp > 0:
+                
+                usages.append( p( ideal_fp ) + ' files' )
+                
+            
+            if ideal_tp > 0 and ideal_tp != ideal_fp:
+                
+                usages.append( p( ideal_tp ) + ' full-size thumbnails' )
+                
+            
+            if ideal_rp > 0 and ideal_rp != ideal_fp:
+                
+                usages.append( p( ideal_rp ) + ' resized thumbnails' )
+                
+            
+            pretty_ideal_usage = ','.join( usages )
+            
+            display_tuple = ( pretty_location, pretty_portable, pretty_ideal_weight, pretty_ideal_usage, pretty_current_usage )
+            sort_tuple = ( location, portable, ideal_weight, ideal_usage, current_usage )
+            
+            tuples.append( ( display_tuple, sort_tuple ) )
+            
+        
+        return tuples
+        
+    
+    def _Update( self ):
+        
+        approx_total_db_size = self._controller.db.GetApproxTotalFileSize()
+        
+        self._current_db_path_st.SetLabelText( 'database (totalling about ' + HydrusData.ConvertIntToBytes( approx_total_db_size ) + '): ' + self._controller.GetDBDir() )
+        self._current_install_path_st.SetLabelText( 'install: ' + HC.BASE_DIR )
+        
+        service_info = HG.client_controller.Read( 'service_info', CC.COMBINED_LOCAL_FILE_SERVICE_KEY )
+        
+        all_local_files_total_size = service_info[ HC.SERVICE_INFO_TOTAL_SIZE ]
+        
+        approx_total_client_files = all_local_files_total_size * 1.1
+        
+        self._current_media_paths_st.SetLabelText( 'media (totalling about ' + HydrusData.ConvertIntToBytes( approx_total_client_files ) + '):' )
+        
+        self._current_media_locations_listctrl.DeleteAllItems()
+        
+        for ( display_tuple, sort_tuple ) in self._GenerateCurrentMediaTuples():
+            
+            self._current_media_locations_listctrl.Append( display_tuple, sort_tuple )
+            
+        
+    
diff --git a/include/ClientGUITopLevelWindows.py b/include/ClientGUITopLevelWindows.py
index be419527..9ef917a1 100644
--- a/include/ClientGUITopLevelWindows.py
+++ b/include/ClientGUITopLevelWindows.py
@@ -662,7 +662,7 @@ class FrameThatResizes( Frame ):
     
 class FrameThatTakesScrollablePanel( FrameThatResizes ):
     
-    def __init__( self, parent, title, frame_key, float_on_parent = True ):
+    def __init__( self, parent, title, frame_key = 'regular_dialog', float_on_parent = True ):
         
         self._panel = None
         
diff --git a/include/ClientImporting.py b/include/ClientImporting.py
index 10dd7dc6..51aff9c8 100644
--- a/include/ClientImporting.py
+++ b/include/ClientImporting.py
@@ -1353,7 +1353,10 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
         
         self._parser_status = ''
         self._seed_cache_status = ( 'initialising', ( 0, 1 ) )
-        self._file_download_hook = None
+        self._download_control_file_set = None
+        self._download_control_file_clear = None
+        self._download_control_page_set = None
+        self._download_control_page_clear = None
         
         self._lock = threading.Lock()
         
@@ -1398,8 +1401,6 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
     
     def _WorkOnFiles( self, page_key ):
         
-        do_wait = True
-        
         file_url = self._urls_cache.GetNextSeed( CC.STATUS_UNKNOWN )
         
         if file_url is None:
@@ -1409,8 +1410,15 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
         
         try:
             
+            with self._lock:
+                
+                self._RegenerateSeedCacheStatus( page_key )
+                
+            
             ( status, hash ) = HG.client_controller.Read( 'url_status', file_url )
             
+            url_not_known_beforehand = status == CC.STATUS_NEW
+            
             if status == CC.STATUS_DELETED:
                 
                 if not self._import_file_options.GetExcludeDeleted():
@@ -1425,21 +1433,64 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
                 
                 try:
                     
-                    report_hooks = []
+                    network_job = ClientNetworking.NetworkJob( 'GET', file_url, temp_path = temp_path )
                     
                     with self._lock:
                         
-                        if self._file_download_hook is not None:
+                        if self._download_control_file_set is not None:
                             
-                            report_hooks.append( self._file_download_hook )
+                            wx.CallAfter( self._download_control_file_set, network_job )
                             
                         
                     
-                    HG.client_controller.DoHTTP( HC.GET, file_url, report_hooks = report_hooks, temp_path = temp_path )
+                    try:
+                        
+                        HG.client_controller.network_engine.AddJob( network_job )
+                        
+                        while not network_job.IsDone():
+                            
+                            time.sleep( 0.1 )
+                            
+                        
+                    finally:
+                        
+                        if self._download_control_file_clear is not None:
+                            
+                            wx.CallAfter( self._download_control_file_clear )
+                            
+                        
                     
-                    client_files_manager = HG.client_controller.client_files_manager
-                    
-                    ( status, hash ) = client_files_manager.ImportFile( temp_path, import_file_options = self._import_file_options )
+                    if HG.view_shutdown:
+                        
+                        return
+                        
+                    elif network_job.HasError():
+                        
+                        status = CC.STATUS_FAILED
+                        
+                        self._urls_cache.UpdateSeedStatus( file_url, status, note = network_job.GetErrorText() )
+                        
+                        time.sleep( 2 )
+                        
+                    elif network_job.IsCancelled():
+                        
+                        status = CC.STATUS_SKIPPED
+                        
+                        self._urls_cache.UpdateSeedStatus( file_url, status, note = 'cancelled during download!' )
+                        
+                    else:
+                        
+                        ( status, hash ) = HG.client_controller.client_files_manager.ImportFile( temp_path, import_file_options = self._import_file_options )
+                        
+                        self._urls_cache.UpdateSeedStatus( file_url, status )
+                        
+                        if url_not_known_beforehand and hash is not None:
+                            
+                            service_keys_to_content_updates = { CC.COMBINED_LOCAL_FILE_SERVICE_KEY : [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( hash, ( file_url, ) ) ) ] }
+                            
+                            HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
+                            
+                        
                     
                 finally:
                     
@@ -1448,15 +1499,9 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
                 
             else:
                 
-                do_wait = False
+                self._urls_cache.UpdateSeedStatus( file_url, status )
                 
             
-            service_keys_to_content_updates = { CC.COMBINED_LOCAL_FILE_SERVICE_KEY : [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( hash, ( file_url, ) ) ) ] }
-            
-            HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
-            
-            self._urls_cache.UpdateSeedStatus( file_url, status )
-            
             if status in ( CC.STATUS_SUCCESSFUL, CC.STATUS_REDUNDANT ):
                 
                 ( media_result, ) = HG.client_controller.Read( 'media_results', ( hash, ) )
@@ -1477,20 +1522,11 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
             self._urls_cache.UpdateSeedStatus( file_url, status, exception = e )
             
         
-        wx.CallAfter( self._file_download_hook, 1, 0 )
-        
         with self._lock:
             
             self._RegenerateSeedCacheStatus( page_key )
             
         
-        HG.client_controller.pub( 'update_status', page_key )
-        
-        if do_wait:
-            
-            ClientData.WaitPolitely( page_key )
-            
-        
         return True
         
     
@@ -1518,50 +1554,94 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
             
             try:
                 
-                html = HG.client_controller.DoHTTP( HC.GET, page_url )
+                network_job = ClientNetworking.NetworkJob( 'GET', page_url )
                 
-                soup = ClientDownloading.GetSoup( html )
-                
-                #
-                
-                all_links = soup.find_all( 'a' )
-                
-                links_with_images = [ link for link in all_links if len( link.find_all( 'img' ) ) > 0 ]
-                
-                all_linked_images = []
-                
-                for link in all_links:
+                with self._lock:
                     
-                    images = link.find_all( 'img' )
-                    
-                    all_linked_images.extend( images )
+                    if self._download_control_page_set is not None:
+                        
+                        wx.CallAfter( self._download_control_page_set, network_job )
+                        
                     
                 
-                all_images = soup.find_all( 'img' )
-                
-                unlinked_images = [ image for image in all_images if image not in all_linked_images ]
-                
-                #
-                
-                file_urls = []
-                
-                if self._download_image_links:
+                try:
                     
-                    file_urls.extend( [ urlparse.urljoin( page_url, link[ 'href' ] ) for link in links_with_images if link.has_attr( 'href' ) ] )
+                    HG.client_controller.network_engine.AddJob( network_job )
+                    
+                    while not network_job.IsDone():
+                        
+                        time.sleep( 0.1 )
+                        
+                    
+                finally:
+                    
+                    if self._download_control_page_clear is not None:
+                        
+                        wx.CallAfter( self._download_control_page_clear )
+                        
                     
                 
-                if self._download_unlinked_images:
+                if HG.view_shutdown:
                     
-                    file_urls.extend( [ urlparse.urljoin( page_url, image[ 'src' ] ) for image in unlinked_images if image.has_attr( 'src' ) ] )
+                    raise HydrusExceptions.ShutdownException()
+                    
+                elif network_job.HasError():
+                    
+                    e = network_job.GetErrorException()
+                    
+                    raise e
+                    
+                elif network_job.IsCancelled():
+                    
+                    raise Exception( 'Page download cancelled!' )
+                    
+                else:
+                    
+                    html = network_job.GetContent()
+                    
+                    soup = ClientDownloading.GetSoup( html )
+                    
+                    #
+                    
+                    all_links = soup.find_all( 'a' )
+                    
+                    links_with_images = [ link for link in all_links if len( link.find_all( 'img' ) ) > 0 ]
+                    
+                    all_linked_images = []
+                    
+                    for link in all_links:
+                        
+                        images = link.find_all( 'img' )
+                        
+                        all_linked_images.extend( images )
+                        
+                    
+                    all_images = soup.find_all( 'img' )
+                    
+                    unlinked_images = [ image for image in all_images if image not in all_linked_images ]
+                    
+                    #
+                    
+                    file_urls = []
+                    
+                    if self._download_image_links:
+                        
+                        file_urls.extend( [ urlparse.urljoin( page_url, link[ 'href' ] ) for link in links_with_images if link.has_attr( 'href' ) ] )
+                        
+                    
+                    if self._download_unlinked_images:
+                        
+                        file_urls.extend( [ urlparse.urljoin( page_url, image[ 'src' ] ) for image in unlinked_images if image.has_attr( 'src' ) ] )
+                        
+                    
+                    new_urls = [ file_url for file_url in file_urls if not self._urls_cache.HasSeed( file_url ) ]
+                    
+                    num_new = len( new_urls )
+                    
+                    self._urls_cache.AddSeeds( new_urls )
+                    
+                    parser_status = 'page checked OK - ' + HydrusData.ConvertIntToPrettyString( num_new ) + ' new urls'
                     
-                
-                new_urls = [ file_url for file_url in file_urls if not self._urls_cache.HasSeed( file_url ) ]
-                
-                num_new = len( new_urls )
-                
-                self._urls_cache.AddSeeds( new_urls )
-                
-                parser_status = 'page checked OK - ' + HydrusData.ConvertIntToPrettyString( num_new ) + ' new urls'
                 
             except HydrusExceptions.NotFoundException:
                 
@@ -1595,8 +1675,6 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
                     
                 
             
-            ClientData.WaitPolitely( page_key )
-            
         
     
     def _THREADWork( self, page_key ):
@@ -1730,11 +1808,21 @@ class PageOfImagesImport( HydrusSerialisable.SerialisableBase ):
             
         
     
-    def SetDownloadHook( self, hook ):
+    def SetDownloadControlFile( self, download_control ):
         
         with self._lock:
             
-            self._file_download_hook = hook
+            self._download_control_file_set = download_control.SetNetworkJob
+            self._download_control_file_clear = download_control.ClearNetworkJob
+            
+        
+    
+    def SetDownloadControlPage( self, download_control ):
+        
+        with self._lock:
+            
+            self._download_control_page_set = download_control.SetNetworkJob
+            self._download_control_page_clear = download_control.ClearNetworkJob
             
         
     
@@ -2871,7 +2959,11 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
         self._check_period = check_period
         self._last_time_checked = 0
         
-        self._file_download_hook = None
+        self._download_control_file_set = None
+        self._download_control_file_clear = None
+        self._download_control_thread_set = None
+        self._download_control_thread_clear = None
+        
         self._check_now = False
         self._paused = False
         
@@ -2924,8 +3016,6 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
     
     def _WorkOnFiles( self, page_key ):
         
-        do_wait = True
-        
         file_url = self._urls_cache.GetNextSeed( CC.STATUS_UNKNOWN )
         
         if file_url is None:
@@ -2939,6 +3029,7 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
                 
                 self._RegenerateSeedCacheStatus( page_key )
                 
+            
             file_original_filename = self._urls_to_filenames[ file_url ]
             
             downloaded_tags = [ 'filename:' + file_original_filename ]
@@ -2948,6 +3039,8 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
             
             ( status, hash ) = HG.client_controller.Read( 'url_status', file_url )
             
+            url_not_known_beforehand = status == CC.STATUS_NEW
+            
             if status == CC.STATUS_NEW:
                 
                 if file_url in self._urls_to_md5_base64:
@@ -2974,21 +3067,64 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
                 
                 try:
                     
-                    report_hooks = []
+                    network_job = ClientNetworking.NetworkJob( 'GET', file_url, temp_path = temp_path )
                     
                     with self._lock:
                         
-                        if self._file_download_hook is not None:
+                        if self._download_control_file_set is not None:
                             
-                            report_hooks.append( self._file_download_hook )
+                            wx.CallAfter( self._download_control_file_set, network_job )
                             
                         
                     
-                    HG.client_controller.DoHTTP( HC.GET, file_url, report_hooks = report_hooks, temp_path = temp_path )
+                    try:
+                        
+                        HG.client_controller.network_engine.AddJob( network_job )
+                        
+                        while not network_job.IsDone():
+                            
+                            time.sleep( 0.1 )
+                            
+                        
+                    finally:
+                        
+                        if self._download_control_file_clear is not None:
+                            
+                            wx.CallAfter( self._download_control_file_clear )
+                            
+                        
                     
-                    client_files_manager = HG.client_controller.client_files_manager
-                    
-                    ( status, hash ) = client_files_manager.ImportFile( temp_path, import_file_options = self._import_file_options )
+                    if HG.view_shutdown:
+                        
+                        return
+                        
+                    elif network_job.HasError():
+                        
+                        status = CC.STATUS_FAILED
+                        
+                        self._urls_cache.UpdateSeedStatus( file_url, status, note = network_job.GetErrorText() )
+                        
+                        time.sleep( 2 )
+                        
+                    elif network_job.IsCancelled():
+                        
+                        status = CC.STATUS_SKIPPED
+                        
+                        self._urls_cache.UpdateSeedStatus( file_url, status, note = 'cancelled during download!' )
+                        
+                    else:
+                        
+                        ( status, hash ) = HG.client_controller.client_files_manager.ImportFile( temp_path, import_file_options = self._import_file_options )
+                        
+                        self._urls_cache.UpdateSeedStatus( file_url, status )
+                        
+                        if url_not_known_beforehand and hash is not None:
+                            
+                            service_keys_to_content_updates = { CC.COMBINED_LOCAL_FILE_SERVICE_KEY : [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( hash, ( file_url, ) ) ) ] }
+                            
+                            HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
+                            
+                        
                     
                 finally:
                     
@@ -2997,15 +3133,9 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
                 
             else:
                 
-                do_wait = False
+                self._urls_cache.UpdateSeedStatus( file_url, status )
                 
             
-            service_keys_to_content_updates = { CC.COMBINED_LOCAL_FILE_SERVICE_KEY : [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( hash, ( file_url, ) ) ) ] }
-            
-            HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
-            
-            self._urls_cache.UpdateSeedStatus( file_url, status )
-            
             if status in ( CC.STATUS_SUCCESSFUL, CC.STATUS_REDUNDANT ):
                 
                 with self._lock:
@@ -3036,8 +3166,6 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
             self._urls_cache.UpdateSeedStatus( file_url, status, exception = e )
             
         
-        wx.CallAfter( self._file_download_hook, 1, 0 )
-        
         with self._lock:
             
             self._RegenerateSeedCacheStatus( page_key )
@@ -3045,17 +3173,11 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
         
         HG.client_controller.pub( 'update_status', page_key )
         
-        if do_wait:
-            
-            ClientData.WaitPolitely( page_key )
-            
-        
         return True
         
     
     def _WorkOnThread( self, page_key ):
         
-        do_wait = False
         error_occurred = False
         
         with self._lock:
@@ -3077,34 +3199,76 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
                 
                 json_url = ClientDownloading.GetImageboardThreadJSONURL( self._thread_url )
                 
-                do_wait = True
+                network_job = ClientNetworking.NetworkJob( 'GET', json_url )
                 
-                raw_json = HG.client_controller.DoHTTP( HC.GET, json_url )
-                
-                file_infos = ClientDownloading.ParseImageboardFileURLsFromJSON( self._thread_url, raw_json )
-                
-                new_urls = []
-                
-                for ( file_url, file_md5_base64, file_original_filename ) in file_infos:
+                with self._lock:
                     
-                    if not self._urls_cache.HasSeed( file_url ):
+                    if self._download_control_thread_set is not None:
                         
-                        new_urls.append( file_url )
-                        
-                        self._urls_to_filenames[ file_url ] = file_original_filename
-                        
-                        if file_md5_base64 is not None:
-                            
-                            self._urls_to_md5_base64[ file_url ] = file_md5_base64
-                            
+                        wx.CallAfter( self._download_control_thread_set, network_job )
                         
                     
                 
-                self._urls_cache.AddSeeds( new_urls )
+                try:
+                    
+                    HG.client_controller.network_engine.AddJob( network_job )
+                    
+                    while not network_job.IsDone():
+                        
+                        time.sleep( 0.1 )
+                        
+                    
+                finally:
+                    
+                    if self._download_control_thread_clear is not None:
+                        
+                        wx.CallAfter( self._download_control_thread_clear )
+                        
+                    
                 
-                num_new = len( new_urls )
-                
-                watcher_status = 'thread checked OK - ' + HydrusData.ConvertIntToPrettyString( num_new ) + ' new urls'
+                if HG.view_shutdown:
+                    
+                    raise HydrusExceptions.ShutdownException()
+                    
+                elif network_job.HasError():
+                    
+                    e = network_job.GetErrorException()
+                    
+                    raise e
+                    
+                elif network_job.IsCancelled():
+                    
+                    raise Exception( 'Page download cancelled!' )
+                    
+                else:
+                    
+                    raw_json = network_job.GetContent()
+                    
+                    file_infos = ClientDownloading.ParseImageboardFileURLsFromJSON( self._thread_url, raw_json )
+                    
+                    new_urls = []
+                    
+                    for ( file_url, file_md5_base64, file_original_filename ) in file_infos:
+                        
+                        if not self._urls_cache.HasSeed( file_url ):
+                            
+                            new_urls.append( file_url )
+                            
+                            self._urls_to_filenames[ file_url ] = file_original_filename
+                            
+                            if file_md5_base64 is not None:
+                                
+                                self._urls_to_md5_base64[ file_url ] = file_md5_base64
+                                
+                            
+                        
+                    
+                    self._urls_cache.AddSeeds( new_urls )
+                    
+                    num_new = len( new_urls )
+                    
+                    watcher_status = 'thread checked OK - ' + HydrusData.ConvertIntToPrettyString( num_new ) + ' new urls'
+                    
                 
             except HydrusExceptions.NotFoundException:
                 
@@ -3175,11 +3339,6 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
             self._RegenerateSeedCacheStatus( page_key )
             
         
-        if do_wait:
-            
-            ClientData.WaitPolitely( page_key )
-            
-        
         if error_occurred:
             
             time.sleep( 5 )
@@ -3290,11 +3449,21 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
             
         
     
-    def SetDownloadHook( self, hook ):
+    def SetDownloadControlFile( self, download_control ):
         
         with self._lock:
             
-            self._file_download_hook = hook
+            self._download_control_file_set = download_control.SetNetworkJob
+            self._download_control_file_clear = download_control.ClearNetworkJob
+            
+        
+    
+    def SetDownloadControlThread( self, download_control ):
+        
+        with self._lock:
+            
+            self._download_control_thread_set = download_control.SetNetworkJob
+            self._download_control_thread_clear = download_control.ClearNetworkJob
             
         
     
@@ -3353,8 +3522,8 @@ class URLsImport( HydrusSerialisable.SerialisableBase ):
         self._paused = False
         
         self._seed_cache_status = ( 'initialising', ( 0, 1 ) )
-        self._download_control_set = None
-        self._download_control_clear = None
+        self._download_control_file_set = None
+        self._download_control_file_clear = None
         
         self._lock = threading.Lock()
         
@@ -3425,9 +3594,9 @@ class URLsImport( HydrusSerialisable.SerialisableBase ):
                     
                     with self._lock:
                         
-                        if self._download_control_set is not None:
+                        if self._download_control_file_set is not None:
                             
-                            wx.CallAfter( self._download_control_set, network_job )
+                            wx.CallAfter( self._download_control_file_set, network_job )
                             
                         
                     
@@ -3442,9 +3611,9 @@ class URLsImport( HydrusSerialisable.SerialisableBase ):
                         
                     finally:
                         
-                        if self._download_control_clear is not None:
+                        if self._download_control_file_clear is not None:
                             
-                            wx.CallAfter( self._download_control_clear )
+                            wx.CallAfter( self._download_control_file_clear )
                             
                         
                     
@@ -3605,12 +3774,12 @@ class URLsImport( HydrusSerialisable.SerialisableBase ):
             
         
     
-    def SetDownloadControl( self, download_control ):
+    def SetDownloadControlFile( self, download_control ):
         
         with self._lock:
             
-            self._download_control_set = download_control.SetNetworkJob
-            self._download_control_clear = download_control.ClearNetworkJob
+            self._download_control_file_set = download_control.SetNetworkJob
+            self._download_control_file_clear = download_control.ClearNetworkJob
             
         
     
diff --git a/include/ClientNetworking.py b/include/ClientNetworking.py
index 3698970c..1b7b5be7 100644
--- a/include/ClientNetworking.py
+++ b/include/ClientNetworking.py
@@ -90,7 +90,13 @@ def ConvertDomainIntoAllApplicableDomains( domain ):
     
     while domain.count( '.' ) > 0:
         
-        domains.append( domain )
+        # let's discard www.blah.com so we don't end up tracking it separately to blah.com--there's not much point!
+        startswith_www = domain.count( '.' ) > 1 and domain.startswith( 'www' )
+        
+        if not startswith_www:
+            
+            domains.append( domain )
+            
         
         domain = '.'.join( domain.split( '.' )[1:] ) # i.e. strip off the leftmost subdomain maps.google.com -> google.com
         
@@ -1098,7 +1104,7 @@ class NetworkBandwidthManager( HydrusSerialisable.SerialisableBase ):
         self._lock = threading.Lock()
         
         self._network_contexts_to_bandwidth_trackers = collections.defaultdict( HydrusNetworking.BandwidthTracker )
-        self._network_contexts_to_bandwidth_rules = {}
+        self._network_contexts_to_bandwidth_rules = collections.defaultdict( HydrusNetworking.BandwidthRules )
         
         for context_type in [ CC.NETWORK_CONTEXT_GLOBAL, CC.NETWORK_CONTEXT_HYDRUS, CC.NETWORK_CONTEXT_DOMAIN, CC.NETWORK_CONTEXT_DOWNLOADER, CC.NETWORK_CONTEXT_DOWNLOADER_QUERY, CC.NETWORK_CONTEXT_SUBSCRIPTION ]:
             
@@ -1118,7 +1124,8 @@ class NetworkBandwidthManager( HydrusSerialisable.SerialisableBase ):
     
     def _GetSerialisableInfo( self ):
         
-        all_serialisable_trackers = [ ( network_context.GetSerialisableTuple(), tracker.GetSerialisableTuple() ) for ( network_context, tracker ) in self._network_contexts_to_bandwidth_trackers.items() ]
+        # note this discards downloader_query instances, which have page_key-specific identifiers and are temporary, not meant to be hung onto forever, and are generally invisible to the user
+        all_serialisable_trackers = [ ( network_context.GetSerialisableTuple(), tracker.GetSerialisableTuple() ) for ( network_context, tracker ) in self._network_contexts_to_bandwidth_trackers.items() if network_context.context_type != CC.NETWORK_CONTEXT_DOWNLOADER_QUERY ]
         all_serialisable_rules = [ ( network_context.GetSerialisableTuple(), rules.GetSerialisableTuple() ) for ( network_context, rules ) in self._network_contexts_to_bandwidth_rules.items() ]
         
         return ( all_serialisable_trackers, all_serialisable_rules )
@@ -1210,24 +1217,6 @@ class NetworkBandwidthManager( HydrusSerialisable.SerialisableBase ):
             
         
     
-    def GetDomains( self, history_time_delta_threshold = 86400 * 30 ):
-        
-        with self._lock:
-            
-            domains = []
-            
-            for ( network_context, bandwidth_tracker ) in self._network_contexts_to_bandwidth_trackers.items():
-                
-                if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN and bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, history_time_delta_threshold ) > 0:
-                    
-                    domains.append( network_context.content_data )
-                    
-                
-            
-            return domains
-            
-        
-    
     def GetEstimateInfo( self, network_contexts ):
         
         with self._lock:
@@ -1244,6 +1233,29 @@ class NetworkBandwidthManager( HydrusSerialisable.SerialisableBase ):
             
         
     
+    def GetNetworkContextsAndBandwidthTrackersForUser( self, history_time_delta_threshold = 86400 * 30 ):
+        
+        with self._lock:
+            
+            result = []
+            
+            for ( network_context, bandwidth_tracker ) in self._network_contexts_to_bandwidth_trackers.items():
+                
+                if network_context.context_type == CC.NETWORK_CONTEXT_DOWNLOADER_QUERY: # user doesn't want these
+                    
+                    continue
+                    
+                
+                if bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_REQUESTS, history_time_delta_threshold ) > 0:
+                    
+                    result.append( ( network_context, bandwidth_tracker ) )
+                    
+                
+            
+            return result
+            
+        
+    
     def GetTracker( self, network_context ):
         
         with self._lock:
@@ -1332,6 +1344,11 @@ class NetworkContext( HydrusSerialisable.SerialisableBase ):
         return self.__hash__() != other.__hash__()
         
     
+    def __repr__( self ):
+        
+        return self.ToUnicode()
+        
+    
     def _GetSerialisableInfo( self ):
         
         if self.context_data is None:
@@ -1360,6 +1377,18 @@ class NetworkContext( HydrusSerialisable.SerialisableBase ):
             
         
     
+    def ToUnicode( self ):
+        
+        if self.context_data is None:
+            
+            return CC.network_context_type_string_lookup[ self.context_type ] + ' domain'
+            
+        else:
+            
+            return CC.network_context_type_string_lookup[ self.context_type ] + ': ' + HydrusData.ToUnicode( self.context_data )
+            
+        
+    
 HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_CONTEXT ] = NetworkContext
 
 GLOBAL_NETWORK_CONTEXT = NetworkContext( CC.NETWORK_CONTEXT_GLOBAL )
diff --git a/include/HydrusConstants.py b/include/HydrusConstants.py
index c4ce1f1c..136b0a0d 100755
--- a/include/HydrusConstants.py
+++ b/include/HydrusConstants.py
@@ -49,7 +49,7 @@ options = {}
 # Misc
 
 NETWORK_VERSION = 18
-SOFTWARE_VERSION = 263
+SOFTWARE_VERSION = 264
 
 UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )
 
diff --git a/include/HydrusController.py b/include/HydrusController.py
index f78def05..5b29f92e 100644
--- a/include/HydrusController.py
+++ b/include/HydrusController.py
@@ -25,18 +25,18 @@ class HydrusController( object ):
         
         self._name = 'hydrus'
         
-        self._db_dir = db_dir
+        self.db_dir = db_dir
         self._no_daemons = no_daemons
         self._no_wal = no_wal
         
-        self._no_wal_path = os.path.join( self._db_dir, 'no-wal' )
+        self._no_wal_path = os.path.join( self.db_dir, 'no-wal' )
         
         if os.path.exists( self._no_wal_path ):
             
             self._no_wal = True
             
         
-        self._db = None
+        self.db = None
         
         self._model_shutdown = False
         self._view_shutdown = False
@@ -96,7 +96,7 @@ class HydrusController( object ):
     
     def _Read( self, action, *args, **kwargs ):
         
-        result = self._db.Read( action, HC.HIGH_PRIORITY, *args, **kwargs )
+        result = self.db.Read( action, HC.HIGH_PRIORITY, *args, **kwargs )
         
         return result
         
@@ -118,7 +118,7 @@ class HydrusController( object ):
     
     def _Write( self, action, priority, synchronous, *args, **kwargs ):
         
-        result = self._db.Write( action, priority, synchronous, *args, **kwargs )
+        result = self.db.Write( action, priority, synchronous, *args, **kwargs )
         
         return result
         
@@ -182,24 +182,24 @@ class HydrusController( object ):
     
     def DBCurrentlyDoingJob( self ):
         
-        if self._db is None:
+        if self.db is None:
             
             return False
             
         else:
             
-            return self._db.CurrentlyDoingJob()
+            return self.db.CurrentlyDoingJob()
             
         
     
     def GetDBDir( self ):
         
-        return self._db_dir
+        return self.db_dir
         
     
     def GetDBStatus( self ):
         
-        return self._db.GetStatus()
+        return self.db.GetStatus()
         
     
     def GetCache( self, name ):
@@ -231,7 +231,7 @@ class HydrusController( object ):
     
     def InitModel( self ):
         
-        self._db = self._InitDB()
+        self.db = self._InitDB()
         
     
     def InitView( self ):
@@ -248,13 +248,13 @@ class HydrusController( object ):
     
     def IsFirstStart( self ):
         
-        if self._db is None:
+        if self.db is None:
             
             return False
             
         else:
             
-            return self._db.IsFirstStart()
+            return self.db.IsFirstStart()
             
         
     
@@ -287,7 +287,7 @@ class HydrusController( object ):
         
         profile_log_filename = self._name + ' profile - ' + boot_pretty_timestamp + '.log'
         
-        profile_log_path = os.path.join( self._db_dir, profile_log_filename )
+        profile_log_path = os.path.join( self.db_dir, profile_log_filename )
         
         with open( profile_log_path, 'a' ) as f:
             
@@ -333,9 +333,9 @@ class HydrusController( object ):
         self._model_shutdown = True
         HG.model_shutdown = True
         
-        if self._db is not None:
+        if self.db is not None:
             
-            while not self._db.LoopIsFinished(): time.sleep( 0.1 )
+            while not self.db.LoopIsFinished(): time.sleep( 0.1 )
             
         
     
diff --git a/include/HydrusDB.py b/include/HydrusDB.py
index 5c1e0b05..718f4291 100644
--- a/include/HydrusDB.py
+++ b/include/HydrusDB.py
@@ -62,22 +62,6 @@ def CanVacuum( db_path, stop_time = None ):
         return False
         
     
-def SetupDBCreatePragma( c, no_wal = False ):
-    
-    c.execute( 'PRAGMA auto_vacuum = 0;' ) # none
-    
-    if HC.PLATFORM_WINDOWS:
-        
-        c.execute( 'PRAGMA page_size = 4096;' )
-        
-    
-    if not no_wal:
-        
-        c.execute( 'PRAGMA journal_mode = WAL;' )
-        
-    
-    c.execute( 'PRAGMA synchronous = 2;' )
-    
 def VacuumDB( db_path ):
     
     db = sqlite3.connect( db_path, isolation_level = None, detect_types = sqlite3.PARSE_DECLTYPES )
@@ -110,6 +94,8 @@ def VacuumDB( db_path ):
         c.execute( 'PRAGMA page_size = ' + str( ideal_page_size ) + ';' )
         
     
+    c.execute( 'PRAGMA auto_vacuum = 0;' ) # none
+    
     c.execute( 'VACUUM;' )
     
     if previous_journal_mode == 'wal':
@@ -131,6 +117,7 @@ class HydrusDB( object ):
         
         self._transaction_started = 0
         self._in_transaction = False
+        self._transaction_contains_writes = False
         
         self._connection_timestamp = 0
         
@@ -263,34 +250,20 @@ class HydrusDB( object ):
             
             db_path = os.path.join( self._db_dir, self._db_filenames[ name ] )
             
-            if not os.path.exists( db_path ):
-                
-                db = sqlite3.connect( db_path, isolation_level = None, detect_types = sqlite3.PARSE_DECLTYPES )
-                
-                c = db.cursor()
-                
-                SetupDBCreatePragma( c, no_wal = self._no_wal )
-                
-                del c
-                del db
-                
-            
             self._c.execute( 'ATTACH ? AS ' + name + ';', ( db_path, ) )
             
         
     
     def _BeginImmediate( self ):
         
-        if self._in_transaction:
-            
-            HydrusData.Print( 'Received a call to begin, but was already in a transaction!' )
-            
-        else:
+        if not self._in_transaction:
             
             self._c.execute( 'BEGIN IMMEDIATE;' )
+            self._c.execute( 'SAVEPOINT hydrus_savepoint;' )
             
             self._transaction_started = HydrusData.GetNow()
             self._in_transaction = True
+            self._transaction_contains_writes = False
             
         
     
@@ -406,6 +379,10 @@ class HydrusDB( object ):
             
             self._CreateDB()
             
+            self._Commit()
+            
+            self._BeginImmediate()
+            
         
     
     def _InitDBCursor( self ):
@@ -422,6 +399,8 @@ class HydrusDB( object ):
         
         self._c = self._db.cursor()
         
+        self._c.execute( 'PRAGMA temp_store = 2;' )
+        
         self._c.execute( 'PRAGMA main.cache_size = -10000;' )
         
         self._c.execute( 'ATTACH ":memory:" AS mem;' )
@@ -446,7 +425,9 @@ class HydrusDB( object ):
                 
                 self._c.execute( 'PRAGMA ' + db_name + '.journal_mode = WAL;' )
                 
-                # This was 1 previously, but some interrupted commits were rolling back inconsistently across the attached dbs
+                # if this is set to 1, transactions are not immediately synced to the journal and can be undone following a power-loss
+                # if set to 2, all transactions are synced
+                # either way, transactions are atomically consistent, but let's not mess around when power-cut during heavy file import or w/e
                 self._c.execute( 'PRAGMA ' + db_name + '.synchronous = 2;' )
                 
                 try:
@@ -489,6 +470,15 @@ class HydrusDB( object ):
                 
             
         
+        try:
+            
+            self._BeginImmediate()
+            
+        except Exception as e:
+            
+            raise HydrusExceptions.DBAccessException( HydrusData.ToUnicode( e ) )
+            
+        
     
     def _InitDiskCache( self ):
         
@@ -517,9 +507,7 @@ class HydrusDB( object ):
                 
                 self._current_status = 'db write locked'
                 
-                self.publish_status_update()
-                
-                self._BeginImmediate()
+                self._transaction_contains_writes = True
                 
             else:
                 
@@ -537,7 +525,7 @@ class HydrusDB( object ):
                 result = self._Write( action, *args, **kwargs )
                 
             
-            if self._in_transaction:
+            if self._transaction_contains_writes and HydrusData.TimeHasPassed( self._transaction_started + 10 ):
                 
                 self._current_status = 'db committing'
                 
@@ -545,6 +533,12 @@ class HydrusDB( object ):
                 
                 self._Commit()
                 
+                self._BeginImmediate()
+                
+            else:
+                
+                self._Save()
+                
             
             for ( topic, args, kwargs ) in self._pubsubs:
                 
@@ -558,24 +552,23 @@ class HydrusDB( object ):
             
         except Exception as e:
             
-            if self._in_transaction:
+            try:
                 
-                try:
-                    
-                    self._Rollback()
-                    
-                except Exception as rollback_e:
-                    
-                    HydrusData.Print( 'When the transaction failed, attempting to rollback the database failed.' )
-                    
-                    HydrusData.PrintException( rollback_e )
-                    
+                self._Rollback()
+                
+            except Exception as rollback_e:
+                
+                HydrusData.Print( 'When the transaction failed, attempting to rollback the database failed.' )
+                
+                HydrusData.PrintException( rollback_e )
                 
             
             self._ManageDBError( job, e )
             
         finally:
             
+            self._pubsubs = []
+            
             self._current_status = ''
             
             self.publish_status_update()
@@ -596,9 +589,7 @@ class HydrusDB( object ):
         
         if self._in_transaction:
             
-            self._c.execute( 'ROLLBACK;' )
-            
-            self._in_transaction = False
+            self._c.execute( 'ROLLBACK TO hydrus_savepoint;' )
             
         else:
             
@@ -606,6 +597,13 @@ class HydrusDB( object ):
             
         
     
+    def _Save( self ):
+        
+        self._c.execute( 'RELEASE hydrus_savepoint;' )
+        
+        self._c.execute( 'SAVEPOINT hydrus_savepoint;' )
+        
+    
     def _SelectFromList( self, select_statement, xs ):
         
         # issue here is that doing a simple blah_id = ? is real quick and cacheable but doing a lot of fetchone()s is slow
@@ -680,7 +678,7 @@ class HydrusDB( object ):
         raise NotImplementedError()
         
     
-    def pub_after_commit( self, topic, *args, **kwargs ):
+    def pub_after_job( self, topic, *args, **kwargs ):
         
         self._pubsubs.append( ( topic, args, kwargs ) )
         
@@ -695,6 +693,20 @@ class HydrusDB( object ):
         return self._currently_doing_job
         
     
+    def GetApproxTotalFileSize( self ):
+        
+        total = 0
+        
+        for filename in self._db_filenames.values():
+            
+            path = os.path.join( self._db_dir, filename )
+            
+            total += os.path.getsize( path )
+            
+        
+        return total
+        
+    
     def GetStatus( self ):
         
         return ( self._current_status, self._current_job_name )
@@ -754,8 +766,6 @@ class HydrusDB( object ):
                 
                 self.publish_status_update()
                 
-                self._pubsubs = []
-                
                 try:
                     
                     if HG.db_profile_mode:
diff --git a/include/HydrusPaths.py b/include/HydrusPaths.py
index 8fc094ce..4905cee7 100644
--- a/include/HydrusPaths.py
+++ b/include/HydrusPaths.py
@@ -514,7 +514,7 @@ def MirrorFile( source, dest ):
     
     return True
     
-def MirrorTree( source, dest ):
+def MirrorTree( source, dest, text_update_hook = None, is_cancelled_hook = None ):
     
     pauser = HydrusData.BigJobPauser()
     
@@ -524,6 +524,16 @@ def MirrorTree( source, dest ):
     
     for ( root, dirnames, filenames ) in os.walk( source ):
         
+        if is_cancelled_hook is not None and is_cancelled_hook():
+            
+            return
+            
+        
+        if text_update_hook is not None:
+            
+            text_update_hook( 'Copying ' + root + '.' )
+            
+        
         dest_root = root.replace( source, dest )
         
         surplus_dest_paths = { os.path.join( dest_root, dest_filename ) for dest_filename in os.listdir( dest_root ) }
diff --git a/include/HydrusVideoHandling.py b/include/HydrusVideoHandling.py
index bd624c9c..7654033f 100755
--- a/include/HydrusVideoHandling.py
+++ b/include/HydrusVideoHandling.py
@@ -193,7 +193,7 @@ def GetFFMPEGVideoProperties( path, count_frames_manually = False ):
         
         if fps is None:
             
-            raise HydrusExceptions.MimeException( 'Could not determine either the duration or fps!' )
+            fps = 24 # screw it, let's just put one in there
             
         
         if not count_frames_manually:
@@ -205,7 +205,7 @@ def GetFFMPEGVideoProperties( path, count_frames_manually = False ):
         
         num_frames = ParseFFMPEGNumFramesManually( lines )
         
-        duration = num_frames / fps
+        duration = num_frames / float( fps )
         
     else:
         
diff --git a/include/ServerController.py b/include/ServerController.py
index a9cc5f9d..499e6ff6 100755
--- a/include/ServerController.py
+++ b/include/ServerController.py
@@ -167,7 +167,7 @@ class Controller( HydrusController.HydrusController ):
     
     def _InitDB( self ):
         
-        return ServerDB.DB( self, self._db_dir, 'server', no_wal = self._no_wal )
+        return ServerDB.DB( self, self.db_dir, 'server', no_wal = self._no_wal )
         
     
     def StartService( self, service ):
@@ -209,7 +209,7 @@ class Controller( HydrusController.HydrusController ):
                             return
                             
                         
-                        ( ssl_cert_path, ssl_key_path ) = self._db.GetSSLPaths()
+                        ( ssl_cert_path, ssl_key_path ) = self.db.GetSSLPaths()
                         
                         sslmethod = twisted.internet.ssl.SSL.TLSv1_2_METHOD
                         
@@ -272,12 +272,12 @@ class Controller( HydrusController.HydrusController ):
         
         self.ShutdownModel()
         
-        HydrusData.CleanRunningFile( self._db_dir, 'server' )
+        HydrusData.CleanRunningFile( self.db_dir, 'server' )
         
     
     def GetFilesDir( self ):
         
-        return self._db.GetFilesDir()
+        return self.db.GetFilesDir()
         
     
     def GetServerSessionManager( self ):
@@ -375,7 +375,7 @@ class Controller( HydrusController.HydrusController ):
     
     def Run( self ):
         
-        HydrusData.RecordRunningStart( self._db_dir, 'server' )
+        HydrusData.RecordRunningStart( self.db_dir, 'server' )
         
         HydrusData.Print( u'Initialising db\u2026' )
         
diff --git a/include/ServerDB.py b/include/ServerDB.py
index 16a89bcb..f50c9d05 100755
--- a/include/ServerDB.py
+++ b/include/ServerDB.py
@@ -270,8 +270,6 @@ class DB( HydrusDB.HydrusDB ):
     
     def _Backup( self ):
         
-        self._Commit()
-        
         self._CloseDBCursor()
         
         HG.server_busy = True
@@ -325,8 +323,6 @@ class DB( HydrusDB.HydrusDB ):
             
             self._InitDBCursor()
             
-            self._BeginImmediate()
-            
         
         HydrusData.Print( 'backing up: done!' )
         
@@ -342,10 +338,6 @@ class DB( HydrusDB.HydrusDB ):
             HydrusPaths.MakeSureDirectoryExists( new_dir )
             
         
-        HydrusDB.SetupDBCreatePragma( self._c, no_wal = self._no_wal )
-        
-        self._BeginImmediate()
-        
         self._c.execute( 'CREATE TABLE services ( service_id INTEGER PRIMARY KEY, service_key BLOB_BYTES, service_type INTEGER, name TEXT, port INTEGER, dictionary_string TEXT );' )
         
         self._c.execute( 'CREATE TABLE accounts ( account_id INTEGER PRIMARY KEY, service_id INTEGER, account_key BLOB_BYTES, hashed_access_key BLOB_BYTES, account_type_id INTEGER, created INTEGER, expires INTEGER, dictionary_string TEXT );' )
@@ -393,8 +385,6 @@ class DB( HydrusDB.HydrusDB ):
         
         self._AddService( admin_service ) # this sets up the admin account and a registration key by itself
         
-        self._Commit()
-        
     
     def _DeleteOrphans( self ):
         
@@ -1136,7 +1126,7 @@ class DB( HydrusDB.HydrusDB ):
         
         subject_account_keys = [ subject_account.GetAccountKey() for subject_account in subject_accounts ]
         
-        self.pub_after_commit( 'update_session_accounts', service_key, subject_account_keys )
+        self.pub_after_job( 'update_session_accounts', service_key, subject_account_keys )
         
     
     def _ModifyAccountTypes( self, service_key, account, account_types, deletee_account_type_keys_to_new_account_type_keys ):
@@ -1193,7 +1183,7 @@ class DB( HydrusDB.HydrusDB ):
         
         self._RefreshAccountTypeCache( service_id )
         
-        self.pub_after_commit( 'update_all_session_accounts', service_key )
+        self.pub_after_job( 'update_all_session_accounts', service_key )
         
     
     def _ModifyServices( self, account, services ):
@@ -1202,11 +1192,6 @@ class DB( HydrusDB.HydrusDB ):
         
         self._Commit()
         
-        if not self._fast_big_transaction_wal:
-            
-            self._c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
-            
-        
         self._c.execute( 'PRAGMA foreign_keys = ON;' )
         
         self._BeginImmediate()
@@ -1247,12 +1232,10 @@ class DB( HydrusDB.HydrusDB ):
                 
             
         
-        self._Commit()
+        self._CloseDBCursor()
         
         self._InitDBCursor()
         
-        self._BeginImmediate()
-        
         return service_keys_to_access_keys
         
     
@@ -3796,8 +3779,6 @@ class DB( HydrusDB.HydrusDB ):
             
             HydrusData.Print( 'committing to disk' )
             
-            self._Commit()
-            
             self._CloseDBCursor()
             
             try:
@@ -3826,8 +3807,6 @@ class DB( HydrusDB.HydrusDB ):
                 
                 self._InitDBCursor()
                 
-                self._BeginImmediate()
-                
             
             
             for schema in [ 'main', 'external_master', 'external_mappings' ]:
diff --git a/static/refresh.png b/static/refresh.png
new file mode 100644
index 0000000000000000000000000000000000000000..0de26566d4102eec080253c2d08985ec58b14838
GIT binary patch
literal 685
zcmV;e0#f~nP)<h;3K|Lk000e1NJLTq000mG000mO1^@s6AM^iV00004XF*Lt006JZ
zHwB960000PbVXQnQ*UN;cVTj606}DLVr3vnZDD6+Qe|Oed2z{QJOBU!KS@MER5;6x
zliN$vVI0N>`!iy8(2_#ButL^3%VaH2WCpD^U)OZxp@C)2#hU)y+@T%ZNzJigNk%37
zz-WYJwT%teVfiEI+B*@v4ey@58(ld4VY_&5-ox`e@AKg+0U-I`y79bmuw_~y6+4rZ
zBG5EdFDS+@M0OSE`>d7SUDOzKZ&h*4eB1iX7tOd9RiYtW2mQ--bUahxr1`i{RG@dM
zL#}_X=DDO1{;UI$pFu=dLYT_=5d8WC-sLfjr7UO-HKMAwa<oXG=R>=!>)kEhvuwre
zuW3yF@ZxFCkI*+ad|5kOX%5zu8IQjhan)UqgSrFGA_0nQFn@Z08DSEUToCSz4Z1ls
z&fDbq$T&7|6iq$_uDI$@q1_kQ@dfqk*0>{SDL6V)94@)ete)j++*>bIc9sj}Y;R1o
z#OpH+Yt-^4wfv{nern^i<pj6=(tb^usZ6y;N@k3ZGD6Cffs2gHn1otpmp$hgMP?M4
zQDoMi?vff3cTC%b$c!VijelmFWVR;Cosh-|sjzjAkh;li)swan6*~qN?&bT5?|4i|
z1J;ZQ!-<EW$e_re$mo8fg)7MwkUG*MXq#xxBQ+E~CZtX-WyIX?XaQ+Vm?%Ozk{Bu=
z)z03=k?uo0A(a&g5Yo{{ANvto2^Wxtzk4yBb;6agVt8o?J@Y!W_?0-Uuiy!(Tyz#F
zvagTtg#H!FBLyKvjj0;d_p8<^a-5JF2&rt>Vag8<wJek}Xrh~hR7yw(OA+`Bgsg~}
T=<19p00000NkvXXu0mjf<1{{&

literal 0
HcmV?d00001

diff --git a/test.py b/test.py
index a71b3847..fcc0581f 100644
--- a/test.py
+++ b/test.py
@@ -53,14 +53,14 @@ class Controller( object ):
     
     def __init__( self ):
         
-        self._db_dir = tempfile.mkdtemp()
+        self.db_dir = tempfile.mkdtemp()
         
-        TestConstants.DB_DIR = self._db_dir
+        TestConstants.DB_DIR = self.db_dir
         
-        self._server_files_dir = os.path.join( self._db_dir, 'server_files' )
-        self._updates_dir = os.path.join( self._db_dir, 'test_updates' )
+        self._server_files_dir = os.path.join( self.db_dir, 'server_files' )
+        self._updates_dir = os.path.join( self.db_dir, 'test_updates' )
         
-        client_files_default = os.path.join( self._db_dir, 'client_files' )
+        client_files_default = os.path.join( self.db_dir, 'client_files' )
         
         HydrusPaths.MakeSureDirectoryExists( self._server_files_dir )
         HydrusPaths.MakeSureDirectoryExists( self._updates_dir )
@@ -73,7 +73,7 @@ class Controller( object ):
         
         self._pubsub = HydrusPubSub.HydrusPubSub( self )
         
-        self._new_options = ClientData.ClientOptions( self._db_dir )
+        self._new_options = ClientData.ClientOptions( self.db_dir )
         
         def show_text( text ): pass
         
@@ -315,7 +315,7 @@ class Controller( object ):
         
         time.sleep( 2 )
         
-        HydrusPaths.DeletePath( self._db_dir )
+        HydrusPaths.DeletePath( self.db_dir )
         
     
     def ViewIsShutdown( self ):