Version 241

client.py

@@ -38,7 +38,7 @@ try:
     
     if result.db_dir is None:
         
-        db_dir = os.path.join( HC.BASE_DIR, 'db' )
+        db_dir = HC.DEFAULT_DB_DIR
         
     else:
         
@@ -124,4 +124,4 @@ except Exception as e:
         
         print( 'Critical error occured! Details written to crash.log!' )
         
-    
+    

client.pyw

@@ -38,7 +38,7 @@ try:
     
     if result.db_dir is None:
         
-        db_dir = os.path.join( HC.BASE_DIR, 'db' )
+        db_dir = HC.DEFAULT_DB_DIR
         
     else:
         
@@ -124,4 +124,4 @@ except Exception as e:
         
         print( 'Critical error occured! Details written to crash.log!' )
         
-    
+    

help/changelog.html

@@ -8,6 +8,34 @@
 		<div class="content">
 			<h3>changelog</h3>
 			<ul>
+				<li><h3>version 241</h3></li>
+				<ul>
+					<li>fixed the 'setnondupename' problem that was affecting 'add' actions on manage subscriptions, scripts, and import/export folders</li>
+					<li>added some more tests to catch this problem automatically in future</li>
+					<li>cleaned up some similar files phash regeneration logic</li>
+					<li>cleaned up similar files maintenance code to deal with the new duplicates page</li>
+					<li>wrote a similar files duplicate pair search maintenance routine</li>
+					<li>activated file phash regen button on the new duplicates page</li>
+					<li>activated branch rebalancing button on the new duplicates page</li>
+					<li>activated duplicate search button on the new duplicates page</li>
+					<li>search distance on the new duplicates page is now remembered between sessions</li>
+					<li>improved the phash algorithm to use median instead of mean--it now gives fewer apparent false positives and negatives, but I think it may also be stricter in general</li>
+					<li>the duplicate system now discards phashes for blank, flat colour images (this will be more useful when I reintroduce dupe checking for animations, which often start with a black frame)</li>
+					<li>misc phash code cleanup</li>
+					<li>all local jpegs and pngs will be scheduled for phash regeneration on update as their current phashes are legacies of several older versions of the algorithm</li>
+					<li>debuted a cog menu button on the new duplicates page to refresh the page and reset found potential duplicate pairs--this cog should be making appearances elsewhere to add settings and reduce excess buttons</li>
+					<li>improved some search logic that was refreshing too much info on an 'include current/pending tags' button press</li>
+					<li>fixed pixiv login--for now!</li>
+					<li>system:dimensions now catches an enter key event and passes it to the correct ok button, rather than always num_pixels</li>
+					<li>fixed some bad http->https conversion when uploading files to file repo</li>
+					<li>folder deletion will try to deal better with read-only nested files</li>
+					<li>tag parent uploads will now go one at a time (rather than up to 100 as before) to reduce commit lag</li>
+					<li>updated to python 2.7.13 for windows</li>
+					<li>updated to OpenCV 3.2 for windows--this new version does not crash with the same files that 3.1 does, so I recommend windows users turn off 'load images with pil' under options->media if they have it set</li>
+					<li>I think I improved some unicode error handling</li>
+					<li>added LICENSE_PATH and harmonised various instances of default db dir creation to DEFAULT_DB_DIR, both in HydrusConstants</li>
+					<li>misc code cleanup and bitmap button cleanup</li>
+				</ul>
 				<li><h3>version 240</h3></li>
 				<ul>
 					<li>improved how the client analyzes itself, reducing maintenance latency and also overall cpu usage. syncing a big repo will no longer introduce lingering large lag, and huge analyze jobs will run significantly less frequently</li>

include/ClientCaches.py

@@ -1,4 +1,5 @@
 import ClientDefaults
+import ClientDownloading
 import ClientFiles
 import ClientNetworking
 import ClientRendering
@@ -11,9 +12,11 @@ import HydrusImageHandling
 import HydrusPaths
 import HydrusSessions
 import itertools
+import json
 import os
 import random
 import Queue
+import requests
 import shutil
 import threading
 import time
@@ -2947,22 +2950,9 @@ class WebSessionManagerClient( object ):
                     raise HydrusExceptions.DataMissing( 'You need to set up your pixiv credentials in services->manage pixiv account.' )
                     
                 
-                ( id, password ) = result
+                ( pixiv_id, password ) = result
                 
-                form_fields = {}
+                cookies = self.GetPixivCookies( pixiv_id, password )
-                
-                form_fields[ 'pixiv_id' ] = id
-                form_fields[ 'password' ] = password
-                
-                body = urllib.urlencode( form_fields )
-                
-                headers = {}
-                headers[ 'Content-Type' ] = 'application/x-www-form-urlencoded'
-                
-                ( response_gumpf, cookies ) = self._controller.DoHTTP( HC.POST, 'http://www.pixiv.net/login.php', request_headers = headers, body = body, return_cookies = True )
-                
-                # _ only given to logged in php sessions
-                if 'PHPSESSID' not in cookies: raise Exception( 'Pixiv login credentials not accepted!' )
                 
                 expires = now + 30 * 86400
                 
@@ -2975,3 +2965,60 @@ class WebSessionManagerClient( object ):
             
         
     
+    # This updated login form is cobbled together from the example in PixivUtil2
+    # it is breddy shid because I'm not using mechanize or similar browser emulation (like requests's sessions) yet
+    # Pixiv 400s if cookies and referrers aren't passed correctly
+    # I am leaving this as a mess with the hope the eventual login engine will replace it
+    def GetPixivCookies( self, pixiv_id, password ):
+        
+        ( response, cookies ) = self._controller.DoHTTP( HC.GET, 'https://accounts.pixiv.net/login', return_cookies = True )
+        
+        soup = ClientDownloading.GetSoup( response )
+        
+        # some whocking 20kb bit of json tucked inside a hidden form input wew lad
+        i = soup.find( 'input', id = 'init-config' )
+        
+        raw_json = i['value']
+        
+        j = json.loads( raw_json )
+        
+        if 'pixivAccount.postKey' not in j:
+            
+            raise HydrusExceptions.ForbiddenException( 'When trying to log into Pixiv, I could not find the POST key!' )
+            
+        
+        post_key = j[ 'pixivAccount.postKey' ]
+        
+        form_fields = {}
+        
+        form_fields[ 'pixiv_id' ] = pixiv_id
+        form_fields[ 'password' ] = password
+        form_fields[ 'captcha' ] = ''
+        form_fields[ 'g_recaptcha_response' ] = ''
+        form_fields[ 'return_to' ] = 'http://www.pixiv.net'
+        form_fields[ 'lang' ] = 'en'
+        form_fields[ 'post_key' ] = post_key
+        form_fields[ 'source' ] = 'pc'
+        
+        headers = {}
+        
+        headers[ 'referer' ] = "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
+        headers[ 'origin' ] = "https://accounts.pixiv.net"
+        ClientNetworking.AddCookiesToHeaders( cookies, headers )
+        
+        r = requests.post( 'https://accounts.pixiv.net/api/login?lang=en', data = form_fields, headers = headers )
+        
+        # doesn't work
+        #( response_gumpf, cookies ) = self._controller.DoHTTP( HC.POST, 'https://accounts.pixiv.net/api/login?lang=en', request_headers = headers, body = body, return_cookies = True )
+        
+        cookies = dict( r.cookies )
+        
+        # _ only given to logged-in php sessions
+        if 'PHPSESSID' not in cookies or '_' not in cookies[ 'PHPSESSID' ]:
+            
+            raise HydrusExceptions.ForbiddenException( 'Pixiv login credentials not accepted!' )
+            
+        
+        return cookies
+        
+    

include/ClientConstants.py

@@ -16,6 +16,8 @@ ID_TIMER_UPDATES = wx.NewId()
 
 #
 
+BLANK_PHASH = '\x80\x00\x00\x00\x00\x00\x00\x00' # first bit 1 but everything else 0 means only significant part of dct was [0,0], which represents flat colour
+
 CAN_HIDE_MOUSE = True
 
 # Hue is generally 200, Sat and Lum changes based on need
@@ -433,6 +435,8 @@ class GlobalBMPs( object ):
         GlobalBMPs.dump_recoverable = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'dump_recoverable.png' ) )
         GlobalBMPs.dump_fail = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'dump_fail.png' ) )
         
+        GlobalBMPs.cog = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'cog.png' ) )
+        GlobalBMPs.check = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'check.png' ) )
         GlobalBMPs.pause = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'pause.png' ) )
         GlobalBMPs.play = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'play.png' ) )
         GlobalBMPs.stop = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'stop.png' ) )

include/ClientDB.py

@@ -1596,29 +1596,62 @@ class DB( HydrusDB.HydrusDB ):
         self._c.execute( 'INSERT INTO shape_vptree ( phash_id, parent_id, radius, inner_id, inner_population, outer_id, outer_population ) VALUES ( ?, ?, ?, ?, ?, ?, ? );', ( phash_id, parent_id, radius, inner_id, inner_population, outer_id, outer_population ) )
         
     
-    def _CacheSimilarFilesDelete( self, hash_id, phash_ids = None ):
+    def _CacheSimilarFilesAssociatePHashes( self, hash_id, phashes ):
         
-        if phash_ids is None:
+        phash_ids = set()
+        
+        for phash in phashes:
             
-            phash_ids = { phash_id for ( phash_id, ) in self._c.execute( 'SELECT phash_id FROM shape_perceptual_hash_map WHERE hash_id = ?;', ( hash_id, ) ) }
+            phash_id = self._CacheSimilarFilesGetPHashId( phash )
             
-            self._c.execute( 'DELETE FROM shape_perceptual_hash_map WHERE hash_id = ?;', ( hash_id, ) )
+            phash_ids.add( phash_id )
-            
-        else:
-            
-            phash_ids = set( phash_ids )
-            
-            self._c.executemany( 'DELETE FROM shape_perceptual_hash_map WHERE phash_id = ? AND hash_id = ?;', ( ( phash_id, hash_id ) for phash_id in phash_ids ) )
             
         
-        useful_phash_ids = { phash for ( phash, ) in self._c.execute( 'SELECT phash_id FROM shape_perceptual_hash_map WHERE phash_id IN ' + HydrusData.SplayListForDB( phash_ids ) + ';' ) }
+        self._c.executemany( 'INSERT OR IGNORE INTO shape_perceptual_hash_map ( phash_id, hash_id ) VALUES ( ?, ? );', ( ( phash_id, hash_id ) for phash_id in phash_ids ) )
         
-        deletee_phash_ids = phash_ids.difference( useful_phash_ids )
+        if self._GetRowCount() > 0:
+            
+            self._c.execute( 'REPLACE INTO shape_search_cache ( hash_id, searched_distance ) VALUES ( ?, ? );', ( hash_id, None ) )
+            
         
-        self._c.executemany( 'INSERT OR IGNORE INTO shape_maintenance_branch_regen ( phash_id ) VALUES ( ? );', ( ( phash_id, ) for phash_id in deletee_phash_ids ) )
+        return phash_ids
+        
+    
+    def _CacheSimilarFilesDeleteFile( self, hash_id ):
+        
+        phash_ids = { phash_id for ( phash_id, ) in self._c.execute( 'SELECT phash_id FROM shape_perceptual_hash_map WHERE hash_id = ?;', ( hash_id, ) ) }
+        
+        self._CacheSimilarFilesDisassociatePHashes( hash_id, phash_ids )
         
         self._c.execute( 'DELETE FROM shape_search_cache WHERE hash_id = ?;', ( hash_id, ) )
         self._c.execute( 'DELETE FROM duplicate_pairs WHERE smaller_hash_id = ? or larger_hash_id = ?;', ( hash_id, hash_id ) )
+        self._c.execute( 'DELETE FROM shape_maintenance_phash_regen WHERE hash_id = ?;', ( hash_id, ) )
+        
+    
+    def _CacheSimilarFilesDeleteUnknownDuplicatePairs( self ):
+        
+        hash_ids = set()
+        
+        for ( smaller_hash_id, larger_hash_id ) in self._c.execute( 'SELECT smaller_hash_id, larger_hash_id FROM duplicate_pairs WHERE duplicate_type = ?;', ( HC.DUPLICATE_UNKNOWN, ) ):
+            
+            hash_ids.add( smaller_hash_id )
+            hash_ids.add( larger_hash_id )
+            
+        
+        self._c.execute( 'DELETE FROM duplicate_pairs WHERE duplicate_type = ?;', ( HC.DUPLICATE_UNKNOWN, ) )
+        
+        self._c.executemany( 'UPDATE shape_search_cache SET searched_distance = NULL WHERE hash_id = ?;', ( ( hash_id, ) for hash_id in hash_ids ) )
+        
+    
+    def _CacheSimilarFilesDisassociatePHashes( self, hash_id, phash_ids ):
+        
+        self._c.executemany( 'DELETE FROM shape_perceptual_hash_map WHERE phash_id = ? AND hash_id = ?;', ( ( phash_id, hash_id ) for phash_id in phash_ids ) )
+        
+        useful_phash_ids = { phash for ( phash, ) in self._c.execute( 'SELECT phash_id FROM shape_perceptual_hash_map WHERE phash_id IN ' + HydrusData.SplayListForDB( phash_ids ) + ';' ) }
+        
+        useless_phash_ids = phash_ids.difference( useful_phash_ids )
+        
+        self._c.executemany( 'INSERT OR IGNORE INTO shape_maintenance_branch_regen ( phash_id ) VALUES ( ? );', ( ( phash_id, ) for phash_id in useless_phash_ids ) )
         
     
     def _CacheSimilarFilesGenerateBranch( self, job_key, parent_id, phash_id, phash, children ):
@@ -1739,42 +1772,99 @@ class DB( HydrusDB.HydrusDB ):
         return ( searched_distances_to_count, duplicate_types_to_count, num_phashes_to_regen, num_branches_to_regen )
         
     
-    def _CacheSimilarFilesAssociatePHashes( self, hash_id, phashes ):
+    def _CacheSimilarFilesMaintainDuplicatePairs( self, search_distance, job_key = None, stop_time = None ):
         
-        phash_ids = set()
+        pub_job_key = False
+        job_key_pubbed = False
         
-        for phash in phashes:
+        if job_key is None:
             
-            phash_id = self._CacheSimilarFilesGetPHashId( phash )
+            job_key = ClientThreading.JobKey( cancellable = True )
             
-            self._c.execute( 'INSERT OR IGNORE INTO shape_perceptual_hash_map ( phash_id, hash_id ) VALUES ( ?, ? );', ( phash_id, hash_id ) )
+            pub_job_key = True
-            
-            phash_ids.add( phash_id )
             
         
-        self._c.execute( 'REPLACE INTO shape_search_cache ( hash_id, searched_distance ) VALUES ( ?, ? );', ( hash_id, None ) )
+        hash_ids = [ hash_id for ( hash_id, ) in self._c.execute( 'SELECT hash_id FROM shape_search_cache WHERE searched_distance IS NULL or searched_distance < ?;', ( search_distance, ) ) ]
         
-        return phash_ids
+        pairs_found = 0
+        
+        num_to_do = len( hash_ids )
+        
+        for ( i, hash_id ) in enumerate( hash_ids ):
+            
+            job_key.SetVariable( 'popup_title', 'similar files duplicate pair discovery' )
+            
+            if pub_job_key and not job_key_pubbed:
+                
+                self._controller.pub( 'message', job_key )
+                
+                job_key_pubbed = True
+                
+            
+            ( i_paused, should_quit ) = job_key.WaitIfNeeded()
+            
+            should_stop = stop_time is not None and HydrusData.TimeHasPassed( stop_time )
+            
+            if should_quit or should_stop:
+                
+                return
+                
+            
+            text = 'searched ' + HydrusData.ConvertValueRangeToPrettyString( i, num_to_do ) + ', found ' + HydrusData.ConvertIntToPrettyString( pairs_found )
+            
+            job_key.SetVariable( 'popup_text_1', text )
+            job_key.SetVariable( 'popup_gauge_1', ( i, num_to_do ) )
+            
+            duplicate_hash_ids = [ duplicate_hash_id for duplicate_hash_id in self._CacheSimilarFilesSearch( hash_id, search_distance ) if duplicate_hash_id != hash_id ]
+            
+            self._c.executemany( 'INSERT OR IGNORE INTO duplicate_pairs ( smaller_hash_id, larger_hash_id, duplicate_type ) VALUES ( ?, ?, ? );', ( ( min( hash_id, duplicate_hash_id ), max( hash_id, duplicate_hash_id ), HC.DUPLICATE_UNKNOWN ) for duplicate_hash_id in duplicate_hash_ids ) )
+            
+            pairs_found += self._GetRowCount()
+            
+            self._c.execute( 'UPDATE shape_search_cache SET searched_distance = ? WHERE hash_id = ?;', ( search_distance, hash_id ) )
+            
+        
+        job_key.SetVariable( 'popup_text_1', 'done!' )
+        job_key.DeleteVariable( 'popup_gauge_1' )
+        
+        job_key.Finish()
+        job_key.Delete( 30 )
         
     
-    def _CacheSimilarFilesMaintainFiles( self, job_key ):
+    def _CacheSimilarFilesMaintainFiles( self, job_key = None, stop_time = None ):
         
-        # this should take a cancellable job_key from the gui filter window
+        pub_job_key = False
+        job_key_pubbed = False
+        
+        if job_key is None:
+            
+            job_key = ClientThreading.JobKey( cancellable = True )
+            
+            pub_job_key = True
+            
         
         hash_ids = [ hash_id for ( hash_id, ) in self._c.execute( 'SELECT hash_id FROM shape_maintenance_phash_regen;' ) ]
         
-        # remove hash_id from the pairs cache?
-        # set its search status to False, but don't remove any existing pairs
-        
         client_files_manager = self._controller.GetClientFilesManager()
         
         num_to_do = len( hash_ids )
         
         for ( i, hash_id ) in enumerate( hash_ids ):
             
+            job_key.SetVariable( 'popup_title', 'similar files metadata maintenance' )
+            
+            if pub_job_key and not job_key_pubbed:
+                
+                self._controller.pub( 'message', job_key )
+                
+                job_key_pubbed = True
+                
+            
             ( i_paused, should_quit ) = job_key.WaitIfNeeded()
             
-            if should_quit:
+            should_stop = stop_time is not None and HydrusData.TimeHasPassed( stop_time )
+            
+            if should_quit or should_stop:
                 
                 return
                 
@@ -1823,31 +1913,44 @@ class DB( HydrusDB.HydrusDB ):
             
             correct_phash_ids = self._CacheSimilarFilesAssociatePHashes( hash_id, phashes )
             
-            deletee_phash_ids = existing_phash_ids.difference( correct_phash_ids )
+            incorrect_phash_ids = existing_phash_ids.difference( correct_phash_ids )
             
-            self._CacheSimilarFilesDelete( hash_id, deletee_phash_ids )
+            if len( incorrect_phash_ids ) > 0:
+                
+                self._CacheSimilarFilesDisassociatePHashes( hash_id, incorrect_phash_ids )
+                
             
             self._c.execute( 'DELETE FROM shape_maintenance_phash_regen WHERE hash_id = ?;', ( hash_id, ) )
             
         
+        job_key.SetVariable( 'popup_text_1', 'done!' )
+        job_key.DeleteVariable( 'popup_gauge_1' )
+        
         job_key.Finish()
+        job_key.Delete( 30 )
         
     
-    def _CacheSimilarFilesMaintainTree( self, stop_time ):
+    def _CacheSimilarFilesMaintainTree( self, job_key = None, stop_time = None ):
         
-        job_key = ClientThreading.JobKey( cancellable = True )
+        pub_job_key = False
+        job_key_pubbed = False
+        
+        if job_key is None:
+            
+            job_key = ClientThreading.JobKey( cancellable = True )
+            
+            pub_job_key = True
+            
         
         job_key.SetVariable( 'popup_title', 'similar files metadata maintenance' )
         
-        job_key_pubbed = False
-        
         rebalance_phash_ids = [ phash_id for ( phash_id, ) in self._c.execute( 'SELECT phash_id FROM shape_maintenance_branch_regen;' ) ]
         
         num_to_do = len( rebalance_phash_ids )
         
         while len( rebalance_phash_ids ) > 0:
             
-            if not job_key_pubbed:
+            if pub_job_key and not job_key_pubbed:
                 
                 self._controller.pub( 'message', job_key )
                 
@@ -1856,14 +1959,16 @@ class DB( HydrusDB.HydrusDB ):
             
             ( i_paused, should_quit ) = job_key.WaitIfNeeded()
             
-            if should_quit or HydrusData.TimeHasPassed( stop_time ):
+            should_stop = stop_time is not None and HydrusData.TimeHasPassed( stop_time )
+            
+            if should_quit or should_stop:
                 
                 return
                 
             
             num_done = num_to_do - len( rebalance_phash_ids )
             
-            text = 'regenerating unbalanced similar file search data - ' + HydrusData.ConvertValueRangeToPrettyString( num_done, num_to_do )
+            text = 'rebalancing similar file metadata - ' + HydrusData.ConvertValueRangeToPrettyString( num_done, num_to_do )
             
             HydrusGlobals.client_controller.pub( 'splash_set_status_text', text )
             job_key.SetVariable( 'popup_text_1', text )
@@ -1881,7 +1986,7 @@ class DB( HydrusDB.HydrusDB ):
         
         job_key.SetVariable( 'popup_text_1', 'done!' )
         job_key.DeleteVariable( 'popup_gauge_1' )
-        job_key.DeleteVariable( 'popup_text_2' )
+        job_key.DeleteVariable( 'popup_text_2' ) # used in the regenbranch call
         
         job_key.Finish()
         job_key.Delete( 30 )
@@ -3052,7 +3157,7 @@ class DB( HydrusDB.HydrusDB ):
         
         for hash_id in hash_ids:
             
-            self._CacheSimilarFilesDelete( hash_id )
+            self._CacheSimilarFilesDeleteFile( hash_id )
             
         
     
@@ -5144,7 +5249,7 @@ class DB( HydrusDB.HydrusDB ):
             
             # tag parents
             
-            pending = [ ( ( self._GetNamespaceTag( child_namespace_id, child_tag_id ), self._GetNamespaceTag( parent_namespace_id, parent_tag_id ) ), self._GetText( reason_id ) ) for ( child_namespace_id, child_tag_id, parent_namespace_id, parent_tag_id, reason_id ) in self._c.execute( 'SELECT child_namespace_id, child_tag_id, parent_namespace_id, parent_tag_id, reason_id FROM tag_parent_petitions WHERE service_id = ? AND status = ? ORDER BY reason_id LIMIT 100;', ( service_id, HC.PENDING ) ).fetchall() ]
+            pending = [ ( ( self._GetNamespaceTag( child_namespace_id, child_tag_id ), self._GetNamespaceTag( parent_namespace_id, parent_tag_id ) ), self._GetText( reason_id ) ) for ( child_namespace_id, child_tag_id, parent_namespace_id, parent_tag_id, reason_id ) in self._c.execute( 'SELECT child_namespace_id, child_tag_id, parent_namespace_id, parent_tag_id, reason_id FROM tag_parent_petitions WHERE service_id = ? AND status = ? ORDER BY reason_id LIMIT 1;', ( service_id, HC.PENDING ) ).fetchall() ]
             
             if len( pending ) > 0:
                 
@@ -5209,7 +5314,6 @@ class DB( HydrusDB.HydrusDB ):
                 
             
         
-        
         if len( content_data_dict ) > 0:
             
             hash_ids_to_hashes = self._GetHashIdsToHashes( all_hash_ids )
@@ -8855,7 +8959,18 @@ class DB( HydrusDB.HydrusDB ):
             
             combined_local_file_service_id = self._GetServiceId( CC.COMBINED_LOCAL_FILE_SERVICE_KEY )
             
-            self._c.execute( 'INSERT OR IGNORE INTO shape_search_cache SELECT hash_id, NULL FROM current_files, files_info USING ( hash_id ) WHERE service_id = ? and mime IN ' + HydrusData.SplayListForDB( HC.MIMES_WE_CAN_PHASH ) + ';', ( combined_local_file_service_id, ) )
+            self._c.execute( 'INSERT OR IGNORE INTO shape_search_cache SELECT hash_id, NULL FROM current_files, files_info USING ( hash_id ) WHERE service_id = ? and mime IN ( ?, ? );', ( combined_local_file_service_id, HC.IMAGE_JPEG, HC.IMAGE_PNG ) )
+            
+        
+        if version == 240:
+            
+            combined_local_file_service_id = self._GetServiceId( CC.COMBINED_LOCAL_FILE_SERVICE_KEY )
+            
+            self._c.execute( 'INSERT OR IGNORE INTO shape_maintenance_phash_regen SELECT hash_id FROM current_files, files_info USING ( hash_id ) WHERE service_id = ? AND mime IN ( ?, ? );', ( combined_local_file_service_id, HC.IMAGE_JPEG, HC.IMAGE_PNG ) )
+            
+            #
+            
+            self._c.execute( 'DELETE FROM web_sessions WHERE name = ?;', ( 'pixiv', ) )
             
         
         self._controller.pub( 'splash_set_title_text', 'updated db to v' + str( version + 1 ) )
@@ -9505,12 +9620,15 @@ class DB( HydrusDB.HydrusDB ):
         elif action == 'delete_remote_booru': result = self._DeleteYAMLDump( YAML_DUMP_ID_REMOTE_BOORU, *args, **kwargs )
         elif action == 'delete_serialisable_named': result = self._DeleteJSONDumpNamed( *args, **kwargs )
         elif action == 'delete_service_info': result = self._DeleteServiceInfo( *args, **kwargs )
+        elif action == 'delete_unknown_duplicate_pairs': result = self._CacheSimilarFilesDeleteUnknownDuplicatePairs( *args, **kwargs )
         elif action == 'export_mappings': result = self._ExportToTagArchive( *args, **kwargs )
         elif action == 'file_integrity': result = self._CheckFileIntegrity( *args, **kwargs )
         elif action == 'hydrus_session': result = self._AddHydrusSession( *args, **kwargs )
         elif action == 'imageboard': result = self._SetYAMLDump( YAML_DUMP_ID_IMAGEBOARD, *args, **kwargs )
         elif action == 'import_file': result = self._ImportFile( *args, **kwargs )
         elif action == 'local_booru_share': result = self._SetYAMLDump( YAML_DUMP_ID_LOCAL_BOORU, *args, **kwargs )
+        elif action == 'maintain_similar_files_duplicate_pairs': result = self._CacheSimilarFilesMaintainDuplicatePairs( *args, **kwargs )
+        elif action == 'maintain_similar_files_phashes': result = self._CacheSimilarFilesMaintainFiles( *args, **kwargs )
         elif action == 'maintain_similar_files_tree': result = self._CacheSimilarFilesMaintainTree( *args, **kwargs )
         elif action == 'push_recent_tags': result = self._PushRecentTags( *args, **kwargs )
         elif action == 'regenerate_ac_cache': result = self._RegenerateACCache( *args, **kwargs )

include/ClientData.py

@@ -55,6 +55,8 @@ def CatchExceptionClient( etype, value, tb ):
             first_line = pretty_value
             
         
+        trace = HydrusData.ToUnicode( trace )
+        
         job_key = ClientThreading.JobKey()
         
         if etype == HydrusExceptions.ShutdownException:
@@ -360,6 +362,8 @@ def ShowExceptionClient( e ):
         trace = ''.join( traceback.format_exception( etype, value, tb ) )
         
     
+    trace = HydrusData.ToUnicode( trace )
+    
     pretty_value = HydrusData.ToUnicode( value )
     
     if os.linesep in pretty_value:
@@ -512,7 +516,7 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
         
         if db_dir is None:
             
-            db_dir = os.path.join( HC.BASE_DIR, 'db' )
+            db_dir = HC.DEFAULT_DB_DIR
             
         
         self._dictionary = HydrusSerialisable.SerialisableDictionary()
@@ -570,6 +574,8 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
         
         self._dictionary[ 'integers' ][ 'suggested_tags_width' ] = 300
         
+        self._dictionary[ 'integers' ][ 'similar_files_duplicate_pairs_search_distance' ] = 0
+        
         #
         
         self._dictionary[ 'keys' ] = {}

include/ClientGUI.py

@@ -224,7 +224,7 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
         
         aboutinfo.SetDescription( description )
         
-        with open( os.path.join( HC.BASE_DIR, 'license.txt' ), 'rb' ) as f: license = f.read()
+        with open( HC.LICENSE_PATH, 'rb' ) as f: license = f.read()
         
         aboutinfo.SetLicense( license )
         
@@ -1541,7 +1541,7 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
                 
                 stop_time = HydrusData.GetNow() + 60 * 10
                 
-                self._controller.Write( 'maintain_similar_files_tree', stop_time )
+                self._controller.Write( 'maintain_similar_files_tree', stop_time = stop_time )
                 
             
         

include/ClientGUIACDropdown.py

@@ -1052,10 +1052,10 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
             
             self._file_search_context.SetIncludeCurrentTags( value )
             
-        
+            wx.CallAfter( self.RefreshList )
-        wx.CallAfter( self.RefreshList )
+            
-        
+            HydrusGlobals.client_controller.pub( 'refresh_query', self._page_key )
-        HydrusGlobals.client_controller.pub( 'refresh_query', self._page_key )
+            
         
     
     def IncludePending( self, page_key, value ):
@@ -1064,10 +1064,10 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
             
             self._file_search_context.SetIncludePendingTags( value )
             
-        
+            wx.CallAfter( self.RefreshList )
-        wx.CallAfter( self.RefreshList )
+            
-        
+            HydrusGlobals.client_controller.pub( 'refresh_query', self._page_key )
-        HydrusGlobals.client_controller.pub( 'refresh_query', self._page_key )
+            
         
     
     def SetSynchronisedWait( self, page_key ):

include/ClientGUICommon.py

@@ -185,6 +185,23 @@ class AnimatedStaticTextTimestamp( wx.StaticText ):
             
         
     
+class BetterBitmapButton( wx.BitmapButton ):
+    
+    def __init__( self, parent, bitmap, callable, *args, **kwargs ):
+        
+        wx.BitmapButton.__init__( self, parent, bitmap = bitmap )
+        
+        self._callable = callable
+        self._args = args
+        self._kwargs = kwargs
+        self.Bind( wx.EVT_BUTTON, self.EventButton )
+        
+    
+    def EventButton( self, event ):
+        
+        self._callable( *self._args,  **self._kwargs )
+        
+    
 class BetterButton( wx.Button ):
     
     def __init__( self, parent, label, callable, *args, **kwargs ):
@@ -3207,6 +3224,27 @@ class ListCtrlAutoWidth( wx.ListCtrl, ListCtrlAutoWidthMixin ):
         for index in indices: self.DeleteItem( index )
         
     
+class MenuBitmapButton( BetterBitmapButton ):
+    
+    def __init__( self, parent, bitmap, menu_items ):
+        
+        BetterBitmapButton.__init__( self, parent, bitmap, self.DoMenu )
+        
+        self._menu_items = menu_items
+        
+    
+    def DoMenu( self ):
+        
+        menu = wx.Menu()
+        
+        for ( title, description, callable ) in self._menu_items:
+            
+            ClientGUIMenus.AppendMenuItem( menu, title, description, self, callable )
+            
+        
+        HydrusGlobals.client_controller.PopupMenu( self, menu )
+        
+    
 class MenuButton( BetterButton ):
     
     def __init__( self, parent, label, menu_items ):
@@ -5186,7 +5224,7 @@ class SaneListCtrlForSingleObject( SaneListCtrl ):
         SaneListCtrl.Append( self, display_tuple, sort_tuple )
         
     
-    def GetIndexFromClientData( self, obj ):
+    def GetIndexFromObject( self, obj ):
         
         try:
             
@@ -5227,11 +5265,11 @@ class SaneListCtrlForSingleObject( SaneListCtrl ):
         return datas
         
     
-    def HasClientData( self, data ):
+    def HasObject( self, obj ):
         
         try:
             
-            index = self.GetIndexFromClientData( data )
+            index = self.GetIndexFromObject( obj )
             
             return True
             
@@ -5247,7 +5285,7 @@ class SaneListCtrlForSingleObject( SaneListCtrl ):
         
         name = obj.GetName()
         
-        current_names = { obj.GetName() for obj in self.GetClientData() }
+        current_names = { obj.GetName() for obj in self.GetObjects() }
         
         if name in current_names:
             
@@ -5407,9 +5445,9 @@ class SeedCacheControl( SaneListCtrlForSingleObject ):
         
         if self._seed_cache.HasSeed( seed ):
             
-            if self.HasClientData( seed ):
+            if self.HasObject( seed ):
                 
-                index = self.GetIndexFromClientData( seed )
+                index = self.GetIndexFromObject( seed )
                 
                 ( display_tuple, sort_tuple ) = self._GetListCtrlTuples( seed )
                 
@@ -5422,9 +5460,9 @@ class SeedCacheControl( SaneListCtrlForSingleObject ):
             
         else:
             
-            if self.HasClientData( seed ):
+            if self.HasObject( seed ):
                 
-                index = self.GetIndexFromClientData( seed )
+                index = self.GetIndexFromObject( seed )
                 
                 self.DeleteItem( index )
                 

include/ClientGUIDialogs.py

@@ -1327,7 +1327,6 @@ class DialogInputFileSystemPredicates( Dialog ):
             self._predicate_panel = predicate_class( self )
             
             self._ok = wx.Button( self, id = wx.ID_OK, label = 'Ok' )
-            self._ok.SetDefault()
             self._ok.Bind( wx.EVT_BUTTON, self.EventOK )
             self._ok.SetForegroundColour( ( 0, 128, 0 ) )
             
@@ -1338,14 +1337,33 @@ class DialogInputFileSystemPredicates( Dialog ):
             
             self.SetSizer( hbox )
             
+            self.Bind( wx.EVT_CHAR_HOOK, self.EventCharHook )
+            
         
-        def EventOK( self, event ):
+        def _DoOK( self ):
             
             predicates = self._predicate_panel.GetPredicates()
             
             self.GetParent().SubPanelOK( predicates )
             
         
+        def EventCharHook( self, event ):
+            
+            if event.KeyCode in ( wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER ):
+                
+                self._DoOK()
+                
+            else:
+                
+                event.Skip()
+                
+            
+        
+        def EventOK( self, event ):
+            
+            self._DoOK()
+            
+        
     
 class DialogInputLocalBooruShare( Dialog ):
     

include/ClientGUIDialogsManage.py

@@ -3149,16 +3149,16 @@ class DialogManagePixivAccount( ClientGUIDialogs.Dialog ):
     
     def EventOK( self, event ):
         
-        id = self._id.GetValue()
+        pixiv_id = self._id.GetValue()
         password = self._password.GetValue()
         
-        if id == '' and password == '':
+        if pixiv_id == '' and password == '':
             
             HydrusGlobals.client_controller.Write( 'serialisable_simple', 'pixiv_account', None )
             
         else:
             
-            HydrusGlobals.client_controller.Write( 'serialisable_simple', 'pixiv_account', ( id, password ) )
+            HydrusGlobals.client_controller.Write( 'serialisable_simple', 'pixiv_account', ( pixiv_id, password ) )
             
         
         self.EndModal( wx.ID_OK )
@@ -3166,41 +3166,23 @@ class DialogManagePixivAccount( ClientGUIDialogs.Dialog ):
     
     def EventTest( self, event ):
         
-        id = self._id.GetValue()
+        pixiv_id = self._id.GetValue()
         password = self._password.GetValue()
         
-        form_fields = {}
+        try:
-        
+            
-        # this no longer seems to work--they updated to some javascript gubbins for their main form
+            manager = HydrusGlobals.client_controller.GetManager( 'web_sessions' )
-        # I couldn't see where the POST was going in Firefox dev console, so I guess it is some other thing that doesn't pick up
+            
-        # the old form is still there, but hidden and changed to https://accounts.pixiv.net/login, but even if I do that, it just refreshes in Japanese :/
+            cookies = manager.GetPixivCookies( pixiv_id, password )
-        
-        form_fields[ 'post_key' ] = 'c779b8a16389a7861d584d11d73424a0'
-        form_fields[ 'lang' ] = 'en'
-        form_fields[ 'source' ] = 'pc'
-        form_fields[ 'return_to' ] = 'http://www.pixiv.net'
-        form_fields[ 'pixiv_id' ] = id
-        form_fields[ 'password' ] = password
-        
-        body = urllib.urlencode( form_fields )
-        
-        headers = {}
-        headers[ 'Content-Type' ] = 'application/x-www-form-urlencoded'
-        
-        ( response_gumpf, cookies ) = HydrusGlobals.client_controller.DoHTTP( HC.POST, 'https://accounts.pixiv.net/login', request_headers = headers, body = body, return_cookies = True )
-        
-        # actually, it needs an _ in it to be a logged in session
-        # posting to the old login form gives you a 301 and a session without an underscore
-        if 'PHPSESSID' in cookies:
             
             self._status.SetLabelText( 'OK!' )
             
-        else:
+            wx.CallLater( 5000, self._status.SetLabel, '' )
             
-            self._status.SetLabelText( 'Did not work!' )
+        except HydrusExceptions.ForbiddenException as e:
+            
+            self._status.SetLabelText( 'Did not work! ' + repr( e ) )
             
-        
-        wx.CallLater( 2000, self._status.SetLabel, '' )
         
     
 class DialogManageRatings( ClientGUIDialogs.Dialog ):

include/ClientGUIManagement.py

@@ -1445,17 +1445,25 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
         
         ManagementPanel.__init__( self, parent, page, controller, management_controller )
         
+        self._job = None
+        self._job_key = None
+        
+        menu_items = []
+        
+        menu_items.append( ( 'refresh', 'This panel does not update itself when files are added or deleted elsewhere in the client. Hitting this will refresh the numbers from the database.', self._RefreshAndUpdateStatus ) )
+        menu_items.append( ( 'reset potentials', 'This will delete all the potential duplicate pairs found so far and reset their files\' search status.', self._ResetUnknown ) )
+        
+        self._cog_button = ClientGUICommon.MenuBitmapButton( self, CC.GlobalBMPs.cog, menu_items )
+        
         self._preparing_panel = ClientGUICommon.StaticBox( self, 'preparation' )
         
         # refresh button that just calls update
         
-        self._total_files = wx.StaticText( self._preparing_panel )
-        
         self._num_phashes_to_regen = wx.StaticText( self._preparing_panel )
         self._num_branches_to_regen = wx.StaticText( self._preparing_panel )
         
-        self._phashes_button = wx.BitmapButton( self._preparing_panel, bitmap = CC.GlobalBMPs.play )
+        self._phashes_button = ClientGUICommon.BetterBitmapButton( self._preparing_panel, CC.GlobalBMPs.play, self._RegeneratePhashes )
-        self._branches_button = wx.BitmapButton( self._preparing_panel, bitmap = CC.GlobalBMPs.play )
+        self._branches_button = ClientGUICommon.BetterBitmapButton( self._preparing_panel, CC.GlobalBMPs.play, self._RebalanceTree )
         
         #
         
@@ -1471,10 +1479,11 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
         self._search_distance_button = ClientGUICommon.MenuButton( self._searching_panel, 'similarity', menu_items )
         
         self._search_distance_spinctrl = wx.SpinCtrl( self._searching_panel, min = 0, max = 64, size = ( 50, -1 ) )
+        self._search_distance_spinctrl.Bind( wx.EVT_SPINCTRL, self.EventSearchDistanceChanged )
         
         self._num_searched = ClientGUICommon.TextAndGauge( self._searching_panel )
         
-        self._search_button = wx.BitmapButton( self._searching_panel, bitmap = CC.GlobalBMPs.play )
+        self._search_button = ClientGUICommon.BetterBitmapButton( self._searching_panel, CC.GlobalBMPs.play, self._SearchForDuplicates )
         
         #
         
@@ -1484,30 +1493,32 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
         self._num_same_file_duplicates = wx.StaticText( self._filtering_panel )
         self._num_alternate_duplicates = wx.StaticText( self._filtering_panel )
         
-        # bind spinctrl (which should throw another update on every shift, as well
+        #
-        # bind the buttons (nah, replace with betterbitmapbutton)
+        
+        new_options = self._controller.GetNewOptions()
+        
+        self._search_distance_spinctrl.SetValue( new_options.GetInteger( 'similar_files_duplicate_pairs_search_distance' ) )
         
         #
         
-        # initialise value of spinctrl, label of distance button (which might be 'custom')
+        gridbox_1 = wx.FlexGridSizer( 0, 3 )
-        
-        gridbox_1 = wx.FlexGridSizer( 0, 2 )
         
         gridbox_1.AddGrowableCol( 0, 1 )
         
-        gridbox_1.AddF( self._num_phashes_to_regen, CC.FLAGS_EXPAND_PERPENDICULAR )
+        gridbox_1.AddF( self._num_phashes_to_regen, CC.FLAGS_VCENTER )
+        gridbox_1.AddF( ( 10, 10 ), CC.FLAGS_EXPAND_PERPENDICULAR )
         gridbox_1.AddF( self._phashes_button, CC.FLAGS_VCENTER )
-        gridbox_1.AddF( self._num_branches_to_regen, CC.FLAGS_EXPAND_PERPENDICULAR )
+        gridbox_1.AddF( self._num_branches_to_regen, CC.FLAGS_VCENTER )
+        gridbox_1.AddF( ( 10, 10 ), CC.FLAGS_EXPAND_PERPENDICULAR )
         gridbox_1.AddF( self._branches_button, CC.FLAGS_VCENTER )
         
-        self._preparing_panel.AddF( self._total_files, CC.FLAGS_EXPAND_PERPENDICULAR )
         self._preparing_panel.AddF( gridbox_1, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
         
         #
         
         distance_hbox = wx.BoxSizer( wx.HORIZONTAL )
         
-        distance_hbox.AddF( wx.StaticText( self._searching_panel, label = 'search similarity: ' ), CC.FLAGS_VCENTER )
+        distance_hbox.AddF( wx.StaticText( self._searching_panel, label = 'search distance: ' ), CC.FLAGS_VCENTER )
         distance_hbox.AddF( self._search_distance_button, CC.FLAGS_EXPAND_BOTH_WAYS )
         distance_hbox.AddF( self._search_distance_spinctrl, CC.FLAGS_VCENTER )
         
@@ -1531,22 +1542,63 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
         
         vbox = wx.BoxSizer( wx.VERTICAL )
         
+        vbox.AddF( self._cog_button, CC.FLAGS_LONE_BUTTON )
         vbox.AddF( self._preparing_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
         vbox.AddF( self._searching_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
         vbox.AddF( self._filtering_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
         
         self.SetSizer( vbox )
         
+        self.Bind( wx.EVT_TIMER, self.TIMEREventUpdateDBJob, id = ID_TIMER_UPDATE )
+        self._update_db_job_timer = wx.Timer( self, id = ID_TIMER_UPDATE )
+        
         #
         
-        self._RefreshAndUpdate()
+        self._RefreshAndUpdateStatus()
+        
+    
+    def _RebalanceTree( self ):
+        
+        self._job = 'branches'
+        
+        self._StartStopDBJob()
+        
+    
+    def _RegeneratePhashes( self ):
+        
+        self._job = 'phashes'
+        
+        self._StartStopDBJob()
+        
+    
+    def _ResetUnknown( self ):
+        
+        text = 'This will delete all the potential duplicate pairs and reset their files\' search status.'
+        text += os.linesep * 2
+        text += 'This can be useful if you have accidentally searched too broadly and are now swamped with too many false positives.'
+        
+        with ClientGUIDialogs.DialogYesNo( self, text ) as dlg:
+            
+            if dlg.ShowModal() == wx.ID_YES:
+                
+                self._controller.Write( 'delete_unknown_duplicate_pairs' )
+                
+                self._RefreshAndUpdateStatus()
+            
+        
+    
+    def _SearchForDuplicates( self ):
+        
+        self._job = 'search'
+        
+        self._StartStopDBJob()
         
     
     def _SetSearchDistance( self, value ):
         
-        self._search_distance_spinctrl.SetValue( value ) # does this trigger the update event? check it
+        self._search_distance_spinctrl.SetValue( value )
         
-        # update the label, which prob needs an HC.hamming_str dict or something, which I can then apply everywhere else as well.
+        self._UpdateStatus()
         
     
     def _SetSearchDistanceExact( self ):
@@ -1569,26 +1621,126 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
         self._SetSearchDistance( HC.HAMMING_VERY_SIMILAR )
         
     
-    def _RefreshAndUpdate( self ):
+    def _StartStopDBJob( self ):
+        
+        if self._job_key is None:
+            
+            self._cog_button.Disable()
+            self._phashes_button.Disable()
+            self._branches_button.Disable()
+            self._search_button.Disable()
+            self._search_distance_button.Disable()
+            self._search_distance_spinctrl.Disable()
+            
+            self._job_key = ClientThreading.JobKey( cancellable = True )
+            
+            if self._job == 'phashes':
+                
+                self._phashes_button.Enable()
+                self._phashes_button.SetBitmap( CC.GlobalBMPs.stop )
+                
+                self._controller.Write( 'maintain_similar_files_phashes', job_key = self._job_key )
+                
+            elif self._job == 'branches':
+                
+                self._branches_button.Enable()
+                self._branches_button.SetBitmap( CC.GlobalBMPs.stop )
+                
+                self._controller.Write( 'maintain_similar_files_tree', job_key = self._job_key )
+                
+            elif self._job == 'search':
+                
+                self._search_button.Enable()
+                self._search_button.SetBitmap( CC.GlobalBMPs.stop )
+                
+                search_distance = self._search_distance_spinctrl.GetValue()
+                
+                self._controller.Write( 'maintain_similar_files_duplicate_pairs', search_distance, job_key = self._job_key )
+                
+            
+            self._update_db_job_timer.Start( 250, wx.TIMER_CONTINUOUS )
+            
+        else:
+            
+            self._job_key.Cancel()
+            
+        
+    
+    def _UpdateJob( self ):
+        
+        if self._job_key.IsDone():
+            
+            self._job_key = None
+            
+            self._update_db_job_timer.Stop()
+            
+            self._RefreshAndUpdateStatus()
+            
+            return
+            
+        
+        if self._job == 'phashes':
+            
+            text = self._job_key.GetIfHasVariable( 'popup_text_1' )
+            
+            if text is not None:
+                
+                self._num_phashes_to_regen.SetLabelText( text )
+                
+            
+        elif self._job == 'branches':
+            
+            text = self._job_key.GetIfHasVariable( 'popup_text_1' )
+            
+            if text is not None:
+                
+                self._num_branches_to_regen.SetLabelText( text )
+                
+            
+        elif self._job == 'search':
+            
+            text = self._job_key.GetIfHasVariable( 'popup_text_1' )
+            gauge = self._job_key.GetIfHasVariable( 'popup_gauge_1' )
+            
+            if text is not None and gauge is not None:
+                
+                ( value, range ) = gauge
+                
+                self._num_searched.SetValue( text, value, range )
+                
+            
+        
+    
+    def _RefreshAndUpdateStatus( self ):
         
         self._similar_files_maintenance_status = self._controller.Read( 'similar_files_maintenance_status' )
         
-        self._Update()
+        self._UpdateStatus()
         
     
-    def _Update( self ):
+    def _UpdateStatus( self ):
         
         ( searched_distances_to_count, duplicate_types_to_count, num_phashes_to_regen, num_branches_to_regen ) = self._similar_files_maintenance_status
         
+        self._cog_button.Enable()
+        
+        self._phashes_button.SetBitmap( CC.GlobalBMPs.play )
+        self._branches_button.SetBitmap( CC.GlobalBMPs.play )
+        self._search_button.SetBitmap( CC.GlobalBMPs.play )
+        
+        total_num_files = sum( searched_distances_to_count.values() )
+        
         if num_phashes_to_regen == 0:
             
-            self._num_phashes_to_regen.SetLabelText( 'All files ready!' )
+            self._num_phashes_to_regen.SetLabelText( 'All ' + HydrusData.ConvertIntToPrettyString( total_num_files ) + ' eligible files up to date!' )
             
             self._phashes_button.Disable()
             
         else:
             
-            self._num_phashes_to_regen.SetLabelText( HydrusData.ConvertIntToPrettyString( num_phashes_to_regen ) + ' files to reanalyze.' )
+            num_done = total_num_files - num_phashes_to_regen
+            
+            self._num_phashes_to_regen.SetLabelText( HydrusData.ConvertValueRangeToPrettyString( num_done, total_num_files ) + 'eligible files up to date.' )
             
             self._phashes_button.Enable()
             
@@ -1606,17 +1758,31 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
             self._branches_button.Enable()
             
         
-        total_num_files = sum( searched_distances_to_count.values() )
+        self._search_distance_button.Enable()
-        
+        self._search_distance_spinctrl.Enable()
-        self._total_files.SetLabelText( HydrusData.ConvertIntToPrettyString( total_num_files ) + ' eligable files.' )
         
         search_distance = self._search_distance_spinctrl.GetValue()
         
+        new_options = self._controller.GetNewOptions()
+        
+        new_options.SetInteger( 'similar_files_duplicate_pairs_search_distance', search_distance )
+        
+        if search_distance in HC.hamming_string_lookup:
+            
+            button_label = HC.hamming_string_lookup[ search_distance ]
+            
+        else:
+            
+            button_label = 'custom'
+            
+        
+        self._search_distance_button.SetLabelText( button_label )
+        
         num_searched = sum( ( count for ( value, count ) in searched_distances_to_count.items() if value is not None and value >= search_distance ) )
         
         if num_searched == total_num_files:
             
-            self._num_searched.SetValue( 'All potential duplicates found.', total_num_files, total_num_files )
+            self._num_searched.SetValue( 'All potential duplicates found at this distance.', total_num_files, total_num_files )
             
             self._search_button.Disable()
             
@@ -1624,21 +1790,33 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
             
             if num_searched == 0:
                 
-                self._num_searched.SetValue( 'Have not yet searched at that distance.', 0, total_num_files )
+                self._num_searched.SetValue( 'Have not yet searched at this distance.', 0, total_num_files )
                 
             else:
                 
-                self._num_searched.SetValue( 'Searched ' + HydrusData.ConvertValueRangeToPrettyString( num_searched, total_num_files ) + ' files.', num_searched, total_num_files )
+                self._num_searched.SetValue( 'Searched ' + HydrusData.ConvertValueRangeToPrettyString( num_searched, total_num_files ) + ' files at this distance.', num_searched, total_num_files )
                 
             
             self._search_button.Enable()
             
         
-        self._num_unknown_duplicates.SetLabelText( HydrusData.ConvertIntToPrettyString( duplicate_types_to_count[ HC.DUPLICATE_UNKNOWN ] ) + ' potential duplicates found.' )
+        num_unknown = duplicate_types_to_count[ HC.DUPLICATE_UNKNOWN ]
+        
+        self._num_unknown_duplicates.SetLabelText( HydrusData.ConvertIntToPrettyString( num_unknown ) + ' potential duplicates found.' )
         self._num_same_file_duplicates.SetLabelText( HydrusData.ConvertIntToPrettyString( duplicate_types_to_count[ HC.DUPLICATE_SAME_FILE ] ) + ' same file pairs filtered.' )
         self._num_alternate_duplicates.SetLabelText( HydrusData.ConvertIntToPrettyString( duplicate_types_to_count[ HC.DUPLICATE_ALTERNATE ] ) + ' alternate file pairs filtered.' )
         
     
+    def EventSearchDistanceChanged( self, event ):
+        
+        self._UpdateStatus()
+        
+    
+    def TIMEREventUpdateDBJob( self, event ):
+        
+        self._UpdateJob()
+        
+    
 management_panel_types_to_classes[ MANAGEMENT_TYPE_DUPLICATE_FILTER ] = ManagementPanelDuplicateFilter
 
 class ManagementPanelGalleryImport( ManagementPanel ):

include/ClientImageHandling.py

@@ -152,20 +152,32 @@ def GenerateShapePerceptualHashes( path ):
     
     dct_88 = dct[:8,:8]
     
-    # get mean of dct, excluding [0,0]
+    # get median of dct
+    # exclude [0,0], which represents flat colour
+    # this [0,0] exclusion is apparently important for mean, but maybe it ain't so important for median--w/e
     
-    mask = numpy.ones( ( 8, 8 ) )
+    # old mean code
+    # mask = numpy.ones( ( 8, 8 ) )
+    # mask[0,0] = 0
+    # average = numpy.average( dct_88, weights = mask )
     
-    mask[0,0] = 0
+    median = numpy.median( dct_88.reshape( 64 )[1:] )
     
-    average = numpy.average( dct_88, weights = mask )
+    # make a monochromatic, 64-bit hash of whether the entry is above or below the median
     
-    # make a monochromatic, 64-bit hash of whether the entry is above or below the mean
+    dct_88_boolean = dct_88 > median
+    
+    # convert TTTFTFTF to 11101010 by repeatedly shifting answer and adding 0 or 1
+    # you can even go ( a << 1 ) + b and leave out the initial param on the latel reduce call as bools act like ints for this
+    # but let's not go crazy for another two nanoseconds
+    collapse_bools_to_binary_uint = lambda a, b: ( a << 1 ) + int( b )
     
     bytes = []
     
     for i in range( 8 ):
         
+        '''
+        # old way of doing it, which compared value to median every time
         byte = 0
         
         for j in range( 8 ):
@@ -174,15 +186,27 @@ def GenerateShapePerceptualHashes( path ):
             
             value = dct_88[i,j]
             
-            if value > average: byte |= 1
+            if value > median:
+                
+                byte |= 1
+                
             
+        '''
+        
+        byte = reduce( collapse_bools_to_binary_uint, dct_88_boolean[i], 0 )
         
         bytes.append( byte )
         
     
     phash = str( bytearray( bytes ) )
     
-    phashes = [ phash ]
+    # now discard the blank hash, which is 1000000... and not useful
+    
+    phashes = set()
+    
+    phashes.add( phash )
+    
+    phashes.discard( CC.BLANK_PHASH )
     
     # we good
     
@@ -213,4 +237,4 @@ def ResizeNumpyImage( mime, numpy_image, ( target_x, target_y ) ):
         
         return cv2.resize( numpy_image, ( target_x, target_y ), interpolation = interpolation )
         
-    
+    

include/ClientNetworking.py

@@ -616,6 +616,13 @@ class HTTPConnection( object ):
                 
                 if attempt_number <= 3:
                     
+                    if self._hydrus_network:
+                        
+                        # we are talking to a new hydrus server, which uses https, and hence an http call gives badstatusline
+                        
+                        self._scheme = 'https'
+                        
+                    
                     self._RefreshConnection()
                     
                     return self._GetInitialResponse( method, path_and_query, request_headers, body, attempt_number = attempt_number + 1 )
@@ -638,6 +645,13 @@ class HTTPConnection( object ):
             
             if attempt_number <= 3:
                 
+                if self._hydrus_network:
+                    
+                    # we are talking to a new hydrus server, which uses https, and hence an http call gives badstatusline
+                    
+                    self._scheme = 'https'
+                    
+                
                 self._RefreshConnection()
                 
                 return self._GetInitialResponse( method_string, path_and_query, request_headers, body, attempt_number = attempt_number + 1 )

include/HydrusConstants.py

@@ -25,6 +25,9 @@ HELP_DIR = os.path.join( BASE_DIR, 'help' )
 INCLUDE_DIR = os.path.join( BASE_DIR, 'include' )
 STATIC_DIR = os.path.join( BASE_DIR, 'static' )
 
+DEFAULT_DB_DIR = os.path.join( BASE_DIR, 'db' )
+LICENSE_PATH = os.path.join( BASE_DIR, 'license.txt' )
+
 #
 
 PLATFORM_WINDOWS = False
@@ -46,7 +49,7 @@ options = {}
 # Misc
 
 NETWORK_VERSION = 17
-SOFTWARE_VERSION = 240
+SOFTWARE_VERSION = 241
 
 UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )
 
@@ -126,6 +129,13 @@ HAMMING_VERY_SIMILAR = 2
 HAMMING_SIMILAR = 4
 HAMMING_SPECULATIVE = 8
 
+hamming_string_lookup = {}
+
+hamming_string_lookup[ HAMMING_EXACT_MATCH ] = 'exact match'
+hamming_string_lookup[ HAMMING_VERY_SIMILAR ] = 'very similar'
+hamming_string_lookup[ HAMMING_SIMILAR ] = 'similar'
+hamming_string_lookup[ HAMMING_SPECULATIVE ] = 'speculative'
+
 HYDRUS_CLIENT = 0
 HYDRUS_SERVER = 1
 HYDRUS_TEST = 2

include/HydrusPaths.py

@@ -333,6 +333,19 @@ def MakeFileWritable( path ):
         
         os.chmod( path, stat.S_IWRITE | stat.S_IREAD )
         
+        if os.path.isdir( path ):
+            
+            for ( root, dirnames, filenames ) in os.walk( path ):
+                
+                for filename in filenames:
+                    
+                    sub_path = os.path.join( root, filename )
+                    
+                    os.chmod( sub_path, stat.S_IWRITE | stat.S_IREAD )
+                    
+                
+            
+        
     except:
         
         pass
@@ -568,4 +581,4 @@ def RecyclePath( path ):
             DeletePath( original_path )
             
         
-    
+    

include/TestClientImageHandling.py

@@ -11,5 +11,5 @@ class TestImageHandling( unittest.TestCase ):
         
         phashes = ClientImageHandling.GenerateShapePerceptualHashes( os.path.join( HC.STATIC_DIR, 'hydrus.png' ) )
         
-        self.assertEqual( phashes, [ '\xb0\x08\x83\xb2\x08\x0b8\x08' ] )
+        self.assertEqual( phashes, set( [ '\xb4M\xc7\xb2M\xcb8\x1c' ] ) )
         

include/TestDialogs.py

@@ -1,6 +1,8 @@
 import ClientConstants as CC
 import ClientDefaults
 import ClientGUIDialogs
+import ClientGUIScrolledPanelsManagement
+import ClientGUITopLevelWindows
 import collections
 import HydrusConstants as HC
 import os
@@ -9,9 +11,35 @@ import unittest
 import wx
 import HydrusGlobals
 
+def HitButton( button ): wx.PostEvent( button, wx.CommandEvent( wx.EVT_BUTTON.typeId, button.GetId() ) )
+
 def HitCancelButton( window ): wx.PostEvent( window, wx.CommandEvent( wx.EVT_BUTTON.typeId, wx.ID_CANCEL ) )
 
-def HitButton( button ): wx.PostEvent( button, wx.CommandEvent( wx.EVT_BUTTON.typeId, button.GetId() ) )
+def HitOKButton( window ): wx.PostEvent( window, wx.CommandEvent( wx.EVT_BUTTON.typeId, wx.ID_OK ) )
+
+def CancelChildDialog( window ):
+    
+    children = window.GetChildren()
+    
+    for child in children:
+        
+        if isinstance( child, wx.Dialog ):
+            
+            HitCancelButton( child )
+            
+        
+
+def OKChildDialog( window ):
+    
+    children = window.GetChildren()
+    
+    for child in children:
+        
+        if isinstance( child, wx.Dialog ):
+            
+            HitOKButton( child )
+            
+        
 
 def PressKey( window, key ):
     
@@ -39,6 +67,31 @@ class TestDBDialogs( unittest.TestCase ):
             
         
     
+    def test_dialog_manage_subs( self ):
+        
+        HydrusGlobals.test_controller.SetRead( 'serialisable_named', [] )
+        
+        title = 'subs test'
+        frame_key = 'regular_dialog'
+        
+        with ClientGUITopLevelWindows.DialogManage( None, title, frame_key ) as dlg:
+            
+            panel = ClientGUIScrolledPanelsManagement.ManageSubscriptionsPanel( dlg )
+            
+            dlg.SetPanel( panel )
+            
+            wx.CallAfter( panel.Add )
+            
+            wx.CallLater( 2000, OKChildDialog, panel )
+            
+            wx.CallLater( 4000, HitCancelButton, dlg )
+            
+            result = dlg.ShowModal()
+            
+            self.assertEqual( result, wx.ID_CANCEL )
+            
+        
+    
 class TestNonDBDialogs( unittest.TestCase ):
     
     def test_dialog_choose_new_service_method( self ):
@@ -169,4 +222,4 @@ class TestNonDBDialogs( unittest.TestCase ):
             self.assertEqual( result, wx.ID_NO )
             
         
-    
+    

server.py

@@ -46,7 +46,7 @@ try:
     
     if result.db_dir is None:
         
-        db_dir = os.path.join( HC.BASE_DIR, 'db' )
+        db_dir = HC.DEFAULT_DB_DIR
         
     else:
         
@@ -55,6 +55,7 @@ try:
     
     db_dir = HydrusPaths.ConvertPortablePathToAbsPath( db_dir, HC.BASE_DIR )
     
+    
     try:
         
         HydrusPaths.MakeSureDirectoryExists( db_dir )
@@ -139,4 +140,4 @@ except Exception as e:
         
         print( 'Critical error occured! Details written to crash.log!' )
         
-    
+    

test.py

@@ -156,6 +156,13 @@ class Controller( object ):
         return call_to_thread
         
     
+    def _SetupWx( self ):
+        
+        self.locale = wx.Locale( wx.LANGUAGE_DEFAULT ) # Very important to init this here and keep it non garbage collected
+        
+        CC.GlobalBMPs.STATICInitialise()
+        
+    
     def pub( self, topic, *args, **kwargs ):
         
         pass
@@ -249,6 +256,8 @@ class Controller( object ):
     
     def Run( self ):
         
+        self._SetupWx()
+        
         suites = []
         
         if only_run is None: run_all = True