Version 241
This commit is contained in:
parent
643586ce04
commit
e24ade0548
|
@ -38,7 +38,7 @@ try:
|
|||
|
||||
if result.db_dir is None:
|
||||
|
||||
db_dir = os.path.join( HC.BASE_DIR, 'db' )
|
||||
db_dir = HC.DEFAULT_DB_DIR
|
||||
|
||||
else:
|
||||
|
||||
|
@ -124,4 +124,4 @@ except Exception as e:
|
|||
|
||||
print( 'Critical error occured! Details written to crash.log!' )
|
||||
|
||||
|
||||
|
||||
|
|
|
@ -38,7 +38,7 @@ try:
|
|||
|
||||
if result.db_dir is None:
|
||||
|
||||
db_dir = os.path.join( HC.BASE_DIR, 'db' )
|
||||
db_dir = HC.DEFAULT_DB_DIR
|
||||
|
||||
else:
|
||||
|
||||
|
@ -124,4 +124,4 @@ except Exception as e:
|
|||
|
||||
print( 'Critical error occured! Details written to crash.log!' )
|
||||
|
||||
|
||||
|
||||
|
|
|
@ -8,6 +8,34 @@
|
|||
<div class="content">
|
||||
<h3>changelog</h3>
|
||||
<ul>
|
||||
<li><h3>version 241</h3></li>
|
||||
<ul>
|
||||
<li>fixed the 'setnondupename' problem that was affecting 'add' actions on manage subscriptions, scripts, and import/export folders</li>
|
||||
<li>added some more tests to catch this problem automatically in future</li>
|
||||
<li>cleaned up some similar files phash regeneration logic</li>
|
||||
<li>cleaned up similar files maintenance code to deal with the new duplicates page</li>
|
||||
<li>wrote a similar files duplicate pair search maintenance routine</li>
|
||||
<li>activated file phash regen button on the new duplicates page</li>
|
||||
<li>activated branch rebalancing button on the new duplicates page</li>
|
||||
<li>activated duplicate search button on the new duplicates page</li>
|
||||
<li>search distance on the new duplicates page is now remembered between sessions</li>
|
||||
<li>improved the phash algorithm to use median instead of mean--it now gives fewer apparent false positives and negatives, but I think it may also be stricter in general</li>
|
||||
<li>the duplicate system now discards phashes for blank, flat colour images (this will be more useful when I reintroduce dupe checking for animations, which often start with a black frame)</li>
|
||||
<li>misc phash code cleanup</li>
|
||||
<li>all local jpegs and pngs will be scheduled for phash regeneration on update as their current phashes are legacies of several older versions of the algorithm</li>
|
||||
<li>debuted a cog menu button on the new duplicates page to refresh the page and reset found potential duplicate pairs--this cog should be making appearances elsewhere to add settings and reduce excess buttons</li>
|
||||
<li>improved some search logic that was refreshing too much info on an 'include current/pending tags' button press</li>
|
||||
<li>fixed pixiv login--for now!</li>
|
||||
<li>system:dimensions now catches an enter key event and passes it to the correct ok button, rather than always num_pixels</li>
|
||||
<li>fixed some bad http->https conversion when uploading files to file repo</li>
|
||||
<li>folder deletion will try to deal better with read-only nested files</li>
|
||||
<li>tag parent uploads will now go one at a time (rather than up to 100 as before) to reduce commit lag</li>
|
||||
<li>updated to python 2.7.13 for windows</li>
|
||||
<li>updated to OpenCV 3.2 for windows--this new version does not crash with the same files that 3.1 does, so I recommend windows users turn off 'load images with pil' under options->media if they have it set</li>
|
||||
<li>I think I improved some unicode error handling</li>
|
||||
<li>added LICENSE_PATH and harmonised various instances of default db dir creation to DEFAULT_DB_DIR, both in HydrusConstants</li>
|
||||
<li>misc code cleanup and bitmap button cleanup</li>
|
||||
</ul>
|
||||
<li><h3>version 240</h3></li>
|
||||
<ul>
|
||||
<li>improved how the client analyzes itself, reducing maintenance latency and also overall cpu usage. syncing a big repo will no longer introduce lingering large lag, and huge analyze jobs will run significantly less frequently</li>
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
import ClientDefaults
|
||||
import ClientDownloading
|
||||
import ClientFiles
|
||||
import ClientNetworking
|
||||
import ClientRendering
|
||||
|
@ -11,9 +12,11 @@ import HydrusImageHandling
|
|||
import HydrusPaths
|
||||
import HydrusSessions
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import Queue
|
||||
import requests
|
||||
import shutil
|
||||
import threading
|
||||
import time
|
||||
|
@ -2947,22 +2950,9 @@ class WebSessionManagerClient( object ):
|
|||
raise HydrusExceptions.DataMissing( 'You need to set up your pixiv credentials in services->manage pixiv account.' )
|
||||
|
||||
|
||||
( id, password ) = result
|
||||
( pixiv_id, password ) = result
|
||||
|
||||
form_fields = {}
|
||||
|
||||
form_fields[ 'pixiv_id' ] = id
|
||||
form_fields[ 'password' ] = password
|
||||
|
||||
body = urllib.urlencode( form_fields )
|
||||
|
||||
headers = {}
|
||||
headers[ 'Content-Type' ] = 'application/x-www-form-urlencoded'
|
||||
|
||||
( response_gumpf, cookies ) = self._controller.DoHTTP( HC.POST, 'http://www.pixiv.net/login.php', request_headers = headers, body = body, return_cookies = True )
|
||||
|
||||
# _ only given to logged in php sessions
|
||||
if 'PHPSESSID' not in cookies: raise Exception( 'Pixiv login credentials not accepted!' )
|
||||
cookies = self.GetPixivCookies( pixiv_id, password )
|
||||
|
||||
expires = now + 30 * 86400
|
||||
|
||||
|
@ -2975,3 +2965,60 @@ class WebSessionManagerClient( object ):
|
|||
|
||||
|
||||
|
||||
# This updated login form is cobbled together from the example in PixivUtil2
|
||||
# it is breddy shid because I'm not using mechanize or similar browser emulation (like requests's sessions) yet
|
||||
# Pixiv 400s if cookies and referrers aren't passed correctly
|
||||
# I am leaving this as a mess with the hope the eventual login engine will replace it
|
||||
def GetPixivCookies( self, pixiv_id, password ):
|
||||
|
||||
( response, cookies ) = self._controller.DoHTTP( HC.GET, 'https://accounts.pixiv.net/login', return_cookies = True )
|
||||
|
||||
soup = ClientDownloading.GetSoup( response )
|
||||
|
||||
# some whocking 20kb bit of json tucked inside a hidden form input wew lad
|
||||
i = soup.find( 'input', id = 'init-config' )
|
||||
|
||||
raw_json = i['value']
|
||||
|
||||
j = json.loads( raw_json )
|
||||
|
||||
if 'pixivAccount.postKey' not in j:
|
||||
|
||||
raise HydrusExceptions.ForbiddenException( 'When trying to log into Pixiv, I could not find the POST key!' )
|
||||
|
||||
|
||||
post_key = j[ 'pixivAccount.postKey' ]
|
||||
|
||||
form_fields = {}
|
||||
|
||||
form_fields[ 'pixiv_id' ] = pixiv_id
|
||||
form_fields[ 'password' ] = password
|
||||
form_fields[ 'captcha' ] = ''
|
||||
form_fields[ 'g_recaptcha_response' ] = ''
|
||||
form_fields[ 'return_to' ] = 'http://www.pixiv.net'
|
||||
form_fields[ 'lang' ] = 'en'
|
||||
form_fields[ 'post_key' ] = post_key
|
||||
form_fields[ 'source' ] = 'pc'
|
||||
|
||||
headers = {}
|
||||
|
||||
headers[ 'referer' ] = "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
|
||||
headers[ 'origin' ] = "https://accounts.pixiv.net"
|
||||
ClientNetworking.AddCookiesToHeaders( cookies, headers )
|
||||
|
||||
r = requests.post( 'https://accounts.pixiv.net/api/login?lang=en', data = form_fields, headers = headers )
|
||||
|
||||
# doesn't work
|
||||
#( response_gumpf, cookies ) = self._controller.DoHTTP( HC.POST, 'https://accounts.pixiv.net/api/login?lang=en', request_headers = headers, body = body, return_cookies = True )
|
||||
|
||||
cookies = dict( r.cookies )
|
||||
|
||||
# _ only given to logged-in php sessions
|
||||
if 'PHPSESSID' not in cookies or '_' not in cookies[ 'PHPSESSID' ]:
|
||||
|
||||
raise HydrusExceptions.ForbiddenException( 'Pixiv login credentials not accepted!' )
|
||||
|
||||
|
||||
return cookies
|
||||
|
||||
|
||||
|
|
|
@ -16,6 +16,8 @@ ID_TIMER_UPDATES = wx.NewId()
|
|||
|
||||
#
|
||||
|
||||
BLANK_PHASH = '\x80\x00\x00\x00\x00\x00\x00\x00' # first bit 1 but everything else 0 means only significant part of dct was [0,0], which represents flat colour
|
||||
|
||||
CAN_HIDE_MOUSE = True
|
||||
|
||||
# Hue is generally 200, Sat and Lum changes based on need
|
||||
|
@ -433,6 +435,8 @@ class GlobalBMPs( object ):
|
|||
GlobalBMPs.dump_recoverable = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'dump_recoverable.png' ) )
|
||||
GlobalBMPs.dump_fail = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'dump_fail.png' ) )
|
||||
|
||||
GlobalBMPs.cog = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'cog.png' ) )
|
||||
GlobalBMPs.check = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'check.png' ) )
|
||||
GlobalBMPs.pause = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'pause.png' ) )
|
||||
GlobalBMPs.play = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'play.png' ) )
|
||||
GlobalBMPs.stop = wx.Bitmap( os.path.join( HC.STATIC_DIR, 'stop.png' ) )
|
||||
|
|
|
@ -1596,29 +1596,62 @@ class DB( HydrusDB.HydrusDB ):
|
|||
self._c.execute( 'INSERT INTO shape_vptree ( phash_id, parent_id, radius, inner_id, inner_population, outer_id, outer_population ) VALUES ( ?, ?, ?, ?, ?, ?, ? );', ( phash_id, parent_id, radius, inner_id, inner_population, outer_id, outer_population ) )
|
||||
|
||||
|
||||
def _CacheSimilarFilesDelete( self, hash_id, phash_ids = None ):
|
||||
def _CacheSimilarFilesAssociatePHashes( self, hash_id, phashes ):
|
||||
|
||||
if phash_ids is None:
|
||||
phash_ids = set()
|
||||
|
||||
for phash in phashes:
|
||||
|
||||
phash_ids = { phash_id for ( phash_id, ) in self._c.execute( 'SELECT phash_id FROM shape_perceptual_hash_map WHERE hash_id = ?;', ( hash_id, ) ) }
|
||||
phash_id = self._CacheSimilarFilesGetPHashId( phash )
|
||||
|
||||
self._c.execute( 'DELETE FROM shape_perceptual_hash_map WHERE hash_id = ?;', ( hash_id, ) )
|
||||
|
||||
else:
|
||||
|
||||
phash_ids = set( phash_ids )
|
||||
|
||||
self._c.executemany( 'DELETE FROM shape_perceptual_hash_map WHERE phash_id = ? AND hash_id = ?;', ( ( phash_id, hash_id ) for phash_id in phash_ids ) )
|
||||
phash_ids.add( phash_id )
|
||||
|
||||
|
||||
useful_phash_ids = { phash for ( phash, ) in self._c.execute( 'SELECT phash_id FROM shape_perceptual_hash_map WHERE phash_id IN ' + HydrusData.SplayListForDB( phash_ids ) + ';' ) }
|
||||
self._c.executemany( 'INSERT OR IGNORE INTO shape_perceptual_hash_map ( phash_id, hash_id ) VALUES ( ?, ? );', ( ( phash_id, hash_id ) for phash_id in phash_ids ) )
|
||||
|
||||
deletee_phash_ids = phash_ids.difference( useful_phash_ids )
|
||||
if self._GetRowCount() > 0:
|
||||
|
||||
self._c.execute( 'REPLACE INTO shape_search_cache ( hash_id, searched_distance ) VALUES ( ?, ? );', ( hash_id, None ) )
|
||||
|
||||
|
||||
self._c.executemany( 'INSERT OR IGNORE INTO shape_maintenance_branch_regen ( phash_id ) VALUES ( ? );', ( ( phash_id, ) for phash_id in deletee_phash_ids ) )
|
||||
return phash_ids
|
||||
|
||||
|
||||
def _CacheSimilarFilesDeleteFile( self, hash_id ):
|
||||
|
||||
phash_ids = { phash_id for ( phash_id, ) in self._c.execute( 'SELECT phash_id FROM shape_perceptual_hash_map WHERE hash_id = ?;', ( hash_id, ) ) }
|
||||
|
||||
self._CacheSimilarFilesDisassociatePHashes( hash_id, phash_ids )
|
||||
|
||||
self._c.execute( 'DELETE FROM shape_search_cache WHERE hash_id = ?;', ( hash_id, ) )
|
||||
self._c.execute( 'DELETE FROM duplicate_pairs WHERE smaller_hash_id = ? or larger_hash_id = ?;', ( hash_id, hash_id ) )
|
||||
self._c.execute( 'DELETE FROM shape_maintenance_phash_regen WHERE hash_id = ?;', ( hash_id, ) )
|
||||
|
||||
|
||||
def _CacheSimilarFilesDeleteUnknownDuplicatePairs( self ):
|
||||
|
||||
hash_ids = set()
|
||||
|
||||
for ( smaller_hash_id, larger_hash_id ) in self._c.execute( 'SELECT smaller_hash_id, larger_hash_id FROM duplicate_pairs WHERE duplicate_type = ?;', ( HC.DUPLICATE_UNKNOWN, ) ):
|
||||
|
||||
hash_ids.add( smaller_hash_id )
|
||||
hash_ids.add( larger_hash_id )
|
||||
|
||||
|
||||
self._c.execute( 'DELETE FROM duplicate_pairs WHERE duplicate_type = ?;', ( HC.DUPLICATE_UNKNOWN, ) )
|
||||
|
||||
self._c.executemany( 'UPDATE shape_search_cache SET searched_distance = NULL WHERE hash_id = ?;', ( ( hash_id, ) for hash_id in hash_ids ) )
|
||||
|
||||
|
||||
def _CacheSimilarFilesDisassociatePHashes( self, hash_id, phash_ids ):
|
||||
|
||||
self._c.executemany( 'DELETE FROM shape_perceptual_hash_map WHERE phash_id = ? AND hash_id = ?;', ( ( phash_id, hash_id ) for phash_id in phash_ids ) )
|
||||
|
||||
useful_phash_ids = { phash for ( phash, ) in self._c.execute( 'SELECT phash_id FROM shape_perceptual_hash_map WHERE phash_id IN ' + HydrusData.SplayListForDB( phash_ids ) + ';' ) }
|
||||
|
||||
useless_phash_ids = phash_ids.difference( useful_phash_ids )
|
||||
|
||||
self._c.executemany( 'INSERT OR IGNORE INTO shape_maintenance_branch_regen ( phash_id ) VALUES ( ? );', ( ( phash_id, ) for phash_id in useless_phash_ids ) )
|
||||
|
||||
|
||||
def _CacheSimilarFilesGenerateBranch( self, job_key, parent_id, phash_id, phash, children ):
|
||||
|
@ -1739,42 +1772,99 @@ class DB( HydrusDB.HydrusDB ):
|
|||
return ( searched_distances_to_count, duplicate_types_to_count, num_phashes_to_regen, num_branches_to_regen )
|
||||
|
||||
|
||||
def _CacheSimilarFilesAssociatePHashes( self, hash_id, phashes ):
|
||||
def _CacheSimilarFilesMaintainDuplicatePairs( self, search_distance, job_key = None, stop_time = None ):
|
||||
|
||||
phash_ids = set()
|
||||
pub_job_key = False
|
||||
job_key_pubbed = False
|
||||
|
||||
for phash in phashes:
|
||||
if job_key is None:
|
||||
|
||||
phash_id = self._CacheSimilarFilesGetPHashId( phash )
|
||||
job_key = ClientThreading.JobKey( cancellable = True )
|
||||
|
||||
self._c.execute( 'INSERT OR IGNORE INTO shape_perceptual_hash_map ( phash_id, hash_id ) VALUES ( ?, ? );', ( phash_id, hash_id ) )
|
||||
|
||||
phash_ids.add( phash_id )
|
||||
pub_job_key = True
|
||||
|
||||
|
||||
self._c.execute( 'REPLACE INTO shape_search_cache ( hash_id, searched_distance ) VALUES ( ?, ? );', ( hash_id, None ) )
|
||||
hash_ids = [ hash_id for ( hash_id, ) in self._c.execute( 'SELECT hash_id FROM shape_search_cache WHERE searched_distance IS NULL or searched_distance < ?;', ( search_distance, ) ) ]
|
||||
|
||||
return phash_ids
|
||||
pairs_found = 0
|
||||
|
||||
num_to_do = len( hash_ids )
|
||||
|
||||
for ( i, hash_id ) in enumerate( hash_ids ):
|
||||
|
||||
job_key.SetVariable( 'popup_title', 'similar files duplicate pair discovery' )
|
||||
|
||||
if pub_job_key and not job_key_pubbed:
|
||||
|
||||
self._controller.pub( 'message', job_key )
|
||||
|
||||
job_key_pubbed = True
|
||||
|
||||
|
||||
( i_paused, should_quit ) = job_key.WaitIfNeeded()
|
||||
|
||||
should_stop = stop_time is not None and HydrusData.TimeHasPassed( stop_time )
|
||||
|
||||
if should_quit or should_stop:
|
||||
|
||||
return
|
||||
|
||||
|
||||
text = 'searched ' + HydrusData.ConvertValueRangeToPrettyString( i, num_to_do ) + ', found ' + HydrusData.ConvertIntToPrettyString( pairs_found )
|
||||
|
||||
job_key.SetVariable( 'popup_text_1', text )
|
||||
job_key.SetVariable( 'popup_gauge_1', ( i, num_to_do ) )
|
||||
|
||||
duplicate_hash_ids = [ duplicate_hash_id for duplicate_hash_id in self._CacheSimilarFilesSearch( hash_id, search_distance ) if duplicate_hash_id != hash_id ]
|
||||
|
||||
self._c.executemany( 'INSERT OR IGNORE INTO duplicate_pairs ( smaller_hash_id, larger_hash_id, duplicate_type ) VALUES ( ?, ?, ? );', ( ( min( hash_id, duplicate_hash_id ), max( hash_id, duplicate_hash_id ), HC.DUPLICATE_UNKNOWN ) for duplicate_hash_id in duplicate_hash_ids ) )
|
||||
|
||||
pairs_found += self._GetRowCount()
|
||||
|
||||
self._c.execute( 'UPDATE shape_search_cache SET searched_distance = ? WHERE hash_id = ?;', ( search_distance, hash_id ) )
|
||||
|
||||
|
||||
job_key.SetVariable( 'popup_text_1', 'done!' )
|
||||
job_key.DeleteVariable( 'popup_gauge_1' )
|
||||
|
||||
job_key.Finish()
|
||||
job_key.Delete( 30 )
|
||||
|
||||
|
||||
def _CacheSimilarFilesMaintainFiles( self, job_key ):
|
||||
def _CacheSimilarFilesMaintainFiles( self, job_key = None, stop_time = None ):
|
||||
|
||||
# this should take a cancellable job_key from the gui filter window
|
||||
pub_job_key = False
|
||||
job_key_pubbed = False
|
||||
|
||||
if job_key is None:
|
||||
|
||||
job_key = ClientThreading.JobKey( cancellable = True )
|
||||
|
||||
pub_job_key = True
|
||||
|
||||
|
||||
hash_ids = [ hash_id for ( hash_id, ) in self._c.execute( 'SELECT hash_id FROM shape_maintenance_phash_regen;' ) ]
|
||||
|
||||
# remove hash_id from the pairs cache?
|
||||
# set its search status to False, but don't remove any existing pairs
|
||||
|
||||
client_files_manager = self._controller.GetClientFilesManager()
|
||||
|
||||
num_to_do = len( hash_ids )
|
||||
|
||||
for ( i, hash_id ) in enumerate( hash_ids ):
|
||||
|
||||
job_key.SetVariable( 'popup_title', 'similar files metadata maintenance' )
|
||||
|
||||
if pub_job_key and not job_key_pubbed:
|
||||
|
||||
self._controller.pub( 'message', job_key )
|
||||
|
||||
job_key_pubbed = True
|
||||
|
||||
|
||||
( i_paused, should_quit ) = job_key.WaitIfNeeded()
|
||||
|
||||
if should_quit:
|
||||
should_stop = stop_time is not None and HydrusData.TimeHasPassed( stop_time )
|
||||
|
||||
if should_quit or should_stop:
|
||||
|
||||
return
|
||||
|
||||
|
@ -1823,31 +1913,44 @@ class DB( HydrusDB.HydrusDB ):
|
|||
|
||||
correct_phash_ids = self._CacheSimilarFilesAssociatePHashes( hash_id, phashes )
|
||||
|
||||
deletee_phash_ids = existing_phash_ids.difference( correct_phash_ids )
|
||||
incorrect_phash_ids = existing_phash_ids.difference( correct_phash_ids )
|
||||
|
||||
self._CacheSimilarFilesDelete( hash_id, deletee_phash_ids )
|
||||
if len( incorrect_phash_ids ) > 0:
|
||||
|
||||
self._CacheSimilarFilesDisassociatePHashes( hash_id, incorrect_phash_ids )
|
||||
|
||||
|
||||
self._c.execute( 'DELETE FROM shape_maintenance_phash_regen WHERE hash_id = ?;', ( hash_id, ) )
|
||||
|
||||
|
||||
job_key.SetVariable( 'popup_text_1', 'done!' )
|
||||
job_key.DeleteVariable( 'popup_gauge_1' )
|
||||
|
||||
job_key.Finish()
|
||||
job_key.Delete( 30 )
|
||||
|
||||
|
||||
def _CacheSimilarFilesMaintainTree( self, stop_time ):
|
||||
def _CacheSimilarFilesMaintainTree( self, job_key = None, stop_time = None ):
|
||||
|
||||
job_key = ClientThreading.JobKey( cancellable = True )
|
||||
pub_job_key = False
|
||||
job_key_pubbed = False
|
||||
|
||||
if job_key is None:
|
||||
|
||||
job_key = ClientThreading.JobKey( cancellable = True )
|
||||
|
||||
pub_job_key = True
|
||||
|
||||
|
||||
job_key.SetVariable( 'popup_title', 'similar files metadata maintenance' )
|
||||
|
||||
job_key_pubbed = False
|
||||
|
||||
rebalance_phash_ids = [ phash_id for ( phash_id, ) in self._c.execute( 'SELECT phash_id FROM shape_maintenance_branch_regen;' ) ]
|
||||
|
||||
num_to_do = len( rebalance_phash_ids )
|
||||
|
||||
while len( rebalance_phash_ids ) > 0:
|
||||
|
||||
if not job_key_pubbed:
|
||||
if pub_job_key and not job_key_pubbed:
|
||||
|
||||
self._controller.pub( 'message', job_key )
|
||||
|
||||
|
@ -1856,14 +1959,16 @@ class DB( HydrusDB.HydrusDB ):
|
|||
|
||||
( i_paused, should_quit ) = job_key.WaitIfNeeded()
|
||||
|
||||
if should_quit or HydrusData.TimeHasPassed( stop_time ):
|
||||
should_stop = stop_time is not None and HydrusData.TimeHasPassed( stop_time )
|
||||
|
||||
if should_quit or should_stop:
|
||||
|
||||
return
|
||||
|
||||
|
||||
num_done = num_to_do - len( rebalance_phash_ids )
|
||||
|
||||
text = 'regenerating unbalanced similar file search data - ' + HydrusData.ConvertValueRangeToPrettyString( num_done, num_to_do )
|
||||
text = 'rebalancing similar file metadata - ' + HydrusData.ConvertValueRangeToPrettyString( num_done, num_to_do )
|
||||
|
||||
HydrusGlobals.client_controller.pub( 'splash_set_status_text', text )
|
||||
job_key.SetVariable( 'popup_text_1', text )
|
||||
|
@ -1881,7 +1986,7 @@ class DB( HydrusDB.HydrusDB ):
|
|||
|
||||
job_key.SetVariable( 'popup_text_1', 'done!' )
|
||||
job_key.DeleteVariable( 'popup_gauge_1' )
|
||||
job_key.DeleteVariable( 'popup_text_2' )
|
||||
job_key.DeleteVariable( 'popup_text_2' ) # used in the regenbranch call
|
||||
|
||||
job_key.Finish()
|
||||
job_key.Delete( 30 )
|
||||
|
@ -3052,7 +3157,7 @@ class DB( HydrusDB.HydrusDB ):
|
|||
|
||||
for hash_id in hash_ids:
|
||||
|
||||
self._CacheSimilarFilesDelete( hash_id )
|
||||
self._CacheSimilarFilesDeleteFile( hash_id )
|
||||
|
||||
|
||||
|
||||
|
@ -5144,7 +5249,7 @@ class DB( HydrusDB.HydrusDB ):
|
|||
|
||||
# tag parents
|
||||
|
||||
pending = [ ( ( self._GetNamespaceTag( child_namespace_id, child_tag_id ), self._GetNamespaceTag( parent_namespace_id, parent_tag_id ) ), self._GetText( reason_id ) ) for ( child_namespace_id, child_tag_id, parent_namespace_id, parent_tag_id, reason_id ) in self._c.execute( 'SELECT child_namespace_id, child_tag_id, parent_namespace_id, parent_tag_id, reason_id FROM tag_parent_petitions WHERE service_id = ? AND status = ? ORDER BY reason_id LIMIT 100;', ( service_id, HC.PENDING ) ).fetchall() ]
|
||||
pending = [ ( ( self._GetNamespaceTag( child_namespace_id, child_tag_id ), self._GetNamespaceTag( parent_namespace_id, parent_tag_id ) ), self._GetText( reason_id ) ) for ( child_namespace_id, child_tag_id, parent_namespace_id, parent_tag_id, reason_id ) in self._c.execute( 'SELECT child_namespace_id, child_tag_id, parent_namespace_id, parent_tag_id, reason_id FROM tag_parent_petitions WHERE service_id = ? AND status = ? ORDER BY reason_id LIMIT 1;', ( service_id, HC.PENDING ) ).fetchall() ]
|
||||
|
||||
if len( pending ) > 0:
|
||||
|
||||
|
@ -5209,7 +5314,6 @@ class DB( HydrusDB.HydrusDB ):
|
|||
|
||||
|
||||
|
||||
|
||||
if len( content_data_dict ) > 0:
|
||||
|
||||
hash_ids_to_hashes = self._GetHashIdsToHashes( all_hash_ids )
|
||||
|
@ -8855,7 +8959,18 @@ class DB( HydrusDB.HydrusDB ):
|
|||
|
||||
combined_local_file_service_id = self._GetServiceId( CC.COMBINED_LOCAL_FILE_SERVICE_KEY )
|
||||
|
||||
self._c.execute( 'INSERT OR IGNORE INTO shape_search_cache SELECT hash_id, NULL FROM current_files, files_info USING ( hash_id ) WHERE service_id = ? and mime IN ' + HydrusData.SplayListForDB( HC.MIMES_WE_CAN_PHASH ) + ';', ( combined_local_file_service_id, ) )
|
||||
self._c.execute( 'INSERT OR IGNORE INTO shape_search_cache SELECT hash_id, NULL FROM current_files, files_info USING ( hash_id ) WHERE service_id = ? and mime IN ( ?, ? );', ( combined_local_file_service_id, HC.IMAGE_JPEG, HC.IMAGE_PNG ) )
|
||||
|
||||
|
||||
if version == 240:
|
||||
|
||||
combined_local_file_service_id = self._GetServiceId( CC.COMBINED_LOCAL_FILE_SERVICE_KEY )
|
||||
|
||||
self._c.execute( 'INSERT OR IGNORE INTO shape_maintenance_phash_regen SELECT hash_id FROM current_files, files_info USING ( hash_id ) WHERE service_id = ? AND mime IN ( ?, ? );', ( combined_local_file_service_id, HC.IMAGE_JPEG, HC.IMAGE_PNG ) )
|
||||
|
||||
#
|
||||
|
||||
self._c.execute( 'DELETE FROM web_sessions WHERE name = ?;', ( 'pixiv', ) )
|
||||
|
||||
|
||||
self._controller.pub( 'splash_set_title_text', 'updated db to v' + str( version + 1 ) )
|
||||
|
@ -9505,12 +9620,15 @@ class DB( HydrusDB.HydrusDB ):
|
|||
elif action == 'delete_remote_booru': result = self._DeleteYAMLDump( YAML_DUMP_ID_REMOTE_BOORU, *args, **kwargs )
|
||||
elif action == 'delete_serialisable_named': result = self._DeleteJSONDumpNamed( *args, **kwargs )
|
||||
elif action == 'delete_service_info': result = self._DeleteServiceInfo( *args, **kwargs )
|
||||
elif action == 'delete_unknown_duplicate_pairs': result = self._CacheSimilarFilesDeleteUnknownDuplicatePairs( *args, **kwargs )
|
||||
elif action == 'export_mappings': result = self._ExportToTagArchive( *args, **kwargs )
|
||||
elif action == 'file_integrity': result = self._CheckFileIntegrity( *args, **kwargs )
|
||||
elif action == 'hydrus_session': result = self._AddHydrusSession( *args, **kwargs )
|
||||
elif action == 'imageboard': result = self._SetYAMLDump( YAML_DUMP_ID_IMAGEBOARD, *args, **kwargs )
|
||||
elif action == 'import_file': result = self._ImportFile( *args, **kwargs )
|
||||
elif action == 'local_booru_share': result = self._SetYAMLDump( YAML_DUMP_ID_LOCAL_BOORU, *args, **kwargs )
|
||||
elif action == 'maintain_similar_files_duplicate_pairs': result = self._CacheSimilarFilesMaintainDuplicatePairs( *args, **kwargs )
|
||||
elif action == 'maintain_similar_files_phashes': result = self._CacheSimilarFilesMaintainFiles( *args, **kwargs )
|
||||
elif action == 'maintain_similar_files_tree': result = self._CacheSimilarFilesMaintainTree( *args, **kwargs )
|
||||
elif action == 'push_recent_tags': result = self._PushRecentTags( *args, **kwargs )
|
||||
elif action == 'regenerate_ac_cache': result = self._RegenerateACCache( *args, **kwargs )
|
||||
|
|
|
@ -55,6 +55,8 @@ def CatchExceptionClient( etype, value, tb ):
|
|||
first_line = pretty_value
|
||||
|
||||
|
||||
trace = HydrusData.ToUnicode( trace )
|
||||
|
||||
job_key = ClientThreading.JobKey()
|
||||
|
||||
if etype == HydrusExceptions.ShutdownException:
|
||||
|
@ -360,6 +362,8 @@ def ShowExceptionClient( e ):
|
|||
trace = ''.join( traceback.format_exception( etype, value, tb ) )
|
||||
|
||||
|
||||
trace = HydrusData.ToUnicode( trace )
|
||||
|
||||
pretty_value = HydrusData.ToUnicode( value )
|
||||
|
||||
if os.linesep in pretty_value:
|
||||
|
@ -512,7 +516,7 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
|
|||
|
||||
if db_dir is None:
|
||||
|
||||
db_dir = os.path.join( HC.BASE_DIR, 'db' )
|
||||
db_dir = HC.DEFAULT_DB_DIR
|
||||
|
||||
|
||||
self._dictionary = HydrusSerialisable.SerialisableDictionary()
|
||||
|
@ -570,6 +574,8 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
|
|||
|
||||
self._dictionary[ 'integers' ][ 'suggested_tags_width' ] = 300
|
||||
|
||||
self._dictionary[ 'integers' ][ 'similar_files_duplicate_pairs_search_distance' ] = 0
|
||||
|
||||
#
|
||||
|
||||
self._dictionary[ 'keys' ] = {}
|
||||
|
|
|
@ -224,7 +224,7 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
|
|||
|
||||
aboutinfo.SetDescription( description )
|
||||
|
||||
with open( os.path.join( HC.BASE_DIR, 'license.txt' ), 'rb' ) as f: license = f.read()
|
||||
with open( HC.LICENSE_PATH, 'rb' ) as f: license = f.read()
|
||||
|
||||
aboutinfo.SetLicense( license )
|
||||
|
||||
|
@ -1541,7 +1541,7 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
|
|||
|
||||
stop_time = HydrusData.GetNow() + 60 * 10
|
||||
|
||||
self._controller.Write( 'maintain_similar_files_tree', stop_time )
|
||||
self._controller.Write( 'maintain_similar_files_tree', stop_time = stop_time )
|
||||
|
||||
|
||||
|
||||
|
|
|
@ -1052,10 +1052,10 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
|
|||
|
||||
self._file_search_context.SetIncludeCurrentTags( value )
|
||||
|
||||
|
||||
wx.CallAfter( self.RefreshList )
|
||||
|
||||
HydrusGlobals.client_controller.pub( 'refresh_query', self._page_key )
|
||||
wx.CallAfter( self.RefreshList )
|
||||
|
||||
HydrusGlobals.client_controller.pub( 'refresh_query', self._page_key )
|
||||
|
||||
|
||||
|
||||
def IncludePending( self, page_key, value ):
|
||||
|
@ -1064,10 +1064,10 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
|
|||
|
||||
self._file_search_context.SetIncludePendingTags( value )
|
||||
|
||||
|
||||
wx.CallAfter( self.RefreshList )
|
||||
|
||||
HydrusGlobals.client_controller.pub( 'refresh_query', self._page_key )
|
||||
wx.CallAfter( self.RefreshList )
|
||||
|
||||
HydrusGlobals.client_controller.pub( 'refresh_query', self._page_key )
|
||||
|
||||
|
||||
|
||||
def SetSynchronisedWait( self, page_key ):
|
||||
|
|
|
@ -185,6 +185,23 @@ class AnimatedStaticTextTimestamp( wx.StaticText ):
|
|||
|
||||
|
||||
|
||||
class BetterBitmapButton( wx.BitmapButton ):
|
||||
|
||||
def __init__( self, parent, bitmap, callable, *args, **kwargs ):
|
||||
|
||||
wx.BitmapButton.__init__( self, parent, bitmap = bitmap )
|
||||
|
||||
self._callable = callable
|
||||
self._args = args
|
||||
self._kwargs = kwargs
|
||||
self.Bind( wx.EVT_BUTTON, self.EventButton )
|
||||
|
||||
|
||||
def EventButton( self, event ):
|
||||
|
||||
self._callable( *self._args, **self._kwargs )
|
||||
|
||||
|
||||
class BetterButton( wx.Button ):
|
||||
|
||||
def __init__( self, parent, label, callable, *args, **kwargs ):
|
||||
|
@ -3207,6 +3224,27 @@ class ListCtrlAutoWidth( wx.ListCtrl, ListCtrlAutoWidthMixin ):
|
|||
for index in indices: self.DeleteItem( index )
|
||||
|
||||
|
||||
class MenuBitmapButton( BetterBitmapButton ):
|
||||
|
||||
def __init__( self, parent, bitmap, menu_items ):
|
||||
|
||||
BetterBitmapButton.__init__( self, parent, bitmap, self.DoMenu )
|
||||
|
||||
self._menu_items = menu_items
|
||||
|
||||
|
||||
def DoMenu( self ):
|
||||
|
||||
menu = wx.Menu()
|
||||
|
||||
for ( title, description, callable ) in self._menu_items:
|
||||
|
||||
ClientGUIMenus.AppendMenuItem( menu, title, description, self, callable )
|
||||
|
||||
|
||||
HydrusGlobals.client_controller.PopupMenu( self, menu )
|
||||
|
||||
|
||||
class MenuButton( BetterButton ):
|
||||
|
||||
def __init__( self, parent, label, menu_items ):
|
||||
|
@ -5186,7 +5224,7 @@ class SaneListCtrlForSingleObject( SaneListCtrl ):
|
|||
SaneListCtrl.Append( self, display_tuple, sort_tuple )
|
||||
|
||||
|
||||
def GetIndexFromClientData( self, obj ):
|
||||
def GetIndexFromObject( self, obj ):
|
||||
|
||||
try:
|
||||
|
||||
|
@ -5227,11 +5265,11 @@ class SaneListCtrlForSingleObject( SaneListCtrl ):
|
|||
return datas
|
||||
|
||||
|
||||
def HasClientData( self, data ):
|
||||
def HasObject( self, obj ):
|
||||
|
||||
try:
|
||||
|
||||
index = self.GetIndexFromClientData( data )
|
||||
index = self.GetIndexFromObject( obj )
|
||||
|
||||
return True
|
||||
|
||||
|
@ -5247,7 +5285,7 @@ class SaneListCtrlForSingleObject( SaneListCtrl ):
|
|||
|
||||
name = obj.GetName()
|
||||
|
||||
current_names = { obj.GetName() for obj in self.GetClientData() }
|
||||
current_names = { obj.GetName() for obj in self.GetObjects() }
|
||||
|
||||
if name in current_names:
|
||||
|
||||
|
@ -5407,9 +5445,9 @@ class SeedCacheControl( SaneListCtrlForSingleObject ):
|
|||
|
||||
if self._seed_cache.HasSeed( seed ):
|
||||
|
||||
if self.HasClientData( seed ):
|
||||
if self.HasObject( seed ):
|
||||
|
||||
index = self.GetIndexFromClientData( seed )
|
||||
index = self.GetIndexFromObject( seed )
|
||||
|
||||
( display_tuple, sort_tuple ) = self._GetListCtrlTuples( seed )
|
||||
|
||||
|
@ -5422,9 +5460,9 @@ class SeedCacheControl( SaneListCtrlForSingleObject ):
|
|||
|
||||
else:
|
||||
|
||||
if self.HasClientData( seed ):
|
||||
if self.HasObject( seed ):
|
||||
|
||||
index = self.GetIndexFromClientData( seed )
|
||||
index = self.GetIndexFromObject( seed )
|
||||
|
||||
self.DeleteItem( index )
|
||||
|
||||
|
|
|
@ -1327,7 +1327,6 @@ class DialogInputFileSystemPredicates( Dialog ):
|
|||
self._predicate_panel = predicate_class( self )
|
||||
|
||||
self._ok = wx.Button( self, id = wx.ID_OK, label = 'Ok' )
|
||||
self._ok.SetDefault()
|
||||
self._ok.Bind( wx.EVT_BUTTON, self.EventOK )
|
||||
self._ok.SetForegroundColour( ( 0, 128, 0 ) )
|
||||
|
||||
|
@ -1338,14 +1337,33 @@ class DialogInputFileSystemPredicates( Dialog ):
|
|||
|
||||
self.SetSizer( hbox )
|
||||
|
||||
self.Bind( wx.EVT_CHAR_HOOK, self.EventCharHook )
|
||||
|
||||
|
||||
def EventOK( self, event ):
|
||||
def _DoOK( self ):
|
||||
|
||||
predicates = self._predicate_panel.GetPredicates()
|
||||
|
||||
self.GetParent().SubPanelOK( predicates )
|
||||
|
||||
|
||||
def EventCharHook( self, event ):
|
||||
|
||||
if event.KeyCode in ( wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER ):
|
||||
|
||||
self._DoOK()
|
||||
|
||||
else:
|
||||
|
||||
event.Skip()
|
||||
|
||||
|
||||
|
||||
def EventOK( self, event ):
|
||||
|
||||
self._DoOK()
|
||||
|
||||
|
||||
|
||||
class DialogInputLocalBooruShare( Dialog ):
|
||||
|
||||
|
|
|
@ -3149,16 +3149,16 @@ class DialogManagePixivAccount( ClientGUIDialogs.Dialog ):
|
|||
|
||||
def EventOK( self, event ):
|
||||
|
||||
id = self._id.GetValue()
|
||||
pixiv_id = self._id.GetValue()
|
||||
password = self._password.GetValue()
|
||||
|
||||
if id == '' and password == '':
|
||||
if pixiv_id == '' and password == '':
|
||||
|
||||
HydrusGlobals.client_controller.Write( 'serialisable_simple', 'pixiv_account', None )
|
||||
|
||||
else:
|
||||
|
||||
HydrusGlobals.client_controller.Write( 'serialisable_simple', 'pixiv_account', ( id, password ) )
|
||||
HydrusGlobals.client_controller.Write( 'serialisable_simple', 'pixiv_account', ( pixiv_id, password ) )
|
||||
|
||||
|
||||
self.EndModal( wx.ID_OK )
|
||||
|
@ -3166,41 +3166,23 @@ class DialogManagePixivAccount( ClientGUIDialogs.Dialog ):
|
|||
|
||||
def EventTest( self, event ):
|
||||
|
||||
id = self._id.GetValue()
|
||||
pixiv_id = self._id.GetValue()
|
||||
password = self._password.GetValue()
|
||||
|
||||
form_fields = {}
|
||||
|
||||
# this no longer seems to work--they updated to some javascript gubbins for their main form
|
||||
# I couldn't see where the POST was going in Firefox dev console, so I guess it is some other thing that doesn't pick up
|
||||
# the old form is still there, but hidden and changed to https://accounts.pixiv.net/login, but even if I do that, it just refreshes in Japanese :/
|
||||
|
||||
form_fields[ 'post_key' ] = 'c779b8a16389a7861d584d11d73424a0'
|
||||
form_fields[ 'lang' ] = 'en'
|
||||
form_fields[ 'source' ] = 'pc'
|
||||
form_fields[ 'return_to' ] = 'http://www.pixiv.net'
|
||||
form_fields[ 'pixiv_id' ] = id
|
||||
form_fields[ 'password' ] = password
|
||||
|
||||
body = urllib.urlencode( form_fields )
|
||||
|
||||
headers = {}
|
||||
headers[ 'Content-Type' ] = 'application/x-www-form-urlencoded'
|
||||
|
||||
( response_gumpf, cookies ) = HydrusGlobals.client_controller.DoHTTP( HC.POST, 'https://accounts.pixiv.net/login', request_headers = headers, body = body, return_cookies = True )
|
||||
|
||||
# actually, it needs an _ in it to be a logged in session
|
||||
# posting to the old login form gives you a 301 and a session without an underscore
|
||||
if 'PHPSESSID' in cookies:
|
||||
try:
|
||||
|
||||
manager = HydrusGlobals.client_controller.GetManager( 'web_sessions' )
|
||||
|
||||
cookies = manager.GetPixivCookies( pixiv_id, password )
|
||||
|
||||
self._status.SetLabelText( 'OK!' )
|
||||
|
||||
else:
|
||||
wx.CallLater( 5000, self._status.SetLabel, '' )
|
||||
|
||||
self._status.SetLabelText( 'Did not work!' )
|
||||
except HydrusExceptions.ForbiddenException as e:
|
||||
|
||||
self._status.SetLabelText( 'Did not work! ' + repr( e ) )
|
||||
|
||||
|
||||
wx.CallLater( 2000, self._status.SetLabel, '' )
|
||||
|
||||
|
||||
class DialogManageRatings( ClientGUIDialogs.Dialog ):
|
||||
|
|
|
@ -1445,17 +1445,25 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
|
|||
|
||||
ManagementPanel.__init__( self, parent, page, controller, management_controller )
|
||||
|
||||
self._job = None
|
||||
self._job_key = None
|
||||
|
||||
menu_items = []
|
||||
|
||||
menu_items.append( ( 'refresh', 'This panel does not update itself when files are added or deleted elsewhere in the client. Hitting this will refresh the numbers from the database.', self._RefreshAndUpdateStatus ) )
|
||||
menu_items.append( ( 'reset potentials', 'This will delete all the potential duplicate pairs found so far and reset their files\' search status.', self._ResetUnknown ) )
|
||||
|
||||
self._cog_button = ClientGUICommon.MenuBitmapButton( self, CC.GlobalBMPs.cog, menu_items )
|
||||
|
||||
self._preparing_panel = ClientGUICommon.StaticBox( self, 'preparation' )
|
||||
|
||||
# refresh button that just calls update
|
||||
|
||||
self._total_files = wx.StaticText( self._preparing_panel )
|
||||
|
||||
self._num_phashes_to_regen = wx.StaticText( self._preparing_panel )
|
||||
self._num_branches_to_regen = wx.StaticText( self._preparing_panel )
|
||||
|
||||
self._phashes_button = wx.BitmapButton( self._preparing_panel, bitmap = CC.GlobalBMPs.play )
|
||||
self._branches_button = wx.BitmapButton( self._preparing_panel, bitmap = CC.GlobalBMPs.play )
|
||||
self._phashes_button = ClientGUICommon.BetterBitmapButton( self._preparing_panel, CC.GlobalBMPs.play, self._RegeneratePhashes )
|
||||
self._branches_button = ClientGUICommon.BetterBitmapButton( self._preparing_panel, CC.GlobalBMPs.play, self._RebalanceTree )
|
||||
|
||||
#
|
||||
|
||||
|
@ -1471,10 +1479,11 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
|
|||
self._search_distance_button = ClientGUICommon.MenuButton( self._searching_panel, 'similarity', menu_items )
|
||||
|
||||
self._search_distance_spinctrl = wx.SpinCtrl( self._searching_panel, min = 0, max = 64, size = ( 50, -1 ) )
|
||||
self._search_distance_spinctrl.Bind( wx.EVT_SPINCTRL, self.EventSearchDistanceChanged )
|
||||
|
||||
self._num_searched = ClientGUICommon.TextAndGauge( self._searching_panel )
|
||||
|
||||
self._search_button = wx.BitmapButton( self._searching_panel, bitmap = CC.GlobalBMPs.play )
|
||||
self._search_button = ClientGUICommon.BetterBitmapButton( self._searching_panel, CC.GlobalBMPs.play, self._SearchForDuplicates )
|
||||
|
||||
#
|
||||
|
||||
|
@ -1484,30 +1493,32 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
|
|||
self._num_same_file_duplicates = wx.StaticText( self._filtering_panel )
|
||||
self._num_alternate_duplicates = wx.StaticText( self._filtering_panel )
|
||||
|
||||
# bind spinctrl (which should throw another update on every shift, as well
|
||||
# bind the buttons (nah, replace with betterbitmapbutton)
|
||||
#
|
||||
|
||||
new_options = self._controller.GetNewOptions()
|
||||
|
||||
self._search_distance_spinctrl.SetValue( new_options.GetInteger( 'similar_files_duplicate_pairs_search_distance' ) )
|
||||
|
||||
#
|
||||
|
||||
# initialise value of spinctrl, label of distance button (which might be 'custom')
|
||||
|
||||
gridbox_1 = wx.FlexGridSizer( 0, 2 )
|
||||
gridbox_1 = wx.FlexGridSizer( 0, 3 )
|
||||
|
||||
gridbox_1.AddGrowableCol( 0, 1 )
|
||||
|
||||
gridbox_1.AddF( self._num_phashes_to_regen, CC.FLAGS_EXPAND_PERPENDICULAR )
|
||||
gridbox_1.AddF( self._num_phashes_to_regen, CC.FLAGS_VCENTER )
|
||||
gridbox_1.AddF( ( 10, 10 ), CC.FLAGS_EXPAND_PERPENDICULAR )
|
||||
gridbox_1.AddF( self._phashes_button, CC.FLAGS_VCENTER )
|
||||
gridbox_1.AddF( self._num_branches_to_regen, CC.FLAGS_EXPAND_PERPENDICULAR )
|
||||
gridbox_1.AddF( self._num_branches_to_regen, CC.FLAGS_VCENTER )
|
||||
gridbox_1.AddF( ( 10, 10 ), CC.FLAGS_EXPAND_PERPENDICULAR )
|
||||
gridbox_1.AddF( self._branches_button, CC.FLAGS_VCENTER )
|
||||
|
||||
self._preparing_panel.AddF( self._total_files, CC.FLAGS_EXPAND_PERPENDICULAR )
|
||||
self._preparing_panel.AddF( gridbox_1, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
|
||||
|
||||
#
|
||||
|
||||
distance_hbox = wx.BoxSizer( wx.HORIZONTAL )
|
||||
|
||||
distance_hbox.AddF( wx.StaticText( self._searching_panel, label = 'search similarity: ' ), CC.FLAGS_VCENTER )
|
||||
distance_hbox.AddF( wx.StaticText( self._searching_panel, label = 'search distance: ' ), CC.FLAGS_VCENTER )
|
||||
distance_hbox.AddF( self._search_distance_button, CC.FLAGS_EXPAND_BOTH_WAYS )
|
||||
distance_hbox.AddF( self._search_distance_spinctrl, CC.FLAGS_VCENTER )
|
||||
|
||||
|
@ -1531,22 +1542,63 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
|
|||
|
||||
vbox = wx.BoxSizer( wx.VERTICAL )
|
||||
|
||||
vbox.AddF( self._cog_button, CC.FLAGS_LONE_BUTTON )
|
||||
vbox.AddF( self._preparing_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
|
||||
vbox.AddF( self._searching_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
|
||||
vbox.AddF( self._filtering_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
|
||||
|
||||
self.SetSizer( vbox )
|
||||
|
||||
self.Bind( wx.EVT_TIMER, self.TIMEREventUpdateDBJob, id = ID_TIMER_UPDATE )
|
||||
self._update_db_job_timer = wx.Timer( self, id = ID_TIMER_UPDATE )
|
||||
|
||||
#
|
||||
|
||||
self._RefreshAndUpdate()
|
||||
self._RefreshAndUpdateStatus()
|
||||
|
||||
|
||||
def _RebalanceTree( self ):
|
||||
|
||||
self._job = 'branches'
|
||||
|
||||
self._StartStopDBJob()
|
||||
|
||||
|
||||
def _RegeneratePhashes( self ):
|
||||
|
||||
self._job = 'phashes'
|
||||
|
||||
self._StartStopDBJob()
|
||||
|
||||
|
||||
def _ResetUnknown( self ):
|
||||
|
||||
text = 'This will delete all the potential duplicate pairs and reset their files\' search status.'
|
||||
text += os.linesep * 2
|
||||
text += 'This can be useful if you have accidentally searched too broadly and are now swamped with too many false positives.'
|
||||
|
||||
with ClientGUIDialogs.DialogYesNo( self, text ) as dlg:
|
||||
|
||||
if dlg.ShowModal() == wx.ID_YES:
|
||||
|
||||
self._controller.Write( 'delete_unknown_duplicate_pairs' )
|
||||
|
||||
self._RefreshAndUpdateStatus()
|
||||
|
||||
|
||||
|
||||
def _SearchForDuplicates( self ):
|
||||
|
||||
self._job = 'search'
|
||||
|
||||
self._StartStopDBJob()
|
||||
|
||||
|
||||
def _SetSearchDistance( self, value ):
|
||||
|
||||
self._search_distance_spinctrl.SetValue( value ) # does this trigger the update event? check it
|
||||
self._search_distance_spinctrl.SetValue( value )
|
||||
|
||||
# update the label, which prob needs an HC.hamming_str dict or something, which I can then apply everywhere else as well.
|
||||
self._UpdateStatus()
|
||||
|
||||
|
||||
def _SetSearchDistanceExact( self ):
|
||||
|
@ -1569,26 +1621,126 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
|
|||
self._SetSearchDistance( HC.HAMMING_VERY_SIMILAR )
|
||||
|
||||
|
||||
def _RefreshAndUpdate( self ):
|
||||
def _StartStopDBJob( self ):
|
||||
|
||||
if self._job_key is None:
|
||||
|
||||
self._cog_button.Disable()
|
||||
self._phashes_button.Disable()
|
||||
self._branches_button.Disable()
|
||||
self._search_button.Disable()
|
||||
self._search_distance_button.Disable()
|
||||
self._search_distance_spinctrl.Disable()
|
||||
|
||||
self._job_key = ClientThreading.JobKey( cancellable = True )
|
||||
|
||||
if self._job == 'phashes':
|
||||
|
||||
self._phashes_button.Enable()
|
||||
self._phashes_button.SetBitmap( CC.GlobalBMPs.stop )
|
||||
|
||||
self._controller.Write( 'maintain_similar_files_phashes', job_key = self._job_key )
|
||||
|
||||
elif self._job == 'branches':
|
||||
|
||||
self._branches_button.Enable()
|
||||
self._branches_button.SetBitmap( CC.GlobalBMPs.stop )
|
||||
|
||||
self._controller.Write( 'maintain_similar_files_tree', job_key = self._job_key )
|
||||
|
||||
elif self._job == 'search':
|
||||
|
||||
self._search_button.Enable()
|
||||
self._search_button.SetBitmap( CC.GlobalBMPs.stop )
|
||||
|
||||
search_distance = self._search_distance_spinctrl.GetValue()
|
||||
|
||||
self._controller.Write( 'maintain_similar_files_duplicate_pairs', search_distance, job_key = self._job_key )
|
||||
|
||||
|
||||
self._update_db_job_timer.Start( 250, wx.TIMER_CONTINUOUS )
|
||||
|
||||
else:
|
||||
|
||||
self._job_key.Cancel()
|
||||
|
||||
|
||||
|
||||
def _UpdateJob( self ):
|
||||
|
||||
if self._job_key.IsDone():
|
||||
|
||||
self._job_key = None
|
||||
|
||||
self._update_db_job_timer.Stop()
|
||||
|
||||
self._RefreshAndUpdateStatus()
|
||||
|
||||
return
|
||||
|
||||
|
||||
if self._job == 'phashes':
|
||||
|
||||
text = self._job_key.GetIfHasVariable( 'popup_text_1' )
|
||||
|
||||
if text is not None:
|
||||
|
||||
self._num_phashes_to_regen.SetLabelText( text )
|
||||
|
||||
|
||||
elif self._job == 'branches':
|
||||
|
||||
text = self._job_key.GetIfHasVariable( 'popup_text_1' )
|
||||
|
||||
if text is not None:
|
||||
|
||||
self._num_branches_to_regen.SetLabelText( text )
|
||||
|
||||
|
||||
elif self._job == 'search':
|
||||
|
||||
text = self._job_key.GetIfHasVariable( 'popup_text_1' )
|
||||
gauge = self._job_key.GetIfHasVariable( 'popup_gauge_1' )
|
||||
|
||||
if text is not None and gauge is not None:
|
||||
|
||||
( value, range ) = gauge
|
||||
|
||||
self._num_searched.SetValue( text, value, range )
|
||||
|
||||
|
||||
|
||||
|
||||
def _RefreshAndUpdateStatus( self ):
|
||||
|
||||
self._similar_files_maintenance_status = self._controller.Read( 'similar_files_maintenance_status' )
|
||||
|
||||
self._Update()
|
||||
self._UpdateStatus()
|
||||
|
||||
|
||||
def _Update( self ):
|
||||
def _UpdateStatus( self ):
|
||||
|
||||
( searched_distances_to_count, duplicate_types_to_count, num_phashes_to_regen, num_branches_to_regen ) = self._similar_files_maintenance_status
|
||||
|
||||
self._cog_button.Enable()
|
||||
|
||||
self._phashes_button.SetBitmap( CC.GlobalBMPs.play )
|
||||
self._branches_button.SetBitmap( CC.GlobalBMPs.play )
|
||||
self._search_button.SetBitmap( CC.GlobalBMPs.play )
|
||||
|
||||
total_num_files = sum( searched_distances_to_count.values() )
|
||||
|
||||
if num_phashes_to_regen == 0:
|
||||
|
||||
self._num_phashes_to_regen.SetLabelText( 'All files ready!' )
|
||||
self._num_phashes_to_regen.SetLabelText( 'All ' + HydrusData.ConvertIntToPrettyString( total_num_files ) + ' eligible files up to date!' )
|
||||
|
||||
self._phashes_button.Disable()
|
||||
|
||||
else:
|
||||
|
||||
self._num_phashes_to_regen.SetLabelText( HydrusData.ConvertIntToPrettyString( num_phashes_to_regen ) + ' files to reanalyze.' )
|
||||
num_done = total_num_files - num_phashes_to_regen
|
||||
|
||||
self._num_phashes_to_regen.SetLabelText( HydrusData.ConvertValueRangeToPrettyString( num_done, total_num_files ) + 'eligible files up to date.' )
|
||||
|
||||
self._phashes_button.Enable()
|
||||
|
||||
|
@ -1606,17 +1758,31 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
|
|||
self._branches_button.Enable()
|
||||
|
||||
|
||||
total_num_files = sum( searched_distances_to_count.values() )
|
||||
|
||||
self._total_files.SetLabelText( HydrusData.ConvertIntToPrettyString( total_num_files ) + ' eligable files.' )
|
||||
self._search_distance_button.Enable()
|
||||
self._search_distance_spinctrl.Enable()
|
||||
|
||||
search_distance = self._search_distance_spinctrl.GetValue()
|
||||
|
||||
new_options = self._controller.GetNewOptions()
|
||||
|
||||
new_options.SetInteger( 'similar_files_duplicate_pairs_search_distance', search_distance )
|
||||
|
||||
if search_distance in HC.hamming_string_lookup:
|
||||
|
||||
button_label = HC.hamming_string_lookup[ search_distance ]
|
||||
|
||||
else:
|
||||
|
||||
button_label = 'custom'
|
||||
|
||||
|
||||
self._search_distance_button.SetLabelText( button_label )
|
||||
|
||||
num_searched = sum( ( count for ( value, count ) in searched_distances_to_count.items() if value is not None and value >= search_distance ) )
|
||||
|
||||
if num_searched == total_num_files:
|
||||
|
||||
self._num_searched.SetValue( 'All potential duplicates found.', total_num_files, total_num_files )
|
||||
self._num_searched.SetValue( 'All potential duplicates found at this distance.', total_num_files, total_num_files )
|
||||
|
||||
self._search_button.Disable()
|
||||
|
||||
|
@ -1624,21 +1790,33 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
|
|||
|
||||
if num_searched == 0:
|
||||
|
||||
self._num_searched.SetValue( 'Have not yet searched at that distance.', 0, total_num_files )
|
||||
self._num_searched.SetValue( 'Have not yet searched at this distance.', 0, total_num_files )
|
||||
|
||||
else:
|
||||
|
||||
self._num_searched.SetValue( 'Searched ' + HydrusData.ConvertValueRangeToPrettyString( num_searched, total_num_files ) + ' files.', num_searched, total_num_files )
|
||||
self._num_searched.SetValue( 'Searched ' + HydrusData.ConvertValueRangeToPrettyString( num_searched, total_num_files ) + ' files at this distance.', num_searched, total_num_files )
|
||||
|
||||
|
||||
self._search_button.Enable()
|
||||
|
||||
|
||||
self._num_unknown_duplicates.SetLabelText( HydrusData.ConvertIntToPrettyString( duplicate_types_to_count[ HC.DUPLICATE_UNKNOWN ] ) + ' potential duplicates found.' )
|
||||
num_unknown = duplicate_types_to_count[ HC.DUPLICATE_UNKNOWN ]
|
||||
|
||||
self._num_unknown_duplicates.SetLabelText( HydrusData.ConvertIntToPrettyString( num_unknown ) + ' potential duplicates found.' )
|
||||
self._num_same_file_duplicates.SetLabelText( HydrusData.ConvertIntToPrettyString( duplicate_types_to_count[ HC.DUPLICATE_SAME_FILE ] ) + ' same file pairs filtered.' )
|
||||
self._num_alternate_duplicates.SetLabelText( HydrusData.ConvertIntToPrettyString( duplicate_types_to_count[ HC.DUPLICATE_ALTERNATE ] ) + ' alternate file pairs filtered.' )
|
||||
|
||||
|
||||
def EventSearchDistanceChanged( self, event ):
|
||||
|
||||
self._UpdateStatus()
|
||||
|
||||
|
||||
def TIMEREventUpdateDBJob( self, event ):
|
||||
|
||||
self._UpdateJob()
|
||||
|
||||
|
||||
management_panel_types_to_classes[ MANAGEMENT_TYPE_DUPLICATE_FILTER ] = ManagementPanelDuplicateFilter
|
||||
|
||||
class ManagementPanelGalleryImport( ManagementPanel ):
|
||||
|
|
|
@ -152,20 +152,32 @@ def GenerateShapePerceptualHashes( path ):
|
|||
|
||||
dct_88 = dct[:8,:8]
|
||||
|
||||
# get mean of dct, excluding [0,0]
|
||||
# get median of dct
|
||||
# exclude [0,0], which represents flat colour
|
||||
# this [0,0] exclusion is apparently important for mean, but maybe it ain't so important for median--w/e
|
||||
|
||||
mask = numpy.ones( ( 8, 8 ) )
|
||||
# old mean code
|
||||
# mask = numpy.ones( ( 8, 8 ) )
|
||||
# mask[0,0] = 0
|
||||
# average = numpy.average( dct_88, weights = mask )
|
||||
|
||||
mask[0,0] = 0
|
||||
median = numpy.median( dct_88.reshape( 64 )[1:] )
|
||||
|
||||
average = numpy.average( dct_88, weights = mask )
|
||||
# make a monochromatic, 64-bit hash of whether the entry is above or below the median
|
||||
|
||||
# make a monochromatic, 64-bit hash of whether the entry is above or below the mean
|
||||
dct_88_boolean = dct_88 > median
|
||||
|
||||
# convert TTTFTFTF to 11101010 by repeatedly shifting answer and adding 0 or 1
|
||||
# you can even go ( a << 1 ) + b and leave out the initial param on the latel reduce call as bools act like ints for this
|
||||
# but let's not go crazy for another two nanoseconds
|
||||
collapse_bools_to_binary_uint = lambda a, b: ( a << 1 ) + int( b )
|
||||
|
||||
bytes = []
|
||||
|
||||
for i in range( 8 ):
|
||||
|
||||
'''
|
||||
# old way of doing it, which compared value to median every time
|
||||
byte = 0
|
||||
|
||||
for j in range( 8 ):
|
||||
|
@ -174,15 +186,27 @@ def GenerateShapePerceptualHashes( path ):
|
|||
|
||||
value = dct_88[i,j]
|
||||
|
||||
if value > average: byte |= 1
|
||||
if value > median:
|
||||
|
||||
byte |= 1
|
||||
|
||||
|
||||
'''
|
||||
|
||||
byte = reduce( collapse_bools_to_binary_uint, dct_88_boolean[i], 0 )
|
||||
|
||||
bytes.append( byte )
|
||||
|
||||
|
||||
phash = str( bytearray( bytes ) )
|
||||
|
||||
phashes = [ phash ]
|
||||
# now discard the blank hash, which is 1000000... and not useful
|
||||
|
||||
phashes = set()
|
||||
|
||||
phashes.add( phash )
|
||||
|
||||
phashes.discard( CC.BLANK_PHASH )
|
||||
|
||||
# we good
|
||||
|
||||
|
@ -213,4 +237,4 @@ def ResizeNumpyImage( mime, numpy_image, ( target_x, target_y ) ):
|
|||
|
||||
return cv2.resize( numpy_image, ( target_x, target_y ), interpolation = interpolation )
|
||||
|
||||
|
||||
|
||||
|
|
|
@ -616,6 +616,13 @@ class HTTPConnection( object ):
|
|||
|
||||
if attempt_number <= 3:
|
||||
|
||||
if self._hydrus_network:
|
||||
|
||||
# we are talking to a new hydrus server, which uses https, and hence an http call gives badstatusline
|
||||
|
||||
self._scheme = 'https'
|
||||
|
||||
|
||||
self._RefreshConnection()
|
||||
|
||||
return self._GetInitialResponse( method, path_and_query, request_headers, body, attempt_number = attempt_number + 1 )
|
||||
|
@ -638,6 +645,13 @@ class HTTPConnection( object ):
|
|||
|
||||
if attempt_number <= 3:
|
||||
|
||||
if self._hydrus_network:
|
||||
|
||||
# we are talking to a new hydrus server, which uses https, and hence an http call gives badstatusline
|
||||
|
||||
self._scheme = 'https'
|
||||
|
||||
|
||||
self._RefreshConnection()
|
||||
|
||||
return self._GetInitialResponse( method_string, path_and_query, request_headers, body, attempt_number = attempt_number + 1 )
|
||||
|
|
|
@ -25,6 +25,9 @@ HELP_DIR = os.path.join( BASE_DIR, 'help' )
|
|||
INCLUDE_DIR = os.path.join( BASE_DIR, 'include' )
|
||||
STATIC_DIR = os.path.join( BASE_DIR, 'static' )
|
||||
|
||||
DEFAULT_DB_DIR = os.path.join( BASE_DIR, 'db' )
|
||||
LICENSE_PATH = os.path.join( BASE_DIR, 'license.txt' )
|
||||
|
||||
#
|
||||
|
||||
PLATFORM_WINDOWS = False
|
||||
|
@ -46,7 +49,7 @@ options = {}
|
|||
# Misc
|
||||
|
||||
NETWORK_VERSION = 17
|
||||
SOFTWARE_VERSION = 240
|
||||
SOFTWARE_VERSION = 241
|
||||
|
||||
UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )
|
||||
|
||||
|
@ -126,6 +129,13 @@ HAMMING_VERY_SIMILAR = 2
|
|||
HAMMING_SIMILAR = 4
|
||||
HAMMING_SPECULATIVE = 8
|
||||
|
||||
hamming_string_lookup = {}
|
||||
|
||||
hamming_string_lookup[ HAMMING_EXACT_MATCH ] = 'exact match'
|
||||
hamming_string_lookup[ HAMMING_VERY_SIMILAR ] = 'very similar'
|
||||
hamming_string_lookup[ HAMMING_SIMILAR ] = 'similar'
|
||||
hamming_string_lookup[ HAMMING_SPECULATIVE ] = 'speculative'
|
||||
|
||||
HYDRUS_CLIENT = 0
|
||||
HYDRUS_SERVER = 1
|
||||
HYDRUS_TEST = 2
|
||||
|
|
|
@ -333,6 +333,19 @@ def MakeFileWritable( path ):
|
|||
|
||||
os.chmod( path, stat.S_IWRITE | stat.S_IREAD )
|
||||
|
||||
if os.path.isdir( path ):
|
||||
|
||||
for ( root, dirnames, filenames ) in os.walk( path ):
|
||||
|
||||
for filename in filenames:
|
||||
|
||||
sub_path = os.path.join( root, filename )
|
||||
|
||||
os.chmod( sub_path, stat.S_IWRITE | stat.S_IREAD )
|
||||
|
||||
|
||||
|
||||
|
||||
except:
|
||||
|
||||
pass
|
||||
|
@ -568,4 +581,4 @@ def RecyclePath( path ):
|
|||
DeletePath( original_path )
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
@ -11,5 +11,5 @@ class TestImageHandling( unittest.TestCase ):
|
|||
|
||||
phashes = ClientImageHandling.GenerateShapePerceptualHashes( os.path.join( HC.STATIC_DIR, 'hydrus.png' ) )
|
||||
|
||||
self.assertEqual( phashes, [ '\xb0\x08\x83\xb2\x08\x0b8\x08' ] )
|
||||
self.assertEqual( phashes, set( [ '\xb4M\xc7\xb2M\xcb8\x1c' ] ) )
|
||||
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
import ClientConstants as CC
|
||||
import ClientDefaults
|
||||
import ClientGUIDialogs
|
||||
import ClientGUIScrolledPanelsManagement
|
||||
import ClientGUITopLevelWindows
|
||||
import collections
|
||||
import HydrusConstants as HC
|
||||
import os
|
||||
|
@ -9,9 +11,35 @@ import unittest
|
|||
import wx
|
||||
import HydrusGlobals
|
||||
|
||||
def HitButton( button ): wx.PostEvent( button, wx.CommandEvent( wx.EVT_BUTTON.typeId, button.GetId() ) )
|
||||
|
||||
def HitCancelButton( window ): wx.PostEvent( window, wx.CommandEvent( wx.EVT_BUTTON.typeId, wx.ID_CANCEL ) )
|
||||
|
||||
def HitButton( button ): wx.PostEvent( button, wx.CommandEvent( wx.EVT_BUTTON.typeId, button.GetId() ) )
|
||||
def HitOKButton( window ): wx.PostEvent( window, wx.CommandEvent( wx.EVT_BUTTON.typeId, wx.ID_OK ) )
|
||||
|
||||
def CancelChildDialog( window ):
|
||||
|
||||
children = window.GetChildren()
|
||||
|
||||
for child in children:
|
||||
|
||||
if isinstance( child, wx.Dialog ):
|
||||
|
||||
HitCancelButton( child )
|
||||
|
||||
|
||||
|
||||
def OKChildDialog( window ):
|
||||
|
||||
children = window.GetChildren()
|
||||
|
||||
for child in children:
|
||||
|
||||
if isinstance( child, wx.Dialog ):
|
||||
|
||||
HitOKButton( child )
|
||||
|
||||
|
||||
|
||||
def PressKey( window, key ):
|
||||
|
||||
|
@ -39,6 +67,31 @@ class TestDBDialogs( unittest.TestCase ):
|
|||
|
||||
|
||||
|
||||
def test_dialog_manage_subs( self ):
|
||||
|
||||
HydrusGlobals.test_controller.SetRead( 'serialisable_named', [] )
|
||||
|
||||
title = 'subs test'
|
||||
frame_key = 'regular_dialog'
|
||||
|
||||
with ClientGUITopLevelWindows.DialogManage( None, title, frame_key ) as dlg:
|
||||
|
||||
panel = ClientGUIScrolledPanelsManagement.ManageSubscriptionsPanel( dlg )
|
||||
|
||||
dlg.SetPanel( panel )
|
||||
|
||||
wx.CallAfter( panel.Add )
|
||||
|
||||
wx.CallLater( 2000, OKChildDialog, panel )
|
||||
|
||||
wx.CallLater( 4000, HitCancelButton, dlg )
|
||||
|
||||
result = dlg.ShowModal()
|
||||
|
||||
self.assertEqual( result, wx.ID_CANCEL )
|
||||
|
||||
|
||||
|
||||
class TestNonDBDialogs( unittest.TestCase ):
|
||||
|
||||
def test_dialog_choose_new_service_method( self ):
|
||||
|
@ -169,4 +222,4 @@ class TestNonDBDialogs( unittest.TestCase ):
|
|||
self.assertEqual( result, wx.ID_NO )
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
@ -46,7 +46,7 @@ try:
|
|||
|
||||
if result.db_dir is None:
|
||||
|
||||
db_dir = os.path.join( HC.BASE_DIR, 'db' )
|
||||
db_dir = HC.DEFAULT_DB_DIR
|
||||
|
||||
else:
|
||||
|
||||
|
@ -55,6 +55,7 @@ try:
|
|||
|
||||
db_dir = HydrusPaths.ConvertPortablePathToAbsPath( db_dir, HC.BASE_DIR )
|
||||
|
||||
|
||||
try:
|
||||
|
||||
HydrusPaths.MakeSureDirectoryExists( db_dir )
|
||||
|
@ -139,4 +140,4 @@ except Exception as e:
|
|||
|
||||
print( 'Critical error occured! Details written to crash.log!' )
|
||||
|
||||
|
||||
|
||||
|
|
Binary file not shown.
After Width: | Height: | Size: 512 B |
9
test.py
9
test.py
|
@ -156,6 +156,13 @@ class Controller( object ):
|
|||
return call_to_thread
|
||||
|
||||
|
||||
def _SetupWx( self ):
|
||||
|
||||
self.locale = wx.Locale( wx.LANGUAGE_DEFAULT ) # Very important to init this here and keep it non garbage collected
|
||||
|
||||
CC.GlobalBMPs.STATICInitialise()
|
||||
|
||||
|
||||
def pub( self, topic, *args, **kwargs ):
|
||||
|
||||
pass
|
||||
|
@ -249,6 +256,8 @@ class Controller( object ):
|
|||
|
||||
def Run( self ):
|
||||
|
||||
self._SetupWx()
|
||||
|
||||
suites = []
|
||||
|
||||
if only_run is None: run_all = True
|
||||
|
|
Loading…
Reference in New Issue