RepoMirrors
/
hydrus
mirror of https://github.com/hydrusnetwork/hydrus.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Version 244

This commit is contained in:
Hydrus Network Developer 2017-02-08 16:27:00 -06:00
parent 79c2bf2d9f
commit 257ef475d9
22 changed files with 1286 additions and 3341 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
help/changelog.html
View File

@ -8,6 +8,30 @@
<div class="content">
<h3>changelog</h3>
<ul>
<li><h3>version 244</h3></li>
<ul>
<li>updated client database to compact ( namespace_id, tag_id ) pair into a single id for storage</li>
<li>added some bells and whistles to the update code</li>
<li>added a free space check and messagebox warning before the update</li>
<li>updated db, service, and a/c cache creation code to reflect new schema</li>
<li>updated absolutely everything else in the db to reflect the new schema</li>
<li>for users with plenty of tags, the db should now be about 33% smaller!</li>
<li>unified how unnamespaced tag searching counts are totalled</li>
<li>unnamespaced tag searching counts are now totalled when the tags are fetched from the in-view ui media</li>
<li>unified how tags are split into ( namespace, subtag ) across the program</li>
<li>fixed deviantart gallery thumbnail parser</li>
<li>fixed linux session load page key event handling bug</li>
<li>os x can now support notebooks with zero pages open</li>
<li>fixed an issue where os x was losing the first page of some session loads</li>
<li>fixed some similar files shutdown work false positive calculation</li>
<li>reduced server bandwidth check period from 24 hours to 1 hour</li>
<li>improved calltothread scheduling under heavy load</li>
<li>improved scheduling of how files are physically deleted</li>
<li>numerous laggy temp_table replacement/cleanup</li>
<li>more temp_table replacement</li>
<li>misc efficiency improvements and general db code cleanup</li>
<li>misc path code cleanup</li>
</ul>
<li><h3>version 243</h3></li>
<ul>
<li>updated more menu code to the new system</li>

14
include/ClientCaches.py
View File

@ -828,9 +828,12 @@ class ClientFilesManager( object ):
def DelayedDeleteFiles( self, hashes ):
def DelayedDeleteFiles( self, hashes, time_to_delete ):
time.sleep( 2 )
while not HydrusData.TimeHasPassed( time_to_delete ):
time.sleep( 0.5 )
with self._lock:
@ -849,9 +852,12 @@ class ClientFilesManager( object ):
def DelayedDeleteThumbnails( self, hashes ):
def DelayedDeleteThumbnails( self, hashes, time_to_delete ):
time.sleep( 2 )
while not HydrusData.TimeHasPassed( time_to_delete ):
time.sleep( 0.5 )
with self._lock:

4
include/ClientController.py
View File

@ -1130,11 +1130,13 @@ class Controller( HydrusController.HydrusController ):
def THREADDoFileQuery( self, query_key, search_context ):
QUERY_CHUNK_SIZE = 256
query_hash_ids = self.Read( 'file_query_ids', search_context )
media_results = []
for sub_query_hash_ids in HydrusData.SplitListIntoChunks( query_hash_ids, 256 ):
for sub_query_hash_ids in HydrusData.SplitListIntoChunks( query_hash_ids, QUERY_CHUNK_SIZE ):
if query_key.IsCancelled(): return

1527
include/ClientDB.py
View File

File diff suppressed because it is too large Load Diff

60
include/ClientData.py
View File

@ -326,7 +326,7 @@ def MergeCounts( min_a, max_a, min_b, max_b ):
return ( min_answer, max_answer )
def MergePredicates( predicates ):
def MergePredicates( predicates, add_namespaceless = False ):
master_predicate_dict = {}
@ -344,6 +344,45 @@ def MergePredicates( predicates ):
if add_namespaceless:
# we want to include the count for namespaced tags in the namespaceless version when:
# there exists more than one instance of the subtag with different namespaces, including '', that has nonzero count
unnamespaced_predicate_dict = {}
subtag_nonzero_instance_counter = collections.Counter()
for predicate in master_predicate_dict.values():
if predicate.HasNonZeroCount():
unnamespaced_predicate = predicate.GetUnnamespacedCopy()
subtag_nonzero_instance_counter[ unnamespaced_predicate ] += 1
if unnamespaced_predicate in unnamespaced_predicate_dict:
unnamespaced_predicate_dict[ unnamespaced_predicate ].AddCounts( unnamespaced_predicate )
else:
unnamespaced_predicate_dict[ unnamespaced_predicate ] = unnamespaced_predicate
for ( unnamespaced_predicate, count ) in subtag_nonzero_instance_counter.items():
# if there were indeed several instances of this subtag, overwrte the master dict's instance with our new count total
if count > 1:
master_predicate_dict[ unnamespaced_predicate ] = unnamespaced_predicate_dict[ unnamespaced_predicate ]
return master_predicate_dict.values()
def ShowExceptionClient( e ):
@ -439,22 +478,17 @@ def SortTagsList( tags, sort_type ):
def key( tag ):
if ':' in tag:
# '{' is above 'z' in ascii, so this works for most situations
( namespace, subtag ) = HydrusTags.SplitTag( tag )
if namespace == '':
( namespace, subtag ) = tag.split( ':', 1 )
if namespace == '':
return ( '{', subtag )
else:
return ( namespace, subtag )
return ( '{', subtag )
else:
return ( '{', tag ) # '{' is above 'z' in ascii, so this works for most situations
return ( namespace, subtag )

26
include/ClientDownloading.py
View File

@ -1073,15 +1073,15 @@ class GalleryDeviantArt( Gallery ):
soup = GetSoup( html )
thumbs_container = soup.find( class_ = 'zones-container' )
thumbs_container = soup.find( 'div', class_ = 'torpedo-container' )
artist = url_base.split( 'http://' )[1].split( '.deviantart.com' )[0]
links = thumbs_container.find_all( 'a', class_ = 'thumb' )
thumbs = thumbs_container.find_all( 'span', class_ = 'thumb' )
for link in links:
for thumb in thumbs:
url = link[ 'href' ] # something in the form of blah.da.com/art/blah-123456
url = thumb[ 'href' ] # something in the form of blah.da.com/art/blah-123456
urls.append( url )
@ -1089,19 +1089,17 @@ class GalleryDeviantArt( Gallery ):
tags.append( 'creator:' + artist )
try: # starts_with_thumb picks up some false positives, but they break
title_tag = thumb.find( 'span', class_ = 'title' )
if title_tag is not None:
raw_title = link[ 'title' ] # sweet dolls by AngeniaC, date, blah blah blah
title = title_tag.string
raw_title_reversed = raw_title[::-1] # trAtnaiveD no CainegnA yb sllod teews
if title is not None and title != '':
tags.append( 'title:' + title )
( creator_and_gumpf_reversed, title_reversed ) = raw_title_reversed.split( ' yb ', 1 )
title = title_reversed[::-1] # sweet dolls
tags.append( 'title:' + title )
except: pass
SetExtraURLInfo( url, tags )

24
include/ClientExporting.py
View File

@ -7,6 +7,7 @@ import HydrusData
import HydrusGlobals
import HydrusPaths
import HydrusSerialisable
import HydrusTags
import os
import re
import stat
@ -29,7 +30,11 @@ def GenerateExportFilename( media, terms ):
tags = tags_manager.GetNamespaceSlice( ( term, ) )
filename += ', '.join( [ tag.split( ':' )[1] for tag in tags ] )
subtags = [ HydrusTags.SplitTag( tag )[1] for tag in tags ]
subtags.sort()
filename += ', '.join( subtags )
elif term_type == 'predicate':
@ -40,8 +45,14 @@ def GenerateExportFilename( media, terms ):
tags = list( current.union( pending ) )
if term == 'nn tags': tags = [ tag for tag in tags if ':' not in tag ]
else: tags = [ tag if ':' not in tag else tag.split( ':' )[1] for tag in tags ]
if term == 'nn tags':
tags = [ tag for tag in tags if ':' not in tag ]
else:
tags = [ HydrusTags.SplitTag( tag )[1] for tag in tags ]
tags.sort()
@ -56,9 +67,12 @@ def GenerateExportFilename( media, terms ):
elif term_type == 'tag':
if ':' in term: term = term.split( ':' )[1]
( namespace, subtag ) = HydrusTags.SplitTag( tag )
if tags_manager.HasTag( term ): filename += term
if tags_manager.HasTag( subtag ):
filename += subtag

35
include/ClientGUI.py
View File

@ -642,7 +642,10 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
selection = self._notebook.GetSelection()
if selection != wx.NOT_FOUND: self._ClosePage( selection, polite = polite )
if selection != wx.NOT_FOUND:
self._ClosePage( selection, polite = polite )
def _ClosePage( self, selection, polite = True ):
@ -655,12 +658,6 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
return
# issue with having all pages closed
if HC.PLATFORM_OSX and self._notebook.GetPageCount() == 1:
return
page = self._notebook.GetPage( selection )
if polite:
@ -1478,7 +1475,10 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
for page in [ self._notebook.GetPage( i ) for i in range( self._notebook.GetPageCount() ) ]:
try: page.TestAbleToClose()
try:
page.TestAbleToClose()
except HydrusExceptions.PermissionException:
return
@ -1494,7 +1494,12 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
try:
self._notebook.Disable()
if not HC.PLATFORM_LINUX:
# on linux, this stops session pages from accepting keyboard input, wew
wx.CallAfter( self._notebook.Disable )
for ( page_name, management_controller, initial_hashes ) in session.IteratePages():
@ -1528,19 +1533,15 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
if HC.PLATFORM_OSX:
wx.CallAfter( self._ClosePage, 0 )
finally:
self._loading_session = False
self._media_status_override = None
self._notebook.Enable()
wx.CallAfter( self.Layout )
if not HC.PLATFORM_LINUX:
wx.CallAfter( self._notebook.Enable )

50
include/ClientGUIACDropdown.py
View File

@ -1,5 +1,6 @@
import ClientCaches
import ClientConstants as CC
import ClientData
import ClientGUICommon
import ClientGUIListBoxes
import ClientSearch
@ -854,19 +855,19 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
must_do_a_search = True
if ':' in search_text:
( namespace, half_complete_tag ) = search_text.split( ':', 1 )
( namespace, half_complete_subtag ) = HydrusTags.SplitTag( search_text )
if namespace != '':
if namespace != self._current_namespace:
self._current_namespace = namespace # do a new search, no matter what half_complete tag is
if half_complete_tag != '': must_do_a_search = True
if half_complete_subtag != '': must_do_a_search = True
else:
if self._cache_text == self._current_namespace + ':' and half_complete_tag != '':
if self._cache_text == self._current_namespace + ':' and half_complete_subtag != '':
must_do_a_search = True
@ -874,14 +875,12 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
else:
self._current_namespace = ''
half_complete_tag = search_text
self._current_namespace = namespace
siblings_manager = HydrusGlobals.client_controller.GetManager( 'tag_siblings' )
if half_complete_tag == '':
if half_complete_subtag == '':
self._cache_text = self._current_namespace + ':'
@ -908,7 +907,7 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
include_current = self._file_search_context.IncludeCurrentTags()
include_pending = self._file_search_context.IncludePendingTags()
if len( half_complete_tag ) < num_autocomplete_chars and '*' not in search_text:
if len( half_complete_subtag ) < num_autocomplete_chars and '*' not in search_text:
predicates = HydrusGlobals.client_controller.Read( 'autocomplete_predicates', file_service_key = self._file_service_key, tag_service_key = self._tag_service_key, search_text = search_text, exact_match = True, inclusive = inclusive, include_current = include_current, include_pending = include_pending, add_namespaceless = True, collapse_siblings = True )
@ -977,6 +976,11 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
predicates = siblings_manager.CollapsePredicates( self._tag_service_key, predicates )
if self._current_namespace == '':
predicates = ClientData.MergePredicates( predicates, add_namespaceless = True )
self._next_updatelist_is_probably_fast = True
@ -989,14 +993,14 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
if self._current_namespace != '':
if '*' not in self._current_namespace and half_complete_tag == '':
if '*' not in self._current_namespace and half_complete_subtag == '':
matches.insert( 0, ClientSearch.Predicate( HC.PREDICATE_TYPE_NAMESPACE, self._current_namespace, inclusive ) )
if half_complete_tag != '':
if half_complete_subtag != '':
if '*' in self._current_namespace or ( '*' in half_complete_tag and half_complete_tag != '*' ):
if '*' in self._current_namespace or ( '*' in half_complete_subtag and half_complete_subtag != '*' ):
matches.insert( 0, ClientSearch.Predicate( HC.PREDICATE_TYPE_WILDCARD, search_text, inclusive ) )
@ -1182,11 +1186,11 @@ class AutoCompleteDropdownTagsWrite( AutoCompleteDropdownTags ):
must_do_a_search = False
if ':' in search_text:
( namespace, half_complete_subtag ) = HydrusTags.SplitTag( search_text )
if namespace != '':
( namespace, other_half ) = search_text.split( ':', 1 )
if other_half != '' and namespace != self._current_namespace:
if half_complete_subtag != '' and namespace != self._current_namespace:
self._current_namespace = namespace # do a new search, no matter what half_complete tag is
@ -1195,20 +1199,18 @@ class AutoCompleteDropdownTagsWrite( AutoCompleteDropdownTags ):
else:
self._current_namespace = ''
self._current_namespace = namespace
half_complete_tag = search_text
if len( half_complete_tag ) < num_autocomplete_chars and '*' not in search_text:
if len( half_complete_subtag ) < num_autocomplete_chars and '*' not in search_text:
predicates = HydrusGlobals.client_controller.Read( 'autocomplete_predicates', file_service_key = self._file_service_key, tag_service_key = self._tag_service_key, search_text = search_text, exact_match = True, add_namespaceless = False, collapse_siblings = False )
else:
if must_do_a_search or self._cache_text == '' or not half_complete_tag.startswith( self._cache_text ):
if must_do_a_search or self._cache_text == '' or not half_complete_subtag.startswith( self._cache_text ):
self._cache_text = half_complete_tag
self._cache_text = half_complete_subtag
self._cached_results = HydrusGlobals.client_controller.Read( 'autocomplete_predicates', file_service_key = self._file_service_key, tag_service_key = self._tag_service_key, search_text = search_text, add_namespaceless = False, collapse_siblings = False )
@ -1218,7 +1220,7 @@ class AutoCompleteDropdownTagsWrite( AutoCompleteDropdownTags ):
self._next_updatelist_is_probably_fast = True
matches = ClientSearch.FilterPredicatesBySearchEntry( self._tag_service_key, half_complete_tag, predicates )
matches = ClientSearch.FilterPredicatesBySearchEntry( self._tag_service_key, half_complete_subtag, predicates )
matches = ClientSearch.SortPredicates( matches )

11
include/ClientGUICanvas.py
View File

@ -1697,16 +1697,15 @@ class CanvasWithDetails( Canvas ):
display_string += ' (-)'
if ':' in tag:
( namespace, subtag ) = HydrusTags.SplitTag( tag )
if namespace in namespace_colours:
( namespace, sub_tag ) = tag.split( ':', 1 )
if namespace in namespace_colours: ( r, g, b ) = namespace_colours[ namespace ]
else: ( r, g, b ) = namespace_colours[ None ]
( r, g, b ) = namespace_colours[ namespace ]
else:
( r, g, b ) = namespace_colours[ '' ]
( r, g, b ) = namespace_colours[ None ]
dc.SetTextForeground( wx.Colour( r, g, b ) )

47
include/ClientGUIListBoxes.py
View File

@ -527,20 +527,29 @@ class ListBoxTags( ListBox ):
namespace_colours = self._GetNamespaceColours()
if ':' in tag_string:
( namespace, sub_tag ) = tag_string.split( ':', 1 )
( namespace, subtag ) = HydrusTags.SplitTag( tag_string )
if namespace != '':
if namespace.startswith( '-' ): namespace = namespace[1:]
if namespace.startswith( '(+) ' ): namespace = namespace[4:]
if namespace.startswith( '(-) ' ): namespace = namespace[4:]
if namespace.startswith( '(X) ' ): namespace = namespace[4:]
if namespace.startswith( ' ' ): namespace = namespace[4:]
elif namespace.startswith( '(+) ' ): namespace = namespace[4:]
elif namespace.startswith( '(-) ' ): namespace = namespace[4:]
elif namespace.startswith( '(X) ' ): namespace = namespace[4:]
elif namespace.startswith( ' ' ): namespace = namespace[4:]
if namespace in namespace_colours: ( r, g, b ) = namespace_colours[ namespace ]
else: ( r, g, b ) = namespace_colours[ None ]
if namespace in namespace_colours:
( r, g, b ) = namespace_colours[ namespace ]
else:
( r, g, b ) = namespace_colours[ None ]
else:
( r, g, b ) = namespace_colours[ '' ]
else: ( r, g, b ) = namespace_colours[ '' ]
return ( r, g, b )
@ -599,9 +608,9 @@ class ListBoxTags( ListBox ):
text = HydrusData.ToUnicode( term )
if command == 'copy_sub_terms' and ':' in text:
if command == 'copy_sub_terms':
( namespace_gumpf, text ) = text.split( ':', 1 )
( namespace_gumpf, text ) = HydrusTags.SplitTag( text )
texts.append( text )
@ -752,9 +761,11 @@ class ListBoxTags( ListBox ):
if len( self._selected_terms ) == 1:
if ':' in selection_string:
( namespace, subtag ) = HydrusTags.SplitTag( selection_string )
if namespace != '':
sub_selection_string = '"' + selection_string.split( ':', 1 )[1]
sub_selection_string = '"' + subtag
menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_sub_terms' ), 'copy ' + sub_selection_string )
@ -1741,11 +1752,9 @@ class ListBoxTagsSelection( ListBoxTags ):
tag = self._strings_to_terms[ unordered_string ]
if ':' in tag:
( namespace, subtag ) = tag.split( ':', 1 )
else:
( namespace, subtag ) = HydrusTags.SplitTag( tag )
if namespace == '':
namespace = '{' # '{' is above 'z' in ascii, so this works for most situations

18
include/ClientMedia.py
View File

@ -1835,7 +1835,9 @@ class TagsManagerSimple( object ):
combined_current = combined_statuses_to_tags[ HC.CURRENT ]
combined_pending = combined_statuses_to_tags[ HC.PENDING ]
self._combined_namespaces_cache = HydrusData.BuildKeyToSetDict( tag.split( ':', 1 ) for tag in combined_current.union( combined_pending ) if ':' in tag )
pairs = ( HydrusTags.SplitTag( tag ) for tag in combined_current.union( combined_pending ) )
self._combined_namespaces_cache = HydrusData.BuildKeyToSetDict( ( namespace, subtag ) for ( namespace, subtag ) in pairs if namespace != '' )
result = { namespace : self._combined_namespaces_cache[ namespace ] for namespace in namespaces }
@ -1854,19 +1856,17 @@ class TagsManagerSimple( object ):
combined = combined_current.union( combined_pending )
pairs = [ HydrusTags.SplitTag( tag ) for tag in combined ]
slice = []
for namespace in namespaces:
for desired_namespace in namespaces:
tags = [ tag for tag in combined if tag.startswith( namespace + ':' ) ]
subtags = [ HydrusTags.ConvertTagToSortable( subtag ) for ( namespace, subtag ) in pairs if namespace == desired_namespace ]
tags = [ tag.split( ':', 1 )[1] for tag in tags ]
subtags.sort()
tags = HydrusTags.SortNumericTags( tags )
tags = tuple( ( HydrusTags.ConvertTagToSortable( tag ) for tag in tags ) )
slice.append( tags )
slice.append( tuple( subtags ) )
return tuple( slice )

83
include/ClientSearch.py
View File

@ -69,29 +69,29 @@ def FilterTagsBySearchEntry( service_key, search_entry, tags, search_siblings =
if ':' in search_entry:
search_entry = ConvertTagToSearchable( search_entry )
( namespace, half_complete_subtag ) = HydrusTags.SplitTag( search_entry )
if namespace != '':
search_namespace = True
( namespace_entry, search_entry ) = search_entry.split( ':', 1 )
namespace_entry = ConvertTagToSearchable( namespace_entry )
namespace_re_predicate = compile_re( namespace_entry )
namespace_re_predicate = compile_re( ConvertTagToSearchable( namespace ) )
else:
search_namespace = False
search_entry = ConvertTagToSearchable( search_entry )
if '*' not in search_entry:
search_entry += '*'
namespace_re_predicate = None
re_predicate = compile_re( search_entry )
if '*' not in half_complete_subtag:
half_complete_subtag += '*'
half_complete_subtag_re_predicate = compile_re( half_complete_subtag )
sibling_manager = HydrusGlobals.client_controller.GetManager( 'tag_siblings' )
@ -110,19 +110,17 @@ def FilterTagsBySearchEntry( service_key, search_entry, tags, search_siblings =
for possible_tag in possible_tags:
if ':' in possible_tag:
( possible_namespace, possible_subtag ) = HydrusTags.SplitTag( possible_tag )
if possible_namespace != '':
( n, t ) = possible_tag.split( ':', 1 )
possible_namespace = ConvertTagToSearchable( possible_namespace )
n = ConvertTagToSearchable( n )
if search_namespace and re.search( namespace_re_predicate, n ) is None:
if search_namespace and re.search( namespace_re_predicate, possible_namespace ) is None:
continue
comparee = t
else:
if search_namespace:
@ -130,14 +128,12 @@ def FilterTagsBySearchEntry( service_key, search_entry, tags, search_siblings =
continue
comparee = possible_tag
comparee = ConvertTagToSearchable( comparee )
possible_subtag = ConvertTagToSearchable( possible_subtag )
if re.search( re_predicate, comparee ) is not None:
if re.search( half_complete_subtag_re_predicate, possible_subtag ) is not None:
result.append( tag )
result.append( possible_tag )
break
@ -791,9 +787,15 @@ class Predicate( HydrusSerialisable.SerialisableBase ):
return ( self._min_current_count, self._max_current_count, self._min_pending_count, self._max_pending_count )
def GetCopy( self ): return Predicate( self._predicate_type, self._value, self._inclusive, self._min_current_count, self._min_pending_count, self._max_current_count, self._max_pending_count )
def GetCopy( self ):
return Predicate( self._predicate_type, self._value, self._inclusive, self._min_current_count, self._min_pending_count, self._max_current_count, self._max_pending_count )
def GetCountlessCopy( self ): return Predicate( self._predicate_type, self._value, self._inclusive )
def GetCountlessCopy( self ):
return Predicate( self._predicate_type, self._value, self._inclusive )
def GetCount( self, current_or_pending = None ):
@ -830,7 +832,10 @@ class Predicate( HydrusSerialisable.SerialisableBase ):
return self._inclusive
def GetInfo( self ): return ( self._predicate_type, self._value, self._inclusive )
def GetInfo( self ):
return ( self._predicate_type, self._value, self._inclusive )
def GetInverseCopy( self ):
@ -1137,9 +1142,29 @@ class Predicate( HydrusSerialisable.SerialisableBase ):
return base
def GetUnnamespacedCopy( self ):
if self._predicate_type == HC.PREDICATE_TYPE_TAG:
( namespace, subtag ) = HydrusTags.SplitTag( self._value )
return Predicate( self._predicate_type, subtag, self._inclusive, self._min_current_count, self._min_pending_count, self._max_current_count, self._max_pending_count )
return self.GetCopy()
def GetValue( self ): return self._value
def SetInclusive( self, inclusive ): self._inclusive = inclusive
def HasNonZeroCount( self ):
return self._min_current_count > 0 or self._min_pending_count > 0
def SetInclusive( self, inclusive ):
self._inclusive = inclusive
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_PREDICATE ] = Predicate

2
include/HydrusConstants.py
View File

@ -49,7 +49,7 @@ options = {}
# Misc
NETWORK_VERSION = 17
SOFTWARE_VERSION = 243
SOFTWARE_VERSION = 244
UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )

18
include/HydrusController.py
View File

@ -67,16 +67,18 @@ class HydrusController( object ):
if len( self._call_to_threads ) > 100:
if len( self._call_to_threads ) < 10:
raise Exception( 'Too many call to threads!' )
call_to_thread = HydrusThreading.THREADCallToThread( self )
self._call_to_threads.append( call_to_thread )
call_to_thread.start()
else:
call_to_thread = random.choice( self._call_to_threads )
call_to_thread = HydrusThreading.THREADCallToThread( self )
self._call_to_threads.append( call_to_thread )
call_to_thread.start()
return call_to_thread

15
include/HydrusDB.py
View File

@ -50,28 +50,28 @@ def CanVacuum( db_path, stop_time = None ):
temp_dir = tempfile.gettempdir()
( db_dir, db_filename ) = os.path.split( db_path )
temp_disk_usage = psutil.disk_usage( temp_dir )
temp_disk_free_space = HydrusPaths.GetFreeSpace( temp_dir )
a = HydrusPaths.GetDevice( temp_dir )
b = HydrusPaths.GetDevice( db_dir )
if HydrusPaths.GetDevice( temp_dir ) == HydrusPaths.GetDevice( db_dir ):
if temp_disk_usage.free < db_size * 2.2:
if temp_disk_free_space < db_size * 2.2:
return False
else:
if temp_disk_usage.free < db_size * 1.1:
if temp_disk_free_space < db_size * 1.1:
return False
db_disk_usage = psutil.disk_usage( db_dir )
db_disk_free_space = HydrusPaths.GetFreeSpace( db_dir )
if db_disk_usage.free < db_size * 1.1:
if db_disk_free_space < db_size * 1.1:
return False
@ -551,6 +551,11 @@ class HydrusDB( object ):
def _SelectFromListFetchAll( self, select_statement, xs ):
return [ row for row in self._SelectFromList( select_statement, xs ) ]
def _UpdateDB( self, version ):
raise NotImplementedError()

6
include/HydrusPaths.py
View File

@ -256,6 +256,12 @@ def GetDevice( path ):
return None
def GetFreeSpace( path ):
disk_usage = psutil.disk_usage( path )
return disk_usage.free
def GetTempFile(): return tempfile.TemporaryFile()
def GetTempFileQuick(): return tempfile.SpooledTemporaryFile( max_size = 1024 * 1024 * 4 )
def GetTempPath( suffix = '' ):

84
include/HydrusTags.py
View File

@ -16,14 +16,18 @@ def CensorshipMatch( tag, censorships ):
if censorship == '': # '' - all non namespaced tags
if ':' not in tag:
( namespace, subtag ) = SplitTag( tag )
if namespace == '':
return True
elif censorship == ':': # ':' - all namespaced tags
if ':' in tag:
( namespace, subtag ) = SplitTag( tag )
if namespace != '':
return True
@ -32,7 +36,9 @@ def CensorshipMatch( tag, censorships ):
if censorship.endswith( ':' ): # 'series:' - namespaced tags
if tag.startswith( censorship ):
( namespace, subtag ) = SplitTag( tag )
if namespace == censorship:
return True
@ -49,21 +55,11 @@ def CensorshipMatch( tag, censorships ):
# 'table' - normal tag, or namespaced version of same
if ':' in tag:
( namespace, subtag ) = SplitTag( tag )
if subtag == censorship:
( namespace, comparison_tag ) = tag.split( ':', 1 )
if comparison_tag == censorship:
return True
else:
if tag == censorship:
return True
return True
@ -72,7 +68,7 @@ def CensorshipMatch( tag, censorships ):
def ConvertTagToSortable( t ):
if t[0].isdecimal():
if len( t ) > 0 and t[0].isdecimal():
# We want to maintain that:
# 0 < 0a < 0b < 1 ( lexicographic comparison )
@ -110,22 +106,23 @@ def FilterNamespaces( tags, namespaces ):
for tag in tags:
if ':' in tag:
( namespace, subtag ) = tag.split( ':', 1 )
processed_tags[ namespace ].add( tag )
else: processed_tags[ '' ].add( tag )
( namespace, subtag ) = SplitTag( tag )
processed_tags[ namespace ].add( tag )
result = set()
for namespace in namespaces:
if namespace in ( '', None ): result.update( processed_tags[ '' ] )
result.update( processed_tags[ namespace ] )
if namespace == None:
result.update( processed_tags[ '' ] )
else:
result.update( processed_tags[ namespace ] )
return result
@ -142,16 +139,12 @@ def CheckTagNotEmpty( tag ):
empty_tag = False
if tag == '': empty_tag = True
( namespace, subtag ) = SplitTag( tag )
if ':' in tag:
if subtag == '':
( namespace, subtag ) = tag.split( ':', 1 )
raise HydrusExceptions.SizeException( 'Received a zero-length tag!' )
if subtag == '': empty_tag = True
if empty_tag: raise HydrusExceptions.SizeException( 'Received a zero-length tag!' )
def CleanTag( tag ):
@ -204,22 +197,22 @@ def CleanTags( tags ):
return clean_tags
def CombineTag( namespace, tag ):
def CombineTag( namespace, subtag ):
if namespace == '':
if tag.startswith( ':' ):
if subtag.startswith( ':' ):
return ':' + tag
return ':' + subtag
else:
return tag
return subtag
else:
return namespace + ':' + tag
return namespace + ':' + subtag
def RenderTag( tag ):
@ -233,3 +226,14 @@ def RenderTag( tag ):
return tag
def SplitTag( tag ):
if ':' in tag:
return tag.split( ':', 1 )
else:
return ( '', tag )

4
include/ServerController.py
View File

@ -204,8 +204,6 @@ class Controller( HydrusController.HydrusController ):
self._services[ service_key ] = reactor.listenSSL( port, service_object, context_factory )
#self._services[ service_key ] = reactor.listenTCP( port, service_object )
try:
connection = HydrusNetworking.GetLocalConnection( port )
@ -291,7 +289,7 @@ class Controller( HydrusController.HydrusController ):
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'ClearBans', ServerDaemons.DAEMONClearBans, period = 3600 ) )
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'DeleteOrphans', ServerDaemons.DAEMONDeleteOrphans, period = 86400 ) )
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'GenerateUpdates', ServerDaemons.DAEMONGenerateUpdates, period = 600 ) )
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'CheckDataUsage', ServerDaemons.DAEMONCheckDataUsage, period = 86400 ) )
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'CheckDataUsage', ServerDaemons.DAEMONCheckDataUsage, period = 3600 ) )
self._daemons.append( HydrusThreading.DAEMONWorker( self, 'UPnP', ServerDaemons.DAEMONUPnP, ( 'notify_new_options', ), period = 43200 ) )

47
include/TestClientDownloading.py
View File

@ -25,53 +25,6 @@ class TestDownloaders( unittest.TestCase ):
HydrusGlobals.test_controller.SetHTTP( self.old_http )
def test_deviantart( self ):
with open( os.path.join( HC.STATIC_DIR, 'testing', 'da_gallery.html' ) ) as f: da_gallery = f.read()
with open( os.path.join( HC.STATIC_DIR, 'testing', 'da_page.html' ) ) as f: da_page = f.read()
HydrusGlobals.test_controller.GetHTTP().SetResponse( HC.GET, 'http://sakimichan.deviantart.com/gallery/?catpath=/&offset=0', da_gallery )
HydrusGlobals.test_controller.GetHTTP().SetResponse( HC.GET, 'http://sakimichan.deviantart.com/art/Sailor-moon-in-PJs-506918040', da_page )
HydrusGlobals.test_controller.GetHTTP().SetResponse( HC.GET, 'http://fc00.deviantart.net/fs71/f/2015/013/3/c/3c026edbe356b22c802e7be0db6fbd0b-d8dt0go.jpg', 'image file' )
#
gallery = ClientDownloading.GalleryDeviantArt()
#
( page_of_urls, definitely_no_more_pages ) = gallery.GetPage( 'sakimichan', 0 )
expected_gallery_urls = ['http://sakimichan.deviantart.com/art/Sailor-moon-in-PJs-506918040', 'http://sakimichan.deviantart.com/art/Johnny-Bravo-505601401', 'http://sakimichan.deviantart.com/art/Daphne-505394693', 'http://sakimichan.deviantart.com/art/kim-Possible-505195132', 'http://sakimichan.deviantart.com/art/Levi-s-evil-plan-504966437', 'http://sakimichan.deviantart.com/art/Velma-504483448', 'http://sakimichan.deviantart.com/art/Scoobydoo-504238131', 'http://sakimichan.deviantart.com/art/Kerrigan-chilling-503477012', 'http://sakimichan.deviantart.com/art/Kiki-498525851', 'http://sakimichan.deviantart.com/art/Waiter-Howl-502377515', 'http://sakimichan.deviantart.com/art/Modern-Loki-497985045', 'http://sakimichan.deviantart.com/art/Emma-501919103', 'http://sakimichan.deviantart.com/art/Lola-494941222', 'http://sakimichan.deviantart.com/art/Elsas-501262184', 'http://sakimichan.deviantart.com/art/Tsunade-499517356', 'http://sakimichan.deviantart.com/art/A-little-cold-out-commission-498326494', 'http://sakimichan.deviantart.com/art/Girl-496999831', 'http://sakimichan.deviantart.com/art/Green-elf-496797148', 'http://sakimichan.deviantart.com/art/Itachi-496625357', 'http://sakimichan.deviantart.com/art/Sesshomaru-495474394', 'http://sakimichan.deviantart.com/art/Mononoke-years-later-502160436', 'http://sakimichan.deviantart.com/art/Jinx-488513585', 'http://sakimichan.deviantart.com/art/Alex-in-wonderland-485819661', 'http://sakimichan.deviantart.com/art/Ariels-476991263' ]
self.assertEqual( page_of_urls, expected_gallery_urls )
self.assertFalse( definitely_no_more_pages )
#
tags = ['title:Sailor moon in PJs', 'creator:sakimichan']
( os_file_handle, temp_path ) = HydrusPaths.GetTempPath()
try:
tags = gallery.GetFileAndTags( temp_path, 'http://sakimichan.deviantart.com/art/Sailor-moon-in-PJs-506918040' )
with open( temp_path, 'rb' ) as f: data = f.read()
finally:
HydrusPaths.CleanUpTempPath( os_file_handle, temp_path )
info = ( data, tags )
expected_info = ('image file', tags)
self.assertEqual( info, expected_info )
def test_newgrounds( self ):
with open( os.path.join( HC.STATIC_DIR, 'testing', 'newgrounds_gallery_games.html' ) ) as f: newgrounds_gallery_games = f.read()

1094
static/testing/da_gallery.html
View File

File diff suppressed because one or more lines are too long

1434
static/testing/da_page.html
View File

File diff suppressed because one or more lines are too long