1225 lines
38 KiB
Python
1225 lines
38 KiB
Python
import os
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
|
|
from hydrus.core import HydrusConstants as HC
|
|
from hydrus.core import HydrusData
|
|
from hydrus.core import HydrusExceptions
|
|
from hydrus.core import HydrusGlobals as HG
|
|
from hydrus.core import HydrusSerialisable
|
|
|
|
from hydrus.client import ClientConstants as CC
|
|
from hydrus.client.importing import ClientImporting
|
|
from hydrus.client.importing import ClientImportFileSeeds
|
|
from hydrus.client.importing import ClientImportGallerySeeds
|
|
from hydrus.client.importing.options import TagImportOptions
|
|
from hydrus.client.metadata import ClientTags
|
|
from hydrus.client.networking import ClientNetworkingJobs
|
|
|
|
class SimpleDownloaderImport( HydrusSerialisable.SerialisableBase ):
|
|
|
|
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_SIMPLE_DOWNLOADER_IMPORT
|
|
SERIALISABLE_NAME = 'Simple Downloader Import'
|
|
SERIALISABLE_VERSION = 5
|
|
|
|
def __init__( self ):
|
|
|
|
HydrusSerialisable.SerialisableBase.__init__( self )
|
|
|
|
file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud' )
|
|
|
|
self._pending_jobs = []
|
|
self._gallery_seed_log = ClientImportGallerySeeds.GallerySeedLog()
|
|
self._file_seed_cache = ClientImportFileSeeds.FileSeedCache()
|
|
self._file_import_options = file_import_options
|
|
self._formula_name = 'all files linked by images in page'
|
|
self._queue_paused = False
|
|
self._files_paused = False
|
|
|
|
self._downloader_key = HydrusData.GenerateKey()
|
|
|
|
self._parser_status = ''
|
|
self._current_action = ''
|
|
|
|
self._lock = threading.Lock()
|
|
|
|
self._have_started = False
|
|
|
|
self._files_network_job = None
|
|
self._page_network_job = None
|
|
|
|
self._files_repeating_job = None
|
|
self._queue_repeating_job = None
|
|
|
|
HG.client_controller.sub( self, 'NotifyFileSeedsUpdated', 'file_seed_cache_file_seeds_updated' )
|
|
|
|
|
|
def _FileNetworkJobPresentationContextFactory( self, network_job ):
|
|
|
|
def enter_call():
|
|
|
|
with self._lock:
|
|
|
|
self._files_network_job = network_job
|
|
|
|
|
|
|
|
def exit_call():
|
|
|
|
with self._lock:
|
|
|
|
self._files_network_job = None
|
|
|
|
|
|
|
|
return ClientImporting.NetworkJobPresentationContext( enter_call, exit_call )
|
|
|
|
|
|
def _GetSerialisableInfo( self ):
|
|
|
|
serialisable_pending_jobs = [ ( url, simple_downloader_formula.GetSerialisableTuple() ) for ( url, simple_downloader_formula ) in self._pending_jobs ]
|
|
|
|
serialisable_gallery_seed_log = self._gallery_seed_log.GetSerialisableTuple()
|
|
serialisable_file_seed_cache = self._file_seed_cache.GetSerialisableTuple()
|
|
serialisable_file_import_options = self._file_import_options.GetSerialisableTuple()
|
|
|
|
return ( serialisable_pending_jobs, serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, self._formula_name, self._queue_paused, self._files_paused )
|
|
|
|
|
|
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
|
|
|
|
( serialisable_pending_jobs, serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, self._formula_name, self._queue_paused, self._files_paused ) = serialisable_info
|
|
|
|
self._pending_jobs = [ ( url, HydrusSerialisable.CreateFromSerialisableTuple( serialisable_simple_downloader_formula ) ) for ( url, serialisable_simple_downloader_formula ) in serialisable_pending_jobs ]
|
|
|
|
self._gallery_seed_log = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_gallery_seed_log )
|
|
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
|
|
self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_import_options )
|
|
|
|
|
|
def _NetworkJobFactory( self, *args, **kwargs ):
|
|
|
|
network_job = ClientNetworkingJobs.NetworkJobDownloader( self._downloader_key, *args, **kwargs )
|
|
|
|
return network_job
|
|
|
|
|
|
def _PageNetworkJobPresentationContextFactory( self, network_job ):
|
|
|
|
def enter_call():
|
|
|
|
with self._lock:
|
|
|
|
self._page_network_job = network_job
|
|
|
|
|
|
|
|
def exit_call():
|
|
|
|
with self._lock:
|
|
|
|
self._page_network_job = None
|
|
|
|
|
|
|
|
return ClientImporting.NetworkJobPresentationContext( enter_call, exit_call )
|
|
|
|
|
|
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
|
|
|
|
if version == 1:
|
|
|
|
( pending_page_urls, serialisable_file_seed_cache, serialisable_file_import_options, download_image_links, download_unlinked_images, paused ) = old_serialisable_info
|
|
|
|
queue_paused = paused
|
|
files_paused = paused
|
|
|
|
new_serialisable_info = ( pending_page_urls, serialisable_file_seed_cache, serialisable_file_import_options, download_image_links, download_unlinked_images, queue_paused, files_paused )
|
|
|
|
return ( 2, new_serialisable_info )
|
|
|
|
|
|
if version == 2:
|
|
|
|
( pending_page_urls, serialisable_file_seed_cache, serialisable_file_import_options, download_image_links, download_unlinked_images, queue_paused, files_paused ) = old_serialisable_info
|
|
|
|
pending_jobs = []
|
|
|
|
new_serialisable_info = ( pending_jobs, serialisable_file_seed_cache, serialisable_file_import_options, queue_paused, files_paused )
|
|
|
|
return ( 3, new_serialisable_info )
|
|
|
|
|
|
if version == 3:
|
|
|
|
( pending_jobs, serialisable_file_seed_cache, serialisable_file_import_options, queue_paused, files_paused ) = old_serialisable_info
|
|
|
|
pending_jobs = []
|
|
|
|
formula_name = 'all files linked by images in page'
|
|
|
|
new_serialisable_info = ( pending_jobs, serialisable_file_seed_cache, serialisable_file_import_options, formula_name, queue_paused, files_paused )
|
|
|
|
return ( 4, new_serialisable_info )
|
|
|
|
|
|
if version == 4:
|
|
|
|
( pending_jobs, serialisable_file_seed_cache, serialisable_file_import_options, formula_name, queue_paused, files_paused ) = old_serialisable_info
|
|
|
|
gallery_seed_log = ClientImportGallerySeeds.GallerySeedLog()
|
|
|
|
serialisable_gallery_seed_log = gallery_seed_log.GetSerialisableTuple()
|
|
|
|
new_serialisable_info = ( pending_jobs, serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, formula_name, queue_paused, files_paused )
|
|
|
|
return ( 5, new_serialisable_info )
|
|
|
|
|
|
|
|
def _WorkOnFiles( self, page_key ):
|
|
|
|
file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
|
|
|
|
if file_seed is None:
|
|
|
|
return
|
|
|
|
|
|
did_substantial_work = False
|
|
|
|
def status_hook( text ):
|
|
|
|
with self._lock:
|
|
|
|
if len( text ) > 0:
|
|
|
|
text = text.splitlines()[0]
|
|
|
|
|
|
self._current_action = text
|
|
|
|
|
|
|
|
tag_import_options = TagImportOptions.TagImportOptions( is_default = True )
|
|
|
|
did_substantial_work = file_seed.WorkOnURL( self._file_seed_cache, status_hook, self._NetworkJobFactory, self._FileNetworkJobPresentationContextFactory, self._file_import_options, tag_import_options )
|
|
|
|
if file_seed.ShouldPresent( self._file_import_options ):
|
|
|
|
file_seed.PresentToPage( page_key )
|
|
|
|
did_substantial_work = True
|
|
|
|
|
|
with self._lock:
|
|
|
|
self._current_action = ''
|
|
|
|
|
|
if did_substantial_work:
|
|
|
|
time.sleep( ClientImporting.DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )
|
|
|
|
|
|
|
|
def _WorkOnQueue( self, page_key ):
|
|
|
|
if len( self._pending_jobs ) > 0:
|
|
|
|
with self._lock:
|
|
|
|
( url, simple_downloader_formula ) = self._pending_jobs.pop( 0 )
|
|
|
|
self._parser_status = 'checking ' + url
|
|
|
|
|
|
error_occurred = False
|
|
|
|
try:
|
|
|
|
gallery_seed = ClientImportGallerySeeds.GallerySeed( url, can_generate_more_pages = False )
|
|
|
|
self._gallery_seed_log.AddGallerySeeds( ( gallery_seed, ) )
|
|
|
|
network_job = self._NetworkJobFactory( 'GET', url )
|
|
|
|
network_job.OverrideBandwidth( 30 )
|
|
|
|
HG.client_controller.network_engine.AddJob( network_job )
|
|
|
|
with self._PageNetworkJobPresentationContextFactory( network_job ):
|
|
|
|
network_job.WaitUntilDone()
|
|
|
|
|
|
parsing_text = network_job.GetContentText()
|
|
|
|
#
|
|
|
|
parsing_context = {}
|
|
|
|
parsing_context[ 'url' ] = url
|
|
|
|
parsing_formula = simple_downloader_formula.GetFormula()
|
|
|
|
file_seeds = []
|
|
|
|
for parsed_text in parsing_formula.Parse( parsing_context, parsing_text ):
|
|
|
|
try:
|
|
|
|
file_url = urllib.parse.urljoin( url, parsed_text )
|
|
|
|
file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, file_url )
|
|
|
|
file_seed.SetReferralURL( url )
|
|
|
|
file_seeds.append( file_seed )
|
|
|
|
except:
|
|
|
|
continue
|
|
|
|
|
|
|
|
num_new = self._file_seed_cache.AddFileSeeds( file_seeds )
|
|
|
|
if num_new > 0:
|
|
|
|
ClientImporting.WakeRepeatingJob( self._files_repeating_job )
|
|
|
|
|
|
parser_status = 'page checked OK with formula "' + simple_downloader_formula.GetName() + '" - ' + HydrusData.ToHumanInt( num_new ) + ' new urls'
|
|
|
|
num_already_in_file_seed_cache = len( file_seeds ) - num_new
|
|
|
|
if num_already_in_file_seed_cache > 0:
|
|
|
|
parser_status += ' (' + HydrusData.ToHumanInt( num_already_in_file_seed_cache ) + ' already in queue)'
|
|
|
|
|
|
gallery_seed_status = CC.STATUS_SUCCESSFUL_AND_NEW
|
|
|
|
except HydrusExceptions.ShutdownException:
|
|
|
|
gallery_seed_status = CC.STATUS_VETOED
|
|
parser_status = 'program is shutting down'
|
|
|
|
return
|
|
|
|
except HydrusExceptions.NotFoundException:
|
|
|
|
gallery_seed_status = CC.STATUS_VETOED
|
|
|
|
error_occurred = True
|
|
|
|
parser_status = 'page 404'
|
|
|
|
except Exception as e:
|
|
|
|
gallery_seed_status = CC.STATUS_ERROR
|
|
|
|
error_occurred = True
|
|
|
|
parser_status = str( e )
|
|
|
|
finally:
|
|
|
|
gallery_seed_note = parser_status
|
|
|
|
gallery_seed.SetStatus( gallery_seed_status, note = gallery_seed_note )
|
|
|
|
self._gallery_seed_log.NotifyGallerySeedsUpdated( ( gallery_seed, ) )
|
|
|
|
|
|
with self._lock:
|
|
|
|
self._parser_status = parser_status
|
|
|
|
|
|
if error_occurred:
|
|
|
|
time.sleep( 5 )
|
|
|
|
|
|
return True
|
|
|
|
else:
|
|
|
|
with self._lock:
|
|
|
|
self._parser_status = ''
|
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
def AdvanceJob( self, job ):
|
|
|
|
with self._lock:
|
|
|
|
if job in self._pending_jobs:
|
|
|
|
index = self._pending_jobs.index( job )
|
|
|
|
if index - 1 >= 0:
|
|
|
|
self._pending_jobs.remove( job )
|
|
|
|
self._pending_jobs.insert( index - 1, job )
|
|
|
|
|
|
|
|
|
|
|
|
def CurrentlyWorking( self ):
|
|
|
|
with self._lock:
|
|
|
|
finished = not self._file_seed_cache.WorkToDo() or len( self._pending_jobs ) > 0
|
|
|
|
return not finished and not self._files_paused
|
|
|
|
|
|
|
|
def DelayJob( self, job ):
|
|
|
|
with self._lock:
|
|
|
|
if job in self._pending_jobs:
|
|
|
|
index = self._pending_jobs.index( job )
|
|
|
|
if index + 1 < len( self._pending_jobs ):
|
|
|
|
self._pending_jobs.remove( job )
|
|
|
|
self._pending_jobs.insert( index + 1, job )
|
|
|
|
|
|
|
|
|
|
|
|
def DeleteJob( self, job ):
|
|
|
|
with self._lock:
|
|
|
|
if job in self._pending_jobs:
|
|
|
|
self._pending_jobs.remove( job )
|
|
|
|
|
|
|
|
|
|
def GetAPIInfoDict( self, simple ):
|
|
|
|
with self._lock:
|
|
|
|
d = {}
|
|
|
|
d[ 'imports' ] = self._file_seed_cache.GetAPIInfoDict( simple )
|
|
|
|
d[ 'gallery_log' ] = self._gallery_seed_log.GetAPIInfoDict( simple )
|
|
|
|
d[ 'files_paused' ] = self._files_paused
|
|
|
|
d[ 'gallery_paused' ] = self._queue_paused
|
|
|
|
return d
|
|
|
|
|
|
|
|
def GetFileSeedCache( self ):
|
|
|
|
with self._lock:
|
|
|
|
return self._file_seed_cache
|
|
|
|
|
|
|
|
def GetFileImportOptions( self ):
|
|
|
|
with self._lock:
|
|
|
|
return self._file_import_options
|
|
|
|
|
|
|
|
def GetFormulaName( self ):
|
|
|
|
with self._lock:
|
|
|
|
return self._formula_name
|
|
|
|
|
|
|
|
def GetGallerySeedLog( self ):
|
|
|
|
with self._lock:
|
|
|
|
return self._gallery_seed_log
|
|
|
|
|
|
|
|
def GetNetworkJobs( self ):
|
|
|
|
with self._lock:
|
|
|
|
return ( self._files_network_job, self._page_network_job )
|
|
|
|
|
|
|
|
def GetNumSeeds( self ):
|
|
|
|
with self._lock:
|
|
|
|
return len( self._file_seed_cache ) + len( self._gallery_seed_log )
|
|
|
|
|
|
|
|
def GetStatus( self ):
|
|
|
|
with self._lock:
|
|
|
|
return ( list( self._pending_jobs ), self._parser_status, self._current_action, self._queue_paused, self._files_paused )
|
|
|
|
|
|
|
|
def GetValueRange( self ):
|
|
|
|
with self._lock:
|
|
|
|
return self._file_seed_cache.GetValueRange()
|
|
|
|
|
|
|
|
def NotifyFileSeedsUpdated( self, file_seed_cache_key, file_seeds ):
|
|
|
|
if file_seed_cache_key == self._file_seed_cache.GetFileSeedCacheKey():
|
|
|
|
ClientImporting.WakeRepeatingJob( self._files_repeating_job )
|
|
|
|
|
|
|
|
def PausePlayFiles( self ):
|
|
|
|
with self._lock:
|
|
|
|
self._files_paused = not self._files_paused
|
|
|
|
ClientImporting.WakeRepeatingJob( self._files_repeating_job )
|
|
|
|
|
|
|
|
def PausePlayQueue( self ):
|
|
|
|
with self._lock:
|
|
|
|
self._queue_paused = not self._queue_paused
|
|
|
|
ClientImporting.WakeRepeatingJob( self._queue_repeating_job )
|
|
|
|
|
|
|
|
def PendJob( self, job ):
|
|
|
|
with self._lock:
|
|
|
|
if job not in self._pending_jobs:
|
|
|
|
self._pending_jobs.append( job )
|
|
|
|
ClientImporting.WakeRepeatingJob( self._queue_repeating_job )
|
|
|
|
|
|
|
|
|
|
def SetFileImportOptions( self, file_import_options ):
|
|
|
|
with self._lock:
|
|
|
|
self._file_import_options = file_import_options
|
|
|
|
|
|
|
|
def SetFormulaName( self, formula_name ):
|
|
|
|
with self._lock:
|
|
|
|
self._formula_name = formula_name
|
|
|
|
|
|
|
|
def Start( self, page_key ):
|
|
|
|
with self._lock:
|
|
|
|
if self._have_started:
|
|
|
|
return
|
|
|
|
|
|
self._files_repeating_job = HG.client_controller.CallRepeating( ClientImporting.GetRepeatingJobInitialDelay(), ClientImporting.REPEATING_JOB_TYPICAL_PERIOD, self.REPEATINGWorkOnFiles, page_key )
|
|
self._queue_repeating_job = HG.client_controller.CallRepeating( ClientImporting.GetRepeatingJobInitialDelay(), ClientImporting.REPEATING_JOB_TYPICAL_PERIOD, self.REPEATINGWorkOnQueue, page_key )
|
|
|
|
self._files_repeating_job.SetThreadSlotType( 'misc' )
|
|
self._queue_repeating_job.SetThreadSlotType( 'misc' )
|
|
|
|
self._have_started = True
|
|
|
|
|
|
|
|
def CanDoFileWork( self, page_key ):
|
|
|
|
with self._lock:
|
|
|
|
if ClientImporting.PageImporterShouldStopWorking( page_key ):
|
|
|
|
self._files_repeating_job.Cancel()
|
|
|
|
return False
|
|
|
|
|
|
files_paused = self._files_paused or HG.client_controller.new_options.GetBoolean( 'pause_all_file_queues' )
|
|
|
|
if files_paused:
|
|
|
|
return False
|
|
|
|
|
|
work_to_do = self._file_seed_cache.WorkToDo()
|
|
|
|
if not work_to_do:
|
|
|
|
return False
|
|
|
|
|
|
|
|
return self.CanDoNetworkWork( page_key )
|
|
|
|
|
|
def CanDoNetworkWork( self, page_key ):
|
|
|
|
with self._lock:
|
|
|
|
page_shown = not HG.client_controller.PageClosedButNotDestroyed( page_key )
|
|
|
|
if not page_shown:
|
|
|
|
return False
|
|
|
|
|
|
network_engine_good = not HG.client_controller.network_engine.IsBusy()
|
|
|
|
if not network_engine_good:
|
|
|
|
return False
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
|
def REPEATINGWorkOnFiles( self, page_key ):
|
|
|
|
while self.CanDoFileWork( page_key ):
|
|
|
|
try:
|
|
|
|
self._WorkOnFiles( page_key )
|
|
|
|
HG.client_controller.WaitUntilViewFree()
|
|
|
|
except Exception as e:
|
|
|
|
HydrusData.ShowException( e )
|
|
|
|
|
|
|
|
|
|
def CanDoQueueWork( self, page_key ):
|
|
|
|
with self._lock:
|
|
|
|
if ClientImporting.PageImporterShouldStopWorking( page_key ):
|
|
|
|
self._queue_repeating_job.Cancel()
|
|
|
|
return False
|
|
|
|
|
|
queue_paused = self._queue_paused or HG.client_controller.new_options.GetBoolean( 'pause_all_gallery_searches' )
|
|
|
|
if queue_paused:
|
|
|
|
return False
|
|
|
|
|
|
|
|
return self.CanDoNetworkWork( page_key )
|
|
|
|
|
|
def REPEATINGWorkOnQueue( self, page_key ):
|
|
|
|
while self.CanDoQueueWork( page_key ):
|
|
|
|
try:
|
|
|
|
did_work = self._WorkOnQueue( page_key )
|
|
|
|
if did_work:
|
|
|
|
time.sleep( ClientImporting.DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )
|
|
|
|
else:
|
|
|
|
return
|
|
|
|
|
|
HG.client_controller.WaitUntilViewFree()
|
|
|
|
except Exception as e:
|
|
|
|
HydrusData.ShowException( e )
|
|
|
|
|
|
|
|
|
|
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_SIMPLE_DOWNLOADER_IMPORT ] = SimpleDownloaderImport
|
|
|
|
class URLsImport( HydrusSerialisable.SerialisableBase ):
|
|
|
|
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_URLS_IMPORT
|
|
SERIALISABLE_NAME = 'URL Import'
|
|
SERIALISABLE_VERSION = 3
|
|
|
|
def __init__( self ):
|
|
|
|
HydrusSerialisable.SerialisableBase.__init__( self )
|
|
|
|
self._gallery_seed_log = ClientImportGallerySeeds.GallerySeedLog()
|
|
self._file_seed_cache = ClientImportFileSeeds.FileSeedCache()
|
|
self._file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud' )
|
|
self._tag_import_options = TagImportOptions.TagImportOptions( is_default = True )
|
|
self._paused = False
|
|
|
|
self._downloader_key = HydrusData.GenerateKey()
|
|
|
|
self._lock = threading.Lock()
|
|
|
|
self._have_started = False
|
|
|
|
self._files_network_job = None
|
|
self._gallery_network_job = None
|
|
|
|
self._files_repeating_job = None
|
|
self._gallery_repeating_job = None
|
|
|
|
HG.client_controller.sub( self, 'NotifyFileSeedsUpdated', 'file_seed_cache_file_seeds_updated' )
|
|
HG.client_controller.sub( self, 'NotifyGallerySeedsUpdated', 'gallery_seed_log_gallery_seeds_updated' )
|
|
|
|
|
|
def _FileNetworkJobPresentationContextFactory( self, network_job ):
|
|
|
|
def enter_call():
|
|
|
|
with self._lock:
|
|
|
|
self._files_network_job = network_job
|
|
|
|
|
|
|
|
def exit_call():
|
|
|
|
with self._lock:
|
|
|
|
self._files_network_job = None
|
|
|
|
|
|
|
|
return ClientImporting.NetworkJobPresentationContext( enter_call, exit_call )
|
|
|
|
|
|
def _GalleryNetworkJobPresentationContextFactory( self, network_job ):
|
|
|
|
def enter_call():
|
|
|
|
with self._lock:
|
|
|
|
self._gallery_network_job = network_job
|
|
|
|
|
|
|
|
def exit_call():
|
|
|
|
with self._lock:
|
|
|
|
self._gallery_network_job = None
|
|
|
|
|
|
|
|
return ClientImporting.NetworkJobPresentationContext( enter_call, exit_call )
|
|
|
|
|
|
def _GetSerialisableInfo( self ):
|
|
|
|
serialisable_gallery_seed_log = self._gallery_seed_log.GetSerialisableTuple()
|
|
serialisable_file_seed_cache = self._file_seed_cache.GetSerialisableTuple()
|
|
serialisable_file_import_options = self._file_import_options.GetSerialisableTuple()
|
|
serialisable_tag_import_options = self._tag_import_options.GetSerialisableTuple()
|
|
|
|
return ( serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, serialisable_tag_import_options, self._paused )
|
|
|
|
|
|
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
|
|
|
|
( serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, serialisable_tag_import_options, self._paused ) = serialisable_info
|
|
|
|
self._gallery_seed_log = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_gallery_seed_log )
|
|
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
|
|
self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_import_options )
|
|
self._tag_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_import_options )
|
|
|
|
|
|
def _NetworkJobFactory( self, *args, **kwargs ):
|
|
|
|
network_job = ClientNetworkingJobs.NetworkJobDownloader( self._downloader_key, *args, **kwargs )
|
|
|
|
return network_job
|
|
|
|
|
|
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
|
|
|
|
if version == 1:
|
|
|
|
( serialisable_file_seed_cache, serialisable_file_import_options, paused ) = old_serialisable_info
|
|
|
|
gallery_seed_log = ClientImportGallerySeeds.GallerySeedLog()
|
|
|
|
serialisable_gallery_seed_log = gallery_seed_log.GetSerialisableTuple()
|
|
|
|
new_serialisable_info = ( serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, paused )
|
|
|
|
return ( 2, new_serialisable_info )
|
|
|
|
|
|
if version == 2:
|
|
|
|
( serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, paused ) = old_serialisable_info
|
|
|
|
tag_import_options = TagImportOptions.TagImportOptions( is_default = True )
|
|
|
|
serialisable_tag_import_options = tag_import_options.GetSerialisableTuple()
|
|
|
|
new_serialisable_info = ( serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, serialisable_tag_import_options, paused )
|
|
|
|
return ( 3, new_serialisable_info )
|
|
|
|
|
|
|
|
def _WorkOnFiles( self, page_key ):
|
|
|
|
file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
|
|
|
|
if file_seed is None:
|
|
|
|
return
|
|
|
|
|
|
did_substantial_work = False
|
|
|
|
url = file_seed.file_seed_data
|
|
|
|
try:
|
|
|
|
status_hook = lambda s: s # do nothing for now
|
|
|
|
did_substantial_work = file_seed.WorkOnURL( self._file_seed_cache, status_hook, self._NetworkJobFactory, self._FileNetworkJobPresentationContextFactory, self._file_import_options, self._tag_import_options )
|
|
|
|
if file_seed.ShouldPresent( self._file_import_options ):
|
|
|
|
file_seed.PresentToPage( page_key )
|
|
|
|
did_substantial_work = True
|
|
|
|
|
|
except Exception as e:
|
|
|
|
status = CC.STATUS_ERROR
|
|
|
|
file_seed.SetStatus( status, exception = e )
|
|
|
|
time.sleep( 3 )
|
|
|
|
|
|
if did_substantial_work:
|
|
|
|
time.sleep( ClientImporting.DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )
|
|
|
|
|
|
|
|
def _WorkOnGallery( self, page_key ):
|
|
|
|
gallery_seed = self._gallery_seed_log.GetNextGallerySeed( CC.STATUS_UNKNOWN )
|
|
|
|
if gallery_seed is None:
|
|
|
|
return
|
|
|
|
|
|
try:
|
|
|
|
status_hook = lambda s: s
|
|
title_hook = lambda s: s
|
|
|
|
def file_seeds_callable( file_seeds ):
|
|
|
|
return ClientImporting.UpdateFileSeedCacheWithFileSeeds( self._file_seed_cache, file_seeds )
|
|
|
|
|
|
gallery_seed.WorkOnURL( 'download page', self._gallery_seed_log, file_seeds_callable, status_hook, title_hook, self._NetworkJobFactory, self._GalleryNetworkJobPresentationContextFactory, self._file_import_options )
|
|
|
|
except Exception as e:
|
|
|
|
status = CC.STATUS_ERROR
|
|
|
|
gallery_seed.SetStatus( status, exception = e )
|
|
|
|
time.sleep( 3 )
|
|
|
|
|
|
time.sleep( 1 )
|
|
|
|
|
|
def CurrentlyWorking( self ):
|
|
|
|
with self._lock:
|
|
|
|
finished = not self._file_seed_cache.WorkToDo()
|
|
|
|
return not finished and not self._paused
|
|
|
|
|
|
|
|
def GetAPIInfoDict( self, simple ):
|
|
|
|
with self._lock:
|
|
|
|
d = {}
|
|
|
|
d[ 'imports' ] = self._file_seed_cache.GetAPIInfoDict( simple )
|
|
|
|
d[ 'gallery_log' ] = self._gallery_seed_log.GetAPIInfoDict( simple )
|
|
|
|
d[ 'files_paused' ] = self._paused
|
|
|
|
return d
|
|
|
|
|
|
|
|
def GetFileSeedCache( self ):
|
|
|
|
with self._lock:
|
|
|
|
return self._file_seed_cache
|
|
|
|
|
|
|
|
def GetGallerySeedLog( self ):
|
|
|
|
with self._lock:
|
|
|
|
return self._gallery_seed_log
|
|
|
|
|
|
|
|
def GetNetworkJobs( self ):
|
|
|
|
with self._lock:
|
|
|
|
return ( self._files_network_job, self._gallery_network_job )
|
|
|
|
|
|
|
|
def GetNumSeeds( self ):
|
|
|
|
with self._lock:
|
|
|
|
return len( self._file_seed_cache ) + len( self._gallery_seed_log )
|
|
|
|
|
|
|
|
def GetOptions( self ):
|
|
|
|
with self._lock:
|
|
|
|
return ( self._file_import_options, self._tag_import_options )
|
|
|
|
|
|
|
|
def GetValueRange( self ):
|
|
|
|
with self._lock:
|
|
|
|
return self._file_seed_cache.GetValueRange()
|
|
|
|
|
|
|
|
def IsPaused( self ):
|
|
|
|
with self._lock:
|
|
|
|
return self._paused
|
|
|
|
|
|
|
|
def NotifyFileSeedsUpdated( self, file_seed_cache_key, file_seeds ):
|
|
|
|
if file_seed_cache_key == self._file_seed_cache.GetFileSeedCacheKey():
|
|
|
|
ClientImporting.WakeRepeatingJob( self._files_repeating_job )
|
|
|
|
|
|
|
|
def NotifyGallerySeedsUpdated( self, gallery_seed_log_key, gallery_seeds ):
|
|
|
|
if gallery_seed_log_key == self._gallery_seed_log.GetGallerySeedLogKey():
|
|
|
|
ClientImporting.WakeRepeatingJob( self._gallery_repeating_job )
|
|
|
|
|
|
|
|
def PausePlay( self ):
|
|
|
|
with self._lock:
|
|
|
|
self._paused = not self._paused
|
|
|
|
ClientImporting.WakeRepeatingJob( self._files_repeating_job )
|
|
ClientImporting.WakeRepeatingJob( self._gallery_repeating_job )
|
|
|
|
|
|
|
|
def PendURLs( self, urls, filterable_tags = None, additional_service_keys_to_tags = None ):
|
|
|
|
if filterable_tags is None:
|
|
|
|
filterable_tags = set()
|
|
|
|
|
|
if additional_service_keys_to_tags is None:
|
|
|
|
additional_service_keys_to_tags = ClientTags.ServiceKeysToTags()
|
|
|
|
|
|
with self._lock:
|
|
|
|
urls = [ u for u in urls if len( u ) > 1 ] # > _1_ to take out the occasional whitespace
|
|
|
|
file_seeds = []
|
|
|
|
gallery_seeds = []
|
|
|
|
for url in urls:
|
|
|
|
try:
|
|
|
|
url_class = HG.client_controller.network_engine.domain_manager.GetURLClass( url )
|
|
|
|
except HydrusExceptions.URLClassException:
|
|
|
|
continue
|
|
|
|
|
|
if url_class is None or url_class.GetURLType() in ( HC.URL_TYPE_FILE, HC.URL_TYPE_POST ):
|
|
|
|
file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url )
|
|
|
|
file_seed.SetExternalFilterableTags( filterable_tags )
|
|
file_seed.SetExternalAdditionalServiceKeysToTags( additional_service_keys_to_tags )
|
|
|
|
file_seeds.append( file_seed )
|
|
|
|
else:
|
|
|
|
can_generate_more_pages = False
|
|
|
|
gallery_seed = ClientImportGallerySeeds.GallerySeed( url, can_generate_more_pages = can_generate_more_pages )
|
|
|
|
gallery_seed.SetExternalFilterableTags( filterable_tags )
|
|
gallery_seed.SetExternalAdditionalServiceKeysToTags( additional_service_keys_to_tags )
|
|
|
|
gallery_seeds.append( gallery_seed )
|
|
|
|
|
|
|
|
if len( gallery_seeds ) > 0:
|
|
|
|
self._gallery_seed_log.AddGallerySeeds( gallery_seeds )
|
|
|
|
ClientImporting.WakeRepeatingJob( self._gallery_repeating_job )
|
|
|
|
|
|
if len( file_seeds ) > 0:
|
|
|
|
self._file_seed_cache.AddFileSeeds( file_seeds )
|
|
|
|
ClientImporting.WakeRepeatingJob( self._files_repeating_job )
|
|
|
|
|
|
|
|
|
|
def SetFileImportOptions( self, file_import_options ):
|
|
|
|
with self._lock:
|
|
|
|
self._file_import_options = file_import_options
|
|
|
|
|
|
|
|
def SetTagImportOptions( self, tag_import_options ):
|
|
|
|
with self._lock:
|
|
|
|
self._tag_import_options = tag_import_options
|
|
|
|
|
|
|
|
def Start( self, page_key ):
|
|
|
|
with self._lock:
|
|
|
|
if self._have_started:
|
|
|
|
return
|
|
|
|
|
|
self._files_repeating_job = HG.client_controller.CallRepeating( ClientImporting.GetRepeatingJobInitialDelay(), ClientImporting.REPEATING_JOB_TYPICAL_PERIOD, self.REPEATINGWorkOnFiles, page_key )
|
|
self._gallery_repeating_job = HG.client_controller.CallRepeating( ClientImporting.GetRepeatingJobInitialDelay(), ClientImporting.REPEATING_JOB_TYPICAL_PERIOD, self.REPEATINGWorkOnGallery, page_key )
|
|
|
|
self._files_repeating_job.SetThreadSlotType( 'misc' )
|
|
self._gallery_repeating_job.SetThreadSlotType( 'misc' )
|
|
|
|
self._have_started = True
|
|
|
|
|
|
|
|
def CanDoFileWork( self, page_key ):
|
|
|
|
with self._lock:
|
|
|
|
if ClientImporting.PageImporterShouldStopWorking( page_key ):
|
|
|
|
self._files_repeating_job.Cancel()
|
|
|
|
return False
|
|
|
|
|
|
files_paused = self._paused or HG.client_controller.new_options.GetBoolean( 'pause_all_file_queues' )
|
|
|
|
if files_paused:
|
|
|
|
return False
|
|
|
|
|
|
work_to_do = self._file_seed_cache.WorkToDo()
|
|
|
|
if not work_to_do:
|
|
|
|
return False
|
|
|
|
|
|
|
|
return self.CanDoNetworkWork( page_key )
|
|
|
|
|
|
def CanDoNetworkWork( self, page_key ):
|
|
|
|
with self._lock:
|
|
|
|
page_shown = not HG.client_controller.PageClosedButNotDestroyed( page_key )
|
|
|
|
if not page_shown:
|
|
|
|
return False
|
|
|
|
|
|
network_engine_good = not HG.client_controller.network_engine.IsBusy()
|
|
|
|
if not network_engine_good:
|
|
|
|
return False
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
|
def REPEATINGWorkOnFiles( self, page_key ):
|
|
|
|
while self.CanDoFileWork( page_key ):
|
|
|
|
try:
|
|
|
|
self._WorkOnFiles( page_key )
|
|
|
|
HG.client_controller.WaitUntilViewFree()
|
|
|
|
except Exception as e:
|
|
|
|
HydrusData.ShowException( e )
|
|
|
|
|
|
|
|
|
|
def CanDoGalleryWork( self, page_key ):
|
|
|
|
with self._lock:
|
|
|
|
if ClientImporting.PageImporterShouldStopWorking( page_key ):
|
|
|
|
self._gallery_repeating_job.Cancel()
|
|
|
|
return False
|
|
|
|
|
|
gallery_paused = self._paused or HG.client_controller.new_options.GetBoolean( 'pause_all_gallery_searches' )
|
|
|
|
if gallery_paused:
|
|
|
|
return False
|
|
|
|
|
|
work_to_do = self._gallery_seed_log.WorkToDo()
|
|
|
|
if not work_to_do:
|
|
|
|
return False
|
|
|
|
|
|
|
|
return self.CanDoNetworkWork( page_key )
|
|
|
|
|
|
def REPEATINGWorkOnGallery( self, page_key ):
|
|
|
|
while self.CanDoGalleryWork( page_key ):
|
|
|
|
try:
|
|
|
|
self._WorkOnGallery( page_key )
|
|
|
|
HG.client_controller.WaitUntilViewFree()
|
|
|
|
except Exception as e:
|
|
|
|
HydrusData.ShowException( e )
|
|
|
|
|
|
|
|
|
|
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_URLS_IMPORT ] = URLsImport
|