2017-06-28 20:23:21 +00:00
import ClientConstants as CC
2017-06-07 22:05:15 +00:00
import collections
2017-06-21 21:15:59 +00:00
import cStringIO
2015-10-21 21:53:10 +00:00
import HydrusConstants as HC
2017-10-25 21:45:15 +00:00
import HydrusData
2015-10-21 21:53:10 +00:00
import HydrusExceptions
2017-10-25 21:45:15 +00:00
import HydrusGlobals as HG
2017-03-02 02:14:56 +00:00
import HydrusNetwork
2017-06-07 22:05:15 +00:00
import HydrusNetworking
2015-11-04 22:30:28 +00:00
import HydrusPaths
2015-10-21 21:53:10 +00:00
import HydrusSerialisable
2017-10-25 21:45:15 +00:00
import itertools
2018-10-17 21:00:09 +00:00
import multipart
2015-10-21 21:53:10 +00:00
import os
2017-06-21 21:15:59 +00:00
import random
2016-02-24 21:42:54 +00:00
import requests
2017-06-21 21:15:59 +00:00
import urllib3
from urllib3 . exceptions import InsecureRequestWarning
2015-10-21 21:53:10 +00:00
import threading
import time
2017-06-21 21:15:59 +00:00
import traceback
2015-10-21 21:53:10 +00:00
import urllib
import urlparse
import yaml
2017-06-21 21:15:59 +00:00
urllib3 . disable_warnings ( InsecureRequestWarning )
2017-01-25 22:56:55 +00:00
2018-04-18 22:10:15 +00:00
JOB_STATUS_AWAITING_VALIDITY = 0
JOB_STATUS_AWAITING_BANDWIDTH = 1
JOB_STATUS_AWAITING_LOGIN = 2
JOB_STATUS_AWAITING_SLOT = 3
JOB_STATUS_RUNNING = 4
job_status_str_lookup = { }
job_status_str_lookup [ JOB_STATUS_AWAITING_VALIDITY ] = ' waiting for validation '
job_status_str_lookup [ JOB_STATUS_AWAITING_BANDWIDTH ] = ' waiting for bandwidth '
job_status_str_lookup [ JOB_STATUS_AWAITING_LOGIN ] = ' waiting for login '
job_status_str_lookup [ JOB_STATUS_AWAITING_SLOT ] = ' waiting for slot '
job_status_str_lookup [ JOB_STATUS_RUNNING ] = ' running '
2018-10-17 21:00:09 +00:00
def GenerateMultipartFormDataCTAndBodyFromDict ( fields ) :
m = multipart . Multipart ( )
for ( name , value ) in fields . items ( ) :
m . field ( name , HydrusData . ToByteString ( value ) )
return m . get ( )
2018-04-18 22:10:15 +00:00
class NetworkEngine ( object ) :
2017-10-25 21:45:15 +00:00
2018-04-18 22:10:15 +00:00
def __init__ ( self , controller , bandwidth_manager , session_manager , domain_manager , login_manager ) :
2015-10-21 21:53:10 +00:00
2018-04-18 22:10:15 +00:00
self . controller = controller
2015-10-21 21:53:10 +00:00
2018-04-18 22:10:15 +00:00
self . bandwidth_manager = bandwidth_manager
self . session_manager = session_manager
self . domain_manager = domain_manager
self . login_manager = login_manager
2015-10-21 21:53:10 +00:00
2018-04-18 22:10:15 +00:00
self . bandwidth_manager . engine = self
self . session_manager . engine = self
self . domain_manager . engine = self
self . login_manager . engine = self
2015-10-21 21:53:10 +00:00
2018-04-18 22:10:15 +00:00
self . _lock = threading . Lock ( )
2015-10-21 21:53:10 +00:00
2018-09-26 19:05:12 +00:00
self . RefreshOptions ( )
2018-04-18 22:10:15 +00:00
self . _new_work_to_do = threading . Event ( )
2015-10-21 21:53:10 +00:00
2018-11-21 22:22:36 +00:00
self . _domains_to_login = [ ]
2018-08-22 21:10:59 +00:00
self . _active_domains_counter = collections . Counter ( )
2018-04-18 22:10:15 +00:00
self . _jobs_awaiting_validity = [ ]
self . _current_validation_process = None
self . _jobs_awaiting_bandwidth = [ ]
self . _jobs_awaiting_login = [ ]
self . _current_login_process = None
self . _jobs_awaiting_slot = [ ]
self . _jobs_running = [ ]
2017-10-25 21:45:15 +00:00
2018-04-18 22:10:15 +00:00
self . _pause_all_new_network_traffic = self . controller . new_options . GetBoolean ( ' pause_all_new_network_traffic ' )
2015-10-21 21:53:10 +00:00
2018-04-18 22:10:15 +00:00
self . _is_running = False
self . _is_shutdown = False
self . _local_shutdown = False
2015-10-21 21:53:10 +00:00
2018-09-26 19:05:12 +00:00
self . controller . sub ( self , ' RefreshOptions ' , ' notify_new_options ' )
2018-04-18 22:10:15 +00:00
def AddJob ( self , job ) :
2015-10-21 21:53:10 +00:00
2018-07-04 20:48:28 +00:00
if HG . network_report_mode :
2018-08-22 21:10:59 +00:00
HydrusData . ShowText ( ' Network Job Added: ' + job . _method + ' ' + job . _url )
2018-07-04 20:48:28 +00:00
2018-04-18 22:10:15 +00:00
with self . _lock :
2015-10-21 21:53:10 +00:00
2018-04-18 22:10:15 +00:00
job . engine = self
2017-05-31 21:50:53 +00:00
2018-04-18 22:10:15 +00:00
self . _jobs_awaiting_validity . append ( job )
2017-05-31 21:50:53 +00:00
2018-04-18 22:10:15 +00:00
self . _new_work_to_do . set ( )
2017-06-07 22:05:15 +00:00
2018-11-21 22:22:36 +00:00
def ForceLogins ( self , domains_to_login ) :
with self . _lock :
self . _domains_to_login . extend ( domains_to_login )
self . _domains_to_login = HydrusData . DedupeList ( self . _domains_to_login )
2018-04-18 22:10:15 +00:00
def GetJobsSnapshot ( self ) :
2017-08-16 21:58:06 +00:00
2018-04-18 22:10:15 +00:00
with self . _lock :
2017-08-16 21:58:06 +00:00
2018-04-18 22:10:15 +00:00
jobs = [ ]
2017-08-16 21:58:06 +00:00
2018-04-18 22:10:15 +00:00
jobs . extend ( ( ( JOB_STATUS_AWAITING_VALIDITY , j ) for j in self . _jobs_awaiting_validity ) )
jobs . extend ( ( ( JOB_STATUS_AWAITING_BANDWIDTH , j ) for j in self . _jobs_awaiting_bandwidth ) )
jobs . extend ( ( ( JOB_STATUS_AWAITING_LOGIN , j ) for j in self . _jobs_awaiting_login ) )
jobs . extend ( ( ( JOB_STATUS_AWAITING_SLOT , j ) for j in self . _jobs_awaiting_slot ) )
jobs . extend ( ( ( JOB_STATUS_RUNNING , j ) for j in self . _jobs_running ) )
2017-06-07 22:05:15 +00:00
2018-04-18 22:10:15 +00:00
return jobs
2017-06-07 22:05:15 +00:00
2017-06-21 21:15:59 +00:00
2018-09-19 21:54:51 +00:00
def IsBusy ( self ) :
with self . _lock :
return len ( self . _jobs_awaiting_validity ) + len ( self . _jobs_awaiting_bandwidth ) + len ( self . _jobs_awaiting_login ) + len ( self . _jobs_awaiting_slot ) + len ( self . _jobs_running ) > 50
2018-04-18 22:10:15 +00:00
def IsRunning ( self ) :
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
with self . _lock :
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
return self . _is_running
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
def IsShutdown ( self ) :
2017-08-16 21:58:06 +00:00
2018-04-18 22:10:15 +00:00
with self . _lock :
2017-08-16 21:58:06 +00:00
2018-04-18 22:10:15 +00:00
return self . _is_shutdown
2017-08-16 21:58:06 +00:00
2017-07-05 21:09:28 +00:00
2018-04-18 22:10:15 +00:00
def MainLoop ( self ) :
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
def ProcessValidationJob ( job ) :
2017-06-28 20:23:21 +00:00
2018-04-18 22:10:15 +00:00
if job . IsDone ( ) :
return False
elif job . IsAsleep ( ) :
2017-06-28 20:23:21 +00:00
2018-04-18 22:10:15 +00:00
return True
2017-06-28 20:23:21 +00:00
2018-04-18 22:10:15 +00:00
elif not job . IsValid ( ) :
2017-06-28 20:23:21 +00:00
2018-04-18 22:10:15 +00:00
if job . CanValidateInPopup ( ) :
if self . _current_validation_process is None :
validation_process = job . GenerateValidationPopupProcess ( )
self . controller . CallToThread ( validation_process . Start )
self . _current_validation_process = validation_process
job . SetStatus ( u ' validation presented to user \u2026 ' )
else :
job . SetStatus ( u ' waiting in user validation queue \u2026 ' )
job . Sleep ( 5 )
return True
else :
error_text = u ' network context not currently valid! '
job . SetError ( HydrusExceptions . ValidationException ( error_text ) , error_text )
2017-06-28 20:23:21 +00:00
return False
2018-04-18 22:10:15 +00:00
else :
self . _jobs_awaiting_bandwidth . append ( job )
return False
2017-06-28 20:23:21 +00:00
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
def ProcessCurrentValidationJob ( ) :
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
if self . _current_validation_process is not None :
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
if self . _current_validation_process . IsDone ( ) :
2017-07-27 00:47:13 +00:00
2018-04-18 22:10:15 +00:00
self . _current_validation_process = None
2017-07-27 00:47:13 +00:00
2017-06-28 20:23:21 +00:00
2018-04-18 22:10:15 +00:00
def ProcessBandwidthJob ( job ) :
2017-06-28 20:23:21 +00:00
2018-04-18 22:10:15 +00:00
if job . IsDone ( ) :
2017-06-28 20:23:21 +00:00
2018-04-18 22:10:15 +00:00
return False
elif job . IsAsleep ( ) :
return True
elif not job . BandwidthOK ( ) :
return True
2017-06-28 20:23:21 +00:00
else :
2018-04-18 22:10:15 +00:00
self . _jobs_awaiting_login . append ( job )
return False
2017-06-28 20:23:21 +00:00
2017-07-19 21:21:41 +00:00
2018-11-21 22:22:36 +00:00
def ProcessForceLogins ( ) :
if len ( self . _domains_to_login ) > 0 and self . _current_login_process is None :
try :
login_domain = self . _domains_to_login . pop ( 0 )
login_process = self . login_manager . GenerateLoginProcessForDomain ( login_domain )
except Exception as e :
HydrusData . ShowException ( e )
return
self . controller . CallToThread ( login_process . Start )
self . _current_login_process = login_process
2018-04-18 22:10:15 +00:00
def ProcessLoginJob ( job ) :
2017-07-19 21:21:41 +00:00
2018-04-18 22:10:15 +00:00
if job . IsDone ( ) :
return False
elif job . IsAsleep ( ) :
return True
elif job . NeedsLogin ( ) :
2017-07-19 21:21:41 +00:00
2018-04-18 22:10:15 +00:00
try :
2017-07-19 21:21:41 +00:00
2018-04-18 22:10:15 +00:00
job . CheckCanLogin ( )
2017-07-19 21:21:41 +00:00
2018-04-18 22:10:15 +00:00
except Exception as e :
2018-10-31 21:41:14 +00:00
if job . WillingToWaitOnInvalidLogin ( ) :
job . SetStatus ( HydrusData . ToUnicode ( e ) )
job . Sleep ( 60 )
return True
else :
2018-11-28 22:31:04 +00:00
if job . IsHydrusJob ( ) :
message = ' This hydrus service ( ' + job . GetLoginNetworkContext ( ) . ToUnicode ( ) + ' ) recently failed to log in. Please hit its \' refresh account \' under \' review services \' and try again. '
else :
message = ' This job \' s network context ( ' + job . GetLoginNetworkContext ( ) . ToUnicode ( ) + ' ) seems to have an invalid login, and it is not willing to wait! Please review its login details and then try again. '
job . Cancel ( message )
2018-10-31 21:41:14 +00:00
return False
2017-07-19 21:21:41 +00:00
2018-04-18 22:10:15 +00:00
if self . _current_login_process is None :
2018-10-17 21:00:09 +00:00
try :
login_process = job . GenerateLoginProcess ( )
except Exception as e :
HydrusData . ShowException ( e )
job . SetStatus ( HydrusData . ToUnicode ( e ) )
job . Sleep ( 60 )
return True
2018-04-18 22:10:15 +00:00
self . controller . CallToThread ( login_process . Start )
self . _current_login_process = login_process
job . SetStatus ( u ' logging in \u2026 ' )
else :
job . SetStatus ( u ' waiting in login queue \u2026 ' )
job . Sleep ( 5 )
return True
else :
self . _jobs_awaiting_slot . append ( job )
return False
2017-07-19 21:21:41 +00:00
2018-04-18 22:10:15 +00:00
def ProcessCurrentLoginJob ( ) :
2017-07-19 21:21:41 +00:00
2018-04-18 22:10:15 +00:00
if self . _current_login_process is not None :
2017-07-19 21:21:41 +00:00
2018-04-18 22:10:15 +00:00
if self . _current_login_process . IsDone ( ) :
2017-07-19 21:21:41 +00:00
2018-04-18 22:10:15 +00:00
self . _current_login_process = None
2017-07-19 21:21:41 +00:00
2017-07-12 20:03:45 +00:00
2018-04-18 22:10:15 +00:00
def ProcessReadyJob ( job ) :
2017-08-16 21:58:06 +00:00
2018-04-18 22:10:15 +00:00
if job . IsDone ( ) :
2017-07-12 20:03:45 +00:00
2018-04-18 22:10:15 +00:00
return False
2017-07-12 20:03:45 +00:00
2018-09-12 21:36:26 +00:00
elif job . IsAsleep ( ) :
return True
2018-04-18 22:10:15 +00:00
elif len ( self . _jobs_running ) < self . MAX_JOBS :
if self . _pause_all_new_network_traffic :
2017-07-12 20:03:45 +00:00
2018-04-18 22:10:15 +00:00
job . SetStatus ( u ' all new network traffic is paused \u2026 ' )
2017-06-21 21:15:59 +00:00
2018-09-12 21:36:26 +00:00
job . Sleep ( 2 )
2018-04-18 22:10:15 +00:00
return True
2017-06-21 21:15:59 +00:00
2018-07-04 20:48:28 +00:00
elif self . controller . JustWokeFromSleep ( ) :
job . SetStatus ( u ' looks like computer just woke up, waiting a bit ' )
2018-09-12 21:36:26 +00:00
job . Sleep ( 5 )
2018-07-04 20:48:28 +00:00
return True
2018-08-22 21:10:59 +00:00
elif self . _active_domains_counter [ job . GetSecondLevelDomain ( ) ] > = self . MAX_JOBS_PER_DOMAIN :
job . SetStatus ( u ' waiting for a slot on this domain ' )
2018-09-12 21:36:26 +00:00
job . Sleep ( 2 )
2018-08-22 21:10:59 +00:00
return True
elif not job . TokensOK ( ) :
return True
2018-04-18 22:10:15 +00:00
else :
2017-06-21 21:15:59 +00:00
2018-08-22 21:10:59 +00:00
if HG . network_report_mode :
HydrusData . ShowText ( ' Network Job Starting: ' + job . _method + ' ' + job . _url )
self . _active_domains_counter [ job . GetSecondLevelDomain ( ) ] + = 1
2018-04-18 22:10:15 +00:00
self . controller . CallToThread ( job . Start )
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
self . _jobs_running . append ( job )
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
return False
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
else :
2017-06-21 21:15:59 +00:00
2018-09-12 21:36:26 +00:00
job . SetStatus ( u ' waiting for a slot \u2026 ' )
2017-07-27 00:47:13 +00:00
2018-04-18 22:10:15 +00:00
return True
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
def ProcessRunningJob ( job ) :
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
if job . IsDone ( ) :
2017-06-21 21:15:59 +00:00
2018-08-22 21:10:59 +00:00
if HG . network_report_mode :
HydrusData . ShowText ( ' Network Job Done: ' + job . _method + ' ' + job . _url )
second_level_domain = job . GetSecondLevelDomain ( )
self . _active_domains_counter [ second_level_domain ] - = 1
if self . _active_domains_counter [ second_level_domain ] == 0 :
del self . _active_domains_counter [ second_level_domain ]
2018-04-18 22:10:15 +00:00
return False
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
else :
2017-09-27 21:52:54 +00:00
2018-04-18 22:10:15 +00:00
return True
2017-09-27 21:52:54 +00:00
2017-09-06 20:18:20 +00:00
2018-04-18 22:10:15 +00:00
self . _is_running = True
while not ( self . _local_shutdown or self . controller . ModelIsShutdown ( ) ) :
2017-09-06 20:18:20 +00:00
2018-04-18 22:10:15 +00:00
with self . _lock :
2017-09-06 20:18:20 +00:00
2018-04-18 22:10:15 +00:00
self . _jobs_awaiting_validity = filter ( ProcessValidationJob , self . _jobs_awaiting_validity )
2017-09-06 20:18:20 +00:00
2018-04-18 22:10:15 +00:00
ProcessCurrentValidationJob ( )
2017-09-06 20:18:20 +00:00
2018-04-18 22:10:15 +00:00
self . _jobs_awaiting_bandwidth = filter ( ProcessBandwidthJob , self . _jobs_awaiting_bandwidth )
2017-09-06 20:18:20 +00:00
2018-11-21 22:22:36 +00:00
ProcessForceLogins ( )
2018-04-18 22:10:15 +00:00
self . _jobs_awaiting_login = filter ( ProcessLoginJob , self . _jobs_awaiting_login )
2017-10-25 21:45:15 +00:00
2018-04-18 22:10:15 +00:00
ProcessCurrentLoginJob ( )
2017-10-25 21:45:15 +00:00
2018-04-18 22:10:15 +00:00
self . _jobs_awaiting_slot = filter ( ProcessReadyJob , self . _jobs_awaiting_slot )
2017-10-25 21:45:15 +00:00
2018-04-18 22:10:15 +00:00
self . _jobs_running = filter ( ProcessRunningJob , self . _jobs_running )
2017-10-25 21:45:15 +00:00
2018-04-18 22:10:15 +00:00
# we want to catch the rollover of the second for bandwidth jobs
2017-06-28 20:23:21 +00:00
2018-04-18 22:10:15 +00:00
now_with_subsecond = time . time ( )
subsecond_part = now_with_subsecond % 1
2017-06-28 20:23:21 +00:00
2018-04-18 22:10:15 +00:00
time_until_next_second = 1.0 - subsecond_part
2017-06-28 20:23:21 +00:00
2018-04-18 22:10:15 +00:00
self . _new_work_to_do . wait ( time_until_next_second )
2017-07-27 00:47:13 +00:00
2018-04-18 22:10:15 +00:00
self . _new_work_to_do . clear ( )
2017-06-28 20:23:21 +00:00
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
self . _is_running = False
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
self . _is_shutdown = True
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
def PausePlayNewJobs ( self ) :
2018-03-22 00:03:33 +00:00
2018-04-18 22:10:15 +00:00
self . _pause_all_new_network_traffic = not self . _pause_all_new_network_traffic
2017-06-21 21:15:59 +00:00
2018-04-18 22:10:15 +00:00
self . controller . new_options . SetBoolean ( ' pause_all_new_network_traffic ' , self . _pause_all_new_network_traffic )
2017-06-21 21:15:59 +00:00
2018-09-26 19:05:12 +00:00
def RefreshOptions ( self ) :
with self . _lock :
self . MAX_JOBS = self . controller . new_options . GetInteger ( ' max_network_jobs ' )
self . MAX_JOBS_PER_DOMAIN = self . controller . new_options . GetInteger ( ' max_network_jobs_per_domain ' )
2018-04-18 22:10:15 +00:00
def Shutdown ( self ) :
2017-07-05 21:09:28 +00:00
2018-04-18 22:10:15 +00:00
self . _local_shutdown = True
2017-07-05 21:09:28 +00:00
2018-04-18 22:10:15 +00:00
self . _new_work_to_do . set ( )
2017-07-05 21:09:28 +00:00