RepoMirrors
/
hydrus
mirror of https://github.com/hydrusnetwork/hydrus.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Version 101

This commit is contained in:
Hydrus 2014-01-29 15:59:42 -06:00
parent 4ccda3c806
commit d0a9c8d662
18 changed files with 807 additions and 342 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
help/changelog.html
View File

@ -8,6 +8,25 @@
<div class="content">
<h3>changelog</h3>
<ul>
<li><h3>version 101</h3></li>
<ul>
<li>fixed another update problem for clients &lt;v95</li>
<li>uploading pending content should be less gui-blocking</li>
<li>fixed numpad navigation of page chooser to work for qwerty rather than dvorak as default</li>
<li>os x page picker now obeys arrow and numpad keys for navigation</li>
<li>os x command key should be treated as ctrl now, for better cross-platform compatibility</li>
<li>a couple more cmd/ctrl confusions cleared up</li>
<li>os x menubar disappearing after fullscreen problem fixed</li>
<li>added stateless http connection manager</li>
<li>simplified how http works in hydrus</li>
<li>http connections now used more efficiently</li>
<li>better cookie parsing</li>
<li>better http application shutdown handling</li>
<li>some misc http bugs fixed</li>
<li>new http error handling should limit occasional spam-errors</li>
<li>moved all hydrus http connections to new manager</li>
<li>moved many non-hydrus http connections to new manager</li>
</ul>
<li><h3>version 100</h3></li>
<ul>
<li>MVC import_controller system created</li>

86
include/ClientConstants.py
View File

@ -9,6 +9,7 @@ import HydrusExceptions
import HydrusFileHandling
import HydrusImageHandling
import HydrusMessageHandling
import HydrusNetworking
import HydrusTags
import itertools
import multipart
@ -2109,8 +2110,6 @@ class Service( HC.HydrusYAMLBase ):
def GetAccount( self ): return self._info[ 'account' ]
def GetConnection( self ): return ConnectionToService( self._service_identifier, self.GetCredentials() )
def GetCredentials( self ):
host = self._info[ 'host' ]
@ -2216,6 +2215,89 @@ class Service( HC.HydrusYAMLBase ):
def Request( self, method, command, request_args = {}, report_hooks = [], response_to_path = False, return_cookies = False ):
try:
credentials = self.GetCredentials()
request_headers = {}
if command == 'init': pass
elif command in ( 'session_key', 'access_key_verification' ): HydrusNetworking.AddHydrusCredentialsToHeaders( credentials, request_headers )
else: HydrusNetworking.AddHydrusSessionKeyToHeaders( self._service_identifier, request_headers )
path = '/' + command
if method == HC.GET:
query = HydrusNetworking.ConvertHydrusGETArgsToQuery( request_args )
body = ''
elif method == HC.POST:
query = ''
if command == 'file':
content_type = HC.APPLICATION_OCTET_STREAM
body = request_args[ 'file' ]
else:
content_type = HC.APPLICATION_YAML
body = yaml.safe_dump( request_args )
request_headers[ 'Content-Type' ] = HC.mime_string_lookup[ content_type ]
if query != '': path_and_query = path + '?' + query
else: path_and_query = path
( host, port ) = credentials.GetAddress()
url = 'http://' + host + ':' + HC.u( port ) + path_and_query
( response, size_of_response, response_headers, cookies ) = HC.http.Request( method, url, request_headers, body, report_hooks = report_hooks, response_to_path = response_to_path, return_everything = True )
HydrusNetworking.CheckHydrusVersion( self._service_identifier, response_headers )
if method == HC.GET: data_used = size_of_response
elif method == HC.POST: data_used = len( body )
HydrusNetworking.DoHydrusBandwidth( self._service_identifier, method, command, data_used )
if return_cookies: return ( response, cookies )
else: return response
except Exception as e:
if isinstance( e, HydrusExceptions.ForbiddenException ):
if HC.u( e ) == 'Session not found!':
session_manager = HC.app.GetManager( 'hydrus_sessions' )
session_manager.DeleteSessionKey( self._service_identifier )
HC.app.Write( 'service_updates', { self._service_identifier : [ HC.ServiceUpdate( HC.SERVICE_UPDATE_ERROR, HC.u( e ) ) ] } )
if isinstance( e, ( HydrusExceptions.PermissionException, HydrusExceptions.NetworkVersionException ) ):
HC.app.Write( 'service_updates', { self._service_identifier : [ HC.ServiceUpdate( HC.SERVICE_UPDATE_ACCOUNT, HC.GetUnknownAccount() ) ] } )
raise
def SetCredentials( self, credentials ):
( host, port ) = credentials.GetAddress()

2
include/ClientController.py
View File

@ -3,6 +3,7 @@ import hashlib
import httplib
import HydrusConstants as HC
import HydrusExceptions
import HydrusNetworking
import HydrusImageHandling
import HydrusSessions
import HydrusServer
@ -168,6 +169,7 @@ The database will be locked while the backup occurs, which may lock up your gui
def OnInit( self ):
HC.app = self
HC.http = HydrusNetworking.HTTPConnectionManager()
self._local_service = None
self._server = None

194
include/ClientDB.py
View File

@ -4986,13 +4986,16 @@ class DB( ServiceDB ):
c.execute( 'BEGIN IMMEDIATE' )
for ( service_id, info ) in c.execute( 'SELECT service_id, info FROM services;' ).fetchall():
if version > 95:
if 'account' in info:
for ( service_id, info ) in c.execute( 'SELECT service_id, info FROM services;' ).fetchall():
info[ 'account' ].MakeStale()
c.execute( 'UPDATE services SET info = ? WHERE service_id = ?;', ( info, service_id ) )
if 'account' in info:
info[ 'account' ].MakeStale()
c.execute( 'UPDATE services SET info = ? WHERE service_id = ?;', ( info, service_id ) )
@ -7017,8 +7020,6 @@ def DAEMONCheckImportFolders():
def DAEMONDownloadFiles():
service_identifiers_to_connections = {}
hashes = HC.app.ReadDaemon( 'downloads' )
num_downloads = len( hashes )
@ -7044,15 +7045,9 @@ def DAEMONDownloadFiles():
try:
if service_identifier not in service_identifiers_to_connections: service_identifiers_to_connections[ service_identifier ] = file_repository.GetConnection()
request_args = { 'hash' : hash.encode( 'hex' ) }
connection = service_identifiers_to_connections[ service_identifier ]
file = connection.Get( 'file', hash = hash.encode( 'hex' ) )
temp_path = HC.GetTempPath()
with open( temp_path, 'wb' ) as f: f.write( file )
temp_path = file_repository.Request( HC.GET, 'file', request_args = request_args, response_to_path = True )
num_downloads -= 1
@ -7102,8 +7097,6 @@ def DAEMONDownloadThumbnails():
try:
connection = file_repository.GetConnection()
num_per_round = 50
for i in range( 0, len( thumbnail_hashes_i_need ), num_per_round ):
@ -7114,7 +7107,9 @@ def DAEMONDownloadThumbnails():
for hash in thumbnail_hashes_i_need[ i : i + num_per_round ]:
thumbnail = connection.Get( 'thumbnail', hash = hash.encode( 'hex' ) )
request_args = { 'hash' : hash.encode( 'hex' ) }
thumbnail = file_repository.Request( HC.GET, 'thumbnail', request_args = request_args )
thumbnails.append( ( hash, thumbnail ) )
@ -7209,9 +7204,7 @@ def DAEMONSynchroniseAccounts():
try:
connection = service.GetConnection()
response = connection.Get( 'account' )
response = service.Request( HC.GET, 'account' )
account = response[ 'account' ]
@ -7437,100 +7430,95 @@ def DAEMONSynchroniseRepositoriesAndSubscriptions():
try: service = HC.app.ReadDaemon( 'service', service_identifier )
except: continue
if service.CanUpdate():
while service.CanUpdate():
connection = service.GetConnection()
while service.CanUpdate():
while HC.options[ 'pause_repo_sync' ]:
while HC.options[ 'pause_repo_sync' ]:
HC.pubsub.pub( 'service_status', 'Repository synchronisation paused' )
time.sleep( 5 )
if HC.shutdown: raise Exception( 'Application shutting down!' )
if HC.repos_or_subs_changed:
HC.pubsub.pub( 'service_status', 'Sync daemon restarting' )
HC.pubsub.pub( 'notify_restart_sync_daemon' )
return
HC.pubsub.pub( 'service_status', 'Repository synchronisation paused' )
time.sleep( 5 )
if HC.shutdown: raise Exception( 'Application shutting down!' )
first_begin = service.GetFirstBegin()
next_begin = service.GetNextBegin()
if first_begin == 0: update_index_string = 'initial update'
else: update_index_string = 'update ' + HC.u( ( ( next_begin - first_begin ) / HC.UPDATE_DURATION ) + 1 )
prefix_string = name + ' ' + update_index_string + ': '
HC.pubsub.pub( 'service_status', prefix_string + 'downloading and parsing' )
update = connection.Get( 'update', begin = next_begin )
if service_type == HC.TAG_REPOSITORY:
if HC.repos_or_subs_changed:
HC.pubsub.pub( 'service_status', 'Generating tags for ' + name )
HC.pubsub.pub( 'service_status', 'Sync daemon restarting' )
HC.app.WriteSynchronous( 'generate_tag_ids', update.GetTags() )
HC.pubsub.pub( 'notify_restart_sync_daemon' )
i = 0
num_content_updates = update.GetNumContentUpdates()
content_updates = []
current_weight = 0
for content_update in update.IterateContentUpdates():
return
content_updates.append( content_update )
current_weight += len( content_update.GetHashes() )
i += 1
if current_weight > 50:
HC.pubsub.pub( 'service_status', prefix_string + 'processing content ' + HC.ConvertIntToPrettyString( i ) + '/' + HC.ConvertIntToPrettyString( num_content_updates ) )
HC.app.WaitUntilGoodTimeToUseGUIThread()
time.sleep( 0.0001 )
HC.app.WriteSynchronous( 'content_updates', { service_identifier : content_updates } )
content_updates = []
current_weight = 0
if len( content_updates ) > 0: HC.app.WriteSynchronous( 'content_updates', { service_identifier : content_updates } )
HC.pubsub.pub( 'service_status', prefix_string + 'processing service info' )
service_updates = [ service_update for service_update in update.IterateServiceUpdates() ]
service_identifiers_to_service_updates = { service_identifier : service_updates }
HC.app.WriteSynchronous( 'service_updates', service_identifiers_to_service_updates )
HC.pubsub.pub( 'notify_new_pending' )
time.sleep( 0.10 )
try: service = HC.app.ReadDaemon( 'service', service_identifier )
except: break
HC.pubsub.pub( 'service_status', '' )
if HC.shutdown: raise Exception( 'Application shutting down!' )
first_begin = service.GetFirstBegin()
next_begin = service.GetNextBegin()
if first_begin == 0: update_index_string = 'initial update'
else: update_index_string = 'update ' + HC.u( ( ( next_begin - first_begin ) / HC.UPDATE_DURATION ) + 1 )
prefix_string = name + ' ' + update_index_string + ': '
HC.pubsub.pub( 'service_status', prefix_string + 'downloading and parsing' )
update = service.Request( HC.GET, 'update', { 'begin' : next_begin } )
if service_type == HC.TAG_REPOSITORY:
HC.pubsub.pub( 'service_status', 'Generating tags for ' + name )
HC.app.WriteSynchronous( 'generate_tag_ids', update.GetTags() )
i = 0
num_content_updates = update.GetNumContentUpdates()
content_updates = []
current_weight = 0
for content_update in update.IterateContentUpdates():
content_updates.append( content_update )
current_weight += len( content_update.GetHashes() )
i += 1
if current_weight > 50:
HC.pubsub.pub( 'service_status', prefix_string + 'processing content ' + HC.ConvertIntToPrettyString( i ) + '/' + HC.ConvertIntToPrettyString( num_content_updates ) )
HC.app.WaitUntilGoodTimeToUseGUIThread()
time.sleep( 0.0001 )
HC.app.WriteSynchronous( 'content_updates', { service_identifier : content_updates } )
content_updates = []
current_weight = 0
if len( content_updates ) > 0: HC.app.WriteSynchronous( 'content_updates', { service_identifier : content_updates } )
HC.pubsub.pub( 'service_status', prefix_string + 'processing service info' )
service_updates = [ service_update for service_update in update.IterateServiceUpdates() ]
service_identifiers_to_service_updates = { service_identifier : service_updates }
HC.app.WriteSynchronous( 'service_updates', service_identifiers_to_service_updates )
HC.pubsub.pub( 'notify_new_pending' )
time.sleep( 0.10 )
try: service = HC.app.ReadDaemon( 'service', service_identifier )
except: break
HC.pubsub.pub( 'service_status', '' )
except Exception as e:

150
include/ClientGUI.py
View File

@ -162,8 +162,6 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
HC.pubsub.pub( 'message_gauge_info', job_key, gauge_range, i, u'connecting to repository' )
connection = service.GetConnection()
good_hashes = []
error_messages = set()
@ -186,7 +184,7 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
with open( path, 'rb' ) as f: file = f.read()
connection.Post( 'file', file = file )
service.Request( HC.POST, 'file', { 'file' : file } )
( hash, inbox, size, mime, timestamp, width, height, duration, num_frames, num_words, tags_manager, file_service_identifiers_cdpp, local_ratings, remote_ratings ) = media_result.ToTuple()
@ -207,6 +205,10 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
time.sleep( 1 )
time.sleep( 0.1 )
HC.app.WaitUntilGoodTimeToUseGUIThread()
if not update.IsEmpty():
@ -214,7 +216,7 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
HC.pubsub.pub( 'message_gauge_info', job_key, gauge_range, i, u'uploading petitions' )
connection.Post( 'update', update = update )
service.Request( HC.POST, 'update', { 'update' : update } )
content_updates = update.GetContentUpdates( for_client = True )
@ -237,8 +239,6 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
HC.pubsub.pub( 'message_gauge_info', job_key, gauge_range, i, u'connecting to repository' )
connection = service.GetConnection()
for update in updates:
if job_key.IsCancelled(): return
@ -247,12 +247,16 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
HC.pubsub.pub( 'message_gauge_info', job_key, gauge_range, i, u'posting update' )
connection.Post( 'update', update = update )
service.Request( HC.POST, 'update', { 'update' : update } )
service_identifiers_to_content_updates = { service_identifier : update.GetContentUpdates( for_client = True ) }
HC.app.Write( 'content_updates', service_identifiers_to_content_updates )
time.sleep( 0.5 )
HC.app.WaitUntilGoodTimeToUseGUIThread()
except Exception as e: HC.ShowException( e )
@ -291,9 +295,7 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
service = HC.app.Read( 'service', service_identifier )
connection = service.GetConnection()
response = connection.Get( 'account_info', subject_access_key = subject_access_key.encode( 'hex' ) )
response = service.Request( HC.GET, 'account_info', { 'subject_access_key' : subject_access_key.encode( 'hex' ) } )
account_info = response[ 'account_info' ]
@ -439,9 +441,7 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
#
connection = service.GetConnection()
response = connection.Get( 'init' )
response = service.Request( HC.GET, 'init' )
access_key = response[ 'access_key' ]
@ -470,9 +470,9 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
edit_log.append( ( HC.ADD, ( tag_server_service_identifier, tag_options ) ) )
edit_log.append( ( HC.ADD, ( file_server_service_identifier, file_options ) ) )
result = connection.Post( 'services', edit_log = edit_log )
response = service.Request( HC.POST, 'services', { 'edit_log' : edit_log } )
service_identifiers_to_access_keys = dict( result[ 'service_identifiers_to_access_keys' ] )
service_identifiers_to_access_keys = dict( response[ 'service_identifiers_to_access_keys' ] )
HC.app.Write( 'update_server_services', admin_service_identifier, edit_log, service_identifiers_to_access_keys )
@ -491,9 +491,7 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
service = HC.app.Read( 'service', service_identifier )
connection = service.GetConnection()
with wx.BusyCursor(): connection.Post( 'backup' )
with wx.BusyCursor(): service.Request( HC.POST, 'backup' )
wx.MessageBox( 'Done!' )
@ -573,9 +571,7 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
service = HC.app.Read( 'service', service_identifier )
connection = service.GetConnection()
with wx.BusyCursor(): response = connection.Get( 'ip', hash = hash.encode( 'hex' ) )
with wx.BusyCursor(): response = service.Request( HC.GET, 'ip', { 'hash' : hash.encode( 'hex' ) } )
ip = response[ 'ip' ]
timestamp = response[ 'timestamp' ]
@ -880,9 +876,7 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
service = HC.app.Read( 'service', service_identifier )
connection = service.GetConnection()
with wx.BusyCursor(): connection.Post( 'news', news = news )
with wx.BusyCursor(): service.Request( HC.POST, 'news', { 'news' : news } )
@ -1115,9 +1109,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
service = HC.app.Read( 'service', service_identifier )
connection = service.GetConnection()
response = connection.Get( 'stats' )
response = service.Request( HC.GET, 'stats' )
stats = response[ 'stats' ]
@ -1910,15 +1902,15 @@ class FramePageChooser( ClientGUICommon.Frame ):
gridbox = wx.GridSizer( 0, 3 )
gridbox.AddF( self._button_1, FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( self._button_2, FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( self._button_3, FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( self._button_4, FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( self._button_5, FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( self._button_6, FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( self._button_7, FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( self._button_8, FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( self._button_9, FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( self._button_4, FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( self._button_5, FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( self._button_6, FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( self._button_1, FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( self._button_2, FLAGS_EXPAND_BOTH_WAYS )
gridbox.AddF( self._button_3, FLAGS_EXPAND_BOTH_WAYS )
self.SetSizer( gridbox )
@ -1929,23 +1921,6 @@ class FramePageChooser( ClientGUICommon.Frame ):
self.Center()
self._keycodes_to_ids = {}
self._keycodes_to_ids[ wx.WXK_NUMPAD1 ] = 1
self._keycodes_to_ids[ wx.WXK_NUMPAD2 ] = 2
self._keycodes_to_ids[ wx.WXK_NUMPAD3 ] = 3
self._keycodes_to_ids[ wx.WXK_NUMPAD4 ] = 4
self._keycodes_to_ids[ wx.WXK_NUMPAD5 ] = 5
self._keycodes_to_ids[ wx.WXK_NUMPAD6 ] = 6
self._keycodes_to_ids[ wx.WXK_NUMPAD7 ] = 7
self._keycodes_to_ids[ wx.WXK_NUMPAD8 ] = 8
self._keycodes_to_ids[ wx.WXK_NUMPAD9 ] = 9
self._keycodes_to_ids[ wx.WXK_UP ] = 2
self._keycodes_to_ids[ wx.WXK_DOWN ] = 8
self._keycodes_to_ids[ wx.WXK_LEFT ] = 4
self._keycodes_to_ids[ wx.WXK_RIGHT ] = 6
InitialiseControls()
PopulateControls()
@ -1960,9 +1935,33 @@ class FramePageChooser( ClientGUICommon.Frame ):
self.Bind( wx.EVT_BUTTON, self.EventButton )
self.Bind( wx.EVT_CHAR_HOOK, self.EventCharHook )
self.Bind( wx.EVT_MENU, self.EventMenu )
self._button_hidden.SetFocus()
#
entries = []
entries.append( ( wx.ACCEL_NORMAL, wx.WXK_UP, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'button', 8 ) ) )
entries.append( ( wx.ACCEL_NORMAL, wx.WXK_LEFT, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'button', 4 ) ) )
entries.append( ( wx.ACCEL_NORMAL, wx.WXK_RIGHT, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'button', 6 ) ) )
entries.append( ( wx.ACCEL_NORMAL, wx.WXK_DOWN, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'button', 2 ) ) )
entries.append( ( wx.ACCEL_NORMAL, wx.WXK_NUMPAD1, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'button', 1 ) ) )
entries.append( ( wx.ACCEL_NORMAL, wx.WXK_NUMPAD2, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'button', 2 ) ) )
entries.append( ( wx.ACCEL_NORMAL, wx.WXK_NUMPAD3, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'button', 3 ) ) )
entries.append( ( wx.ACCEL_NORMAL, wx.WXK_NUMPAD4, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'button', 4 ) ) )
entries.append( ( wx.ACCEL_NORMAL, wx.WXK_NUMPAD5, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'button', 5 ) ) )
entries.append( ( wx.ACCEL_NORMAL, wx.WXK_NUMPAD6, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'button', 6 ) ) )
entries.append( ( wx.ACCEL_NORMAL, wx.WXK_NUMPAD7, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'button', 7 ) ) )
entries.append( ( wx.ACCEL_NORMAL, wx.WXK_NUMPAD8, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'button', 8 ) ) )
entries.append( ( wx.ACCEL_NORMAL, wx.WXK_NUMPAD9, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'button', 9 ) ) )
self.SetAcceleratorTable( wx.AcceleratorTable( entries ) )
#
self.Show( True )
@ -2034,20 +2033,22 @@ class FramePageChooser( ClientGUICommon.Frame ):
self._button_7.Hide()
self._button_9.Hide()
usable_buttons = [ self._button_2, self._button_4, self._button_6, self._button_8 ]
potential_buttons = [ self._button_8, self._button_4, self._button_6, self._button_2 ]
elif len( entries ) <= 9: usable_buttons = [ self._button_1, self._button_2, self._button_3, self._button_4, self._button_5, self._button_6, self._button_7, self._button_8, self._button_9 ]
elif len( entries ) <= 9: potential_buttons = [ self._button_7, self._button_8, self._button_9, self._button_4, self._button_5, self._button_6, self._button_1, self._button_2, self._button_3 ]
else:
pass # sort out a multi-page solution? maybe only if this becomes a big thing; the person can always select from the menus, yeah?
usable_buttons = [ self._button_1, self._button_2, self._button_3, self._button_4, self._button_5, self._button_6, self._button_7, self._button_8, self._button_9 ]
potential_buttons = [ self._button_7, self._button_8, self._button_9, self._button_4, self._button_5, self._button_6, self._button_1, self._button_2, self._button_3 ]
entries = entries[:9]
for entry in entries: self._AddEntry( usable_buttons.pop( 0 ), entry )
for entry in entries: self._AddEntry( potential_buttons.pop( 0 ), entry )
for button in usable_buttons: button.Hide()
unused_buttons = potential_buttons
for button in unused_buttons: button.Hide()
def EventButton( self, event ):
@ -2093,18 +2094,29 @@ class FramePageChooser( ClientGUICommon.Frame ):
def EventCharHook( self, event ):
if event.KeyCode in self._keycodes_to_ids.keys():
id = self._keycodes_to_ids[ event.KeyCode ]
new_event = wx.CommandEvent( wx.wxEVT_COMMAND_BUTTON_CLICKED, winid = id )
self.ProcessEvent( new_event )
elif event.KeyCode == wx.WXK_ESCAPE: self.Close()
if event.KeyCode == wx.WXK_ESCAPE: self.Close()
else: event.Skip()
def EventMenu( self, event ):
event_id = event.GetId()
action = CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetAction( event_id )
if action is not None:
( command, data ) = action
if command == 'button':
new_event = wx.CommandEvent( wx.wxEVT_COMMAND_BUTTON_CLICKED, winid = data )
self.ProcessEvent( new_event )
class FrameReviewServices( ClientGUICommon.Frame ):
def __init__( self ):
@ -2645,11 +2657,7 @@ class FrameReviewServices( ClientGUICommon.Frame ):
def EventServerInitialise( self, event ):
service = HC.app.Read( 'service', self._service_identifier )
connection = service.GetConnection()
response = connection.Get( 'init' )
response = self._service.Request( HC.GET, 'init' )
access_key = response[ 'access_key' ]
@ -2666,9 +2674,7 @@ class FrameReviewServices( ClientGUICommon.Frame ):
self._refresh.Disable()
connection = self._service.GetConnection()
response = connection.Get( 'account' )
response = self._service.Request( HC.GET, 'account' )
account = response[ 'account' ]

11
include/ClientGUICanvas.py
View File

@ -585,7 +585,7 @@ class Canvas():
self._media_container.Hide()
self._media_container.Close()
wx.CallAfter( self._media_container.Destroy )
self._media_container = None
@ -1030,7 +1030,9 @@ class CanvasFullscreenMediaList( ClientGUIMixins.ListeningMediaList, Canvas, Cli
HC.pubsub.pub( 'set_focus', self._page_key, self._current_media )
self.Destroy()
if HC.PLATFORM_OSX and self.IsFullScreen(): self.ShowFullScreen( False )
wx.CallAfter( self.Destroy )
def EventDrag( self, event ):
@ -1154,7 +1156,6 @@ class CanvasFullscreenMediaListBrowser( CanvasFullscreenMediaList ):
self.Bind( wx.EVT_RIGHT_DOWN, self.EventShowMenu )
self.Bind( wx.EVT_MENU, self.EventMenu )
self.Bind( wx.EVT_CHAR_HOOK, self.EventCharHook )
if first_hash is None: self.SetMedia( self._GetFirst() )
@ -3012,7 +3013,9 @@ class RatingsFilterFrameNumerical( ClientGUICommon.FrameThatResizes ):
HC.pubsub.pub( 'set_focus', self._page_key, self._current_media_to_rate )
self.Destroy()
if HC.PLATFORM_OSX and self.IsFullScreen(): self.ShowFullScreen( False )
wx.CallAfter( self.Destroy )
def EventFullscreenSwitch( self, event ): self._FullscreenSwitch()

2
include/ClientGUICommon.py
View File

@ -2972,7 +2972,7 @@ class Shortcut( wx.TextCtrl ):
modifier = wx.ACCEL_NORMAL
if event.AltDown(): modifier = wx.ACCEL_ALT
elif event.ControlDown(): modifier = wx.ACCEL_CTRL
elif event.CmdDown(): modifier = wx.ACCEL_CTRL
elif event.ShiftDown(): modifier = wx.ACCEL_SHIFT
( self._modifier, self._key ) = HC.GetShortcutFromEvent( event )

27
include/ClientGUIDialogs.py
View File

@ -540,9 +540,7 @@ class DialogGenerateNewAccounts( Dialog ):
service = HC.app.Read( 'service', service_identifier )
connection = service.GetConnection()
response = connection.Get( 'account_types' )
response = service.Request( HC.GET, 'account_types' )
account_types = response[ 'account_types' ]
@ -605,10 +603,11 @@ class DialogGenerateNewAccounts( Dialog ):
try:
connection = service.GetConnection()
request_args = { 'num' : num, 'title' : title }
if lifetime is None: response = connection.Get( 'registration_keys', num = num, title = title )
else: response = connection.Get( 'registration_keys', num = num, title = title, lifetime = lifetime )
if lifetime is not None: request_args[ 'lifetime' ] = lifetime
response = service.Request( HC.GET, 'registration_keys', request_args )
registration_keys = response[ 'registration_keys' ]
@ -3254,13 +3253,11 @@ class DialogModifyAccounts( Dialog ):
def PopulateControls():
connection = self._service.GetConnection()
if len( self._subject_identifiers ) == 1:
( subject_identifier, ) = self._subject_identifiers
response = connection.Get( 'account_info', subject_identifier = subject_identifier )
response = self._service.Request( HC.GET, 'account_info', { 'subject_identifier' : subject_identifier } )
subject_string = HC.u( response[ 'account_info' ] )
@ -3270,7 +3267,7 @@ class DialogModifyAccounts( Dialog ):
#
response = connection.Get( 'account_types' )
response = self._service.Request( HC.GET, 'account_types' )
account_types = response[ 'account_types' ]
@ -3359,18 +3356,18 @@ class DialogModifyAccounts( Dialog ):
def _DoModification( self, action, **kwargs ):
connection = self._service.GetConnection()
request_args = kwargs
kwargs[ 'subject_identifiers' ] = self._subject_identifiers
kwargs[ 'action' ] = action
request_args[ 'subject_identifiers' ] = self._subject_identifiers
request_args[ 'action' ] = action
connection.Post( 'account', **kwargs )
self._service.Request( HC.POST, 'account', request_args )
if len( self._subject_identifiers ) == 1:
( subject_identifier, ) = self._subject_identifiers
response = connection.Get( 'account_info', subject_identifier = subject_identifier )
response = self._service.Request( HC.GET, 'account_info', { 'subject_identifier' : subject_identifier } )
account_info = response[ 'account_info' ]

54
include/ClientGUIDialogsManage.py
View File

@ -216,9 +216,7 @@ class DialogManageAccountTypes( ClientGUIDialogs.Dialog ):
service = HC.app.Read( 'service', service_identifier )
connection = service.GetConnection()
response = connection.Get( 'account_types' )
response = service.Request( HC.GET, 'account_types' )
account_types = response[ 'account_types' ]
@ -394,9 +392,7 @@ class DialogManageAccountTypes( ClientGUIDialogs.Dialog ):
service = HC.app.Read( 'service', self._service_identifier )
connection = service.GetConnection()
connection.Post( 'account_types', edit_log = self._edit_log )
service.Request( HC.POST, 'account_types', { 'edit_log' : self._edit_log } )
self.EndModal( wx.ID_OK )
@ -3600,11 +3596,7 @@ class DialogManagePixivAccount( ClientGUIDialogs.Dialog ):
headers = {}
headers[ 'Content-Type' ] = 'application/x-www-form-urlencoded'
connection = HC.get_connection( url = 'http://www.pixiv.net/', accept_cookies = True )
response = connection.request( 'POST', '/login.php', headers = headers, body = body, follow_redirects = False )
cookies = connection.GetCookies()
( response_gumpf, cookies ) = HC.http.Request( HC.POST, 'http://www.pixiv.net/login.php', request_headers = headers, body = body, return_cookies = True )
# _ only given to logged in php sessions
if 'PHPSESSID' in cookies and '_' in cookies[ 'PHPSESSID' ]: self._status.SetLabel( 'OK!' )
@ -4034,9 +4026,7 @@ class DialogManageServer( ClientGUIDialogs.Dialog ):
self._service_types.SetSelection( 0 )
connection = self._service.GetConnection()
response = connection.Get( 'services' )
response = self._service.Request( HC.GET, 'services' )
services_info = response[ 'services_info' ]
@ -4170,11 +4160,9 @@ class DialogManageServer( ClientGUIDialogs.Dialog ):
if len( self._edit_log ) > 0:
connection = self._service.GetConnection()
response = self._service.Request( HC.POST, 'services', { 'edit_log' : self._edit_log } )
result = connection.Post( 'services', edit_log = self._edit_log )
service_identifiers_to_access_keys = dict( result[ 'service_identifiers_to_access_keys' ] )
service_identifiers_to_access_keys = dict( response[ 'service_identifiers_to_access_keys' ] )
HC.app.Write( 'update_server_services', self._service_identifier, self._edit_log, service_identifiers_to_access_keys )
@ -4859,43 +4847,35 @@ class DialogManageServices( ClientGUIDialogs.Dialog ):
def EventCheckService( self, event ):
( service_identifier, info ) = self.GetInfo()
service_type = service_identifier.GetType()
host = info[ 'host' ]
port = info[ 'port' ]
service = CC.Service( service_identifier, info )
if 'access_key' in info: access_key = info[ 'access_key' ]
else: access_key = None
credentials = CC.Credentials( host, port, access_key )
try: connection = CC.ConnectionToService( service_identifier, credentials )
except:
wx.MessageBox( 'Could not connect to the service!' )
return
try: root = connection.Get( '' )
try: root = service.Request( HC.GET, '' )
except HydrusExceptions.WrongServiceTypeException:
wx.MessageBox( 'Connection was made, but the service was not a ' + HC.service_string_lookup[ service_type ] + '.' )
return
except:
wx.MessageBox( 'Could not connect!' )
return
if service_type in HC.RESTRICTED_SERVICES:
if access_key is None:
if 'access_key' not in info or info[ 'access_key' ] is None:
wx.MessageBox( 'No access key!' )
return
response = connection.Get( 'access_key_verification' )
response = service.Request( HC.GET, 'access_key_verification' )
if not response[ 'verified' ]:

67
include/ClientGUIManagement.py
View File

@ -247,36 +247,27 @@ class CaptchaControl( wx.Panel ):
def EventRefreshCaptcha( self, event ):
try:
connection = HC.get_connection( scheme = 'http', host = 'www.google.com', port = 80 )
javascript_string = connection.request( 'GET', '/recaptcha/api/challenge?k=' + self._captcha_key )
( trash, rest ) = javascript_string.split( 'challenge : \'', 1 )
( self._captcha_challenge, trash ) = rest.split( '\'', 1 )
jpeg = connection.request( 'GET', '/recaptcha/api/image?c=' + self._captcha_challenge )
temp_path = HC.GetTempPath()
with open( temp_path, 'wb' ) as f: f.write( jpeg )
self._bitmap = HydrusImageHandling.GenerateHydrusBitmap( temp_path )
self._captcha_runs_out = HC.GetNow() + 5 * 60 - 15
self._DrawMain()
self._DrawEntry( '' )
self._DrawReady( False )
self._timer.Start( 1000, wx.TIMER_CONTINUOUS )
except:
wx.MessageBox( traceback.format_exc() )
javascript_string = HC.http.Request( HC.GET, 'http://www.google.com/recaptcha/api/challenge?k=' + self._captcha_key )
( trash, rest ) = javascript_string.split( 'challenge : \'', 1 )
( self._captcha_challenge, trash ) = rest.split( '\'', 1 )
jpeg = HC.http.Request( HC.GET, 'http://www.google.com/recaptcha/api/image?c=' + self._captcha_challenge )
temp_path = HC.GetTempPath()
with open( temp_path, 'wb' ) as f: f.write( jpeg )
self._bitmap = HydrusImageHandling.GenerateHydrusBitmap( temp_path )
self._captcha_runs_out = HC.GetNow() + 5 * 60 - 15
self._DrawMain()
self._DrawEntry( '' )
self._DrawReady( False )
self._timer.Start( 1000, wx.TIMER_CONTINUOUS )
def EventTimer( self, event ):
@ -1927,9 +1918,7 @@ class ManagementPanelPetitions( ManagementPanel ):
update = self._current_petition.GetApproval()
connection = self._service.GetConnection()
connection.Post( 'update', update = update )
self._service.Request( HC.POST, 'update', { 'update' : update } )
HC.app.Write( 'content_updates', { self._petition_service_identifier : update.GetContentUpdates( for_client = True ) } )
@ -1944,9 +1933,7 @@ class ManagementPanelPetitions( ManagementPanel ):
update = self._current_petition.GetDenial()
connection = self._service.GetConnection()
connection.Post( 'update', update = update )
self._service.Request( HC.POST, 'update', { 'update' : update } )
self._current_petition = None
@ -1959,9 +1946,7 @@ class ManagementPanelPetitions( ManagementPanel ):
try:
connection = self._service.GetConnection()
response = connection.Get( 'petition' )
response = self._service.Request( HC.GET, 'petition' )
self._current_petition = response[ 'petition' ]
@ -1988,9 +1973,7 @@ class ManagementPanelPetitions( ManagementPanel ):
try:
connection = self._service.GetConnection()
response = connection.Get( 'num_petitions' )
response = self._service.Request( HC.GET, 'num_petitions' )
self._num_petitions = response[ 'num_petitions' ]

2
include/ClientGUIMedia.py
View File

@ -2026,7 +2026,7 @@ class MediaPanelThumbnails( MediaPanel ):
( wx.ACCEL_SHIFT, wx.WXK_NUMPAD_LEFT, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'key_shift_left' ) ),
( wx.ACCEL_SHIFT, wx.WXK_RIGHT, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'key_shift_right' ) ),
( wx.ACCEL_SHIFT, wx.WXK_NUMPAD_RIGHT, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'key_shift_right' ) ),
( wx.ACCEL_CMD, ord( 'A' ), CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'select', 'all' ) ),
( wx.ACCEL_CTRL, ord( 'A' ), CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'select', 'all' ) ),
( wx.ACCEL_CTRL, ord( 'c' ), CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'copy_files' ) ),
( wx.ACCEL_CTRL, wx.WXK_SPACE, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'ctrl-space' ) )
]

6
include/ClientGUIMessages.py
View File

@ -781,9 +781,9 @@ class DraftBodyPanel( wx.Panel ):
self.SetSizer( vbox )
self.SetAcceleratorTable( wx.AcceleratorTable( [
( wx.ACCEL_CMD, ord( 'b' ), self.ID_BOLD ),
( wx.ACCEL_CMD, ord( 'i' ), self.ID_ITALIC ),
( wx.ACCEL_CMD, ord( 'u' ), self.ID_UNDERLINE )
( wx.ACCEL_CTRL, ord( 'b' ), self.ID_BOLD ),
( wx.ACCEL_CTRL, ord( 'i' ), self.ID_ITALIC ),
( wx.ACCEL_CTRL, ord( 'u' ), self.ID_UNDERLINE )
] ) )
self.Bind( wx.EVT_TOOL, self.EventToolBar )

8
include/HydrusConstants.py
View File

@ -4,6 +4,7 @@ import collections
import cStringIO
import httplib
import HydrusExceptions
import HydrusNetworking
import HydrusPubSub
import itertools
import locale
@ -45,7 +46,7 @@ TEMP_DIR = BASE_DIR + os.path.sep + 'temp'
# Misc
NETWORK_VERSION = 13
SOFTWARE_VERSION = 100
SOFTWARE_VERSION = 101
UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )
@ -57,10 +58,13 @@ lifetimes = [ ( 'one month', 31 * 86400 ), ( 'three months', 3 * 31 * 86400 ), (
app = None
shutdown = False
is_first_start = False
is_db_updated = False
repos_or_subs_changed = False
http = None
busy_doing_pubsub = False
# Enums
@ -1050,7 +1054,7 @@ def GetShortcutFromEvent( event ):
modifier = wx.ACCEL_NORMAL
if event.AltDown(): modifier = wx.ACCEL_ALT
elif event.ControlDown(): modifier = wx.ACCEL_CTRL
elif event.CmdDown(): modifier = wx.ACCEL_CTRL
elif event.ShiftDown(): modifier = wx.ACCEL_SHIFT
key = event.KeyCode

43
include/HydrusDownloading.py
View File

@ -1280,23 +1280,13 @@ class ImportArgsGeneratorThread( ImportArgsGenerator ):
url = 'http://images.4chan.org/' + board + '/src/' + image_name + ext
parse_result = urlparse.urlparse( url )
( scheme, host, port ) = ( parse_result.scheme, parse_result.hostname, parse_result.port )
connection = HC.get_connection( scheme = scheme, host = host, port = port )
def hook( range, value ):
self._job_key.SetVariable( 'range', range )
self._job_key.SetVariable( 'value', value )
connection.AddReportHook( hook )
temp_path = connection.geturl( url, response_to_path = True )
connection.ClearReportHooks()
temp_path = HC.http.Request( HC.GET, url, report_hooks = [ hook ], response_to_path = True )
tags = [ 'filename:' + filename + ext ]
@ -1332,23 +1322,13 @@ class ImportArgsGeneratorURLs( ImportArgsGenerator ):
self._job_key.SetVariable( 'status', 'downloading' )
parse_result = urlparse.urlparse( url )
( scheme, host, port ) = ( parse_result.scheme, parse_result.hostname, parse_result.port )
connection = HC.get_connection( scheme = scheme, host = host, port = port )
def hook( range, value ):
self._job_key.SetVariable( 'range', range )
self._job_key.SetVariable( 'value', value )
connection.AddReportHook( hook )
temp_path = connection.geturl( url, response_to_path = True )
connection.ClearReportHooks()
temp_path = HC.http.Request( HC.GET, url, report_hooks = [ hook ], response_to_path = True )
service_identifiers_to_tags = {}
@ -1481,22 +1461,9 @@ class ImportQueueGeneratorURLs( ImportQueueGenerator ):
url = self._item
self._job_key.SetVariable( 'status', 'parsing url' )
try:
parse_result = urlparse.urlparse( url )
( scheme, host, port ) = ( parse_result.scheme, parse_result.hostname, parse_result.port )
except: raise Exception( 'Could not parse that URL' )
self._job_key.SetVariable( 'status', 'Connecting to address' )
try: connection = HC.get_connection( scheme = scheme, host = host, port = port )
except: raise Exception( 'Could not connect to server' )
try: html = connection.geturl( url )
try: html = HC.http.Request( HC.GET, url )
except: raise Exception( 'Could not download that url' )
self._job_key.SetVariable( 'status', 'parsing html' )
@ -1555,9 +1522,7 @@ class ImportQueueGeneratorThread( ImportQueueGenerator ):
try:
connection = HC.get_connection( url = url )
raw_json = connection.geturl( url )
raw_json = HC.http.Request( HC.GET, url )
json_dict = json.loads( raw_json )

452
include/HydrusNetworking.py Normal file
View File

@ -0,0 +1,452 @@
import HydrusConstants as HC
import HydrusExceptions
import httplib
import threading
import time
import urlparse
import yaml
def AddHydrusCredentialsToHeaders( credentials, request_headers ):
if credentials.HasAccessKey():
access_key = credentials.GetAccessKey()
if access_key != '': request_headers[ 'Hydrus-Key' ] = access_key.encode( 'hex' )
else: raise Exception( 'No access key!' )
def AddHydrusSessionKeyToHeaders( service_identifier, request_headers ):
session_manager = HC.app.GetManager( 'hydrus_sessions' )
session_key = session_manager.GetSessionKey( service_identifier )
request_headers[ 'Cookie' ] = 'session_key=' + session_key.encode( 'hex' )
def AddCookiesToHeaders( cookies, request_headers ):
request_headers[ 'Cookie' ] = '; '.join( [ k + '=' + v for ( k, v ) in cookies.items() ] )
def CheckHydrusVersion( service_identifier, response_headers ):
service_type = service_identifier.GetType()
service_string = HC.service_string_lookup[ service_type ]
if 'server' not in response_headers or service_string not in response_headers[ 'server' ]:
HC.app.Write( 'service_updates', { service_identifier : [ HC.ServiceUpdate( HC.SERVICE_UPDATE_ACCOUNT, HC.GetUnknownAccount() ) ] })
raise HydrusExceptions.WrongServiceTypeException( 'Target was not a ' + service_string + '!' )
server_header = response_headers[ 'server' ]
( service_string_gumpf, network_version ) = server_header.split( '/' )
network_version = int( network_version )
if network_version != HC.NETWORK_VERSION:
if network_version > HC.NETWORK_VERSION: message = 'Your client is out of date; please download the latest release.'
else: message = 'The server is out of date; please ask its admin to update to the latest release.'
raise HydrusExceptions.NetworkVersionException( 'Network version mismatch! The server\'s network version was ' + u( network_version ) + ', whereas your client\'s is ' + u( HC.NETWORK_VERSION ) + '! ' + message )
def ConvertHydrusGETArgsToQuery( request_args ):
if 'subject_identifier' in request_args:
subject_identifier = request_args[ 'subject_identifier' ]
del request_args[ 'subject_identifier' ]
data = subject_identifier.GetData()
if subject_identifier.HasAccessKey(): request_args[ 'subject_access_key' ] = data.encode( 'hex' )
elif subject_identifier.HasAccountId(): request_args[ 'subject_account_id' ] = data
elif subject_identifier.HasHash(): request_args[ 'subject_hash' ] = data.encode( 'hex' )
if subject_identifier.HasMapping():
( subject_hash, subject_tag ) = data
request_args[ 'subject_hash' ] = subject_hash.encode( 'hex' )
request_args[ 'subject_tag' ] = subject_tag.encode( 'hex' )
if 'title' in request_args:
request_args[ 'title' ] = request_args[ 'title' ].encode( 'hex' )
query = '&'.join( [ key + '=' + HC.u( value ) for ( key, value ) in request_args.items() ] )
return query
def DoHydrusBandwidth( service_identifier, method, command, size ):
service_type = service_identifier.GetType()
if ( service_type, method, command ) in HC.BANDWIDTH_CONSUMING_REQUESTS: HC.pubsub.pub( 'service_updates_delayed', { service_identifier : [ HC.ServiceUpdate( HC.SERVICE_UPDATE_REQUEST_MADE, size ) ] } )
def ParseURL( url ):
try:
parse_result = urlparse.urlparse( url )
( scheme, host, port ) = ( parse_result.scheme, parse_result.hostname, parse_result.port )
parse_result = urlparse.urlparse( url )
location = ( parse_result.scheme, parse_result.hostname, parse_result.port )
path = parse_result.path
query = parse_result.query
except: raise Exception( 'Could not parse that URL' )
return ( location, path, query )
class HTTPConnectionManager():
def __init__( self ):
self._connections = {}
self._lock = threading.Lock()
threading.Thread( target = self.MaintainConnections, name = 'Maintain Connections' ).start()
def _DoRequest( self, location, method, path_and_query, request_headers, body, follow_redirects = True, report_hooks = [], response_to_path = False, num_redirects_permitted = 4 ):
connection = self._GetConnection( location )
try:
with connection.lock:
( parsed_response, redirect_info, size_of_response, response_headers, cookies ) = connection.Request( method, path_and_query, request_headers, body, report_hooks = report_hooks, response_to_path = response_to_path )
if redirect_info is None or not follow_redirects: return ( parsed_response, size_of_response, response_headers, cookies )
else:
if num_redirects_permitted == 0: raise Exception( 'Too many redirects!' )
( new_method, new_url ) = redirect_info
( new_location, new_path, new_query ) = ParseURL( new_url )
if new_query != '': new_path_and_query = new_path + '?' + new_query
else: new_path_and_query = new_path
return self._DoRequest( new_location, new_method, new_path_and_query, request_headers, body, report_hooks = report_hooks, response_to_path = response_to_path, num_redirects_permitted = num_redirects_permitted - 1 )
except:
time.sleep( 2 )
raise
def _GetConnection( self, location ):
with self._lock:
if location not in self._connections:
connection = HTTPConnection( location )
self._connections[ location ] = connection
return self._connections[ location ]
def Request( self, method, url, request_headers = {}, body = '', return_everything = False, return_cookies = False, report_hooks = [], response_to_path = False ):
( location, path, query ) = ParseURL( url )
if query != '': path_and_query = path + '?' + query
else: path_and_query = path
follow_redirects = not return_cookies
( response, size_of_response, response_headers, cookies ) = self._DoRequest( location, method, path_and_query, request_headers, body, follow_redirects = follow_redirects, report_hooks = report_hooks, response_to_path = response_to_path )
if return_everything: return ( response, size_of_response, response_headers, cookies )
elif return_cookies: return ( response, cookies )
else: return response
def MaintainConnections( self ):
while True:
if HC.shutdown: break
with self._lock:
connections_copy = dict( self._connections )
for ( location, connection ) in connections_copy.items():
with connection.lock:
if connection.IsStale():
del self._connections[ location ]
time.sleep( 30 )
class HTTPConnection():
timeout = 30
read_block_size = 64 * 1024
def __init__( self, location ):
( self._scheme, self._host, self._port ) = location
self.lock = threading.Lock()
self._last_request_time = HC.GetNow()
self._RefreshConnection()
def _ParseCookies( self, raw_cookies_string ):
cookies = {}
if raw_cookies_string is not None:
raw_cookie_strings = raw_cookies_string.split( ', ' )
for raw_cookie_string in raw_cookie_strings:
try:
# HSID=AYQEVnDKrdst; Domain=.foo.com; Path=/; Expires=Wed, 13 Jan 2021 22:23:01 GMT; HttpOnly
if ';' in raw_cookie_string: ( raw_cookie_string, gumpf ) = raw_cookie_string.split( ';', 1 )
( cookie_name, cookie_value ) = raw_cookie_string.split( '=' )
cookies[ cookie_name ] = cookie_value
except Exception as e: pass
return cookies
def _ParseResponse( self, response, report_hooks ):
content_length = response.getheader( 'Content-Length' )
if content_length is not None: content_length = int( content_length )
data = ''
next_block = response.read( self.read_block_size )
while next_block != '':
if HC.shutdown: raise Exception( 'Application is shutting down!' )
data += next_block
if content_length is not None and len( data ) > content_length:
raise Exception( 'Response was longer than suggested!' )
for hook in report_hooks: hook( content_length, len( data ) )
next_block = response.read( self.read_block_size )
size_of_response = len( data )
content_type = response.getheader( 'Content-Type' )
if content_type is None: parsed_response = data
else:
if '; ' in content_type: ( mime_string, additional_info ) = content_type.split( '; ', 1 )
else: ( mime_string, additional_info ) = ( content_type, '' )
if 'charset=' in additional_info:
# this does utf-8, ISO-8859-4, whatever
( gumpf, charset ) = additional_info.split( '=' )
try: parsed_response = data.decode( charset )
except: parsed_response = data
elif content_type in HC.mime_enum_lookup and HC.mime_enum_lookup[ content_type ] == HC.APPLICATION_YAML:
try: parsed_response = yaml.safe_load( data )
except Exception as e: raise HydrusExceptions.NetworkVersionException( 'Failed to parse a response object!' + os.linesep + u( e ) )
elif content_type == 'text/html':
try: parsed_response = data.decode( 'utf-8' )
except: parsed_response = data
else: parsed_response = data
return ( parsed_response, size_of_response )
def _RefreshConnection( self ):
if self._scheme == 'http': self._connection = httplib.HTTPConnection( self._host, self._port, timeout = self.timeout )
elif self._scheme == 'https': self._connection = httplib.HTTPSConnection( self._host, self._port, timeout = self.timeout )
try: self._connection.connect()
except: raise Exception( 'Could not connect to ' + self._host + '!' )
def _WriteResponseToPath( self, response, report_hooks ):
content_length = response.getheader( 'Content-Length' )
if content_length is not None: content_length = int( content_length )
temp_path = HC.GetTempPath()
size_of_response = 0
with open( temp_path, 'wb' ) as f:
next_block = response.read( self.read_block_size )
while next_block != '':
if HC.shutdown: raise Exception( 'Application is shutting down!' )
size_of_response += len( next_block )
if content_length is not None and size_of_response > content_length:
raise Exception( 'Response was longer than suggested!' )
f.write( next_block )
for hook in report_hooks: hook( content_length, size_of_response )
next_block = response.read( self.read_block_size )
return ( temp_path, size_of_response )
def IsStale( self ):
time_since_last_request = HC.GetNow() - self._last_request_time
return time_since_last_request > self.timeout
def Request( self, method, path_and_query, request_headers, body, report_hooks = [], response_to_path = False ):
if method == HC.GET: method_string = 'GET'
elif method == HC.POST: method_string = 'POST'
if 'User-Agent' not in request_headers: request_headers[ 'User-Agent' ] = 'hydrus/' + HC.u( HC.NETWORK_VERSION )
try:
self._connection.request( method_string, path_and_query, headers = request_headers, body = body )
response = self._connection.getresponse()
except ( httplib.CannotSendRequest, httplib.BadStatusLine ):
# for some reason, we can't send a request on the current connection, so let's make a new one and try again!
self._RefreshConnection()
self._connection.request( method_string, path_and_query, headers = request_headers, body = body )
response = self._connection.getresponse()
if response.status == 200 and response_to_path:
( temp_path, size_of_response ) = self._WriteResponseToPath( response, report_hooks )
parsed_response = temp_path
else:
( parsed_response, size_of_response ) = self._ParseResponse( response, report_hooks )
response_headers = { k : v for ( k, v ) in response.getheaders() if k != 'set-cookie' }
cookies = self._ParseCookies( response.getheader( 'set-cookie' ) )
self._last_request_time = HC.GetNow()
if response.status == 200: return ( parsed_response, None, size_of_response, response_headers, cookies )
elif response.status in ( 301, 302, 303, 307 ):
location = response.getheader( 'Location' )
if location is None: raise Exception( parsed_response )
else:
url = location
if response.status in ( 301, 307 ):
# 301: moved permanently, repeat request
# 307: moved temporarily, repeat request
redirect_info = ( method, url )
elif response.status in ( 302, 303 ):
# 302: moved temporarily, repeat request (except everyone treats it like 303 for no good fucking reason)
# 303: thanks, now go here with GET
redirect_info = ( HC.GET, url )
return ( parsed_response, redirect_info, size_of_response, response_headers, cookies )
elif response.status == 304: raise HydrusExceptions.NotModifiedException()
else:
if response.status == 401: raise HydrusExceptions.PermissionException( parsed_response )
elif response.status == 403: raise HydrusExceptions.ForbiddenException( parsed_response )
elif response.status == 404: raise HydrusExceptions.NotFoundException( parsed_response )
elif response.status == 426: raise HydrusExceptions.NetworkVersionException( parsed_response )
elif response.status in ( 500, 501, 502, 503 ): raise Exception( parsed_response )
else: raise Exception( parsed_response )

22
include/HydrusSessions.py
View File

@ -121,11 +121,7 @@ class HydrusSessionManagerClient():
service = HC.app.Read( 'service', service_identifier )
connection = service.GetConnection()
connection.Get( 'session_key' )
cookies = connection.GetCookies()
( response, cookies ) = service.Request( HC.GET, 'session_key', return_cookies = True )
try: session_key = cookies[ 'session_key' ].decode( 'hex' )
except: raise Exception( 'Service did not return a session key!' )
@ -296,12 +292,7 @@ class WebSessionManagerClient():
if name == 'hentai foundry':
connection = HC.get_connection( url = 'http://www.hentai-foundry.com', accept_cookies = True )
# this establishes the php session cookie, the csrf cookie, and tells hf that we are 18 years of age
connection.request( 'GET', '/?enterAgree=1' )
cookies = connection.GetCookies()
( response_gumpf, cookies ) = HC.http.Request( HC.GET, 'http://www.hentai-foundry.com/?enterAgree=1', return_cookies = True )
expiry = now + 60 * 60
@ -314,8 +305,6 @@ class WebSessionManagerClient():
raise Exception( 'You need to set up your pixiv credentials in services->manage pixiv account.' )
connection = HC.get_connection( url = 'http://www.pixiv.net', accept_cookies = True )
form_fields = {}
form_fields[ 'mode' ] = 'login'
@ -328,13 +317,10 @@ class WebSessionManagerClient():
headers = {}
headers[ 'Content-Type' ] = 'application/x-www-form-urlencoded'
# this logs in and establishes the php session cookie
response = connection.request( 'POST', '/login.php', headers = headers, body = body, follow_redirects = False )
cookies = connection.GetCookies()
( response_gumpf, cookies ) = HC.http.Request( HC.POST, 'http://www.pixiv.net/login.php', request_headers = headers, body = body, return_cookies = True )
# _ only given to logged in php sessions
if 'PHPSESSID' not in cookies or '_' not in cookies[ 'PHPSESSID' ]: raise Exception( 'Login credentials not accepted!' )
if 'PHPSESSID' not in cookies or '_' not in cookies[ 'PHPSESSID' ]: raise Exception( 'Pixiv login credentials not accepted!' )
expiry = now + 30 * 86400

2
include/TestConstants.py
View File

@ -140,7 +140,7 @@ class FakePubSub():
def sub( self, object, method_name, topic ): pass
def WXpubimmediate( self, topic, *args, **kwargs ):
return
with self._lock:
callables = self._GetCallables( topic )

2
include/TestHydrusServer.py
View File

@ -307,8 +307,6 @@ class TestServer( unittest.TestCase ):
service_for_session_manager = CC.Service( service_identifier, info )
connection = service_for_session_manager.GetConnection()
HC.app.SetRead( 'service', service_for_session_manager )
HC.app.SetRead( 'account', self._account )