RepoMirrors
/
hydrus
mirror of https://github.com/hydrusnetwork/hydrus.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Version 87

This commit is contained in:
Hydrus 2013-10-02 17:06:06 -05:00
parent 90b2f8fa26
commit 756e6ad6d1
17 changed files with 2375 additions and 1852 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
client.pyw
View File

@ -18,7 +18,7 @@ import sys
from include import HydrusConstants as HC
from include import ClientController
import threading
#from twisted.internet import reactor
from twisted.internet import reactor
initial_sys_stdout = sys.stdout
initial_sys_stderr = sys.stderr
@ -30,7 +30,7 @@ with open( HC.LOGS_DIR + os.path.sep + 'client.log', 'a' ) as f:
try:
#threading.Thread( target = reactor.run, kwargs = { 'installSignalHandlers' : 0 } ).start()
threading.Thread( target = reactor.run, kwargs = { 'installSignalHandlers' : 0 } ).start()
app = ClientController.Controller()
@ -47,6 +47,6 @@ sys.stderr = initial_sys_stderr
HC.shutdown = True
#reactor.callFromThread( reactor.stop )
reactor.callFromThread( reactor.stop )
HC.pubsub.pubimmediate( 'shutdown' )

44
help/changelog.html
View File

@ -8,6 +8,50 @@
<div class="content">
<h3>changelog</h3>
<ul>
<li><h3>version 87</h3></li>
<ul>
<li>misc:</li>
<li><ul>
<li>fixed system:untagged, which was doing numtags>0 by accident</li>
</ul></li>
<li>basics:</li>
<li><ul>
<li>moved client local service to twisted</li>
<li>moved server to twisted</li>
<li>cleaned up a whole lot of unrelated server stuff I haven't touched in a while</li>
<li>fixed some misc typos</li>
</ul></li>
<li>details:</li>
<li><ul>
<li>built twisted server framework to plug into hydrus</li>
<li>changed hydrus authorisation header</li>
<li>changed sessions to manage accounts, reducing server db load</li>
<li>reworked session cookie to be neater</li>
<li>cleaned up a bunch of server code</li>
<li>fixed how certain server errors were being printed to the log</li>
<li>reworked response_context to a body/path dichotomy to reduce cpu+memory on file requests</li>
<li>collapsed a bunch of GET POST requests to the same path</li>
<li>reworked server_service_identifiers to be port-independant</li>
<li>improved data use logging to support new session management system</li>
<li>moved session management to new system</li>
<li>moved error management to new system</li>
<li>integrated twisted thread into hydrus controller</li>
<li>reworked root and favicon</li>
<li>reworked local file and thumbnail requests</li>
<li>reworked restricted service requests</li>
<li>reworked admin service requests</li>
<li>reworked repository service requests</li>
<li>removed manage options query; it'll be rolled into manage services admin queries</li>
<li>updated serverdb to manage with new system, harmonising internal and external requests to one workflow</li>
<li>made server-side data use tracking simpler</li>
<li>added Content-Type header to most requests</li>
<li>improved how key registration, init, and account GET requests are handled client-side</li>
<li>harmonised how account_keys are created server-side</li>
<li>moved server management from serverdb to servercontroller, with better pubsub restart</li>
<li>moved several useful functions to the new serverconstants.py</li>
<li>fixed a max_age sessions issue</li>
</ul></li>
</ul>
<li><h3>version 86</h3></li>
<ul>
<li>timeout on connections improved</li>

71
include/ClientConstants.py
View File

@ -1160,7 +1160,7 @@ class ConnectionToService():
access_key = self._credentials.GetAccessKey()
if access_key != '': headers[ 'Authorization' ] = 'hydrus_network ' + access_key.encode( 'hex' )
if access_key != '': headers[ 'Hydrus-Key' ] = access_key.encode( 'hex' )
else: raise Exception( 'No access key!' )
@ -1178,6 +1178,8 @@ class ConnectionToService():
# prepare
headers = self._GetHeaders( request )
if request_type == HC.GET:
request_type_string = 'GET'
@ -1232,11 +1234,21 @@ class ConnectionToService():
request_string = '/' + request
if request == 'file': body = request_args[ 'file' ]
else: body = yaml.safe_dump( request_args )
if request == 'file':
content_type = HC.APPLICATION_OCTET_STREAM
body = request_args[ 'file' ]
else:
content_type = HC.APPLICATION_YAML
body = yaml.safe_dump( request_args )
headers[ 'Content-Type' ] = HC.mime_string_lookup[ content_type ]
headers = self._GetHeaders( request )
# send
@ -1261,54 +1273,7 @@ class ConnectionToService():
except: pass
def Get( self, request, **kwargs ):
response = self._SendRequest( HC.GET, request, kwargs )
if request in ( 'registration_keys', 'init' ):
if request == 'registration_keys':
body = os.linesep.join( [ 'r' + key.encode( 'hex' ) for key in response ] )
filename = 'registration keys.txt'
elif request == 'init':
filename = 'admin access key.txt'
access_key = response
body = access_key.encode( 'hex' )
( host, port ) = self._credentials.GetAddress()
self._credentials = Credentials( host, port, access_key )
edit_log = [ ( 'edit', ( self._service_identifier, ( self._service_identifier, self._credentials, None ) ) ) ]
HC.app.Write( 'update_services', edit_log )
with wx.FileDialog( None, style=wx.FD_SAVE, defaultFile = filename ) as dlg:
if dlg.ShowModal() == wx.ID_OK:
with open( dlg.GetPath(), 'wb' ) as f: f.write( body )
elif request == 'account':
account = response
account.MakeFresh()
HC.app.Write( 'service_updates', { self._service_identifier : [ HC.ServiceUpdate( HC.SERVICE_UPDATE_ACCOUNT, account ) ] } )
return response
def Get( self, request, **kwargs ): return self._SendRequest( HC.GET, request, kwargs )
def GetCookies( self ): return self._connection.GetCookies()

96
include/ClientController.py
View File

@ -20,7 +20,7 @@ import time
import traceback
import wx
import wx.richtext
#from twisted.internet import reactor
from twisted.internet import reactor
ID_ANIMATED_EVENT_TIMER = wx.NewId()
ID_MAINTENANCE_EVENT_TIMER = wx.NewId()
@ -210,7 +210,8 @@ class Controller( wx.App ):
except HydrusExceptions.DBAccessException as e:
print( repr( HC.u( e ) ) )
try: print( HC.u( e ) )
except: print( repr( HC.u( e ) ) )
message = 'This instance of the client had a problem connecting to the database, which probably means an old instance is still closing.'
message += os.linesep + os.linesep
@ -324,55 +325,7 @@ class Controller( wx.App ):
def RestartServer( self ):
def do_it():
# stick it on a thread because the connection test stuff blocks
if self._server is not None: self._server.shutdown()
port = HC.options[ 'local_port' ]
time.sleep( 1 )
connection = httplib.HTTPConnection( '127.0.0.1', port, timeout = 10 )
try:
connection.connect()
connection.close()
message = 'Something was already bound to port ' + HC.u( port )
wx.CallAfter( HC.pubsub.pub, 'message', HC.Message( HC.MESSAGE_TYPE_TEXT, message ) )
except:
local_file_server_service_identifier = HC.ServerServiceIdentifier( HC.LOCAL_FILE, port )
self._server = HydrusServer.HydrusHTTPServer( local_file_server_service_identifier )
server_thread = threading.Thread( target=self._server.serve_forever )
server_thread.start()
time.sleep( 1 )
connection = httplib.HTTPConnection( '127.0.0.1', port, timeout = 10 )
try:
connection.connect()
connection.close()
except:
message = 'Tried to bind port ' + HC.u( port ) + ' but it failed'
wx.CallAfter( HC.pubsub.pub, 'message', HC.Message( HC.MESSAGE_TYPE_TEXT, message ) )
threading.Thread( target = do_it ).start()
port = HC.options[ 'local_port' ]
def TWISTEDRestartServer():
@ -380,13 +333,42 @@ class Controller( wx.App ):
try:
local_file_server_service_identifier = HC.ServerServiceIdentifier( HC.LOCAL_FILE, 1050 )
connection = httplib.HTTPConnection( '127.0.0.1', port, timeout = 10 )
self._local_service = reactor.listenTCP( 1050, HydrusServer.HydrusServiceLocal( local_file_server_service_identifier, 'hello' ) )
try:
connection.connect()
connection.close()
message = 'Something was already bound to port ' + HC.u( port )
wx.CallAfter( HC.pubsub.pub, 'message', HC.Message( HC.MESSAGE_TYPE_TEXT, message ) )
except:
local_file_server_service_identifier = HC.ServerServiceIdentifier( 'local file', HC.LOCAL_FILE )
self._local_service = reactor.listenTCP( port, HydrusServer.HydrusServiceLocal( local_file_server_service_identifier, 'hello' ) )
connection = httplib.HTTPConnection( '127.0.0.1', port, timeout = 10 )
try:
connection.connect()
connection.close()
except:
message = 'Tried to bind port ' + HC.u( port ) + ' but it failed'
wx.CallAfter( HC.pubsub.pub, 'message', HC.Message( HC.MESSAGE_TYPE_TEXT, message ) )
except Exception as e:
wx.CallAfter( HC.ShowException, e )
except Exception as e: HC.ShowException( e )
# handle problems
if self._local_service is None: StartServer()
@ -398,7 +380,7 @@ class Controller( wx.App ):
#reactor.callFromThread( TWISTEDRestartServer )
reactor.callFromThread( TWISTEDRestartServer )
def SetSplashText( self, text ):

132
include/ClientDB.py
View File

@ -1892,7 +1892,10 @@ class ServiceDB( FileDB, MessageDB, TagDB, RatingDB ):
if num_tags_zero or num_tags_nonzero:
query_hash_ids = { id for ( id, ) in c.execute( 'SELECT hash_id FROM mappings WHERE service_id = ? AND hash_id IN ' + HC.SplayListForDB( query_hash_ids ) + ' AND status IN ' + HC.SplayListForDB( statuses ) + ';', ( tag_service_id, ) ) }
tag_query_hash_ids = { id for ( id, ) in c.execute( 'SELECT hash_id FROM mappings WHERE service_id = ? AND hash_id IN ' + HC.SplayListForDB( query_hash_ids ) + ' AND status IN ' + HC.SplayListForDB( statuses ) + ';', ( tag_service_id, ) ) }
if num_tags_zero: query_hash_ids.difference_update( tag_query_hash_ids )
elif num_tags_nonzero: query_hash_ids = tag_query_hash_ids
if len( tag_predicates ) > 0:
@ -4079,33 +4082,25 @@ class ServiceDB( FileDB, MessageDB, TagDB, RatingDB ):
if action == HC.ADD:
server_service_identifier = data
( service_type, port ) = data
service_key = os.urandom( 32 )
service_type = server_service_identifier.GetType()
service_port = server_service_identifier.GetPort()
port = server_service_identifier.GetPort()
service_name = HC.service_string_lookup[ service_type ] + ' at ' + host + ':' + HC.u( service_port )
service_name = HC.service_string_lookup[ service_type ] + ' at ' + host + ':' + HC.u( port )
client_service_identifier = HC.ClientServiceIdentifier( service_key, service_type, service_name )
credentials = CC.Credentials( host, service_port, access_key )
credentials = CC.Credentials( host, port, access_key )
if service_type == HC.MESSAGE_DEPOT: extra_info = ( 'identity@' + service_name, 180, HydrusEncryption.GenerateNewPrivateKey(), True )
else: extra_info = None
self._AddService( c, client_service_identifier, credentials, extra_info )
elif action == HC.EDIT:
( server_service_identifier, new_port ) = data
current_port = server_service_identifier.GetPort()
c.execute( 'UPDATE addresses SET port = ? WHERE host = ? AND port = ?;', ( new_port, host, current_port ) )
elif action == HC.DELETE:
server_service_identifier = data
@ -6473,61 +6468,63 @@ class DB( ServiceDB ):
def ProcessWrite( action, args, kwargs ):
if action == '4chan_pass': self._Set4chanPass( c, *args, **kwargs )
elif action == 'add_downloads': self._AddDownloads( c, *args, **kwargs )
elif action == 'add_uploads': self._AddUploads( c, *args, **kwargs )
elif action == 'archive_conversation': self._ArchiveConversation( c, *args, **kwargs )
elif action == 'contact_associated': self._AssociateContact( c, *args, **kwargs )
elif action == 'content_updates': self._ProcessContentUpdates( c, *args, **kwargs )
elif action == 'copy_files': self._CopyFiles( c, *args, **kwargs )
elif action == 'delete_conversation': self._DeleteConversation( c, *args, **kwargs )
elif action == 'delete_draft': self._DeleteDraft( c, *args, **kwargs )
elif action == 'delete_orphans': self._DeleteOrphans( c, *args, **kwargs )
elif action == 'delete_pending': self._DeletePending( c, *args, **kwargs )
elif action == 'delete_hydrus_session_key': self._DeleteHydrusSessionKey( c, *args, **kwargs )
elif action == 'draft_message': self._DraftMessage( c, *args, **kwargs )
elif action == 'fatten_autocomplete_cache': self._FattenAutocompleteCache( c, *args, **kwargs )
elif action == 'favourite_custom_filter_actions': self._SetFavouriteCustomFilterActions( c, *args, **kwargs )
elif action == 'flush_message_statuses': self._FlushMessageStatuses( c, *args, **kwargs )
elif action == 'generate_tag_ids': self._GenerateTagIdsEfficiently( c, *args, **kwargs )
if action == '4chan_pass': result = self._Set4chanPass( c, *args, **kwargs )
elif action == 'add_downloads': result = self._AddDownloads( c, *args, **kwargs )
elif action == 'add_uploads': result = self._AddUploads( c, *args, **kwargs )
elif action == 'archive_conversation': result = self._ArchiveConversation( c, *args, **kwargs )
elif action == 'contact_associated': result = self._AssociateContact( c, *args, **kwargs )
elif action == 'content_updates': result = self._ProcessContentUpdates( c, *args, **kwargs )
elif action == 'copy_files': result = self._CopyFiles( c, *args, **kwargs )
elif action == 'delete_conversation': result = self._DeleteConversation( c, *args, **kwargs )
elif action == 'delete_draft': result = self._DeleteDraft( c, *args, **kwargs )
elif action == 'delete_orphans': result = self._DeleteOrphans( c, *args, **kwargs )
elif action == 'delete_pending': result = self._DeletePending( c, *args, **kwargs )
elif action == 'delete_hydrus_session_key': result = self._DeleteHydrusSessionKey( c, *args, **kwargs )
elif action == 'draft_message': result = self._DraftMessage( c, *args, **kwargs )
elif action == 'fatten_autocomplete_cache': result = self._FattenAutocompleteCache( c, *args, **kwargs )
elif action == 'favourite_custom_filter_actions': result = self._SetFavouriteCustomFilterActions( c, *args, **kwargs )
elif action == 'flush_message_statuses': result = self._FlushMessageStatuses( c, *args, **kwargs )
elif action == 'generate_tag_ids': result = self._GenerateTagIdsEfficiently( c, *args, **kwargs )
elif action == 'hydrus_session': result = self._AddHydrusSession( c, *args, **kwargs )
elif action == 'import_file': return self._ImportFile( c, *args, **kwargs )
elif action == 'import_file_from_page': self._ImportFilePage( c, *args, **kwargs )
elif action == 'import_folder': self._UpdateImportFolder( c, *args, **kwargs )
elif action == 'import_folders': self._SetImportFolders( c, *args, **kwargs )
elif action == 'inbox_conversation': self._InboxConversation( c, *args, **kwargs )
elif action == 'message': self._AddMessage( c, *args, **kwargs )
elif action == 'message_info_since': self._AddMessageInfoSince( c, *args, **kwargs )
elif action == 'message_statuses': self._UpdateMessageStatuses( c, *args, **kwargs )
elif action == 'pixiv_account': self._SetPixivAccount( c, *args, **kwargs )
elif action == 'reset_service': self._ResetService( c, *args, **kwargs )
elif action == 'save_options': self._SaveOptions( c, *args, **kwargs )
elif action == 'service_updates': self._ProcessServiceUpdates( c, *args, **kwargs )
elif action == 'session': self._AddSession( c, *args, **kwargs )
elif action == 'set_password': self._SetPassword( c, *args, **kwargs )
elif action == 'set_tag_service_precedence': self._SetTagServicePrecedence( c, *args, **kwargs )
elif action == 'subscription': self._SetSubscription( c, *args, **kwargs )
elif action == 'subscriptions': self._SetSubscriptions( c, *args, **kwargs )
elif action == 'tag_parents': self._UpdateTagParents( c, *args, **kwargs )
elif action == 'tag_siblings': self._UpdateTagSiblings( c, *args, **kwargs )
elif action == 'thumbnails': self._AddThumbnails( c, *args, **kwargs )
elif action == 'update': self._AddUpdate( c, *args, **kwargs )
elif action == 'update_boorus': self._UpdateBoorus( c, *args, **kwargs )
elif action == 'update_contacts': self._UpdateContacts( c, *args, **kwargs )
elif action == 'update_imageboards': self._UpdateImageboards( c, *args, **kwargs )
elif action == 'update_server_services': self._UpdateServerServices( c, *args, **kwargs )
elif action == 'update_services': self._UpdateServices( c, *args, **kwargs )
elif action == 'vacuum': self._Vacuum()
elif action == 'import_file': result = self._ImportFile( c, *args, **kwargs )
elif action == 'import_file_from_page': result = self._ImportFilePage( c, *args, **kwargs )
elif action == 'import_folder': result = self._UpdateImportFolder( c, *args, **kwargs )
elif action == 'import_folders': result = self._SetImportFolders( c, *args, **kwargs )
elif action == 'inbox_conversation': result = self._InboxConversation( c, *args, **kwargs )
elif action == 'message': result = self._AddMessage( c, *args, **kwargs )
elif action == 'message_info_since': result = self._AddMessageInfoSince( c, *args, **kwargs )
elif action == 'message_statuses': result = self._UpdateMessageStatuses( c, *args, **kwargs )
elif action == 'pixiv_account': result = self._SetPixivAccount( c, *args, **kwargs )
elif action == 'reset_service': result = self._ResetService( c, *args, **kwargs )
elif action == 'save_options': result = self._SaveOptions( c, *args, **kwargs )
elif action == 'service_updates': result = self._ProcessServiceUpdates( c, *args, **kwargs )
elif action == 'session': result = self._AddSession( c, *args, **kwargs )
elif action == 'set_password': result = self._SetPassword( c, *args, **kwargs )
elif action == 'set_tag_service_precedence': result = self._SetTagServicePrecedence( c, *args, **kwargs )
elif action == 'subscription': result = self._SetSubscription( c, *args, **kwargs )
elif action == 'subscriptions': result = self._SetSubscriptions( c, *args, **kwargs )
elif action == 'tag_parents': result = self._UpdateTagParents( c, *args, **kwargs )
elif action == 'tag_siblings': result = self._UpdateTagSiblings( c, *args, **kwargs )
elif action == 'thumbnails': result = self._AddThumbnails( c, *args, **kwargs )
elif action == 'update': result = self._AddUpdate( c, *args, **kwargs )
elif action == 'update_boorus': result = self._UpdateBoorus( c, *args, **kwargs )
elif action == 'update_contacts': result = self._UpdateContacts( c, *args, **kwargs )
elif action == 'update_imageboards': result = self._UpdateImageboards( c, *args, **kwargs )
elif action == 'update_server_services': result = self._UpdateServerServices( c, *args, **kwargs )
elif action == 'update_services': result = self._UpdateServices( c, *args, **kwargs )
elif action == 'vacuum': result = self._Vacuum()
elif action == 'web_session': result = self._AddWebSession( c, *args, **kwargs )
else: raise Exception( 'db received an unknown write command: ' + action )
return result
HC.pubsub.pub( 'db_locked_status', 'db locked' )
action = job.GetAction()
job_type = job.GetType()
action = job.GetAction()
args = job.GetArgs()
kwargs = job.GetKWArgs()
@ -6864,7 +6861,12 @@ def DAEMONDownloadThumbnails():
thumbnails = []
for hash in thumbnail_hashes_i_need[ i : i + num_per_round ]: thumbnails.append( ( hash, connection.Get( 'thumbnail', hash = hash.encode( 'hex' ) ) ) )
for hash in thumbnail_hashes_i_need[ i : i + num_per_round ]:
thumbnail = connection.Get( 'thumbnail', hash = hash.encode( 'hex' ) )
thumbnails.append( ( hash, thumbnail ) )
HC.app.WaitUntilGoodTimeToUseGUIThread()
@ -6958,12 +6960,20 @@ def DAEMONSynchroniseAccounts():
connection = service.GetConnection()
connection.Get( 'account' )
response = connection.Get( 'account' )
account = response[ 'account' ]
account.MakeFresh()
HC.app.Write( 'service_updates', { service_identifier : [ HC.ServiceUpdate( HC.SERVICE_UPDATE_ACCOUNT, account ) ] } )
do_notify = True
except Exception as e:
print( traceback.format_exc() )
name = service_identifier.GetName()
message = 'Failed to refresh account for ' + name + ':' + os.linesep + os.linesep + HC.u( e )

102
include/ClientGUI.py
View File

@ -184,9 +184,9 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
content_updates = [ HC.ContentUpdate( HC.CONTENT_DATA_TYPE_FILES, HC.CONTENT_UPDATE_ADD, content_update_row ) ]
service_identfiers_to_content_updates = { service_identifier : content_updates }
service_identifiers_to_content_updates = { service_identifier : content_updates }
HC.app.Write( 'content_updates', service_identfiers_to_content_updates )
HC.app.Write( 'content_updates', service_identifiers_to_content_updates )
except Exception as e:
@ -206,9 +206,9 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
content_updates = update.GetContentUpdates( for_client = True )
service_identfiers_to_content_updates = { service_identifier : content_updates }
service_identifiers_to_content_updates = { service_identifier : content_updates }
HC.app.Write( 'content_updates', service_identfiers_to_content_updates )
HC.app.Write( 'content_updates', service_identifiers_to_content_updates )
elif service_type == HC.TAG_REPOSITORY:
@ -237,9 +237,9 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
connection.Post( 'update', update = update )
service_identfiers_to_content_updates = { service_identifier : update.GetContentUpdates( for_client = True ) }
service_identifiers_to_content_updates = { service_identifier : update.GetContentUpdates( for_client = True ) }
HC.app.Write( 'content_updates', service_identfiers_to_content_updates )
HC.app.Write( 'content_updates', service_identifiers_to_content_updates )
@ -281,7 +281,9 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
connection = service.GetConnection()
account_info = connection.Get( 'account_info', subject_access_key = subject_access_key.encode( 'hex' ) )
response = connection.Get( 'account_info', subject_access_key = subject_access_key.encode( 'hex' ) )
account_info = response[ 'account_info' ]
wx.MessageBox( HC.u( account_info ) )
@ -409,17 +411,36 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
connection = service.GetConnection()
connection.Get( 'init' )
response = connection.Get( 'init' )
filename = 'admin access key.txt'
access_key = response[ 'access_key' ]
body = access_key.encode( 'hex' )
( host, port ) = service.GetCredentials().GetAddress()
credentials = Credentials( host, port, access_key )
edit_log = [ ( 'edit', ( self._service_identifier, ( self._service_identifier, credentials, None ) ) ) ]
HC.app.Write( 'update_services', edit_log )
with wx.FileDialog( None, style=wx.FD_SAVE, defaultFile = filename ) as dlg:
if dlg.ShowModal() == wx.ID_OK:
with open( dlg.GetPath(), 'wb' ) as f: f.write( body )
edit_log = []
tag_service_identifier = HC.ServerServiceIdentifier( HC.TAG_REPOSITORY, HC.DEFAULT_SERVICE_PORT )
file_service_identifier = HC.ServerServiceIdentifier( HC.FILE_REPOSITORY, HC.DEFAULT_SERVICE_PORT + 1 )
edit_log.append( ( HC.ADD, ( HC.TAG_REPOSITORY, HC.DEFAULT_SERVICE_PORT ) ) )
edit_log.append( ( HC.ADD, ( HC.FILE_REPOSITORY, HC.DEFAULT_SERVICE_PORT + 1 ) ) )
edit_log.append( ( HC.ADD, tag_service_identifier ) )
edit_log.append( ( HC.ADD, file_service_identifier ) )
connection.Post( 'services_modification', edit_log = edit_log )
connection.Post( 'services', edit_log = edit_log )
HC.app.Write( 'update_server_services', admin_service_identifier, edit_log )
@ -508,7 +529,9 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
connection = service.GetConnection()
stats = connection.Get( 'stats' )
response = connection.Get( 'stats' )
stats = response[ 'stats' ]
wx.MessageBox( HC.u( stats ) )
@ -525,7 +548,10 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
connection = service.GetConnection()
with wx.BusyCursor(): ( ip, timestamp ) = connection.Get( 'ip', hash = hash.encode( 'hex' ) )
with wx.BusyCursor(): response = connection.Get( 'ip', hash = hash.encode( 'hex' ) )
ip = response[ 'ip' ]
timestamp = response[ 'timestamp' ]
message = 'File Hash: ' + hash.encode( 'hex' ) + os.linesep + 'Uploader\'s IP: ' + ip + os.linesep + 'Upload Time (GMT): ' + time.asctime( time.gmtime( int( timestamp ) ) )
@ -1498,8 +1524,6 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
submenu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'modify_account', s_i ), p( '&Modify an Account' ), p( 'Modify a specific account\'s type and expiration.' ) )
submenu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'account_info', s_i ), p( '&Get an Account\'s Info' ), p( 'Fetch information about an account from the tag repository.' ) )
submenu.AppendSeparator()
submenu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'options', s_i ), p( '&Options' ), p( 'Set the tag repository\'s options.' ) )
submenu.AppendSeparator()
submenu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'stats', s_i ), p( '&Get Stats' ), p( 'Fetch operating statistics from the tag repository.' ) )
submenu.AppendSeparator()
submenu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'post_news', s_i ), p( '&Post News' ), p( 'Post a news item to the tag repository.' ) )
@ -1517,8 +1541,6 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
submenu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'account_info', s_i ), p( '&Get an Account\'s Info' ), p( 'Fetch information about an account from the file repository.' ) )
submenu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'fetch_ip', s_i ), p( '&Get an Uploader\'s IP Address' ), p( 'Fetch an uploader\'s ip address.' ) )
submenu.AppendSeparator()
submenu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'options', s_i ), p( '&Options' ), p( 'Set the file repository\'s options.' ) )
submenu.AppendSeparator()
submenu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'stats', s_i ), p( '&Get Stats' ), p( 'Fetch operating statistics from the file repository.' ) )
submenu.AppendSeparator()
submenu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'post_news', s_i ), p( '&Post News' ), p( 'Post a news item to the file repository.' ) )
@ -1541,9 +1563,8 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
submenu = wx.Menu()
submenu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'options', s_i ), p( '&Options' ), p( 'Set the server\'s options.' ) )
submenu.AppendSeparator()
submenu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'manage_server_services', s_i ), p( 'Manage &Services' ), p( 'Add, edit, and delete this server\'s services.' ) )
menu_item = submenu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'manage_server_services', s_i ), p( 'Manage &Services' ), p( 'Add, edit, and delete this server\'s services.' ) )
submenu.Enable( menu_item.GetId(), False )
submenu.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'backup_service', s_i ), p( 'Make a &Backup' ), p( 'Back up this server\'s database.' ) )
admin.AppendMenu( CC.ID_NULL, p( s_i.GetName() ), submenu )
@ -1556,7 +1577,8 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
help.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'help' ), p( '&Help' ) )
dont_know = wx.Menu()
dont_know.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'auto_repo_setup' ), p( 'Just set up some repositories for me, please' ) )
dont_know.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'auto_server_setup' ), p( 'Just set up the server on this computer, please' ) )
menu_item = dont_know.Append( CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'auto_server_setup' ), p( 'Just set up the server on this computer, please' ) )
dont_know.Enable( menu_item.GetId(), False )
help.AppendMenu( wx.ID_NONE, p( 'I don\'t know what I am doing' ), dont_know )
links = wx.Menu()
tumblr = wx.MenuItem( links, CC.MENU_EVENT_ID_TO_ACTION_CACHE.GetId( 'tumblr' ), p( 'Tumblr' ) )
@ -2378,7 +2400,29 @@ class FrameReviewServices( ClientGUICommon.Frame ):
connection = service.GetConnection()
connection.Get( 'init' )
response = connection.Get( 'init' )
access_key = response[ 'access_key' ]
body = access_key.encode( 'hex' )
( host, port ) = service.GetCredentials().GetAddress()
credentials = CC.Credentials( host, port, access_key )
edit_log = [ ( 'edit', ( self._service_identifier, ( self._service_identifier, credentials, None ) ) ) ]
HC.app.Write( 'update_services', edit_log )
filename = 'admin access key.txt'
with wx.FileDialog( None, style=wx.FD_SAVE, defaultFile = filename ) as dlg:
if dlg.ShowModal() == wx.ID_OK:
with open( dlg.GetPath(), 'wb' ) as f: f.write( body )
def EventServiceRefreshAccount( self, event ):
@ -2387,7 +2431,13 @@ class FrameReviewServices( ClientGUICommon.Frame ):
connection = self._service.GetConnection()
connection.Get( 'account' )
response = connection.Get( 'account' )
account = response[ 'account' ]
account.MakeFresh()
HC.app.Write( 'service_updates', { self._service_identifier : [ HC.ServiceUpdate( HC.SERVICE_UPDATE_ACCOUNT, account ) ] } )
def EventServiceReset( self, event ):

44
include/ClientGUIDialogs.py
View File

@ -1994,7 +1994,9 @@ class DialogInputNewAccounts( Dialog ):
connection = service.GetConnection()
account_types = connection.Get( 'account_types' )
response = connection.Get( 'account_types' )
account_types = response[ 'account_types' ]
for account_type in account_types: self._account_types.Append( account_type.ConvertToString(), account_type )
self._account_types.SetSelection( 0 ) # admin
@ -2057,8 +2059,22 @@ class DialogInputNewAccounts( Dialog ):
connection = service.GetConnection()
if expiration is None: access_keys = connection.Get( 'registration_keys', num = num, title = title )
else: access_keys = connection.Get( 'registration_keys', num = num, title = title, expiration = expiration )
if expiration is None: response = connection.Get( 'registration_keys', num = num, title = title )
else: response = connection.Get( 'registration_keys', num = num, title = title, expiration = expiration )
registration_keys = response[ 'registration_keys' ]
filename = 'registration keys.txt'
with wx.FileDialog( None, style=wx.FD_SAVE, defaultFile = filename ) as dlg:
if dlg.ShowModal() == wx.ID_OK:
body = os.linesep.join( [ 'r' + key.encode( 'hex' ) for key in registration_keys ] )
with open( dlg.GetPath(), 'wb' ) as f: f.write( body )
finally: self.EndModal( wx.ID_OK )
@ -2582,7 +2598,9 @@ class DialogModifyAccounts( Dialog ):
( subject_identifier, ) = self._subject_identifiers
subject_string = connection.Get( 'account_info', subject_identifier = subject_identifier )
response = connection.Get( 'account_info', subject_identifier = subject_identifier )
subject_string = HC.u( response[ 'account_info' ] )
else: subject_string = 'modifying ' + HC.ConvertIntToPrettyString( len( self._subject_identifiers ) ) + ' accounts'
@ -2590,7 +2608,9 @@ class DialogModifyAccounts( Dialog ):
#
account_types = connection.Get( 'account_types' )
response = connection.Get( 'account_types' )
account_types = response[ 'account_types' ]
for account_type in account_types: self._account_types.Append( account_type.ConvertToString(), account_type )
@ -2682,13 +2702,17 @@ class DialogModifyAccounts( Dialog ):
kwargs[ 'subject_identifiers' ] = list( self._subject_identifiers )
kwargs[ 'action' ] = action
connection.Post( 'account_modification', **kwargs )
connection.Post( 'account', **kwargs )
if len( self._subject_identifiers ) == 1:
( subject_identifier, ) = self._subject_identifiers
self._subject_text.SetLabel( HC.u( connection.Get( 'account_info', subject_identifier = subject_identifier ) ) )
response = connection.Get( 'account_info', subject_identifier = subject_identifier )
account_info = response[ 'account_info' ]
self._subject_text.SetLabel( HC.u( account_info ) )
if len( self._subject_identifiers ) > 1: wx.MessageBox( 'Done!' )
@ -3550,9 +3574,11 @@ class DialogRegisterService( Dialog ):
headers = {}
headers[ 'Authorization' ] = 'hydrus_network ' + registration_key.encode( 'hex' )
headers[ 'Hydrus-Key' ] = registration_key.encode( 'hex' )
access_key = connection.request( 'GET', '/access_key', headers = headers )
response = connection.request( 'GET', '/access_key', headers = headers )
access_key = response[ 'access_key' ]
self._credentials = CC.Credentials( host, port, access_key )

142
include/ClientGUIDialogsManage.py
View File

@ -218,7 +218,9 @@ class DialogManageAccountTypes( ClientGUIDialogs.Dialog ):
connection = service.GetConnection()
account_types = connection.Get( 'account_types' )
response = connection.Get( 'account_types' )
account_types = response[ 'account_types' ]
self._titles_to_account_types = {}
@ -394,7 +396,7 @@ class DialogManageAccountTypes( ClientGUIDialogs.Dialog ):
connection = service.GetConnection()
connection.Post( 'account_types_modification', edit_log = self._edit_log )
connection.Post( 'account_types', edit_log = self._edit_log )
self.EndModal( wx.ID_OK )
@ -2453,7 +2455,9 @@ class DialogManageOptionsFileRepository( ClientGUIDialogs.Dialog ):
connection = self._service.GetConnection()
options = connection.Get( 'options' )
response = connection.Get( 'options' )
options = response[ 'options' ]
self._max_monthly_data.SetValue( options[ 'max_monthly_data' ] )
self._max_storage.SetValue( options[ 'max_storage' ] )
@ -3446,7 +3450,9 @@ class DialogManageOptionsServerAdmin( ClientGUIDialogs.Dialog ):
connection = self._service.GetConnection()
options = connection.Get( 'options' )
response = connection.Get( 'options' )
options = response[ 'options' ]
self._max_monthly_data.SetValue( options[ 'max_monthly_data' ] )
self._max_storage.SetValue( options[ 'max_storage' ] )
@ -3547,7 +3553,9 @@ class DialogManageOptionsTagRepository( ClientGUIDialogs.Dialog ):
connection = self._service.GetConnection()
options = connection.Get( 'options' )
response = connection.Get( 'options' )
options = response[ 'options' ]
self._max_monthly_data.SetValue( options[ 'max_monthly_data' ] )
@ -3817,7 +3825,7 @@ class DialogManageRatings( ClientGUIDialogs.Dialog ):
try:
service_identfiers_to_content_updates = {}
service_identifiers_to_content_updates = {}
for panel in self._panels:
@ -3825,11 +3833,11 @@ class DialogManageRatings( ClientGUIDialogs.Dialog ):
( service_identifier, content_updates ) = panel.GetContentUpdates()
service_identfiers_to_content_updates[ service_identifier ] = content_updates
service_identifiers_to_content_updates[ service_identifier ] = content_updates
HC.app.Write( 'content_updates', service_identfiers_to_content_updates )
HC.app.Write( 'content_updates', service_identifiers_to_content_updates )
except Exception as e: wx.MessageBox( 'Saving ratings changes to DB raised this error: ' + HC.u( e ) )
@ -4125,8 +4133,6 @@ class DialogManageServer( ClientGUIDialogs.Dialog ):
def InitialiseControls():
self._edit_log = []
self._services_listbook = ClientGUICommon.ListBook( self )
self._services_listbook.Bind( wx.EVT_NOTEBOOK_PAGE_CHANGED, self.EventServiceChanged )
self._services_listbook.Bind( wx.EVT_NOTEBOOK_PAGE_CHANGING, self.EventServiceChanging )
@ -4141,24 +4147,8 @@ class DialogManageServer( ClientGUIDialogs.Dialog ):
self._remove.Bind( wx.EVT_BUTTON, self.EventRemove )
self._remove.SetForegroundColour( ( 128, 0, 0 ) )
self._ok = wx.Button( self, label='ok' )
self._ok.Bind( wx.EVT_BUTTON, self.EventOK )
self._ok.SetForegroundColour( ( 0, 128, 0 ) )
self._cancel = wx.Button( self, id = wx.ID_CANCEL, label='cancel' )
self._cancel.Bind( wx.EVT_BUTTON, self.EventCancel )
self._cancel.SetForegroundColour( ( 128, 0, 0 ) )
# goes after self._remove, because of events
for service_identifier in self._service_identifiers:
page = self._Panel( self._services_listbook, service_identifier )
name = HC.service_string_lookup[ service_identifier.GetType() ]
self._services_listbook.AddPage( page, name )
self._done = wx.Button( self, label='done' )
self._done.Bind( wx.EVT_BUTTON, self.EventDone )
def PopulateControls():
@ -4167,6 +4157,21 @@ class DialogManageServer( ClientGUIDialogs.Dialog ):
self._service_types.SetSelection( 0 )
connection = self._service.GetConnection()
response = connection.Get( 'services' )
services_info = response[ 'services_info' ]
for ( service_identifier, options ) in services_info:
page = self._Panel( self._services_listbook, service_identifier, options )
name = HC.service_string_lookup[ service_identifier.GetType() ]
self._services_listbook.AddPage( page, name )
def ArrangeControls():
@ -4175,14 +4180,10 @@ class DialogManageServer( ClientGUIDialogs.Dialog ):
add_remove_hbox.AddF( self._add, FLAGS_MIXED )
add_remove_hbox.AddF( self._remove, FLAGS_MIXED )
ok_hbox = wx.BoxSizer( wx.HORIZONTAL )
ok_hbox.AddF( self._ok, FLAGS_MIXED )
ok_hbox.AddF( self._cancel, FLAGS_MIXED )
vbox = wx.BoxSizer( wx.VERTICAL )
vbox.AddF( self._services_listbook, FLAGS_EXPAND_BOTH_WAYS )
vbox.AddF( add_remove_hbox, FLAGS_SMALL_INDENT )
vbox.AddF( ok_hbox, FLAGS_BUTTON_SIZERS )
vbox.AddF( self._done, FLAGS_LONE_BUTTON )
self.SetSizer( vbox )
@ -4199,7 +4200,9 @@ class DialogManageServer( ClientGUIDialogs.Dialog ):
connection = self._service.GetConnection()
self._service_identifiers = connection.Get( 'services' )
response = connection.Get( 'services' )
self._service_identifiers = response[ 'service_identifiers' ]
InitialiseControls()
@ -4221,11 +4224,11 @@ class DialogManageServer( ClientGUIDialogs.Dialog ):
if service_panel is not None:
port = service_panel.GetInfo()
( service_identifier, options ) = service_panel.GetInfo()
for existing_port in [ page.GetInfo() for page in self._services_listbook.GetNameToPageDict().values() if page != service_panel ]:
for ( existing_service_identifier, existing_options ) in [ page.GetInfo() for page in self._services_listbook.GetNameToPageDict().values() if page != service_panel ]:
if port == existing_port: raise Exception( 'That port is already in use!' )
if options[ 'port' ] == existing_options[ 'port' ]: raise Exception( 'That port is already in use!' )
@ -4236,26 +4239,23 @@ class DialogManageServer( ClientGUIDialogs.Dialog ):
service_type = self._service_types.GetClientData( self._service_types.GetSelection() )
existing_ports = [ page.GetInfo() for page in self._services_listbook.GetNameToPageDict().values() ]
connection = self._service.GetConnection()
port = HC.DEFAULT_SERVICE_PORT
response = connection.Post( 'services', action = 'add', data = service_type )
while port in existing_ports: port += 1
service_identifier = response[ 'service_identifier' ]
options = response[ 'options' ]
service_identifier = HC.ServerServiceIdentifier( service_type, port )
# commit to local db now
self._edit_log.append( ( HC.ADD, service_identifier ) )
page = self._Panel( self._services_listbook, service_identifier )
page = self._Panel( self._services_listbook, service_identifier, options )
name = HC.service_string_lookup[ service_type ]
self._services_listbook.AddPage( page, name, select = True )
def EventCancel( self, event ): self.EndModal( wx.ID_CANCEL )
def EventOK( self, event ):
def EventDone( self, event ):
try: self._CheckCurrentServiceIsValid()
except Exception as e:
@ -4267,22 +4267,9 @@ class DialogManageServer( ClientGUIDialogs.Dialog ):
for page in self._services_listbook.GetNameToPageDict().values():
if page.HasChanges(): self._edit_log.append( ( HC.EDIT, ( page.GetOriginalServiceIdentifier(), page.GetInfo() ) ) )
if page.HasChanges(): page.CommitChanges()
try:
if len( self._edit_log ) > 0:
connection = self._service.GetConnection()
connection.Post( 'services_modification', edit_log = self._edit_log )
HC.app.Write( 'update_server_services', self._service.GetServiceIdentifier(), self._edit_log )
except Exception as e: wx.MessageBox( 'Saving services to server raised this error: ' + HC.u( e ) )
self.EndModal( wx.ID_OK )
@ -4292,9 +4279,11 @@ class DialogManageServer( ClientGUIDialogs.Dialog ):
if service_panel is not None:
service_identifier = service_panel.GetOriginalServiceIdentifier()
service_identifier = service_panel.GetServiceIdentifier()
self._edit_log.append( ( HC.DELETE, service_identifier ) )
connection = self._service.GetConnection()
response = connection.Post( 'services', action = 'delete', data = service_identifier )
self._services_listbook.DeleteCurrentPage()
@ -4321,18 +4310,23 @@ class DialogManageServer( ClientGUIDialogs.Dialog ):
class _Panel( wx.Panel ):
def __init__( self, parent, service_identifier ):
def __init__( self, parent, service_identifier, options ):
wx.Panel.__init__( self, parent )
self._service_identifier = service_identifier
self._options = options
def InitialiseControls():
self._service_panel = ClientGUICommon.StaticBox( self, 'service' )
# all possible options here. hide as appropriate in arrange
self._service_port = wx.SpinCtrl( self._service_panel, min = 1, max = 65535 )
# a button to commit options, should enable when changes
def PopulateControls():
@ -4366,6 +4360,20 @@ class DialogManageServer( ClientGUIDialogs.Dialog ):
ArrangeControls()
def EventUploadOptions( self, event ):
# create service/connection to service/whatever
connection.Post( 'options', options = options )
pass
def CommitChanges( self ):
pass
def GetInfo( self ):
port = self._service_port.GetValue()
@ -6866,16 +6874,16 @@ class DialogManageTags( ClientGUIDialogs.Dialog ):
try:
service_identfiers_to_content_updates = {}
service_identifiers_to_content_updates = {}
for page in self._tag_repositories.GetNameToPageDict().values():
( service_identifier, content_updates ) = page.GetContentUpdates()
service_identfiers_to_content_updates[ service_identifier ] = content_updates
service_identifiers_to_content_updates[ service_identifier ] = content_updates
if len( service_identfiers_to_content_updates ) > 0: HC.app.Write( 'content_updates', service_identfiers_to_content_updates )
if len( service_identifiers_to_content_updates ) > 0: HC.app.Write( 'content_updates', service_identifiers_to_content_updates )
except Exception as e: wx.MessageBox( 'Saving mapping changes to DB raised this error: ' + HC.u( e ) )

8
include/ClientGUIManagement.py
View File

@ -2604,7 +2604,9 @@ class ManagementPanelPetitions( ManagementPanel ):
connection = self._service.GetConnection()
self._current_petition = connection.Get( 'petition' )
response = connection.Get( 'petition' )
self._current_petition = response[ 'petition' ]
self._DrawCurrentPetition()
@ -2631,7 +2633,9 @@ class ManagementPanelPetitions( ManagementPanel ):
connection = self._service.GetConnection()
self._num_petitions = connection.Get( 'num_petitions' )
response = connection.Get( 'num_petitions' )
self._num_petitions = response[ 'num_petitions' ]
self._DrawNumPetitions()

64
include/HydrusConstants.py
View File

@ -37,8 +37,8 @@ TEMP_DIR = BASE_DIR + os.path.sep + 'temp'
# Misc
NETWORK_VERSION = 10
SOFTWARE_VERSION = 86
NETWORK_VERSION = 11
SOFTWARE_VERSION = 87
UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )
@ -127,7 +127,15 @@ service_string_lookup[ TAG_REPOSITORY ] = 'hydrus tag repository'
service_string_lookup[ FILE_REPOSITORY ] = 'hydrus file repository'
service_string_lookup[ LOCAL_FILE ] = 'hydrus local file service'
service_string_lookup[ MESSAGE_DEPOT ] = 'hydrus message depot'
service_string_lookup[ LOCAL_TAG ] = 'local tag service'
service_string_lookup[ LOCAL_RATING_NUMERICAL ] = 'local numerical rating service'
service_string_lookup[ LOCAL_RATING_LIKE ] = 'local like/dislike rating service'
service_string_lookup[ RATING_NUMERICAL_REPOSITORY ] = 'hydrus numerical rating repository'
service_string_lookup[ RATING_LIKE_REPOSITORY ] = 'hydrus like/dislike rating repository'
service_string_lookup[ COMBINED_TAG ] = 'virtual combined tag service'
service_string_lookup[ COMBINED_FILE ] = 'virtual combined file service'
service_string_lookup[ SERVER_ADMIN ] = 'hydrus server administration'
service_string_lookup[ NULL_SERVICE ] = 'null service'
RATINGS_SERVICES = [ LOCAL_RATING_LIKE, LOCAL_RATING_NUMERICAL, RATING_LIKE_REPOSITORY, RATING_NUMERICAL_REPOSITORY ]
REPOSITORIES = [ TAG_REPOSITORY, FILE_REPOSITORY, RATING_LIKE_REPOSITORY, RATING_NUMERICAL_REPOSITORY ]
@ -453,15 +461,15 @@ restricted_requests.append( ( GET, 'options', GENERAL_ADMIN ) )
restricted_requests.append( ( GET, 'registration_keys', GENERAL_ADMIN ) )
restricted_requests.append( ( GET, 'session_key', None ) )
restricted_requests.append( ( GET, 'stats', GENERAL_ADMIN ) )
restricted_requests.append( ( POST, 'account_modification', ( MANAGE_USERS, GENERAL_ADMIN ) ) )
restricted_requests.append( ( POST, 'account_types_modification', GENERAL_ADMIN ) )
restricted_requests.append( ( POST, 'account', ( MANAGE_USERS, GENERAL_ADMIN ) ) )
restricted_requests.append( ( POST, 'account_types', GENERAL_ADMIN ) )
restricted_requests.append( ( POST, 'options', GENERAL_ADMIN ) )
admin_requests = list( restricted_requests )
admin_requests.append( ( GET, 'init', None ) )
admin_requests.append( ( GET, 'services', EDIT_SERVICES ) )
admin_requests.append( ( POST, 'backup', EDIT_SERVICES ) )
admin_requests.append( ( POST, 'services_modification', EDIT_SERVICES ) )
admin_requests.append( ( POST, 'services', EDIT_SERVICES ) )
repository_requests = list( restricted_requests )
repository_requests.append( ( GET, 'num_petitions', RESOLVE_PETITIONS ) )
@ -984,7 +992,8 @@ def ShowExceptionDefault( e ):
message = u( etype.__name__ ) + ': ' + u( value ) + os.linesep + u( trace )
print( repr( message ) )
try: print( message )
except: print( repr( message ) )
ShowException = ShowExceptionDefault
@ -1431,9 +1440,7 @@ class Account( HydrusYAMLBase ):
else: return GetNow() > self._expires
def CheckPermissions( self, permissions ):
if type( permissions ) == int: permissions = ( permissions, )
def CheckPermission( self, permission ):
if self._IsBanned(): raise HydrusExceptions.PermissionException( 'This account is banned!' )
@ -1443,11 +1450,11 @@ class Account( HydrusYAMLBase ):
( used_bytes, used_requests ) = self._used_data
if max_num_bytes is not None and used_bytes > max_num_bytes: raise HydrusExceptions.PermissionException( 'You have hit your data transfer limit (' + ConvertIntToBytes( max_num_bytes ) + '), and cannot download any more for the month.' )
if max_num_bytes is not None and used_bytes > max_num_bytes: raise HydrusExceptions.PermissionException( 'You have hit your data transfer limit (' + ConvertIntToBytes( max_num_bytes ) + '), and cannot make any more requests for the month.' )
if max_num_requests is not None and used_requests > max_num_requests: raise HydrusExceptions.PermissionException( 'You have hit your requests limit (' + ConvertIntToPrettyString( max_num_requests ) + '), and cannot download any more for the month.' )
if max_num_requests is not None and used_requests > max_num_requests: raise HydrusExceptions.PermissionException( 'You have hit your requests limit (' + ConvertIntToPrettyString( max_num_requests ) + '), and cannot make any more requests for the month.' )
if len( permissions ) > 0 and True not in [ self._account_type.HasPermission( permission ) for permission in permissions ]: raise HydrusExceptions.PermissionException( 'You do not have permission to do that.' )
if not self._account_type.HasPermission( permission ): raise HydrusExceptions.PermissionException( 'You do not have permission to do that.' )
def ConvertToString( self ): return ConvertTimestampToPrettyAge( self._created ) + os.linesep + self._account_type.ConvertToString( self._used_data ) + os.linesep + 'which '+ ConvertTimestampToPrettyExpires( self._expires )
@ -1497,6 +1504,8 @@ class Account( HydrusYAMLBase ):
def HasPermission( self, permission ):
if self._IsBanned(): return False
if self._IsExpired(): return False
( max_num_bytes, max_num_requests ) = self._account_type.GetMaxMonthlyData()
@ -1636,7 +1645,7 @@ class ClientServiceIdentifier( HydrusYAMLBase ):
def __ne__( self, other ): return self.__hash__() != other.__hash__()
def __repr__( self ): return 'Client Service Identifier: ' + u( ( self._name, self._type, self._service_key ) )
def __repr__( self ): return 'Client Service Identifier: ' + u( ( self._name, service_string_lookup[ self._type ] ) )
def GetInfo( self ): return ( self._service_key, self._type, self._name )
@ -2256,53 +2265,58 @@ SYSTEM_PREDICATE_NOT_LOCAL = Predicate( PREDICATE_TYPE_SYSTEM, ( SYSTEM_PREDICAT
class ResponseContext():
def __init__( self, status_code, mime = None, body = '', filename = None, cookies = [] ):
def __init__( self, status_code, mime = APPLICATION_YAML, body = None, path = None, is_yaml = False, cookies = [] ):
self._status_code = status_code
self._mime = mime
self._body = body
self._filename = filename
self._path = path
self._is_yaml = is_yaml
self._cookies = cookies
def GetCookies( self ): return self._cookies
def GetFilename( self ): return self._filename
def GetLength( self ): return len( self._body )
def GetMimeBody( self ): return ( self._mime, self._body )
def GetPath( self ): return self._path
def GetStatusCode( self ): return self._status_code
def HasBody( self ): return self._body != ''
def HasBody( self ): return self._body is not None
def HasPath( self ): return self._path is not None
def IsYAML( self ): return self._is_yaml
def HasFilename( self ): return self._filename is not None
class ServerServiceIdentifier( HydrusYAMLBase ):
yaml_tag = u'!ServerServiceIdentifier'
def __init__( self, type, port ):
def __init__( self, service_key, type ):
HydrusYAMLBase.__init__( self )
self._service_key = service_key
self._type = type
self._port = port
def __eq__( self, other ): return self.__hash__() == other.__hash__()
def __hash__( self ): return ( self._type, self._port ).__hash__()
def __hash__( self ): return self._service_key.__hash__()
def __ne__( self, other ): return self.__hash__() != other.__hash__()
def __repr__( self ): return 'Server Service Identifier: ' + u( ( self._type, self._port ) )
def __repr__( self ): return 'Server Service Identifier: ' + service_string_lookup[ self._type ]
def GetPort( self ): return self._port
def GetServiceKey( self ): return self._service_key
def GetType( self ): return self._type
SERVER_ADMIN_IDENTIFIER = ServerServiceIdentifier( 'server admin', SERVER_ADMIN )
class ServiceUpdate():
def __init__( self, action, row = None ):

710
include/HydrusServer.py
View File

@ -10,17 +10,21 @@ import HydrusExceptions
import HydrusFileHandling
import HydrusFlashHandling
import HydrusImageHandling
import HydrusServerResources
import HydrusVideoHandling
import os
import random
import ServerConstants as SC
import SocketServer
import traceback
import urllib
import wx
import yaml
#from twisted.web.server import Site
#from twisted.web.resource import Resource
#from twisted.web.static import File as FileResource
from twisted.internet import reactor, defer
from twisted.internet.threads import deferToThread
from twisted.web.server import Request, Site, NOT_DONE_YET
from twisted.web.resource import Resource
from twisted.web.static import File as FileResource, NoRangeStaticProducer
eris = '''<html><head><title>hydrus</title></head><body><pre>
<font color="red">8888 8888888</font>
@ -226,384 +230,7 @@ ROOT_MESSAGE_BEGIN = '''<html>
ROOT_MESSAGE_END = '''</p>
</body>
</html>'''
def ParseAccessKey( authorisation_text ):
if authorisation_text is None: access_key = None
else:
( format, access_key_encoded ) = authorisation_text.split( ' ' )
if format != 'hydrus_network': raise HydrusExceptions.ForbiddenException( 'Authorisation format error!' )
try: access_key = access_key_encoded.decode( 'hex' )
except: raise HydrusExceptions.ForbiddenException( 'Attempted to parse access key, but could not understand it.' )
return access_key
def ParseFileArguments( path ):
HydrusImageHandling.ConvertToPngIfBmp( path )
hash = HydrusFileHandling.GetHashFromPath( path )
try: ( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path, hash )
except HydrusExceptions.SizeException: raise HydrusExceptions.ForbiddenException( 'File is of zero length!' )
except HydrusExceptions.MimeException: raise HydrusExceptions.ForbiddenException( 'Filetype is not permitted!' )
except Exception as e: raise HydrusExceptions.ForbiddenException( HC.u( e ) )
args = {}
args[ 'path' ] = path
args[ 'hash' ] = hash
args[ 'size' ] = size
args[ 'mime' ] = mime
if width is not None: args[ 'width' ] = width
if height is not None: args[ 'height' ] = height
if duration is not None: args[ 'duration' ] = duration
if num_frames is not None: args[ 'num_frames' ] = num_frames
if num_words is not None: args[ 'num_words' ] = num_words
if mime in HC.IMAGES:
try: thumbnail = HydrusImageHandling.GenerateThumbnail( path )
except: raise HydrusExceptions.ForbiddenException( 'Could not generate thumbnail from that file.' )
args[ 'thumbnail' ] = thumbnail
return args
def ParseHTTPGETArguments( path ):
path = urllib.unquote( path )
arguments = {}
if '?' in path:
raw_arguments = path.split( '?', 1 )[1]
for raw_argument in raw_arguments.split( '&' ):
if '=' in raw_argument:
[ name, value ] = raw_argument.split( '=', 1 )
if name in ( 'begin', 'num', 'expiration', 'subject_account_id', 'service_type', 'service_port', 'since' ):
try: arguments[ name ] = int( value )
except: raise HydrusExceptions.ForbiddenException( 'I was expecting to parse \'' + name + '\' as an integer, but it failed.' )
elif name in ( 'access_key', 'title', 'subject_access_key', 'contact_key', 'hash', 'subject_hash', 'subject_tag', 'message_key' ):
try: arguments[ name ] = value.decode( 'hex' )
except: raise HydrusExceptions.ForbiddenException( 'I was expecting to parse \'' + name + '\' as a hex-encoded string, but it failed.' )
if 'subject_account_id' in arguments: arguments[ 'subject_identifier' ] = HC.AccountIdentifier( access_key = arguments[ 'subject_account_id' ] )
elif 'subject_access_key' in arguments: arguments[ 'subject_identifier' ] = HC.AccountIdentifier( access_key = arguments[ 'subject_access_key' ] )
elif 'subject_tag' in arguments and 'subject_hash' in arguments: arguments[ 'subject_identifier' ] = HC.AccountIdentifier( tag = arguments[ 'subject_tag' ], hash = arguments[ 'subject_hash' ] )
elif 'subject_hash' in arguments: arguments[ 'subject_identifier' ] = HC.AccountIdentifier( hash = arguments[ 'subject_hash' ] )
return arguments
def ParseHTTPPOSTArguments( body ):
if body == '': args = ()
else: args = yaml.safe_load( body )
return args
def ParseHTTPRequest( path ):
path = urllib.unquote( path )
if not path.startswith( '/' ): return ''
after_slash = path.split( '/', 1 )[1]
return after_slash.split( '?', 1 )[0]
def ParseSessionKey( cookie_text ):
if cookie_text is None: session_key = None
else:
try:
cookies = Cookie.SimpleCookie( cookie_text )
if 'session_key' not in cookies: session_key = None
else: session_key = cookies[ 'session_key' ].value.decode( 'hex' )
except: raise Exception( 'Problem parsing cookie!' )
return session_key
class HydrusHTTPServer( SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer ):
def __init__( self, service_identifier, message = '' ):
#self.daemon_threads = True
self._service_identifier = service_identifier
self._message = message
port = service_identifier.GetPort()
BaseHTTPServer.HTTPServer.__init__( self, ( '', port ), HydrusHTTPRequestHandler )
HC.pubsub.sub( self, 'shutdown', 'shutdown' )
def GetServiceIdentifier( self ): return self._service_identifier
def GetMessage( self ): return self._message
def SetMessage( self, message ): self._message = message
class HydrusHTTPRequestHandler( BaseHTTPServer.BaseHTTPRequestHandler ):
server_version = 'hydrus/' + HC.u( HC.NETWORK_VERSION )
protocol_version = 'HTTP/1.1'
with open( HC.STATIC_DIR + os.path.sep + 'hydrus.ico', 'rb' ) as f: _favicon = f.read()
def __init__( self, request, client_address, server ):
self._service_identifier = server.GetServiceIdentifier()
try: BaseHTTPServer.BaseHTTPRequestHandler.__init__( self, request, client_address, server )
except: self.log_string( 'Connection reset by peer' )
def _ProcessRequest( self, request_type ):
try:
try:
default_mime = HC.TEXT_HTML
default_encoding = lambda x: HC.u( x )
user_agent_text = self.headers.getheader( 'User-Agent' )
if user_agent_text is not None:
user_agents = user_agent_text.split( ' ' )
for user_agent in user_agents:
if '/' in user_agent:
( client, network_version ) = user_agent.split( '/', 1 )
if client == 'hydrus':
default_mime = HC.APPLICATION_YAML
default_encoding = lambda x: yaml.safe_dump( HC.u( x ) )
network_version = int( network_version )
if network_version != HC.NETWORK_VERSION:
if network_version < HC.NETWORK_VERSION: message = 'Please download the latest release.'
else: message = 'Please ask this server\'s admin to update to the latest release.'
raise HydrusExceptions.NetworkVersionException( 'Network version mismatch! This server\'s network version is ' + HC.u( HC.NETWORK_VERSION ) + ', whereas your client\'s is ' + HC.u( network_version ) + '! ' + message )
( ip, port ) = self.client_address
service_type = self._service_identifier.GetType()
if service_type == HC.LOCAL_FILE and ip != '127.0.0.1': raise HydrusExceptions.ForbiddenException( 'Only local access allowed!' )
request = ParseHTTPRequest( self.path )
if ( service_type, request_type, request ) not in HC.ALLOWED_REQUESTS: raise HydrusExceptions.ForbiddenException( 'This service does not support that request.' )
if request_type == HC.GET:
request_args = ParseHTTPGETArguments( self.path )
request_length = 0
elif request_type == HC.POST:
content_length = int( self.headers.getheader( 'Content-Length', default = 0 ) )
request_length = content_length
# can do customised test here for max file allowed or whatever
if ( request == 'file' and content_length > 100 * 1024 * 1024 ) or ( request != 'file' and content_length > 10 * 1024 * 1024 ): raise HydrusExceptions.ForbiddenException( 'That request is too large!' )
if request == 'file':
path = HC.GetTempPath()
num_bytes_still_to_write = content_length
block_size = 65536
next_block_size = min( block_size, num_bytes_still_to_write )
next_block = self.rfile.read( next_block_size )
num_bytes_still_to_write -= next_block_size
with open( path, 'wb' ) as f:
while True:
f.write( next_block )
if num_bytes_still_to_write == 0: break
next_block_size = min( block_size, num_bytes_still_to_write )
next_block = self.rfile.read( next_block_size )
num_bytes_still_to_write -= next_block_size
request_args = ParseFileArguments( path )
else:
body = self.rfile.read( content_length )
request_args = ParseHTTPPOSTArguments( body )
if request == '':
if service_type == HC.LOCAL_FILE: body = CLIENT_ROOT_MESSAGE
else:
message = self.server.GetMessage()
body = ROOT_MESSAGE_BEGIN + message + ROOT_MESSAGE_END
response_context = HC.ResponseContext( 200, mime = HC.TEXT_HTML, body = body )
elif request == 'favicon.ico': response_context = HC.ResponseContext( 200, mime = HC.IMAGE_ICON, body = self._favicon, filename = 'favicon.ico' )
else:
if service_type == HC.LOCAL_FILE:
if request_type == HC.GET:
if request == 'file':
hash = request_args[ 'hash' ]
path = CC.GetFilePath( hash )
mime = HydrusFileHandling.GetMime( path )
with open( path, 'rb' ) as f: file = f.read()
response_context = HC.ResponseContext( 200, mime = mime, body = file, filename = hash.encode( 'hex' ) + HC.mime_ext_lookup[ mime ] )
elif request == 'thumbnail':
hash = request_args[ 'hash' ]
path = CC.GetThumbnailPath( hash, True )
mime = HydrusFileHandling.GetMime( path )
with open( path, 'rb' ) as f: thumbnail = f.read()
response_context = HC.ResponseContext( 200, mime = mime, body = thumbnail, filename = hash.encode( 'hex' ) + '_thumbnail' + HC.mime_ext_lookup[ mime ] )
elif request_type == HC.POST: pass # nothing here yet!
else:
session_key = ParseSessionKey( self.headers.getheader( 'Cookie' ) )
access_key = ParseAccessKey( self.headers.getheader( 'Authorization' ) )
response_context = HC.app.GetDB().AddJobServer( self._service_identifier, access_key, session_key, ip, request_type, request, request_args, request_length )
except:
# wx.MessageBox( traceback.format_exc() )
raise
except KeyError: response_context = HC.ResponseContext( 403, mime = default_mime, body = default_encoding( 'It appears one or more parameters required for that request were missing.' ) )
except HydrusExceptions.PermissionException as e: response_context = HC.ResponseContext( 401, mime = default_mime, body = default_encoding( e ) )
except HydrusExceptions.ForbiddenException as e: response_context = HC.ResponseContext( 403, mime = default_mime, body = default_encoding( e ) )
except HydrusExceptions.NotFoundException as e: response_context = HC.ResponseContext( 404, mime = default_mime, body = default_encoding( e ) )
except HydrusExceptions.NetworkVersionException as e: response_context = HC.ResponseContext( 426, mime = default_mime, body = default_encoding( e ) )
except HydrusExceptions.SessionException as e: response_context = HC.ResponseContext( 403, mime = default_mime, body = default_encoding( 'Session not found!' ) )
except Exception as e:
self.log_string( traceback.format_exc() )
response_context = HC.ResponseContext( 500, mime = default_mime, body = default_encoding( 'The repository encountered an error it could not handle! Here is a dump of what happened, which will also be written to your client.log file. If it persists, please forward it to hydrus.admin@gmail.com:' + os.linesep + os.linesep + traceback.format_exc() ) )
try: self._Respond( response_context )
except: pass # wx.MessageBox( traceback.format_exc() )
def _Respond( self, response_context ):
status_code = response_context.GetStatusCode()
self.send_response( status_code )
for cookie in response_context.GetCookies(): self.send_header( 'Set-Cookie', cookie )
if response_context.HasBody():
( mime, body ) = response_context.GetMimeBody()
content_type = HC.mime_string_lookup[ mime ]
if response_context.HasFilename():
filename = response_context.GetFilename()
content_type += '; ' + filename
self.send_header( 'Content-Type', content_type )
self.send_header( 'Content-Length', len( body ) )
self.end_headers()
self.wfile.write( body )
else:
self.send_header( 'Content-Length', '0' )
self.end_headers()
def do_GET( self ): self._ProcessRequest( HC.GET )
'''
def do_OPTIONS( self ):
service_type = self._service_identifier.GetType()
@ -622,189 +249,26 @@ class HydrusHTTPRequestHandler( BaseHTTPServer.BaseHTTPRequestHandler ):
self.send_header( 'Allow', ','.join( allowed ) )
self.end_headers()
'''
class HydrusRequest( Request ):
def do_POST( self ): self._ProcessRequest( HC.POST )
def log_message( self, format, *args ): print( "[%s] %s%s" % ( self.log_date_time_string(), format%args, os.linesep ) )
def log_request( self, *args ): pass
def log_string( self, message ): print( repr( message ) )
# this overrides the base method to no longer use the class variable server_version
def version_string( self ):
def __init__( self, *args, **kwargs ):
service_type = self._service_identifier.GetType()
Request.__init__( self, *args, **kwargs )
server_version = HC.service_string_lookup[ service_type ] + '/' + HC.u( HC.NETWORK_VERSION )
return server_version + ' ' + self.sys_version
#
'''
hydrus_favicon = FileResource( HC.STATIC_DIR + os.path.sep + 'hydrus.ico', defaultType = HC.IMAGE_ICON )
class HydrusResourceWelcome( Resource ):
def __init__( self, service_identifier, message ):
Resource.__init__( self )
if service_identifier.GetType() == HC.LOCAL_FILE: body = CLIENT_ROOT_MESSAGE
else: body = ROOT_MESSAGE_BEGIN + message + ROOT_MESSAGE_END
self._body = body.encode( 'utf-8' )
def render_GET( self, request ): return self._body
class HydrusResourceMakeSession( Resource ):
def render_GET( self, request ):
# parse key from authorisation header
# go to db to fetch account, error as appropriate
# 200 OK with session key cookie
pass
self.is_hydrus_client = True
self.hydrus_args = None
self.hydrus_response_context = None
self.hydrus_request_data_usage = 0
class HydrusResourceCommand( Resource ):
class HydrusRequestRestricted( HydrusRequest ):
def __init__( self, local_only ):
def __init__( self, *args, **kwargs ):
Resource.__init__( self )
HydrusRequest.__init__( self, *args, **kwargs )
# set default mime, which is like self.defaultContentType
self._local_only = local_only
def _checkLocal( self, request ):
if self._local_only and request.getClientIP() != '127.0.0.1': raise HydrusExceptions.ForbiddenException( 'Only local access allowed!' )
def _checkRestrictions( self, request ):
self._checkLocal( request )
def _renderResponseContext( self, response_context, request ):
status_code = response_context.GetStatusCode()
request.setResponseCode( status_code )
# this is wrong; addcookie takes k, v
# should move to sessions anyway
for cookie in response_context.GetCookies(): request.addCookie( cookie )
if response_context.HasBody():
( mime, body ) = response_context.GetMimeBody()
content_type = HC.mime_string_lookup[ mime ]
if response_context.HasFilename():
filename = response_context.GetFilename()
content_type += '; ' + filename
request.setHeader( 'Content-Type', content_type )
request.setHeader( 'Content-Length', len( body ) )
return body
else: return ''
def render_GET( self, request ):
try:
self._checkRestrictions( request )
args = FilterGETArgs( request.args )
# WAIT, NO. MAKE EVERY REQUEST DEFERRED. THAT'S 10000% easier
# d = reactor.deferToThread or something
# d.addCallBack( self._renderResponseContext )
# and maybe an errback too, or the same one! whatever!
# return NOT_READY or whatever
except: pass
class HydrusResourceCommandRestricted( HydrusResourceCommand ):
GET_PERMISSION = HC.GENERAL_ADMIN
POST_PERMISSION = HC.GENERAL_ADMIN
def _checkPermission( self, method, request ):
# get session
# get account
# test account for that permission
# depending on method, use class variable GET_PERMISSION or POST_PERMISSION
# fail with a no session found
# fail with you do not have that permission
pass
def _checkRestrictions( self, request ):
HydrusResourceCommand._checkRestrictions( self, request )
self._checkGETPermission( request )
def _doDatabase( self, args ): pass
def _databaseDone( self, response_context ):
# after a deferred, can I set the response code?
pass # renderresponse
class HydrusResourceLocalFile( Resource ):
def render_GET( self, request ):
if request.getClientIP() != '127.0.0.1':
request.setResponseCode( 403 )
return 'nope'
else:
try:
hash = request.args[ 'hash' ][0]
print( 'ok')
# test hash is ok
# test os.path.exists
# return a fileresource
return hydrus_favicon
except:
print( traceback.format_exc())
return NoResource()
self.hydrus_account = None
class HydrusService( Site ):
@ -818,13 +282,15 @@ class HydrusService( Site ):
Site.__init__( self, root )
self.requestFactory = HydrusRequest
def _InitRoot( self ):
root = Resource()
root.putChild( '', HydrusResourceWelcome( self._service_identifier, self._message ) )
root.putChild( 'favicon.ico', hydrus_favicon )
root.putChild( '', HydrusServerResources.HydrusResourceWelcome( self._service_identifier, self._message ) )
root.putChild( 'favicon.ico', HydrusServerResources.hydrus_favicon )
return root
@ -835,95 +301,75 @@ class HydrusServiceLocal( HydrusService ):
root = HydrusService._InitRoot( self )
root.putChild( 'file', HydrusResourceLocalFile() )
root.putChild( 'file', HydrusServerResources.HydrusResourceCommandFileLocal( self._service_identifier ) )
root.putChild( 'thumbnail', HydrusServerResources.HydrusResourceCommandThumbnailLocal( self._service_identifier ) )
return root
class HydrusServiceRestricted( HydrusService ):
def makeSession( self ):
def __init__( self, service_identifier, message ):
uid = os.urandom( 32 ) # only change here is this hydrus-style uid
HydrusService.__init__( self, service_identifier, message )
session = self.sessions[uid] = self.sessionFactory(self, uid)
self.requestFactory = HydrusRequestRestricted
session.startCheckingExpiration()
def _InitRoot( self ):
return session
root = HydrusService._InitRoot( self )
root.putChild( 'access_key', HydrusServerResources.HydrusResourceCommandAccessKey( self._service_identifier ) )
root.putChild( 'session_key', HydrusServerResources.HydrusResourceCommandSessionKey( self._service_identifier ) )
root.putChild( 'account', HydrusServerResources.HydrusResourceCommandRestrictedAccount( self._service_identifier ) )
root.putChild( 'account_info', HydrusServerResources.HydrusResourceCommandRestrictedAccountInfo( self._service_identifier ) )
root.putChild( 'account_types', HydrusServerResources.HydrusResourceCommandRestrictedAccountTypes( self._service_identifier ) )
root.putChild( 'registration_keys', HydrusServerResources.HydrusResourceCommandRestrictedRegistrationKeys( self._service_identifier ) )
root.putChild( 'stats', HydrusServerResources.HydrusResourceCommandRestrictedStats( self._service_identifier ) )
return root
'''
'''
BANDWIDTH_CONSUMING_REQUESTS = set()
class HydrusServiceAdmin( HydrusServiceRestricted ):
BANDWIDTH_CONSUMING_REQUESTS.add( ( TAG_REPOSITORY, GET, 'update' ) )
BANDWIDTH_CONSUMING_REQUESTS.add( ( TAG_REPOSITORY, POST, 'mappings' ) )
BANDWIDTH_CONSUMING_REQUESTS.add( ( TAG_REPOSITORY, POST, 'petitions' ) )
BANDWIDTH_CONSUMING_REQUESTS.add( ( FILE_REPOSITORY, GET, 'update' ) )
BANDWIDTH_CONSUMING_REQUESTS.add( ( FILE_REPOSITORY, GET, 'file' ) )
BANDWIDTH_CONSUMING_REQUESTS.add( ( FILE_REPOSITORY, GET, 'thumbnail' ) )
BANDWIDTH_CONSUMING_REQUESTS.add( ( FILE_REPOSITORY, POST, 'file' ) )
BANDWIDTH_CONSUMING_REQUESTS.add( ( FILE_REPOSITORY, POST, 'petitions' ) )
def _InitRoot( self ):
root = HydrusServiceRestricted._InitRoot( self )
root.putChild( 'backup', HydrusServerResources.HydrusResourceCommandRestrictedBackup( self._service_identifier ) )
root.putChild( 'init', HydrusServerResources.HydrusResourceCommandInit( self._service_identifier ) )
root.putChild( 'services', HydrusServerResources.HydrusResourceCommandRestrictedAccountInfo( self._service_identifier ) )
return root
service_requests = []
service_requests.append( ( GET, '', None ) )
service_requests.append( ( GET, 'favicon.ico', None ) )
class HydrusServiceRepository( HydrusServiceRestricted ):
local_file_requests = list( service_requests )
local_file_requests.append( ( GET, 'file', None ) )
local_file_requests.append( ( GET, 'thumbnail', None ) )
def _InitRoot( self ):
root = HydrusServiceRestricted._InitRoot( self )
root.putChild( 'news', HydrusServerResources.HydrusResourceCommandRestrictedNews( self._service_identifier ) )
root.putChild( 'num_petitions', HydrusServerResources.HydrusResourceCommandRestrictedNumPetitions( self._service_identifier ) )
root.putChild( 'petition', HydrusServerResources.HydrusResourceCommandRestrictedPetition( self._service_identifier ) )
root.putChild( 'update', HydrusServerResources.HydrusResourceCommandRestrictedUpdate( self._service_identifier ) )
return root
restricted_requests = list( service_requests )
restricted_requests.append( ( GET, 'access_key', None ) )
restricted_requests.append( ( GET, 'account', None ) )
restricted_requests.append( ( GET, 'account_info', MANAGE_USERS ) )
restricted_requests.append( ( GET, 'account_types', MANAGE_USERS ) )
restricted_requests.append( ( GET, 'options', GENERAL_ADMIN ) )
restricted_requests.append( ( GET, 'registration_keys', GENERAL_ADMIN ) )
restricted_requests.append( ( GET, 'session_key', None ) )
restricted_requests.append( ( GET, 'stats', GENERAL_ADMIN ) )
restricted_requests.append( ( POST, 'account_modification', ( MANAGE_USERS, GENERAL_ADMIN ) ) )
restricted_requests.append( ( POST, 'account_types_modification', GENERAL_ADMIN ) )
restricted_requests.append( ( POST, 'options', GENERAL_ADMIN ) )
class HydrusServiceRepositoryFile( HydrusServiceRepository ):
admin_requests = list( restricted_requests )
admin_requests.append( ( GET, 'init', None ) )
admin_requests.append( ( GET, 'services', EDIT_SERVICES ) )
admin_requests.append( ( POST, 'backup', EDIT_SERVICES ) )
admin_requests.append( ( POST, 'services_modification', EDIT_SERVICES ) )
def _InitRoot( self ):
root = HydrusServiceRepository._InitRoot( self )
root.putChild( 'file', HydrusServerResources.HydrusResourceCommandRestrictedFileRepository( self._service_identifier ) )
root.putChild( 'ip', HydrusServerResources.HydrusResourceCommandRestrictedIP( self._service_identifier ) )
root.putChild( 'thumbnail', HydrusServerResources.HydrusResourceCommandRestrictedThumbnailRepository( self._service_identifier ) )
return root
repository_requests = list( restricted_requests )
repository_requests.append( ( GET, 'num_petitions', RESOLVE_PETITIONS ) )
repository_requests.append( ( GET, 'petition', RESOLVE_PETITIONS ) )
repository_requests.append( ( GET, 'update', GET_DATA ) )
repository_requests.append( ( POST, 'news', GENERAL_ADMIN ) )
repository_requests.append( ( POST, 'update', POST_DATA ) )
file_repository_requests = list( repository_requests )
file_repository_requests.append( ( GET, 'file', GET_DATA ) )
file_repository_requests.append( ( GET, 'ip', GENERAL_ADMIN ) )
file_repository_requests.append( ( GET, 'thumbnail', GET_DATA ) )
file_repository_requests.append( ( POST, 'file', POST_DATA ) )
tag_repository_requests = list( repository_requests )
message_depot_requests = list( restricted_requests )
message_depot_requests.append( ( GET, 'message', GET_DATA ) )
message_depot_requests.append( ( GET, 'message_info_since', GET_DATA ) )
message_depot_requests.append( ( GET, 'public_key', None ) )
message_depot_requests.append( ( POST, 'contact', POST_DATA ) )
message_depot_requests.append( ( POST, 'message', None ) )
message_depot_requests.append( ( POST, 'message_statuses', None ) )
all_requests = []
all_requests.extend( [ ( LOCAL_FILE, request_type, request, permissions ) for ( request_type, request, permissions ) in local_file_requests ] )
all_requests.extend( [ ( SERVER_ADMIN, request_type, request, permissions ) for ( request_type, request, permissions ) in admin_requests ] )
all_requests.extend( [ ( FILE_REPOSITORY, request_type, request, permissions ) for ( request_type, request, permissions ) in file_repository_requests ] )
all_requests.extend( [ ( TAG_REPOSITORY, request_type, request, permissions ) for ( request_type, request, permissions ) in tag_repository_requests ] )
all_requests.extend( [ ( MESSAGE_DEPOT, request_type, request, permissions ) for ( request_type, request, permissions ) in message_depot_requests ] )
ALLOWED_REQUESTS = { ( service_type, request_type, request ) for ( service_type, request_type, request, permissions ) in all_requests }
REQUESTS_TO_PERMISSIONS = { ( service_type, request_type, request ) : permissions for ( service_type, request_type, request, permissions ) in all_requests }
'''
class HydrusServiceRepositoryTag( HydrusServiceRepository ): pass

1116
include/HydrusServerResources.py Normal file
View File

File diff suppressed because it is too large Load Diff

33
include/HydrusSessions.py
View File

@ -13,6 +13,8 @@ import traceback
import wx
import yaml
HYDRUS_SESSION_EXPIRY_TIME = 30 * 86400
class HydrusSessionManagerClient():
def __init__( self ):
@ -61,7 +63,7 @@ class HydrusSessionManagerClient():
try: session_key = cookies[ 'session_key' ].decode( 'hex' )
except: raise Exception( 'Service did not return a session key!' )
expiry = now + 30 * 86400
expiry = now + HYDRUS_SESSION_EXPIRY_TIME
self._sessions[ service_identifier ] = ( session_key, expiry )
@ -75,21 +77,32 @@ class HydrusSessionManagerServer():
def __init__( self ):
existing_sessions = HC.app.GetDB().Read( 'sessions', HC.HIGH_PRIORITY )
existing_sessions = HC.app.Read( 'sessions' )
self._sessions = { ( session_key, service_identifier ) : ( account_identifier, expiry ) for ( session_key, service_identifier, account_identifier, expiry ) in existing_sessions }
self._sessions = { ( session_key, service_identifier ) : ( account, expiry ) for ( session_key, service_identifier, account, expiry ) in existing_sessions }
self._lock = threading.Lock()
def AddSession( self, session_key, service_identifier, account_identifier, expiry ):
def AddSession( self, service_identifier, account ):
HC.app.GetDB().Write( 'session', HC.HIGH_PRIORITY, session_key, service_identifier, account_identifier, expiry )
session_key = os.urandom( 32 )
self._sessions[ ( session_key, service_identifier ) ] = ( account_identifier, expiry )
now = HC.GetNow()
expiry = now + HYDRUS_SESSION_EXPIRY_TIME
HC.app.Write( 'session', session_key, service_identifier, account, expiry )
with self._lock:
self._sessions[ ( session_key, service_identifier ) ] = ( account, expiry )
return ( session_key, expiry )
def GetAccountIdentifier( self, session_key, service_identifier ):
def GetAccount( self, session_key, service_identifier ):
now = HC.GetNow()
@ -97,14 +110,12 @@ class HydrusSessionManagerServer():
if ( session_key, service_identifier ) in self._sessions:
( account_identifier, expiry ) = self._sessions[ ( session_key, service_identifier ) ]
( account, expiry ) = self._sessions[ ( session_key, service_identifier ) ]
if now > expiry: del self._sessions[ ( session_key, service_identifier ) ]
else: return account_identifier
else: return account
# session not found, or expired
raise HydrusExceptions.SessionException()

49
include/ServerConstants.py Normal file
View File

@ -0,0 +1,49 @@
import dircache
import HydrusConstants as HC
import HydrusExceptions
import itertools
import os
def GetExpectedPath( file_type, hash ):
if file_type == 'file': directory = HC.SERVER_FILES_DIR
elif file_type == 'thumbnail': directory = HC.SERVER_THUMBNAILS_DIR
elif file_type == 'update': directory = HC.SERVER_UPDATES_DIR
elif file_type == 'message': directory = HC.SERVER_MESSAGES_DIR
hash_encoded = hash.encode( 'hex' )
first_two_chars = hash_encoded[:2]
path = directory + os.path.sep + first_two_chars + os.path.sep + hash_encoded
return path
def GetPath( file_type, hash ):
path = GetExpectedPath( file_type, hash )
if not os.path.exists( path ): raise HydrusExceptions.NotFoundException( file_type + ' not found!' )
return path
def GetAllHashes( file_type ): return { os.path.split( path )[1].decode( 'hex' ) for path in IterateAllPaths( file_type ) }
def IterateAllPaths( file_type ):
if file_type == 'file': directory = HC.SERVER_FILES_DIR
elif file_type == 'thumbnail': directory = HC.SERVER_THUMBNAILS_DIR
elif file_type == 'update': directory = HC.SERVER_UPDATES_DIR
elif file_type == 'message': directory = HC.SERVER_MESSAGES_DIR
hex_chars = '0123456789abcdef'
for ( one, two ) in itertools.product( hex_chars, hex_chars ):
dir = directory + os.path.sep + one + two
next_paths = dircache.listdir( dir )
for path in next_paths: yield dir + os.path.sep + path

135
include/ServerController.py
View File

@ -1,6 +1,8 @@
import httplib
import HydrusConstants as HC
import HydrusServer
import HydrusSessions
import ServerConstants as SC
import ServerDB
import os
import random
@ -9,6 +11,7 @@ import time
import traceback
import wx
import yaml
from twisted.internet import reactor
class Controller( wx.App ):
@ -26,36 +29,13 @@ class Controller( wx.App ):
except: return False
def AddSession( self, session_key, service_identifier, account_identifier, expiry ): self._session_manager.AddSession( session_key, service_identifier, account_identifier, expiry )
def _Read( self, action, *args, **kwargs ): return self._db.Read( action, HC.HIGH_PRIORITY, *args, **kwargs )
def ChangePort( self, port ):
new_server = self._server_callable( port )
server_daemon = threading.Thread( target=new_server.serve_forever )
server_daemon.setDaemon( True )
server_daemon.start()
connection = httplib.HTTPConnection( '127.0.0.1', port, timeout = 20 )
connection.connect()
connection.request( 'GET', '/' )
response = connection.getresponse()
data = response.read()
if response.status != 200: raise Exception( yaml.safe_load( data ) )
connection.close()
self._server.shutdown()
self._server = new_server
def _Write( self, action, priority, *args, **kwargs ): return self._db.Write( action, priority, *args, **kwargs )
def GetAccountIdentifier( self, session_key, service_identifier ): return self._session_manager.GetAccountIdentifier( session_key, service_identifier )
def AddSession( self, service_identifier, account ): return self._session_manager.AddSession( service_identifier, account )
def GetAccount( self, session_key, service_identifier ): return self._session_manager.GetAccount( session_key, service_identifier )
def EventExit( self, event ): self._tbicon.Destroy()
@ -72,29 +52,110 @@ class Controller( wx.App ):
pubsubs_queue.task_done()
def GetDB( self ): return self._db
def OnInit( self ):
HC.app = self
try: self._db = ServerDB.DB()
try:
self._db = ServerDB.DB()
self.Bind( wx.EVT_MENU, self.EventExit, id=wx.ID_EXIT )
self.Bind( HC.EVT_PUBSUB, self.EventPubSub )
self._tbicon = TaskBarIcon()
self._session_manager = HydrusSessions.HydrusSessionManagerServer()
HC.pubsub.sub( self, 'RestartService', 'restart_service' )
self._services = {}
services_info = self.Read( 'services' )
for ( service_identifier, options ) in services_info: self.RestartService( service_identifier, options )
return True
except Exception as e:
HC.ShowException( e )
print( traceback.format_exc() )
return False
self._session_manager = HydrusSessions.HydrusSessionManagerServer()
def Read( self, action, *args, **kwargs ):
self.Bind( wx.EVT_MENU, self.EventExit, id=wx.ID_EXIT )
return self._Read( action, *args, **kwargs )
self.Bind( HC.EVT_PUBSUB, self.EventPubSub )
def RestartService( self, service_identifier, options ):
self._tbicon = TaskBarIcon()
def TWISTEDRestartService():
def StartService():
try:
port = options[ 'port' ]
connection = httplib.HTTPConnection( '127.0.0.1', port, timeout = 10 )
try:
connection.connect()
connection.close()
raise Exception( 'Something was already bound to port ' + HC.u( port ) )
except:
message = options[ 'message' ]
service_type = service_identifier.GetType()
if service_type == HC.SERVER_ADMIN: service_object = HydrusServer.HydrusServiceAdmin( service_identifier, message )
elif service_type == HC.FILE_REPOSITORY: service_object = HydrusServer.HydrusServiceRepositoryFile( service_identifier, message )
elif service_type == HC.TAG_REPOSITORY: service_object = HydrusServer.HydrusServiceRepositoryTag( service_identifier, message )
elif service_type == HC.MESSAGE_DEPOT: return
self._services[ service_identifier ] = reactor.listenTCP( port, service_object )
connection = httplib.HTTPConnection( '127.0.0.1', port, timeout = 10 )
try:
connection.connect()
connection.close()
except:
raise Exception( 'Tried to bind port ' + HC.u( port ) + ' but it failed.' )
except Exception as e:
print( traceback.format_exc() )
if service_identifier not in self._services: StartService()
else:
deferred = self._services[ service_identifier ].stopListening()
deferred.AddCallback( StartService )
return True
reactor.callFromThread( TWISTEDRestartService )
def Write( self, action, *args, **kwargs ):
return self._Write( action, HC.HIGH_PRIORITY, *args, **kwargs )
class TaskBarIcon( wx.TaskBarIcon ):

1469
include/ServerDB.py
View File

File diff suppressed because it is too large Load Diff

6
server.pyw
View File

@ -17,6 +17,8 @@ import os
import sys
from include import HydrusConstants as HC
from include import ServerController
import threading
from twisted.internet import reactor
initial_sys_stdout = sys.stdout
initial_sys_stderr = sys.stderr
@ -28,6 +30,8 @@ with open( HC.LOGS_DIR + os.path.sep + 'server.log', 'a' ) as f:
try:
threading.Thread( target = reactor.run, kwargs = { 'installSignalHandlers' : 0 } ).start()
app = ServerController.Controller()
app.MainLoop()
@ -43,4 +47,6 @@ sys.stderr = initial_sys_stderr
HC.shutdown = True
reactor.callFromThread( reactor.stop )
HC.pubsub.pubimmediate( 'shutdown' )