2013-10-02 22:06:06 +00:00
import HydrusConstants as HC
import HydrusExceptions
import HydrusFileHandling
import HydrusImageHandling
2015-11-04 22:30:28 +00:00
import HydrusPaths
2015-09-02 23:16:09 +00:00
import HydrusSerialisable
2013-10-02 22:06:06 +00:00
import os
2014-07-16 20:50:18 +00:00
import time
2013-10-02 22:06:06 +00:00
import traceback
import yaml
from twisted . internet import reactor , defer
from twisted . internet . threads import deferToThread
2015-04-29 19:20:35 +00:00
from twisted . web . server import NOT_DONE_YET
2013-10-02 22:06:06 +00:00
from twisted . web . resource import Resource
from twisted . web . static import File as FileResource , NoRangeStaticProducer
2015-03-25 22:04:19 +00:00
import HydrusData
import HydrusGlobals
2013-10-02 22:06:06 +00:00
CLIENT_ROOT_MESSAGE = ''' <html>
< head >
< title > hydrus client < / title >
< / head >
< body >
2015-11-04 22:30:28 +00:00
< p > This hydrus client uses software version ''' + str( HC.SOFTWARE_VERSION ) + ''' and network version ''' + str( HC.NETWORK_VERSION ) + ''' . < / p >
2013-10-02 22:06:06 +00:00
< p > It only serves requests from 127.0 .0 .1 . < / p >
< / body >
< / html > '''
ROOT_MESSAGE_BEGIN = ''' <html>
< head >
< title > hydrus service < / title >
< / head >
< body >
2015-11-04 22:30:28 +00:00
< p > This hydrus service uses software version ''' + str( HC.SOFTWARE_VERSION ) + ''' and network version ''' + str( HC.NETWORK_VERSION ) + ''' . < / p >
2013-10-02 22:06:06 +00:00
< p > '''
ROOT_MESSAGE_END = ''' </p>
< / body >
< / html > '''
def ParseFileArguments ( path ) :
HydrusImageHandling . ConvertToPngIfBmp ( path )
hash = HydrusFileHandling . GetHashFromPath ( path )
2015-03-04 22:44:32 +00:00
try :
( size , mime , width , height , duration , num_frames , num_words ) = HydrusFileHandling . GetFileInfo ( path )
except HydrusExceptions . SizeException :
raise HydrusExceptions . ForbiddenException ( ' File is of zero length! ' )
except HydrusExceptions . MimeException :
raise HydrusExceptions . ForbiddenException ( ' Filetype is not permitted! ' )
except Exception as e :
2015-11-04 22:30:28 +00:00
raise HydrusExceptions . ForbiddenException ( HydrusData . ToUnicode ( e ) )
2015-03-04 22:44:32 +00:00
2013-10-02 22:06:06 +00:00
args = { }
args [ ' path ' ] = path
args [ ' hash ' ] = hash
args [ ' size ' ] = size
args [ ' mime ' ] = mime
if width is not None : args [ ' width ' ] = width
if height is not None : args [ ' height ' ] = height
if duration is not None : args [ ' duration ' ] = duration
if num_frames is not None : args [ ' num_frames ' ] = num_frames
if num_words is not None : args [ ' num_words ' ] = num_words
2015-11-25 22:00:57 +00:00
if mime in HC . MIMES_WITH_THUMBNAILS :
2013-10-02 22:06:06 +00:00
2016-04-27 19:20:37 +00:00
try :
thumbnail = HydrusFileHandling . GenerateThumbnail ( path )
except Exception as e :
tb = traceback . format_exc ( )
raise HydrusExceptions . ForbiddenException ( ' Could not generate thumbnail from that file: ' + os . linesep + tb )
2013-10-02 22:06:06 +00:00
args [ ' thumbnail ' ] = thumbnail
return args
2015-11-04 22:30:28 +00:00
hydrus_favicon = FileResource ( os . path . join ( HC . STATIC_DIR , ' hydrus.ico ' ) , defaultType = ' image/x-icon ' )
2013-10-02 22:06:06 +00:00
2014-07-09 22:15:14 +00:00
class HydrusDomain ( object ) :
def __init__ ( self , local_only ) :
self . _local_only = local_only
def CheckValid ( self , client_ip ) :
if self . _local_only and client_ip != ' 127.0.0.1 ' : raise HydrusExceptions . ForbiddenException ( ' Only local access allowed! ' )
2013-10-02 22:06:06 +00:00
class HydrusResourceWelcome ( Resource ) :
2015-08-05 18:42:35 +00:00
def __init__ ( self , service_type , message ) :
2013-10-02 22:06:06 +00:00
Resource . __init__ ( self )
2015-08-05 18:42:35 +00:00
if service_type == HC . LOCAL_FILE : body = CLIENT_ROOT_MESSAGE
2013-10-02 22:06:06 +00:00
else : body = ROOT_MESSAGE_BEGIN + message + ROOT_MESSAGE_END
2015-11-04 22:30:28 +00:00
self . _body = HydrusData . ToByteString ( body )
2013-10-02 22:06:06 +00:00
2014-01-01 20:01:00 +00:00
self . _server_version_string = HC . service_string_lookup [ service_type ] + ' / ' + str ( HC . NETWORK_VERSION )
2015-07-22 19:40:39 +00:00
2014-01-01 20:01:00 +00:00
def render_GET ( self , request ) :
2015-12-02 22:32:18 +00:00
request . setResponseCode ( 200 )
2014-01-01 20:01:00 +00:00
request . setHeader ( ' Server ' , self . _server_version_string )
return self . _body
2013-10-02 22:06:06 +00:00
class HydrusResourceCommand ( Resource ) :
2014-08-27 22:15:22 +00:00
def __init__ ( self , service_key , service_type , domain ) :
2013-10-02 22:06:06 +00:00
Resource . __init__ ( self )
2014-08-27 22:15:22 +00:00
self . _service_key = service_key
self . _service_type = service_type
2014-07-09 22:15:14 +00:00
self . _domain = domain
2013-10-02 22:06:06 +00:00
self . _server_version_string = HC . service_string_lookup [ service_type ] + ' / ' + str ( HC . NETWORK_VERSION )
2015-07-22 19:40:39 +00:00
def _checkServerBusy ( self ) :
if HydrusGlobals . server_busy :
raise HydrusExceptions . ServerBusyException ( ' This server is busy, please try again later. ' )
2013-10-02 22:06:06 +00:00
def _callbackCheckRestrictions ( self , request ) :
2015-07-22 19:40:39 +00:00
self . _checkServerBusy ( )
2013-10-02 22:06:06 +00:00
self . _checkUserAgent ( request )
2014-07-09 22:15:14 +00:00
self . _domain . CheckValid ( request . getClientIP ( ) )
2013-10-02 22:06:06 +00:00
return request
def _callbackParseGETArgs ( self , request ) :
hydrus_args = { }
for name in request . args :
values = request . args [ name ]
value = values [ 0 ]
2015-06-03 21:05:13 +00:00
if name in ( ' begin ' , ' expires ' , ' lifetime ' , ' num ' , ' service_type ' , ' service_port ' , ' since ' , ' subindex ' , ' timespan ' ) :
2013-10-02 22:06:06 +00:00
try : hydrus_args [ name ] = int ( value )
except : raise HydrusExceptions . ForbiddenException ( ' I was expecting to parse \' ' + name + ' \' as an integer, but it failed. ' )
2014-10-01 22:58:32 +00:00
elif name in ( ' access_key ' , ' title ' , ' subject_account_key ' , ' contact_key ' , ' hash ' , ' subject_hash ' , ' subject_tag ' , ' message_key ' , ' share_key ' ) :
2013-10-02 22:06:06 +00:00
try : hydrus_args [ name ] = value . decode ( ' hex ' )
except : raise HydrusExceptions . ForbiddenException ( ' I was expecting to parse \' ' + name + ' \' as a hex-encoded string, but it failed. ' )
2015-10-14 21:02:25 +00:00
if ' subject_account_key ' in hydrus_args :
hydrus_args [ ' subject_identifier ' ] = HydrusData . AccountIdentifier ( account_key = hydrus_args [ ' subject_account_key ' ] )
2013-10-02 22:06:06 +00:00
elif ' subject_hash ' in hydrus_args :
2015-10-14 21:02:25 +00:00
hash = hydrus_args [ ' subject_hash ' ]
if ' subject_tag ' in hydrus_args :
tag = hydrus_args [ ' subject_tag ' ]
content = HydrusData . Content ( HC . CONTENT_TYPE_MAPPING , ( tag , hash ) )
else :
content = HydrusData . Content ( HC . CONTENT_TYPE_FILES , [ hash ] )
hydrus_args [ ' subject_identifier ' ] = HydrusData . AccountIdentifier ( content = content )
2013-10-02 22:06:06 +00:00
request . hydrus_args = hydrus_args
return request
def _callbackParsePOSTArgs ( self , request ) :
request . content . seek ( 0 )
2015-09-16 18:11:00 +00:00
if not request . requestHeaders . hasHeader ( ' Content-Type ' ) :
2013-10-02 22:06:06 +00:00
2015-09-16 18:11:00 +00:00
hydrus_args = { }
2013-10-02 22:06:06 +00:00
else :
2015-09-16 18:11:00 +00:00
content_types = request . requestHeaders . getRawHeaders ( ' Content-Type ' )
2015-03-25 22:04:19 +00:00
2015-09-16 18:11:00 +00:00
content_type = content_types [ 0 ]
2015-03-25 22:04:19 +00:00
2015-09-16 18:11:00 +00:00
try : mime = HC . mime_enum_lookup [ content_type ]
except : raise HydrusExceptions . ForbiddenException ( ' Did not recognise Content-Type header! ' )
2013-10-02 22:06:06 +00:00
2015-09-16 18:11:00 +00:00
if mime == HC . APPLICATION_YAML :
2013-10-02 22:06:06 +00:00
2015-09-16 18:11:00 +00:00
yaml_string = request . content . read ( )
request . hydrus_request_data_usage + = len ( yaml_string )
hydrus_args = yaml . safe_load ( yaml_string )
2015-10-14 21:02:25 +00:00
elif mime == HC . APPLICATION_JSON :
json_string = request . content . read ( )
request . hydrus_request_data_usage + = len ( json_string )
hydrus_args = HydrusSerialisable . CreateFromNetworkString ( json_string )
2015-09-16 18:11:00 +00:00
else :
2015-11-04 22:30:28 +00:00
( os_file_handle , temp_path ) = HydrusPaths . GetTempPath ( )
2015-09-16 18:11:00 +00:00
request . temp_file_info = ( os_file_handle , temp_path )
with open ( temp_path , ' wb ' ) as f :
2013-10-02 22:06:06 +00:00
2015-11-04 22:30:28 +00:00
for block in HydrusPaths . ReadFileLikeAsBlocks ( request . content ) :
2015-09-16 18:11:00 +00:00
f . write ( block )
request . hydrus_request_data_usage + = len ( block )
2013-10-02 22:06:06 +00:00
2015-09-16 18:11:00 +00:00
hydrus_args = ParseFileArguments ( temp_path )
2013-10-02 22:06:06 +00:00
request . hydrus_args = hydrus_args
return request
def _callbackRenderResponseContext ( self , request ) :
2015-03-25 22:04:19 +00:00
self . _CleanUpTempFile ( request )
2013-10-02 22:06:06 +00:00
response_context = request . hydrus_response_context
status_code = response_context . GetStatusCode ( )
request . setResponseCode ( status_code )
for ( k , v , kwargs ) in response_context . GetCookies ( ) : request . addCookie ( k , v , * * kwargs )
do_finish = True
if response_context . HasBody ( ) :
( mime , body ) = response_context . GetMimeBody ( )
content_type = HC . mime_string_lookup [ mime ]
content_length = len ( body )
request . setHeader ( ' Content-Type ' , content_type )
request . setHeader ( ' Content-Length ' , str ( content_length ) )
2015-11-04 22:30:28 +00:00
request . write ( HydrusData . ToByteString ( body ) )
2013-10-02 22:06:06 +00:00
elif response_context . HasPath ( ) :
path = response_context . GetPath ( )
2016-04-14 01:54:29 +00:00
size = os . path . getsize ( path )
2013-10-02 22:06:06 +00:00
2015-06-03 21:05:13 +00:00
if response_context . IsJSON ( ) :
2013-10-02 22:06:06 +00:00
2015-06-03 21:05:13 +00:00
mime = HC . APPLICATION_JSON
2013-10-02 22:06:06 +00:00
content_type = HC . mime_string_lookup [ mime ]
else :
mime = HydrusFileHandling . GetMime ( path )
( base , filename ) = os . path . split ( path )
content_type = HC . mime_string_lookup [ mime ] + ' ; ' + filename
content_length = size
2015-11-04 22:30:28 +00:00
# can't be unicode!
request . setHeader ( ' Content-Type ' , str ( content_type ) )
2013-10-02 22:06:06 +00:00
request . setHeader ( ' Content-Length ' , str ( content_length ) )
2014-07-16 20:50:18 +00:00
request . setHeader ( ' Expires ' , time . strftime ( ' %a , %d % b % Y % H: % M: % S GMT ' , time . gmtime ( time . time ( ) + 86400 * 365 ) ) )
request . setHeader ( ' Cache-Control ' , str ( 86400 * 365 ) )
2013-10-02 22:06:06 +00:00
fileObject = open ( path , ' rb ' )
producer = NoRangeStaticProducer ( request , fileObject )
producer . start ( )
do_finish = False
else :
content_length = 0
request . setHeader ( ' Content-Length ' , str ( content_length ) )
request . hydrus_request_data_usage + = content_length
self . _recordDataUsage ( request )
2015-11-18 22:44:07 +00:00
if do_finish :
request . finish ( )
2013-10-02 22:06:06 +00:00
def _callbackDoGETJob ( self , request ) :
def wrap_thread_result ( response_context ) :
request . hydrus_response_context = response_context
return request
d = deferToThread ( self . _threadDoGETJob , request )
d . addCallback ( wrap_thread_result )
return d
def _callbackDoPOSTJob ( self , request ) :
def wrap_thread_result ( response_context ) :
request . hydrus_response_context = response_context
return request
d = deferToThread ( self . _threadDoPOSTJob , request )
d . addCallback ( wrap_thread_result )
return d
def _checkUserAgent ( self , request ) :
request . is_hydrus_user_agent = False
if request . requestHeaders . hasHeader ( ' User-Agent ' ) :
user_agent_texts = request . requestHeaders . getRawHeaders ( ' User-Agent ' )
user_agent_text = user_agent_texts [ 0 ]
try :
user_agents = user_agent_text . split ( ' ' )
2013-12-04 22:44:16 +00:00
except : return # crazy user agent string, so just assume not a hydrus client
for user_agent in user_agents :
if ' / ' in user_agent :
( client , network_version ) = user_agent . split ( ' / ' , 1 )
2013-10-02 22:06:06 +00:00
2013-12-04 22:44:16 +00:00
if client == ' hydrus ' :
2013-10-02 22:06:06 +00:00
2013-12-04 22:44:16 +00:00
request . is_hydrus_user_agent = True
2013-10-02 22:06:06 +00:00
2013-12-04 22:44:16 +00:00
network_version = int ( network_version )
if network_version == HC . NETWORK_VERSION : return
else :
2013-10-02 22:06:06 +00:00
2013-12-04 22:44:16 +00:00
if network_version < HC . NETWORK_VERSION : message = ' Your client is out of date; please download the latest release. '
else : message = ' This server is out of date; please ask its admin to update to the latest release. '
2013-10-02 22:06:06 +00:00
2015-11-04 22:30:28 +00:00
raise HydrusExceptions . NetworkVersionException ( ' Network version mismatch! This server \' s network version is ' + str ( HC . NETWORK_VERSION ) + ' , whereas your client \' s is ' + str ( network_version ) + ' ! ' + message )
2013-10-02 22:06:06 +00:00
2015-11-25 22:00:57 +00:00
def _errbackDisconnected ( self , failure , request_deferred ) :
request_deferred . cancel ( )
2013-10-02 22:06:06 +00:00
def _errbackHandleEmergencyError ( self , failure , request ) :
2015-08-12 20:35:24 +00:00
try : self . _CleanUpTempFile ( request )
except : pass
2015-03-25 22:04:19 +00:00
2015-11-18 22:44:07 +00:00
try : HydrusData . DebugPrint ( failure . getTraceback ( ) )
2015-08-12 20:35:24 +00:00
except : pass
2013-10-02 22:06:06 +00:00
try : request . write ( failure . getTraceback ( ) )
except : pass
2015-11-25 22:00:57 +00:00
try : request . finish ( )
except : pass
2013-10-02 22:06:06 +00:00
def _errbackHandleProcessingError ( self , failure , request ) :
2015-03-25 22:04:19 +00:00
self . _CleanUpTempFile ( request )
2014-01-01 20:01:00 +00:00
do_yaml = True
try :
# the error may have occured before user agent was set up!
if not request . is_hydrus_user_agent : do_yaml = False
except : pass
if do_yaml :
2013-10-02 22:06:06 +00:00
default_mime = HC . APPLICATION_YAML
2015-11-04 22:30:28 +00:00
default_encoding = lambda x : yaml . safe_dump ( HydrusData . ToUnicode ( x ) )
2013-10-02 22:06:06 +00:00
else :
default_mime = HC . TEXT_HTML
2015-11-04 22:30:28 +00:00
default_encoding = lambda x : HydrusData . ToByteString ( x )
2013-10-02 22:06:06 +00:00
2015-03-25 22:04:19 +00:00
if failure . type == KeyError : response_context = ResponseContext ( 403 , mime = default_mime , body = default_encoding ( ' It appears one or more parameters required for that request were missing: ' + os . linesep + failure . getTraceback ( ) ) )
elif failure . type == HydrusExceptions . PermissionException : response_context = ResponseContext ( 401 , mime = default_mime , body = default_encoding ( failure . value ) )
elif failure . type == HydrusExceptions . ForbiddenException : response_context = ResponseContext ( 403 , mime = default_mime , body = default_encoding ( failure . value ) )
elif failure . type == HydrusExceptions . NotFoundException : response_context = ResponseContext ( 404 , mime = default_mime , body = default_encoding ( failure . value ) )
elif failure . type == HydrusExceptions . NetworkVersionException : response_context = ResponseContext ( 426 , mime = default_mime , body = default_encoding ( failure . value ) )
2015-07-22 19:40:39 +00:00
elif failure . type == HydrusExceptions . ServerBusyException : response_context = ResponseContext ( 503 , mime = default_mime , body = default_encoding ( failure . value ) )
2015-05-06 20:26:18 +00:00
elif failure . type == HydrusExceptions . SessionException : response_context = ResponseContext ( 419 , mime = default_mime , body = default_encoding ( failure . value ) )
2013-10-02 22:06:06 +00:00
else :
2015-11-18 22:44:07 +00:00
HydrusData . DebugPrint ( failure . getTraceback ( ) )
2013-10-02 22:06:06 +00:00
2015-03-25 22:04:19 +00:00
response_context = ResponseContext ( 500 , mime = default_mime , body = default_encoding ( ' The repository encountered an error it could not handle! Here is a dump of what happened, which will also be written to your client.log file. If it persists, please forward it to hydrus.admin@gmail.com: ' + os . linesep * 2 + failure . getTraceback ( ) ) )
2013-10-02 22:06:06 +00:00
request . hydrus_response_context = response_context
return request
def _parseAccessKey ( self , request ) :
if not request . requestHeaders . hasHeader ( ' Hydrus-Key ' ) : raise HydrusExceptions . PermissionException ( ' No hydrus key header found! ' )
hex_keys = request . requestHeaders . getRawHeaders ( ' Hydrus-Key ' )
hex_key = hex_keys [ 0 ]
try : access_key = hex_key . decode ( ' hex ' )
except : raise HydrusExceptions . ForbiddenException ( ' Could not parse the hydrus key! ' )
return access_key
2014-07-16 20:50:18 +00:00
def _recordDataUsage ( self , request ) : pass
2013-10-02 22:06:06 +00:00
def _threadDoGETJob ( self , request ) : raise HydrusExceptions . NotFoundException ( ' This service does not support that request! ' )
def _threadDoPOSTJob ( self , request ) : raise HydrusExceptions . NotFoundException ( ' This service does not support that request! ' )
2015-03-25 22:04:19 +00:00
def _CleanUpTempFile ( self , request ) :
if hasattr ( request , ' temp_file_info ' ) :
( os_file_handle , temp_path ) = request . temp_file_info
2015-11-04 22:30:28 +00:00
HydrusPaths . CleanUpTempPath ( os_file_handle , temp_path )
2015-03-25 22:04:19 +00:00
2015-08-12 20:35:24 +00:00
del request . temp_file_info
2015-03-25 22:04:19 +00:00
2013-10-02 22:06:06 +00:00
def render_GET ( self , request ) :
request . setHeader ( ' Server ' , self . _server_version_string )
d = defer . Deferred ( )
d . addCallback ( self . _callbackCheckRestrictions )
d . addCallback ( self . _callbackParseGETArgs )
d . addCallback ( self . _callbackDoGETJob )
d . addErrback ( self . _errbackHandleProcessingError , request )
d . addCallback ( self . _callbackRenderResponseContext )
d . addErrback ( self . _errbackHandleEmergencyError , request )
reactor . callLater ( 0 , d . callback , request )
2015-11-25 22:00:57 +00:00
request . notifyFinish ( ) . addErrback ( self . _errbackDisconnected , d )
2013-10-02 22:06:06 +00:00
return NOT_DONE_YET
def render_POST ( self , request ) :
request . setHeader ( ' Server ' , self . _server_version_string )
d = defer . Deferred ( )
d . addCallback ( self . _callbackCheckRestrictions )
d . addCallback ( self . _callbackParsePOSTArgs )
d . addCallback ( self . _callbackDoPOSTJob )
d . addErrback ( self . _errbackHandleProcessingError , request )
d . addCallback ( self . _callbackRenderResponseContext )
d . addErrback ( self . _errbackHandleEmergencyError , request )
reactor . callLater ( 0 , d . callback , request )
2015-11-25 22:00:57 +00:00
request . notifyFinish ( ) . addErrback ( self . _errbackDisconnected , d )
2013-10-02 22:06:06 +00:00
return NOT_DONE_YET
2015-03-25 22:04:19 +00:00
class ResponseContext ( object ) :
2015-06-03 21:05:13 +00:00
def __init__ ( self , status_code , mime = HC . APPLICATION_YAML , body = None , path = None , is_json = False , cookies = None ) :
2015-03-25 22:04:19 +00:00
if cookies is None : cookies = [ ]
self . _status_code = status_code
self . _mime = mime
self . _body = body
self . _path = path
2015-06-03 21:05:13 +00:00
self . _is_json = is_json
2015-03-25 22:04:19 +00:00
self . _cookies = cookies
def GetCookies ( self ) : return self . _cookies
def GetLength ( self ) : return len ( self . _body )
def GetMimeBody ( self ) : return ( self . _mime , self . _body )
def GetPath ( self ) : return self . _path
def GetStatusCode ( self ) : return self . _status_code
def HasBody ( self ) : return self . _body is not None
def HasPath ( self ) : return self . _path is not None
2015-06-03 21:05:13 +00:00
def IsJSON ( self ) : return self . _is_json
2013-10-02 22:06:06 +00:00
