2019-02-27 23:03:30 +00:00
import collections
2019-03-13 21:04:21 +00:00
import hashlib
2019-02-13 22:26:43 +00:00
import http . client
import json
import os
2019-03-13 21:04:21 +00:00
import random
import shutil
2019-02-13 22:26:43 +00:00
import time
import unittest
import urllib
2020-07-29 20:52:44 +00:00
2019-02-13 22:26:43 +00:00
from twisted . internet import reactor
2020-07-29 20:52:44 +00:00
from hydrus . core import HydrusConstants as HC
2020-04-22 21:00:35 +00:00
from hydrus . core import HydrusData
2020-07-29 20:52:44 +00:00
from hydrus . core import HydrusExceptions
2020-04-22 21:00:35 +00:00
from hydrus . core import HydrusGlobals as HG
2020-07-29 20:52:44 +00:00
from hydrus . core import HydrusTags
from hydrus . core import HydrusText
from hydrus . client import ClientConstants as CC
from hydrus . client import ClientAPI
from hydrus . client import ClientManagers
from hydrus . client import ClientSearch
from hydrus . client import ClientServices
2021-06-30 21:27:35 +00:00
from hydrus . client . importing import ClientImportFiles
2021-08-18 21:10:01 +00:00
from hydrus . client . media import ClientMedia
2020-07-29 20:52:44 +00:00
from hydrus . client . media import ClientMediaManagers
from hydrus . client . media import ClientMediaResult
2020-08-05 20:10:36 +00:00
from hydrus . client . metadata import ClientTags
2021-04-07 21:26:45 +00:00
from hydrus . client . networking import ClientLocalServer
from hydrus . client . networking import ClientLocalServerResources
2021-05-27 00:09:06 +00:00
from hydrus . client . networking import ClientNetworkingContexts
2019-02-13 22:26:43 +00:00
class TestClientAPI ( unittest . TestCase ) :
@classmethod
def setUpClass ( cls ) :
cls . _client_api = ClientServices . GenerateService ( CC . CLIENT_API_SERVICE_KEY , HC . CLIENT_API_SERVICE , ' client api ' )
2019-04-17 21:51:50 +00:00
cls . _client_api_cors = ClientServices . GenerateService ( CC . CLIENT_API_SERVICE_KEY , HC . CLIENT_API_SERVICE , ' client api ' )
cls . _client_api_cors . _support_cors = True
2019-02-13 22:26:43 +00:00
def TWISTEDSetup ( ) :
reactor . listenTCP ( 45869 , ClientLocalServer . HydrusServiceClientAPI ( cls . _client_api , allow_non_local_connections = False ) )
2019-04-17 21:51:50 +00:00
reactor . listenTCP ( 45899 , ClientLocalServer . HydrusServiceClientAPI ( cls . _client_api_cors , allow_non_local_connections = False ) )
2019-02-13 22:26:43 +00:00
reactor . callFromThread ( TWISTEDSetup )
time . sleep ( 1 )
2020-11-11 22:20:16 +00:00
def _compare_content_updates ( self , service_keys_to_content_updates , expected_service_keys_to_content_updates ) :
self . assertEqual ( len ( service_keys_to_content_updates ) , len ( expected_service_keys_to_content_updates ) )
for ( service_key , content_updates ) in service_keys_to_content_updates . items ( ) :
expected_content_updates = expected_service_keys_to_content_updates [ service_key ]
2021-01-27 22:14:03 +00:00
c_u_tuples = sorted ( ( ( c_u . ToTuple ( ) , c_u . GetReason ( ) ) for c_u in content_updates ) )
e_c_u_tuples = sorted ( ( ( e_c_u . ToTuple ( ) , e_c_u . GetReason ( ) ) for e_c_u in expected_content_updates ) )
2020-11-11 22:20:16 +00:00
self . assertEqual ( c_u_tuples , e_c_u_tuples )
2019-02-13 22:26:43 +00:00
def _test_basics ( self , connection ) :
#
connection . request ( ' GET ' , ' / ' )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
#
with open ( os . path . join ( HC . STATIC_DIR , ' hydrus.ico ' ) , ' rb ' ) as f :
favicon = f . read ( )
connection . request ( ' GET ' , ' /favicon.ico ' )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( data , favicon )
2020-04-29 21:44:12 +00:00
time . sleep ( 3 )
2019-02-13 22:26:43 +00:00
def _test_client_api_basics ( self , connection ) :
# /api_version
connection . request ( ' GET ' , ' /api_version ' )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
response_json = json . loads ( text )
self . assertEqual ( response_json [ ' version ' ] , HC . CLIENT_API_VERSION )
2021-05-27 00:09:06 +00:00
self . assertEqual ( response_json [ ' hydrus_version ' ] , HC . SOFTWARE_VERSION )
2019-02-13 22:26:43 +00:00
# /request_new_permissions
def format_request_new_permissions_query ( name , basic_permissions ) :
return ' /request_new_permissions?name= {} &basic_permissions= {} ' . format ( urllib . parse . quote ( name ) , urllib . parse . quote ( json . dumps ( basic_permissions ) ) )
2020-05-06 21:31:41 +00:00
# fail as dialog not open
2019-02-13 22:26:43 +00:00
ClientAPI . api_request_dialog_open = False
connection . request ( ' GET ' , format_request_new_permissions_query ( ' test ' , [ ClientAPI . CLIENT_API_PERMISSION_ADD_FILES ] ) )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
2020-05-06 21:31:41 +00:00
self . assertEqual ( response . status , 409 )
2019-02-13 22:26:43 +00:00
2019-06-26 21:27:18 +00:00
self . assertIn ( ' dialog ' , text )
2019-02-13 22:26:43 +00:00
# success
permissions_to_set_up = [ ]
permissions_to_set_up . append ( ( ' everything ' , list ( ClientAPI . ALLOWED_PERMISSIONS ) ) )
permissions_to_set_up . append ( ( ' add_files ' , [ ClientAPI . CLIENT_API_PERMISSION_ADD_FILES ] ) )
permissions_to_set_up . append ( ( ' add_tags ' , [ ClientAPI . CLIENT_API_PERMISSION_ADD_TAGS ] ) )
permissions_to_set_up . append ( ( ' add_urls ' , [ ClientAPI . CLIENT_API_PERMISSION_ADD_URLS ] ) )
2019-06-05 19:42:39 +00:00
permissions_to_set_up . append ( ( ' manage_pages ' , [ ClientAPI . CLIENT_API_PERMISSION_MANAGE_PAGES ] ) )
2019-08-07 22:59:53 +00:00
permissions_to_set_up . append ( ( ' manage_cookies ' , [ ClientAPI . CLIENT_API_PERMISSION_MANAGE_COOKIES ] ) )
2019-02-13 22:26:43 +00:00
permissions_to_set_up . append ( ( ' search_all_files ' , [ ClientAPI . CLIENT_API_PERMISSION_SEARCH_FILES ] ) )
permissions_to_set_up . append ( ( ' search_green_files ' , [ ClientAPI . CLIENT_API_PERMISSION_SEARCH_FILES ] ) )
set_up_permissions = { }
for ( name , basic_permissions ) in permissions_to_set_up :
ClientAPI . api_request_dialog_open = True
connection . request ( ' GET ' , format_request_new_permissions_query ( name , basic_permissions ) )
response = connection . getresponse ( )
data = response . read ( )
ClientAPI . api_request_dialog_open = False
response_text = str ( data , ' utf-8 ' )
2019-06-26 21:27:18 +00:00
self . assertEqual ( response . status , 200 )
2019-02-13 22:26:43 +00:00
response_json = json . loads ( response_text )
access_key_hex = response_json [ ' access_key ' ]
self . assertEqual ( len ( access_key_hex ) , 64 )
access_key_hex = HydrusText . HexFilter ( access_key_hex )
self . assertEqual ( len ( access_key_hex ) , 64 )
api_permissions = ClientAPI . last_api_permissions_request
if ' green ' in name :
2021-04-07 21:26:45 +00:00
search_tag_filter = HydrusTags . TagFilter ( )
2019-02-13 22:26:43 +00:00
2021-04-07 21:26:45 +00:00
search_tag_filter . SetRule ( ' ' , HC . FILTER_BLACKLIST )
search_tag_filter . SetRule ( ' : ' , HC . FILTER_BLACKLIST )
search_tag_filter . SetRule ( ' green ' , HC . FILTER_WHITELIST )
2019-02-13 22:26:43 +00:00
api_permissions . SetSearchTagFilter ( search_tag_filter )
self . assertEqual ( bytes . fromhex ( access_key_hex ) , api_permissions . GetAccessKey ( ) )
set_up_permissions [ name ] = api_permissions
HG . test_controller . client_api_manager . AddAccess ( api_permissions )
# /verify_access_key
# missing
connection . request ( ' GET ' , ' /verify_access_key ' )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 401 )
2019-06-26 21:27:18 +00:00
# fail header
2019-02-13 22:26:43 +00:00
incorrect_headers = { ' Hydrus-Client-API-Access-Key ' : ' abcd ' }
connection . request ( ' GET ' , ' /verify_access_key ' , headers = incorrect_headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 403 )
2019-06-26 21:27:18 +00:00
# fail get param
connection . request ( ' GET ' , ' /verify_access_key?Hydrus-Client-API-Access-Key=abcd ' )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 403 )
# success
def do_good_verify_test ( api_permissions , key_hex , key_name ) :
for request_type in ( ' header ' , ' get ' ) :
if request_type == ' header ' :
headers = { key_name : key_hex }
connection . request ( ' GET ' , ' /verify_access_key ' , headers = headers )
elif request_type == ' get ' :
connection . request ( ' GET ' , ' /verify_access_key? {} = {} ' . format ( key_name , key_hex ) )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
body_dict = json . loads ( text )
self . assertEqual ( set ( body_dict [ ' basic_permissions ' ] ) , set ( api_permissions . GetBasicPermissions ( ) ) )
self . assertEqual ( body_dict [ ' human_description ' ] , api_permissions . ToHumanString ( ) )
for api_permissions in set_up_permissions . values ( ) :
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
do_good_verify_test ( api_permissions , access_key_hex , ' Hydrus-Client-API-Access-Key ' )
# /session_key
# fail header
incorrect_headers = { ' Hydrus-Client-API-Session-Key ' : ' abcd ' }
connection . request ( ' GET ' , ' /verify_access_key ' , headers = incorrect_headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 419 )
# fail get param
connection . request ( ' GET ' , ' /verify_access_key?Hydrus-Client-API-Session-Key=abcd ' )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 419 )
2019-02-13 22:26:43 +00:00
# success
2019-06-26 21:27:18 +00:00
for api_permissions in set_up_permissions . values ( ) :
2019-02-13 22:26:43 +00:00
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
2019-06-26 21:27:18 +00:00
connection . request ( ' GET ' , ' /session_key ' , headers = headers )
2019-02-13 22:26:43 +00:00
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
body_dict = json . loads ( text )
self . assertEqual ( response . status , 200 )
2019-06-26 21:27:18 +00:00
self . assertIn ( ' session_key ' , body_dict )
session_key_hex = body_dict [ ' session_key ' ]
self . assertEqual ( len ( session_key_hex ) , 64 )
2019-02-13 22:26:43 +00:00
2019-06-26 21:27:18 +00:00
do_good_verify_test ( api_permissions , session_key_hex , ' Hydrus-Client-API-Session-Key ' )
# test access in POST params
# fail
2020-01-16 02:08:23 +00:00
headers = { ' Content-Type ' : HC . mime_mimetype_string_lookup [ HC . APPLICATION_JSON ] }
2019-06-26 21:27:18 +00:00
hash = os . urandom ( 32 )
hash_hex = hash . hex ( )
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_tags/add_tags '
2019-09-18 22:40:39 +00:00
body_dict = { ' Hydrus-Client-API-Access-Key ' : ' abcd ' , ' hash ' : hash_hex , ' service_names_to_tags ' : { ' my tags ' : [ ' test ' , ' test2 ' ] } }
2019-06-26 21:27:18 +00:00
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 403 )
2019-09-18 22:40:39 +00:00
body_dict = { ' Hydrus-Client-API-Session-Key ' : ' abcd ' , ' hash ' : hash_hex , ' service_names_to_tags ' : { ' my tags ' : [ ' test ' , ' test2 ' ] } }
2019-06-26 21:27:18 +00:00
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 419 )
# success
api_permissions = set_up_permissions [ ' everything ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
connection . request ( ' GET ' , ' /session_key ' , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
body_dict = json . loads ( text )
session_key_hex = body_dict [ ' session_key ' ]
2020-01-16 02:08:23 +00:00
headers = { ' Content-Type ' : HC . mime_mimetype_string_lookup [ HC . APPLICATION_JSON ] }
2019-06-26 21:27:18 +00:00
hash = os . urandom ( 32 )
hash_hex = hash . hex ( )
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_tags/add_tags '
2019-09-18 22:40:39 +00:00
body_dict = { ' Hydrus-Client-API-Access-Key ' : access_key_hex , ' hash ' : hash_hex , ' service_names_to_tags ' : { ' my tags ' : [ ' test ' , ' test2 ' ] } }
2019-06-26 21:27:18 +00:00
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
2021-11-03 20:49:56 +00:00
#
HG . test_controller . ClearWrites ( ' content_updates ' )
2019-09-18 22:40:39 +00:00
body_dict = { ' Hydrus-Client-API-Session-Key ' : session_key_hex , ' hash ' : hash_hex , ' service_names_to_tags ' : { ' my tags ' : [ ' test ' , ' test2 ' ] } }
2019-06-26 21:27:18 +00:00
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
2019-02-13 22:26:43 +00:00
2021-11-03 20:49:56 +00:00
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
self . assertIn ( CC . DEFAULT_LOCAL_TAG_SERVICE_KEY , service_keys_to_content_updates )
self . assertTrue ( len ( service_keys_to_content_updates [ CC . DEFAULT_LOCAL_TAG_SERVICE_KEY ] ) > 0 )
#
HG . test_controller . ClearWrites ( ' content_updates ' )
body_dict = { ' Hydrus-Client-API-Session-Key ' : session_key_hex , ' hash ' : hash_hex , ' service_keys_to_tags ' : { CC . DEFAULT_LOCAL_TAG_SERVICE_KEY . hex ( ) : [ ' test ' , ' test2 ' ] } }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
self . assertIn ( CC . DEFAULT_LOCAL_TAG_SERVICE_KEY , service_keys_to_content_updates )
self . assertTrue ( len ( service_keys_to_content_updates [ CC . DEFAULT_LOCAL_TAG_SERVICE_KEY ] ) > 0 )
#
HG . test_controller . ClearWrites ( ' content_updates ' )
body_dict = { ' Hydrus-Client-API-Session-Key ' : session_key_hex , ' hash ' : hash_hex , ' service_keys_to_actions_to_tags ' : { CC . DEFAULT_LOCAL_TAG_SERVICE_KEY . hex ( ) : { str ( HC . CONTENT_UPDATE_ADD ) : [ ' test ' , ' test2 ' ] } } }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
self . assertIn ( CC . DEFAULT_LOCAL_TAG_SERVICE_KEY , service_keys_to_content_updates )
self . assertTrue ( len ( service_keys_to_content_updates [ CC . DEFAULT_LOCAL_TAG_SERVICE_KEY ] ) > 0 )
#
2019-02-13 22:26:43 +00:00
return set_up_permissions
2019-04-17 21:51:50 +00:00
def _test_cors_fails ( self , connection ) :
connection . request ( ' OPTIONS ' , ' /api_version ' )
response = connection . getresponse ( )
data = response . read ( )
2019-05-29 21:34:43 +00:00
self . assertEqual ( response . status , 200 )
2019-04-17 21:51:50 +00:00
self . assertEqual ( response . getheader ( ' Allow ' ) , ' GET ' )
#
connection . request ( ' OPTIONS ' , ' /api_version ' , headers = { ' Origin ' : ' muhsite.com ' } )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 401 )
def _test_cors_succeeds ( self , connection ) :
connection . request ( ' OPTIONS ' , ' /api_version ' )
response = connection . getresponse ( )
data = response . read ( )
2019-05-29 21:34:43 +00:00
self . assertEqual ( response . status , 200 )
2019-04-17 21:51:50 +00:00
self . assertEqual ( response . getheader ( ' Allow ' ) , ' GET ' )
#
connection . request ( ' OPTIONS ' , ' /api_version ' , headers = { ' Origin ' : ' muhsite.com ' } )
response = connection . getresponse ( )
data = response . read ( )
2019-05-29 21:34:43 +00:00
self . assertEqual ( response . status , 200 )
2019-04-17 21:51:50 +00:00
self . assertEqual ( response . getheader ( ' Access-Control-Allow-Methods ' ) , ' GET ' )
2020-06-11 12:01:08 +00:00
self . assertEqual ( response . getheader ( ' Access-Control-Allow-Headers ' ) , ' * ' )
2019-04-17 21:51:50 +00:00
self . assertEqual ( response . getheader ( ' Access-Control-Allow-Origin ' ) , ' * ' )
2021-05-27 00:09:06 +00:00
def _test_get_services ( self , connection , set_up_permissions ) :
should_work = { set_up_permissions [ ' everything ' ] , set_up_permissions [ ' add_files ' ] , set_up_permissions [ ' add_tags ' ] , set_up_permissions [ ' manage_pages ' ] , set_up_permissions [ ' search_all_files ' ] , set_up_permissions [ ' search_green_files ' ] }
should_break = { set_up_permissions [ ' add_urls ' ] , set_up_permissions [ ' manage_cookies ' ] }
expected_answer = {
' local_tags ' : [
{
' name ' : ' my tags ' ,
' service_key ' : ' 6c6f63616c2074616773 '
}
] ,
' tag_repositories ' : [
{
' name ' : ' example tag repo ' ,
' service_key ' : HG . test_controller . example_tag_repo_service_key . hex ( )
}
] ,
' local_files ' : [
{
' name ' : ' my files ' ,
' service_key ' : ' 6c6f63616c2066696c6573 '
}
] ,
' file_repositories ' : [
] ,
' all_local_files ' : [
{
' name ' : ' all local files ' ,
' service_key ' : ' 616c6c206c6f63616c2066696c6573 '
}
] ,
' all_known_files ' : [
{
' name ' : ' all known files ' ,
' service_key ' : ' 616c6c206b6e6f776e2066696c6573 '
}
] ,
' all_known_tags ' : [
{
' name ' : ' all known tags ' ,
' service_key ' : ' 616c6c206b6e6f776e2074616773 '
}
] ,
' trash ' : [
{
' name ' : ' trash ' ,
' service_key ' : ' 7472617368 '
}
]
}
for api_permissions in should_work . union ( should_break ) :
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
#
path = ' /get_services '
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
if api_permissions in should_work :
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
d = json . loads ( text )
self . assertEqual ( d , expected_answer )
else :
self . assertEqual ( response . status , 403 )
2020-11-11 22:20:16 +00:00
def _test_add_files_add_file ( self , connection , set_up_permissions ) :
2019-02-13 22:26:43 +00:00
api_permissions = set_up_permissions [ ' add_files ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
# fail
2021-06-30 21:27:35 +00:00
hash = bytes . fromhex ( ' a593942cb7ea9ffcd8ccf2f0fa23c338e23bfecd9a3e508dfc0bcf07501ead08 ' )
f = ClientImportFiles . FileImportStatus . STATICGetUnknownStatus ( )
f . hash = hash
HG . test_controller . SetRead ( ' hash_status ' , f )
2019-02-13 22:26:43 +00:00
2020-01-16 02:08:23 +00:00
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex , ' Content-Type ' : HC . mime_mimetype_string_lookup [ HC . APPLICATION_OCTET_STREAM ] }
2019-02-13 22:26:43 +00:00
path = ' /add_files/add_file '
body = b ' blarg '
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
response_json = json . loads ( text )
self . assertEqual ( response_json [ ' status ' ] , CC . STATUS_ERROR )
2021-06-30 21:27:35 +00:00
self . assertEqual ( response_json [ ' hash ' ] , hash . hex ( ) )
2019-02-13 22:26:43 +00:00
self . assertIn ( ' Traceback ' , response_json [ ' note ' ] )
# success as body
2021-06-30 21:27:35 +00:00
hash = b ' \xad m5 \x99 \xa6 \xc4 \x89 \xa5 u \xeb \x19 \xc0 & \xfa \xce \x97 \xa9 \xcd ey \xe7 G( \xb0 \xce \x94 \xa6 \x01 \xd2 2 \xf3 \xc3 '
f = ClientImportFiles . FileImportStatus . STATICGetUnknownStatus ( )
f . hash = hash
f . note = ' test note '
HG . test_controller . SetRead ( ' hash_status ' , f )
2019-02-13 22:26:43 +00:00
hydrus_png_path = os . path . join ( HC . STATIC_DIR , ' hydrus.png ' )
with open ( hydrus_png_path , ' rb ' ) as f :
HYDRUS_PNG_BYTES = f . read ( )
2020-01-16 02:08:23 +00:00
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex , ' Content-Type ' : HC . mime_mimetype_string_lookup [ HC . APPLICATION_OCTET_STREAM ] }
2019-02-13 22:26:43 +00:00
path = ' /add_files/add_file '
body = HYDRUS_PNG_BYTES
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
response_json = json . loads ( text )
2021-06-30 21:27:35 +00:00
expected_result = { ' status ' : CC . STATUS_SUCCESSFUL_AND_NEW , ' hash ' : hash . hex ( ) , ' note ' : ' test note ' }
2019-02-13 22:26:43 +00:00
self . assertEqual ( response_json , expected_result )
# do hydrus png as path
2021-06-30 21:27:35 +00:00
f = ClientImportFiles . FileImportStatus . STATICGetUnknownStatus ( )
f . hash = hash
f . note = ' test note '
HG . test_controller . SetRead ( ' hash_status ' , f )
2020-01-16 02:08:23 +00:00
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex , ' Content-Type ' : HC . mime_mimetype_string_lookup [ HC . APPLICATION_JSON ] }
2019-02-13 22:26:43 +00:00
path = ' /add_files/add_file '
body_dict = { ' path ' : hydrus_png_path }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
response_json = json . loads ( text )
2021-06-30 21:27:35 +00:00
expected_result = { ' status ' : CC . STATUS_SUCCESSFUL_AND_NEW , ' hash ' : hash . hex ( ) , ' note ' : ' test note ' }
2019-02-13 22:26:43 +00:00
self . assertEqual ( response_json , expected_result )
2020-11-11 22:20:16 +00:00
def _test_add_files_other_actions ( self , connection , set_up_permissions ) :
api_permissions = set_up_permissions [ ' add_files ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex , ' Content-Type ' : HC . mime_mimetype_string_lookup [ HC . APPLICATION_JSON ] }
#
hash = HydrusData . GenerateKey ( )
hashes = { HydrusData . GenerateKey ( ) for i in range ( 10 ) }
#
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_files/delete_files '
body_dict = { ' hash ' : hash . hex ( ) }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
expected_service_keys_to_content_updates = { CC . LOCAL_FILE_SERVICE_KEY : [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_FILES , HC . CONTENT_UPDATE_DELETE , { hash } ) ] }
self . _compare_content_updates ( service_keys_to_content_updates , expected_service_keys_to_content_updates )
#
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_files/delete_files '
body_dict = { ' hashes ' : [ h . hex ( ) for h in hashes ] }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
expected_service_keys_to_content_updates = { CC . LOCAL_FILE_SERVICE_KEY : [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_FILES , HC . CONTENT_UPDATE_DELETE , hashes ) ] }
self . _compare_content_updates ( service_keys_to_content_updates , expected_service_keys_to_content_updates )
#
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_files/undelete_files '
body_dict = { ' hash ' : hash . hex ( ) }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
2021-04-28 21:43:16 +00:00
expected_service_keys_to_content_updates = { CC . LOCAL_FILE_SERVICE_KEY : [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_FILES , HC . CONTENT_UPDATE_UNDELETE , { hash } ) ] }
2020-11-11 22:20:16 +00:00
self . _compare_content_updates ( service_keys_to_content_updates , expected_service_keys_to_content_updates )
#
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_files/undelete_files '
body_dict = { ' hashes ' : [ h . hex ( ) for h in hashes ] }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
2021-04-28 21:43:16 +00:00
expected_service_keys_to_content_updates = { CC . LOCAL_FILE_SERVICE_KEY : [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_FILES , HC . CONTENT_UPDATE_UNDELETE , hashes ) ] }
2020-11-11 22:20:16 +00:00
self . _compare_content_updates ( service_keys_to_content_updates , expected_service_keys_to_content_updates )
#
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_files/archive_files '
body_dict = { ' hash ' : hash . hex ( ) }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
expected_service_keys_to_content_updates = { CC . COMBINED_LOCAL_FILE_SERVICE_KEY : [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_FILES , HC . CONTENT_UPDATE_ARCHIVE , { hash } ) ] }
self . _compare_content_updates ( service_keys_to_content_updates , expected_service_keys_to_content_updates )
#
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_files/archive_files '
body_dict = { ' hashes ' : [ h . hex ( ) for h in hashes ] }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
expected_service_keys_to_content_updates = { CC . COMBINED_LOCAL_FILE_SERVICE_KEY : [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_FILES , HC . CONTENT_UPDATE_ARCHIVE , hashes ) ] }
self . _compare_content_updates ( service_keys_to_content_updates , expected_service_keys_to_content_updates )
#
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_files/unarchive_files '
body_dict = { ' hash ' : hash . hex ( ) }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
expected_service_keys_to_content_updates = { CC . COMBINED_LOCAL_FILE_SERVICE_KEY : [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_FILES , HC . CONTENT_UPDATE_INBOX , { hash } ) ] }
self . _compare_content_updates ( service_keys_to_content_updates , expected_service_keys_to_content_updates )
#
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_files/unarchive_files '
body_dict = { ' hashes ' : [ h . hex ( ) for h in hashes ] }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
expected_service_keys_to_content_updates = { CC . COMBINED_LOCAL_FILE_SERVICE_KEY : [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_FILES , HC . CONTENT_UPDATE_INBOX , hashes ) ] }
self . _compare_content_updates ( service_keys_to_content_updates , expected_service_keys_to_content_updates )
2019-02-13 22:26:43 +00:00
def _test_add_tags ( self , connection , set_up_permissions ) :
api_permissions = set_up_permissions [ ' everything ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
2019-02-27 23:03:30 +00:00
# clean tags
tags = [ " bikini " , " blue eyes " , " character : samus aran " , " :) " , " " , " " , " 10 " , " 11 " , " 9 " , " system:wew " , " -flower " ]
json_tags = json . dumps ( tags )
path = ' /add_tags/clean_tags?tags= {} ' . format ( urllib . parse . quote ( json_tags , safe = ' ' ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
d = json . loads ( text )
expected_answer = { }
clean_tags = [ " bikini " , " blue eyes " , " character:samus aran " , " ::) " , " 10 " , " 11 " , " 9 " , " wew " , " flower " ]
clean_tags = HydrusTags . SortNumericTags ( clean_tags )
expected_answer [ ' tags ' ] = clean_tags
self . assertEqual ( d , expected_answer )
# add tags
2020-01-16 02:08:23 +00:00
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex , ' Content-Type ' : HC . mime_mimetype_string_lookup [ HC . APPLICATION_JSON ] }
2019-02-27 23:03:30 +00:00
hash = os . urandom ( 32 )
hash_hex = hash . hex ( )
hash2 = os . urandom ( 32 )
hash2_hex = hash2 . hex ( )
# missing hashes
path = ' /add_tags/add_tags '
2019-09-18 22:40:39 +00:00
body_dict = { ' service_names_to_tags ' : { ' my tags ' : [ ' test ' ] } }
2019-02-27 23:03:30 +00:00
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 400 )
# invalid service key
path = ' /add_tags/add_tags '
body_dict = { ' hash ' : hash_hex , ' service_names_to_tags ' : { ' bad tag service ' : [ ' test ' ] } }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 400 )
# add tags to local
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_tags/add_tags '
2019-09-18 22:40:39 +00:00
body_dict = { ' hash ' : hash_hex , ' service_names_to_tags ' : { ' my tags ' : [ ' test ' , ' test2 ' ] } }
2019-02-27 23:03:30 +00:00
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
expected_service_keys_to_content_updates = collections . defaultdict ( list )
2019-09-18 22:40:39 +00:00
expected_service_keys_to_content_updates [ CC . DEFAULT_LOCAL_TAG_SERVICE_KEY ] = [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_ADD , ( ' test ' , set ( [ hash ] ) ) ) , HydrusData . ContentUpdate ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_ADD , ( ' test2 ' , set ( [ hash ] ) ) ) ]
2019-02-27 23:03:30 +00:00
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
2020-11-11 22:20:16 +00:00
self . _compare_content_updates ( service_keys_to_content_updates , expected_service_keys_to_content_updates )
2021-01-27 22:14:03 +00:00
# add tags to local complex
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_tags/add_tags '
body_dict = { ' hash ' : hash_hex , ' service_names_to_actions_to_tags ' : { ' my tags ' : { str ( HC . CONTENT_UPDATE_ADD ) : [ ' test_add ' , ' test_add2 ' ] , str ( HC . CONTENT_UPDATE_DELETE ) : [ ' test_delete ' , ' test_delete2 ' ] } } }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
expected_service_keys_to_content_updates = collections . defaultdict ( list )
expected_service_keys_to_content_updates [ CC . DEFAULT_LOCAL_TAG_SERVICE_KEY ] = [
HydrusData . ContentUpdate ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_ADD , ( ' test_add ' , set ( [ hash ] ) ) ) ,
HydrusData . ContentUpdate ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_ADD , ( ' test_add2 ' , set ( [ hash ] ) ) ) ,
HydrusData . ContentUpdate ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_DELETE , ( ' test_delete ' , set ( [ hash ] ) ) ) ,
HydrusData . ContentUpdate ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_DELETE , ( ' test_delete2 ' , set ( [ hash ] ) ) )
]
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
self . _compare_content_updates ( service_keys_to_content_updates , expected_service_keys_to_content_updates )
# pend tags to repo
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_tags/add_tags '
body_dict = { ' hash ' : hash_hex , ' service_names_to_tags ' : { ' example tag repo ' : [ ' test ' , ' test2 ' ] } }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
expected_service_keys_to_content_updates = collections . defaultdict ( list )
expected_service_keys_to_content_updates [ HG . test_controller . example_tag_repo_service_key ] = [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_PEND , ( ' test ' , set ( [ hash ] ) ) ) , HydrusData . ContentUpdate ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_PEND , ( ' test2 ' , set ( [ hash ] ) ) ) ]
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
self . _compare_content_updates ( service_keys_to_content_updates , expected_service_keys_to_content_updates )
# pend tags to repo complex
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_tags/add_tags '
body_dict = { ' hash ' : hash_hex , ' service_names_to_actions_to_tags ' : { ' example tag repo ' : { str ( HC . CONTENT_UPDATE_PEND ) : [ ' test_add ' , ' test_add2 ' ] , str ( HC . CONTENT_UPDATE_PETITION ) : [ [ ' test_delete ' , ' muh reason ' ] , ' test_delete2 ' ] } } }
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
expected_service_keys_to_content_updates = collections . defaultdict ( list )
expected_service_keys_to_content_updates [ HG . test_controller . example_tag_repo_service_key ] = [
HydrusData . ContentUpdate ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_PEND , ( ' test_add ' , set ( [ hash ] ) ) ) ,
HydrusData . ContentUpdate ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_PEND , ( ' test_add2 ' , set ( [ hash ] ) ) ) ,
HydrusData . ContentUpdate ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_PETITION , ( ' test_delete ' , set ( [ hash ] ) ) , reason = ' muh reason ' ) ,
HydrusData . ContentUpdate ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_PETITION , ( ' test_delete2 ' , set ( [ hash ] ) ) , reason = ' Petitioned from API ' )
]
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
self . _compare_content_updates ( service_keys_to_content_updates , expected_service_keys_to_content_updates )
2019-02-27 23:03:30 +00:00
2019-04-17 21:51:50 +00:00
# add to multiple files
HG . test_controller . ClearWrites ( ' content_updates ' )
path = ' /add_tags/add_tags '
2019-09-18 22:40:39 +00:00
body_dict = { ' hashes ' : [ hash_hex , hash2_hex ] , ' service_names_to_tags ' : { ' my tags ' : [ ' test ' , ' test2 ' ] } }
2019-04-17 21:51:50 +00:00
body = json . dumps ( body_dict )
connection . request ( ' POST ' , path , body = body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
expected_service_keys_to_content_updates = collections . defaultdict ( list )
2019-09-18 22:40:39 +00:00
expected_service_keys_to_content_updates [ CC . DEFAULT_LOCAL_TAG_SERVICE_KEY ] = [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_ADD , ( ' test ' , set ( [ hash , hash2 ] ) ) ) , HydrusData . ContentUpdate ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_ADD , ( ' test2 ' , set ( [ hash , hash2 ] ) ) ) ]
2019-04-17 21:51:50 +00:00
[ ( ( service_keys_to_content_updates , ) , kwargs ) ] = HG . test_controller . GetWrite ( ' content_updates ' )
2020-11-11 22:20:16 +00:00
self . _compare_content_updates ( service_keys_to_content_updates , expected_service_keys_to_content_updates )
2019-04-17 21:51:50 +00:00
2019-02-13 22:26:43 +00:00
def _test_add_urls ( self , connection , set_up_permissions ) :
# get url files
api_permissions = set_up_permissions [ ' everything ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
# none
url = ' https://muhsite.wew/help_compute '
HG . test_controller . SetRead ( ' url_statuses ' , [ ] )
path = ' /add_urls/get_url_files?url= {} ' . format ( urllib . parse . quote ( url , safe = ' ' ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
d = json . loads ( text )
expected_answer = { }
expected_answer [ ' normalised_url ' ] = url
expected_answer [ ' url_file_statuses ' ] = [ ]
self . assertEqual ( d , expected_answer )
# some
2019-11-28 01:11:46 +00:00
url = ' http://safebooru.org/index.php?s=view&page=post&id=2753608 '
2019-02-13 22:26:43 +00:00
normalised_url = ' https://safebooru.org/index.php?id=2753608&page=post&s=view '
hash = os . urandom ( 32 )
2021-06-30 21:27:35 +00:00
url_file_statuses = [ ClientImportFiles . FileImportStatus ( CC . STATUS_SUCCESSFUL_BUT_REDUNDANT , hash , note = ' muh import phrase ' ) ]
2019-02-13 22:26:43 +00:00
json_url_file_statuses = [ { ' status ' : CC . STATUS_SUCCESSFUL_BUT_REDUNDANT , ' hash ' : hash . hex ( ) , ' note ' : ' muh import phrase ' } ]
HG . test_controller . SetRead ( ' url_statuses ' , url_file_statuses )
path = ' /add_urls/get_url_files?url= {} ' . format ( urllib . parse . quote ( url , safe = ' ' ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
d = json . loads ( text )
expected_answer = { }
expected_answer [ ' normalised_url ' ] = normalised_url
expected_answer [ ' url_file_statuses ' ] = json_url_file_statuses
self . assertEqual ( d , expected_answer )
# get url info
api_permissions = set_up_permissions [ ' everything ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
# unknown
url = ' https://muhsite.wew/help_compute '
path = ' /add_urls/get_url_info?url= {} ' . format ( urllib . parse . quote ( url , safe = ' ' ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
d = json . loads ( text )
expected_answer = { }
expected_answer [ ' normalised_url ' ] = url
expected_answer [ ' url_type ' ] = HC . URL_TYPE_UNKNOWN
expected_answer [ ' url_type_string ' ] = ' unknown url '
expected_answer [ ' match_name ' ] = ' unknown url '
expected_answer [ ' can_parse ' ] = False
2021-09-08 21:41:52 +00:00
expected_answer [ ' cannot_parse_reason ' ] = ' unknown url class '
2019-02-13 22:26:43 +00:00
self . assertEqual ( d , expected_answer )
# known
url = ' http://8ch.net/tv/res/1846574.html '
normalised_url = ' https://8ch.net/tv/res/1846574.html '
# http so we can test normalised is https
path = ' /add_urls/get_url_info?url= {} ' . format ( urllib . parse . quote ( url , safe = ' ' ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
d = json . loads ( text )
expected_answer = { }
expected_answer [ ' normalised_url ' ] = normalised_url
expected_answer [ ' url_type ' ] = HC . URL_TYPE_WATCHABLE
expected_answer [ ' url_type_string ' ] = ' watchable url '
expected_answer [ ' match_name ' ] = ' 8chan thread '
expected_answer [ ' can_parse ' ] = True
self . assertEqual ( d , expected_answer )
# known post url
url = ' http://safebooru.org/index.php?page=post&s=view&id=2753608 '
normalised_url = ' https://safebooru.org/index.php?id=2753608&page=post&s=view '
hash = os . urandom ( 32 )
path = ' /add_urls/get_url_info?url= {} ' . format ( urllib . parse . quote ( url , safe = ' ' ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
d = json . loads ( text )
expected_answer = { }
expected_answer [ ' normalised_url ' ] = normalised_url
expected_answer [ ' url_type ' ] = HC . URL_TYPE_POST
expected_answer [ ' url_type_string ' ] = ' post url '
expected_answer [ ' match_name ' ] = ' safebooru file page '
expected_answer [ ' can_parse ' ] = True
self . assertEqual ( d , expected_answer )
# add url
2019-02-27 23:03:30 +00:00
HG . test_controller . ClearWrites ( ' import_url_test ' )
2020-01-16 02:08:23 +00:00
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex , ' Content-Type ' : HC . mime_mimetype_string_lookup [ HC . APPLICATION_JSON ] }
2019-02-13 22:26:43 +00:00
2019-02-27 23:03:30 +00:00
url = ' http://8ch.net/tv/res/1846574.html '
request_dict = { ' url ' : url }
2019-02-13 22:26:43 +00:00
request_body = json . dumps ( request_dict )
connection . request ( ' POST ' , ' /add_urls/add_url ' , body = request_body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
2019-06-26 21:27:18 +00:00
self . assertEqual ( response . status , 200 )
2019-02-13 22:26:43 +00:00
response_json = json . loads ( text )
self . assertEqual ( response_json [ ' human_result_text ' ] , ' " https://8ch.net/tv/res/1846574.html " URL added successfully. ' )
self . assertEqual ( response_json [ ' normalised_url ' ] , ' https://8ch.net/tv/res/1846574.html ' )
2020-09-16 20:46:54 +00:00
self . assertEqual ( HG . test_controller . GetWrite ( ' import_url_test ' ) , [ ( ( url , set ( ) , ClientTags . ServiceKeysToTags ( ) , None , None , False ) , { } ) ] )
2019-02-27 23:03:30 +00:00
# with name
HG . test_controller . ClearWrites ( ' import_url_test ' )
request_dict = { ' url ' : url , ' destination_page_name ' : ' muh /tv/ ' }
request_body = json . dumps ( request_dict )
connection . request ( ' POST ' , ' /add_urls/add_url ' , body = request_body , headers = headers )
response = connection . getresponse ( )
2019-06-26 21:27:18 +00:00
data = response . read ( )
text = str ( data , ' utf-8 ' )
2019-02-27 23:03:30 +00:00
self . assertEqual ( response . status , 200 )
2019-06-26 21:27:18 +00:00
response_json = json . loads ( text )
self . assertEqual ( response_json [ ' human_result_text ' ] , ' " https://8ch.net/tv/res/1846574.html " URL added successfully. ' )
self . assertEqual ( response_json [ ' normalised_url ' ] , ' https://8ch.net/tv/res/1846574.html ' )
2020-09-16 20:46:54 +00:00
self . assertEqual ( HG . test_controller . GetWrite ( ' import_url_test ' ) , [ ( ( url , set ( ) , ClientTags . ServiceKeysToTags ( ) , ' muh /tv/ ' , None , False ) , { } ) ] )
2019-06-26 21:27:18 +00:00
# with page_key
HG . test_controller . ClearWrites ( ' import_url_test ' )
page_key = os . urandom ( 32 )
page_key_hex = page_key . hex ( )
request_dict = { ' url ' : url , ' destination_page_key ' : page_key_hex }
request_body = json . dumps ( request_dict )
connection . request ( ' POST ' , ' /add_urls/add_url ' , body = request_body , headers = headers )
response = connection . getresponse ( )
2019-02-27 23:03:30 +00:00
data = response . read ( )
text = str ( data , ' utf-8 ' )
2019-06-26 21:27:18 +00:00
self . assertEqual ( response . status , 200 )
2019-02-27 23:03:30 +00:00
response_json = json . loads ( text )
self . assertEqual ( response_json [ ' human_result_text ' ] , ' " https://8ch.net/tv/res/1846574.html " URL added successfully. ' )
self . assertEqual ( response_json [ ' normalised_url ' ] , ' https://8ch.net/tv/res/1846574.html ' )
2020-09-16 20:46:54 +00:00
self . assertEqual ( HG . test_controller . GetWrite ( ' import_url_test ' ) , [ ( ( url , set ( ) , ClientTags . ServiceKeysToTags ( ) , None , page_key , False ) , { } ) ] )
2019-02-27 23:03:30 +00:00
2019-03-20 21:22:10 +00:00
# add tags and name, and show destination page
2019-02-27 23:03:30 +00:00
HG . test_controller . ClearWrites ( ' import_url_test ' )
2020-09-16 20:46:54 +00:00
request_dict = { ' url ' : url , ' destination_page_name ' : ' muh /tv/ ' , ' show_destination_page ' : True , ' filterable_tags ' : [ ' filename:yo ' ] , ' service_names_to_additional_tags ' : { ' my tags ' : [ ' /tv/ thread ' ] } }
2019-02-27 23:03:30 +00:00
request_body = json . dumps ( request_dict )
connection . request ( ' POST ' , ' /add_urls/add_url ' , body = request_body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
2019-06-26 21:27:18 +00:00
self . assertEqual ( response . status , 200 )
2019-02-27 23:03:30 +00:00
response_json = json . loads ( text )
self . assertEqual ( response_json [ ' human_result_text ' ] , ' " https://8ch.net/tv/res/1846574.html " URL added successfully. ' )
self . assertEqual ( response_json [ ' normalised_url ' ] , ' https://8ch.net/tv/res/1846574.html ' )
2020-09-16 20:46:54 +00:00
filterable_tags = [ ' filename:yo ' ]
additional_service_keys_to_tags = ClientTags . ServiceKeysToTags ( { CC . DEFAULT_LOCAL_TAG_SERVICE_KEY : set ( [ ' /tv/ thread ' ] ) } )
2019-02-27 23:03:30 +00:00
2020-09-16 20:46:54 +00:00
self . assertEqual ( HG . test_controller . GetWrite ( ' import_url_test ' ) , [ ( ( url , set ( filterable_tags ) , additional_service_keys_to_tags , ' muh /tv/ ' , None , True ) , { } ) ] )
2019-02-27 23:03:30 +00:00
2021-11-03 20:49:56 +00:00
# add tags with service key and name, and show destination page
HG . test_controller . ClearWrites ( ' import_url_test ' )
request_dict = { ' url ' : url , ' destination_page_name ' : ' muh /tv/ ' , ' show_destination_page ' : True , ' filterable_tags ' : [ ' filename:yo ' ] , ' service_keys_to_additional_tags ' : { CC . DEFAULT_LOCAL_TAG_SERVICE_KEY . hex ( ) : [ ' /tv/ thread ' ] } }
request_body = json . dumps ( request_dict )
connection . request ( ' POST ' , ' /add_urls/add_url ' , body = request_body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
response_json = json . loads ( text )
self . assertEqual ( response_json [ ' human_result_text ' ] , ' " https://8ch.net/tv/res/1846574.html " URL added successfully. ' )
self . assertEqual ( response_json [ ' normalised_url ' ] , ' https://8ch.net/tv/res/1846574.html ' )
filterable_tags = [ ' filename:yo ' ]
additional_service_keys_to_tags = ClientTags . ServiceKeysToTags ( { CC . DEFAULT_LOCAL_TAG_SERVICE_KEY : set ( [ ' /tv/ thread ' ] ) } )
self . assertEqual ( HG . test_controller . GetWrite ( ' import_url_test ' ) , [ ( ( url , set ( filterable_tags ) , additional_service_keys_to_tags , ' muh /tv/ ' , None , True ) , { } ) ] )
2019-02-27 23:03:30 +00:00
# associate url
HG . test_controller . ClearWrites ( ' content_updates ' )
hash = bytes . fromhex ( ' 3b820114f658d768550e4e3d4f1dced3ff8db77443472b5ad93700647ad2d3ba ' )
url = ' https://rule34.xxx/index.php?id=2588418&page=post&s=view '
request_dict = { ' url_to_add ' : url , ' hash ' : hash . hex ( ) }
request_body = json . dumps ( request_dict )
connection . request ( ' POST ' , ' /add_urls/associate_url ' , body = request_body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
expected_service_keys_to_content_updates = collections . defaultdict ( list )
expected_service_keys_to_content_updates [ CC . COMBINED_LOCAL_FILE_SERVICE_KEY ] = [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_URLS , HC . CONTENT_UPDATE_ADD , ( [ url ] , [ hash ] ) ) ]
expected_result = [ ( ( expected_service_keys_to_content_updates , ) , { } ) ]
result = HG . test_controller . GetWrite ( ' content_updates ' )
self . assertEqual ( result , expected_result )
#
HG . test_controller . ClearWrites ( ' content_updates ' )
hash = bytes . fromhex ( ' 3b820114f658d768550e4e3d4f1dced3ff8db77443472b5ad93700647ad2d3ba ' )
url = ' https://rule34.xxx/index.php?id=2588418&page=post&s=view '
request_dict = { ' urls_to_add ' : [ url ] , ' hashes ' : [ hash . hex ( ) ] }
request_body = json . dumps ( request_dict )
connection . request ( ' POST ' , ' /add_urls/associate_url ' , body = request_body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
expected_service_keys_to_content_updates = collections . defaultdict ( list )
expected_service_keys_to_content_updates [ CC . COMBINED_LOCAL_FILE_SERVICE_KEY ] = [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_URLS , HC . CONTENT_UPDATE_ADD , ( [ url ] , [ hash ] ) ) ]
expected_result = [ ( ( expected_service_keys_to_content_updates , ) , { } ) ]
result = HG . test_controller . GetWrite ( ' content_updates ' )
self . assertEqual ( result , expected_result )
#
HG . test_controller . ClearWrites ( ' content_updates ' )
hash = bytes . fromhex ( ' 3b820114f658d768550e4e3d4f1dced3ff8db77443472b5ad93700647ad2d3ba ' )
url = ' http://rule34.xxx/index.php?id=2588418&page=post&s=view '
request_dict = { ' url_to_delete ' : url , ' hash ' : hash . hex ( ) }
request_body = json . dumps ( request_dict )
connection . request ( ' POST ' , ' /add_urls/associate_url ' , body = request_body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
expected_service_keys_to_content_updates = collections . defaultdict ( list )
expected_service_keys_to_content_updates [ CC . COMBINED_LOCAL_FILE_SERVICE_KEY ] = [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_URLS , HC . CONTENT_UPDATE_DELETE , ( [ url ] , [ hash ] ) ) ]
expected_result = [ ( ( expected_service_keys_to_content_updates , ) , { } ) ]
result = HG . test_controller . GetWrite ( ' content_updates ' )
self . assertEqual ( result , expected_result )
#
HG . test_controller . ClearWrites ( ' content_updates ' )
hash = bytes . fromhex ( ' 3b820114f658d768550e4e3d4f1dced3ff8db77443472b5ad93700647ad2d3ba ' )
url = ' http://rule34.xxx/index.php?id=2588418&page=post&s=view '
request_dict = { ' urls_to_delete ' : [ url ] , ' hashes ' : [ hash . hex ( ) ] }
request_body = json . dumps ( request_dict )
connection . request ( ' POST ' , ' /add_urls/associate_url ' , body = request_body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
expected_service_keys_to_content_updates = collections . defaultdict ( list )
expected_service_keys_to_content_updates [ CC . COMBINED_LOCAL_FILE_SERVICE_KEY ] = [ HydrusData . ContentUpdate ( HC . CONTENT_TYPE_URLS , HC . CONTENT_UPDATE_DELETE , ( [ url ] , [ hash ] ) ) ]
expected_result = [ ( ( expected_service_keys_to_content_updates , ) , { } ) ]
result = HG . test_controller . GetWrite ( ' content_updates ' )
self . assertEqual ( result , expected_result )
2019-02-13 22:26:43 +00:00
2019-08-07 22:59:53 +00:00
def _test_manage_cookies ( self , connection , set_up_permissions ) :
api_permissions = set_up_permissions [ ' manage_cookies ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
#
path = ' /manage_cookies/get_cookies?domain=somesite.com '
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
d = json . loads ( text )
cookies = d [ ' cookies ' ]
self . assertEqual ( cookies , [ ] )
#
2020-01-16 02:08:23 +00:00
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex , ' Content-Type ' : HC . mime_mimetype_string_lookup [ HC . APPLICATION_JSON ] }
2019-08-07 22:59:53 +00:00
path = ' /manage_cookies/set_cookies '
cookies = [ ]
cookies . append ( [ ' one ' , ' 1 ' , ' .somesite.com ' , ' / ' , HydrusData . GetNow ( ) + 86400 ] )
cookies . append ( [ ' two ' , ' 2 ' , ' somesite.com ' , ' / ' , HydrusData . GetNow ( ) + 86400 ] )
cookies . append ( [ ' three ' , ' 3 ' , ' wew.somesite.com ' , ' / ' , HydrusData . GetNow ( ) + 86400 ] )
cookies . append ( [ ' four ' , ' 4 ' , ' .somesite.com ' , ' / ' , None ] )
request_dict = { ' cookies ' : cookies }
request_body = json . dumps ( request_dict )
connection . request ( ' POST ' , path , body = request_body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
path = ' /manage_cookies/get_cookies?domain=somesite.com '
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
d = json . loads ( text )
result_cookies = d [ ' cookies ' ]
frozen_result_cookies = { tuple ( row ) for row in result_cookies }
frozen_expected_cookies = { tuple ( row ) for row in cookies }
self . assertEqual ( frozen_result_cookies , frozen_expected_cookies )
#
2020-01-16 02:08:23 +00:00
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex , ' Content-Type ' : HC . mime_mimetype_string_lookup [ HC . APPLICATION_JSON ] }
2019-08-07 22:59:53 +00:00
path = ' /manage_cookies/set_cookies '
cookies = [ ]
cookies . append ( [ ' one ' , None , ' .somesite.com ' , ' / ' , None ] )
request_dict = { ' cookies ' : cookies }
request_body = json . dumps ( request_dict )
connection . request ( ' POST ' , path , body = request_body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
path = ' /manage_cookies/get_cookies?domain=somesite.com '
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
d = json . loads ( text )
result_cookies = d [ ' cookies ' ]
expected_cookies = [ ]
expected_cookies . append ( [ ' two ' , ' 2 ' , ' somesite.com ' , ' / ' , HydrusData . GetNow ( ) + 86400 ] )
expected_cookies . append ( [ ' three ' , ' 3 ' , ' wew.somesite.com ' , ' / ' , HydrusData . GetNow ( ) + 86400 ] )
expected_cookies . append ( [ ' four ' , ' 4 ' , ' .somesite.com ' , ' / ' , None ] )
frozen_result_cookies = { tuple ( row ) for row in result_cookies }
frozen_expected_cookies = { tuple ( row ) for row in expected_cookies }
self . assertEqual ( frozen_result_cookies , frozen_expected_cookies )
2021-05-27 00:09:06 +00:00
#
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex , ' Content-Type ' : HC . mime_mimetype_string_lookup [ HC . APPLICATION_JSON ] }
path = ' /manage_headers/set_user_agent '
new_user_agent = ' muh user agent '
request_dict = { ' user-agent ' : new_user_agent }
request_body = json . dumps ( request_dict )
connection . request ( ' POST ' , path , body = request_body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
current_headers = HG . test_controller . network_engine . domain_manager . GetHeaders ( [ ClientNetworkingContexts . GLOBAL_NETWORK_CONTEXT ] )
self . assertEqual ( current_headers [ ' User-Agent ' ] , new_user_agent )
#
request_dict = { ' user-agent ' : ' ' }
request_body = json . dumps ( request_dict )
connection . request ( ' POST ' , path , body = request_body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
current_headers = HG . test_controller . network_engine . domain_manager . GetHeaders ( [ ClientNetworkingContexts . GLOBAL_NETWORK_CONTEXT ] )
from hydrus . client import ClientDefaults
self . assertEqual ( current_headers [ ' User-Agent ' ] , ClientDefaults . DEFAULT_USER_AGENT )
def _test_manage_database ( self , connection , set_up_permissions ) :
api_permissions = set_up_permissions [ ' everything ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
#
self . assertFalse ( HG . client_busy . locked ( ) )
path = ' /manage_database/lock_on '
connection . request ( ' POST ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
self . assertTrue ( HG . client_busy . locked ( ) )
#
path = ' /manage_pages/get_pages '
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 503 )
#
path = ' /manage_database/lock_off '
connection . request ( ' POST ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
self . assertFalse ( HG . client_busy . locked ( ) )
2019-08-07 22:59:53 +00:00
2019-06-05 19:42:39 +00:00
def _test_manage_pages ( self , connection , set_up_permissions ) :
api_permissions = set_up_permissions [ ' manage_pages ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
#
path = ' /manage_pages/get_pages '
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
2019-06-26 21:27:18 +00:00
self . assertEqual ( response . status , 200 )
2019-06-05 19:42:39 +00:00
d = json . loads ( text )
pages = d [ ' pages ' ]
self . assertEqual ( pages [ ' name ' ] , ' top pages notebook ' )
2019-07-17 22:10:19 +00:00
#
2020-01-16 02:08:23 +00:00
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex , ' Content-Type ' : HC . mime_mimetype_string_lookup [ HC . APPLICATION_JSON ] }
2019-07-17 22:10:19 +00:00
path = ' /manage_pages/focus_page '
page_key = os . urandom ( 32 )
request_dict = { ' page_key ' : page_key . hex ( ) }
request_body = json . dumps ( request_dict )
connection . request ( ' POST ' , path , body = request_body , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
result = HG . test_controller . GetWrite ( ' show_page ' )
expected_result = [ ( ( page_key , ) , { } ) ]
self . assertEqual ( result , expected_result )
2019-06-05 19:42:39 +00:00
2019-02-13 22:26:43 +00:00
def _test_search_files ( self , connection , set_up_permissions ) :
2021-08-11 21:14:12 +00:00
hash_ids = [ 1 , 2 , 3 , 4 , 5 , 10 , 15 , 16 , 17 , 18 , 19 , 20 , 21 , 25 , 100 , 101 , 150 ]
2019-03-06 23:06:22 +00:00
# search files failed tag permission
api_permissions = set_up_permissions [ ' search_green_files ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
#
2021-08-11 21:14:12 +00:00
sample_hash_ids = set ( random . sample ( hash_ids , 3 ) )
HG . test_controller . SetRead ( ' file_query_ids ' , set ( sample_hash_ids ) )
2019-03-13 21:04:21 +00:00
tags = [ ]
path = ' /get_files/search_files?tags= {} ' . format ( urllib . parse . quote ( json . dumps ( tags ) ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 403 )
#
2021-08-11 21:14:12 +00:00
sample_hash_ids = set ( random . sample ( hash_ids , 3 ) )
HG . test_controller . SetRead ( ' file_query_ids ' , set ( sample_hash_ids ) )
2019-03-06 23:06:22 +00:00
tags = [ ' kino ' ]
path = ' /get_files/search_files?tags= {} ' . format ( urllib . parse . quote ( json . dumps ( tags ) ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 403 )
2019-02-13 22:26:43 +00:00
# search files
2021-08-18 21:10:01 +00:00
HG . test_controller . ClearReads ( ' file_query_ids ' )
2021-08-11 21:14:12 +00:00
sample_hash_ids = set ( random . sample ( hash_ids , 3 ) )
HG . test_controller . SetRead ( ' file_query_ids ' , set ( sample_hash_ids ) )
2019-03-06 23:06:22 +00:00
tags = [ ' kino ' , ' green ' ]
path = ' /get_files/search_files?tags= {} ' . format ( urllib . parse . quote ( json . dumps ( tags ) ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
2019-06-26 21:27:18 +00:00
self . assertEqual ( response . status , 200 )
2019-03-06 23:06:22 +00:00
d = json . loads ( text )
2021-08-11 21:14:12 +00:00
expected_answer = { ' file_ids ' : list ( sample_hash_ids ) }
2019-03-06 23:06:22 +00:00
self . assertEqual ( d , expected_answer )
2021-08-18 21:10:01 +00:00
[ ( args , kwargs ) ] = HG . test_controller . GetRead ( ' file_query_ids ' )
( file_search_context , ) = args
self . assertEqual ( file_search_context . GetFileServiceKey ( ) , CC . LOCAL_FILE_SERVICE_KEY )
self . assertEqual ( file_search_context . GetTagSearchContext ( ) . service_key , CC . COMBINED_TAG_SERVICE_KEY )
self . assertEqual ( set ( file_search_context . GetPredicates ( ) ) , { ClientSearch . Predicate ( ClientSearch . PREDICATE_TYPE_TAG , tag ) for tag in tags } )
self . assertIn ( ' sort_by ' , kwargs )
sort_by = kwargs [ ' sort_by ' ]
self . assertEqual ( sort_by . sort_type , ( ' system ' , CC . SORT_FILES_BY_IMPORT_TIME ) )
self . assertEqual ( sort_by . sort_order , CC . SORT_DESC )
self . assertIn ( ' apply_implicit_limit ' , kwargs )
self . assertEqual ( kwargs [ ' apply_implicit_limit ' ] , False )
2021-08-11 21:14:12 +00:00
# sort
# this just tests if it parses, we don't have a full test for read params yet
2021-08-18 21:10:01 +00:00
HG . test_controller . ClearReads ( ' file_query_ids ' )
2021-08-11 21:14:12 +00:00
sample_hash_ids = set ( random . sample ( hash_ids , 3 ) )
HG . test_controller . SetRead ( ' file_query_ids ' , set ( sample_hash_ids ) )
tags = [ ' kino ' , ' green ' ]
path = ' /get_files/search_files?tags= {} &file_sort_type= {} ' . format ( urllib . parse . quote ( json . dumps ( tags ) ) , CC . SORT_FILES_BY_FRAMERATE )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
2021-08-18 21:10:01 +00:00
[ ( args , kwargs ) ] = HG . test_controller . GetRead ( ' file_query_ids ' )
( file_search_context , ) = args
self . assertEqual ( file_search_context . GetFileServiceKey ( ) , CC . LOCAL_FILE_SERVICE_KEY )
self . assertEqual ( file_search_context . GetTagSearchContext ( ) . service_key , CC . COMBINED_TAG_SERVICE_KEY )
self . assertEqual ( set ( file_search_context . GetPredicates ( ) ) , { ClientSearch . Predicate ( ClientSearch . PREDICATE_TYPE_TAG , tag ) for tag in tags } )
self . assertIn ( ' sort_by ' , kwargs )
sort_by = kwargs [ ' sort_by ' ]
self . assertEqual ( sort_by . sort_type , ( ' system ' , CC . SORT_FILES_BY_FRAMERATE ) )
self . assertEqual ( sort_by . sort_order , CC . SORT_DESC )
self . assertIn ( ' apply_implicit_limit ' , kwargs )
self . assertEqual ( kwargs [ ' apply_implicit_limit ' ] , False )
2021-08-11 21:14:12 +00:00
# sort
2021-08-18 21:10:01 +00:00
HG . test_controller . ClearReads ( ' file_query_ids ' )
2021-08-11 21:14:12 +00:00
sample_hash_ids = set ( random . sample ( hash_ids , 3 ) )
HG . test_controller . SetRead ( ' file_query_ids ' , set ( sample_hash_ids ) )
tags = [ ' kino ' , ' green ' ]
path = ' /get_files/search_files?tags= {} &file_sort_type= {} &file_sort_asc= {} ' . format ( urllib . parse . quote ( json . dumps ( tags ) ) , CC . SORT_FILES_BY_FRAMERATE , ' true ' )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
2021-08-18 21:10:01 +00:00
[ ( args , kwargs ) ] = HG . test_controller . GetRead ( ' file_query_ids ' )
( file_search_context , ) = args
self . assertEqual ( file_search_context . GetFileServiceKey ( ) , CC . LOCAL_FILE_SERVICE_KEY )
self . assertEqual ( file_search_context . GetTagSearchContext ( ) . service_key , CC . COMBINED_TAG_SERVICE_KEY )
self . assertEqual ( set ( file_search_context . GetPredicates ( ) ) , { ClientSearch . Predicate ( ClientSearch . PREDICATE_TYPE_TAG , tag ) for tag in tags } )
self . assertIn ( ' sort_by ' , kwargs )
sort_by = kwargs [ ' sort_by ' ]
self . assertEqual ( sort_by . sort_type , ( ' system ' , CC . SORT_FILES_BY_FRAMERATE ) )
self . assertEqual ( sort_by . sort_order , CC . SORT_ASC )
self . assertIn ( ' apply_implicit_limit ' , kwargs )
self . assertEqual ( kwargs [ ' apply_implicit_limit ' ] , False )
2021-08-11 21:14:12 +00:00
# file domain
2021-08-18 21:10:01 +00:00
HG . test_controller . ClearReads ( ' file_query_ids ' )
2021-08-11 21:14:12 +00:00
sample_hash_ids = set ( random . sample ( hash_ids , 3 ) )
HG . test_controller . SetRead ( ' file_query_ids ' , set ( sample_hash_ids ) )
tags = [ ' kino ' , ' green ' ]
path = ' /get_files/search_files?tags= {} &file_sort_type= {} &file_sort_asc= {} &file_service_name= {} ' . format (
urllib . parse . quote ( json . dumps ( tags ) ) ,
CC . SORT_FILES_BY_FRAMERATE ,
' true ' ,
' trash '
)
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
2021-08-18 21:10:01 +00:00
[ ( args , kwargs ) ] = HG . test_controller . GetRead ( ' file_query_ids ' )
( file_search_context , ) = args
self . assertEqual ( file_search_context . GetFileServiceKey ( ) , CC . TRASH_SERVICE_KEY )
self . assertEqual ( file_search_context . GetTagSearchContext ( ) . service_key , CC . COMBINED_TAG_SERVICE_KEY )
self . assertEqual ( set ( file_search_context . GetPredicates ( ) ) , { ClientSearch . Predicate ( ClientSearch . PREDICATE_TYPE_TAG , tag ) for tag in tags } )
self . assertIn ( ' sort_by ' , kwargs )
sort_by = kwargs [ ' sort_by ' ]
self . assertEqual ( sort_by . sort_type , ( ' system ' , CC . SORT_FILES_BY_FRAMERATE ) )
self . assertEqual ( sort_by . sort_order , CC . SORT_ASC )
self . assertIn ( ' apply_implicit_limit ' , kwargs )
self . assertEqual ( kwargs [ ' apply_implicit_limit ' ] , False )
2021-08-11 21:14:12 +00:00
# file and tag domain
2021-08-18 21:10:01 +00:00
HG . test_controller . ClearReads ( ' file_query_ids ' )
2021-08-11 21:14:12 +00:00
sample_hash_ids = set ( random . sample ( hash_ids , 3 ) )
HG . test_controller . SetRead ( ' file_query_ids ' , set ( sample_hash_ids ) )
tags = [ ' kino ' , ' green ' ]
path = ' /get_files/search_files?tags= {} &file_sort_type= {} &file_sort_asc= {} &file_service_key= {} &tag_service_name= {} ' . format (
urllib . parse . quote ( json . dumps ( tags ) ) ,
CC . SORT_FILES_BY_FRAMERATE ,
' true ' ,
CC . TRASH_SERVICE_KEY . hex ( ) ,
' all % 20known % 20tags '
)
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
2021-08-18 21:10:01 +00:00
[ ( args , kwargs ) ] = HG . test_controller . GetRead ( ' file_query_ids ' )
( file_search_context , ) = args
self . assertEqual ( file_search_context . GetFileServiceKey ( ) , CC . TRASH_SERVICE_KEY )
self . assertEqual ( file_search_context . GetTagSearchContext ( ) . service_key , CC . COMBINED_TAG_SERVICE_KEY )
self . assertEqual ( set ( file_search_context . GetPredicates ( ) ) , { ClientSearch . Predicate ( ClientSearch . PREDICATE_TYPE_TAG , tag ) for tag in tags } )
self . assertIn ( ' sort_by ' , kwargs )
sort_by = kwargs [ ' sort_by ' ]
self . assertEqual ( sort_by . sort_type , ( ' system ' , CC . SORT_FILES_BY_FRAMERATE ) )
self . assertEqual ( sort_by . sort_order , CC . SORT_ASC )
self . assertIn ( ' apply_implicit_limit ' , kwargs )
self . assertEqual ( kwargs [ ' apply_implicit_limit ' ] , False )
2021-08-11 21:14:12 +00:00
# file and tag domain
# this just tests if it parses, we don't have a full test for read params yet
sample_hash_ids = set ( random . sample ( hash_ids , 3 ) )
HG . test_controller . SetRead ( ' file_query_ids ' , set ( sample_hash_ids ) )
tags = [ ' kino ' , ' green ' ]
path = ' /get_files/search_files?tags= {} &file_sort_type= {} &file_sort_asc= {} &file_service_key= {} &tag_service_key= {} ' . format (
urllib . parse . quote ( json . dumps ( tags ) ) ,
CC . SORT_FILES_BY_FRAMERATE ,
' true ' ,
CC . COMBINED_FILE_SERVICE_KEY . hex ( ) ,
CC . COMBINED_TAG_SERVICE_KEY . hex ( )
)
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 400 )
def _test_search_files_predicate_parsing ( self , connection , set_up_permissions ) :
2019-03-06 23:06:22 +00:00
# some file search param parsing
2019-03-13 21:04:21 +00:00
class PretendRequest ( object ) :
pass
2019-03-06 23:06:22 +00:00
pretend_request = PretendRequest ( )
pretend_request . parsed_request_args = { }
pretend_request . client_api_permissions = set_up_permissions [ ' everything ' ]
predicates = ClientLocalServerResources . ParseClientAPISearchPredicates ( pretend_request )
self . assertEqual ( predicates , [ ] )
#
pretend_request = PretendRequest ( )
pretend_request . parsed_request_args = { ' system_inbox ' : True }
pretend_request . client_api_permissions = set_up_permissions [ ' search_green_files ' ]
with self . assertRaises ( HydrusExceptions . InsufficientCredentialsException ) :
ClientLocalServerResources . ParseClientAPISearchPredicates ( pretend_request )
#
pretend_request = PretendRequest ( )
pretend_request . parsed_request_args = { ' tags ' : [ ' -green ' ] }
pretend_request . client_api_permissions = set_up_permissions [ ' search_green_files ' ]
with self . assertRaises ( HydrusExceptions . InsufficientCredentialsException ) :
ClientLocalServerResources . ParseClientAPISearchPredicates ( pretend_request )
#
pretend_request = PretendRequest ( )
pretend_request . parsed_request_args = { ' tags ' : [ ' green ' , ' -kino ' ] }
pretend_request . client_api_permissions = set_up_permissions [ ' search_green_files ' ]
predicates = ClientLocalServerResources . ParseClientAPISearchPredicates ( pretend_request )
expected_predicates = [ ]
2020-03-11 21:52:11 +00:00
expected_predicates . append ( ClientSearch . Predicate ( predicate_type = ClientSearch . PREDICATE_TYPE_TAG , value = ' green ' ) )
expected_predicates . append ( ClientSearch . Predicate ( predicate_type = ClientSearch . PREDICATE_TYPE_TAG , value = ' kino ' , inclusive = False ) )
2019-03-06 23:06:22 +00:00
self . assertEqual ( set ( predicates ) , set ( expected_predicates ) )
#
pretend_request = PretendRequest ( )
pretend_request . parsed_request_args = { ' tags ' : [ ' green ' ] , ' system_inbox ' : True }
pretend_request . client_api_permissions = set_up_permissions [ ' search_green_files ' ]
predicates = ClientLocalServerResources . ParseClientAPISearchPredicates ( pretend_request )
expected_predicates = [ ]
2020-03-11 21:52:11 +00:00
expected_predicates . append ( ClientSearch . Predicate ( predicate_type = ClientSearch . PREDICATE_TYPE_TAG , value = ' green ' ) )
expected_predicates . append ( ClientSearch . Predicate ( predicate_type = ClientSearch . PREDICATE_TYPE_SYSTEM_INBOX ) )
2019-03-06 23:06:22 +00:00
self . assertEqual ( set ( predicates ) , set ( expected_predicates ) )
#
pretend_request = PretendRequest ( )
pretend_request . parsed_request_args = { ' tags ' : [ ' green ' ] , ' system_archive ' : True }
pretend_request . client_api_permissions = set_up_permissions [ ' search_green_files ' ]
predicates = ClientLocalServerResources . ParseClientAPISearchPredicates ( pretend_request )
expected_predicates = [ ]
2020-03-11 21:52:11 +00:00
expected_predicates . append ( ClientSearch . Predicate ( predicate_type = ClientSearch . PREDICATE_TYPE_TAG , value = ' green ' ) )
expected_predicates . append ( ClientSearch . Predicate ( predicate_type = ClientSearch . PREDICATE_TYPE_SYSTEM_ARCHIVE ) )
2019-03-06 23:06:22 +00:00
self . assertEqual ( set ( predicates ) , set ( expected_predicates ) )
2021-07-28 21:12:00 +00:00
#
pretend_request = PretendRequest ( )
pretend_request . parsed_request_args = { ' tags ' : [ ' green ' , ' system:archive ' ] }
pretend_request . client_api_permissions = set_up_permissions [ ' search_green_files ' ]
predicates = ClientLocalServerResources . ParseClientAPISearchPredicates ( pretend_request )
expected_predicates = [ ]
expected_predicates . append ( ClientSearch . Predicate ( predicate_type = ClientSearch . PREDICATE_TYPE_TAG , value = ' green ' ) )
expected_predicates . append ( ClientSearch . Predicate ( predicate_type = ClientSearch . PREDICATE_TYPE_SYSTEM_ARCHIVE ) )
self . assertEqual ( set ( predicates ) , set ( expected_predicates ) )
2021-09-01 21:09:01 +00:00
#
pretend_request = PretendRequest ( )
pretend_request . parsed_request_args = { ' tags ' : [ ' green ' , [ ' red ' , ' blue ' ] , ' system:archive ' ] }
pretend_request . client_api_permissions = set_up_permissions [ ' search_green_files ' ]
predicates = ClientLocalServerResources . ParseClientAPISearchPredicates ( pretend_request )
expected_predicates = [ ]
expected_predicates . append ( ClientSearch . Predicate ( predicate_type = ClientSearch . PREDICATE_TYPE_TAG , value = ' green ' ) )
expected_predicates . append (
ClientSearch . Predicate (
predicate_type = ClientSearch . PREDICATE_TYPE_OR_CONTAINER ,
value = [
ClientSearch . Predicate ( predicate_type = ClientSearch . PREDICATE_TYPE_TAG , value = ' red ' ) ,
ClientSearch . Predicate ( predicate_type = ClientSearch . PREDICATE_TYPE_TAG , value = ' blue ' )
]
)
)
expected_predicates . append ( ClientSearch . Predicate ( predicate_type = ClientSearch . PREDICATE_TYPE_SYSTEM_ARCHIVE ) )
self . assertEqual ( { pred for pred in predicates if pred . GetType ( ) != ClientSearch . PREDICATE_TYPE_OR_CONTAINER } , { pred for pred in expected_predicates if pred . GetType ( ) != ClientSearch . PREDICATE_TYPE_OR_CONTAINER } )
self . assertEqual ( { frozenset ( pred . GetValue ( ) ) for pred in predicates if pred . GetType ( ) == ClientSearch . PREDICATE_TYPE_OR_CONTAINER } , { frozenset ( pred . GetValue ( ) ) for pred in expected_predicates if pred . GetType ( ) == ClientSearch . PREDICATE_TYPE_OR_CONTAINER } )
2021-08-11 21:14:12 +00:00
def _test_file_metadata ( self , connection , set_up_permissions ) :
2019-03-13 21:04:21 +00:00
# test file metadata
api_permissions = set_up_permissions [ ' search_green_files ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
file_ids_to_hashes = { 1 : bytes . fromhex ( ' a ' * 64 ) , 2 : bytes . fromhex ( ' b ' * 64 ) , 3 : bytes . fromhex ( ' c ' * 64 ) }
metadata = [ ]
for ( file_id , hash ) in file_ids_to_hashes . items ( ) :
metadata_row = { ' file_id ' : file_id , ' hash ' : hash . hex ( ) }
metadata . append ( metadata_row )
expected_identifier_result = { ' metadata ' : metadata }
media_results = [ ]
2019-08-07 22:59:53 +00:00
urls = { " https://gelbooru.com/index.php?page=post&s=view&id=4841557 " , " https://img2.gelbooru.com//images/80/c8/80c8646b4a49395fb36c805f316c49a9.jpg " }
2020-05-13 19:03:16 +00:00
sorted_urls = sorted ( urls )
2019-08-07 22:59:53 +00:00
2019-03-13 21:04:21 +00:00
for ( file_id , hash ) in file_ids_to_hashes . items ( ) :
size = random . randint ( 8192 , 20 * 1048576 )
mime = random . choice ( [ HC . IMAGE_JPEG , HC . VIDEO_WEBM , HC . APPLICATION_PDF ] )
width = random . randint ( 200 , 4096 )
height = random . randint ( 200 , 4096 )
duration = random . choice ( [ 220 , 16.66667 , None ] )
2019-08-07 22:59:53 +00:00
has_audio = random . choice ( [ True , False ] )
2019-03-13 21:04:21 +00:00
2020-05-06 21:31:41 +00:00
file_info_manager = ClientMediaManagers . FileInfoManager ( file_id , hash , size = size , mime = mime , width = width , height = height , duration = duration , has_audio = has_audio )
2019-03-13 21:04:21 +00:00
2020-11-25 22:22:47 +00:00
service_keys_to_statuses_to_tags = { CC . DEFAULT_LOCAL_TAG_SERVICE_KEY : { HC . CONTENT_STATUS_CURRENT : [ ' blue_eyes ' , ' blonde_hair ' ] , HC . CONTENT_STATUS_PENDING : [ ' bodysuit ' ] } }
service_keys_to_statuses_to_display_tags = { CC . DEFAULT_LOCAL_TAG_SERVICE_KEY : { HC . CONTENT_STATUS_CURRENT : [ ' blue eyes ' , ' blonde hair ' ] , HC . CONTENT_STATUS_PENDING : [ ' bodysuit ' , ' clothing ' ] } }
2019-03-13 21:04:21 +00:00
2020-08-19 22:38:20 +00:00
tags_manager = ClientMediaManagers . TagsManager ( service_keys_to_statuses_to_tags , service_keys_to_statuses_to_display_tags )
2019-03-13 21:04:21 +00:00
2021-04-28 21:43:16 +00:00
locations_manager = ClientMediaManagers . LocationsManager ( dict ( ) , dict ( ) , set ( ) , set ( ) , inbox = False , urls = urls )
2020-05-06 21:31:41 +00:00
ratings_manager = ClientMediaManagers . RatingsManager ( { } )
notes_manager = ClientMediaManagers . NotesManager ( { } )
file_viewing_stats_manager = ClientMediaManagers . FileViewingStatsManager ( 0 , 0 , 0 , 0 )
2019-03-13 21:04:21 +00:00
2020-05-20 21:36:02 +00:00
media_result = ClientMediaResult . MediaResult ( file_info_manager , tags_manager , locations_manager , ratings_manager , notes_manager , file_viewing_stats_manager )
2019-03-13 21:04:21 +00:00
media_results . append ( media_result )
metadata = [ ]
2020-06-24 21:25:24 +00:00
detailed_known_urls_metadata = [ ]
2019-03-13 21:04:21 +00:00
services_manager = HG . client_controller . services_manager
service_keys_to_names = { }
for media_result in media_results :
metadata_row = { }
file_info_manager = media_result . GetFileInfoManager ( )
metadata_row [ ' file_id ' ] = file_info_manager . hash_id
metadata_row [ ' hash ' ] = file_info_manager . hash . hex ( )
metadata_row [ ' size ' ] = file_info_manager . size
2020-01-16 02:08:23 +00:00
metadata_row [ ' mime ' ] = HC . mime_mimetype_string_lookup [ file_info_manager . mime ]
2020-04-16 00:09:42 +00:00
metadata_row [ ' ext ' ] = HC . mime_ext_lookup [ file_info_manager . mime ]
2019-03-13 21:04:21 +00:00
metadata_row [ ' width ' ] = file_info_manager . width
metadata_row [ ' height ' ] = file_info_manager . height
metadata_row [ ' duration ' ] = file_info_manager . duration
2019-08-07 22:59:53 +00:00
metadata_row [ ' has_audio ' ] = file_info_manager . has_audio
2019-03-13 21:04:21 +00:00
metadata_row [ ' num_frames ' ] = file_info_manager . num_frames
metadata_row [ ' num_words ' ] = file_info_manager . num_words
2020-06-11 12:01:08 +00:00
metadata_row [ ' is_inbox ' ] = False
metadata_row [ ' is_local ' ] = False
metadata_row [ ' is_trashed ' ] = False
2019-08-07 22:59:53 +00:00
metadata_row [ ' known_urls ' ] = list ( sorted_urls )
2019-03-13 21:04:21 +00:00
tags_manager = media_result . GetTagsManager ( )
service_names_to_statuses_to_tags = { }
2021-11-03 20:49:56 +00:00
api_service_keys_to_statuses_to_tags = { }
2019-03-13 21:04:21 +00:00
2019-10-02 23:38:59 +00:00
service_keys_to_statuses_to_tags = tags_manager . GetServiceKeysToStatusesToTags ( ClientTags . TAG_DISPLAY_STORAGE )
2019-03-13 21:04:21 +00:00
for ( service_key , statuses_to_tags ) in service_keys_to_statuses_to_tags . items ( ) :
if service_key not in service_keys_to_names :
service_keys_to_names [ service_key ] = services_manager . GetName ( service_key )
2021-11-03 20:49:56 +00:00
s = { str ( status ) : sorted ( tags , key = HydrusTags . ConvertTagToSortable ) for ( status , tags ) in statuses_to_tags . items ( ) if len ( tags ) > 0 }
2019-03-13 21:04:21 +00:00
2021-11-03 20:49:56 +00:00
if len ( s ) > 0 :
service_name = service_keys_to_names [ service_key ]
service_names_to_statuses_to_tags [ service_name ] = s
api_service_keys_to_statuses_to_tags [ service_key . hex ( ) ] = s
2019-03-13 21:04:21 +00:00
metadata_row [ ' service_names_to_statuses_to_tags ' ] = service_names_to_statuses_to_tags
2021-11-03 20:49:56 +00:00
metadata_row [ ' service_keys_to_statuses_to_tags ' ] = api_service_keys_to_statuses_to_tags
2019-03-13 21:04:21 +00:00
2021-11-03 20:49:56 +00:00
service_names_to_statuses_to_display_tags = { }
service_keys_to_statuses_to_display_tags = { }
2020-11-25 22:22:47 +00:00
service_keys_to_statuses_to_tags = tags_manager . GetServiceKeysToStatusesToTags ( ClientTags . TAG_DISPLAY_ACTUAL )
for ( service_key , statuses_to_tags ) in service_keys_to_statuses_to_tags . items ( ) :
if service_key not in service_keys_to_names :
service_keys_to_names [ service_key ] = services_manager . GetName ( service_key )
2021-11-03 20:49:56 +00:00
s = { str ( status ) : sorted ( tags , key = HydrusTags . ConvertTagToSortable ) for ( status , tags ) in statuses_to_tags . items ( ) if len ( tags ) > 0 }
2020-11-25 22:22:47 +00:00
2021-11-03 20:49:56 +00:00
if len ( s ) > 0 :
service_name = service_keys_to_names [ service_key ]
service_names_to_statuses_to_display_tags [ service_name ] = s
service_keys_to_statuses_to_display_tags [ service_key . hex ( ) ] = s
2020-11-25 22:22:47 +00:00
2021-11-03 20:49:56 +00:00
metadata_row [ ' service_names_to_statuses_to_display_tags ' ] = service_names_to_statuses_to_display_tags
metadata_row [ ' service_keys_to_statuses_to_display_tags ' ] = service_keys_to_statuses_to_display_tags
2020-11-25 22:22:47 +00:00
2019-03-13 21:04:21 +00:00
metadata . append ( metadata_row )
2020-06-24 21:25:24 +00:00
detailed_known_urls_metadata_row = dict ( metadata_row )
detailed_known_urls_metadata_row [ ' detailed_known_urls ' ] = [
{ ' normalised_url ' : ' https://gelbooru.com/index.php?id=4841557&page=post&s=view ' , ' url_type ' : 0 , ' url_type_string ' : ' post url ' , ' match_name ' : ' gelbooru file page ' , ' can_parse ' : True } ,
2021-09-08 21:41:52 +00:00
{ ' normalised_url ' : ' https://img2.gelbooru.com//images/80/c8/80c8646b4a49395fb36c805f316c49a9.jpg ' , ' url_type ' : 5 , ' url_type_string ' : ' unknown url ' , ' match_name ' : ' unknown url ' , ' can_parse ' : False , ' cannot_parse_reason ' : ' unknown url class ' }
2020-06-24 21:25:24 +00:00
]
detailed_known_urls_metadata . append ( detailed_known_urls_metadata_row )
2019-03-13 21:04:21 +00:00
expected_metadata_result = { ' metadata ' : metadata }
2020-06-24 21:25:24 +00:00
expected_detailed_known_urls_metadata_result = { ' metadata ' : detailed_known_urls_metadata }
2019-03-13 21:04:21 +00:00
HG . test_controller . SetRead ( ' hash_ids_to_hashes ' , file_ids_to_hashes )
HG . test_controller . SetRead ( ' media_results ' , media_results )
HG . test_controller . SetRead ( ' media_results_from_ids ' , media_results )
api_permissions . SetLastSearchResults ( [ 1 , 2 , 3 , 4 , 5 , 6 ] )
# fail on non-permitted files
path = ' /get_files/file_metadata?file_ids= {} &only_return_identifiers=true ' . format ( urllib . parse . quote ( json . dumps ( [ 1 , 2 , 3 , 7 ] ) ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 403 )
# fails on hashes even if the hashes are 'good'
path = ' /get_files/file_metadata?hashes= {} &only_return_identifiers=true ' . format ( urllib . parse . quote ( json . dumps ( [ hash . hex ( ) for hash in file_ids_to_hashes . values ( ) ] ) ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 403 )
# identifiers from file_ids
path = ' /get_files/file_metadata?file_ids= {} &only_return_identifiers=true ' . format ( urllib . parse . quote ( json . dumps ( [ 1 , 2 , 3 ] ) ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
2019-06-26 21:27:18 +00:00
self . assertEqual ( response . status , 200 )
2019-03-13 21:04:21 +00:00
d = json . loads ( text )
self . assertEqual ( d , expected_identifier_result )
# metadata from file_ids
path = ' /get_files/file_metadata?file_ids= {} ' . format ( urllib . parse . quote ( json . dumps ( [ 1 , 2 , 3 ] ) ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
2019-06-26 21:27:18 +00:00
self . assertEqual ( response . status , 200 )
2019-03-13 21:04:21 +00:00
d = json . loads ( text )
self . assertEqual ( d , expected_metadata_result )
# now from hashes
api_permissions = set_up_permissions [ ' everything ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
# identifiers from hashes
path = ' /get_files/file_metadata?hashes= {} &only_return_identifiers=true ' . format ( urllib . parse . quote ( json . dumps ( [ hash . hex ( ) for hash in file_ids_to_hashes . values ( ) ] ) ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
2019-06-26 21:27:18 +00:00
self . assertEqual ( response . status , 200 )
2019-03-13 21:04:21 +00:00
d = json . loads ( text )
self . assertEqual ( d , expected_identifier_result )
# metadata from hashes
path = ' /get_files/file_metadata?hashes= {} ' . format ( urllib . parse . quote ( json . dumps ( [ hash . hex ( ) for hash in file_ids_to_hashes . values ( ) ] ) ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
2019-06-26 21:27:18 +00:00
self . assertEqual ( response . status , 200 )
2019-03-13 21:04:21 +00:00
d = json . loads ( text )
self . assertEqual ( d , expected_metadata_result )
2021-09-01 21:09:01 +00:00
# fails on borked hashes
path = ' /get_files/file_metadata?hashes= {} ' . format ( urllib . parse . quote ( json . dumps ( [ ' deadbeef ' ] ) ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 400 )
2020-06-24 21:25:24 +00:00
# metadata from hashes with detailed url info
path = ' /get_files/file_metadata?hashes= {} &detailed_url_information=true ' . format ( urllib . parse . quote ( json . dumps ( [ hash . hex ( ) for hash in file_ids_to_hashes . values ( ) ] ) ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 200 )
d = json . loads ( text )
self . assertEqual ( d , expected_detailed_known_urls_metadata_result )
2021-09-01 21:09:01 +00:00
# failure on missing file_ids
HG . test_controller . SetRead ( ' media_results_from_ids ' , HydrusExceptions . DataMissing ( ' test missing ' ) )
api_permissions = set_up_permissions [ ' everything ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
path = ' /get_files/file_metadata?file_ids= {} ' . format ( urllib . parse . quote ( json . dumps ( [ 123456 ] ) ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
text = str ( data , ' utf-8 ' )
self . assertEqual ( response . status , 404 )
self . assertIn ( ' test missing ' , text )
2021-08-11 21:14:12 +00:00
def _test_get_files ( self , connection , set_up_permissions ) :
2019-03-13 21:04:21 +00:00
# files and thumbs
2021-08-11 21:14:12 +00:00
file_id = 1
2019-03-13 21:04:21 +00:00
hash = b ' \xad m5 \x99 \xa6 \xc4 \x89 \xa5 u \xeb \x19 \xc0 & \xfa \xce \x97 \xa9 \xcd ey \xe7 G( \xb0 \xce \x94 \xa6 \x01 \xd2 2 \xf3 \xc3 '
hash_hex = hash . hex ( )
size = 100
mime = HC . IMAGE_PNG
width = 20
height = 20
duration = None
2020-05-06 21:31:41 +00:00
file_info_manager = ClientMediaManagers . FileInfoManager ( file_id , hash , size = size , mime = mime , width = width , height = height , duration = duration )
2019-03-13 21:04:21 +00:00
2020-11-25 22:22:47 +00:00
service_keys_to_statuses_to_tags = { CC . DEFAULT_LOCAL_TAG_SERVICE_KEY : { HC . CONTENT_STATUS_CURRENT : [ ' blue_eyes ' , ' blonde_hair ' ] , HC . CONTENT_STATUS_PENDING : [ ' bodysuit ' ] } }
service_keys_to_statuses_to_display_tags = { CC . DEFAULT_LOCAL_TAG_SERVICE_KEY : { HC . CONTENT_STATUS_CURRENT : [ ' blue eyes ' , ' blonde hair ' ] , HC . CONTENT_STATUS_PENDING : [ ' bodysuit ' , ' clothing ' ] } }
2019-03-13 21:04:21 +00:00
2020-08-19 22:38:20 +00:00
tags_manager = ClientMediaManagers . TagsManager ( service_keys_to_statuses_to_tags , service_keys_to_statuses_to_display_tags )
2019-03-13 21:04:21 +00:00
2021-04-28 21:43:16 +00:00
locations_manager = ClientMediaManagers . LocationsManager ( dict ( ) , dict ( ) , set ( ) , set ( ) )
2020-05-06 21:31:41 +00:00
ratings_manager = ClientMediaManagers . RatingsManager ( { } )
notes_manager = ClientMediaManagers . NotesManager ( { } )
file_viewing_stats_manager = ClientMediaManagers . FileViewingStatsManager ( 0 , 0 , 0 , 0 )
2019-03-13 21:04:21 +00:00
2020-05-20 21:36:02 +00:00
media_result = ClientMediaResult . MediaResult ( file_info_manager , tags_manager , locations_manager , ratings_manager , notes_manager , file_viewing_stats_manager )
2019-03-13 21:04:21 +00:00
2020-04-29 21:44:12 +00:00
HG . test_controller . SetRead ( ' media_result ' , media_result )
2019-03-13 21:04:21 +00:00
HG . test_controller . SetRead ( ' media_results_from_ids ' , ( media_result , ) )
path = os . path . join ( HC . STATIC_DIR , ' hydrus.png ' )
file_path = HG . test_controller . client_files_manager . GetFilePath ( hash , HC . IMAGE_PNG , check_file_exists = False )
shutil . copy2 ( path , file_path )
thumb_hash = b ' \x17 \xde \xd6 \xee \x1b \xfa \x00 2 \xbd j \xc0 w \x92 \xce 5 \xf0 \x12 ~ \xfe \x91 5 \xb3 \xb3 tA \xac \x90 F \x95 \xc2 T \xc5 '
path = os . path . join ( HC . STATIC_DIR , ' hydrus_small.png ' )
2019-03-27 22:01:02 +00:00
thumb_path = HG . test_controller . client_files_manager . _GenerateExpectedThumbnailPath ( hash )
2019-03-13 21:04:21 +00:00
shutil . copy2 ( path , thumb_path )
api_permissions = set_up_permissions [ ' search_green_files ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
# let's fail first
path = ' /get_files/file?file_id= {} ' . format ( 10 )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 403 )
#
path = ' /get_files/thumbnail?file_id= {} ' . format ( 10 )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 403 )
#
path = ' /get_files/file?hash= {} ' . format ( hash_hex )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 403 )
#
path = ' /get_files/thumbnail?hash= {} ' . format ( hash_hex )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 403 )
# now succeed
path = ' /get_files/file?file_id= {} ' . format ( 1 )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
self . assertEqual ( hashlib . sha256 ( data ) . digest ( ) , hash )
2021-03-10 23:10:11 +00:00
# range request
path = ' /get_files/file?file_id= {} ' . format ( 1 )
partial_headers = dict ( headers )
partial_headers [ ' Range ' ] = ' bytes=100-199 '
connection . request ( ' GET ' , path , headers = partial_headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 206 )
with open ( file_path , ' rb ' ) as f :
f . seek ( 100 )
actual_data = f . read ( 100 )
self . assertEqual ( data , actual_data )
# n onwards range request
path = ' /get_files/file?file_id= {} ' . format ( 1 )
partial_headers = dict ( headers )
partial_headers [ ' Range ' ] = ' bytes=100- '
connection . request ( ' GET ' , path , headers = partial_headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 206 )
with open ( file_path , ' rb ' ) as f :
f . seek ( 100 )
actual_data = f . read ( )
self . assertEqual ( data , actual_data )
# last n onwards range request
path = ' /get_files/file?file_id= {} ' . format ( 1 )
partial_headers = dict ( headers )
partial_headers [ ' Range ' ] = ' bytes=-100 '
connection . request ( ' GET ' , path , headers = partial_headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 206 )
with open ( file_path , ' rb ' ) as f :
actual_data = f . read ( ) [ - 100 : ]
self . assertEqual ( data , actual_data )
# invalid range request
path = ' /get_files/file?file_id= {} ' . format ( 1 )
partial_headers = dict ( headers )
partial_headers [ ' Range ' ] = ' bytes=200-199 '
connection . request ( ' GET ' , path , headers = partial_headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 416 )
# multi range request, not currently supported
path = ' /get_files/file?file_id= {} ' . format ( 1 )
partial_headers = dict ( headers )
partial_headers [ ' Range ' ] = ' bytes=100-199,300-399 '
connection . request ( ' GET ' , path , headers = partial_headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 416 )
2019-03-13 21:04:21 +00:00
#
path = ' /get_files/thumbnail?file_id= {} ' . format ( 1 )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
self . assertEqual ( hashlib . sha256 ( data ) . digest ( ) , thumb_hash )
#
api_permissions = set_up_permissions [ ' everything ' ]
access_key_hex = api_permissions . GetAccessKey ( ) . hex ( )
headers = { ' Hydrus-Client-API-Access-Key ' : access_key_hex }
#
path = ' /get_files/file?hash= {} ' . format ( hash_hex )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
self . assertEqual ( hashlib . sha256 ( data ) . digest ( ) , hash )
#
path = ' /get_files/thumbnail?hash= {} ' . format ( hash_hex )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
self . assertEqual ( hashlib . sha256 ( data ) . digest ( ) , thumb_hash )
2021-05-27 00:09:06 +00:00
# with "sha256:"" on the front
path = ' /get_files/thumbnail?hash= {} {} ' . format ( ' sha256: ' , hash_hex )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 200 )
self . assertEqual ( hashlib . sha256 ( data ) . digest ( ) , thumb_hash )
2019-03-13 21:04:21 +00:00
# now 404
hash_404 = os . urandom ( 32 )
2020-05-06 21:31:41 +00:00
file_info_manager = ClientMediaManagers . FileInfoManager ( 123456 , hash_404 , size = size , mime = mime , width = width , height = height , duration = duration )
2019-03-13 21:04:21 +00:00
2020-05-20 21:36:02 +00:00
media_result = ClientMediaResult . MediaResult ( file_info_manager , tags_manager , locations_manager , ratings_manager , notes_manager , file_viewing_stats_manager )
2019-03-13 21:04:21 +00:00
2020-04-29 21:44:12 +00:00
HG . test_controller . SetRead ( ' media_result ' , media_result )
2019-03-13 21:04:21 +00:00
HG . test_controller . SetRead ( ' media_results_from_ids ' , ( media_result , ) )
#
path = ' /get_files/file?hash= {} ' . format ( hash_404 . hex ( ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 404 )
#
path = ' /get_files/thumbnail?hash= {} ' . format ( hash_404 . hex ( ) )
connection . request ( ' GET ' , path , headers = headers )
response = connection . getresponse ( )
data = response . read ( )
self . assertEqual ( response . status , 404 )
#
os . unlink ( file_path )
os . unlink ( thumb_path )
2019-02-13 22:26:43 +00:00
def _test_permission_failures ( self , connection , set_up_permissions ) :
pass
# failed permission tests
def test_client_api ( self ) :
host = ' 127.0.0.1 '
port = 45869
connection = http . client . HTTPConnection ( host , port , timeout = 10 )
self . _test_basics ( connection )
set_up_permissions = self . _test_client_api_basics ( connection )
2021-05-27 00:09:06 +00:00
self . _test_get_services ( connection , set_up_permissions )
self . _test_manage_database ( connection , set_up_permissions )
2020-11-11 22:20:16 +00:00
self . _test_add_files_add_file ( connection , set_up_permissions )
self . _test_add_files_other_actions ( connection , set_up_permissions )
2019-02-13 22:26:43 +00:00
self . _test_add_tags ( connection , set_up_permissions )
self . _test_add_urls ( connection , set_up_permissions )
2019-08-07 22:59:53 +00:00
self . _test_manage_cookies ( connection , set_up_permissions )
2019-06-05 19:42:39 +00:00
self . _test_manage_pages ( connection , set_up_permissions )
2019-02-13 22:26:43 +00:00
self . _test_search_files ( connection , set_up_permissions )
2021-08-11 21:14:12 +00:00
self . _test_search_files_predicate_parsing ( connection , set_up_permissions )
self . _test_file_metadata ( connection , set_up_permissions )
self . _test_get_files ( connection , set_up_permissions )
2019-02-13 22:26:43 +00:00
self . _test_permission_failures ( connection , set_up_permissions )
2019-04-17 21:51:50 +00:00
self . _test_cors_fails ( connection )
connection . close ( )
#
port = 45899
connection = http . client . HTTPConnection ( host , port , timeout = 10 )
self . _test_cors_succeeds ( connection )
2019-02-13 22:26:43 +00:00