From 8fb8779dfbef0ffc9d98b5c7fc3b73f972cb076f Mon Sep 17 00:00:00 2001
From: thatfuckingbird <67429906+thatfuckingbird@users.noreply.github.com>
Date: Mon, 7 Mar 2022 03:41:36 +0100
Subject: [PATCH 1/4] increase URL length limit. this fixes internal server
 errors on long GET request lines when using API

---
 docs/developer_api.md                  | 5 +++++
 hydrus/core/networking/HydrusServer.py | 3 ++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/docs/developer_api.md b/docs/developer_api.md
index b224c5fc..da239847 100644
--- a/docs/developer_api.md
+++ b/docs/developer_api.md
@@ -47,6 +47,11 @@ In general, the API deals with standard UTF-8 JSON. POST requests and 200 OK res
 
 On 200 OK, the API returns JSON for everything except actual file/thumbnail requests. On 4XX and 5XX, assume it will return plain text, which may be a raw traceback that I'd be interested in seeing. You'll typically get 400 for a missing parameter, 401/403/419 for missing/insufficient/expired access, and 500 for a real deal serverside error.
 
+!!! note
+    For any request sent to the API, the total size of the initial request line (this includes the URL and any parameters) and the headers must not be larger than 2 megabytes.
+    Exceeding this limit will cause the request to fail. Make sure to use pagination if you are passing very large JSON arrays as parameters in a GET request.
+    
+
 ## Access and permissions
 
 The client gives access to its API through different 'access keys', which are the typical 64-character hex used in many other places across hydrus. Each guarantees different permissions such as handling files or tags. Most of the time, a user will provide full access, but do not assume this. If the access header or parameter is not provided, you will get 401, and all insufficient permission problems will return 403 with appropriate error text.
diff --git a/hydrus/core/networking/HydrusServer.py b/hydrus/core/networking/HydrusServer.py
index c8ac872a..d6beb941 100644
--- a/hydrus/core/networking/HydrusServer.py
+++ b/hydrus/core/networking/HydrusServer.py
@@ -12,7 +12,8 @@ REMOTE_DOMAIN = HydrusServerResources.HydrusDomain( False )
 
 class FatHTTPChannel( HTTPChannel ):
     
-    totalHeadersSize = 1048576 # :^)
+    MAX_LENGTH = 2 * 1048576
+    totalHeadersSize = 2 * 1048576 # :^)
     
 class HydrusService( Site ):
     

From 78a24b5d986ce108ecadebb2b68ed764447304ff Mon Sep 17 00:00:00 2001
From: thatfuckingbird <67429906+thatfuckingbird@users.noreply.github.com>
Date: Mon, 7 Mar 2022 03:44:01 +0100
Subject: [PATCH 2/4] implement note predicate parsing + note api

---
 docs/developer_api.md                         | 70 ++++++++++++++++
 hydrus/client/ClientAPI.py                    |  4 +-
 .../ClientSearchParseSystemPredicates.py      |  7 +-
 hydrus/client/networking/ClientLocalServer.py |  7 ++
 .../networking/ClientLocalServerResources.py  | 81 ++++++++++++++++++-
 hydrus/external/SystemPredicateParser.py      | 45 ++++++++---
 6 files changed, 200 insertions(+), 14 deletions(-)

diff --git a/docs/developer_api.md b/docs/developer_api.md
index da239847..3e7ebd05 100644
--- a/docs/developer_api.md
+++ b/docs/developer_api.md
@@ -799,6 +799,67 @@ Response:
 :   200 with no content. Like when adding tags, this is safely idempotent--do not worry about re-adding URLs associations that already exist or accidentally trying to delete ones that don't.
     
 
+## Adding Notes
+
+### **POST `/add_notes/set_notes`** { id="add_notes_set_notes" }
+
+_Add or update notes associated with a file._
+
+Restricted access: 
+:   YES. Add Notes permission needed.
+    
+Required Headers:
+:       
+    *   `Content-Type`: `application/json`
+    
+Arguments (in percent-encoded JSON):
+:   
+*   `notes`: a dictionary mapping note names to note contents
+*   `hash`: the SHA256 of the target file
+*   `file_id`: the identifier of the target file (an integer)
+
+    You must provide one of `hash` or `file_id`. Existing notes will be overwritten.
+```json title="Example request body"
+{
+  "notes": {
+      "note name": "content of note",
+      "another note": "asdf"
+  },
+  "hash": "3b820114f658d768550e4e3d4f1dced3ff8db77443472b5ad93700647ad2d3ba"
+}
+```
+
+Response: 
+:   200 with no content. This operation is idempotent.
+
+### **POST `/add_notes/delete_notes`** { id="add_notes_delete_notes" }
+
+_Remove notes associated with a file._
+
+Restricted access: 
+:   YES. Add Notes permission needed.
+    
+Required Headers:
+:       
+    *   `Content-Type`: `application/json`
+    
+Arguments (in percent-encoded JSON):
+:   
+*   `note_names`: a list of note names to delete
+*   `hash`: the SHA256 of the target file
+*   `file_id`: the identifier of the target file (an integer)
+
+    You must provide one of `hash` or `file_id`.
+```json title="Example request body"
+{
+  "note_names": ["note name", "another note"],
+  "hash": "3b820114f658d768550e4e3d4f1dced3ff8db77443472b5ad93700647ad2d3ba"
+}
+```
+
+Response: 
+:   200 with no content. This operation is idempotent.
+
 ## Managing Cookies and HTTP Headers
 
 This refers to the cookies held in the client's session manager, which are sent with network requests to different domains.
@@ -1227,6 +1288,14 @@ Arguments (in percent-encoded JSON):
         *   system:has a url with class safebooru file page
         *   system:does not have a url with url class safebooru file page
         *   system:tag as number page < 5
+        *   system:has notes
+        *   system:no notes
+        *   system:does not have notes
+        *   system:num notes is 5
+        *   system:num notes > 1
+        *   system:has note with name note name
+        *   system:no note with name note name
+        *   system:does not have note with name note name
 
     More system predicate types and input formats will be available in future. Please test out the system predicates you want to send. Reverse engineering system predicate data from text is obviously tricky. If a system predicate does not parse, you'll get 400.
 
@@ -1302,6 +1371,7 @@ Arguments (in percent-encoded JSON):
     *   `only_return_identifiers`: true or false (optional, defaulting to false)
     *   `detailed_url_information`: true or false (optional, defaulting to false)
     *   `hide_service_names_tags`: true or false (optional, defaulting to false)
+    *   `include_notes`: true or false (optional, defaulting to false)
 
     You need one of file_ids or hashes. If your access key is restricted by tag, you cannot search by hashes, and **the file_ids you search for must have been in the most recent search result**.
 
diff --git a/hydrus/client/ClientAPI.py b/hydrus/client/ClientAPI.py
index d4ad3e5d..824d13ec 100644
--- a/hydrus/client/ClientAPI.py
+++ b/hydrus/client/ClientAPI.py
@@ -13,8 +13,9 @@ CLIENT_API_PERMISSION_SEARCH_FILES = 3
 CLIENT_API_PERMISSION_MANAGE_PAGES = 4
 CLIENT_API_PERMISSION_MANAGE_COOKIES = 5
 CLIENT_API_PERMISSION_MANAGE_DATABASE = 6
+CLIENT_API_PERMISSION_ADD_NOTES = 7
 
-ALLOWED_PERMISSIONS = ( CLIENT_API_PERMISSION_ADD_FILES, CLIENT_API_PERMISSION_ADD_TAGS, CLIENT_API_PERMISSION_ADD_URLS, CLIENT_API_PERMISSION_SEARCH_FILES, CLIENT_API_PERMISSION_MANAGE_PAGES, CLIENT_API_PERMISSION_MANAGE_COOKIES, CLIENT_API_PERMISSION_MANAGE_DATABASE )
+ALLOWED_PERMISSIONS = ( CLIENT_API_PERMISSION_ADD_FILES, CLIENT_API_PERMISSION_ADD_TAGS, CLIENT_API_PERMISSION_ADD_URLS, CLIENT_API_PERMISSION_SEARCH_FILES, CLIENT_API_PERMISSION_MANAGE_PAGES, CLIENT_API_PERMISSION_MANAGE_COOKIES, CLIENT_API_PERMISSION_MANAGE_DATABASE, CLIENT_API_PERMISSION_ADD_NOTES )
 
 basic_permission_to_str_lookup = {}
 
@@ -25,6 +26,7 @@ basic_permission_to_str_lookup[ CLIENT_API_PERMISSION_SEARCH_FILES ] = 'search f
 basic_permission_to_str_lookup[ CLIENT_API_PERMISSION_MANAGE_PAGES ] = 'manage pages'
 basic_permission_to_str_lookup[ CLIENT_API_PERMISSION_MANAGE_COOKIES ] = 'manage cookies'
 basic_permission_to_str_lookup[ CLIENT_API_PERMISSION_MANAGE_DATABASE ] = 'manage database'
+basic_permission_to_str_lookup[ CLIENT_API_PERMISSION_ADD_NOTES ] = 'add notes to files'
 
 SEARCH_RESULTS_CACHE_TIMEOUT = 4 * 3600
 
diff --git a/hydrus/client/ClientSearchParseSystemPredicates.py b/hydrus/client/ClientSearchParseSystemPredicates.py
index b2a67fed..bb3b0af7 100644
--- a/hydrus/client/ClientSearchParseSystemPredicates.py
+++ b/hydrus/client/ClientSearchParseSystemPredicates.py
@@ -175,7 +175,12 @@ pred_generators = {
     SystemPredicateParser.Predicate.LAST_VIEWED_TIME : lambda o, v, u: date_pred_generator( ClientSearch.PREDICATE_TYPE_SYSTEM_LAST_VIEWED_TIME, o, v ),
     SystemPredicateParser.Predicate.TIME_IMPORTED : lambda o, v, u: date_pred_generator( ClientSearch.PREDICATE_TYPE_SYSTEM_AGE, o, v ),
     SystemPredicateParser.Predicate.FILE_SERVICE : file_service_pred_generator,
-    SystemPredicateParser.Predicate.NUM_FILE_RELS : num_file_relationships_pred_generator
+    SystemPredicateParser.Predicate.NUM_FILE_RELS : num_file_relationships_pred_generator,
+    SystemPredicateParser.Predicate.HAS_NOTES : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_NOTES, ( '>', 0 ) ),
+    SystemPredicateParser.Predicate.NO_NOTES : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_NOTES, ( '=', 0 ) ),
+    SystemPredicateParser.Predicate.NUM_NOTES : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_NOTES, ( o, v ) ),
+    SystemPredicateParser.Predicate.HAS_NOTE_NAME : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_NOTE_NAME, ( True, v ) ),
+    SystemPredicateParser.Predicate.NO_NOTE_NAME : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_NOTE_NAME, ( False, v ) )
 }
 
 def ParseSystemPredicateStringsToPredicates( system_predicate_strings: typing.Collection[ str ] ) -> typing.List[ ClientSearch.Predicate ]:
diff --git a/hydrus/client/networking/ClientLocalServer.py b/hydrus/client/networking/ClientLocalServer.py
index f1f71d14..c3c74561 100644
--- a/hydrus/client/networking/ClientLocalServer.py
+++ b/hydrus/client/networking/ClientLocalServer.py
@@ -84,6 +84,13 @@ class HydrusServiceClientAPI( HydrusClientService ):
         get_files.putChild( b'file', ClientLocalServerResources.HydrusResourceClientAPIRestrictedGetFilesGetFile( self._service, self._client_requests_domain ) )
         get_files.putChild( b'thumbnail', ClientLocalServerResources.HydrusResourceClientAPIRestrictedGetFilesGetThumbnail( self._service, self._client_requests_domain ) )
         
+        add_notes = NoResource()
+        
+        root.putChild( b'add_notes', add_notes )
+        
+        add_notes.putChild( b'set_notes', ClientLocalServerResources.HydrusResourceClientAPIRestrictedAddNotesSetNotes( self._service, self._client_requests_domain ) )
+        add_notes.putChild( b'delete_notes', ClientLocalServerResources.HydrusResourceClientAPIRestrictedAddNotesDeleteNotes( self._service, self._client_requests_domain ) )
+        
         manage_cookies = NoResource()
         
         root.putChild( b'manage_cookies', manage_cookies )
diff --git a/hydrus/client/networking/ClientLocalServerResources.py b/hydrus/client/networking/ClientLocalServerResources.py
index 558a8a0f..b3d92458 100644
--- a/hydrus/client/networking/ClientLocalServerResources.py
+++ b/hydrus/client/networking/ClientLocalServerResources.py
@@ -43,7 +43,7 @@ LOCAL_BOORU_JSON_BYTE_LIST_PARAMS = set()
 CLIENT_API_INT_PARAMS = { 'file_id', 'file_sort_type' }
 CLIENT_API_BYTE_PARAMS = { 'hash', 'destination_page_key', 'page_key', 'Hydrus-Client-API-Access-Key', 'Hydrus-Client-API-Session-Key', 'tag_service_key', 'file_service_key' }
 CLIENT_API_STRING_PARAMS = { 'name', 'url', 'domain', 'search', 'file_service_name', 'tag_service_name' }
-CLIENT_API_JSON_PARAMS = { 'basic_permissions', 'system_inbox', 'system_archive', 'tags', 'file_ids', 'only_return_identifiers', 'detailed_url_information', 'hide_service_names_tags', 'simple', 'file_sort_asc', 'return_hashes' }
+CLIENT_API_JSON_PARAMS = { 'basic_permissions', 'system_inbox', 'system_archive', 'tags', 'file_ids', 'only_return_identifiers', 'detailed_url_information', 'hide_service_names_tags', 'simple', 'file_sort_asc', 'return_hashes', 'include_notes', 'notes', 'note_names' }
 CLIENT_API_JSON_BYTE_LIST_PARAMS = { 'hashes' }
 CLIENT_API_JSON_BYTE_DICT_PARAMS = { 'service_keys_to_tags', 'service_keys_to_actions_to_tags', 'service_keys_to_additional_tags' }
 
@@ -1007,6 +1007,7 @@ class HydrusResourceClientAPIRestrictedGetServices( HydrusResourceClientAPIRestr
             (
                 ClientAPI.CLIENT_API_PERMISSION_ADD_FILES,
                 ClientAPI.CLIENT_API_PERMISSION_ADD_TAGS,
+                ClientAPI.CLIENT_API_PERMISSION_ADD_NOTES,
                 ClientAPI.CLIENT_API_PERMISSION_MANAGE_PAGES,
                 ClientAPI.CLIENT_API_PERMISSION_SEARCH_FILES
             )
@@ -1245,6 +1246,79 @@ class HydrusResourceClientAPIRestrictedAddFilesUndeleteFiles( HydrusResourceClie
         return response_context
         
     
+class HydrusResourceClientAPIRestrictedAddNotes( HydrusResourceClientAPIRestricted ):
+    
+    def _CheckAPIPermissions( self, request: HydrusServerRequest.HydrusRequest ):
+        
+        request.client_api_permissions.CheckPermission( ClientAPI.CLIENT_API_PERMISSION_ADD_NOTES )
+        
+    
+class HydrusResourceClientAPIRestrictedAddNotesSetNotes( HydrusResourceClientAPIRestrictedAddNotes ):
+    
+    def _threadDoPOSTJob( self, request: HydrusServerRequest.HydrusRequest ):
+        
+        if 'hash' in request.parsed_request_args:
+            
+            hash = request.parsed_request_args.GetValue( 'hash', bytes )
+        
+        elif 'file_id' in request.parsed_request_args:
+            
+            hash_id = request.parsed_request_args.GetValue( 'file_id', int )
+            
+            hash_ids_to_hashes = HG.client_controller.Read( 'hash_ids_to_hashes', hash_ids = [ hash_id ] )
+            
+            hash = hash_ids_to_hashes[ hash_id ]
+        
+        else:
+            
+            raise HydrusExceptions.BadRequestException( 'There was no file identifier or hash given!' )
+        
+        notes = request.parsed_request_args.GetValue( 'notes', dict )
+        
+        content_updates = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_NOTES, HC.CONTENT_UPDATE_SET, ( hash, name, note ) ) for ( name, note ) in notes.items() ]
+        
+        service_keys_to_content_updates = { CC.LOCAL_NOTES_SERVICE_KEY : content_updates }
+        
+        HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
+        
+        response_context = HydrusServerResources.ResponseContext( 200 )
+        
+        return response_context
+        
+    
+class HydrusResourceClientAPIRestrictedAddNotesDeleteNotes( HydrusResourceClientAPIRestrictedAddNotes ):
+    
+    def _threadDoPOSTJob( self, request: HydrusServerRequest.HydrusRequest ):
+        
+        if 'hash' in request.parsed_request_args:
+            
+            hash = request.parsed_request_args.GetValue( 'hash', bytes )
+        
+        elif 'file_id' in request.parsed_request_args:
+            
+            hash_id = request.parsed_request_args.GetValue( 'file_id', int )
+            
+            hash_ids_to_hashes = HG.client_controller.Read( 'hash_ids_to_hashes', hash_ids = [ hash_id ] )
+            
+            hash = hash_ids_to_hashes[ hash_id ]
+        
+        else:
+            
+            raise HydrusExceptions.BadRequestException( 'There was no file identifier or hash given!' )
+        
+        note_names = request.parsed_request_args.GetValue( 'note_names', list, expected_list_type = str )
+        
+        content_updates = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_NOTES, HC.CONTENT_UPDATE_DELETE, ( hash, name ) ) for name in note_names ]
+        
+        service_keys_to_content_updates = { CC.LOCAL_NOTES_SERVICE_KEY : content_updates }
+        
+        HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
+        
+        response_context = HydrusServerResources.ResponseContext( 200 )
+        
+        return response_context
+        
+    
 class HydrusResourceClientAPIRestrictedAddTags( HydrusResourceClientAPIRestricted ):
     
     def _CheckAPIPermissions( self, request: HydrusServerRequest.HydrusRequest ):
@@ -2132,6 +2206,7 @@ class HydrusResourceClientAPIRestrictedGetFilesFileMetadata( HydrusResourceClien
         only_return_identifiers = request.parsed_request_args.GetValue( 'only_return_identifiers', bool, default_value = False )
         hide_service_names_tags = request.parsed_request_args.GetValue( 'hide_service_names_tags', bool, default_value = False )
         detailed_url_information = request.parsed_request_args.GetValue( 'detailed_url_information', bool, default_value = False )
+        include_notes = request.parsed_request_args.GetValue( 'include_notes', bool, default_value = False )
         
         try:
             
@@ -2217,6 +2292,10 @@ class HydrusResourceClientAPIRestrictedGetFilesFileMetadata( HydrusResourceClien
                 metadata_row[ 'num_words' ] = file_info_manager.num_words
                 metadata_row[ 'has_audio' ] = file_info_manager.has_audio
                 
+                if include_notes:
+                    
+                    metadata_row[ 'notes' ] = media_result.GetNotesManager().GetNamesToNotes()
+                
                 locations_manager = media_result.GetLocationsManager()
                 
                 metadata_row[ 'file_services' ] = {
diff --git a/hydrus/external/SystemPredicateParser.py b/hydrus/external/SystemPredicateParser.py
index 80ccfee9..311750e4 100644
--- a/hydrus/external/SystemPredicateParser.py
+++ b/hydrus/external/SystemPredicateParser.py
@@ -97,6 +97,11 @@ class Predicate( Enum ):
     URL_CLASS = auto()
     NO_URL_CLASS = auto()
     TAG_AS_NUMBER = auto()
+    HAS_NOTES = auto()
+    NO_NOTES = auto()
+    NUM_NOTES = auto()
+    HAS_NOTE_NAME = auto()
+    NO_NOTE_NAME = auto()
 
 
 # This enum lists the possible value formats a predicate can have (if it has a value).
@@ -121,6 +126,7 @@ class Value( Enum ):
 # Implemented in parse_operator
 class Operators( Enum ):
     RELATIONAL = auto()  # One of '=', '<', '>', '\u2248' ('≈') (takes '~=' too)
+    RELATIONAL_EXACT = auto() # Like RELATIONAL but without the approximately equal operator
     EQUAL = auto()  # One of '=' or '!='
     FILESERVICE_STATUS = auto()  # One of 'is not currently in', 'is currently in', 'is not pending to', 'is pending to'
     TAG_RELATIONAL = auto()  # A tuple of a string (a potential tag name) and a relational operator (as a string)
@@ -186,7 +192,12 @@ SYSTEM_PREDICATES = {
     '(does not|doesn\'t) have (a )?(url with )?domain': (Predicate.NO_DOMAIN, None, Value.ANY_STRING, None),
     'has (a )?url with (url )?class': (Predicate.URL_CLASS, None, Value.ANY_STRING, None),
     '(does not|doesn\'t) have (a )?url with (url )?class': (Predicate.NO_URL_CLASS, None, Value.ANY_STRING, None),
-    'tag as number': (Predicate.TAG_AS_NUMBER, Operators.TAG_RELATIONAL, Value.INTEGER, None)
+    'tag as number': (Predicate.TAG_AS_NUMBER, Operators.TAG_RELATIONAL, Value.INTEGER, None),
+    'has notes?': (Predicate.HAS_NOTES, None, None, None),
+    '(no|does not have|doesn\'t have) notes': (Predicate.NO_NOTES, None, None, None),
+    'num(ber of)? notes': (Predicate.NUM_NOTES, Operators.RELATIONAL_EXACT, Value.NATURAL, None),
+    '(has (a )?)?note with name': (Predicate.HAS_NOTE_NAME, None, Value.ANY_STRING, None),
+    '(no|does not have|doesn\'t have) note with name': (Predicate.NO_NOTE_NAME, None, Value.ANY_STRING, None),
 }
 
 
@@ -344,17 +355,21 @@ def parse_operator( string: str, spec ):
     string = string.strip()
     if spec is None:
         return string, None
-    elif spec == Operators.RELATIONAL:
-        ops = [ '\u2248', '=', '<', '>', '\u2260' ]
+    elif spec == Operators.RELATIONAL or spec == Operators.RELATIONAL_EXACT:
+        exact = spec == Operators.RELATIONAL_EXACT
+        ops = [ '=', '<', '>' ]
+        if not exact:
+            ops = ops + [ '\u2260', '\u2248' ]
         if string.startswith( '==' ): return string[ 2: ], '='
-        if string.startswith( '!=' ): return string[ 2: ], '\u2260'
-        if string.startswith( 'is not' ): return string[ 6: ], '\u2260'
-        if string.startswith( 'isn\'t' ): return string[ 5: ], '\u2260'
-        if string.startswith( '~=' ): return string[ 2: ], '\u2248'
+        if not exact:
+            if string.startswith( '!=' ): return string[ 2: ], '\u2260'
+            if string.startswith( 'is not' ): return string[ 6: ], '\u2260'
+            if string.startswith( 'isn\'t' ): return string[ 5: ], '\u2260'
+            if string.startswith( '~=' ): return string[ 2: ], '\u2248'
         for op in ops:
             if string.startswith( op ): return string[ len( op ): ], op
         if string.startswith( 'is' ): return string[ 2: ], '='
-        raise ValueError( "Invalid relation operator" )
+        raise ValueError( "Invalid relational operator" )
     elif spec == Operators.EQUAL:
         if string.startswith( '==' ): return string[ 2: ], '='
         if string.startswith( '=' ): return string[ 1: ], '='
@@ -436,8 +451,8 @@ examples = [
     "system:similar to abcdef distance 5",
     "system:limit is 5000",
     "system:limit = 100",
-    "system:filetype is jpeg",
-    "system:filetype =   image/jpg, image/png, apng",
+    #"system:filetype is jpeg",
+    #"system:filetype =   image/jpg, image/png, apng",
     "system:hash = abcdef1 abcdef2 abcdef3",
     "system:hash = abcdef1 abcdef, abcdef4 md5",
     "system:modified date < 7  years 45 days 70h",
@@ -487,7 +502,15 @@ examples = [
     "system:doesn't have domain test.com",
     "system:has a url with class safebooru file page",
     "system:doesn't have a url with url class safebooru file page ",
-    "system:tag as number page < 5"
+    "system:tag as number page < 5",
+    "system:has notes",
+    "system:no notes",
+    "system:does not have notes",
+    "system:num notes is 5",
+    "system:num notes > 1",
+    "system:has note with name note name",
+    "system:no note with name note name",
+    "system:does not have note with name note name"
 ]
 
 if __name__ == "__main__":

From 4c12f6fac2e0033ed459922e561ab40dc76058ac Mon Sep 17 00:00:00 2001
From: thatfuckingbird <67429906+thatfuckingbird@users.noreply.github.com>
Date: Mon, 7 Mar 2022 03:46:01 +0100
Subject: [PATCH 3/4] support CBOR in client api

---
 hydrus/client/gui/ClientGUI.py                |  12 ++
 .../networking/ClientLocalServerResources.py  | 108 ++++++++++++------
 hydrus/core/HydrusConstants.py                |   4 +
 .../HydrusNetworkVariableHandling.py          |  26 ++++-
 4 files changed, 110 insertions(+), 40 deletions(-)

diff --git a/hydrus/client/gui/ClientGUI.py b/hydrus/client/gui/ClientGUI.py
index fcdab82d..8e6098bd 100644
--- a/hydrus/client/gui/ClientGUI.py
+++ b/hydrus/client/gui/ClientGUI.py
@@ -687,6 +687,18 @@ class FrameGUI( ClientGUITopLevelWindows.MainFrameThatResizes ):
             library_versions.append( ( 'PyQt5', PYQT_VERSION_STR ) )
             library_versions.append( ( 'sip', SIP_VERSION_STR ) )
             
+        CBOR_AVAILABLE = False
+        
+        try:
+            
+            import cbor2
+            CBOR_AVAILABLE = True
+            
+        except:
+            
+            pass
+        
+        library_versions.append( ( 'cbor2 present: ', str( CBOR_AVAILABLE ) ) )
         
         from hydrus.client.networking import ClientNetworkingJobs
         
diff --git a/hydrus/client/networking/ClientLocalServerResources.py b/hydrus/client/networking/ClientLocalServerResources.py
index b3d92458..75309c32 100644
--- a/hydrus/client/networking/ClientLocalServerResources.py
+++ b/hydrus/client/networking/ClientLocalServerResources.py
@@ -7,6 +7,13 @@ import time
 import traceback
 import typing
 
+CBOR_AVAILABLE = False
+try:
+    import cbor2
+    CBOR_AVAILABLE = True
+except:
+    pass
+
 from twisted.web.static import File as FileResource
 
 from hydrus.core import HydrusConstants as HC
@@ -47,6 +54,17 @@ CLIENT_API_JSON_PARAMS = { 'basic_permissions', 'system_inbox', 'system_archive'
 CLIENT_API_JSON_BYTE_LIST_PARAMS = { 'hashes' }
 CLIENT_API_JSON_BYTE_DICT_PARAMS = { 'service_keys_to_tags', 'service_keys_to_actions_to_tags', 'service_keys_to_additional_tags' }
 
+def Dumps( data, mime ):
+    
+    if CBOR_AVAILABLE and mime == HC.APPLICATION_CBOR:
+        
+        return cbor2.dumps( data )
+    
+    else:
+        
+        return json.dumps( data )
+        
+    
 def CheckHashLength( hashes, hash_type = 'sha256' ):
     
     hash_types_to_length = {
@@ -276,7 +294,17 @@ def ParseClientAPIPOSTArgs( request ):
             args = json.loads( json_string )
             
             parsed_request_args = ParseClientAPIPOSTByteArgs( args )
+        
+        elif mime == HC.APPLICATION_CBOR and CBOR_AVAILABLE:
             
+            cbor_bytes = request.content.read()
+            
+            total_bytes_read += len( cbor_bytes )
+            
+            args = cbor2.loads( cbor_bytes )
+            
+            parsed_request_args = ParseClientAPIPOSTByteArgs( args )
+        
         else:
             
             parsed_request_args = HydrusNetworkVariableHandling.ParsedRequestArguments()
@@ -297,7 +325,7 @@ def ParseClientAPIPOSTArgs( request ):
             
         
     
-    return ( parsed_request_args, total_bytes_read )
+    return ( parsed_request_args, total_bytes_read, mime )
     
 def ParseClientAPISearchPredicates( request ):
     
@@ -744,17 +772,21 @@ class HydrusResourceClientAPI( HydrusServerResources.HydrusResource ):
         
         request.parsed_request_args = parsed_request_args
         
+        request.preferred_mime = HC.APPLICATION_CBOR if CBOR_AVAILABLE and b'cbor' in request.args else HC.APPLICATION_JSON
+        
         return request
         
     
     def _callbackParsePOSTArgs( self, request: HydrusServerRequest.HydrusRequest ):
         
-        ( parsed_request_args, total_bytes_read ) = ParseClientAPIPOSTArgs( request )
+        ( parsed_request_args, total_bytes_read, mime ) = ParseClientAPIPOSTArgs( request )
         
         self._reportDataUsed( request, total_bytes_read )
         
         request.parsed_request_args = parsed_request_args
         
+        request.preferred_mime = mime
+        
         return request
         
     
@@ -810,9 +842,9 @@ class HydrusResourceClientAPIPermissionsRequest( HydrusResourceClientAPI ):
         
         body_dict[ 'access_key' ] = access_key.hex()
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
         
@@ -826,9 +858,9 @@ class HydrusResourceClientAPIVersion( HydrusResourceClientAPI ):
         body_dict[ 'version' ] = HC.CLIENT_API_VERSION
         body_dict[ 'hydrus_version' ] = HC.SOFTWARE_VERSION
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
         
@@ -971,9 +1003,9 @@ class HydrusResourceClientAPIRestrictedAccountSessionKey( HydrusResourceClientAP
         
         body_dict[ 'session_key' ] = new_session_key.hex()
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
         
@@ -992,9 +1024,9 @@ class HydrusResourceClientAPIRestrictedAccountVerify( HydrusResourceClientAPIRes
         body_dict[ 'basic_permissions' ] = list( basic_permissions ) # set->list for json
         body_dict[ 'human_description' ] = human_description
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
         
@@ -1036,9 +1068,9 @@ class HydrusResourceClientAPIRestrictedGetServices( HydrusResourceClientAPIRestr
             body_dict[ name ] = [ { 'name' : service.GetName(), 'service_key' : service.GetServiceKey().hex() } for service in services ]
             
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
         
@@ -1091,9 +1123,9 @@ class HydrusResourceClientAPIRestrictedAddFilesAddFile( HydrusResourceClientAPIR
         body_dict[ 'hash' ] = HydrusData.BytesToNoneOrHex( file_import_status.hash )
         body_dict[ 'note' ] = file_import_status.note
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
         
@@ -1534,9 +1566,9 @@ class HydrusResourceClientAPIRestrictedAddTagsGetTagServices( HydrusResourceClie
         body_dict[ 'local_tags' ] = [ service.GetName() for service in local_tags ]
         body_dict[ 'tag_repositories' ] = [ service.GetName() for service in tag_repos ]
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
 
@@ -1648,9 +1680,9 @@ class HydrusResourceClientAPIRestrictedAddTagsSearchTags( HydrusResourceClientAP
 
         body_dict[ 'tags' ] = tags
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
 
@@ -1669,9 +1701,9 @@ class HydrusResourceClientAPIRestrictedAddTagsCleanTags( HydrusResourceClientAPI
         
         body_dict[ 'tags' ] = tags
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
         
@@ -1836,9 +1868,9 @@ class HydrusResourceClientAPIRestrictedAddURLsGetURLFiles( HydrusResourceClientA
         
         body_dict = { 'normalised_url' : normalised_url, 'url_file_statuses' : json_happy_url_statuses }
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
         
@@ -1872,9 +1904,9 @@ class HydrusResourceClientAPIRestrictedAddURLsGetURLInfo( HydrusResourceClientAP
             body_dict[ 'cannot_parse_reason' ] = cannot_parse_reason
             
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
         
@@ -1981,9 +2013,9 @@ class HydrusResourceClientAPIRestrictedAddURLsImportURL( HydrusResourceClientAPI
         
         body_dict = { 'human_result_text' : result_text, 'normalised_url' : normalised_url }
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
         
@@ -2138,9 +2170,9 @@ class HydrusResourceClientAPIRestrictedGetFilesSearchFiles( HydrusResourceClient
             body_dict = { 'file_ids' : list( hash_ids ) }
             
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
         
@@ -2440,8 +2472,8 @@ class HydrusResourceClientAPIRestrictedGetFilesFileMetadata( HydrusResourceClien
         
         body_dict[ 'metadata' ] = metadata
         
-        mime = HC.APPLICATION_JSON
-        body = json.dumps( body_dict )
+        mime = request.preferred_mime
+        body = Dumps( body_dict, mime )
         
         response_context = HydrusServerResources.ResponseContext( 200, mime = mime, body = body )
         
@@ -2539,9 +2571,9 @@ class HydrusResourceClientAPIRestrictedManageCookiesGetCookies( HydrusResourceCl
         
         body_dict = { 'cookies' : body_cookies_list }
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
         
@@ -2711,8 +2743,8 @@ class HydrusResourceClientAPIRestrictedManageDatabaseMrBones( HydrusResourceClie
         
         body_dict = { 'boned_stats' : boned_stats }
         
-        mime = HC.APPLICATION_JSON
-        body = json.dumps( body_dict )
+        mime = request.preferred_mime
+        body = Dumps( body_dict, mime )
         
         response_context = HydrusServerResources.ResponseContext( 200, mime = mime, body = body )
         
@@ -2827,9 +2859,9 @@ class HydrusResourceClientAPIRestrictedManagePagesGetPages( HydrusResourceClient
         
         body_dict = { 'pages' : page_info_dict }
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
         
@@ -2856,9 +2888,9 @@ class HydrusResourceClientAPIRestrictedManagePagesGetPageInfo( HydrusResourceCli
         
         body_dict = { 'page_info' : page_info_dict }
         
-        body = json.dumps( body_dict )
+        body = Dumps( body_dict, request.preferred_mime )
         
-        response_context = HydrusServerResources.ResponseContext( 200, mime = HC.APPLICATION_JSON, body = body )
+        response_context = HydrusServerResources.ResponseContext( 200, mime = request.preferred_mime, body = body )
         
         return response_context
         
diff --git a/hydrus/core/HydrusConstants.py b/hydrus/core/HydrusConstants.py
index dbad27e3..6a269339 100644
--- a/hydrus/core/HydrusConstants.py
+++ b/hydrus/core/HydrusConstants.py
@@ -551,6 +551,7 @@ VIDEO_OGV = 47
 AUDIO_MKV = 48
 AUDIO_MP4 = 49
 UNDETERMINED_MP4 = 50
+APPLICATION_CBOR = 51
 APPLICATION_OCTET_STREAM = 100
 APPLICATION_UNKNOWN = 101
 
@@ -634,6 +635,7 @@ mime_enum_lookup = {
     'application/vnd.rar' : APPLICATION_RAR,
     'application/x-7z-compressed' : APPLICATION_7Z,
     'application/json' : APPLICATION_JSON,
+    'application/cbor': APPLICATION_CBOR,
     'application/hydrus-encrypted-zip' : APPLICATION_HYDRUS_ENCRYPTED_ZIP,
     'application/hydrus-update-content' : APPLICATION_HYDRUS_UPDATE_CONTENT,
     'application/hydrus-update-definitions' : APPLICATION_HYDRUS_UPDATE_DEFINITIONS,
@@ -679,6 +681,7 @@ mime_string_lookup = {
     APPLICATION_OCTET_STREAM : 'application/octet-stream',
     APPLICATION_YAML : 'yaml',
     APPLICATION_JSON : 'json',
+    APPLICATION_CBOR : 'cbor',
     APPLICATION_PDF : 'pdf',
     APPLICATION_PSD : 'photoshop psd',
     APPLICATION_CLIP : 'clip',
@@ -735,6 +738,7 @@ mime_mimetype_string_lookup = {
     APPLICATION_OCTET_STREAM : 'application/octet-stream',
     APPLICATION_YAML : 'application/x-yaml',
     APPLICATION_JSON : 'application/json',
+    APPLICATION_CBOR : 'application/cbor',
     APPLICATION_PDF : 'application/pdf',
     APPLICATION_PSD : 'application/x-photoshop',
     APPLICATION_CLIP : 'application/clip',
diff --git a/hydrus/core/networking/HydrusNetworkVariableHandling.py b/hydrus/core/networking/HydrusNetworkVariableHandling.py
index 8efddf59..aab23c94 100644
--- a/hydrus/core/networking/HydrusNetworkVariableHandling.py
+++ b/hydrus/core/networking/HydrusNetworkVariableHandling.py
@@ -4,6 +4,14 @@ import traceback
 import typing
 import urllib
 
+CBOR_AVAILABLE = False
+try:
+    import cbor2
+    import base64
+    CBOR_AVAILABLE = True
+except:
+    pass
+
 from hydrus.core import HydrusConstants as HC
 from hydrus.core import HydrusData
 from hydrus.core import HydrusExceptions
@@ -299,6 +307,8 @@ def ParseTwistedRequestGETArgs( requests_args, int_params, byte_params, string_p
     
     args = ParsedRequestArguments()
     
+    cbor_requested = b'cbor' in requests_args
+    
     for name_bytes in requests_args:
         
         values_bytes = requests_args[ name_bytes ]
@@ -365,7 +375,13 @@ def ParseTwistedRequestGETArgs( requests_args, int_params, byte_params, string_p
             
             try:
                 
-                args[ name ] = json.loads( urllib.parse.unquote( value ) )
+                if CBOR_AVAILABLE and cbor_requested:
+                    
+                    args[ name ] = cbor2.loads( base64.urlsafe_b64decode( value ) )
+                    
+                else:
+                    
+                    args[ name ] = json.loads( urllib.parse.unquote( value ) )
                 
             except Exception as e:
                 
@@ -376,7 +392,13 @@ def ParseTwistedRequestGETArgs( requests_args, int_params, byte_params, string_p
             
             try:
                 
-                list_of_hex_strings = json.loads( urllib.parse.unquote( value ) )
+                if CBOR_AVAILABLE and cbor_requested:
+                    
+                    list_of_hex_strings = cbor2.loads( base64.urlsafe_b64decode( value ) )
+                    
+                else:
+                    
+                    list_of_hex_strings = json.loads( urllib.parse.unquote( value ) )
                 
                 args[ name ] = [ bytes.fromhex( hex_string ) for hex_string in list_of_hex_strings ]
                 

From 8011554dff06cb7135ef593ecf2b187463ea7d00 Mon Sep 17 00:00:00 2001
From: thatfuckingbird <67429906+thatfuckingbird@users.noreply.github.com>
Date: Mon, 7 Mar 2022 03:46:13 +0100
Subject: [PATCH 4/4] example script

---
 apitest.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 apitest.py

diff --git a/apitest.py b/apitest.py
new file mode 100644
index 00000000..6193c19f
--- /dev/null
+++ b/apitest.py
@@ -0,0 +1,59 @@
+import requests
+import cbor2
+import base64
+import json
+import urllib.parse
+
+hydrus_api_url = "http://localhost:45888"
+metadata =  hydrus_api_url+"/get_files/file_metadata"
+del_note = hydrus_api_url+"/add_notes/delete_notes"
+set_note = hydrus_api_url+"/add_notes/set_notes"
+search = hydrus_api_url+"/get_files/search_files"
+
+hsh="1b625544bcfbd7151000a816e6db6388ba0ef4dc3a664b62e2cb4e9d3036bed8"
+key="222f3c82f4f7e8ce57747ff1cccfaf7014357dc509cdb77af20ff910c26ea05b"
+
+# search for notes
+print(json.loads((requests.get(url = search, params = {
+    "Hydrus-Client-API-Access-Key": key,
+    "tags": urllib.parse.quote("[\"system:has notes\"]")
+}).text)))
+
+# retrieve notes
+print(json.loads((requests.get(url = metadata, params = {
+    "Hydrus-Client-API-Access-Key": key,
+    "include_notes": "true",
+    "hashes": urllib.parse.quote("[\""+hsh+"\"]")
+}).text))["metadata"][0]["notes"])
+
+# retrieve notes, request that the response is CBOR encoded
+print(cbor2.loads((requests.get(url = metadata, params = {
+    "Hydrus-Client-API-Access-Key": key,
+    "include_notes": base64.urlsafe_b64encode(cbor2.dumps(True)),
+    "hashes": base64.urlsafe_b64encode(cbor2.dumps([hsh])),
+    "cbor": ""
+}).content))["metadata"][0]["notes"])
+
+# Add notes
+
+headers = {"Hydrus-Client-API-Access-Key": key, "Content-Type": "application/json"}
+print(requests.post(url = set_note, headers = headers, data = json.dumps({
+    "notes": {"note1":"content1", "note2":"content2"},
+    "hash": hsh
+})))
+
+# Delete notes
+
+headers = {"Hydrus-Client-API-Access-Key": key, "Content-Type": "application/json"}
+print(requests.post(url = del_note, headers = headers, data = json.dumps({
+    "note_names": ["note1","note2","asgasgasgasgaa"],
+    "hash": hsh
+})))
+
+# Add notes, but send CBOR instead of json
+
+headers = {"Hydrus-Client-API-Access-Key": key, "Content-Type": "application/cbor"}
+print(requests.post(url = set_note, headers = headers, data = cbor2.dumps({
+    "notes": {"note1":"content1", "note2":"content2"},
+    "hash": hsh
+})))
\ No newline at end of file