hydrus/hydrus/client/search/ClientSearchParseSystemPred...

320 lines
18 KiB
Python
Raw Normal View History

2021-07-28 21:12:00 +00:00
import datetime
import typing
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusGlobals as HG
2022-10-12 20:18:22 +00:00
from hydrus.client import ClientConstants as CC
2023-05-31 20:54:09 +00:00
from hydrus.client.search import ClientSearch
2021-07-28 21:12:00 +00:00
from hydrus.external import SystemPredicateParser
2023-08-16 20:46:51 +00:00
def convert_timetuple_to_seconds( v ):
( days, hours, minutes, seconds ) = v
return days * 86400 + hours * 3600 + minutes * 60 + seconds
def convert_double_hex_hashlist_and_other_to_double_bytes_and_other( hex_hashlist_and_other ):
try:
bytes_hashlist_1 = tuple( ( bytes.fromhex( hex_hash ) for hex_hash in hex_hashlist_and_other[0] ) )
bytes_hashlist_2 = tuple( ( bytes.fromhex( hex_hash ) for hex_hash in hex_hashlist_and_other[1] ) )
except HydrusExceptions.DataMissing as e:
raise ValueError( str( e ) )
return ( bytes_hashlist_1, bytes_hashlist_2, hex_hashlist_and_other[2] )
def convert_hex_hashlist_and_other_to_bytes_and_other( hex_hashlist_and_other ):
try:
bytes_hashlist = tuple( ( bytes.fromhex( hex_hash ) for hex_hash in hex_hashlist_and_other[0] ) )
except HydrusExceptions.DataMissing as e:
raise ValueError( str( e ) )
return ( bytes_hashlist, hex_hashlist_and_other[1] )
def date_pred_generator( pred_type, o, v ):
#Either a tuple of 4 non-negative integers: (years, months, days, hours) where the latter is < 24 OR
#a datetime.date object. For the latter, only the YYYY-MM-DD format is accepted.
if isinstance( v, datetime.datetime ):
date_type = 'date'
v = ( v.year, v.month, v.day, v.hour, v.minute )
else:
date_type = 'delta'
if o == '=':
2023-09-06 19:49:46 +00:00
o = HC.UNICODE_APPROX_EQUAL
2023-08-16 20:46:51 +00:00
return ClientSearch.Predicate( pred_type, ( o, date_type, tuple( v ) ) )
2021-07-28 21:12:00 +00:00
def file_service_pred_generator( o, v, u ):
if o.startswith( 'is not' ):
is_in = False
else:
is_in = True
2021-07-28 21:12:00 +00:00
o_dict = {
'currently in' : HC.CONTENT_STATUS_CURRENT,
'deleted from' : HC.CONTENT_STATUS_DELETED,
'pending to' : HC.CONTENT_STATUS_PENDING,
'petitioned from' : HC.CONTENT_STATUS_PETITIONED
2021-07-28 21:12:00 +00:00
}
status = None
for ( phrase, possible_status ) in o_dict.items():
if phrase in o:
status = possible_status
break
if status is None:
raise HydrusExceptions.BadRequestException( 'Did not understand the file service status!' )
2021-07-28 21:12:00 +00:00
try:
service_name = v
service_key = HG.client_controller.services_manager.GetServiceKeyFromName( HC.FILE_SERVICES, service_name )
except:
raise HydrusExceptions.BadRequestException( 'Could not find the service "{}"!'.format( service_name ) )
return ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_FILE_SERVICE, ( is_in, status, service_key ) )
2023-08-16 20:46:51 +00:00
2021-07-28 21:12:00 +00:00
def filetype_pred_generator( v ):
# v is a list of non-hydrus-standard filetype strings
mimes = ( 1, )
return ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_MIME, mimes )
2023-07-26 20:57:00 +00:00
2021-07-28 21:12:00 +00:00
def num_file_relationships_pred_generator( o, v, u ):
u_dict = {
'not related/false positive' : HC.DUPLICATE_FALSE_POSITIVE,
'duplicates' : HC.DUPLICATE_MEMBER,
'alternates' : HC.DUPLICATE_ALTERNATE,
'potential duplicates' : HC.DUPLICATE_POTENTIAL
}
dupe_type = u_dict[ u ]
return ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_FILE_RELATIONSHIPS_COUNT, ( o, v, dupe_type ) )
2023-05-31 20:54:09 +00:00
2023-08-16 20:46:51 +00:00
def rating_service_pred_generator( operator, value_and_service_name ):
try:
( value, service_name ) = value_and_service_name
service_key = HG.client_controller.services_manager.GetServiceKeyFromName( HC.RATINGS_SERVICES, service_name )
service = HG.client_controller.services_manager.GetService( service_key )
except:
raise HydrusExceptions.BadRequestException( 'Could not find the service "{}"!'.format( service_name ) )
if service.GetServiceType() == HC.LOCAL_RATING_NUMERICAL and isinstance( value, int ):
value = service.ConvertStarsToRating( value )
predicate_value = ( operator, value, service_key )
return ClientSearch.Predicate( predicate_type = ClientSearch.PREDICATE_TYPE_SYSTEM_RATING, value = predicate_value )
2023-05-31 20:54:09 +00:00
def strip_quotes( s: str ) -> str:
if len( s ) > 2:
for c in [ '\'', '"' ]:
if s.startswith( c ) and s.endswith( c ):
return s[1:-1]
return s
2021-07-28 21:12:00 +00:00
def url_class_pred_generator( include, url_class_name ):
description = ( 'has {} url' if include else 'does not have {} url' ).format( url_class_name )
try:
url_class = HG.client_controller.network_engine.domain_manager.GetURLClassFromName( url_class_name )
except HydrusExceptions.DataMissing as e:
raise ValueError( str( e ) )
return ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_KNOWN_URLS, ( include, 'url_class', url_class, description ) )
2023-05-31 20:54:09 +00:00
2021-07-28 21:12:00 +00:00
SystemPredicateParser.InitialiseFiletypes( HC.mime_enum_lookup )
2023-08-23 20:43:26 +00:00
SystemPredicateParser.InitialiseFiletypes( HC.string_enum_lookup )
2023-09-06 19:49:46 +00:00
SystemPredicateParser.InitialiseFiletypes( { 'gif' : [ HC.IMAGE_GIF, HC.ANIMATION_GIF ] } )
2021-07-28 21:12:00 +00:00
pred_generators = {
SystemPredicateParser.Predicate.EVERYTHING : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_EVERYTHING ),
SystemPredicateParser.Predicate.INBOX : lambda o, v, u: ClientSearch.SYSTEM_PREDICATE_INBOX.Duplicate(),
SystemPredicateParser.Predicate.ARCHIVE : lambda o, v, u: ClientSearch.SYSTEM_PREDICATE_ARCHIVE.Duplicate(),
SystemPredicateParser.Predicate.BEST_QUALITY_OF_GROUP : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_FILE_RELATIONSHIPS_KING, True ),
SystemPredicateParser.Predicate.NOT_BEST_QUALITY_OF_GROUP : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_FILE_RELATIONSHIPS_KING, False ),
SystemPredicateParser.Predicate.HAS_AUDIO : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_AUDIO, True ),
SystemPredicateParser.Predicate.NO_AUDIO : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_AUDIO, False ),
2023-11-15 22:40:54 +00:00
SystemPredicateParser.Predicate.HAS_TRANSPARENCY : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_TRANSPARENCY, True ),
SystemPredicateParser.Predicate.NO_TRANSPARENCY : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_TRANSPARENCY, False ),
SystemPredicateParser.Predicate.HAS_EXIF : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_EXIF, True ),
SystemPredicateParser.Predicate.NO_EXIF : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_EXIF, False ),
SystemPredicateParser.Predicate.HAS_HUMAN_READABLE_EMBEDDED_METADATA : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_HUMAN_READABLE_EMBEDDED_METADATA, True ),
SystemPredicateParser.Predicate.NO_HUMAN_READABLE_EMBEDDED_METADATA : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_HUMAN_READABLE_EMBEDDED_METADATA, False ),
2021-12-08 22:40:59 +00:00
SystemPredicateParser.Predicate.HAS_ICC_PROFILE : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_ICC_PROFILE, True ),
SystemPredicateParser.Predicate.NO_ICC_PROFILE : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_ICC_PROFILE, False ),
2023-12-13 22:29:24 +00:00
SystemPredicateParser.Predicate.HAS_FORCED_FILETYPE : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_FORCED_FILETYPE, True ),
SystemPredicateParser.Predicate.NO_FORCED_FILETYPE : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_FORCED_FILETYPE, False ),
2021-07-28 21:12:00 +00:00
SystemPredicateParser.Predicate.LIMIT : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_LIMIT, v ),
SystemPredicateParser.Predicate.FILETYPE : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_MIME, tuple( v ) ),
SystemPredicateParser.Predicate.HAS_DURATION : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_DURATION, ( '>', 0 ) ),
SystemPredicateParser.Predicate.NO_DURATION : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_DURATION, ( '=', 0 ) ),
2022-09-28 17:15:23 +00:00
SystemPredicateParser.Predicate.HAS_TAGS : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_TAGS, ( '*', '>', 0 ) ),
SystemPredicateParser.Predicate.UNTAGGED : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_TAGS, ( '*', '=', 0 ) ),
SystemPredicateParser.Predicate.NUM_OF_TAGS : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_TAGS, ( '*', o, v ) ),
2023-11-15 22:40:54 +00:00
SystemPredicateParser.Predicate.NUM_OF_TAGS_WITH_NAMESPACE : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_TAGS, v ),
2024-01-10 21:27:29 +00:00
SystemPredicateParser.Predicate.NUM_OF_URLS : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_URLS, ( o, v ) ),
2021-07-28 21:12:00 +00:00
SystemPredicateParser.Predicate.NUM_OF_WORDS : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_WORDS, ( o, v ) ),
SystemPredicateParser.Predicate.HEIGHT : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HEIGHT, ( o, v ) ),
SystemPredicateParser.Predicate.WIDTH : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_WIDTH, ( o, v ) ),
SystemPredicateParser.Predicate.FILESIZE : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_SIZE, ( o, v, HydrusData.ConvertUnitToInt( u ) ) ),
2023-05-24 20:44:12 +00:00
SystemPredicateParser.Predicate.SIMILAR_TO_FILES : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_SIMILAR_TO_FILES, convert_hex_hashlist_and_other_to_bytes_and_other( v ) ),
2023-05-31 20:54:09 +00:00
SystemPredicateParser.Predicate.SIMILAR_TO_DATA : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_SIMILAR_TO_DATA, convert_double_hex_hashlist_and_other_to_double_bytes_and_other( v ) ),
SystemPredicateParser.Predicate.HASH : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HASH, convert_hex_hashlist_and_other_to_bytes_and_other( v ), inclusive = o == '=' ),
2021-07-28 21:12:00 +00:00
SystemPredicateParser.Predicate.DURATION : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_DURATION, ( o, v[0] * 1000 + v[1] ) ),
2023-03-29 20:57:59 +00:00
SystemPredicateParser.Predicate.FRAMERATE : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_FRAMERATE, ( o, v ) ),
SystemPredicateParser.Predicate.NUM_OF_FRAMES : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_FRAMES, ( o, v ) ),
2021-07-28 21:12:00 +00:00
SystemPredicateParser.Predicate.NUM_PIXELS : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_PIXELS, ( o, v, HydrusData.ConvertPixelsToInt( u ) ) ),
SystemPredicateParser.Predicate.RATIO : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_RATIO, ( o, v[0], v[1] ) ),
2023-04-19 20:38:13 +00:00
SystemPredicateParser.Predicate.RATIO_SPECIAL : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_RATIO, ( o, v[0], v[1] ) ),
2021-07-28 21:12:00 +00:00
SystemPredicateParser.Predicate.TAG_AS_NUMBER : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_TAG_AS_NUMBER, ( o[0], o[1], v ) ),
SystemPredicateParser.Predicate.MEDIA_VIEWS : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_FILE_VIEWING_STATS, ( 'views', ( 'media', ), o, v ) ),
SystemPredicateParser.Predicate.PREVIEW_VIEWS : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_FILE_VIEWING_STATS, ( 'views', ( 'preview', ), o, v ) ),
SystemPredicateParser.Predicate.ALL_VIEWS : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_FILE_VIEWING_STATS, ( 'views', ( 'media', 'preview' ), o, v ) ),
SystemPredicateParser.Predicate.MEDIA_VIEWTIME : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_FILE_VIEWING_STATS, ( 'viewtime', ( 'media', ), o, convert_timetuple_to_seconds( v ) ) ),
SystemPredicateParser.Predicate.PREVIEW_VIEWTIME : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_FILE_VIEWING_STATS, ( 'viewtime', ( 'preview', ), o, convert_timetuple_to_seconds( v ) ) ),
SystemPredicateParser.Predicate.ALL_VIEWTIME : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_FILE_VIEWING_STATS, ( 'viewtime', ( 'media', 'preview' ), o, convert_timetuple_to_seconds( v ) ) ),
SystemPredicateParser.Predicate.URL_REGEX : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_KNOWN_URLS, ( True, 'regex', v, 'has a url matching regex: {}'.format( v ) ) ),
SystemPredicateParser.Predicate.NO_URL_REGEX : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_KNOWN_URLS, ( False, 'regex', v, 'does not have a url matching regex: {}'.format( v ) ) ),
SystemPredicateParser.Predicate.URL : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_KNOWN_URLS, ( True, 'exact_match', v, 'has url: {}'.format( v ) ) ),
SystemPredicateParser.Predicate.NO_URL : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_KNOWN_URLS, ( False, 'exact_match', v, 'does not have url: {}'.format( v ) ) ),
SystemPredicateParser.Predicate.DOMAIN : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_KNOWN_URLS, ( True, 'domain', v, 'has a url with domain: {}'.format( v ) ) ),
SystemPredicateParser.Predicate.NO_DOMAIN : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_KNOWN_URLS, ( False, 'domain', v, 'does not have a url with domain: {}'.format( v ) ) ),
SystemPredicateParser.Predicate.URL_CLASS : lambda o, v, u: url_class_pred_generator( True, v ),
SystemPredicateParser.Predicate.NO_URL_CLASS : lambda o, v, u: url_class_pred_generator( False, v ),
SystemPredicateParser.Predicate.MOD_DATE : lambda o, v, u: date_pred_generator( ClientSearch.PREDICATE_TYPE_SYSTEM_MODIFIED_TIME, o, v ),
2023-04-12 20:34:43 +00:00
SystemPredicateParser.Predicate.ARCHIVED_DATE : lambda o, v, u: date_pred_generator( ClientSearch.PREDICATE_TYPE_SYSTEM_ARCHIVED_TIME, o, v ),
2022-01-26 21:57:04 +00:00
SystemPredicateParser.Predicate.LAST_VIEWED_TIME : lambda o, v, u: date_pred_generator( ClientSearch.PREDICATE_TYPE_SYSTEM_LAST_VIEWED_TIME, o, v ),
2021-07-28 21:12:00 +00:00
SystemPredicateParser.Predicate.TIME_IMPORTED : lambda o, v, u: date_pred_generator( ClientSearch.PREDICATE_TYPE_SYSTEM_AGE, o, v ),
SystemPredicateParser.Predicate.FILE_SERVICE : file_service_pred_generator,
SystemPredicateParser.Predicate.NUM_FILE_RELS : num_file_relationships_pred_generator,
SystemPredicateParser.Predicate.HAS_NOTES : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_NOTES, ( '>', 0 ) ),
SystemPredicateParser.Predicate.NO_NOTES : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_NOTES, ( '=', 0 ) ),
SystemPredicateParser.Predicate.NUM_NOTES : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_NUM_NOTES, ( o, v ) ),
2023-05-31 20:54:09 +00:00
SystemPredicateParser.Predicate.HAS_NOTE_NAME : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_NOTE_NAME, ( True, strip_quotes( v ) ) ),
2023-08-16 20:46:51 +00:00
SystemPredicateParser.Predicate.NO_NOTE_NAME : lambda o, v, u: ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_HAS_NOTE_NAME, ( False, strip_quotes( v ) ) ),
SystemPredicateParser.Predicate.HAS_RATING : lambda o, v, u: rating_service_pred_generator( '=', ( 'rated', v ) ),
SystemPredicateParser.Predicate.NO_RATING : lambda o, v, u: rating_service_pred_generator( '=', ( 'not rated', v ) ),
SystemPredicateParser.Predicate.RATING_SPECIFIC_NUMERICAL : lambda o, v, u: rating_service_pred_generator( o, v ),
SystemPredicateParser.Predicate.RATING_SPECIFIC_LIKE_DISLIKE : lambda o, v, u: rating_service_pred_generator( '=', v ),
SystemPredicateParser.Predicate.RATING_SPECIFIC_INCDEC : lambda o, v, u: rating_service_pred_generator( o, v )
2021-07-28 21:12:00 +00:00
}
2023-05-31 20:54:09 +00:00
def ParseSystemPredicateStringsToPredicates( system_predicate_strings: typing.Collection[ str ], discard_failures = False ) -> typing.List[ ClientSearch.Predicate ]:
2021-07-28 21:12:00 +00:00
system_predicates = []
for s in system_predicate_strings:
try:
( ext_pred_type, operator, value, unit ) = SystemPredicateParser.parse_system_predicate( s )
if ext_pred_type not in pred_generators:
raise HydrusExceptions.BadRequestException( 'Sorry, do not know how to parse "{}" yet!'.format( s ) )
predicate = pred_generators[ ext_pred_type ]( operator, value, unit )
system_predicates.append( predicate )
except ValueError as e:
2023-05-31 20:54:09 +00:00
if discard_failures:
continue
2021-07-28 21:12:00 +00:00
raise HydrusExceptions.BadRequestException( 'Could not parse system predicate "{}"!'.format( s ) )
except Exception as e:
2023-05-31 20:54:09 +00:00
if discard_failures:
continue
2021-07-28 21:12:00 +00:00
raise HydrusExceptions.BadRequestException( 'Problem when trying to parse this system predicate: "{}"!'.format( s ) )
return system_predicates
2023-05-31 20:54:09 +00:00