hydrus/include/HydrusConstants.py

import bisect
import bs4
import collections
import cStringIO
import httplib
import HydrusExceptions
import HydrusPubSub
import itertools
import locale
import os
import Queue
import re
import sqlite3
import sys
import threading
import time
import traceback
import urlparse
import wx
import yaml

PLATFORM_WINDOWS = False
PLATFORM_OSX  = False

if sys.platform == 'win32': PLATFORM_WINDOWS = True
elif sys.platform == 'darwin': PLATFORM_OSX = True

options = {}

# dirs

BASE_DIR = sys.path[0]

DB_DIR = BASE_DIR + os.path.sep + 'db'
CLIENT_FILES_DIR = DB_DIR + os.path.sep + 'client_files'
SERVER_FILES_DIR = DB_DIR + os.path.sep + 'server_files'
CLIENT_THUMBNAILS_DIR = DB_DIR + os.path.sep + 'client_thumbnails'
SERVER_THUMBNAILS_DIR = DB_DIR + os.path.sep + 'server_thumbnails'
SERVER_MESSAGES_DIR = DB_DIR + os.path.sep + 'server_messages'
SERVER_UPDATES_DIR = DB_DIR + os.path.sep + 'server_updates'
LOGS_DIR = BASE_DIR + os.path.sep + 'logs'
STATIC_DIR = BASE_DIR + os.path.sep + 'static'
TEMP_DIR = BASE_DIR + os.path.sep + 'temp'

# Misc

NETWORK_VERSION = 12
SOFTWARE_VERSION = 97

UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )

HYDRUS_KEY_LENGTH = 32

UPDATE_DURATION = 100000

lifetimes = [ ( 'one month', 31 * 86400 ), ( 'three months', 3 * 31 * 86400 ), ( 'six months', 6 * 31 * 86400 ), ( 'one year', 12 * 31 * 86400 ), ( 'two years', 24 * 31 * 86400 ), ( 'five years', 60 * 31 * 86400 ), ( 'does not expire', None ) ]

app = None
shutdown = False
is_first_start = False
is_db_updated = False
repos_or_subs_changed = False

busy_doing_pubsub = False

# Enums

CONTENT_DATA_TYPE_MAPPINGS = 0
CONTENT_DATA_TYPE_TAG_SIBLINGS = 1
CONTENT_DATA_TYPE_TAG_PARENTS = 2
CONTENT_DATA_TYPE_FILES = 3
CONTENT_DATA_TYPE_RATINGS = 4

CONTENT_UPDATE_ADD = 0
CONTENT_UPDATE_DELETE = 1
CONTENT_UPDATE_PENDING = 2
CONTENT_UPDATE_RESCIND_PENDING = 3
CONTENT_UPDATE_PETITION = 4
CONTENT_UPDATE_RESCIND_PETITION = 5
CONTENT_UPDATE_EDIT_LOG = 6
CONTENT_UPDATE_ARCHIVE = 7
CONTENT_UPDATE_INBOX = 8
CONTENT_UPDATE_RATING = 9
CONTENT_UPDATE_RATINGS_FILTER = 10
CONTENT_UPDATE_DENY_PEND = 11
CONTENT_UPDATE_DENY_PETITION = 12
CONTENT_UPDATE_ADVANCED = 13

content_update_string_lookup = {}

content_update_string_lookup[ CONTENT_UPDATE_ADD ] = 'add'
content_update_string_lookup[ CONTENT_UPDATE_DELETE ] = 'delete'
content_update_string_lookup[ CONTENT_UPDATE_PENDING ] = 'pending'
content_update_string_lookup[ CONTENT_UPDATE_RESCIND_PENDING ] = 'rescind pending'
content_update_string_lookup[ CONTENT_UPDATE_PETITION ] = 'petition'
content_update_string_lookup[ CONTENT_UPDATE_RESCIND_PETITION ] = 'rescind petition'
content_update_string_lookup[ CONTENT_UPDATE_EDIT_LOG ] = 'edit log'
content_update_string_lookup[ CONTENT_UPDATE_ARCHIVE ] = 'archive'
content_update_string_lookup[ CONTENT_UPDATE_INBOX ] = 'inbox'
content_update_string_lookup[ CONTENT_UPDATE_RATING ] = 'rating'
content_update_string_lookup[ CONTENT_UPDATE_RATINGS_FILTER ] = 'incomplete ratings'
content_update_string_lookup[ CONTENT_UPDATE_DENY_PEND ] = 'deny pend'
content_update_string_lookup[ CONTENT_UPDATE_DENY_PETITION ] = 'deny petition'

IMPORT_FOLDER_TYPE_DELETE = 0
IMPORT_FOLDER_TYPE_SYNCHRONISE = 1

MESSAGE_TYPE_TEXT = 0
MESSAGE_TYPE_ERROR = 1
MESSAGE_TYPE_FILES = 2
MESSAGE_TYPE_GAUGE = 3

GET_DATA = 0
POST_DATA = 1
POST_PETITIONS = 2
RESOLVE_PETITIONS = 3
MANAGE_USERS = 4
GENERAL_ADMIN = 5
EDIT_SERVICES = 6

CREATABLE_PERMISSIONS = [ GET_DATA, POST_DATA, POST_PETITIONS, RESOLVE_PETITIONS, MANAGE_USERS, GENERAL_ADMIN ]
ADMIN_PERMISSIONS = [ RESOLVE_PETITIONS, MANAGE_USERS, GENERAL_ADMIN, EDIT_SERVICES ]

permissions_string_lookup = {}

permissions_string_lookup[ GET_DATA ] = 'get data'
permissions_string_lookup[ POST_DATA ] = 'post data'
permissions_string_lookup[ POST_PETITIONS ] = 'post petitions'
permissions_string_lookup[ RESOLVE_PETITIONS ] = 'resolve petitions'
permissions_string_lookup[ MANAGE_USERS ] = 'manage users'
permissions_string_lookup[ GENERAL_ADMIN ] = 'general administration'
permissions_string_lookup[ EDIT_SERVICES ] = 'edit services'

TAG_REPOSITORY = 0
FILE_REPOSITORY = 1
LOCAL_FILE = 2
MESSAGE_DEPOT = 3
LOCAL_TAG = 5
LOCAL_RATING_NUMERICAL = 6
LOCAL_RATING_LIKE = 7
RATING_NUMERICAL_REPOSITORY = 8
RATING_LIKE_REPOSITORY = 9
COMBINED_TAG = 10
COMBINED_FILE = 11
SERVER_ADMIN = 99
NULL_SERVICE = 100

service_string_lookup = {}

service_string_lookup[ TAG_REPOSITORY ] = 'hydrus tag repository'
service_string_lookup[ FILE_REPOSITORY ] = 'hydrus file repository'
service_string_lookup[ LOCAL_FILE ] = 'hydrus local file service'
service_string_lookup[ MESSAGE_DEPOT ] = 'hydrus message depot'
service_string_lookup[ LOCAL_TAG ] = 'local tag service'
service_string_lookup[ LOCAL_RATING_NUMERICAL ] = 'local numerical rating service'
service_string_lookup[ LOCAL_RATING_LIKE ] = 'local like/dislike rating service'
service_string_lookup[ RATING_NUMERICAL_REPOSITORY ] = 'hydrus numerical rating repository'
service_string_lookup[ RATING_LIKE_REPOSITORY ] = 'hydrus like/dislike rating repository'
service_string_lookup[ COMBINED_TAG ] = 'virtual combined tag service'
service_string_lookup[ COMBINED_FILE ] = 'virtual combined file service'
service_string_lookup[ SERVER_ADMIN ] = 'hydrus server administration'
service_string_lookup[ NULL_SERVICE ] = 'null service'

RATINGS_SERVICES = [ LOCAL_RATING_LIKE, LOCAL_RATING_NUMERICAL, RATING_LIKE_REPOSITORY, RATING_NUMERICAL_REPOSITORY ]
REPOSITORIES = [ TAG_REPOSITORY, FILE_REPOSITORY, RATING_LIKE_REPOSITORY, RATING_NUMERICAL_REPOSITORY ]
RESTRICTED_SERVICES = ( REPOSITORIES ) + [ SERVER_ADMIN, MESSAGE_DEPOT ]
REMOTE_SERVICES = list( RESTRICTED_SERVICES )
ALL_SERVICES = list( REMOTE_SERVICES ) + [ LOCAL_FILE, LOCAL_TAG, LOCAL_RATING_LIKE, LOCAL_RATING_NUMERICAL ]

SERVICES_WITH_THUMBNAILS = [ FILE_REPOSITORY, LOCAL_FILE ]

DELETE_FILES_PETITION = 0
DELETE_TAG_PETITION = 1

BAN = 0
SUPERBAN = 1
CHANGE_ACCOUNT_TYPE = 2
ADD_TO_EXPIRY = 3
SET_EXPIRY = 4

CURRENT = 0
PENDING = 1
DELETED = 2
PETITIONED = 3
DELETED_PENDING = 4

HIGH_PRIORITY = 0
LOW_PRIORITY = 2

SCORE_PETITION = 0

SERVICE_INFO_NUM_FILES = 0
SERVICE_INFO_NUM_INBOX = 1
SERVICE_INFO_NUM_LOCAL = 2
SERVICE_INFO_NUM_MAPPINGS = 3
SERVICE_INFO_NUM_DELETED_MAPPINGS = 4
SERVICE_INFO_NUM_DELETED_FILES = 5
SERVICE_INFO_NUM_THUMBNAILS = 6
SERVICE_INFO_NUM_THUMBNAILS_LOCAL = 7
SERVICE_INFO_TOTAL_SIZE = 8
SERVICE_INFO_NUM_NAMESPACES = 9
SERVICE_INFO_NUM_TAGS = 10
SERVICE_INFO_NUM_PENDING = 11
SERVICE_INFO_NUM_CONVERSATIONS = 12
SERVICE_INFO_NUM_UNREAD = 13
SERVICE_INFO_NUM_DRAFTS = 14
SERVICE_INFO_NUM_PENDING_MAPPINGS = 15
SERVICE_INFO_NUM_PETITIONED_MAPPINGS = 16
SERVICE_INFO_NUM_PENDING_FILES = 15
SERVICE_INFO_NUM_PETITIONED_FILES = 16
SERVICE_INFO_NUM_PENDING_TAG_SIBLINGS = 17
SERVICE_INFO_NUM_PETITIONED_TAG_SIBLINGS = 18
SERVICE_INFO_NUM_PENDING_TAG_PARENTS = 19
SERVICE_INFO_NUM_PETITIONED_TAG_PARENTS = 20

SERVICE_UPDATE_ACCOUNT = 0
SERVICE_UPDATE_DELETE_PENDING = 1
SERVICE_UPDATE_ERROR = 2
SERVICE_UPDATE_NEXT_BEGIN = 3
SERVICE_UPDATE_RESET = 4
SERVICE_UPDATE_REQUEST_MADE = 5
SERVICE_UPDATE_LAST_CHECK = 6
SERVICE_UPDATE_NEWS = 7

ADD = 0
DELETE = 1
EDIT = 2

APPROVE = 0
DENY = 1

GET = 0
POST = 1
OPTIONS = 2

APPLICATION_HYDRUS_CLIENT_COLLECTION = 0
IMAGE_JPEG = 1
IMAGE_PNG = 2
IMAGE_GIF = 3
IMAGE_BMP = 4
APPLICATION_FLASH = 5
APPLICATION_YAML = 6
IMAGE_ICON = 7
TEXT_HTML = 8
VIDEO_FLV = 9
APPLICATION_PDF = 10
APPLICATION_ZIP = 11
APPLICATION_HYDRUS_ENCRYPTED_ZIP = 12
AUDIO_MP3 = 13
VIDEO_MP4 = 14
AUDIO_OGG = 15
AUDIO_FLAC = 16
AUDIO_WMA = 17
VIDEO_WMV = 18
UNDETERMINED_WM = 19
APPLICATION_OCTET_STREAM = 100
APPLICATION_UNKNOWN = 101

ALLOWED_MIMES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP, APPLICATION_FLASH, VIDEO_FLV, VIDEO_MP4, APPLICATION_PDF, AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA, VIDEO_WMV )

IMAGES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP )

AUDIO = ( AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA )

VIDEO = ( VIDEO_FLV, VIDEO_MP4, VIDEO_WMV )

APPLICATIONS = ( APPLICATION_FLASH, APPLICATION_PDF, APPLICATION_ZIP )

NOISY_MIMES = tuple( [ APPLICATION_FLASH ] + list( AUDIO ) + list( VIDEO ) )

ARCHIVES = ( APPLICATION_ZIP, APPLICATION_HYDRUS_ENCRYPTED_ZIP )

MIMES_WITH_THUMBNAILS = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP )

# mp3 header is complicated

mime_enum_lookup = {}

mime_enum_lookup[ 'collection' ] = APPLICATION_HYDRUS_CLIENT_COLLECTION
mime_enum_lookup[ 'image/jpe' ] = IMAGE_JPEG
mime_enum_lookup[ 'image/jpeg' ] = IMAGE_JPEG
mime_enum_lookup[ 'image/jpg' ] = IMAGE_JPEG
mime_enum_lookup[ 'image/x-png' ] = IMAGE_PNG
mime_enum_lookup[ 'image/png' ] = IMAGE_PNG
mime_enum_lookup[ 'image/gif' ] = IMAGE_GIF
mime_enum_lookup[ 'image/bmp' ] = IMAGE_BMP
mime_enum_lookup[ 'image' ] = IMAGES
mime_enum_lookup[ 'image/vnd.microsoft.icon' ] = IMAGE_ICON
mime_enum_lookup[ 'application/x-shockwave-flash' ] = APPLICATION_FLASH
mime_enum_lookup[ 'application/octet-stream' ] = APPLICATION_OCTET_STREAM
mime_enum_lookup[ 'application/x-yaml' ] = APPLICATION_YAML
mime_enum_lookup[ 'PDF document' ] = APPLICATION_PDF
mime_enum_lookup[ 'application/pdf' ] = APPLICATION_PDF
mime_enum_lookup[ 'application/zip' ] = APPLICATION_ZIP
mime_enum_lookup[ 'application/hydrus-encrypted-zip' ] = APPLICATION_HYDRUS_ENCRYPTED_ZIP
mime_enum_lookup[ 'application' ] = APPLICATIONS
mime_enum_lookup[ 'audio/mp3' ] = AUDIO_MP3
mime_enum_lookup[ 'audio/ogg' ] = AUDIO_OGG
mime_enum_lookup[ 'audio/flac' ] = AUDIO_FLAC
mime_enum_lookup[ 'audio/x-ms-wma' ] = AUDIO_WMA
mime_enum_lookup[ 'text/html' ] = TEXT_HTML
mime_enum_lookup[ 'video/x-flv' ] = VIDEO_FLV
mime_enum_lookup[ 'video/mp4' ] = VIDEO_MP4
mime_enum_lookup[ 'video/x-ms-wmv' ] = VIDEO_WMV
mime_enum_lookup[ 'video' ] = VIDEO
mime_enum_lookup[ 'unknown mime' ] = APPLICATION_UNKNOWN

mime_string_lookup = {}

mime_string_lookup[ APPLICATION_HYDRUS_CLIENT_COLLECTION ] = 'collection'
mime_string_lookup[ IMAGE_JPEG ] = 'image/jpg'
mime_string_lookup[ IMAGE_PNG ] = 'image/png'
mime_string_lookup[ IMAGE_GIF ] = 'image/gif'
mime_string_lookup[ IMAGE_BMP ] = 'image/bmp'
mime_string_lookup[ IMAGES ] = 'image'
mime_string_lookup[ IMAGE_ICON ] = 'image/vnd.microsoft.icon'
mime_string_lookup[ APPLICATION_FLASH ] = 'application/x-shockwave-flash'
mime_string_lookup[ APPLICATION_OCTET_STREAM ] = 'application/octet-stream'
mime_string_lookup[ APPLICATION_YAML ] = 'application/x-yaml'
mime_string_lookup[ APPLICATION_PDF ] = 'application/pdf'
mime_string_lookup[ APPLICATION_ZIP ] = 'application/zip'
mime_string_lookup[ APPLICATION_HYDRUS_ENCRYPTED_ZIP ] = 'application/hydrus-encrypted-zip'
mime_string_lookup[ APPLICATIONS ] = 'application'
mime_string_lookup[ AUDIO_MP3 ] = 'audio/mp3'
mime_string_lookup[ AUDIO_OGG ] = 'audio/ogg'
mime_string_lookup[ AUDIO_FLAC ] = 'audio/flac'
mime_string_lookup[ AUDIO_WMA ] = 'audio/x-ms-wma'
mime_string_lookup[ AUDIO ] = 'audio'
mime_string_lookup[ TEXT_HTML ] = 'text/html'
mime_string_lookup[ VIDEO_FLV ] = 'video/x-flv'
mime_string_lookup[ VIDEO_MP4 ] = 'video/mp4'
mime_string_lookup[ VIDEO_WMV ] = 'video/x-ms-wmv'
mime_string_lookup[ VIDEO ] = 'video'
mime_string_lookup[ UNDETERMINED_WM ] = 'audio/x-ms-wma or video/x-ms-wmv'
mime_string_lookup[ APPLICATION_UNKNOWN ] = 'unknown mime'

mime_ext_lookup = {}

mime_ext_lookup[ APPLICATION_HYDRUS_CLIENT_COLLECTION ] = '.collection'
mime_ext_lookup[ IMAGE_JPEG ] = '.jpg'
mime_ext_lookup[ IMAGE_PNG ] = '.png'
mime_ext_lookup[ IMAGE_GIF ] = '.gif'
mime_ext_lookup[ IMAGE_BMP ] = '.bmp'
mime_ext_lookup[ IMAGE_ICON ] = '.ico'
mime_ext_lookup[ APPLICATION_FLASH ] = '.swf'
mime_ext_lookup[ APPLICATION_OCTET_STREAM ] = '.bin'
mime_ext_lookup[ APPLICATION_YAML ] = '.yaml'
mime_ext_lookup[ APPLICATION_PDF ] = '.pdf'
mime_ext_lookup[ APPLICATION_ZIP ] = '.zip'
mime_ext_lookup[ APPLICATION_HYDRUS_ENCRYPTED_ZIP ] = '.zip.encrypted'
mime_ext_lookup[ AUDIO_MP3 ] = '.mp3'
mime_ext_lookup[ AUDIO_OGG ] = '.ogg'
mime_ext_lookup[ AUDIO_FLAC ] = '.flac'
mime_ext_lookup[ AUDIO_WMA ] = '.wma'
mime_ext_lookup[ TEXT_HTML ] = '.html'
mime_ext_lookup[ VIDEO_FLV ] = '.flv'
mime_ext_lookup[ VIDEO_MP4 ] = '.mp4'
mime_ext_lookup[ VIDEO_WMV ] = '.wmv'
mime_ext_lookup[ APPLICATION_UNKNOWN ] = ''
#mime_ext_lookup[ 'application/x-rar-compressed' ] = '.rar'

ALLOWED_MIME_EXTENSIONS = [ mime_ext_lookup[ mime ] for mime in ALLOWED_MIMES ]

PREDICATE_TYPE_SYSTEM = 0
PREDICATE_TYPE_TAG = 1
PREDICATE_TYPE_NAMESPACE = 2
PREDICATE_TYPE_PARENT = 3

SITE_DOWNLOAD_TYPE_DEVIANT_ART = 0
SITE_DOWNLOAD_TYPE_GIPHY = 1
SITE_DOWNLOAD_TYPE_PIXIV = 2
SITE_DOWNLOAD_TYPE_BOORU = 3
SITE_DOWNLOAD_TYPE_TUMBLR = 4
SITE_DOWNLOAD_TYPE_HENTAI_FOUNDRY = 5
SITE_DOWNLOAD_TYPE_NEWGROUNDS = 6

SYSTEM_PREDICATE_TYPE_EVERYTHING = 0
SYSTEM_PREDICATE_TYPE_INBOX = 1
SYSTEM_PREDICATE_TYPE_ARCHIVE = 2
SYSTEM_PREDICATE_TYPE_UNTAGGED = 3
SYSTEM_PREDICATE_TYPE_NUM_TAGS = 4
SYSTEM_PREDICATE_TYPE_LIMIT = 5
SYSTEM_PREDICATE_TYPE_SIZE = 6
SYSTEM_PREDICATE_TYPE_AGE = 7
SYSTEM_PREDICATE_TYPE_HASH = 8
SYSTEM_PREDICATE_TYPE_WIDTH = 9
SYSTEM_PREDICATE_TYPE_HEIGHT = 10
SYSTEM_PREDICATE_TYPE_RATIO = 11
SYSTEM_PREDICATE_TYPE_DURATION = 12
SYSTEM_PREDICATE_TYPE_MIME = 13
SYSTEM_PREDICATE_TYPE_RATING = 14
SYSTEM_PREDICATE_TYPE_SIMILAR_TO = 15
SYSTEM_PREDICATE_TYPE_LOCAL = 17
SYSTEM_PREDICATE_TYPE_NOT_LOCAL = 18
SYSTEM_PREDICATE_TYPE_NUM_WORDS = 19
SYSTEM_PREDICATE_TYPE_FILE_SERVICE = 20

wxk_code_string_lookup = {
    wx.WXK_SPACE: 'space',
    wx.WXK_BACK: 'backspace',
    wx.WXK_TAB: 'tab',
    wx.WXK_RETURN: 'return',
    wx.WXK_NUMPAD_ENTER: 'enter',
    wx.WXK_PAUSE: 'pause',
    wx.WXK_ESCAPE: 'escape',
    wx.WXK_INSERT: 'insert',
    wx.WXK_DELETE: 'delete',
    wx.WXK_UP: 'up',
    wx.WXK_DOWN: 'down',
    wx.WXK_LEFT: 'left',
    wx.WXK_RIGHT: 'right',
    wx.WXK_HOME: 'home',
    wx.WXK_END: 'end',
    wx.WXK_PAGEDOWN: 'page up',
    wx.WXK_PAGEUP: 'page down',
    wx.WXK_F1: 'f1',
    wx.WXK_F2: 'f2',
    wx.WXK_F3: 'f3',
    wx.WXK_F4: 'f4',
    wx.WXK_F5: 'f5',
    wx.WXK_F6: 'f6',
    wx.WXK_F7: 'f7',
    wx.WXK_F8: 'f8',
    wx.WXK_F9: 'f9',
    wx.WXK_F10: 'f10',
    wx.WXK_F11: 'f11',
    wx.WXK_F12: 'f12',
    wx.WXK_ADD: '+',
    wx.WXK_DIVIDE: '/',
    wx.WXK_SUBTRACT: '-',
    wx.WXK_MULTIPLY: '*',
    wx.WXK_NUMPAD1: 'numpad 1',
    wx.WXK_NUMPAD2: 'numpad 2',
    wx.WXK_NUMPAD3: 'numpad 3',
    wx.WXK_NUMPAD4: 'numpad 4',
    wx.WXK_NUMPAD5: 'numpad 5',
    wx.WXK_NUMPAD6: 'numpad 6',
    wx.WXK_NUMPAD7: 'numpad 7',
    wx.WXK_NUMPAD8: 'numpad 8',
    wx.WXK_NUMPAD9: 'numpad 9',
    wx.WXK_NUMPAD0: 'numpad 0',
    wx.WXK_NUMPAD_UP: 'numpad up',
    wx.WXK_NUMPAD_DOWN: 'numpad down',
    wx.WXK_NUMPAD_LEFT: 'numpad left',
    wx.WXK_NUMPAD_RIGHT: 'numpad right',
    wx.WXK_NUMPAD_HOME: 'numpad home',
    wx.WXK_NUMPAD_END: 'numpad end',
    wx.WXK_NUMPAD_PAGEDOWN: 'numpad page up',
    wx.WXK_NUMPAD_PAGEUP: 'numpad page down',
    wx.WXK_NUMPAD_ADD: 'numpad +',
    wx.WXK_NUMPAD_DIVIDE: 'numpad /',
    wx.WXK_NUMPAD_SUBTRACT: 'numpad -',
    wx.WXK_NUMPAD_MULTIPLY: 'numpad *',
    wx.WXK_NUMPAD_DELETE: 'numpad delete',
    wx.WXK_NUMPAD_DECIMAL: 'numpad decimal'
    }

# request checking

BANDWIDTH_CONSUMING_REQUESTS = set()

BANDWIDTH_CONSUMING_REQUESTS.add( ( TAG_REPOSITORY, GET, 'update' ) )
BANDWIDTH_CONSUMING_REQUESTS.add( ( TAG_REPOSITORY, POST, 'mappings' ) )
BANDWIDTH_CONSUMING_REQUESTS.add( ( TAG_REPOSITORY, POST, 'petitions' ) )
BANDWIDTH_CONSUMING_REQUESTS.add( ( FILE_REPOSITORY, GET, 'update' ) )
BANDWIDTH_CONSUMING_REQUESTS.add( ( FILE_REPOSITORY, GET, 'file' ) )
BANDWIDTH_CONSUMING_REQUESTS.add( ( FILE_REPOSITORY, GET, 'thumbnail' ) )
BANDWIDTH_CONSUMING_REQUESTS.add( ( FILE_REPOSITORY, POST, 'file' ) )
BANDWIDTH_CONSUMING_REQUESTS.add( ( FILE_REPOSITORY, POST, 'petitions' ) )

service_requests = []
service_requests.append( ( GET, '', None ) )
service_requests.append( ( GET, 'favicon.ico', None ) )

local_file_requests = list( service_requests )
local_file_requests.append( ( GET, 'file', None ) )
local_file_requests.append( ( GET, 'thumbnail', None ) )

restricted_requests = list( service_requests )
restricted_requests.append( ( GET, 'access_key', None ) )
restricted_requests.append( ( GET, 'account', None ) )
restricted_requests.append( ( GET, 'account_info', MANAGE_USERS ) )
restricted_requests.append( ( GET, 'account_types', MANAGE_USERS ) )
restricted_requests.append( ( GET, 'options', GENERAL_ADMIN ) )
restricted_requests.append( ( GET, 'registration_keys', GENERAL_ADMIN ) )
restricted_requests.append( ( GET, 'session_key', None ) )
restricted_requests.append( ( GET, 'stats', GENERAL_ADMIN ) )
restricted_requests.append( ( POST, 'account', ( MANAGE_USERS, GENERAL_ADMIN ) ) )
restricted_requests.append( ( POST, 'account_types', GENERAL_ADMIN ) )
restricted_requests.append( ( POST, 'options', GENERAL_ADMIN ) )

admin_requests = list( restricted_requests )
admin_requests.append( ( GET, 'init', None ) )
admin_requests.append( ( GET, 'services', EDIT_SERVICES ) )
admin_requests.append( ( POST, 'backup', EDIT_SERVICES ) )
admin_requests.append( ( POST, 'services', EDIT_SERVICES ) )

repository_requests = list( restricted_requests )
repository_requests.append( ( GET, 'num_petitions', RESOLVE_PETITIONS ) )
repository_requests.append( ( GET, 'petition', RESOLVE_PETITIONS ) )
repository_requests.append( ( GET, 'update', GET_DATA ) )
repository_requests.append( ( POST, 'news', GENERAL_ADMIN ) )
repository_requests.append( ( POST, 'update', POST_DATA ) )

file_repository_requests = list( repository_requests )
file_repository_requests.append( ( GET, 'file', GET_DATA ) )
file_repository_requests.append( ( GET, 'ip', GENERAL_ADMIN ) )
file_repository_requests.append( ( GET, 'thumbnail', GET_DATA ) )
file_repository_requests.append( ( POST, 'file', POST_DATA ) )

tag_repository_requests = list( repository_requests )

message_depot_requests = list( restricted_requests )
message_depot_requests.append( ( GET, 'message', GET_DATA ) )
message_depot_requests.append( ( GET, 'message_info_since', GET_DATA ) )
message_depot_requests.append( ( GET, 'public_key', None ) )
message_depot_requests.append( ( POST, 'contact', POST_DATA ) )
message_depot_requests.append( ( POST, 'message', None ) )
message_depot_requests.append( ( POST, 'message_statuses', None ) )

all_requests = []
all_requests.extend( [ ( LOCAL_FILE, request_type, request, permissions ) for ( request_type, request, permissions ) in local_file_requests ] )
all_requests.extend( [ ( SERVER_ADMIN, request_type, request, permissions ) for ( request_type, request, permissions ) in admin_requests ] )
all_requests.extend( [ ( FILE_REPOSITORY, request_type, request, permissions ) for ( request_type, request, permissions ) in file_repository_requests ] )
all_requests.extend( [ ( TAG_REPOSITORY, request_type, request, permissions ) for ( request_type, request, permissions ) in tag_repository_requests ] )
all_requests.extend( [ ( MESSAGE_DEPOT, request_type, request, permissions ) for ( request_type, request, permissions ) in message_depot_requests ] )

ALLOWED_REQUESTS = { ( service_type, request_type, request ) for ( service_type, request_type, request, permissions ) in all_requests }

REQUESTS_TO_PERMISSIONS = { ( service_type, request_type, request ) : permissions for ( service_type, request_type, request, permissions ) in all_requests }

# default options

DEFAULT_LOCAL_FILE_PORT = 45865
DEFAULT_SERVER_ADMIN_PORT = 45870
DEFAULT_SERVICE_PORT = 45871

DEFAULT_OPTIONS = {}

DEFAULT_OPTIONS[ SERVER_ADMIN ] = {}
DEFAULT_OPTIONS[ SERVER_ADMIN ][ 'max_monthly_data' ] = None
DEFAULT_OPTIONS[ SERVER_ADMIN ][ 'max_storage' ] = None
DEFAULT_OPTIONS[ SERVER_ADMIN ][ 'message' ] = 'hydrus server administration service'
DEFAULT_OPTIONS[ SERVER_ADMIN ][ 'upnp' ] = None

DEFAULT_OPTIONS[ FILE_REPOSITORY ] = {}
DEFAULT_OPTIONS[ FILE_REPOSITORY ][ 'max_monthly_data' ] = None
DEFAULT_OPTIONS[ FILE_REPOSITORY ][ 'max_storage' ] = None
DEFAULT_OPTIONS[ FILE_REPOSITORY ][ 'log_uploader_ips' ] = False
DEFAULT_OPTIONS[ FILE_REPOSITORY ][ 'message' ] = 'hydrus file repository'
DEFAULT_OPTIONS[ FILE_REPOSITORY ][ 'upnp' ] = None

DEFAULT_OPTIONS[ TAG_REPOSITORY ] = {}
DEFAULT_OPTIONS[ TAG_REPOSITORY ][ 'max_monthly_data' ] = None
DEFAULT_OPTIONS[ TAG_REPOSITORY ][ 'message' ] = 'hydrus tag repository'
DEFAULT_OPTIONS[ TAG_REPOSITORY ][ 'upnp' ] = None

DEFAULT_OPTIONS[ MESSAGE_DEPOT ] = {}
DEFAULT_OPTIONS[ MESSAGE_DEPOT ][ 'max_monthly_data' ] = None
DEFAULT_OPTIONS[ MESSAGE_DEPOT ][ 'max_storage' ] = None
DEFAULT_OPTIONS[ MESSAGE_DEPOT ][ 'message' ] = 'hydrus message depot'
DEFAULT_OPTIONS[ MESSAGE_DEPOT ][ 'upnp' ] = None

# Hydrus pubsub

EVT_PUBSUB = HydrusPubSub.EVT_PUBSUB
pubsub = HydrusPubSub.HydrusPubSub()

def default_dict_list(): return collections.defaultdict( list )

def default_dict_set(): return collections.defaultdict( set )

def b( text_producing_object ):
    
    if type( text_producing_object ) == unicode: return text_producing_object.encode( 'utf-8' )
    else: return str( text_producing_object )
    
def ChunkifyList( the_list, chunk_size ):
    
    for i in xrange( 0, len( the_list ), chunk_size ): yield the_list[ i : i + chunk_size ]
    
def BuildKeyToListDict( pairs ):
    
    d = collections.defaultdict( list )
    
    for ( key, value ) in pairs: d[ key ].append( value )
    
    return d
    
def BuildKeyToSetDict( pairs ):
    
    d = collections.defaultdict( set )
    
    for ( key, value ) in pairs: d[ key ].add( value )
    
    return d
    
def CalculateScoreFromRating( count, rating ):
    
    # http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
    
    count = float( count )
    
    positive = count * rating
    negative = count * ( 1.0 - rating )
    
    # positive + negative = count
    
    # I think I've parsed this correctly from the website! Not sure though!
    score = ( ( positive + 1.9208 ) / count - 1.96 * ( ( ( positive * negative ) / count + 0.9604 ) ** 0.5 ) / count ) / ( 1 + 3.8416 / count )
    
    return score
    
def CleanTag( tag ):
    
    tag = tag[:1024]
    
    if tag == '': return ''
    
    tag = tag.lower()
    
    tag = u( tag )
    
    tag = re.sub( '[\s]+', ' ', tag, flags = re.UNICODE ) # turns multiple spaces into single spaces
    
    tag = re.sub( '\s\Z', '', tag, flags = re.UNICODE ) # removes space at the end
    
    while re.match( '\s|-|system:', tag, flags = re.UNICODE ) is not None:
        
        tag = re.sub( '\A(\s|-|system:)', '', tag, flags = re.UNICODE ) # removes space at the beginning
        
    
    return tag
    
def ConvertAbsPathToPortablePath( abs_path ):
    
    if abs_path == '': return None
    
    try: return os.path.relpath( abs_path, BASE_DIR )
    except: return abs_path
    
def ConvertIntToBytes( size ):
    
    if size is None: return 'unknown size'
    
    suffixes = ( '', 'K', 'M', 'G', 'T', 'P' )
    
    suffix_index = 0
    
    size = float( size )
    
    while size > 1024.0:
        
        size = size / 1024.0
        
        suffix_index += 1
        
    
    if size < 10.0: return '%.1f' % size + suffixes[ suffix_index ] + 'B'
    
    return '%.0f' % size + suffixes[ suffix_index ] + 'B'
    
def ConvertIntToPrettyString( num ): return u( locale.format( "%d", num, grouping = True ) )

def ConvertMillisecondsToPrettyTime( ms ):
    
    hours = ms / 3600000
    
    if hours == 1: hours_result = '1 hour'
    else: hours_result = u( hours ) + ' hours'
    
    ms = ms % 3600000
    
    minutes = ms / 60000
    
    if minutes == 1: minutes_result = '1 minute'
    else: minutes_result = u( minutes ) + ' minutes'
    
    ms = ms % 60000
    
    seconds = ms / 1000
    
    if seconds == 1: seconds_result = '1 second'
    else: seconds_result = u( seconds ) + ' seconds'
    
    detailed_seconds = float( ms ) / 1000.0
    
    if detailed_seconds == 1.0: detailed_seconds_result = '1.0 seconds'
    else:detailed_seconds_result = '%.1f' % detailed_seconds + ' seconds'
    
    ms = ms % 1000
    
    if ms == 1: milliseconds_result = '1 millisecond'
    else: milliseconds_result = u( ms ) + ' milliseconds'
    
    if hours > 0: return hours_result + ' ' + minutes_result
    
    if minutes > 0: return minutes_result + ' ' + seconds_result
    
    if seconds > 0: return detailed_seconds_result
    
    return milliseconds_result
    
def ConvertNumericalRatingToPrettyString( lower, upper, rating, rounded_result = False, out_of = True ):
    
    rating_converted = ( rating * ( upper - lower ) ) + lower
    
    if rounded_result: s = u( '%.2f' % round( rating_converted ) )
    else: s = u( '%.2f' % rating_converted )
    
    if out_of:
        
        if lower in ( 0, 1 ): s += '/' + u( '%.2f' % upper )
        
    
    return s
    
def ConvertPortablePathToAbsPath( portable_path ):
    
    if portable_path is None: return None
    
    if os.path.isabs( portable_path ): abs_path = portable_path
    else: abs_path = os.path.normpath( BASE_DIR + os.path.sep + portable_path )
    
    if os.path.exists( abs_path ): return abs_path
    else: return None
    
def ConvertServiceIdentifiersToContentUpdatesToPrettyString( service_identifiers_to_content_updates ):
    
    num_files = 0
    actions = set()
    locations = set()
    
    extra_words = ''
    
    for ( service_identifier, content_updates ) in service_identifiers_to_content_updates.items():
        
        if len( content_updates ) > 0: locations.add( service_identifier.GetName() )
        
        for content_update in content_updates:
            
            ( data_type, action, row ) = content_update.ToTuple()
            
            if data_type == CONTENT_DATA_TYPE_MAPPINGS: extra_words = ' tags for'
            
            actions.add( content_update_string_lookup[ action ] )
            
            num_files += len( content_update.GetHashes() )
            
        
    
    s = ', '.join( locations ) + '->' + ', '.join( actions ) + extra_words + ' ' + ConvertIntToPrettyString( num_files ) + ' files'
    
    return s
    
def ConvertShortcutToPrettyShortcut( modifier, key, action ):
    
    if modifier == wx.ACCEL_NORMAL: modifier = ''
    elif modifier == wx.ACCEL_ALT: modifier = 'alt'
    elif modifier == wx.ACCEL_CTRL: modifier = 'ctrl'
    elif modifier == wx.ACCEL_SHIFT: modifier = 'shift'
    
    if key in range( 65, 91 ): key = chr( key + 32 ) # + 32 for converting ascii A -> a
    elif key in range( 97, 123 ): key = chr( key )
    else: key = wxk_code_string_lookup[ key ]
    
    return ( modifier, key, action )
    
def ConvertStatusToPrefix( status ):
    
    if status == CURRENT: return ''
    elif status == PENDING: return '(+) '
    elif status == PETITIONED: return '(-) '
    elif status == DELETED: return '(X) '
    elif status == DELETED_PENDING: return '(X+) '
    
def ConvertTimestampToPrettyAge( timestamp ):
    
    if timestamp == 0 or timestamp is None: return 'unknown age'
    
    age = GetNow() - timestamp
    
    seconds = age % 60
    if seconds == 1: s = '1 second'
    else: s = u( seconds ) + ' seconds'
    
    age = age / 60
    minutes = age % 60
    if minutes == 1: m = '1 minute'
    else: m = u( minutes ) + ' minutes'
    
    age = age / 60
    hours = age % 24
    if hours == 1: h = '1 hour'
    else: h = u( hours ) + ' hours'
    
    age = age / 24
    days = age % 30
    if days == 1: d = '1 day'
    else: d = u( days ) + ' days'
    
    age = age / 30
    months = age % 12
    if months == 1: mo = '1 month'
    else: mo = u( months ) + ' months'
    
    years = age / 12
    if years == 1: y = '1 year'
    else: y = u( years ) + ' years'
    
    if years > 0: return ' '.join( ( y, mo ) ) + ' old'
    elif months > 0: return ' '.join( ( mo, d ) ) + ' old'
    elif days > 0: return ' '.join( ( d, h ) ) + ' old'
    elif hours > 0: return ' '.join( ( h, m ) ) + ' old'
    else: return ' '.join( ( m, s ) ) + ' old'
    
def ConvertTimestampToPrettyAgo( timestamp ):
    
    if timestamp is None or timestamp == 0: return 'unknown time'
    
    age = GetNow() - timestamp
    
    seconds = age % 60
    if seconds == 1: s = '1 second'
    else: s = u( seconds ) + ' seconds'
    
    age = age / 60
    minutes = age % 60
    if minutes == 1: m = '1 minute'
    else: m = u( minutes ) + ' minutes'
    
    age = age / 60
    hours = age % 24
    if hours == 1: h = '1 hour'
    else: h = u( hours ) + ' hours'
    
    age = age / 24
    days = age % 30
    if days == 1: d = '1 day'
    else: d = u( days ) + ' days'
    
    age = age / 30
    months = age % 12
    if months == 1: mo = '1 month'
    else: mo = u( months ) + ' months'
    
    years = age / 12
    if years == 1: y = '1 year'
    else: y = u( years ) + ' years'
    
    if years > 0: return ' '.join( ( y, mo ) ) + ' ago'
    elif months > 0: return ' '.join( ( mo, d ) ) + ' ago'
    elif days > 0: return ' '.join( ( d, h ) ) + ' ago'
    elif hours > 0: return ' '.join( ( h, m ) ) + ' ago'
    else: return ' '.join( ( m, s ) ) + ' ago'
    
def ConvertTimestampToPrettyExpiry( timestamp ):
    
    if timestamp is None: return 'does not expire'
    if timestamp == 0: return 'unknown expiration'
    
    expiry = GetNow() - timestamp
    
    if expiry >= 0: already_happend = True
    else:
        
        expiry *= -1
        
        already_happend = False
        
    
    seconds = expiry % 60
    if seconds == 1: s = '1 second'
    else: s = u( seconds ) + ' seconds'
    
    expiry = expiry / 60
    minutes = expiry % 60
    if minutes == 1: m = '1 minute'
    else: m = u( minutes ) + ' minutes'
    
    expiry = expiry / 60
    hours = expiry % 24
    if hours == 1: h = '1 hour'
    else: h = u( hours ) + ' hours'
    
    expiry = expiry / 24
    days = expiry % 30
    if days == 1: d = '1 day'
    else: d = u( days ) + ' days'
    
    expiry = expiry / 30
    months = expiry % 12
    if months == 1: mo = '1 month'
    else: mo = u( months ) + ' months'
    
    years = expiry / 12
    if years == 1: y = '1 year'
    else: y = u( years ) + ' years'
    
    if already_happend:
        
        if years > 0: return 'expired ' + ' '.join( ( y, mo ) ) + ' ago'
        elif months > 0: return 'expired ' + ' '.join( ( mo, d ) ) + ' ago'
        elif days > 0: return 'expired ' + ' '.join( ( d, h ) ) + ' ago'
        elif hours > 0: return 'expired ' + ' '.join( ( h, m ) ) + ' ago'
        else: return 'expired ' + ' '.join( ( m, s ) ) + ' ago'
        
    else:
        
        if years > 0: return 'expires in ' + ' '.join( ( y, mo ) )
        elif months > 0: return 'expires in ' + ' '.join( ( mo, d ) )
        elif days > 0: return 'expires in ' + ' '.join( ( d, h ) )
        elif hours > 0: return 'expires in ' + ' '.join( ( h, m ) )
        else: return 'expires in ' + ' '.join( ( m, s ) )
        
    
def ConvertTimestampToPrettyPending( timestamp ):
    
    if timestamp is None: return ''
    if timestamp == 0: return 'imminent'
    
    pending = GetNow() - timestamp
    
    if pending >= 0: return 'imminent'
    else: pending *= -1
    
    seconds = pending % 60
    if seconds == 1: s = '1 second'
    else: s = u( seconds ) + ' seconds'
    
    pending = pending / 60
    minutes = pending % 60
    if minutes == 1: m = '1 minute'
    else: m = u( minutes ) + ' minutes'
    
    pending = pending / 60
    hours = pending % 24
    if hours == 1: h = '1 hour'
    else: h = u( hours ) + ' hours'
    
    pending = pending / 24
    days = pending % 30
    if days == 1: d = '1 day'
    else: d = u( days ) + ' days'
    
    pending = pending / 30
    months = pending % 12
    if months == 1: mo = '1 month'
    else: mo = u( months ) + ' months'
    
    years = pending / 12
    if years == 1: y = '1 year'
    else: y = u( years ) + ' years'
    
    if years > 0: return 'in ' + ' '.join( ( y, mo ) )
    elif months > 0: return 'in ' + ' '.join( ( mo, d ) )
    elif days > 0: return 'in ' + ' '.join( ( d, h ) )
    elif hours > 0: return 'in ' + ' '.join( ( h, m ) )
    else: return 'in ' + ' '.join( ( m, s ) )
    
def ConvertTimestampToPrettySync( timestamp ):
    
    if timestamp is None or timestamp == 0: return 'not updated'
    
    age = GetNow() - timestamp
    
    seconds = age % 60
    if seconds == 1: s = '1 second'
    else: s = u( seconds ) + ' seconds'
    
    age = age / 60
    minutes = age % 60
    if minutes == 1: m = '1 minute'
    else: m = u( minutes ) + ' minutes'
    
    age = age / 60
    hours = age % 24
    if hours == 1: h = '1 hour'
    else: h = u( hours ) + ' hours'
    
    age = age / 24
    days = age % 30
    if days == 1: d = '1 day'
    else: d = u( days ) + ' days'
    
    age = age / 30
    months = age % 12
    if months == 1: mo = '1 month'
    else: mo = u( months ) + ' months'
    
    years = age / 12
    if years == 1: y = '1 year'
    else: y = u( years ) + ' years'
    
    if years > 0: return 'updated to ' + ' '.join( ( y, mo ) ) + ' ago'
    elif months > 0: return 'updated to ' + ' '.join( ( mo, d ) ) + ' ago'
    elif days > 0: return 'updated to ' + ' '.join( ( d, h ) ) + ' ago'
    elif hours > 0: return 'updated to ' + ' '.join( ( h, m ) ) + ' ago'
    else: return 'updated to ' + ' '.join( ( m, s ) ) + ' ago'
    
def ConvertTimestampToPrettyTime( timestamp ): return time.strftime( '%Y/%m/%d %H:%M:%S', time.localtime( timestamp ) )

def ConvertTimestampToHumanPrettyTime( timestamp ):
    
    now = GetNow()
    
    difference = now - timestamp
    
    if difference < 60: return 'just now'
    elif difference < 86400 * 7: return ConvertTimestampToPrettyAgo( timestamp )
    else: return ConvertTimestampToPrettyTime( timestamp )
    
def ConvertTimeToPrettyTime( secs ):
    
    return time.strftime( '%H:%M:%S', time.gmtime( secs ) )
    
def ConvertUnitToInteger( unit ):
    
    if unit == 'B': return 1
    elif unit == 'KB': return 1024
    elif unit == 'MB': return 1048576
    elif unit == 'GB': return 1073741824
    
def ConvertUnitToString( unit ):
    
    if unit == 8: return 'B'
    elif unit == 1024: return 'KB'
    elif unit == 1048576: return 'MB'
    elif unit == 1073741824: return 'GB'
    
def ConvertZoomToPercentage( zoom ):
    
    zoom = zoom * 100.0
    
    pretty_zoom = '%.0f' % zoom + '%'
    
    return pretty_zoom
    
def ShowExceptionDefault( e ):
    
    etype = type( e )
    
    value = u( e )
    
    trace_list = traceback.format_stack()
    
    trace = ''.join( trace_list )
    
    message = u( etype.__name__ ) + ': ' + u( value ) + os.linesep + u( trace )
    
    try: print( message )
    except: print( repr( message ) )
    
ShowException = ShowExceptionDefault

def GetEmptyDataDict():
    
    data = collections.defaultdict( default_dict_list )
    
    return data
    
def GetNow(): return int( time.time() )

def GetShortcutFromEvent( event ):
    
    modifier = wx.ACCEL_NORMAL
    
    if event.AltDown(): modifier = wx.ACCEL_ALT
    elif event.ControlDown(): modifier = wx.ACCEL_CTRL
    elif event.ShiftDown(): modifier = wx.ACCEL_SHIFT
    
    key = event.KeyCode
    
    return ( modifier, key )
    
def GetTempPath(): return TEMP_DIR + os.path.sep + os.urandom( 32 ).encode( 'hex' )

def IntelligentMassIntersect( sets_to_reduce ):
    
    answer = None
    
    sets_to_reduce = list( sets_to_reduce )
    
    sets_to_reduce.sort( cmp = lambda x, y: cmp( len( x ), len( y ) ) )
    
    for set_to_reduce in sets_to_reduce:
        
        if len( set_to_reduce ) == 0: return set()
        
        if answer is None: answer = set( set_to_reduce )
        else:
            
            # same thing as union; I could go &= here, but I want to be quick, so use the function call
            if len( answer ) == 0: return set()
            else: answer.intersection_update( set_to_reduce )
            
        
    
    if answer is None: return set()
    else: return answer
    
def IsCollection( mime ): return mime in ( APPLICATION_HYDRUS_CLIENT_COLLECTION, ) # this is a little convoluted, but I want to keep it similar to IsImage, IsText, IsAudio, IsX

def IsImage( mime ): return mime in ( IMAGE_JPEG, IMAGE_GIF, IMAGE_PNG, IMAGE_BMP )

def MergeKeyToListDicts( key_to_list_dicts ):
    
    result = collections.defaultdict( list )
    
    for key_to_list_dict in key_to_list_dicts:
        
        for ( key, value ) in key_to_list_dict.items(): result[ key ].extend( value )
        
    
    return result

def u( text_producing_object ):
    
    if type( text_producing_object ) in ( str, unicode, bs4.element.NavigableString ): text = text_producing_object
    else:
        try: text = str( text_producing_object ) # dealing with exceptions, etc...
        except: text = repr( text_producing_object )
    
    try: return unicode( text )
    except:
        
        try: return text.decode( locale.getpreferredencoding() )
        except: return str( text )
        
    
def SearchEntryMatchesPredicate( search_entry, predicate ):
    
    ( predicate_type, info ) = predicate.GetInfo()
    
    if predicate_type == PREDICATE_TYPE_TAG:
        
        ( operator, value ) = info
        
        return SearchEntryMatchesTag( search_entry, value )
        
    else: return False
    
def SearchEntryMatchesTag( search_entry, tag, search_siblings = True ):
    
    if ':' in search_entry: ( search_entry_namespace, search_entry ) = search_entry.split( ':', 1 )
    
    if search_siblings:
        
        sibling_manager = app.GetManager( 'tag_siblings' )
        
        tags = sibling_manager.GetAllSiblings( tag )
        
    else: tags = [ tag ]
    
    for tag in tags:
        
        if ':' in tag:
            
            ( n, t ) = tag.split( ':', 1 )
            
            comparee = t
            
        else: comparee = tag
        
        if comparee.startswith( search_entry ) or ' ' + search_entry in comparee: return True
        
    
    return False
    
def SplayListForDB( xs ): return '(' + ','.join( [ '"' + u( x ) + '"' for x in xs ] ) + ')'

def SplayTupleListForDB( first_column_name, second_column_name, xys ): return ' OR '.join( [ '( ' + first_column_name + '=' + u( x ) + ' AND ' + second_column_name + ' IN ' + SplayListForDB( ys ) + ' )' for ( x, ys ) in xys ] )

def ThumbnailResolution( original_resolution, target_resolution ):
    
    ( original_width, original_height ) = original_resolution
    ( target_width, target_height ) = target_resolution
    
    if original_width > target_width:
        
        original_height = max( original_height * target_width / float( original_width ), 1 )
        original_width = target_width
        
    
    if round( original_height ) > target_height:
        
        original_width = max( original_width * target_height / float( original_height ), 1 )
        original_height = target_height
        
    
    return ( int( round( original_width ) ), int( round( original_height ) ) )

class AdvancedHTTPConnection():
    
    def __init__( self, url = '', scheme = 'http', host = '', port = None, service_identifier = None, accept_cookies = False ):
        
        if len( url ) > 0:
            
            parse_result = urlparse.urlparse( url )
            
            ( scheme, host, port ) = ( parse_result.scheme, parse_result.hostname, parse_result.port )
            
        
        self._report_hooks = []
        
        self._scheme = scheme
        self._host = host
        self._port = port
        self._service_identifier = service_identifier
        self._accept_cookies = accept_cookies
        
        self._cookies = {}
        
        self._RefreshConnection()
        
    
    def _AcceptCookies( self, response ):
    
        for cookie in response.msg.getallmatchingheaders( 'Set-Cookie' ): # msg is a mimetools.Message
            
            try:
                
                cookie = cookie.replace( 'Set-Cookie: ', '' )
                
                if ';' in cookie: ( cookie, expiry_gumpf ) = cookie.split( ';', 1 )
                
                ( k, v ) = cookie.split( '=' )
                
                self._cookies[ k ] = v
                
            except: pass
            
        
    
    def _CheckHydrusVersion( self, response ):
        
        service_type = self._service_identifier.GetType()
        
        service_string = service_string_lookup[ service_type ]
        
        server_header = response.getheader( 'Server' )
        
        if server_header is None or service_string not in server_header:
            
            app.Write( 'service_updates', { self._service_identifier : [ ServiceUpdate( SERVICE_UPDATE_ACCOUNT, GetUnknownAccount() ) ] })
            
            raise HydrusExceptions.WrongServiceTypeException( 'Target was not a ' + service_string + '!' )
            
        
        ( service_string_gumpf, network_version ) = server_header.split( '/' )
        
        network_version = int( network_version )
        
        if network_version != NETWORK_VERSION:
            
            if network_version > NETWORK_VERSION: message = 'Your client is out of date; please download the latest release.'
            else: message = 'The server is out of date; please ask its admin to update to the latest release.'
            
            raise HydrusExceptions.NetworkVersionException( 'Network version mismatch! The server\'s network version was ' + u( network_version ) + ', whereas your client\'s is ' + u( NETWORK_VERSION ) + '! ' + message )
            
        
    
    def _DoHydrusBandwidth( self, request, response, size_of_request, size_of_response ):
        
        service_type = self._service_identifier.GetType()
        
        if '?' in request: request_command = request.split( '?' )[0]
        else: request_command = request
        
        if '/' in request_command: request_command = request_command.split( '/' )[1]
        
        if request_type == 'GET':
            
            if ( service_type, GET, request_command ) in BANDWIDTH_CONSUMING_REQUESTS: pubsub.pub( 'service_updates_delayed', { self._service_identifier : [ ServiceUpdate( SERVICE_UPDATE_REQUEST_MADE, size_of_response ) ] } )
            
        elif ( service_type, POST, request_command ) in BANDWIDTH_CONSUMING_REQUESTS: pubsub.pub( 'service_updates_delayed', { self._service_identifier : [ ServiceUpdate( SERVICE_UPDATE_REQUEST_MADE, size_of_request ) ] } )
        
    
    def _DoRequest( self, request_type, request, headers, body, response_to_path ):
        
        if request == '/backup' and self._service_identifier is not None and self._service_identifier.GetType() == SERVER_ADMIN:
            
            timeout = 600
            
            if self._scheme == 'http': connection = httplib.HTTPConnection( self._host, self._port, timeout = timeout )
            else: connection = httplib.HTTPSConnection( self._host, self._port, timeout = timeout )
            
            connection.request( request_type, request, headers = headers, body = body )
            
            response = connection.getresponse()
            
        else:
            
            self._connection.request( request_type, request, headers = headers, body = body )
            
            response = self._connection.getresponse()
            
        
        if self._service_identifier is not None: self._CheckHydrusVersion( response )
        
        content_length = response.getheader( 'Content-Length' )
        
        if content_length is not None: content_length = int( content_length )
        
        block_size = 64 * 1024
        
        if response.status == 200 and response_to_path:
            
            temp_path = GetTempPath()
            
            size_of_response = 0
            
            with open( temp_path, 'wb' ) as f:
                
                next_block = response.read( block_size )
                
                while next_block != '':
                    
                    size_of_response += len( next_block )
                    
                    f.write( next_block )
                    
                    for hook in self._report_hooks: hook( content_length, size_of_response )
                    
                    next_block = response.read( block_size )
                    
                
            
            parsed_response = temp_path
            
        else:
            
            data = ''
            
            next_block = response.read( block_size )
            
            while next_block != '':
                
                data += next_block
                
                for hook in self._report_hooks: hook( content_length, len( data ) )
                
                next_block = response.read( block_size )
                
            
            size_of_response = len( data )
            
            parsed_response = self._TryToParseResponse( response, data )
            
        
        return ( response, parsed_response, size_of_response )
        
    
    def _RefreshConnection( self ):
        
        timeout = 30
        
        if self._scheme == 'http': self._connection = httplib.HTTPConnection( self._host, self._port, timeout = timeout )
        else: self._connection = httplib.HTTPSConnection( self._host, self._port, timeout = timeout )
        
    
    def _TryToParseResponse( self, response, data ):
        
        content_type = response.getheader( 'Content-Type' )
        
        if content_type is None: parsed_response = data
        else:
            
            if '; ' in content_type: ( mime_string, additional_info ) = content_type.split( '; ', 1 )
            else: ( mime_string, additional_info ) = ( content_type, '' )
            
            if 'charset=' in additional_info:
        
                # this does utf-8, ISO-8859-4, whatever
                
                ( gumpf, charset ) = additional_info.split( '=' )
                
                try: parsed_response = data.decode( charset )
                except: parsed_response = data
                
            elif content_type in mime_enum_lookup and mime_enum_lookup[ content_type ] == APPLICATION_YAML:
                
                try: parsed_response = yaml.safe_load( data )
                except Exception as e: raise HydrusExceptions.NetworkVersionException( 'Failed to parse a response object!' + os.linesep + u( e ) )
                
            elif content_type == 'text/html':
                
                try: parsed_response = data.decode( 'utf-8' )
                except: parsed_response = data
                
            else: parsed_response = data
            
        
        return parsed_response
        
    
    def AddReportHook( self, hook ): self._report_hooks.append( hook )
    
    def ClearReportHooks( self ): self._report_hooks = []
    
    def close( self ): self._connection.close()
    
    def connect( self ): self._connection.connect()
    
    def GetCookies( self ): return self._cookies
    
    def geturl( self, url, headers = {}, response_to_path = False, is_redirect = False, follow_redirects = True ):
        
        parse_result = urlparse.urlparse( url )
        
        request = parse_result.path
        
        query = parse_result.query
        
        if query != '': request += '?' + query
        
        return self.request( 'GET', request, headers = headers, response_to_path = response_to_path, is_redirect = is_redirect, follow_redirects = follow_redirects )
        
    
    def request( self, request_type, request, headers = {}, body = None, response_to_path = False, is_redirect = False, follow_redirects = True ):
        
        if body is None: size_of_request = 0
        else: size_of_request = len( body )
        
        if 'User-Agent' not in headers: headers[ 'User-Agent' ] = 'hydrus/' + u( NETWORK_VERSION )
        
        if len( self._cookies ) > 0: headers[ 'Cookie' ] = '; '.join( [ k + '=' + v for ( k, v ) in self._cookies.items() ] )
        
        try: ( response, parsed_response, size_of_response ) = self._DoRequest( request_type, request, headers, body, response_to_path )
        except ( httplib.CannotSendRequest, httplib.BadStatusLine ):
            
            # for some reason, we can't send a request on the current connection, so let's make a new one and try again!
            
            self._RefreshConnection()
            
            ( response, parsed_response, size_of_response ) = self._DoRequest( request_type, request, headers, body, response_to_path )
            
        
        if self._accept_cookies: self._AcceptCookies( response )
        
        if self._service_identifier is not None: self._DoHydrusBandwidth( request, response, size_of_request, size_of_response )
        
        if response.status == 200: return parsed_response
        elif response.status == 205: return
        elif response.status in ( 301, 302, 303, 307 ):
            
            location = response.getheader( 'Location' )
            
            if location is None: raise Exception( data )
            else:
                
                if not follow_redirects: return ''
                
                if is_redirect: raise Exception( 'Too many redirects!' )
                
                url = location
                
                parse_result = urlparse.urlparse( url )
                
                redirected_request = parse_result.path
                
                redirected_query = parse_result.query
                
                if redirected_query != '': redirected_request += '?' + redirected_query
                
                ( scheme, host, port ) = ( parse_result.scheme, parse_result.hostname, parse_result.port )
                
                if ( scheme is None or scheme == self._scheme ) and ( request == redirected_request or request in redirected_request or redirected_request in request ): raise Exception( 'Redirection problem' )
                else:
                    
                    if host is None or ( scheme == self._scheme and host == self._host and port == self._port ): new_connection = self
                    else: new_connection = AdvancedHTTPConnection( url )
                    
                    if response.status in ( 301, 307 ):
                        
                        # 301: moved permanently, repeat request
                        # 307: moved temporarily, repeat request
                        
                        return new_connection.request( request_type, redirected_request, headers = headers, body = body, response_to_path = response_to_path, is_redirect = True )
                        
                    elif response.status in ( 302, 303 ):
                        
                        # 302: moved temporarily, repeat request (except everyone treats it like 303 for no good fucking reason)
                        # 303: thanks, now go here with GET
                        
                        return new_connection.request( 'GET', redirected_request, response_to_path = response_to_path, is_redirect = True )
                        
                    
                
            
        elif response.status == 304: raise HydrusExceptions.NotModifiedException()
        else:
            
            if self._service_identifier is not None:
                
                app.Write( 'service_updates', { self._service_identifier : [ ServiceUpdate( SERVICE_UPDATE_ERROR, parsed_response ) ] } )
                
                if response.status in ( 401, 426 ): app.Write( 'service_updates', { self._service_identifier : [ ServiceUpdate( SERVICE_UPDATE_ACCOUNT, GetUnknownAccount() ) ] } )
                
            
            if response.status == 401: raise HydrusExceptions.PermissionException( parsed_response )
            elif response.status == 403: raise HydrusExceptions.ForbiddenException( parsed_response )
            elif response.status == 404: raise HydrusExceptions.NotFoundException( parsed_response )
            elif response.status == 426: raise HydrusExceptions.NetworkVersionException( parsed_response )
            elif response.status in ( 500, 501, 502, 503 ): raise Exception( parsed_response )
            else: raise Exception( parsed_response )
           
        
    
    def SetCookie( self, key, value ): self._cookies[ key ] = value
    
get_connection = AdvancedHTTPConnection

class HydrusYAMLBase( yaml.YAMLObject ):
    
    yaml_loader = yaml.SafeLoader
    yaml_dumper = yaml.SafeDumper
    
class Account( HydrusYAMLBase ):
    
    yaml_tag = u'!Account'
    
    def __init__( self, account_id, account_type, created, expiry, used_data, banned_info = None ):
        
        HydrusYAMLBase.__init__( self )
        
        self._account_id = account_id
        self._account_type = account_type
        self._created = created
        self._expiry = expiry
        self._used_data = used_data
        self._banned_info = banned_info
        
        self._object_instantiation_timestamp = GetNow()
        
    
    def __repr__( self ): return self.ConvertToString()
    
    def __str__( self ): return self.ConvertToString()
    
    def _IsBanned( self ):
        
        if self._banned_info is None: return False
        else:
            
            ( reason, created, expiry ) = self._banned_info
            
            if expiry is None: return True
            else: return GetNow() > expiry
            
        
    
    def _IsExpired( self ):
        
        if self._expiry is None: return False
        else: return GetNow() > self._expiry
        
    
    def CheckPermission( self, permission ):
        
        if self._IsBanned(): raise HydrusExceptions.PermissionException( 'This account is banned!' )
        
        if self._IsExpired(): raise HydrusExceptions.PermissionException( 'This account is expired.' )
        
        ( max_num_bytes, max_num_requests ) = self._account_type.GetMaxMonthlyData()
        
        ( used_bytes, used_requests ) = self._used_data
        
        if max_num_bytes is not None and used_bytes > max_num_bytes: raise HydrusExceptions.PermissionException( 'You have hit your data transfer limit (' + ConvertIntToBytes( max_num_bytes ) + '), and cannot make any more requests for the month.' )
        
        if max_num_requests is not None and used_requests > max_num_requests: raise HydrusExceptions.PermissionException( 'You have hit your requests limit (' + ConvertIntToPrettyString( max_num_requests ) + '), and cannot make any more requests for the month.' )
        
        if not self._account_type.HasPermission( permission ): raise HydrusExceptions.PermissionException( 'You do not have permission to do that.' )
        
    
    def ConvertToString( self ): return ConvertTimestampToPrettyAge( self._created ) + os.linesep + self._account_type.ConvertToString( self._used_data ) + os.linesep + 'which '+ ConvertTimestampToPrettyExpiry( self._expiry )
    
    def GetAccountIdentifier( self ): return AccountIdentifier( account_id = self._account_id )
    
    def GetAccountType( self ): return self._account_type
    
    def GetBannedInfo( self ): return self._banned_info
    
    def GetCreated( self ): return self._created
    
    def GetExpiry( self ): return self._expiry
    
    def GetExpiryString( self ):
        
        if self._IsBanned():
            
            ( reason, created, expiry ) = self._banned_info
            
            return 'banned ' + ConvertTimestampToPrettyAge( created ) + ', ' + ConvertTimestampToPrettyExpiry( expiry ) + ' because: ' + reason
            
        else: return ConvertTimestampToPrettyAge( self._created ) + ' and ' + ConvertTimestampToPrettyExpiry( self._expiry )
        
    
    def GetAccountId( self ): return self._account_id
    
    def GetUsedBytesString( self ):
        
        ( max_num_bytes, max_num_requests ) = self._account_type.GetMaxMonthlyData()
        ( used_bytes, used_requests ) = self._used_data
        
        if max_num_bytes is None: return ConvertIntToBytes( used_bytes ) + ' used this month'
        else: return ConvertIntToBytes( used_bytes ) + '/' + ConvertIntToBytes( max_num_bytes ) + ' used this month'
        
    
    def GetUsedRequestsString( self ):
        
        ( max_num_bytes, max_num_requests ) = self._account_type.GetMaxMonthlyData()
        ( used_bytes, used_requests ) = self._used_data
        
        if max_num_requests is None: return ConvertIntToPrettyString( used_requests ) + ' requests used this month'
        else: return ConvertIntToPrettyString( used_requests ) + '/' + ConvertIntToPrettyString( max_num_requests ) + ' requests used this month'
        
    
    def GetUsedData( self ): return self._used_data
    
    def HasPermission( self, permission ):
        
        if self._IsBanned(): return False
        
        if self._IsExpired(): return False
        
        ( max_num_bytes, max_num_requests ) = self._account_type.GetMaxMonthlyData()
        
        ( used_bytes, used_requests ) = self._used_data
        
        if max_num_bytes is not None and used_bytes >= max_num_bytes: return False
        if max_num_requests is not None and used_requests >= max_num_requests: return False
        
        return self._account_type.HasPermission( permission )
        
    
    def IsAdmin( self ): return True in [ self.HasPermissions( permission ) for permission in ADMIN_PERMISSIONS ]
    
    def IsBanned( self ): return self._IsBanned()
    
    def IsStale( self ): return self._object_instantiation_timestamp + UPDATE_DURATION * 5 < GetNow()
    
    def MakeFresh( self ): self._object_instantiation_timestamp = GetNow()
    
    def MakeStale( self ): self._object_instantiation_timestamp = 0
    
    def RequestMade( self, num_bytes ):
        
        ( used_bytes, used_requests ) = self._used_data
        
        used_bytes += num_bytes
        used_requests += 1
        
        self._used_data = ( used_bytes, used_requests )
        
    
class AccountIdentifier( HydrusYAMLBase ):
    
    yaml_tag = u'!AccountIdentifier'
    
    def __init__( self, access_key = None, hash = None, tag = None, account_id = None ):
        
        HydrusYAMLBase.__init__( self )
        
        self._access_key = access_key
        self._hash = hash
        self._tag = tag
        self._account_id = account_id
        
    
    def __eq__( self, other ): return self.__hash__() == other.__hash__()
    
    def __hash__( self ): return ( self._access_key, self._hash, self._tag, self._account_id ).__hash__()
    
    def __ne__( self, other ): return self.__hash__() != other.__hash__()
    
    def __repr__( self ): return 'Account Identifier: ' + u( ( self._access_key.encode( 'hex' ), self._hash.encode( 'hex' ), self._tag, self._account_id ) )
    
    def GetAccessKey( self ): return self._access_key
    
    def GetAccountId( self ): return self._account_id
    
    def GetHash( self ): return self._hash
    
    def GetMapping( self ): return ( self._tag, self._hash )
    
    def HasAccessKey( self ): return self._access_key is not None
    
    def HasHash( self ): return self._hash is not None and self._tag is None
    
    def HasAccountId( self ): return self._account_id is not None
    
    def HasMapping( self ): return self._tag is not None and self._hash is not None
    
class AccountType( HydrusYAMLBase ):
    
    yaml_tag = u'!AccountType'
    
    def __init__( self, title, permissions, max_monthly_data ):
        
        HydrusYAMLBase.__init__( self )
        
        self._title = title
        self._permissions = permissions
        self._max_monthly_data = max_monthly_data
        
    
    def __repr__( self ): return self.ConvertToString()
    
    def GetPermissions( self ): return self._permissions
    
    def GetTitle( self ): return self._title
    
    def GetMaxMonthlyData( self ): return self._max_monthly_data
    
    def GetMaxMonthlyDataString( self ):
        
        ( max_num_bytes, max_num_requests ) = self._max_monthly_data
        
        if max_num_bytes is None: max_num_bytes_string = 'No limit'
        else: max_num_bytes_string = ConvertIntToBytes( max_num_bytes )
        
        if max_num_requests is None: max_num_requests_string = 'No limit'
        else: max_num_requests_string = ConvertIntToPrettyString( max_num_requests )
        
        return ( max_num_bytes_string, max_num_requests_string )
        
    
    def ConvertToString( self, data_usage = None ):
        
        result_string = self._title + ' with '
        
        if len( self._permissions ) == 0: result_string += 'no permissions'
        else: result_string += ', '.join( [ permissions_string_lookup[ permission ] for permission in self._permissions ] ) + ' permissions'
        
        return result_string
        
    
    def HasPermission( self, permission ): return permission in self._permissions
    
UNKNOWN_ACCOUNT_TYPE = AccountType( 'unknown account', [], ( None, None ) )

def GetUnknownAccount(): return Account( 0, UNKNOWN_ACCOUNT_TYPE, 0, None, ( 0, 0 ) )

class ClientServiceIdentifier( HydrusYAMLBase ):
    
    yaml_tag = u'!ClientServiceIdentifier'
    
    def __init__( self, service_key, type, name ):
        
        HydrusYAMLBase.__init__( self )
        
        self._service_key = service_key
        self._type = type
        self._name = name
        
    
    def __eq__( self, other ): return self.__hash__() == other.__hash__()
    
    def __hash__( self ): return self._service_key.__hash__()
    
    def __ne__( self, other ): return self.__hash__() != other.__hash__()
    
    def __repr__( self ): return 'Client Service Identifier: ' + u( ( self._name, service_string_lookup[ self._type ] ) )
    
    def GetInfo( self ): return ( self._service_key, self._type, self._name )
    
    def GetName( self ): return self._name
    
    def GetServiceKey( self ): return self._service_key
    
    def GetType( self ): return self._type
    
LOCAL_FILE_SERVICE_IDENTIFIER = ClientServiceIdentifier( 'local files', LOCAL_FILE, 'local files' )
LOCAL_TAG_SERVICE_IDENTIFIER = ClientServiceIdentifier( 'local tags', LOCAL_TAG, 'local tags' )
COMBINED_FILE_SERVICE_IDENTIFIER = ClientServiceIdentifier( 'all known files', COMBINED_FILE, 'all known files' )
COMBINED_TAG_SERVICE_IDENTIFIER = ClientServiceIdentifier( 'all known tags', COMBINED_TAG, 'all known tags' )
NULL_SERVICE_IDENTIFIER = ClientServiceIdentifier( '', NULL_SERVICE, 'no service' )

class ClientToServerUpdate( HydrusYAMLBase ):
    
    yaml_tag = u'!ClientToServerUpdate'
    
    def __init__( self, content_data, hash_ids_to_hashes ):
        
        HydrusYAMLBase.__init__( self )
        
        # we need to flatten it from a collections.defaultdict to just a normal dict so it can be YAMLised
        
        self._content_data = {}
        
        for data_type in content_data:
            
            self._content_data[ data_type ] = {}
            
            for action in content_data[ data_type ]: self._content_data[ data_type ][ action ] = content_data[ data_type ][ action ]
            
        
        self._hash_ids_to_hashes = hash_ids_to_hashes
        
    
    def GetContentUpdates( self, for_client = False ):
        
        data_types = [ CONTENT_DATA_TYPE_FILES, CONTENT_DATA_TYPE_MAPPINGS, CONTENT_DATA_TYPE_TAG_SIBLINGS, CONTENT_DATA_TYPE_TAG_PARENTS ]
        actions = [ CONTENT_UPDATE_PENDING, CONTENT_UPDATE_PETITION, CONTENT_UPDATE_DENY_PEND, CONTENT_UPDATE_DENY_PETITION ]
        
        content_updates = []
        
        for ( data_type, action ) in itertools.product( data_types, actions ):
            
            munge_row = lambda row: row
            
            if for_client:
                
                if action == CONTENT_UPDATE_PENDING: new_action = CONTENT_UPDATE_ADD
                elif action == CONTENT_UPDATE_PETITION: new_action = CONTENT_UPDATE_DELETE
                else: continue
                
                if data_type in ( CONTENT_DATA_TYPE_TAG_SIBLINGS, CONTENT_DATA_TYPE_TAG_PARENTS ) and action in ( CONTENT_UPDATE_PENDING, CONTENT_UPDATE_PETITION ):
                    
                    munge_row = lambda ( pair, reason ): pair
                    
                elif data_type == CONTENT_DATA_TYPE_FILES and action == CONTENT_UPDATE_PETITION:
                    
                    munge_row = lambda ( hashes, reason ): hashes
                    
                elif data_type == CONTENT_DATA_TYPE_MAPPINGS and action == CONTENT_UPDATE_PETITION:
                    
                    munge_row = lambda ( tag, hashes, reason ): ( tag, hashes )
                    
                
            else: new_action = action
            
            content_updates.extend( [ ContentUpdate( data_type, new_action, munge_row( row ) ) for row in self.GetContentDataIterator( data_type, action ) ] )
            
        
        return content_updates
        
    
    def GetHashes( self ): return self._hash_ids_to_hashes.values()
    
    def GetContentDataIterator( self, data_type, action ):
        
        if data_type not in self._content_data or action not in self._content_data[ data_type ]: return ()
        
        data = self._content_data[ data_type ][ action ]
        
        if data_type == CONTENT_DATA_TYPE_FILES:
            
            if action == CONTENT_UPDATE_PETITION: return ( ( { self._hash_ids_to_hashes[ hash_id ] for hash_id in hash_ids }, reason ) for ( hash_ids, reason ) in data )
            else: return ( { self._hash_ids_to_hashes[ hash_id ] for hash_id in hash_ids } for hash_ids in ( data, ) )
            
        if data_type == CONTENT_DATA_TYPE_MAPPINGS:
            
            if action == CONTENT_UPDATE_PETITION: return ( ( tag, [ self._hash_ids_to_hashes[ hash_id ] for hash_id in hash_ids ], reason ) for ( tag, hash_ids, reason ) in data )
            else: return ( ( tag, [ self._hash_ids_to_hashes[ hash_id ] for hash_id in hash_ids ] ) for ( tag, hash_ids ) in data )
            
        else: return data.__iter__()
        
    
    def GetTags( self ):
        
        tags = set()
        
        try: tags.update( ( tag for ( tag, hash_ids ) in self._content_data[ CONTENT_DATA_TYPE_MAPPINGS ][ CONTENT_UPDATE_PENDING ] ) )
        except: pass
        
        try: tags.update( ( tag for ( tag, hash_ids, reason ) in self._content_data[ CONTENT_DATA_TYPE_MAPPINGS ][ CONTENT_UPDATE_PETITION ] ) )
        except: pass
        
        return tags
        
    
    def IsEmpty( self ):
        
        num_total = 0
        
        for data_type in self._content_data:
            
            for action in self._content_data[ data_type ]: num_total += len( self._content_data[ data_type ][ action ] )
            
        
        return num_total == 0
        
    
class ContentUpdate():
    
    def __init__( self, data_type, action, row ):
        
        self._data_type = data_type
        self._action = action
        self._row = row
        
    
    def __eq__( self, other ): return self._data_type == other._data_type and self._action == other._action and self._row == other._row
    
    def __ne__( self, other ): return not self.__eq__( other )
    
    def __repr__( self ): return 'Content Update: ' + u( ( self._data_type, self._action, self._row ) )
    
    def GetHashes( self ):
        
        if self._data_type == CONTENT_DATA_TYPE_FILES:
            
            if self._action == CONTENT_UPDATE_ADD:
                
                hash = self._row[0]
                
                hashes = set( ( hash, ) )
                
            elif self._action in ( CONTENT_UPDATE_ARCHIVE, CONTENT_UPDATE_DELETE, CONTENT_UPDATE_INBOX, CONTENT_UPDATE_PENDING, CONTENT_UPDATE_RESCIND_PENDING, CONTENT_UPDATE_RESCIND_PETITION ): hashes = self._row
            elif self._action == CONTENT_UPDATE_PETITION: ( hashes, reason ) = self._row
            
        elif self._data_type == CONTENT_DATA_TYPE_MAPPINGS:
            
            if self._action == CONTENT_UPDATE_PETITION: ( tag, hashes, reason ) = self._row
            else: ( tag, hashes ) = self._row
            
        elif self._data_type in ( CONTENT_DATA_TYPE_TAG_PARENTS, CONTENT_DATA_TYPE_TAG_SIBLINGS ): hashes = set()
        elif self._data_type == CONTENT_DATA_TYPE_RATINGS:
            
            if self._action == CONTENT_UPDATE_ADD: ( rating, hashes ) = self._row
            elif self._action == CONTENT_UPDATE_RATINGS_FILTER: ( min, max, hashes ) = self._row
            
        
        if type( hashes ) != set: hashes = set( hashes )
        
        return hashes
        
    
    def ToTuple( self ): return ( self._data_type, self._action, self._row )
    
class DAEMON( threading.Thread ):
    
    def __init__( self, name, callable, period = 1200 ):
        
        threading.Thread.__init__( self, name = name )
        
        self._name = name
        self._callable = callable
        self._period = period
        
        self._event = threading.Event()
        
        pubsub.sub( self, 'shutdown', 'shutdown' )
        
    
    def shutdown( self ): self._event.set()
    
class DAEMONQueue( DAEMON ):
    
    def __init__( self, name, callable, queue_topic, period = 10 ):
        
        DAEMON.__init__( self, name, callable, period )
        
        self._queue = Queue.Queue()
        self._queue_topic = queue_topic
        
        self.start()
        
        pubsub.sub( self, 'put', queue_topic )
        
    
    def put( self, data ): self._queue.put( data )
    
    def run( self ):
        
        time.sleep( 3 )
        
        while True:
            
            while self._queue.qsize() == 0:
                
                if shutdown: return
                
                self._event.wait( self._period )
                
                self._event.clear()
                
            
            items = []
            
            while self._queue.qsize() > 0: items.append( self._queue.get() )
            
            try: self._callable( items )
            except Exception as e: ShowException( e )
            
        
    
class DAEMONWorker( DAEMON ):
    
    def __init__( self, name, callable, topics = [], period = 1200, init_wait = 3 ):
        
        DAEMON.__init__( self, name, callable, period )
        
        self._topics = topics
        self._init_wait = init_wait
        
        self.start()
        
        for topic in topics: pubsub.sub( self, 'set', topic )
        
    
    def run( self ):
        
        self._event.wait( self._init_wait )
        
        while True:
            
            if shutdown: return
            
            try: self._callable()
            except Exception as e: ShowException( e )
            
            if shutdown: return
            
            self._event.wait( self._period )
            
            self._event.clear()
            
        
    
    def set( self, *args, **kwargs ): self._event.set()
    
class Job():
    
    yaml_tag = u'!Job'
    
    def __init__( self ):
        
        self._result = None
        self._result_ready = threading.Event()
        
    
    def GetResult( self ):
        
        while True:
            
            if self._result_ready.wait( 5 ) == True: break
            elif shutdown: raise Exception( 'Application quit before db could serve result!' )
            
        
        if issubclass( type( self._result ), Exception ):
            
            etype = type( self._result )
            
            db_traceback = unicode( self._result )
            
            trace_list = traceback.format_stack()
            
            my_trace = ''.join( trace_list )
            
            raise etype( my_trace + os.linesep + db_traceback )
            
        else: return self._result
        
    
    def PutResult( self, result ):
        
        self._result = result
        
        self._result_ready.set()
        
    
class JobInternal( Job ):
    
    yaml_tag = u'!JobInternal'
    
    def __init__( self, action, type, synchronous, *args, **kwargs ):
        
        Job.__init__( self )
        
        self._action = action
        self._type = type
        self._synchronous = synchronous
        self._args = args
        self._kwargs = kwargs
        
    
    def GetAction( self ): return self._action
    
    def GetArgs( self ): return self._args
    
    def GetKWArgs( self ): return self._kwargs
    
    def GetType( self ): return self._type
    
    def IsSynchronous( self ): return self._synchronous
    
class JobKey():
    
    def __init__( self ):
        
        self._key = os.urandom( 32 )
        
        self._cancelled = threading.Event()
        
    
    def __eq__( self, other ): return self.__hash__() == other.__hash__()
    
    def __hash__( self ): return self._key.__hash__()
    
    def __ne__( self, other ): return self.__hash__() != other.__hash__()
    
    def Cancel( self ): self._cancelled.set()
    
    def GetKey( self ): return self._key
    
    def IsCancelled( self ): return self._cancelled.is_set()
    
class JobServer():
    
    yaml_tag = u'!JobServer'
    
    def __init__( self, *args ):
        
        self._args = args
        
        self._result = None
        self._result_ready = threading.Event()
        
    
    def GetArgs( self ): return self._args
    
    def GetResult( self ):
        
        while True:
            
            if self._result_ready.wait( 5 ) == True: break
            elif shutdown: raise Exception( 'Application quit before db could serve result!' )
            
        
        if issubclass( type( self._result ), Exception ): raise self._result
        else: return self._result
        
    
    def PutResult( self, result ):
        
        self._result = result
        
        self._result_ready.set()
        
    
class Message():
    
    def __init__( self, message_type, info ):
        
        self._message_type = message_type
        self._info = info
        
    
    def GetInfo( self ): return self._info
    
    def GetType( self ): return self._message_type
    
class Predicate( HydrusYAMLBase ):
    
    yaml_tag = u'!Predicate'
    
    def __init__( self, predicate_type, value, count ):
        
        self._predicate_type = predicate_type
        self._value = value
        self._count = count
        
    
    def __eq__( self, other ): return self.__hash__() == other.__hash__()
    
    def __hash__( self ): return ( self._predicate_type, self._value ).__hash__()
    
    def __ne__( self, other ): return self.__hash__() != other.__hash__()
    
    def __repr__( self ): return 'Predicate: ' + u( ( self._predicate_type, self._value, self._count ) )
    
    def AddToCount( self, count ): self._count += count
    
    def GetCopy( self ): return Predicate( self._predicate_type, self._value, self._count )
    
    def GetCountlessCopy( self ): return Predicate( self._predicate_type, self._value, None )
    
    def GetCount( self ): return self._count
    
    def GetInfo( self ): return ( self._predicate_type, self._value )
    
    def GetPredicateType( self ): return self._predicate_type
    
    def GetUnicode( self, with_count = True ):
        
        if self._predicate_type == PREDICATE_TYPE_SYSTEM:
            
            ( system_predicate_type, info ) = self._value
            
            if system_predicate_type == SYSTEM_PREDICATE_TYPE_EVERYTHING: base = u'system:everything'
            elif system_predicate_type == SYSTEM_PREDICATE_TYPE_INBOX: base = u'system:inbox'
            elif system_predicate_type == SYSTEM_PREDICATE_TYPE_ARCHIVE: base = u'system:archive'
            elif system_predicate_type == SYSTEM_PREDICATE_TYPE_UNTAGGED: base = u'system:untagged'
            elif system_predicate_type == SYSTEM_PREDICATE_TYPE_LOCAL: base = u'system:local'
            elif system_predicate_type == SYSTEM_PREDICATE_TYPE_NOT_LOCAL: base = u'system:not local'
            elif system_predicate_type in ( SYSTEM_PREDICATE_TYPE_NUM_TAGS, SYSTEM_PREDICATE_TYPE_WIDTH, SYSTEM_PREDICATE_TYPE_HEIGHT, SYSTEM_PREDICATE_TYPE_DURATION, SYSTEM_PREDICATE_TYPE_NUM_WORDS ):
                
                if system_predicate_type == SYSTEM_PREDICATE_TYPE_NUM_TAGS: base = u'system:number of tags'
                elif system_predicate_type == SYSTEM_PREDICATE_TYPE_WIDTH: base = u'system:width'
                elif system_predicate_type == SYSTEM_PREDICATE_TYPE_HEIGHT: base = u'system:height'
                elif system_predicate_type == SYSTEM_PREDICATE_TYPE_DURATION: base = u'system:duration'
                elif system_predicate_type == SYSTEM_PREDICATE_TYPE_NUM_WORDS: base = u'system:number of words'
                
                if info is not None:
                    
                    ( operator, value ) = info
                    
                    base += u' ' + operator + u' ' + u( value )
                    
                
            elif system_predicate_type == SYSTEM_PREDICATE_TYPE_RATIO:
                
                base = u'system:ratio'
                
                if info is not None:
                    
                    ( operator, ratio_width, ratio_height ) = info
                    
                    base += u' ' + operator + u' ' + u( ratio_width ) + u':' + u( ratio_height )
                    
                
            elif system_predicate_type == SYSTEM_PREDICATE_TYPE_SIZE:
                
                base = u'system:size'
                
                if info is not None:
                    
                    ( operator, size, unit ) = info
                    
                    base += u' ' + operator + u' ' + u( size ) + ConvertUnitToString( unit )
                    
                
            elif system_predicate_type == SYSTEM_PREDICATE_TYPE_LIMIT:
                
                base = u'system:limit'
                
                if info is not None:
                    
                    value = info
                    
                    base += u' is ' + u( value )
                    
                
            elif system_predicate_type == SYSTEM_PREDICATE_TYPE_AGE:
                
                base = u'system:age'
                
                if info is not None:
                    
                    ( operator, years, months, days, hours ) = info
                    
                    base += u' ' + operator + u' ' + u( years ) + u'y' + u( months ) + u'm' + u( days ) + u'd' + u( hours ) + u'h'
                    
                
            elif system_predicate_type == SYSTEM_PREDICATE_TYPE_HASH:
                
                base = u'system:hash'
                
                if info is not None:
                    
                    hash = info
                    
                    base += u' is ' + hash.encode( 'hex' )
                    
                
            elif system_predicate_type == SYSTEM_PREDICATE_TYPE_MIME:
                
                base = u'system:mime'
                
                if info is not None:
                    
                    mime = info
                    
                    base += u' is ' + mime_string_lookup[ mime ]
                    
                
            elif system_predicate_type == SYSTEM_PREDICATE_TYPE_RATING:
                
                base = u'system:rating'
                
                if info is not None:
                    
                    ( service_identifier, operator, value ) = info
                    
                    base += u' for ' + service_identifier.GetName() + u' ' + operator + u' ' + u( value )
                    
                
            elif system_predicate_type == SYSTEM_PREDICATE_TYPE_SIMILAR_TO:
                
                base = u'system:similar to '
                
                if info is not None:
                    
                    ( hash, max_hamming ) = info
                    
                    base += u' ' + hash.encode( 'hex' ) + u' using max hamming of ' + u( max_hamming )
                    
                
            elif system_predicate_type == SYSTEM_PREDICATE_TYPE_FILE_SERVICE:
                
                base = u'system:file service'
                
                if info is not None:
                    
                    ( operator, type, service_identifier ) = info
                    
                    base += u':'
                    
                    if operator == True: base += u' is'
                    else: base += u' is not'
                    
                    if type == PENDING: base += u' pending to '
                    else: base += u' currently in '
                    
                    base += service_identifier.GetName()
                    
                
            
            if with_count and self._count is not None: base += u' (' + ConvertIntToPrettyString( self._count ) + u')'
            
        elif self._predicate_type == PREDICATE_TYPE_TAG:
            
            ( operator, tag ) = self._value
            
            if operator == '-': base = u'-'
            elif operator == '+': base = u''
            
            base += tag
            
            if self._count is not None: base += u' (' + ConvertIntToPrettyString( self._count ) + u')'
            
            siblings_manager = app.GetManager( 'tag_siblings' )
            
            sibling = siblings_manager.GetSibling( tag )
            
            if sibling is not None: base += u' (' + sibling + u')'
            
        elif self._predicate_type == PREDICATE_TYPE_PARENT:
            
            base = '    '
            
            tag = self._value
            
            base += tag
            
            if with_count and self._count is not None: base += u' (' + ConvertIntToPrettyString( self._count ) + u')'
            
        elif self._predicate_type == PREDICATE_TYPE_NAMESPACE:
            
            ( operator, namespace ) = self._value
            
            if operator == '-': base = u'-'
            elif operator == '+': base = u''
            
            base += namespace + u':*'
            
        
        return base
        
    
    def GetTag( self ):
        
        if self._predicate_type == PREDICATE_TYPE_TAG:
            
            ( operator, tag ) = self._value
            
            return tag
            
        else: return 'no tag'
        
    
    def GetValue( self ): return self._value
    
    def SetOperator( self, operator ):
        
        if self._predicate_type == PREDICATE_TYPE_TAG:
            
            ( old_operator, tag ) = self._value
            
            self._value = ( operator, tag )
            
        
    
SYSTEM_PREDICATE_INBOX = Predicate( PREDICATE_TYPE_SYSTEM, ( SYSTEM_PREDICATE_TYPE_INBOX, None ), None )
SYSTEM_PREDICATE_ARCHIVE = Predicate( PREDICATE_TYPE_SYSTEM, ( SYSTEM_PREDICATE_TYPE_ARCHIVE, None ), None )
SYSTEM_PREDICATE_LOCAL = Predicate( PREDICATE_TYPE_SYSTEM, ( SYSTEM_PREDICATE_TYPE_LOCAL, None ), None )
SYSTEM_PREDICATE_NOT_LOCAL = Predicate( PREDICATE_TYPE_SYSTEM, ( SYSTEM_PREDICATE_TYPE_NOT_LOCAL, None ), None )

class ResponseContext():
    
    def __init__( self, status_code, mime = APPLICATION_YAML, body = None, path = None, is_yaml = False, cookies = [] ):
        
        self._status_code = status_code
        self._mime = mime
        self._body = body
        self._path = path
        self._is_yaml = is_yaml
        self._cookies = cookies
        
    
    def GetCookies( self ): return self._cookies
    
    def GetLength( self ): return len( self._body )
    
    def GetMimeBody( self ): return ( self._mime, self._body )
    
    def GetPath( self ): return self._path
    
    def GetStatusCode( self ): return self._status_code
    
    def HasBody( self ): return self._body is not None
    
    def HasPath( self ): return self._path is not None
    
    def IsYAML( self ): return self._is_yaml
    
class ServerServiceIdentifier( HydrusYAMLBase ):
    
    yaml_tag = u'!ServerServiceIdentifier'
    
    def __init__( self, service_key, type ):
        
        HydrusYAMLBase.__init__( self )
        
        self._service_key = service_key
        self._type = type
        
    
    def __eq__( self, other ): return self.__hash__() == other.__hash__()
    
    def __hash__( self ): return self._service_key.__hash__()
    
    def __ne__( self, other ): return self.__hash__() != other.__hash__()
    
    def __repr__( self ): return 'Server Service Identifier: ' + service_string_lookup[ self._type ]
    
    def GetServiceKey( self ): return self._service_key
    
    def GetType( self ): return self._type
    
SERVER_ADMIN_IDENTIFIER = ServerServiceIdentifier( 'server admin', SERVER_ADMIN )

class ServiceUpdate():
    
    def __init__( self, action, row = None ):
        
        self._action = action
        self._row = row
        
    
    def ToTuple( self ): return ( self._action, self._row )
        
class ServerToClientPetition( HydrusYAMLBase ):
    
    yaml_tag = u'!ServerToClientPetition'
    
    def __init__( self, petition_type, action, petitioner_account_identifier, petition_data, reason ):
        
        HydrusYAMLBase.__init__( self )
        
        self._petition_type = petition_type
        self._action = action
        self._petitioner_account_identifier = petitioner_account_identifier
        self._petition_data = petition_data
        self._reason = reason
        
    
    def GetApproval( self, reason = None ):
        
        if reason is None: reason = self._reason
        
        if self._petition_type == CONTENT_DATA_TYPE_FILES: hashes = self._petition_data
        elif self._petition_type == CONTENT_DATA_TYPE_MAPPINGS: ( tag, hashes ) = self._petition_data
        else: hashes = set()
        
        hash_ids_to_hashes = dict( enumerate( hashes ) )
        
        hash_ids = hash_ids_to_hashes.keys()
        
        content_data = GetEmptyDataDict()
        
        if self._petition_type == CONTENT_DATA_TYPE_FILES: row = ( hash_ids, reason )
        elif self._petition_type == CONTENT_DATA_TYPE_MAPPINGS: row = ( tag, hash_ids, reason )
        else:
            
            ( old, new ) = self._petition_data
            
            row = ( ( old, new ), reason )
            
        
        content_data[ self._petition_type ][ self._action ].append( row )
        
        return ClientToServerUpdate( content_data, hash_ids_to_hashes )
        
    
    def GetDenial( self ):
        
        if self._petition_type == CONTENT_DATA_TYPE_FILES: hashes = self._petition_data
        elif self._petition_type == CONTENT_DATA_TYPE_MAPPINGS: ( tag, hashes ) = self._petition_data
        else: hashes = set()
        
        hash_ids_to_hashes = dict( enumerate( hashes ) )
        
        hash_ids = hash_ids_to_hashes.keys()
        
        if self._petition_type == CONTENT_DATA_TYPE_FILES: row_list = hash_ids
        elif self._petition_type == CONTENT_DATA_TYPE_MAPPINGS: row_list = [ ( tag, hash_ids ) ]
        else: row_list = [ self._petition_data ]
        
        content_data = GetEmptyDataDict()
        
        if self._action == CONTENT_UPDATE_PENDING: denial_action = CONTENT_UPDATE_DENY_PEND
        elif self._action == CONTENT_UPDATE_PETITION: denial_action = CONTENT_UPDATE_DENY_PETITION
        
        content_data[ self._petition_type ][ denial_action ] = row_list
        
        return ClientToServerUpdate( content_data, hash_ids_to_hashes )
        
    
    def GetHashes( self ):
        
        if self._petition_type == CONTENT_DATA_TYPE_FILES: hashes = self._petition_data
        elif self._petition_type == CONTENT_DATA_TYPE_MAPPINGS: ( tag, hashes ) = self._petition_data
        else: hashes = set()
        
        return hashes
        
    
    def GetPetitioner( self ): return self._petitioner_account_identifier
    
    def GetPetitionString( self ):
        
        if self._action == CONTENT_UPDATE_PENDING: action_word = 'Add '
        elif self._action == CONTENT_UPDATE_PETITION: action_word = 'Remove '
        
        if self._petition_type == CONTENT_DATA_TYPE_FILES:
            
            hashes = self._petition_data
            
            content_phrase = ConvertIntToPrettyString( len( hashes ) ) + ' files'
            
        elif self._petition_type == CONTENT_DATA_TYPE_MAPPINGS:
            
            ( tag, hashes ) = self._petition_data
            
            content_phrase = tag + ' for ' + ConvertIntToPrettyString( len( hashes ) ) + ' files'
            
        elif self._petition_type == CONTENT_DATA_TYPE_TAG_SIBLINGS:
            
            ( old_tag, new_tag ) = self._petition_data
            
            content_phrase = ' sibling ' + old_tag + '->' + new_tag
            
        elif self._petition_type == CONTENT_DATA_TYPE_TAG_PARENTS:
            
            ( old_tag, new_tag ) = self._petition_data
            
            content_phrase = ' parent ' + old_tag + '->' + new_tag
            
        
        return action_word + content_phrase + os.linesep + os.linesep + self._reason
        
    
class ServerToClientUpdate( HydrusYAMLBase ):

    yaml_tag = u'!ServerToClientUpdate'
    
    def __init__( self, service_data, content_data, hash_ids_to_hashes ):
        
        HydrusYAMLBase.__init__( self )
        
        self._service_data = service_data
        
        self._content_data = {}
        
        for data_type in content_data:
            
            self._content_data[ data_type ] = {}
            
            for action in content_data[ data_type ]: self._content_data[ data_type ][ action ] = content_data[ data_type ][ action ]
            
        
        self._hash_ids_to_hashes = hash_ids_to_hashes
        
    
    def _GetContentData( data_type, action ):
        
        if data_type in self._content_data and action in self._content_data[ data_type ]: return self._content_data[ data_type ][ action ]
        else: return []
        
    
    def GetContentDataIterator( self, data_type, action ):
        
        if data_type not in self._content_data or action not in self._content_data[ data_type ]: return ()
        
        data = self._content_data[ data_type ][ action ]
        
        if data_type == CONTENT_DATA_TYPE_FILES:
            
            if action == CONTENT_UPDATE_ADD: return ( ( self._hash_ids_to_hashes[ hash_id ], size, mime, timestamp, width, height, duration, num_frames, num_words ) for ( hash_id, size, mime, timestamp, width, height, duration, num_frames, num_words ) in data )
            else: return ( { self._hash_ids_to_hashes[ hash_id ] for hash_id in hash_ids } for hash_ids in ( data, ) )
            
        elif data_type == CONTENT_DATA_TYPE_MAPPINGS: return ( ( tag, [ self._hash_ids_to_hashes[ hash_id ] for hash_id in hash_ids ] ) for ( tag, hash_ids ) in data )
        else: return data.__iter__()
        
    
    def GetNumContentUpdates( self ):
        
        num = 0
        
        data_types = [ CONTENT_DATA_TYPE_FILES, CONTENT_DATA_TYPE_MAPPINGS, CONTENT_DATA_TYPE_TAG_SIBLINGS, CONTENT_DATA_TYPE_TAG_PARENTS ]
        actions = [ CONTENT_UPDATE_ADD, CONTENT_UPDATE_DELETE ]
        
        for ( data_type, action ) in itertools.product( data_types, actions ):
            
            for row in self.GetContentDataIterator( data_type, action ): num += 1
            
        
        return num
        
    
    # this needs work!
    # we need a universal content_update for both client and server
    def IterateContentUpdates( self ):
        
        data_types = [ CONTENT_DATA_TYPE_FILES, CONTENT_DATA_TYPE_MAPPINGS, CONTENT_DATA_TYPE_TAG_SIBLINGS, CONTENT_DATA_TYPE_TAG_PARENTS ]
        actions = [ CONTENT_UPDATE_ADD, CONTENT_UPDATE_DELETE ]
        
        for ( data_type, action ) in itertools.product( data_types, actions ):
            
            for row in self.GetContentDataIterator( data_type, action ): yield ContentUpdate( data_type, action, row )
            
        
    
    def IterateServiceUpdates( self ):
        
        service_types = [ SERVICE_UPDATE_NEWS, SERVICE_UPDATE_NEXT_BEGIN ]
        
        for service_type in service_types:
            
            if service_type in self._service_data:
                
                data = self._service_data[ service_type ]
                
                yield ServiceUpdate( service_type, data )
                
            
        
    
    def GetHashes( self ): return set( hash_ids_to_hashes.values() )
    
    def GetTags( self ):
        
        tags = set()
        
        tags.update( ( tag for ( tag, hash_ids ) in self.GetContentDataIterator( CONTENT_DATA_TYPE_MAPPINGS, CURRENT ) ) )
        tags.update( ( tag for ( tag, hash_ids ) in self.GetContentDataIterator( CONTENT_DATA_TYPE_MAPPINGS, DELETED ) ) )
        
        tags.update( ( old_tag for ( old_tag, new_tag ) in self.GetContentDataIterator( CONTENT_DATA_TYPE_TAG_SIBLINGS, CURRENT ) ) )
        tags.update( ( new_tag for ( old_tag, new_tag ) in self.GetContentDataIterator( CONTENT_DATA_TYPE_TAG_SIBLINGS, CURRENT ) ) )
        tags.update( ( old_tag for ( old_tag, new_tag ) in self.GetContentDataIterator( CONTENT_DATA_TYPE_TAG_SIBLINGS, DELETED ) ) )
        tags.update( ( new_tag for ( old_tag, new_tag ) in self.GetContentDataIterator( CONTENT_DATA_TYPE_TAG_SIBLINGS, DELETED ) ) )
        
        return tags
        
    
    def GetServiceData( self, service_type ): return self._service_data[ service_type ]
    
class SortedList():
    
    def __init__( self, initial_items = [], sort_function = None ):
        
        do_sort = sort_function is not None
        
        if sort_function is None: sort_function = lambda x: x
        
        self._sorted_list = [ ( sort_function( item ), item ) for item in initial_items ]
        
        self._items_to_indices = None
        
        self._sort_function = sort_function
        
        if do_sort: self.sort()
        
    
    def __contains__( self, item ): return self._items_to_indices.__contains__( item )
    
    def __getitem__( self, value ):
        
        if type( value ) == int: return self._sorted_list.__getitem__( value )[1]
        elif type( value ) == slice: return [ item for ( sort_item, item ) in self._sorted_list.__getitem__( value ) ]
        
    
    def __iter__( self ):
        
        for ( sorting_value, item ) in self._sorted_list: yield item
        
    
    def __len__( self ): return self._sorted_list.__len__()
    
    def _DirtyIndices( self ): self._items_to_indices = None
    
    def _RecalcIndices( self ): self._items_to_indices = { item : index for ( index, ( sort_item, item ) ) in enumerate( self._sorted_list ) }
    
    def append_items( self, items ):
        
        self._sorted_list.extend( [ ( self._sort_function( item ), item ) for item in items ] )
        
        self._DirtyIndices()
        
    
    def index( self, item ):
        
        if self._items_to_indices is None: self._RecalcIndices()
        
        return self._items_to_indices[ item ]
        
    
    def insert_items( self, items ):
        
        for item in items: bisect.insort( self._sorted_list, ( self._sort_function( item ), item ) )
        
        self._DirtyIndices()
        
    
    def remove_items( self, items ):
        
        deletee_indices = [ self.index( item ) for item in items ]
        
        deletee_indices.sort()
        
        deletee_indices.reverse()
        
        for index in deletee_indices: self._sorted_list.pop( index )
        
        self._DirtyIndices()
        
    
    def sort( self, f = None ):
        
        if f is not None: self._sort_function = f
        
        self._sorted_list = [ ( self._sort_function( item ), item ) for ( old_value, item ) in self._sorted_list ]
        
        self._sorted_list.sort()
        
        self._DirtyIndices()
        

# adding tuple to yaml

def construct_python_tuple( self, node ): return tuple( self.construct_sequence( node ) )
def represent_python_tuple( self, data ): return self.represent_sequence( u'tag:yaml.org,2002:python/tuple', data )

yaml.SafeLoader.add_constructor( u'tag:yaml.org,2002:python/tuple', construct_python_tuple )
yaml.SafeDumper.add_representer( tuple, represent_python_tuple)

# sqlite mod

sqlite3.register_adapter( dict, yaml.safe_dump )
sqlite3.register_adapter( list, yaml.safe_dump )
sqlite3.register_adapter( Account, yaml.safe_dump )
sqlite3.register_adapter( AccountType, yaml.safe_dump )
sqlite3.register_converter( 'TEXT_YAML', yaml.safe_load )

sqlite3.register_converter( 'BLOB_BYTES', str )

# for some reason, sqlite doesn't parse to int before this, despite the column affinity
# it gives the register_converter function a bytestring :/
def integer_boolean_to_bool( integer_boolean ): return bool( int( integer_boolean ) )

sqlite3.register_adapter( bool, int )
sqlite3.register_converter( 'INTEGER_BOOLEAN', integer_boolean_to_bool )

# no converters in this case, since we always want to send the dumped string, not the object, to the network
sqlite3.register_adapter( ServerToClientUpdate, yaml.safe_dump )