2013-02-19 00:11:43 +00:00
import collections
import hashlib
import httplib
import HydrusConstants as HC
2015-04-22 22:57:25 +00:00
import HydrusDB
2013-07-24 20:26:00 +00:00
import HydrusExceptions
2015-11-25 22:00:57 +00:00
import HydrusFileHandling
2013-11-06 18:22:07 +00:00
import HydrusNATPunch
2015-11-04 22:30:28 +00:00
import HydrusPaths
2015-06-03 21:05:13 +00:00
import HydrusSerialisable
2013-05-01 17:21:53 +00:00
import itertools
2015-06-24 22:10:14 +00:00
import json
import lz4
2013-02-19 00:11:43 +00:00
import os
import Queue
2013-06-12 22:53:31 +00:00
import random
2015-03-18 21:46:29 +00:00
import ServerFiles
2013-02-19 00:11:43 +00:00
import shutil
import sqlite3
import sys
import threading
import time
import traceback
import yaml
2015-03-25 22:04:19 +00:00
import HydrusData
import HydrusTags
import HydrusGlobals
2015-11-11 21:20:41 +00:00
'''
2015-03-04 22:44:32 +00:00
class MessageDB ( object ) :
def _AddMessage ( self , contact_key , message ) :
try : ( service_id , account_id ) = self . _c . execute ( ' SELECT service_id, account_id FROM contacts WHERE contact_key = ?; ' , ( sqlite3 . Binary ( contact_key ) , ) ) . fetchone ( )
except : raise HydrusExceptions . ForbiddenException ( ' Did not find that contact key for the message depot! ' )
2015-07-01 22:02:07 +00:00
message_key = HydrusData . GenerateKey ( )
2015-03-04 22:44:32 +00:00
2015-03-25 22:04:19 +00:00
self . _c . execute ( ' INSERT OR IGNORE INTO messages ( message_key, service_id, account_id, timestamp ) VALUES ( ?, ?, ?, ? ); ' , ( sqlite3 . Binary ( message_key ) , service_id , account_id , HydrusData . GetNow ( ) ) )
2015-03-04 22:44:32 +00:00
2015-03-18 21:46:29 +00:00
dest_path = ServerFiles . GetExpectedPath ( ' message ' , message_key )
2015-03-04 22:44:32 +00:00
with open ( dest_path , ' wb ' ) as f : f . write ( message )
def _AddStatuses ( self , contact_key , statuses ) :
try : ( service_id , account_id ) = self . _c . execute ( ' SELECT service_id, account_id FROM contacts WHERE contact_key = ?; ' , ( sqlite3 . Binary ( contact_key ) , ) ) . fetchone ( )
except : raise HydrusExceptions . ForbiddenException ( ' Did not find that contact key for the message depot! ' )
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2015-03-04 22:44:32 +00:00
self . _c . executemany ( ' INSERT OR REPLACE INTO message_statuses ( status_key, service_id, account_id, status, timestamp ) VALUES ( ?, ?, ?, ?, ? ); ' , [ ( sqlite3 . Binary ( status_key ) , service_id , account_id , sqlite3 . Binary ( status ) , now ) for ( status_key , status ) in statuses ] )
def _CreateContact ( self , service_id , account_id , public_key ) :
result = self . _c . execute ( ' SELECT public_key FROM contacts WHERE service_id = ? AND account_id = ?; ' , ( service_id , account_id ) ) . fetchone ( )
if result is not None :
( existing_public_key , ) = result
if existing_public_key != public_key : raise HydrusExceptions . ForbiddenException ( ' This account already has a public key! ' )
else : return
contact_key = hashlib . sha256 ( public_key ) . digest ( )
self . _c . execute ( ' INSERT INTO contacts ( service_id, account_id, contact_key, public_key ) VALUES ( ?, ?, ?, ? ); ' , ( service_id , account_id , sqlite3 . Binary ( contact_key ) , public_key ) )
def _GetMessage ( self , service_id , account_id , message_key ) :
result = self . _c . execute ( ' SELECT 1 FROM messages WHERE service_id = ? AND account_id = ? AND message_key = ?; ' , ( service_id , account_id , sqlite3 . Binary ( message_key ) ) ) . fetchone ( )
if result is None : raise HydrusExceptions . ForbiddenException ( ' Could not find that message key on message depot! ' )
2015-03-18 21:46:29 +00:00
path = ServerFiles . GetPath ( ' message ' , message_key )
2015-03-04 22:44:32 +00:00
with open ( path , ' rb ' ) as f : message = f . read ( )
return message
def _GetMessageInfoSince ( self , service_id , account_id , timestamp ) :
message_keys = [ message_key for ( message_key , ) in self . _c . execute ( ' SELECT message_key FROM messages WHERE service_id = ? AND account_id = ? AND timestamp > ? ORDER BY timestamp ASC; ' , ( service_id , account_id , timestamp ) ) ]
statuses = [ status for ( status , ) in self . _c . execute ( ' SELECT status FROM message_statuses WHERE service_id = ? AND account_id = ? AND timestamp > ? ORDER BY timestamp ASC; ' , ( service_id , account_id , timestamp ) ) ]
return ( message_keys , statuses )
def _GetPublicKey ( self , contact_key ) :
( public_key , ) = self . _c . execute ( ' SELECT public_key FROM contacts WHERE contact_key = ?; ' , ( sqlite3 . Binary ( contact_key ) , ) ) . fetchone ( )
return public_key
2015-11-11 21:20:41 +00:00
'''
2015-04-22 22:57:25 +00:00
class DB ( HydrusDB . HydrusDB ) :
2016-04-20 20:42:21 +00:00
READ_WRITE_ACTIONS = [ ' access_key ' , ' immediate_content_update ' , ' init ' , ' registration_keys ' ]
2015-03-04 22:44:32 +00:00
def _AccountTypeExists ( self , service_id , title ) : return self . _c . execute ( ' SELECT 1 FROM account_types WHERE service_id = ? AND title = ?; ' , ( service_id , title ) ) . fetchone ( ) is not None
2013-05-01 17:21:53 +00:00
2014-11-20 01:48:04 +00:00
def _AddFile ( self , service_key , account_key , file_dict ) :
2013-05-01 17:21:53 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2013-05-01 17:21:53 +00:00
2014-11-20 01:48:04 +00:00
account_id = self . _GetAccountId ( account_key )
2013-02-19 00:11:43 +00:00
hash = file_dict [ ' hash ' ]
2014-11-20 01:48:04 +00:00
hash_id = self . _GetHashId ( hash )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
if self . _c . execute ( ' SELECT 1 FROM file_map WHERE service_id = ? AND hash_id = ?; ' , ( service_id , hash_id ) ) . fetchone ( ) is None or self . _c . execute ( ' SELECT 1 FROM file_petitions WHERE service_id = ? AND hash_id = ? AND status = ?; ' , ( service_id , hash_id , HC . DELETED ) ) . fetchone ( ) is None :
2013-02-19 00:11:43 +00:00
size = file_dict [ ' size ' ]
mime = file_dict [ ' mime ' ]
if ' width ' in file_dict : width = file_dict [ ' width ' ]
else : width = None
if ' height ' in file_dict : height = file_dict [ ' height ' ]
else : height = None
if ' duration ' in file_dict : duration = file_dict [ ' duration ' ]
else : duration = None
if ' num_frames ' in file_dict : num_frames = file_dict [ ' num_frames ' ]
else : num_frames = None
if ' num_words ' in file_dict : num_words = file_dict [ ' num_words ' ]
else : num_words = None
2014-11-20 01:48:04 +00:00
options = self . _GetOptions ( service_key )
2013-02-19 00:11:43 +00:00
max_storage = options [ ' max_storage ' ]
if max_storage is not None :
# this is wrong! no service_id in files_info. need to cross with file_map or w/e
2014-11-20 01:48:04 +00:00
( current_storage , ) = self . _c . execute ( ' SELECT SUM( size ) FROM file_map, files_info USING ( hash_id ) WHERE service_id = ?; ' , ( service_id , ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
2013-07-24 20:26:00 +00:00
if current_storage + size > max_storage : raise HydrusExceptions . ForbiddenException ( ' The service is full! It cannot take any more files! ' )
2013-02-19 00:11:43 +00:00
2013-08-07 22:25:18 +00:00
source_path = file_dict [ ' path ' ]
2015-03-18 21:46:29 +00:00
dest_path = ServerFiles . GetExpectedPath ( ' file ' , hash )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
with open ( source_path , ' rb ' ) as f_source :
with open ( dest_path , ' wb ' ) as f_dest :
2015-11-04 22:30:28 +00:00
HydrusPaths . CopyFileLikeToFileLike ( f_source , f_dest )
2015-03-25 22:04:19 +00:00
2013-02-19 00:11:43 +00:00
if ' thumbnail ' in file_dict :
2015-03-18 21:46:29 +00:00
thumbnail_dest_path = ServerFiles . GetExpectedPath ( ' thumbnail ' , hash )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
thumbnail = file_dict [ ' thumbnail ' ]
2015-11-18 22:44:07 +00:00
with open ( thumbnail_dest_path , ' wb ' ) as f :
f . write ( thumbnail )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
if self . _c . execute ( ' SELECT 1 FROM files_info WHERE hash_id = ?; ' , ( hash_id , ) ) . fetchone ( ) is None :
2013-07-17 20:56:13 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' INSERT OR IGNORE INTO files_info ( hash_id, size, mime, width, height, duration, num_frames, num_words ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ? ); ' , ( hash_id , size , mime , width , height , duration , num_frames , num_words ) )
2013-07-17 20:56:13 +00:00
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' INSERT OR IGNORE INTO file_map ( service_id, hash_id, account_id, timestamp ) VALUES ( ?, ?, ?, ? ); ' , ( service_id , hash_id , account_id , now ) )
2013-02-19 00:11:43 +00:00
if options [ ' log_uploader_ips ' ] :
ip = file_dict [ ' ip ' ]
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' INSERT INTO ip_addresses ( service_id, hash_id, ip, timestamp ) VALUES ( ?, ?, ?, ? ); ' , ( service_id , hash_id , ip , now ) )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
def _AddFilePetition ( self , service_id , account_id , hash_ids , reason_id ) :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _ApproveFilePetitionOptimised ( service_id , account_id , hash_ids )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
valid_hash_ids = [ hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT hash_id FROM file_map WHERE service_id = ? AND hash_id IN ' + HydrusData . SplayListForDB ( hash_ids ) + ' ; ' , ( service_id , ) ) ]
2013-02-19 00:11:43 +00:00
# this clears out any old reasons, if the user wants to overwrite them
2015-03-25 22:04:19 +00:00
self . _c . execute ( ' DELETE FROM file_petitions WHERE service_id = ? AND account_id = ? AND hash_id IN ' + HydrusData . SplayListForDB ( valid_hash_ids ) + ' AND status = ?; ' , ( service_id , account_id , HC . PETITIONED ) )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-06-12 22:53:31 +00:00
2014-11-20 01:48:04 +00:00
self . _c . executemany ( ' INSERT OR IGNORE INTO file_petitions ( service_id, account_id, hash_id, reason_id, timestamp, status ) VALUES ( ?, ?, ?, ?, ?, ? ); ' , [ ( service_id , account_id , hash_id , reason_id , now , HC . PETITIONED ) for hash_id in valid_hash_ids ] )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _AddNews ( self , service_key , news ) :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
service_id = self . _GetServiceId ( service_key )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' INSERT INTO news ( service_id, news, timestamp ) VALUES ( ?, ?, ? ); ' , ( service_id , news , now ) )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _AddMappings ( self , service_id , account_id , tag_id , hash_ids , overwrite_deleted ) :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
if overwrite_deleted :
2015-03-25 22:04:19 +00:00
splayed_hash_ids = HydrusData . SplayListForDB ( hash_ids )
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' DELETE FROM mapping_petitions WHERE service_id = ? AND tag_id = ? AND hash_id IN ' + splayed_hash_ids + ' AND status = ?; ' , ( service_id , tag_id , HC . DELETED ) )
else :
2015-03-25 22:04:19 +00:00
already_deleted = [ hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT hash_id FROM mapping_petitions WHERE service_id = ? AND tag_id = ? AND hash_id IN ' + HydrusData . SplayListForDB ( hash_ids ) + ' AND status = ?; ' , ( service_id , tag_id , HC . DELETED ) ) ]
2015-03-04 22:44:32 +00:00
hash_ids = set ( hash_ids ) . difference ( already_deleted )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2015-03-04 22:44:32 +00:00
self . _c . executemany ( ' INSERT OR IGNORE INTO mappings ( service_id, tag_id, hash_id, account_id, timestamp ) VALUES ( ?, ?, ?, ?, ? ); ' , [ ( service_id , tag_id , hash_id , account_id , now ) for hash_id in hash_ids ] )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _AddMappingPetition ( self , service_id , account_id , tag_id , hash_ids , reason_id ) :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _ApproveMappingPetitionOptimised ( service_id , account_id , tag_id , hash_ids )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
valid_hash_ids = [ hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT hash_id FROM mappings WHERE service_id = ? AND tag_id = ? AND hash_id IN ' + HydrusData . SplayListForDB ( hash_ids ) + ' ; ' , ( service_id , tag_id ) ) ]
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
self . _c . execute ( ' DELETE FROM mapping_petitions WHERE service_id = ? AND account_id = ? AND tag_id = ? AND hash_id IN ' + HydrusData . SplayListForDB ( valid_hash_ids ) + ' AND STATUS = ?; ' , ( service_id , account_id , tag_id , HC . PETITIONED ) )
2013-06-12 22:53:31 +00:00
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
self . _c . executemany ( ' INSERT OR IGNORE INTO mapping_petitions ( service_id, account_id, tag_id, hash_id, reason_id, timestamp, status ) VALUES ( ?, ?, ?, ?, ?, ?, ? ); ' , [ ( service_id , account_id , tag_id , hash_id , reason_id , now , HC . PETITIONED ) for hash_id in valid_hash_ids ] )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _AddMessagingSession ( self , service_key , session_key , account_key , name , expires ) :
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
service_id = self . _GetServiceId ( service_key )
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
account_id = self . _GetAccountId ( account_key )
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' INSERT INTO sessions ( service_id, session_key, account_id, identifier, name, expiry ) VALUES ( ?, ?, ?, ?, ?, ? ); ' , ( service_id , sqlite3 . Binary ( session_key ) , account_id , sqlite3 . Binary ( account_key ) , name , expires ) )
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
def _AddSession ( self , session_key , service_key , account_key , expires ) :
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
service_id = self . _GetServiceId ( service_key )
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
account_id = self . _GetAccountId ( account_key )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' INSERT INTO sessions ( session_key, service_id, account_id, expiry ) VALUES ( ?, ?, ?, ? ); ' , ( sqlite3 . Binary ( session_key ) , service_id , account_id , expires ) )
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
def _AddTagParentPetition ( self , service_id , account_id , old_tag_id , new_tag_id , reason_id , status ) :
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
result = self . _c . execute ( ' SELECT 1 WHERE EXISTS ( SELECT 1 FROM tag_parents WHERE service_id = ? AND old_tag_id = ? AND new_tag_id = ? AND status = ? ); ' , ( service_id , old_tag_id , new_tag_id , HC . CURRENT ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
it_already_exists = result is not None
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
if status == HC . PENDING :
if it_already_exists : return
elif status == HC . PETITIONED :
if not it_already_exists : return
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' DELETE FROM tag_parents WHERE service_id = ? AND account_id = ? AND old_tag_id = ? AND new_tag_id = ? AND status = ?; ' , ( service_id , account_id , old_tag_id , new_tag_id , status ) )
2013-06-12 22:53:31 +00:00
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' INSERT OR IGNORE INTO tag_parents ( service_id, account_id, old_tag_id, new_tag_id, reason_id, status, timestamp ) VALUES ( ?, ?, ?, ?, ?, ?, ? ); ' , ( service_id , account_id , old_tag_id , new_tag_id , reason_id , status , now ) )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _AddTagSiblingPetition ( self , service_id , account_id , old_tag_id , new_tag_id , reason_id , status ) :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
result = self . _c . execute ( ' SELECT 1 WHERE EXISTS ( SELECT 1 FROM tag_siblings WHERE service_id = ? AND old_tag_id = ? AND new_tag_id = ? AND status = ? ); ' , ( service_id , old_tag_id , new_tag_id , HC . CURRENT ) ) . fetchone ( )
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
it_already_exists = result is not None
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
if status == HC . PENDING :
if it_already_exists : return
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
elif status == HC . PETITIONED :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
if not it_already_exists : return
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' DELETE FROM tag_siblings WHERE service_id = ? AND account_id = ? AND old_tag_id = ? AND new_tag_id = ? AND status = ?; ' , ( service_id , account_id , old_tag_id , new_tag_id , status ) )
2013-08-14 20:21:49 +00:00
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' INSERT OR IGNORE INTO tag_siblings ( service_id, account_id, old_tag_id, new_tag_id, reason_id, status, timestamp ) VALUES ( ?, ?, ?, ?, ?, ?, ? ); ' , ( service_id , account_id , old_tag_id , new_tag_id , reason_id , status , now ) )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
def _AddToExpires ( self , account_ids , timespan ) : self . _c . execute ( ' UPDATE accounts SET expires = expires + ? WHERE account_id IN ' + HydrusData . SplayListForDB ( account_ids ) + ' ; ' , ( timespan , ) )
2015-03-04 22:44:32 +00:00
2016-04-20 20:42:21 +00:00
def _Analyze ( self , stop_time ) :
stale_time_delta = 30 * 86400
2016-01-06 21:17:20 +00:00
existing_names_to_timestamps = dict ( self . _c . execute ( ' SELECT name, timestamp FROM analyze_timestamps; ' ) . fetchall ( ) )
2016-04-27 19:20:37 +00:00
db_names = [ name for ( index , name , path ) in self . _c . execute ( ' PRAGMA database_list; ' ) if name not in ( ' mem ' , ' temp ' ) ]
all_names = set ( )
for db_name in db_names :
all_names . update ( ( name for ( name , ) in self . _c . execute ( ' SELECT name FROM ' + db_name + ' .sqlite_master; ' ) ) )
2016-01-06 21:17:20 +00:00
names_to_analyze = [ name for name in all_names if name not in existing_names_to_timestamps or HydrusData . TimeHasPassed ( existing_names_to_timestamps [ name ] + stale_time_delta ) ]
random . shuffle ( names_to_analyze )
if len ( names_to_analyze ) > 0 :
HydrusGlobals . server_busy = True
2016-04-27 19:20:37 +00:00
for name in names_to_analyze :
2016-01-06 21:17:20 +00:00
started = HydrusData . GetNowPrecise ( )
self . _c . execute ( ' ANALYZE ' + name + ' ; ' )
2016-05-11 18:16:39 +00:00
self . _c . execute ( ' DELETE FROM analyze_timestamps WHERE name = ?; ' , ( name , ) )
self . _c . execute ( ' INSERT OR IGNORE INTO analyze_timestamps ( name, timestamp ) VALUES ( ?, ? ); ' , ( name , HydrusData . GetNow ( ) ) )
2016-01-06 21:17:20 +00:00
time_took = HydrusData . GetNowPrecise ( ) - started
2016-04-20 20:42:21 +00:00
if time_took > 1 :
HydrusData . Print ( ' Analyzed ' + name + ' in ' + HydrusData . ConvertTimeDeltaToPrettyString ( time_took ) )
2016-01-06 21:17:20 +00:00
if HydrusData . TimeHasPassed ( stop_time ) :
break
self . _c . execute ( ' ANALYZE sqlite_master; ' ) # this reloads the current stats into the query planner
HydrusGlobals . server_busy = False
2015-03-04 22:44:32 +00:00
def _ApproveFilePetition ( self , service_id , account_id , hash_ids , reason_id ) :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _ApproveFilePetitionOptimised ( service_id , account_id , hash_ids )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
valid_hash_ids = [ hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT hash_id FROM file_map WHERE service_id = ? AND hash_id IN ' + HydrusData . SplayListForDB ( hash_ids ) + ' ; ' , ( service_id , ) ) ]
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _RewardFilePetitioners ( service_id , valid_hash_ids , 1 )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _DeleteFiles ( service_id , account_id , valid_hash_ids , reason_id )
2014-10-01 22:58:32 +00:00
2015-03-04 22:44:32 +00:00
def _ApproveFilePetitionOptimised ( self , service_id , account_id , hash_ids ) :
2013-02-19 00:11:43 +00:00
2016-04-20 20:42:21 +00:00
( biggest_end , ) = self . _c . execute ( ' SELECT end FROM update_cache WHERE service_id = ? ORDER BY end DESC LIMIT 1; ' , ( service_id , ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
self . _c . execute ( ' DELETE FROM file_map WHERE service_id = ? AND account_id = ? AND hash_id IN ' + HydrusData . SplayListForDB ( hash_ids ) + ' AND timestamp > ?; ' , ( service_id , account_id , biggest_end ) )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _ApproveMappingPetition ( self , service_id , account_id , tag_id , hash_ids , reason_id ) :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _ApproveMappingPetitionOptimised ( service_id , account_id , tag_id , hash_ids )
2013-06-12 22:53:31 +00:00
2015-03-25 22:04:19 +00:00
valid_hash_ids = [ hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT hash_id FROM mappings WHERE service_id = ? AND tag_id = ? AND hash_id IN ' + HydrusData . SplayListForDB ( hash_ids ) + ' ; ' , ( service_id , tag_id ) ) ]
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _RewardMappingPetitioners ( service_id , tag_id , valid_hash_ids , 1 )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _DeleteMappings ( service_id , account_id , tag_id , valid_hash_ids , reason_id )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _ApproveMappingPetitionOptimised ( self , service_id , account_id , tag_id , hash_ids ) :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
( biggest_end , ) = self . _c . execute ( ' SELECT end FROM update_cache WHERE service_id = ? ORDER BY end DESC LIMIT 1; ' , ( service_id , ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
self . _c . execute ( ' DELETE FROM mappings WHERE service_id = ? AND account_id = ? AND tag_id = ? AND hash_id IN ' + HydrusData . SplayListForDB ( hash_ids ) + ' AND timestamp > ?; ' , ( service_id , account_id , tag_id , biggest_end ) )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _ApproveTagParentPetition ( self , service_id , account_id , old_tag_id , new_tag_id , reason_id , status ) :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
result = self . _c . execute ( ' SELECT 1 WHERE EXISTS ( SELECT 1 FROM tag_parents WHERE service_id = ? AND old_tag_id = ? AND new_tag_id = ? AND status = ? ); ' , ( service_id , old_tag_id , new_tag_id , HC . CURRENT ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
it_already_exists = result is not None
if status == HC . PENDING :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
if it_already_exists : return
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
elif status == HC . PETITIONED :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
if not it_already_exists : return
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _RewardTagParentPetitioners ( service_id , old_tag_id , new_tag_id , 1 )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' DELETE FROM tag_parents WHERE service_id = ? AND old_tag_id = ? AND new_tag_id = ?; ' , ( service_id , old_tag_id , new_tag_id ) )
2013-06-12 22:53:31 +00:00
if status == HC . PENDING : new_status = HC . CURRENT
elif status == HC . PETITIONED : new_status = HC . DELETED
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-06-12 22:53:31 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' INSERT OR IGNORE INTO tag_parents ( service_id, account_id, old_tag_id, new_tag_id, reason_id, status, timestamp ) VALUES ( ?, ?, ?, ?, ?, ?, ? ); ' , ( service_id , account_id , old_tag_id , new_tag_id , reason_id , new_status , now ) )
2013-06-12 22:53:31 +00:00
2016-05-04 21:50:55 +00:00
if new_status == HC . CURRENT :
child_hash_ids = [ hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT hash_id FROM mappings WHERE service_id = ? AND tag_id = ?; ' , ( service_id , old_tag_id ) ) ]
self . _AddMappings ( service_id , account_id , new_tag_id , child_hash_ids , True )
2013-06-12 22:53:31 +00:00
2014-11-20 01:48:04 +00:00
def _ApproveTagSiblingPetition ( self , service_id , account_id , old_tag_id , new_tag_id , reason_id , status ) :
2013-06-12 22:53:31 +00:00
2014-11-20 01:48:04 +00:00
result = self . _c . execute ( ' SELECT 1 WHERE EXISTS ( SELECT 1 FROM tag_siblings WHERE service_id = ? AND old_tag_id = ? AND new_tag_id = ? AND status = ? ); ' , ( service_id , old_tag_id , new_tag_id , HC . CURRENT ) ) . fetchone ( )
2013-06-12 22:53:31 +00:00
it_already_exists = result is not None
if status == HC . PENDING :
if it_already_exists : return
elif status == HC . PETITIONED :
if not it_already_exists : return
2014-11-20 01:48:04 +00:00
self . _RewardTagSiblingPetitioners ( service_id , old_tag_id , new_tag_id , 1 )
2013-06-12 22:53:31 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' DELETE FROM tag_siblings WHERE service_id = ? AND old_tag_id = ? AND new_tag_id = ?; ' , ( service_id , old_tag_id , new_tag_id ) )
2013-06-12 22:53:31 +00:00
if status == HC . PENDING : new_status = HC . CURRENT
elif status == HC . PETITIONED : new_status = HC . DELETED
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-06-12 22:53:31 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' INSERT OR IGNORE INTO tag_siblings ( service_id, account_id, old_tag_id, new_tag_id, reason_id, status, timestamp ) VALUES ( ?, ?, ?, ?, ?, ?, ? ); ' , ( service_id , account_id , old_tag_id , new_tag_id , reason_id , new_status , now ) )
2013-06-12 22:53:31 +00:00
2016-04-14 01:54:29 +00:00
def _Backup ( self ) :
self . _c . execute ( ' COMMIT; ' )
self . _CloseDBCursor ( )
try :
stop_time = HydrusData . GetNow ( ) + 300
for filename in self . _db_filenames . values ( ) :
db_path = os . path . join ( self . _db_dir , filename )
if HydrusDB . CanVacuum ( db_path , stop_time ) :
HydrusData . Print ( ' backing up: vacuuming ' + filename )
HydrusDB . VacuumDB ( db_path )
backup_path = os . path . join ( HC . DB_DIR , ' server_backup ' )
if not os . path . exists ( backup_path ) :
os . makedirs ( backup_path )
for filename in self . _db_filenames . values ( ) :
HydrusData . Print ( ' backing up: copying ' + filename )
source = os . path . join ( self . _db_dir , filename )
dest = os . path . join ( backup_path , filename )
2016-05-18 20:07:14 +00:00
if not HydrusPaths . PathsHaveSameSizeAndDate ( source , dest ) :
shutil . copy2 ( source , dest )
2016-04-14 01:54:29 +00:00
HydrusData . Print ( ' backing up: copying files ' )
HydrusPaths . MirrorTree ( HC . SERVER_FILES_DIR , os . path . join ( backup_path , ' server_files ' ) )
HydrusData . Print ( ' backing up: copying thumbnails ' )
HydrusPaths . MirrorTree ( HC . SERVER_THUMBNAILS_DIR , os . path . join ( backup_path , ' server_thumbnails ' ) )
HydrusData . Print ( ' backing up: copying updates ' )
HydrusPaths . MirrorTree ( HC . SERVER_UPDATES_DIR , os . path . join ( backup_path , ' server_updates ' ) )
finally :
self . _InitDBCursor ( )
self . _c . execute ( ' BEGIN IMMEDIATE; ' )
HydrusData . Print ( ' backing up: done! ' )
2016-03-30 22:56:50 +00:00
2015-03-04 22:44:32 +00:00
def _Ban ( self , service_id , action , admin_account_id , subject_account_ids , reason_id , expires = None , lifetime = None ) :
2013-06-12 22:53:31 +00:00
2015-03-25 22:04:19 +00:00
splayed_subject_account_ids = HydrusData . SplayListForDB ( subject_account_ids )
2013-06-12 22:53:31 +00:00
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
if expires is not None : pass
elif lifetime is not None : expires = now + lifetime
else : expires = None
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . executemany ( ' INSERT OR IGNORE INTO bans ( service_id, account_id, admin_account_id, reason_id, created, expires ) VALUES ( ?, ?, ?, ?, ? ); ' , [ ( service_id , subject_account_id , admin_account_id , reason_id , now , expires ) for subject_account_id in subject_account_ids ] )
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' DELETE FROM file_petitions WHERE service_id = ? AND account_id IN ' + splayed_subject_account_ids + ' AND status = ?; ' , ( service_id , HC . PETITIONED ) )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' DELETE FROM mapping_petitions WHERE service_id = ? AND account_id IN ' + splayed_subject_account_ids + ' AND status = ?; ' , ( service_id , HC . PETITIONED ) )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' DELETE FROM tag_siblings WHERE service_id = ? AND account_id IN ' + splayed_subject_account_ids + ' AND status IN ( ?, ? ); ' , ( service_id , HC . PENDING , HC . PETITIONED ) )
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' DELETE FROM tag_parents WHERE service_id = ? AND account_id IN ' + splayed_subject_account_ids + ' AND status IN ( ?, ? ); ' , ( service_id , HC . PENDING , HC . PETITIONED ) )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
if action == HC . SUPERBAN :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
hash_ids = [ hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT hash_id FROM files_info WHERE service_id = ? AND account_id IN ' + splayed_subject_account_ids + ' ; ' , ( service_id , ) ) ]
2013-02-19 00:11:43 +00:00
2015-04-22 22:57:25 +00:00
if len ( hash_ids ) > 0 : self . _DeleteFiles ( service_id , admin_account_id , hash_ids , reason_id )
2015-03-04 22:44:32 +00:00
2015-03-25 22:04:19 +00:00
mappings_dict = HydrusData . BuildKeyToListDict ( self . _c . execute ( ' SELECT tag_id, hash_id FROM mappings WHERE service_id = ? AND account_id IN ' + splayed_subject_account_ids + ' ; ' , ( service_id , ) ) )
2015-03-04 22:44:32 +00:00
if len ( mappings_dict ) > 0 :
for ( tag_id , hash_ids ) in mappings_dict . items ( ) : self . _DeleteMappings ( service_id , admin_account_id , tag_id , hash_ids , reason_id )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
def _ChangeAccountType ( self , account_ids , account_type_id ) : self . _c . execute ( ' UPDATE accounts SET account_type_id = ? WHERE account_id IN ' + HydrusData . SplayListForDB ( account_ids ) + ' ; ' , ( account_type_id , ) )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _CheckDataUsage ( self ) :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
( version_year , version_month ) = self . _c . execute ( ' SELECT year, month FROM version; ' ) . fetchone ( )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
current_time_struct = time . gmtime ( )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
( current_year , current_month ) = ( current_time_struct . tm_year , current_time_struct . tm_mon )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
if version_year != current_year or version_month != current_month :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' UPDATE version SET year = ?, month = ?; ' , ( current_year , current_month ) )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' UPDATE accounts SET used_bytes = ?, used_requests = ?; ' , ( 0 , 0 ) )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . pub_after_commit ( ' update_all_session_accounts ' )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _CheckMonthlyData ( self ) :
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
service_info = self . _GetServicesInfo ( )
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
running_total = 0
self . _services_over_monthly_data = set ( )
for ( service_key , service_type , options ) in service_info :
service_id = self . _GetServiceId ( service_key )
if service_type != HC . SERVER_ADMIN :
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
( total_used_bytes , ) = self . _c . execute ( ' SELECT SUM( used_bytes ) FROM accounts WHERE service_id = ?; ' , ( service_id , ) ) . fetchone ( )
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
if total_used_bytes is None : total_used_bytes = 0
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
running_total + = total_used_bytes
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
if ' max_monthly_data ' in options :
max_monthly_data = options [ ' max_monthly_data ' ]
if max_monthly_data is not None and total_used_bytes > max_monthly_data : self . _services_over_monthly_data . add ( service_key )
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
# have to do this after
server_admin_options = self . _GetOptions ( HC . SERVER_ADMIN_KEY )
self . _over_monthly_data = False
if ' max_monthly_data ' in server_admin_options :
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
max_monthly_data = server_admin_options [ ' max_monthly_data ' ]
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
if max_monthly_data is not None and running_total > max_monthly_data : self . _over_monthly_data = True
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
def _ClearBans ( self ) :
2013-06-12 22:53:31 +00:00
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-06-12 22:53:31 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' DELETE FROM bans WHERE expires < ?; ' , ( now , ) )
2013-06-12 22:53:31 +00:00
2015-04-22 22:57:25 +00:00
def _CreateDB ( self ) :
2015-11-18 22:44:07 +00:00
dirs = ( HC . SERVER_FILES_DIR , HC . SERVER_THUMBNAILS_DIR , HC . SERVER_UPDATES_DIR )
2015-04-22 22:57:25 +00:00
for dir in dirs :
2016-01-06 21:17:20 +00:00
if not os . path . exists ( dir ) :
os . makedirs ( dir )
2015-04-22 22:57:25 +00:00
2015-11-18 22:44:07 +00:00
dirs = ( HC . SERVER_FILES_DIR , HC . SERVER_THUMBNAILS_DIR )
2015-04-22 22:57:25 +00:00
for dir in dirs :
2015-12-02 22:32:18 +00:00
for prefix in HydrusData . IterateHexPrefixes ( ) :
2015-04-22 22:57:25 +00:00
2015-12-02 22:32:18 +00:00
new_dir = os . path . join ( dir , prefix )
2015-04-22 22:57:25 +00:00
2015-11-04 22:30:28 +00:00
if not os . path . exists ( new_dir ) :
2016-01-06 21:17:20 +00:00
os . makedirs ( new_dir )
2015-11-04 22:30:28 +00:00
2015-04-22 22:57:25 +00:00
2016-04-14 01:54:29 +00:00
HydrusDB . SetupDBCreatePragma ( self . _c , no_wal = self . _no_wal )
2015-04-22 22:57:25 +00:00
2016-04-14 01:54:29 +00:00
self . _c . execute ( ' BEGIN IMMEDIATE ' )
2015-04-22 22:57:25 +00:00
now = HydrusData . GetNow ( )
self . _c . execute ( ' CREATE TABLE services ( service_id INTEGER PRIMARY KEY, service_key BLOB_BYTES, type INTEGER, options TEXT_YAML ); ' )
self . _c . execute ( ' CREATE TABLE accounts( account_id INTEGER PRIMARY KEY, service_id INTEGER REFERENCES services ON DELETE CASCADE, account_key BLOB_BYTES, hashed_access_key BLOB_BYTES, account_type_id INTEGER, created INTEGER, expires INTEGER, used_bytes INTEGER, used_requests INTEGER ); ' )
self . _c . execute ( ' CREATE UNIQUE INDEX accounts_account_key_index ON accounts ( account_key ); ' )
self . _c . execute ( ' CREATE UNIQUE INDEX accounts_hashed_access_key_index ON accounts ( hashed_access_key ); ' )
self . _c . execute ( ' CREATE UNIQUE INDEX accounts_service_id_account_id_index ON accounts ( service_id, account_id ); ' )
self . _c . execute ( ' CREATE INDEX accounts_service_id_account_type_id_index ON accounts ( service_id, account_type_id ); ' )
self . _c . execute ( ' CREATE TABLE account_scores ( service_id INTEGER REFERENCES services ON DELETE CASCADE, account_id INTEGER, score_type INTEGER, score INTEGER, PRIMARY KEY( service_id, account_id, score_type ) ); ' )
self . _c . execute ( ' CREATE TABLE account_types ( account_type_id INTEGER PRIMARY KEY, service_id INTEGER REFERENCES services ON DELETE CASCADE, title TEXT, account_type TEXT_YAML ); ' )
self . _c . execute ( ' CREATE UNIQUE INDEX account_types_service_id_account_type_id_index ON account_types ( service_id, account_type_id ); ' )
2016-01-06 21:17:20 +00:00
self . _c . execute ( ' CREATE TABLE analyze_timestamps ( name TEXT, timestamp INTEGER ); ' )
2015-04-22 22:57:25 +00:00
self . _c . execute ( ' CREATE TABLE bans ( service_id INTEGER REFERENCES services ON DELETE CASCADE, account_id INTEGER, admin_account_id INTEGER, reason_id INTEGER, created INTEGER, expires INTEGER, PRIMARY KEY( service_id, account_id ) ); ' )
self . _c . execute ( ' CREATE INDEX bans_expires ON bans ( expires ); ' )
self . _c . execute ( ' CREATE TABLE contacts ( service_id INTEGER REFERENCES services ON DELETE CASCADE, account_id INTEGER, contact_key BLOB, public_key TEXT, PRIMARY KEY( service_id, account_id ) ); ' )
self . _c . execute ( ' CREATE UNIQUE INDEX contacts_contact_key_index ON contacts ( contact_key ); ' )
self . _c . execute ( ' CREATE TABLE files_info ( hash_id INTEGER PRIMARY KEY, size INTEGER, mime INTEGER, width INTEGER, height INTEGER, duration INTEGER, num_frames INTEGER, num_words INTEGER ); ' )
self . _c . execute ( ' CREATE TABLE file_map ( service_id INTEGER REFERENCES services ON DELETE CASCADE, hash_id INTEGER, account_id INTEGER, timestamp INTEGER, PRIMARY KEY( service_id, hash_id, account_id ) ); ' )
self . _c . execute ( ' CREATE INDEX file_map_service_id_account_id_index ON file_map ( service_id, account_id ); ' )
self . _c . execute ( ' CREATE INDEX file_map_service_id_timestamp_index ON file_map ( service_id, timestamp ); ' )
self . _c . execute ( ' CREATE TABLE file_petitions ( service_id INTEGER REFERENCES services ON DELETE CASCADE, account_id INTEGER, hash_id INTEGER, reason_id INTEGER, timestamp INTEGER, status INTEGER, PRIMARY KEY( service_id, account_id, hash_id, status ) ); ' )
self . _c . execute ( ' CREATE INDEX file_petitions_service_id_account_id_reason_id_index ON file_petitions ( service_id, account_id, reason_id ); ' )
self . _c . execute ( ' CREATE INDEX file_petitions_service_id_hash_id_index ON file_petitions ( service_id, hash_id ); ' )
self . _c . execute ( ' CREATE INDEX file_petitions_service_id_status_index ON file_petitions ( service_id, status ); ' )
self . _c . execute ( ' CREATE INDEX file_petitions_service_id_timestamp_index ON file_petitions ( service_id, timestamp ); ' )
self . _c . execute ( ' CREATE TABLE ip_addresses ( service_id INTEGER REFERENCES services ON DELETE CASCADE, hash_id INTEGER, ip TEXT, timestamp INTEGER, PRIMARY KEY( service_id, hash_id ) ); ' )
self . _c . execute ( ' CREATE TABLE messages ( message_key BLOB_BYTES PRIMARY KEY, service_id INTEGER REFERENCES services ON DELETE CASCADE, account_id INTEGER, timestamp INTEGER ); ' )
self . _c . execute ( ' CREATE INDEX messages_service_id_account_id_index ON messages ( service_id, account_id ); ' )
self . _c . execute ( ' CREATE INDEX messages_timestamp_index ON messages ( timestamp ); ' )
self . _c . execute ( ' CREATE TABLE message_statuses ( status_key BLOB_BYTES PRIMARY KEY, service_id INTEGER REFERENCES services ON DELETE CASCADE, account_id INTEGER, status BLOB_BYTES, timestamp INTEGER ); ' )
self . _c . execute ( ' CREATE INDEX message_statuses_service_id_account_id_index ON message_statuses ( service_id, account_id ); ' )
self . _c . execute ( ' CREATE INDEX message_statuses_timestamp_index ON message_statuses ( timestamp ); ' )
self . _c . execute ( ' CREATE TABLE messaging_sessions ( service_id INTEGER REFERENCES services ON DELETE CASCADE, session_key BLOB_BYTES, account_id INTEGER, identifier BLOB_BYTES, name TEXT, expiry INTEGER ); ' )
self . _c . execute ( ' CREATE TABLE news ( service_id INTEGER REFERENCES services ON DELETE CASCADE, news TEXT, timestamp INTEGER ); ' )
self . _c . execute ( ' CREATE INDEX news_timestamp_index ON news ( timestamp ); ' )
self . _c . execute ( ' CREATE TABLE reasons ( reason_id INTEGER PRIMARY KEY, reason TEXT ); ' )
self . _c . execute ( ' CREATE UNIQUE INDEX reasons_reason_index ON reasons ( reason ); ' )
self . _c . execute ( ' CREATE TABLE registration_keys ( registration_key BLOB_BYTES PRIMARY KEY, service_id INTEGER REFERENCES services ON DELETE CASCADE, account_type_id INTEGER, account_key BLOB_BYTES, access_key BLOB_BYTES, expiry INTEGER ); ' )
self . _c . execute ( ' CREATE UNIQUE INDEX registration_keys_access_key_index ON registration_keys ( access_key ); ' )
self . _c . execute ( ' CREATE TABLE sessions ( session_key BLOB_BYTES, service_id INTEGER REFERENCES services ON DELETE CASCADE, account_id INTEGER, expiry INTEGER ); ' )
self . _c . execute ( ' CREATE TABLE tag_parents ( service_id INTEGER REFERENCES services ON DELETE CASCADE, account_id INTEGER, old_tag_id INTEGER, new_tag_id INTEGER, timestamp INTEGER, status INTEGER, reason_id INTEGER, PRIMARY KEY ( service_id, account_id, old_tag_id, new_tag_id, status ) ); ' )
self . _c . execute ( ' CREATE INDEX tag_parents_service_id_old_tag_id_index ON tag_parents ( service_id, old_tag_id, new_tag_id ); ' )
self . _c . execute ( ' CREATE INDEX tag_parents_service_id_timestamp_index ON tag_parents ( service_id, timestamp ); ' )
self . _c . execute ( ' CREATE INDEX tag_parents_service_id_status_index ON tag_parents ( service_id, status ); ' )
self . _c . execute ( ' CREATE TABLE tag_siblings ( service_id INTEGER REFERENCES services ON DELETE CASCADE, account_id INTEGER, old_tag_id INTEGER, new_tag_id INTEGER, timestamp INTEGER, status INTEGER, reason_id INTEGER, PRIMARY KEY ( service_id, account_id, old_tag_id, status ) ); ' )
self . _c . execute ( ' CREATE INDEX tag_siblings_service_id_old_tag_id_index ON tag_siblings ( service_id, old_tag_id ); ' )
self . _c . execute ( ' CREATE INDEX tag_siblings_service_id_timestamp_index ON tag_siblings ( service_id, timestamp ); ' )
self . _c . execute ( ' CREATE INDEX tag_siblings_service_id_status_index ON tag_siblings ( service_id, status ); ' )
self . _c . execute ( ' CREATE TABLE update_cache ( service_id INTEGER REFERENCES services ON DELETE CASCADE, begin INTEGER, end INTEGER, dirty INTEGER_BOOLEAN, PRIMARY KEY( service_id, begin ) ); ' )
self . _c . execute ( ' CREATE UNIQUE INDEX update_cache_service_id_end_index ON update_cache ( service_id, end ); ' )
self . _c . execute ( ' CREATE INDEX update_cache_service_id_dirty_index ON update_cache ( service_id, dirty ); ' )
self . _c . execute ( ' CREATE TABLE version ( version INTEGER, year INTEGER, month INTEGER ); ' )
2016-04-20 20:42:21 +00:00
# mappings
self . _c . execute ( ' CREATE TABLE IF NOT EXISTS external_mappings.mapping_petitions ( service_id INTEGER, account_id INTEGER, tag_id INTEGER, hash_id INTEGER, reason_id INTEGER, timestamp INTEGER, status INTEGER, PRIMARY KEY( service_id, account_id, tag_id, hash_id, status ) ) WITHOUT ROWID; ' )
self . _c . execute ( ' CREATE INDEX IF NOT EXISTS external_mappings.mapping_petitions_service_id_account_id_reason_id_tag_id_index ON mapping_petitions ( service_id, account_id, reason_id, tag_id ); ' )
self . _c . execute ( ' CREATE INDEX IF NOT EXISTS external_mappings.mapping_petitions_service_id_tag_id_hash_id_index ON mapping_petitions ( service_id, tag_id, hash_id ); ' )
self . _c . execute ( ' CREATE INDEX IF NOT EXISTS external_mappings.mapping_petitions_service_id_status_index ON mapping_petitions ( service_id, status ); ' )
self . _c . execute ( ' CREATE INDEX IF NOT EXISTS external_mappings.mapping_petitions_service_id_timestamp_index ON mapping_petitions ( service_id, timestamp ); ' )
self . _c . execute ( ' CREATE TABLE IF NOT EXISTS external_mappings.mappings ( service_id INTEGER, tag_id INTEGER, hash_id INTEGER, account_id INTEGER, timestamp INTEGER, PRIMARY KEY( service_id, tag_id, hash_id ) ) WITHOUT ROWID; ' )
self . _c . execute ( ' CREATE INDEX IF NOT EXISTS external_mappings.mappings_account_id_index ON mappings ( account_id ); ' )
self . _c . execute ( ' CREATE INDEX IF NOT EXISTS external_mappings.mappings_timestamp_index ON mappings ( timestamp ); ' )
# master
self . _c . execute ( ' CREATE TABLE IF NOT EXISTS external_master.hashes ( hash_id INTEGER PRIMARY KEY, hash BLOB_BYTES UNIQUE ); ' )
self . _c . execute ( ' CREATE TABLE IF NOT EXISTS external_master.tags ( tag_id INTEGER PRIMARY KEY, tag TEXT UNIQUE ); ' )
# inserts
2015-04-22 22:57:25 +00:00
current_time_struct = time . gmtime ( )
( current_year , current_month ) = ( current_time_struct . tm_year , current_time_struct . tm_mon )
self . _c . execute ( ' INSERT INTO version ( version, year, month ) VALUES ( ?, ?, ? ); ' , ( HC . SOFTWARE_VERSION , current_year , current_month ) )
# set up server admin
service_key = HC . SERVER_ADMIN_KEY
service_type = HC . SERVER_ADMIN
options = HC . DEFAULT_OPTIONS [ HC . SERVER_ADMIN ]
options [ ' port ' ] = HC . DEFAULT_SERVER_ADMIN_PORT
self . _c . execute ( ' INSERT INTO services ( service_key, type, options ) VALUES ( ?, ?, ? ); ' , ( sqlite3 . Binary ( service_key ) , service_type , options ) )
server_admin_service_id = self . _c . lastrowid
server_admin_account_type = HydrusData . AccountType ( ' server admin ' , [ HC . MANAGE_USERS , HC . GENERAL_ADMIN , HC . EDIT_SERVICES ] , ( None , None ) )
self . _c . execute ( ' INSERT INTO account_types ( service_id, title, account_type ) VALUES ( ?, ?, ? ); ' , ( server_admin_service_id , ' server admin ' , server_admin_account_type ) )
self . _c . execute ( ' COMMIT ' )
2015-03-04 22:44:32 +00:00
def _CreateUpdate ( self , service_key , begin , end ) :
2013-06-12 22:53:31 +00:00
2015-06-03 21:05:13 +00:00
self . _GenerateUpdate ( service_key , begin , end )
2013-06-12 22:53:31 +00:00
2015-06-03 21:05:13 +00:00
service_id = self . _GetServiceId ( service_key )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' INSERT OR REPLACE INTO update_cache ( service_id, begin, end, dirty ) VALUES ( ?, ?, ?, ? ); ' , ( service_id , begin , end , False ) )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _DeleteFiles ( self , service_id , account_id , hash_ids , reason_id ) :
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
splayed_hash_ids = HydrusData . SplayListForDB ( hash_ids )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' DELETE FROM file_map WHERE service_id = ? AND hash_id IN ' + splayed_hash_ids + ' ; ' , ( service_id , ) )
self . _c . execute ( ' DELETE FROM file_petitions WHERE service_id = ? AND hash_id IN ' + splayed_hash_ids + ' AND status = ?; ' , ( service_id , HC . PETITIONED ) )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . executemany ( ' INSERT OR IGNORE INTO file_petitions ( service_id, account_id, hash_id, reason_id, timestamp, status ) VALUES ( ?, ?, ?, ?, ?, ? ); ' , ( ( service_id , account_id , hash_id , reason_id , now , HC . DELETED ) for hash_id in hash_ids ) )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _DeleteMappings ( self , service_id , account_id , tag_id , hash_ids , reason_id ) :
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
splayed_hash_ids = HydrusData . SplayListForDB ( hash_ids )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' DELETE FROM mappings WHERE service_id = ? AND tag_id = ? AND hash_id IN ' + splayed_hash_ids + ' ; ' , ( service_id , tag_id ) )
self . _c . execute ( ' DELETE FROM mapping_petitions WHERE service_id = ? AND tag_id = ? AND hash_id IN ' + splayed_hash_ids + ' AND status = ?; ' , ( service_id , tag_id , HC . PETITIONED ) )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
self . _c . executemany ( ' INSERT OR IGNORE INTO mapping_petitions ( service_id, tag_id, hash_id, account_id, reason_id, timestamp, status ) VALUES ( ?, ?, ?, ?, ?, ?, ? ); ' , ( ( service_id , tag_id , hash_id , account_id , reason_id , HydrusData . GetNow ( ) , HC . DELETED ) for hash_id in hash_ids ) )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _DeleteOrphans ( self ) :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
# files
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
deletees = [ hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT DISTINCT hash_id FROM files_info EXCEPT SELECT DISTINCT hash_id FROM file_map; ' ) ]
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
if len ( deletees ) > 0 :
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
deletee_hashes = set ( self . _GetHashes ( deletees ) )
2013-02-19 00:11:43 +00:00
2015-03-18 21:46:29 +00:00
local_files_hashes = ServerFiles . GetAllHashes ( ' file ' )
thumbnails_hashes = ServerFiles . GetAllHashes ( ' thumbnail ' )
2013-02-19 00:11:43 +00:00
2015-08-19 21:48:21 +00:00
for hash in local_files_hashes & deletee_hashes :
path = ServerFiles . GetPath ( ' file ' , hash )
2015-11-25 22:00:57 +00:00
HydrusPaths . RecyclePath ( path )
2015-08-19 21:48:21 +00:00
for hash in thumbnails_hashes & deletee_hashes :
path = ServerFiles . GetPath ( ' thumbnail ' , hash )
2015-11-25 22:00:57 +00:00
HydrusPaths . RecyclePath ( path )
2015-08-19 21:48:21 +00:00
2015-03-04 22:44:32 +00:00
2015-03-25 22:04:19 +00:00
self . _c . execute ( ' DELETE FROM files_info WHERE hash_id IN ' + HydrusData . SplayListForDB ( deletees ) + ' ; ' )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _DenyFilePetition ( self , service_id , hash_ids ) :
2013-10-02 22:06:06 +00:00
2015-03-04 22:44:32 +00:00
self . _RewardFilePetitioners ( service_id , hash_ids , - 1 )
2013-10-02 22:06:06 +00:00
2015-03-25 22:04:19 +00:00
self . _c . execute ( ' DELETE FROM file_petitions WHERE service_id = ? AND hash_id IN ' + HydrusData . SplayListForDB ( hash_ids ) + ' AND status = ?; ' , ( service_id , HC . PETITIONED ) )
2013-10-02 22:06:06 +00:00
2015-03-04 22:44:32 +00:00
def _DenyMappingPetition ( self , service_id , tag_id , hash_ids ) :
self . _RewardMappingPetitioners ( service_id , tag_id , hash_ids , - 1 )
2015-03-25 22:04:19 +00:00
self . _c . execute ( ' DELETE FROM mapping_petitions WHERE service_id = ? AND tag_id = ? AND hash_id IN ' + HydrusData . SplayListForDB ( hash_ids ) + ' AND status = ?; ' , ( service_id , tag_id , HC . PETITIONED ) )
2013-10-02 22:06:06 +00:00
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _DenyTagParentPetition ( self , service_id , old_tag_id , new_tag_id , action ) :
2013-11-27 18:27:11 +00:00
2015-03-04 22:44:32 +00:00
self . _RewardTagParentPetitioners ( service_id , old_tag_id , new_tag_id , - 1 )
2013-11-27 18:27:11 +00:00
2015-03-04 22:44:32 +00:00
if action == HC . CONTENT_UPDATE_DENY_PEND : status = HC . PENDING
elif action == HC . CONTENT_UPDATE_DENY_PETITION : status = HC . PETITIONED
2013-11-27 18:27:11 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' DELETE FROM tag_parents WHERE service_id = ? AND old_tag_id = ? AND new_tag_id = ? AND status = ?; ' , ( service_id , old_tag_id , new_tag_id , status ) )
2013-11-27 18:27:11 +00:00
2015-03-04 22:44:32 +00:00
def _DenyTagSiblingPetition ( self , service_id , old_tag_id , new_tag_id , action ) :
2013-03-15 02:38:12 +00:00
2015-03-04 22:44:32 +00:00
self . _RewardTagSiblingPetitioners ( service_id , old_tag_id , new_tag_id , - 1 )
2013-03-15 02:38:12 +00:00
2015-03-04 22:44:32 +00:00
if action == HC . CONTENT_UPDATE_DENY_PEND : status = HC . PENDING
elif action == HC . CONTENT_UPDATE_DENY_PETITION : status = HC . PETITIONED
2013-03-15 02:38:12 +00:00
2015-03-04 22:44:32 +00:00
self . _c . execute ( ' DELETE FROM tag_siblings WHERE service_id = ? AND old_tag_id = ? AND new_tag_id = ? AND status = ?; ' , ( service_id , old_tag_id , new_tag_id , status ) )
2013-03-15 02:38:12 +00:00
2015-03-04 22:44:32 +00:00
def _FlushRequestsMade ( self , all_requests ) :
2015-03-25 22:04:19 +00:00
requests_dict = HydrusData . BuildKeyToListDict ( all_requests )
2015-03-04 22:44:32 +00:00
self . _c . executemany ( ' UPDATE accounts SET used_bytes = used_bytes + ?, used_requests = used_requests + ? WHERE account_key = ?; ' , [ ( sum ( num_bytes_list ) , len ( num_bytes_list ) , sqlite3 . Binary ( account_key ) ) for ( account_key , num_bytes_list ) in requests_dict . items ( ) ] )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _GenerateRegistrationKeys ( self , service_key , num , title , lifetime = None ) :
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2014-03-26 21:23:10 +00:00
2015-03-04 22:44:32 +00:00
account_type_id = self . _GetAccountTypeId ( service_id , title )
2013-02-19 00:11:43 +00:00
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
if lifetime is not None : expires = now + lifetime
else : expires = None
2013-05-01 17:21:53 +00:00
2015-03-04 22:44:32 +00:00
keys = [ ( os . urandom ( HC . HYDRUS_KEY_LENGTH ) , os . urandom ( HC . HYDRUS_KEY_LENGTH ) , os . urandom ( HC . HYDRUS_KEY_LENGTH ) ) for i in range ( num ) ]
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
self . _c . executemany ( ' INSERT INTO registration_keys ( registration_key, service_id, account_type_id, account_key, access_key, expiry ) VALUES ( ?, ?, ?, ?, ?, ? ); ' , [ ( sqlite3 . Binary ( hashlib . sha256 ( registration_key ) . digest ( ) ) , service_id , account_type_id , sqlite3 . Binary ( account_key ) , sqlite3 . Binary ( access_key ) , expires ) for ( registration_key , account_key , access_key ) in keys ] )
return [ registration_key for ( registration_key , account_key , access_key ) in keys ]
2013-02-19 00:11:43 +00:00
2015-09-02 23:16:09 +00:00
def _GenerateImmediateContentUpdate ( self , service_key ) :
update_ends = self . _GetUpdateEnds ( )
latest_end = update_ends [ service_key ]
begin = latest_end + 1
end = HydrusData . GetNow ( )
service_id = self . _GetServiceId ( service_key )
service_type = self . _GetServiceType ( service_id )
if service_type == HC . FILE_REPOSITORY :
iterator = self . _IterateFileUpdateContentData
elif service_type == HC . TAG_REPOSITORY :
iterator = self . _IterateTagUpdateContentData
subindex = 0
weight = 0
content_update_package = HydrusData . ServerToClientContentUpdatePackage ( )
smaller_time_step = max ( 10 , ( end - begin ) / 100 )
sub_begin = begin
while sub_begin < = end :
sub_end = min ( ( sub_begin + smaller_time_step ) - 1 , end )
for ( data_type , action , rows , hash_ids_to_hashes , rows_weight ) in iterator ( service_id , sub_begin , sub_end ) :
content_update_package . AddContentData ( data_type , action , rows , hash_ids_to_hashes )
sub_begin + = smaller_time_step
return content_update_package
2015-06-03 21:05:13 +00:00
def _GenerateUpdate ( self , service_key , begin , end ) :
2013-02-19 00:11:43 +00:00
2015-06-03 21:05:13 +00:00
service_id = self . _GetServiceId ( service_key )
2013-02-19 00:11:43 +00:00
2015-06-03 21:05:13 +00:00
service_type = self . _GetServiceType ( service_id )
2013-02-19 00:11:43 +00:00
2015-06-03 21:05:13 +00:00
if service_type == HC . FILE_REPOSITORY :
iterator = self . _IterateFileUpdateContentData
elif service_type == HC . TAG_REPOSITORY :
iterator = self . _IterateTagUpdateContentData
2013-03-15 02:38:12 +00:00
2015-06-03 21:05:13 +00:00
subindex = 0
weight = 0
2013-03-15 02:38:12 +00:00
2015-06-03 21:05:13 +00:00
content_update_package = HydrusData . ServerToClientContentUpdatePackage ( )
2013-03-15 02:38:12 +00:00
2015-06-03 21:05:13 +00:00
smaller_time_step = ( end - begin ) / 100
2013-02-19 00:11:43 +00:00
2015-06-03 21:05:13 +00:00
sub_begin = begin
2013-02-19 00:11:43 +00:00
2015-06-03 21:05:13 +00:00
while sub_begin < = end :
sub_end = min ( ( sub_begin + smaller_time_step ) - 1 , end )
for ( data_type , action , rows , hash_ids_to_hashes , rows_weight ) in iterator ( service_id , sub_begin , sub_end ) :
content_update_package . AddContentData ( data_type , action , rows , hash_ids_to_hashes )
weight + = rows_weight
if weight > = 100000 :
path = ServerFiles . GetExpectedContentUpdatePackagePath ( service_key , begin , subindex )
2015-10-14 21:02:25 +00:00
network_string = content_update_package . DumpToNetworkString ( )
2015-06-03 21:05:13 +00:00
2015-11-18 22:44:07 +00:00
with open ( path , ' wb ' ) as f :
f . write ( network_string )
2015-06-03 21:05:13 +00:00
subindex + = 1
weight = 0
content_update_package = HydrusData . ServerToClientContentUpdatePackage ( )
sub_begin + = smaller_time_step
2013-02-19 00:11:43 +00:00
2015-06-03 21:05:13 +00:00
if weight > 0 :
path = ServerFiles . GetExpectedContentUpdatePackagePath ( service_key , begin , subindex )
2015-10-14 21:02:25 +00:00
network_string = content_update_package . DumpToNetworkString ( )
2015-06-03 21:05:13 +00:00
2015-11-18 22:44:07 +00:00
with open ( path , ' wb ' ) as f :
f . write ( network_string )
2015-06-03 21:05:13 +00:00
subindex + = 1
2013-02-19 00:11:43 +00:00
2015-06-03 21:05:13 +00:00
subindex_count = subindex
2013-03-15 02:38:12 +00:00
2015-06-03 21:05:13 +00:00
service_update_package = HydrusData . ServerToClientServiceUpdatePackage ( )
2013-03-15 02:38:12 +00:00
2015-06-03 21:05:13 +00:00
service_update_package . SetBeginEnd ( begin , end )
service_update_package . SetSubindexCount ( subindex_count )
2013-03-15 02:38:12 +00:00
2015-06-03 21:05:13 +00:00
news_rows = self . _c . execute ( ' SELECT news, timestamp FROM news WHERE service_id = ? AND timestamp BETWEEN ? AND ?; ' , ( service_id , begin , end ) ) . fetchall ( )
2015-03-04 22:44:32 +00:00
2015-06-03 21:05:13 +00:00
service_update_package . SetNews ( news_rows )
2015-03-04 22:44:32 +00:00
2015-06-03 21:05:13 +00:00
path = ServerFiles . GetExpectedServiceUpdatePackagePath ( service_key , begin )
2015-03-04 22:44:32 +00:00
2015-10-14 21:02:25 +00:00
network_string = service_update_package . DumpToNetworkString ( )
2015-03-04 22:44:32 +00:00
2015-11-18 22:44:07 +00:00
with open ( path , ' wb ' ) as f :
f . write ( network_string )
2013-03-15 02:38:12 +00:00
2014-11-20 01:48:04 +00:00
def _GetAccessKey ( self , registration_key ) :
2013-03-15 02:38:12 +00:00
2014-10-29 21:39:01 +00:00
# we generate a new access_key every time this is requested so that if the registration_key leaks, no one grab the access_key before the legit user does
# the reg_key is deleted when the last-requested access_key is used to create a session, which calls getaccountkeyfromaccesskey
2014-11-20 01:48:04 +00:00
try : ( one , ) = self . _c . execute ( ' SELECT 1 FROM registration_keys WHERE registration_key = ?; ' , ( sqlite3 . Binary ( hashlib . sha256 ( registration_key ) . digest ( ) ) , ) ) . fetchone ( )
2013-07-24 20:26:00 +00:00
except : raise HydrusExceptions . ForbiddenException ( ' The service could not find that registration key in its database. ' )
2013-03-15 02:38:12 +00:00
2014-10-29 21:39:01 +00:00
new_access_key = os . urandom ( HC . HYDRUS_KEY_LENGTH )
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' UPDATE registration_keys SET access_key = ? WHERE registration_key = ?; ' , ( sqlite3 . Binary ( new_access_key ) , sqlite3 . Binary ( hashlib . sha256 ( registration_key ) . digest ( ) ) ) )
2014-10-29 21:39:01 +00:00
return new_access_key
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
def _GetAccount ( self , account_key ) :
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
( account_id , service_id , account_key , account_type , created , expires , used_bytes , used_requests ) = self . _c . execute ( ' SELECT account_id, service_id, account_key, account_type, created, expires, used_bytes, used_requests FROM accounts, account_types USING ( service_id, account_type_id ) WHERE account_key = ?; ' , ( sqlite3 . Binary ( account_key ) , ) ) . fetchone ( )
2014-09-10 22:37:38 +00:00
2014-11-20 01:48:04 +00:00
banned_info = self . _c . execute ( ' SELECT reason, created, expires FROM bans, reasons USING ( reason_id ) WHERE service_id = ? AND account_id = ?; ' , ( service_id , account_id ) ) . fetchone ( )
2014-09-10 22:37:38 +00:00
2015-03-25 22:04:19 +00:00
return HydrusData . Account ( account_key , account_type , created , expires , used_bytes , used_requests , banned_info = banned_info )
2014-09-10 22:37:38 +00:00
2014-11-20 01:48:04 +00:00
def _GetAccountFileInfo ( self , service_id , account_id ) :
2014-09-10 22:37:38 +00:00
2014-11-20 01:48:04 +00:00
( num_files , num_files_bytes ) = self . _c . execute ( ' SELECT COUNT( * ), SUM( size ) FROM file_map, files_info USING ( hash_id ) WHERE service_id = ? AND account_id = ?; ' , ( service_id , account_id ) ) . fetchone ( )
2014-09-10 22:37:38 +00:00
if num_files_bytes is None : num_files_bytes = 0
2014-11-20 01:48:04 +00:00
result = self . _c . execute ( ' SELECT score FROM account_scores WHERE service_id = ? AND account_id = ? AND score_type = ?; ' , ( service_id , account_id , HC . SCORE_PETITION ) ) . fetchone ( )
2014-09-10 22:37:38 +00:00
if result is None : petition_score = 0
else : ( petition_score , ) = result
2014-11-20 01:48:04 +00:00
( num_petitions , ) = self . _c . execute ( ' SELECT COUNT( DISTINCT reason_id ) FROM file_petitions WHERE service_id = ? AND account_id = ? AND status = ?; ' , ( service_id , account_id , HC . PETITIONED ) ) . fetchone ( )
2014-09-10 22:37:38 +00:00
account_info = { }
account_info [ ' num_files ' ] = num_files
account_info [ ' num_files_bytes ' ] = num_files_bytes
account_info [ ' petition_score ' ] = petition_score
account_info [ ' num_petitions ' ] = num_petitions
return account_info
2014-11-20 01:48:04 +00:00
def _GetAccountKeyFromAccessKey ( self , service_key , access_key ) :
2014-09-10 22:37:38 +00:00
2014-11-20 01:48:04 +00:00
try : ( account_key , ) = self . _c . execute ( ' SELECT account_key FROM accounts WHERE hashed_access_key = ?; ' , ( sqlite3 . Binary ( hashlib . sha256 ( access_key ) . digest ( ) ) , ) ) . fetchone ( )
2014-10-01 22:58:32 +00:00
except :
2013-02-19 00:11:43 +00:00
2014-10-01 22:58:32 +00:00
# we do not delete the registration_key (and hence the raw unhashed access_key)
# until the first attempt to create a session to make sure the user
# has the access_key saved
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
try : ( account_type_id , account_key , expires ) = self . _c . execute ( ' SELECT account_type_id, account_key, expiry FROM registration_keys WHERE access_key = ?; ' , ( sqlite3 . Binary ( access_key ) , ) ) . fetchone ( )
2014-10-01 22:58:32 +00:00
except : raise HydrusExceptions . ForbiddenException ( ' The service could not find that account in its database. ' )
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' DELETE FROM registration_keys WHERE access_key = ?; ' , ( sqlite3 . Binary ( access_key ) , ) )
2014-10-01 22:58:32 +00:00
#
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2014-10-01 22:58:32 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2014-10-01 22:58:32 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' INSERT INTO accounts ( service_id, account_key, hashed_access_key, account_type_id, created, expires, used_bytes, used_requests ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ? ); ' , ( service_id , sqlite3 . Binary ( account_key ) , sqlite3 . Binary ( hashlib . sha256 ( access_key ) . digest ( ) ) , account_type_id , now , expires , 0 , 0 ) )
2014-10-01 22:58:32 +00:00
return account_key
2014-11-20 01:48:04 +00:00
def _GetAccountKeyFromAccountId ( self , account_id ) :
2014-10-01 22:58:32 +00:00
2014-11-20 01:48:04 +00:00
try : ( account_key , ) = self . _c . execute ( ' SELECT account_key FROM accounts WHERE account_id = ?; ' , ( account_id , ) ) . fetchone ( )
2014-10-01 22:58:32 +00:00
except : raise HydrusExceptions . ForbiddenException ( ' The service could not find that account_id in its database. ' )
return account_key
2014-11-20 01:48:04 +00:00
def _GetAccountKeyFromIdentifier ( self , service_key , account_identifier ) :
2014-10-01 22:58:32 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2014-10-01 22:58:32 +00:00
if account_identifier . HasAccountKey ( ) :
account_key = account_identifier . GetData ( )
2014-11-20 01:48:04 +00:00
result = self . _c . execute ( ' SELECT 1 FROM accounts WHERE service_id = ? AND account_key = ?; ' , ( service_id , sqlite3 . Binary ( account_key ) ) ) . fetchone ( )
2014-10-01 22:58:32 +00:00
if result is None : raise HydrusExceptions . ForbiddenException ( ' The service could not find that hash in its database. ' )
2013-02-19 00:11:43 +00:00
2015-10-14 21:02:25 +00:00
elif account_identifier . HasContent ( ) :
content = account_identifier . GetData ( )
2013-02-19 00:11:43 +00:00
2015-10-14 21:02:25 +00:00
content_type = content . GetContentType ( )
content_data = content . GetContent ( )
if content_type == HC . CONTENT_TYPE_FILES :
2013-02-19 00:11:43 +00:00
2014-09-10 22:37:38 +00:00
try :
2015-10-14 21:02:25 +00:00
hash = content_data [ 0 ]
2014-09-10 22:37:38 +00:00
2014-11-20 01:48:04 +00:00
hash_id = self . _GetHashId ( hash )
2014-09-10 22:37:38 +00:00
2014-11-20 01:48:04 +00:00
result = self . _c . execute ( ' SELECT account_id FROM file_map WHERE service_id = ? AND hash_id = ?; ' , ( service_id , hash_id ) ) . fetchone ( )
2014-09-10 22:37:38 +00:00
2014-11-20 01:48:04 +00:00
if result is None : result = self . _c . execute ( ' SELECT account_id FROM file_petitions WHERE service_id = ? AND hash_id = ? AND status = ?; ' , ( service_id , hash_id , HC . DELETED ) ) . fetchone ( )
2014-09-10 22:37:38 +00:00
( account_id , ) = result
except : raise HydrusExceptions . ForbiddenException ( ' The service could not find that hash in its database. ' )
2013-02-19 00:11:43 +00:00
2015-10-14 21:02:25 +00:00
elif content_type == HC . CONTENT_TYPE_MAPPING :
2013-02-19 00:11:43 +00:00
2014-09-10 22:37:38 +00:00
try :
2015-10-14 21:02:25 +00:00
( tag , hash ) = content_data
2014-09-10 22:37:38 +00:00
2014-11-20 01:48:04 +00:00
hash_id = self . _GetHashId ( hash )
tag_id = self . _GetTagId ( tag )
2014-09-10 22:37:38 +00:00
2014-11-20 01:48:04 +00:00
( account_id , ) = self . _c . execute ( ' SELECT account_id FROM mappings WHERE service_id = ? AND tag_id = ? AND hash_id = ?; ' , ( service_id , tag_id , hash_id ) ) . fetchone ( )
2014-09-10 22:37:38 +00:00
except : raise HydrusExceptions . ForbiddenException ( ' The service could not find that mapping in its database. ' )
2014-01-01 20:01:00 +00:00
2014-11-20 01:48:04 +00:00
try : ( account_key , ) = self . _c . execute ( ' SELECT account_key FROM accounts WHERE account_id = ?; ' , ( account_id , ) ) . fetchone ( )
2013-07-24 20:26:00 +00:00
except : raise HydrusExceptions . ForbiddenException ( ' The service could not find that account in its database. ' )
2013-02-19 00:11:43 +00:00
2014-09-10 22:37:38 +00:00
return account_key
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
def _GetAccountIdFromContactKey ( self , service_id , contact_key ) :
2013-02-19 00:11:43 +00:00
try :
2014-11-20 01:48:04 +00:00
( account_id , ) = self . _c . execute ( ' SELECT account_id FROM contacts WHERE service_id = ? AND contact_key = ?; ' , ( service_id , sqlite3 . Binary ( contact_key ) ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
2016-02-17 22:06:47 +00:00
except :
raise HydrusExceptions . NotFoundException ( ' Could not find that contact key! ' )
2013-02-19 00:11:43 +00:00
return account_id
2014-11-20 01:48:04 +00:00
def _GetAccountId ( self , account_key ) :
2014-09-24 21:50:07 +00:00
2014-11-20 01:48:04 +00:00
result = self . _c . execute ( ' SELECT account_id FROM accounts WHERE account_key = ?; ' , ( sqlite3 . Binary ( account_key ) , ) ) . fetchone ( )
2014-09-24 21:50:07 +00:00
2014-10-01 22:58:32 +00:00
if result is None : raise HydrusExceptions . ForbiddenException ( ' The service could not find that account key in its database. ' )
2014-09-24 21:50:07 +00:00
( account_id , ) = result
return account_id
2014-11-20 01:48:04 +00:00
def _GetAccountInfo ( self , service_key , account_key ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
account = self . _GetAccount ( account_key )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
account_id = self . _GetAccountId ( account_key )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
service_type = self . _GetServiceType ( service_id )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
if service_type == HC . FILE_REPOSITORY : account_info = self . _GetAccountFileInfo ( service_id , account_id )
elif service_type == HC . TAG_REPOSITORY : account_info = self . _GetAccountMappingInfo ( service_id , account_id )
2013-10-02 22:06:06 +00:00
else : account_info = { }
account_info [ ' account ' ] = account
return account_info
2014-11-20 01:48:04 +00:00
def _GetAccountMappingInfo ( self , service_id , account_id ) :
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
num_mappings = len ( self . _c . execute ( ' SELECT 1 FROM mappings WHERE service_id = ? AND account_id = ? LIMIT 5000; ' , ( service_id , account_id ) ) . fetchall ( ) )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
result = self . _c . execute ( ' SELECT score FROM account_scores WHERE service_id = ? AND account_id = ? AND score_type = ?; ' , ( service_id , account_id , HC . SCORE_PETITION ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
if result is None : petition_score = 0
else : ( petition_score , ) = result
# crazy query here because two distinct columns
2014-11-20 01:48:04 +00:00
( num_petitions , ) = self . _c . execute ( ' SELECT COUNT( * ) FROM ( SELECT DISTINCT tag_id, reason_id FROM mapping_petitions WHERE service_id = ? AND account_id = ? AND status = ? ); ' , ( service_id , account_id , HC . PETITIONED ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
account_info = { }
account_info [ ' num_mappings ' ] = num_mappings
account_info [ ' petition_score ' ] = petition_score
account_info [ ' num_petitions ' ] = num_petitions
return account_info
2014-11-20 01:48:04 +00:00
def _GetAccountTypeId ( self , service_id , title ) :
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
result = self . _c . execute ( ' SELECT account_type_id FROM account_types WHERE service_id = ? AND title = ?; ' , ( service_id , title ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
2016-02-17 22:06:47 +00:00
if result is None :
raise HydrusExceptions . NotFoundException ( ' Could not find account title ' + HydrusData . ToUnicode ( title ) + ' in db for this service. ' )
2013-02-19 00:11:43 +00:00
( account_type_id , ) = result
return account_type_id
2014-11-20 01:48:04 +00:00
def _GetAccountTypes ( self , service_key ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
return [ account_type for ( account_type , ) in self . _c . execute ( ' SELECT account_type FROM account_types WHERE service_id = ?; ' , ( service_id , ) ) ]
def _GetFile ( self , hash ) :
2015-03-18 21:46:29 +00:00
path = ServerFiles . GetPath ( ' file ' , hash )
2015-03-04 22:44:32 +00:00
with open ( path , ' rb ' ) as f : file = f . read ( )
return file
def _GetFilePetition ( self , service_id ) :
2015-10-14 21:02:25 +00:00
data_type = HC . CONTENT_TYPE_FILES
status = HC . PETITIONED
action = HC . CONTENT_UPDATE_PETITION
result = self . _c . execute ( ' SELECT DISTINCT account_id, reason_id FROM file_petitions WHERE service_id = ? AND status = ? ORDER BY RANDOM() LIMIT 1; ' , ( service_id , status ) ) . fetchone ( )
2015-03-04 22:44:32 +00:00
2016-02-17 22:06:47 +00:00
if result is None :
raise HydrusExceptions . NotFoundException ( ' No petitions! ' )
2015-03-04 22:44:32 +00:00
( account_id , reason_id ) = result
account_key = self . _GetAccountKeyFromAccountId ( account_id )
2015-03-25 22:04:19 +00:00
petitioner_account_identifier = HydrusData . AccountIdentifier ( account_key = account_key )
2015-03-04 22:44:32 +00:00
reason = self . _GetReason ( reason_id )
2015-10-14 21:02:25 +00:00
hash_ids = [ hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT hash_id FROM file_petitions WHERE service_id = ? AND account_id = ? AND reason_id = ? AND status = ?; ' , ( service_id , account_id , reason_id , status ) ) ]
2015-03-04 22:44:32 +00:00
hashes = self . _GetHashes ( hash_ids )
2015-10-14 21:02:25 +00:00
contents = [ HydrusData . Content ( data_type , hashes ) ]
2015-03-04 22:44:32 +00:00
2015-10-14 21:02:25 +00:00
return HydrusData . ServerToClientPetition ( action , petitioner_account_identifier , reason , contents )
2015-03-04 22:44:32 +00:00
def _GetHash ( self , hash_id ) :
result = self . _c . execute ( ' SELECT hash FROM hashes WHERE hash_id = ?; ' , ( hash_id , ) ) . fetchone ( )
if result is None : raise Exception ( ' File hash error in database ' )
( hash , ) = result
return hash
2015-03-25 22:04:19 +00:00
def _GetHashes ( self , hash_ids ) : return [ hash for ( hash , ) in self . _c . execute ( ' SELECT hash FROM hashes WHERE hash_id IN ' + HydrusData . SplayListForDB ( hash_ids ) + ' ; ' ) ]
2015-03-04 22:44:32 +00:00
def _GetHashId ( self , hash ) :
result = self . _c . execute ( ' SELECT hash_id FROM hashes WHERE hash = ?; ' , ( sqlite3 . Binary ( hash ) , ) ) . fetchone ( )
if result is None :
self . _c . execute ( ' INSERT INTO hashes ( hash ) VALUES ( ? ); ' , ( sqlite3 . Binary ( hash ) , ) )
hash_id = self . _c . lastrowid
return hash_id
else :
( hash_id , ) = result
return hash_id
def _GetHashIds ( self , hashes ) :
hash_ids = set ( )
hashes_not_in_db = set ( )
for hash in hashes :
result = self . _c . execute ( ' SELECT hash_id FROM hashes WHERE hash = ?; ' , ( sqlite3 . Binary ( hash ) , ) ) . fetchone ( )
if result is None : hashes_not_in_db . add ( hash )
else :
( hash_id , ) = result
hash_ids . add ( hash_id )
if len ( hashes_not_in_db ) > 0 :
self . _c . executemany ( ' INSERT INTO hashes ( hash ) VALUES( ? ); ' , ( ( sqlite3 . Binary ( hash ) , ) for hash in hashes_not_in_db ) )
hash_ids . update ( self . _GetHashIds ( hashes ) )
return hash_ids
2015-03-25 22:04:19 +00:00
def _GetHashIdsToHashes ( self , hash_ids ) : return { hash_id : hash for ( hash_id , hash ) in self . _c . execute ( ' SELECT hash_id, hash FROM hashes WHERE hash_id IN ' + HydrusData . SplayListForDB ( hash_ids ) + ' ; ' ) }
2015-03-04 22:44:32 +00:00
def _GetIPTimestamp ( self , service_key , hash ) :
service_id = self . _GetServiceId ( service_key )
hash_id = self . _GetHashId ( hash )
2014-03-26 21:23:10 +00:00
2015-03-04 22:44:32 +00:00
result = self . _c . execute ( ' SELECT ip, timestamp FROM ip_addresses WHERE service_id = ? AND hash_id = ?; ' , ( service_id , hash_id ) ) . fetchone ( )
2014-03-26 21:23:10 +00:00
2015-03-04 22:44:32 +00:00
if result is None : raise HydrusExceptions . ForbiddenException ( ' Did not find ip information for that hash. ' )
2014-03-26 21:23:10 +00:00
2015-03-04 22:44:32 +00:00
return result
2014-03-26 21:23:10 +00:00
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
def _GetMessagingSessions ( self ) :
2013-11-27 18:27:11 +00:00
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-11-27 18:27:11 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' DELETE FROM messaging_sessions WHERE ? > expiry; ' , ( now , ) )
2013-11-27 18:27:11 +00:00
2015-03-25 22:04:19 +00:00
existing_session_ids = HydrusData . BuildKeyToListDict ( [ ( service_id , ( session_key , account_id , identifier , name , expires ) ) for ( service_id , session_key , account_id , identifier , name , expires ) in self . _c . execute ( ' SELECT service_id, session_key, account_id, identifier, name, expiry FROM messaging_sessions; ' ) ] )
2013-11-27 18:27:11 +00:00
existing_sessions = { }
for ( service_id , tuples ) in existing_session_ids . items ( ) :
2014-11-20 01:48:04 +00:00
service_key = self . _GetServiceKey ( service_id )
2013-11-27 18:27:11 +00:00
processed_tuples = [ ]
2014-10-01 22:58:32 +00:00
for ( account_id , identifier , name , expires ) in tuples :
2013-11-27 18:27:11 +00:00
2014-11-20 01:48:04 +00:00
account_key = self . _GetAccountKeyFromAccountId ( account_id )
2014-09-10 22:37:38 +00:00
2014-11-20 01:48:04 +00:00
account = self . _GetAccount ( account_key )
2013-11-27 18:27:11 +00:00
2014-10-01 22:58:32 +00:00
processed_tuples . append ( ( account , name , expires ) )
2013-11-27 18:27:11 +00:00
2014-08-27 22:15:22 +00:00
existing_sessions [ service_key ] = processed_tuples
2013-11-27 18:27:11 +00:00
return existing_sessions
2014-11-20 01:48:04 +00:00
def _GetNumPetitions ( self , service_key ) :
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
service_type = self . _GetServiceType ( service_id )
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
if service_type == HC . FILE_REPOSITORY :
2014-11-20 01:48:04 +00:00
( num_petitions , ) = self . _c . execute ( ' SELECT COUNT( * ) FROM ( SELECT DISTINCT account_id, reason_id FROM file_petitions WHERE service_id = ? AND status = ? ); ' , ( service_id , HC . PETITIONED ) ) . fetchone ( )
2013-10-02 22:06:06 +00:00
elif service_type == HC . TAG_REPOSITORY :
2014-11-20 01:48:04 +00:00
( num_mapping_petitions , ) = self . _c . execute ( ' SELECT COUNT( * ) FROM ( SELECT DISTINCT account_id, tag_id, reason_id FROM mapping_petitions WHERE service_id = ? AND status = ? ); ' , ( service_id , HC . PETITIONED ) ) . fetchone ( )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
( num_tag_sibling_petitions , ) = self . _c . execute ( ' SELECT COUNT( * ) FROM tag_siblings WHERE service_id = ? AND status IN ( ?, ? ); ' , ( service_id , HC . PENDING , HC . PETITIONED ) ) . fetchone ( )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
( num_tag_parent_petitions , ) = self . _c . execute ( ' SELECT COUNT( * ) FROM tag_parents WHERE service_id = ? AND status IN ( ?, ? ); ' , ( service_id , HC . PENDING , HC . PETITIONED ) ) . fetchone ( )
2013-10-02 22:06:06 +00:00
num_petitions = num_mapping_petitions + num_tag_sibling_petitions + num_tag_parent_petitions
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
return num_petitions
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
def _GetOptions ( self , service_key ) :
2013-05-01 17:21:53 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2013-05-01 17:21:53 +00:00
2014-11-20 01:48:04 +00:00
( options , ) = self . _c . execute ( ' SELECT options FROM services WHERE service_id = ?; ' , ( service_id , ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
return options
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
def _GetPetition ( self , service_key ) :
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
service_type = self . _GetServiceType ( service_id )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
if service_type == HC . FILE_REPOSITORY : petition = self . _GetFilePetition ( service_id )
elif service_type == HC . TAG_REPOSITORY : petition = self . _GetTagPetition ( service_id )
2013-02-19 00:11:43 +00:00
2013-10-09 18:13:42 +00:00
return petition
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
def _GetReason ( self , reason_id ) :
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
result = self . _c . execute ( ' SELECT reason FROM reasons WHERE reason_id = ?; ' , ( reason_id , ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
if result is None : raise Exception ( ' Reason error in database ' )
( reason , ) = result
return reason
2014-11-20 01:48:04 +00:00
def _GetReasonId ( self , reason ) :
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
result = self . _c . execute ( ' SELECT reason_id FROM reasons WHERE reason = ?; ' , ( reason , ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
if result is None :
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' INSERT INTO reasons ( reason ) VALUES ( ? ); ' , ( reason , ) )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
reason_id = self . _c . lastrowid
2013-02-19 00:11:43 +00:00
return reason_id
else :
( reason_id , ) = result
return reason_id
2014-11-20 01:48:04 +00:00
def _GetServiceId ( self , service_key ) :
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
result = self . _c . execute ( ' SELECT service_id FROM services WHERE service_key = ?; ' , ( sqlite3 . Binary ( service_key ) , ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
if result is None : raise Exception ( ' Service id error in database ' )
( service_id , ) = result
return service_id
2015-03-25 22:04:19 +00:00
def _GetServiceIds ( self , limited_types = HC . ALL_SERVICES ) : return [ service_id for ( service_id , ) in self . _c . execute ( ' SELECT service_id FROM services WHERE type IN ' + HydrusData . SplayListForDB ( limited_types ) + ' ; ' ) ]
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
def _GetServiceKey ( self , service_id ) :
2013-03-15 02:38:12 +00:00
2014-11-20 01:48:04 +00:00
( service_key , ) = self . _c . execute ( ' SELECT service_key FROM services WHERE service_id = ?; ' , ( service_id , ) ) . fetchone ( )
2013-03-15 02:38:12 +00:00
2014-08-27 22:15:22 +00:00
return service_key
2013-03-15 02:38:12 +00:00
2015-03-25 22:04:19 +00:00
def _GetServiceKeys ( self , limited_types = HC . ALL_SERVICES ) : return [ service_key for ( service_key , ) in self . _c . execute ( ' SELECT service_key FROM services WHERE type IN ' + HydrusData . SplayListForDB ( limited_types ) + ' ; ' ) ]
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
def _GetServiceType ( self , service_id ) :
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
result = self . _c . execute ( ' SELECT type FROM services WHERE service_id = ?; ' , ( service_id , ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
if result is None : raise Exception ( ' Service id error in database ' )
( service_type , ) = result
return service_type
2014-11-20 01:48:04 +00:00
def _GetServiceInfo ( self , service_key ) : return self . _c . execute ( ' SELECT type, options FROM services WHERE service_key = ?; ' , ( sqlite3 . Binary ( service_key ) , ) ) . fetchone ( )
2014-09-17 21:28:26 +00:00
2015-03-25 22:04:19 +00:00
def _GetServicesInfo ( self , limited_types = HC . ALL_SERVICES ) : return self . _c . execute ( ' SELECT service_key, type, options FROM services WHERE type IN ' + HydrusData . SplayListForDB ( limited_types ) + ' ; ' ) . fetchall ( )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
def _GetSessions ( self ) :
2013-03-15 02:38:12 +00:00
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-03-15 02:38:12 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' DELETE FROM sessions WHERE ? > expiry; ' , ( now , ) )
2013-03-15 02:38:12 +00:00
sessions = [ ]
2014-11-20 01:48:04 +00:00
results = self . _c . execute ( ' SELECT session_key, service_id, account_id, expiry FROM sessions; ' ) . fetchall ( )
2013-03-15 02:38:12 +00:00
2014-08-27 22:15:22 +00:00
service_ids_to_service_keys = { }
2013-03-15 02:38:12 +00:00
2014-10-01 22:58:32 +00:00
account_ids_to_accounts = { }
2014-09-24 21:50:07 +00:00
2014-10-01 22:58:32 +00:00
for ( session_key , service_id , account_id , expires ) in results :
2013-03-15 02:38:12 +00:00
2014-11-20 01:48:04 +00:00
if service_id not in service_ids_to_service_keys : service_ids_to_service_keys [ service_id ] = self . _GetServiceKey ( service_id )
2013-03-15 02:38:12 +00:00
2014-08-27 22:15:22 +00:00
service_key = service_ids_to_service_keys [ service_id ]
2013-03-15 02:38:12 +00:00
2014-10-01 22:58:32 +00:00
if account_id not in account_ids_to_accounts :
2014-09-24 21:50:07 +00:00
2014-11-20 01:48:04 +00:00
account_key = self . _GetAccountKeyFromAccountId ( account_id )
2014-09-24 21:50:07 +00:00
2014-11-20 01:48:04 +00:00
account = self . _GetAccount ( account_key )
2014-09-24 21:50:07 +00:00
2014-10-01 22:58:32 +00:00
account_ids_to_accounts [ account_id ] = account
2014-09-24 21:50:07 +00:00
2014-09-10 22:37:38 +00:00
2014-10-01 22:58:32 +00:00
account = account_ids_to_accounts [ account_id ]
2013-10-02 22:06:06 +00:00
2014-10-01 22:58:32 +00:00
sessions . append ( ( session_key , service_key , account , expires ) )
2013-03-15 02:38:12 +00:00
return sessions
2014-11-20 01:48:04 +00:00
def _GetStats ( self , service_key ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2013-10-02 22:06:06 +00:00
stats = { }
2014-11-20 01:48:04 +00:00
( stats [ ' num_accounts ' ] , ) = self . _c . execute ( ' SELECT COUNT( * ) FROM accounts WHERE service_id = ?; ' , ( service_id , ) ) . fetchone ( )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
( stats [ ' num_banned ' ] , ) = self . _c . execute ( ' SELECT COUNT( * ) FROM bans WHERE service_id = ?; ' , ( service_id , ) ) . fetchone ( )
2013-10-02 22:06:06 +00:00
return stats
2015-03-04 22:44:32 +00:00
def _GetTag ( self , tag_id ) :
result = self . _c . execute ( ' SELECT tag FROM tags WHERE tag_id = ?; ' , ( tag_id , ) ) . fetchone ( )
if result is None : raise Exception ( ' Tag error in database ' )
( tag , ) = result
return tag
def _GetTagId ( self , tag ) :
2015-03-25 22:04:19 +00:00
tag = HydrusTags . CleanTag ( tag )
2015-03-04 22:44:32 +00:00
2015-03-25 22:04:19 +00:00
HydrusTags . CheckTagNotEmpty ( tag )
2015-03-04 22:44:32 +00:00
result = self . _c . execute ( ' SELECT tag_id FROM tags WHERE tag = ?; ' , ( tag , ) ) . fetchone ( )
if result is None :
self . _c . execute ( ' INSERT INTO tags ( tag ) VALUES ( ? ); ' , ( tag , ) )
tag_id = self . _c . lastrowid
return tag_id
else :
( tag_id , ) = result
return tag_id
def _GetTagPetition ( self , service_id ) :
2015-10-14 21:02:25 +00:00
content_types = [ HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_TYPE_TAG_SIBLINGS , HC . CONTENT_TYPE_TAG_PARENTS ]
random . shuffle ( content_types )
2015-03-04 22:44:32 +00:00
2015-10-14 21:02:25 +00:00
contents = [ ]
2015-03-04 22:44:32 +00:00
2015-10-14 21:02:25 +00:00
for content_type in content_types :
2015-03-04 22:44:32 +00:00
2015-10-14 21:02:25 +00:00
if content_type == HC . CONTENT_TYPE_MAPPINGS :
2015-03-04 22:44:32 +00:00
2015-10-14 21:02:25 +00:00
status = HC . PETITIONED
2015-03-04 22:44:32 +00:00
action = HC . CONTENT_UPDATE_PETITION
2015-10-14 21:02:25 +00:00
result = self . _c . execute ( ' SELECT account_id, reason_id FROM mapping_petitions WHERE service_id = ? AND status = ? ORDER BY RANDOM() LIMIT 1; ' , ( service_id , status ) ) . fetchone ( )
2015-03-04 22:44:32 +00:00
2015-10-14 21:02:25 +00:00
if result is None : continue
( account_id , reason_id ) = result
2015-03-04 22:44:32 +00:00
2015-10-14 21:02:25 +00:00
tag_ids_to_hash_ids = HydrusData . BuildKeyToListDict ( self . _c . execute ( ' SELECT tag_id, hash_id FROM mapping_petitions WHERE service_id = ? AND account_id = ? AND reason_id = ? AND status = ?; ' , ( service_id , account_id , reason_id , status ) ) )
2015-03-04 22:44:32 +00:00
2015-10-14 21:02:25 +00:00
for ( tag_id , hash_ids ) in tag_ids_to_hash_ids . items ( ) :
tag = self . _GetTag ( tag_id )
hashes = self . _GetHashes ( hash_ids )
content = HydrusData . Content ( content_type , ( tag , hashes ) )
contents . append ( content )
2015-03-04 22:44:32 +00:00
2015-10-14 21:02:25 +00:00
elif content_type in ( HC . CONTENT_TYPE_TAG_SIBLINGS , HC . CONTENT_TYPE_TAG_PARENTS ) :
2015-03-04 22:44:32 +00:00
2015-10-14 21:02:25 +00:00
if content_type == HC . CONTENT_TYPE_TAG_SIBLINGS : result = self . _c . execute ( ' SELECT account_id, reason_id, status FROM tag_siblings WHERE service_id = ? AND status IN ( ?, ? ) ORDER BY RANDOM() LIMIT 1; ' , ( service_id , HC . PENDING , HC . PETITIONED ) ) . fetchone ( )
elif content_type == HC . CONTENT_TYPE_TAG_PARENTS : result = self . _c . execute ( ' SELECT account_id, reason_id, status FROM tag_parents WHERE service_id = ? AND status IN ( ?, ? ) ORDER BY RANDOM() LIMIT 1; ' , ( service_id , HC . PENDING , HC . PETITIONED ) ) . fetchone ( )
2015-03-04 22:44:32 +00:00
if result is None : continue
2015-10-14 21:02:25 +00:00
( account_id , reason_id , status ) = result
2015-03-04 22:44:32 +00:00
2015-10-14 21:02:25 +00:00
if status == HC . PENDING : action = HC . CONTENT_UPDATE_PEND
elif status == HC . PETITIONED : action = HC . CONTENT_UPDATE_PETITION
2015-03-04 22:44:32 +00:00
2015-10-14 21:02:25 +00:00
if content_type == HC . CONTENT_TYPE_TAG_SIBLINGS : tag_pairs = self . _c . execute ( ' SELECT old_tag_id, new_tag_id FROM tag_siblings WHERE service_id = ? AND account_id = ? AND reason_id = ? AND status = ?; ' , ( service_id , account_id , reason_id , status ) ) . fetchall ( )
elif content_type == HC . CONTENT_TYPE_TAG_PARENTS : tag_pairs = self . _c . execute ( ' SELECT old_tag_id, new_tag_id FROM tag_parents WHERE service_id = ? AND account_id = ? AND reason_id = ? AND status = ?; ' , ( service_id , account_id , reason_id , status ) ) . fetchall ( )
2015-03-04 22:44:32 +00:00
2015-10-14 21:02:25 +00:00
for ( old_tag_id , new_tag_id ) in tag_pairs :
old_tag = self . _GetTag ( old_tag_id )
new_tag = self . _GetTag ( new_tag_id )
content = HydrusData . Content ( content_type , ( old_tag , new_tag ) )
contents . append ( content )
2015-03-04 22:44:32 +00:00
account_key = self . _GetAccountKeyFromAccountId ( account_id )
2015-03-25 22:04:19 +00:00
petitioner_account_identifier = HydrusData . AccountIdentifier ( account_key = account_key )
2015-03-04 22:44:32 +00:00
reason = self . _GetReason ( reason_id )
2015-10-14 21:02:25 +00:00
return HydrusData . ServerToClientPetition ( action , petitioner_account_identifier , reason , contents )
2015-03-04 22:44:32 +00:00
raise HydrusExceptions . NotFoundException ( ' No petitions! ' )
def _GetThumbnail ( self , hash ) :
2015-03-18 21:46:29 +00:00
path = ServerFiles . GetPath ( ' thumbnail ' , hash )
2015-03-04 22:44:32 +00:00
with open ( path , ' rb ' ) as f : thumbnail = f . read ( )
return thumbnail
2014-11-20 01:48:04 +00:00
def _GetUpdateEnds ( self ) :
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
service_ids = self . _GetServiceIds ( HC . REPOSITORIES )
2013-02-19 00:11:43 +00:00
2014-03-26 21:23:10 +00:00
results = { }
2013-10-02 22:06:06 +00:00
2014-03-26 21:23:10 +00:00
for service_id in service_ids :
2014-11-20 01:48:04 +00:00
( end , ) = self . _c . execute ( ' SELECT end FROM update_cache WHERE service_id = ? ORDER BY end DESC LIMIT 1; ' , ( service_id , ) ) . fetchone ( )
2014-03-26 21:23:10 +00:00
2014-11-20 01:48:04 +00:00
service_key = self . _GetServiceKey ( service_id )
2014-03-26 21:23:10 +00:00
2014-08-27 22:15:22 +00:00
results [ service_key ] = end
2014-03-26 21:23:10 +00:00
2013-10-02 22:06:06 +00:00
2014-03-26 21:23:10 +00:00
return results
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
def _InitAdmin ( self ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
if self . _c . execute ( ' SELECT 1 FROM accounts; ' ) . fetchone ( ) is not None : raise HydrusExceptions . ForbiddenException ( ' This server is already initialised! ' )
2013-10-02 22:06:06 +00:00
num = 1
title = ' server admin '
2014-11-20 01:48:04 +00:00
( registration_key , ) = self . _GenerateRegistrationKeys ( HC . SERVER_ADMIN_KEY , num , title )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
access_key = self . _GetAccessKey ( registration_key )
2013-10-02 22:06:06 +00:00
return access_key
2015-04-22 22:57:25 +00:00
def _InitCaches ( self ) :
self . _over_monthly_data = False
self . _services_over_monthly_data = set ( )
2016-04-20 20:42:21 +00:00
def _InitExternalDatabases ( self ) :
2016-05-11 18:16:39 +00:00
self . _db_filenames [ ' external_mappings ' ] = ' server.mappings.db '
self . _db_filenames [ ' external_master ' ] = ' server.master.db '
2016-04-20 20:42:21 +00:00
2015-06-03 21:05:13 +00:00
def _IterateFileUpdateContentData ( self , service_id , begin , end ) :
#
files_info = [ ( hash_id , size , mime , timestamp , width , height , duration , num_frames , num_words ) for ( hash_id , size , mime , timestamp , width , height , duration , num_frames , num_words ) in self . _c . execute ( ' SELECT hash_id, size, mime, timestamp, width, height, duration, num_frames, num_words FROM file_map, files_info USING ( hash_id ) WHERE service_id = ? AND timestamp BETWEEN ? AND ?; ' , ( service_id , begin , end ) ) ]
for block_of_files_info in HydrusData . SplitListIntoChunks ( files_info , 10000 ) :
hash_ids = { file_info [ 0 ] for file_info in block_of_files_info }
hash_ids_to_hashes = self . _GetHashIdsToHashes ( hash_ids )
2015-10-14 21:02:25 +00:00
yield ( HC . CONTENT_TYPE_FILES , HC . CONTENT_UPDATE_ADD , block_of_files_info , hash_ids_to_hashes , len ( hash_ids ) )
2015-06-03 21:05:13 +00:00
#
deleted_files_info = [ hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT hash_id FROM file_petitions WHERE service_id = ? AND timestamp BETWEEN ? AND ? AND status = ?; ' , ( service_id , begin , end , HC . DELETED ) ) ]
for block_of_deleted_files_info in HydrusData . SplitListIntoChunks ( deleted_files_info , 10000 ) :
hash_ids = block_of_deleted_files_info
hash_ids_to_hashes = self . _GetHashIdsToHashes ( hash_ids )
2015-10-14 21:02:25 +00:00
yield ( HC . CONTENT_TYPE_FILES , HC . CONTENT_UPDATE_DELETE , block_of_deleted_files_info , hash_ids_to_hashes , len ( hash_ids ) )
2015-06-03 21:05:13 +00:00
def _IterateTagUpdateContentData ( self , service_id , begin , end ) :
# mappings
mappings_dict = HydrusData . BuildKeyToListDict ( self . _c . execute ( ' SELECT tag, hash_id FROM tags, mappings USING ( tag_id ) WHERE service_id = ? AND timestamp BETWEEN ? AND ?; ' , ( service_id , begin , end ) ) )
for ( tag , hash_ids ) in mappings_dict . items ( ) :
for block_of_hash_ids in HydrusData . SplitListIntoChunks ( hash_ids , 10000 ) :
hash_ids_to_hashes = self . _GetHashIdsToHashes ( block_of_hash_ids )
2015-10-14 21:02:25 +00:00
yield ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_ADD , [ ( tag , block_of_hash_ids ) ] , hash_ids_to_hashes , len ( block_of_hash_ids ) )
2015-06-03 21:05:13 +00:00
#
deleted_mappings_dict = HydrusData . BuildKeyToListDict ( self . _c . execute ( ' SELECT tag, hash_id FROM tags, mapping_petitions USING ( tag_id ) WHERE service_id = ? AND timestamp BETWEEN ? AND ? AND status = ?; ' , ( service_id , begin , end , HC . DELETED ) ) )
for ( tag , hash_ids ) in deleted_mappings_dict . items ( ) :
for block_of_hash_ids in HydrusData . SplitListIntoChunks ( hash_ids , 10000 ) :
hash_ids_to_hashes = self . _GetHashIdsToHashes ( block_of_hash_ids )
2015-10-14 21:02:25 +00:00
yield ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_DELETE , [ ( tag , block_of_hash_ids ) ] , hash_ids_to_hashes , len ( block_of_hash_ids ) )
2015-06-03 21:05:13 +00:00
# tag siblings
tag_sibling_ids = self . _c . execute ( ' SELECT old_tag_id, new_tag_id FROM tag_siblings WHERE service_id = ? AND status = ? AND timestamp BETWEEN ? AND ?; ' , ( service_id , HC . CURRENT , begin , end ) ) . fetchall ( )
for block_of_tag_sibling_ids in HydrusData . SplitListIntoChunks ( tag_sibling_ids , 10000 ) :
tag_siblings = [ ( self . _GetTag ( old_tag_id ) , self . _GetTag ( new_tag_id ) ) for ( old_tag_id , new_tag_id ) in block_of_tag_sibling_ids ]
hash_ids_to_hashes = { }
2015-10-14 21:02:25 +00:00
yield ( HC . CONTENT_TYPE_TAG_SIBLINGS , HC . CONTENT_UPDATE_ADD , tag_siblings , hash_ids_to_hashes , len ( tag_siblings ) )
2015-06-03 21:05:13 +00:00
#
deleted_tag_sibling_ids = self . _c . execute ( ' SELECT old_tag_id, new_tag_id FROM tag_siblings WHERE service_id = ? AND status = ? AND timestamp BETWEEN ? AND ?; ' , ( service_id , HC . DELETED , begin , end ) ) . fetchall ( )
for block_of_deleted_tag_sibling_ids in HydrusData . SplitListIntoChunks ( deleted_tag_sibling_ids , 10000 ) :
deleted_tag_siblings = [ ( self . _GetTag ( old_tag_id ) , self . _GetTag ( new_tag_id ) ) for ( old_tag_id , new_tag_id ) in block_of_deleted_tag_sibling_ids ]
hash_ids_to_hashes = { }
2015-10-14 21:02:25 +00:00
yield ( HC . CONTENT_TYPE_TAG_SIBLINGS , HC . CONTENT_UPDATE_DELETE , deleted_tag_siblings , hash_ids_to_hashes , len ( deleted_tag_siblings ) )
2015-06-03 21:05:13 +00:00
# tag parents
tag_parent_ids = self . _c . execute ( ' SELECT old_tag_id, new_tag_id FROM tag_parents WHERE service_id = ? AND status = ? AND timestamp BETWEEN ? AND ?; ' , ( service_id , HC . CURRENT , begin , end ) ) . fetchall ( )
for block_of_tag_parent_ids in HydrusData . SplitListIntoChunks ( tag_parent_ids , 10000 ) :
tag_parents = [ ( self . _GetTag ( old_tag_id ) , self . _GetTag ( new_tag_id ) ) for ( old_tag_id , new_tag_id ) in block_of_tag_parent_ids ]
hash_ids_to_hashes = { }
2015-10-14 21:02:25 +00:00
yield ( HC . CONTENT_TYPE_TAG_PARENTS , HC . CONTENT_UPDATE_ADD , tag_parents , hash_ids_to_hashes , len ( tag_parents ) )
2015-06-03 21:05:13 +00:00
#
deleted_tag_parent_ids = self . _c . execute ( ' SELECT old_tag_id, new_tag_id FROM tag_parents WHERE service_id = ? AND status = ? AND timestamp BETWEEN ? AND ?; ' , ( service_id , HC . DELETED , begin , end ) ) . fetchall ( )
for block_of_deleted_tag_parent_ids in HydrusData . SplitListIntoChunks ( deleted_tag_parent_ids , 10000 ) :
deleted_tag_parents = [ ( self . _GetTag ( old_tag_id ) , self . _GetTag ( new_tag_id ) ) for ( old_tag_id , new_tag_id ) in block_of_deleted_tag_parent_ids ]
hash_ids_to_hashes = { }
2015-10-14 21:02:25 +00:00
yield ( HC . CONTENT_TYPE_TAG_PARENTS , HC . CONTENT_UPDATE_DELETE , deleted_tag_parents , hash_ids_to_hashes , len ( deleted_tag_parents ) )
2015-06-03 21:05:13 +00:00
2015-04-22 22:57:25 +00:00
def _ManageDBError ( self , job , e ) :
( exception_type , value , tb ) = sys . exc_info ( )
new_e = type ( e ) ( os . linesep . join ( traceback . format_exception ( exception_type , value , tb ) ) )
job . PutResult ( new_e )
2014-11-20 01:48:04 +00:00
def _ModifyAccount ( self , service_key , admin_account_key , action , subject_account_keys , kwargs ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
admin_account_id = self . _GetAccountId ( admin_account_key )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
subject_account_ids = [ self . _GetAccountId ( subject_account_key ) for subject_account_key in subject_account_keys ]
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
if action in ( HC . BAN , HC . SUPERBAN ) :
2013-10-30 22:28:06 +00:00
reason = kwargs [ ' reason ' ]
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
reason_id = self . _GetReasonId ( reason )
2013-10-02 22:06:06 +00:00
2015-02-25 19:34:30 +00:00
if ' lifetime ' in kwargs : lifetime = kwargs [ ' lifetime ' ]
2013-12-04 22:44:16 +00:00
else : lifetime = None
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
self . _Ban ( service_id , action , admin_account_id , subject_account_ids , reason_id , lifetime ) # fold ban and superban together, yo
2013-10-02 22:06:06 +00:00
else :
2015-02-25 19:34:30 +00:00
admin_account = self . _GetAccount ( admin_account_key )
2013-10-30 22:28:06 +00:00
admin_account . CheckPermission ( HC . GENERAL_ADMIN ) # special case, don't let manage_users people do these:
2013-10-02 22:06:06 +00:00
if action == HC . CHANGE_ACCOUNT_TYPE :
2013-10-30 22:28:06 +00:00
title = kwargs [ ' title ' ]
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
account_type_id = self . _GetAccountTypeId ( service_id , title )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
self . _ChangeAccountType ( subject_account_ids , account_type_id )
2013-10-02 22:06:06 +00:00
2014-10-01 22:58:32 +00:00
elif action == HC . ADD_TO_EXPIRES :
2013-10-02 22:06:06 +00:00
2013-12-04 22:44:16 +00:00
timespan = kwargs [ ' timespan ' ]
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
self . _AddToExpires ( subject_account_ids , timespan )
2013-10-02 22:06:06 +00:00
2014-10-01 22:58:32 +00:00
elif action == HC . SET_EXPIRES :
2013-10-02 22:06:06 +00:00
2014-10-01 22:58:32 +00:00
expires = kwargs [ ' expires ' ]
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
self . _SetExpires ( subject_account_ids , expires )
2013-10-02 22:06:06 +00:00
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
def _ModifyAccountTypes ( self , service_key , edit_log ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2013-02-19 00:11:43 +00:00
for ( action , details ) in edit_log :
2013-10-09 18:13:42 +00:00
if action == HC . ADD :
2013-02-19 00:11:43 +00:00
account_type = details
title = account_type . GetTitle ( )
2015-11-04 22:30:28 +00:00
if self . _AccountTypeExists ( service_id , title ) : raise HydrusExceptions . ForbiddenException ( ' Already found account type ' + HydrusData . ToUnicode ( title ) + ' in the db for this service, so could not add! ' )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' INSERT OR IGNORE INTO account_types ( service_id, title, account_type ) VALUES ( ?, ?, ? ); ' , ( service_id , title , account_type ) )
2013-02-19 00:11:43 +00:00
2013-10-09 18:13:42 +00:00
elif action == HC . DELETE :
2013-02-19 00:11:43 +00:00
( title , new_title ) = details
2014-11-20 01:48:04 +00:00
account_type_id = self . _GetAccountTypeId ( service_id , title )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
new_account_type_id = self . _GetAccountTypeId ( service_id , new_title )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' UPDATE accounts SET account_type_id = ? WHERE account_type_id = ?; ' , ( new_account_type_id , account_type_id ) )
self . _c . execute ( ' UPDATE registration_keys SET account_type_id = ? WHERE account_type_id = ?; ' , ( new_account_type_id , account_type_id ) )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' DELETE FROM account_types WHERE account_type_id = ?; ' , ( account_type_id , ) )
2013-02-19 00:11:43 +00:00
2013-10-09 18:13:42 +00:00
elif action == HC . EDIT :
2013-02-19 00:11:43 +00:00
( old_title , account_type ) = details
title = account_type . GetTitle ( )
2015-11-04 22:30:28 +00:00
if old_title != title and self . _AccountTypeExists ( service_id , title ) : raise HydrusExceptions . ForbiddenException ( ' Already found account type ' + HydrusData . ToUnicode ( title ) + ' in the database, so could not rename ' + HydrusData . ToUnicode ( old_title ) + ' ! ' )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
account_type_id = self . _GetAccountTypeId ( service_id , old_title )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' UPDATE account_types SET title = ?, account_type = ? WHERE account_type_id = ?; ' , ( title , account_type , account_type_id ) )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
def _ModifyServices ( self , account_key , edit_log ) :
2013-10-02 22:06:06 +00:00
2016-02-03 22:12:53 +00:00
self . _c . execute ( ' COMMIT; ' )
2016-03-02 21:00:30 +00:00
if not self . _fast_big_transaction_wal :
self . _c . execute ( ' PRAGMA journal_mode = TRUNCATE; ' )
2016-02-03 22:12:53 +00:00
self . _c . execute ( ' PRAGMA foreign_keys = ON; ' )
self . _c . execute ( ' BEGIN IMMEDIATE; ' )
2014-11-20 01:48:04 +00:00
account_id = self . _GetAccountId ( account_key )
2013-02-19 00:11:43 +00:00
2014-08-27 22:15:22 +00:00
service_keys_to_access_keys = { }
2013-11-27 18:27:11 +00:00
2015-03-25 22:04:19 +00:00
now = HydrusData . GetNow ( )
2013-02-19 00:11:43 +00:00
2013-10-09 18:13:42 +00:00
for ( action , data ) in edit_log :
2013-10-02 22:06:06 +00:00
2013-10-09 18:13:42 +00:00
if action == HC . ADD :
2013-02-19 00:11:43 +00:00
2014-08-27 22:15:22 +00:00
( service_key , service_type , options ) = data
2013-10-09 18:13:42 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' INSERT INTO services ( service_key, type, options ) VALUES ( ?, ?, ? ); ' , ( sqlite3 . Binary ( service_key ) , service_type , options ) )
2013-10-09 18:13:42 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _c . lastrowid
2013-10-09 18:13:42 +00:00
2015-03-25 22:04:19 +00:00
service_admin_account_type = HydrusData . AccountType ( ' service admin ' , [ HC . GET_DATA , HC . POST_DATA , HC . POST_PETITIONS , HC . RESOLVE_PETITIONS , HC . MANAGE_USERS , HC . GENERAL_ADMIN ] , ( None , None ) )
2013-10-09 18:13:42 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' INSERT INTO account_types ( service_id, title, account_type ) VALUES ( ?, ?, ? ); ' , ( service_id , ' service admin ' , service_admin_account_type ) )
2013-10-09 18:13:42 +00:00
2014-11-20 01:48:04 +00:00
[ registration_key ] = self . _GenerateRegistrationKeys ( service_key , 1 , ' service admin ' , None )
2013-11-27 18:27:11 +00:00
2014-11-20 01:48:04 +00:00
access_key = self . _GetAccessKey ( registration_key )
2013-11-27 18:27:11 +00:00
2014-08-27 22:15:22 +00:00
service_keys_to_access_keys [ service_key ] = access_key
2013-10-09 18:13:42 +00:00
if service_type in HC . REPOSITORIES :
2015-11-04 22:30:28 +00:00
update_dir = ServerFiles . GetExpectedUpdateDir ( service_key )
if not os . path . exists ( update_dir ) :
2016-01-06 21:17:20 +00:00
os . makedirs ( update_dir )
2015-11-04 22:30:28 +00:00
2013-10-09 18:13:42 +00:00
begin = 0
2015-03-25 22:04:19 +00:00
end = HydrusData . GetNow ( )
2013-10-09 18:13:42 +00:00
2014-11-20 01:48:04 +00:00
self . _CreateUpdate ( service_key , begin , end )
2013-10-09 18:13:42 +00:00
2014-09-17 21:28:26 +00:00
self . pub_after_commit ( ' action_service ' , service_key , ' start ' )
2013-10-09 18:13:42 +00:00
elif action == HC . DELETE :
2014-08-27 22:15:22 +00:00
service_key = data
2013-10-09 18:13:42 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2013-10-09 18:13:42 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' DELETE FROM services WHERE service_id = ?; ' , ( service_id , ) )
2016-03-30 22:56:50 +00:00
self . _c . execute ( ' DELETE FROM mappings WHERE service_id = ?; ' , ( service_id , ) )
self . _c . execute ( ' DELETE FROM mapping_petitions WHERE service_id = ?; ' , ( service_id , ) )
2013-10-09 18:13:42 +00:00
2015-11-04 22:30:28 +00:00
update_dir = ServerFiles . GetExpectedUpdateDir ( service_key )
if os . path . exists ( update_dir ) :
2015-11-25 22:00:57 +00:00
HydrusPaths . DeletePath ( update_dir )
2015-11-04 22:30:28 +00:00
2014-09-17 21:28:26 +00:00
self . pub_after_commit ( ' action_service ' , service_key , ' stop ' )
2013-10-09 18:13:42 +00:00
elif action == HC . EDIT :
2014-08-27 22:15:22 +00:00
( service_key , service_type , options ) = data
2013-10-09 18:13:42 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2013-10-09 18:13:42 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' UPDATE services SET options = ? WHERE service_id = ?; ' , ( options , service_id ) )
2013-10-09 18:13:42 +00:00
2014-09-17 21:28:26 +00:00
self . pub_after_commit ( ' action_service ' , service_key , ' restart ' )
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
2016-02-03 22:12:53 +00:00
self . _c . execute ( ' COMMIT; ' )
self . _InitDBCursor ( )
self . _c . execute ( ' BEGIN IMMEDIATE; ' )
2014-08-27 22:15:22 +00:00
return service_keys_to_access_keys
2013-11-27 18:27:11 +00:00
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
def _ProcessUpdate ( self , service_key , account_key , update ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
service_id = self . _GetServiceId ( service_key )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
account_id = self . _GetAccountId ( account_key )
2014-10-01 22:58:32 +00:00
2014-11-20 01:48:04 +00:00
account = self . _GetAccount ( account_key )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
service_type = self . _GetServiceType ( service_id )
2013-10-02 22:06:06 +00:00
if service_type == HC . FILE_REPOSITORY :
if account . HasPermission ( HC . RESOLVE_PETITIONS ) or account . HasPermission ( HC . POST_PETITIONS ) :
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
if account . HasPermission ( HC . RESOLVE_PETITIONS ) : petition_method = self . _ApproveFilePetition
elif account . HasPermission ( HC . POST_PETITIONS ) : petition_method = self . _AddFilePetition
2013-02-19 00:11:43 +00:00
2015-10-14 21:02:25 +00:00
for ( hashes , reason ) in update . GetContentDataIterator ( HC . CONTENT_TYPE_FILES , HC . CONTENT_UPDATE_PETITION ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
hash_ids = self . _GetHashIds ( hashes )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
reason_id = self . _GetReasonId ( reason )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
petition_method ( service_id , account_id , hash_ids , reason_id )
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
if account . HasPermission ( HC . RESOLVE_PETITIONS ) :
2013-02-19 00:11:43 +00:00
2015-10-14 21:02:25 +00:00
for hashes in update . GetContentDataIterator ( HC . CONTENT_TYPE_FILES , HC . CONTENT_UPDATE_DENY_PETITION ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
hash_ids = self . _GetHashIds ( hashes )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
self . _DenyFilePetition ( service_id , hash_ids )
2013-10-02 22:06:06 +00:00
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
elif service_type == HC . TAG_REPOSITORY :
tags = update . GetTags ( )
hashes = update . GetHashes ( )
#
overwrite_deleted = account . HasPermission ( HC . RESOLVE_PETITIONS )
2015-10-14 21:02:25 +00:00
for ( tag , hashes ) in update . GetContentDataIterator ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_PEND ) :
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
tag_id = self . _GetTagId ( tag )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
hash_ids = self . _GetHashIds ( hashes )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
self . _AddMappings ( service_id , account_id , tag_id , hash_ids , overwrite_deleted )
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
if account . HasPermission ( HC . RESOLVE_PETITIONS ) or account . HasPermission ( HC . POST_PETITIONS ) :
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
if account . HasPermission ( HC . RESOLVE_PETITIONS ) : petition_method = self . _ApproveMappingPetition
elif account . HasPermission ( HC . POST_PETITIONS ) : petition_method = self . _AddMappingPetition
2013-02-19 00:11:43 +00:00
2015-10-14 21:02:25 +00:00
for ( tag , hashes , reason ) in update . GetContentDataIterator ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_PETITION ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
tag_id = self . _GetTagId ( tag )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
hash_ids = self . _GetHashIds ( hashes )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
reason_id = self . _GetReasonId ( reason )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
petition_method ( service_id , account_id , tag_id , hash_ids , reason_id )
2013-10-02 22:06:06 +00:00
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
if account . HasPermission ( HC . RESOLVE_PETITIONS ) :
2015-10-14 21:02:25 +00:00
for ( tag , hashes ) in update . GetContentDataIterator ( HC . CONTENT_TYPE_MAPPINGS , HC . CONTENT_UPDATE_DENY_PETITION ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
tag_id = self . _GetTagId ( tag )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
hash_ids = self . _GetHashIds ( hashes )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
self . _DenyMappingPetition ( service_id , tag_id , hash_ids )
2013-10-02 22:06:06 +00:00
#
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
if account . HasPermission ( HC . RESOLVE_PETITIONS ) or account . HasPermission ( HC . POST_PETITIONS ) :
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
if account . HasPermission ( HC . RESOLVE_PETITIONS ) : petition_method = self . _ApproveTagSiblingPetition
elif account . HasPermission ( HC . POST_PETITIONS ) : petition_method = self . _AddTagSiblingPetition
2013-02-19 00:11:43 +00:00
2015-10-14 21:02:25 +00:00
for ( ( old_tag , new_tag ) , reason ) in update . GetContentDataIterator ( HC . CONTENT_TYPE_TAG_SIBLINGS , HC . CONTENT_UPDATE_PEND ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
old_tag_id = self . _GetTagId ( old_tag )
new_tag_id = self . _GetTagId ( new_tag )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
reason_id = self . _GetReasonId ( reason )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
petition_method ( service_id , account_id , old_tag_id , new_tag_id , reason_id , HC . PENDING )
2013-10-02 22:06:06 +00:00
if account . HasPermission ( HC . RESOLVE_PETITIONS ) : petition_method = self . _ApproveTagSiblingPetition
elif account . HasPermission ( HC . POST_PETITIONS ) : petition_method = self . _AddTagSiblingPetition
2015-10-14 21:02:25 +00:00
for ( ( old_tag , new_tag ) , reason ) in update . GetContentDataIterator ( HC . CONTENT_TYPE_TAG_SIBLINGS , HC . CONTENT_UPDATE_PETITION ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
old_tag_id = self . _GetTagId ( old_tag )
new_tag_id = self . _GetTagId ( new_tag )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
reason_id = self . _GetReasonId ( reason )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
petition_method ( service_id , account_id , old_tag_id , new_tag_id , reason_id , HC . PETITIONED )
2013-10-02 22:06:06 +00:00
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
if account . HasPermission ( HC . RESOLVE_PETITIONS ) :
2015-10-14 21:02:25 +00:00
for ( old_tag , new_tag ) in update . GetContentDataIterator ( HC . CONTENT_TYPE_TAG_SIBLINGS , HC . CONTENT_UPDATE_DENY_PEND ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
old_tag_id = self . _GetTagId ( old_tag )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
new_tag_id = self . _GetTagId ( new_tag )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
self . _DenyTagSiblingPetition ( service_id , old_tag_id , new_tag_id , HC . CONTENT_UPDATE_DENY_PEND )
2013-10-02 22:06:06 +00:00
2015-10-14 21:02:25 +00:00
for ( old_tag , new_tag ) in update . GetContentDataIterator ( HC . CONTENT_TYPE_TAG_SIBLINGS , HC . CONTENT_UPDATE_DENY_PETITION ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
old_tag_id = self . _GetTagId ( old_tag )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
new_tag_id = self . _GetTagId ( new_tag )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
self . _DenyTagSiblingPetition ( service_id , old_tag_id , new_tag_id , HC . CONTENT_UPDATE_DENY_PETITION )
2013-10-02 22:06:06 +00:00
#
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
if account . HasPermission ( HC . RESOLVE_PETITIONS ) or account . HasPermission ( HC . POST_PETITIONS ) :
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
if account . HasPermission ( HC . RESOLVE_PETITIONS ) : petition_method = self . _ApproveTagParentPetition
elif account . HasPermission ( HC . POST_PETITIONS ) : petition_method = self . _AddTagParentPetition
2013-02-19 00:11:43 +00:00
2015-10-14 21:02:25 +00:00
for ( ( old_tag , new_tag ) , reason ) in update . GetContentDataIterator ( HC . CONTENT_TYPE_TAG_PARENTS , HC . CONTENT_UPDATE_PEND ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
old_tag_id = self . _GetTagId ( old_tag )
new_tag_id = self . _GetTagId ( new_tag )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
reason_id = self . _GetReasonId ( reason )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
petition_method ( service_id , account_id , old_tag_id , new_tag_id , reason_id , HC . PENDING )
2013-10-02 22:06:06 +00:00
if account . HasPermission ( HC . RESOLVE_PETITIONS ) : petition_method = self . _ApproveTagParentPetition
elif account . HasPermission ( HC . POST_PETITIONS ) : petition_method = self . _AddTagParentPetition
2013-02-19 00:11:43 +00:00
2015-10-14 21:02:25 +00:00
for ( ( old_tag , new_tag ) , reason ) in update . GetContentDataIterator ( HC . CONTENT_TYPE_TAG_PARENTS , HC . CONTENT_UPDATE_PETITION ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
old_tag_id = self . _GetTagId ( old_tag )
new_tag_id = self . _GetTagId ( new_tag )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
reason_id = self . _GetReasonId ( reason )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
petition_method ( service_id , account_id , old_tag_id , new_tag_id , reason_id , HC . PETITIONED )
2013-10-02 22:06:06 +00:00
2013-02-19 00:11:43 +00:00
2013-10-02 22:06:06 +00:00
if account . HasPermission ( HC . RESOLVE_PETITIONS ) :
2013-02-19 00:11:43 +00:00
2015-10-14 21:02:25 +00:00
for ( old_tag , new_tag ) in update . GetContentDataIterator ( HC . CONTENT_TYPE_TAG_PARENTS , HC . CONTENT_UPDATE_DENY_PEND ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
old_tag_id = self . _GetTagId ( old_tag )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
new_tag_id = self . _GetTagId ( new_tag )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
self . _DenyTagParentPetition ( service_id , old_tag_id , new_tag_id , HC . CONTENT_UPDATE_DENY_PEND )
2013-10-02 22:06:06 +00:00
2013-02-19 00:11:43 +00:00
2015-10-14 21:02:25 +00:00
for ( old_tag , new_tag ) in update . GetContentDataIterator ( HC . CONTENT_TYPE_TAG_PARENTS , HC . CONTENT_UPDATE_DENY_PETITION ) :
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
old_tag_id = self . _GetTagId ( old_tag )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
new_tag_id = self . _GetTagId ( new_tag )
2013-10-02 22:06:06 +00:00
2014-11-20 01:48:04 +00:00
self . _DenyTagParentPetition ( service_id , old_tag_id , new_tag_id , HC . CONTENT_UPDATE_DENY_PETITION )
2013-10-02 22:06:06 +00:00
2013-02-19 00:11:43 +00:00
2015-04-22 22:57:25 +00:00
def _Read ( self , action , * args , * * kwargs ) :
if action == ' access_key ' : result = self . _GetAccessKey ( * args , * * kwargs )
elif action == ' account ' : result = self . _GetAccount ( * args , * * kwargs )
elif action == ' account_info ' : result = self . _GetAccountInfo ( * args , * * kwargs )
elif action == ' account_key_from_access_key ' : result = self . _GetAccountKeyFromAccessKey ( * args , * * kwargs )
elif action == ' account_key_from_identifier ' : result = self . _GetAccountKeyFromIdentifier ( * args , * * kwargs )
elif action == ' account_types ' : result = self . _GetAccountTypes ( * args , * * kwargs )
2015-09-02 23:16:09 +00:00
elif action == ' immediate_content_update ' : result = self . _GenerateImmediateContentUpdate ( * args , * * kwargs )
2015-04-22 22:57:25 +00:00
elif action == ' init ' : result = self . _InitAdmin ( * args , * * kwargs )
elif action == ' ip ' : result = self . _GetIPTimestamp ( * args , * * kwargs )
elif action == ' messaging_sessions ' : result = self . _GetMessagingSessions ( * args , * * kwargs )
elif action == ' num_petitions ' : result = self . _GetNumPetitions ( * args , * * kwargs )
elif action == ' petition ' : result = self . _GetPetition ( * args , * * kwargs )
elif action == ' registration_keys ' : result = self . _GenerateRegistrationKeys ( * args , * * kwargs )
elif action == ' service_keys ' : result = self . _GetServiceKeys ( * args , * * kwargs )
elif action == ' service_info ' : result = self . _GetServiceInfo ( * args , * * kwargs )
elif action == ' services_info ' : result = self . _GetServicesInfo ( * args , * * kwargs )
elif action == ' sessions ' : result = self . _GetSessions ( * args , * * kwargs )
elif action == ' stats ' : result = self . _GetStats ( * args , * * kwargs )
elif action == ' update_ends ' : result = self . _GetUpdateEnds ( * args , * * kwargs )
elif action == ' verify_access_key ' : result = self . _VerifyAccessKey ( * args , * * kwargs )
else : raise Exception ( ' db received an unknown read command: ' + action )
return result
2014-11-20 01:48:04 +00:00
def _RewardAccounts ( self , service_id , score_type , scores ) :
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
self . _c . executemany ( ' INSERT OR IGNORE INTO account_scores ( service_id, account_id, score_type, score ) VALUES ( ?, ?, ?, ? ); ' , [ ( service_id , account_id , score_type , 0 ) for ( account_id , score ) in scores ] )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
self . _c . executemany ( ' UPDATE account_scores SET score = score + ? WHERE service_id = ? AND account_id = ? and score_type = ?; ' , [ ( score , service_id , account_id , score_type ) for ( account_id , score ) in scores ] )
2013-02-19 00:11:43 +00:00
2015-03-04 22:44:32 +00:00
def _RewardFilePetitioners ( self , service_id , hash_ids , multiplier ) :
2015-03-25 22:04:19 +00:00
scores = [ ( account_id , count * multiplier ) for ( account_id , count ) in self . _c . execute ( ' SELECT account_id, COUNT( * ) FROM file_petitions WHERE service_id = ? AND hash_id IN ' + HydrusData . SplayListForDB ( hash_ids ) + ' AND status = ? GROUP BY account_id; ' , ( service_id , HC . PETITIONED ) ) ]
2015-03-04 22:44:32 +00:00
self . _RewardAccounts ( service_id , HC . SCORE_PETITION , scores )
def _RewardMappingPetitioners ( self , service_id , tag_id , hash_ids , multiplier ) :
2015-03-25 22:04:19 +00:00
scores = [ ( account_id , count * multiplier ) for ( account_id , count ) in self . _c . execute ( ' SELECT account_id, COUNT( * ) FROM mapping_petitions WHERE service_id = ? AND tag_id = ? AND hash_id IN ' + HydrusData . SplayListForDB ( hash_ids ) + ' AND status = ? GROUP BY account_id; ' , ( service_id , tag_id , HC . PETITIONED ) ) ]
2015-03-04 22:44:32 +00:00
self . _RewardAccounts ( service_id , HC . SCORE_PETITION , scores )
def _RewardTagParentPetitioners ( self , service_id , old_tag_id , new_tag_id , multiplier ) :
hash_ids_with_old_tag = { hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT hash_id FROM mappings WHERE service_id = ? AND tag_id = ?; ' , ( service_id , old_tag_id ) ) }
hash_ids_with_new_tag = { hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT hash_id FROM mappings WHERE service_id = ? AND tag_id = ?; ' , ( service_id , new_tag_id ) ) }
score = len ( hash_ids_with_old_tag . intersection ( hash_ids_with_new_tag ) )
weighted_score = score * multiplier
account_ids = [ account_id for ( account_id , ) in self . _c . execute ( ' SELECT account_id FROM tag_parents WHERE service_id = ? AND old_tag_id = ? AND new_tag_id = ? AND status IN ( ?, ? ); ' , ( service_id , old_tag_id , new_tag_id , HC . PENDING , HC . PETITIONED ) ) ]
scores = [ ( account_id , weighted_score ) for account_id in account_ids ]
self . _RewardAccounts ( service_id , HC . SCORE_PETITION , scores )
def _RewardTagSiblingPetitioners ( self , service_id , old_tag_id , new_tag_id , multiplier ) :
hash_ids_with_old_tag = { hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT hash_id FROM mappings WHERE service_id = ? AND tag_id = ?; ' , ( service_id , old_tag_id ) ) }
hash_ids_with_new_tag = { hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT hash_id FROM mappings WHERE service_id = ? AND tag_id = ?; ' , ( service_id , new_tag_id ) ) }
score = len ( hash_ids_with_old_tag . intersection ( hash_ids_with_new_tag ) )
weighted_score = score * multiplier
account_ids = [ account_id for ( account_id , ) in self . _c . execute ( ' SELECT account_id FROM tag_siblings WHERE service_id = ? AND old_tag_id = ? AND new_tag_id = ? AND status IN ( ?, ? ); ' , ( service_id , old_tag_id , new_tag_id , HC . PENDING , HC . PETITIONED ) ) ]
scores = [ ( account_id , weighted_score ) for account_id in account_ids ]
self . _RewardAccounts ( service_id , HC . SCORE_PETITION , scores )
2015-03-25 22:04:19 +00:00
def _SetExpires ( self , account_ids , expires ) : self . _c . execute ( ' UPDATE accounts SET expires = ? WHERE account_id IN ' + HydrusData . SplayListForDB ( account_ids ) + ' ; ' , ( expires , ) )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
def _UnbanKey ( self , service_id , account_id ) : self . _c . execute ( ' DELETE FROM bans WHERE service_id = ? AND account_id = ?; ' , ( account_id , ) )
2013-02-19 00:11:43 +00:00
2014-11-20 01:48:04 +00:00
def _UpdateDB ( self , version ) :
2013-02-19 00:11:43 +00:00
2016-03-30 22:56:50 +00:00
HydrusData . Print ( ' The server is updating to version ' + str ( version + 1 ) )
2015-05-06 20:26:18 +00:00
if version == 155 :
results = self . _c . execute ( ' SELECT service_id, account_type_id, account_type FROM account_types; ' ) . fetchall ( )
for ( service_id , account_type_id , account_type ) in results :
title = account_type . GetTitle ( )
self . _c . execute ( ' UPDATE account_types SET title = ? WHERE service_id = ? AND account_type_id = ?; ' , ( title , service_id , account_type_id ) )
2015-06-03 21:05:13 +00:00
if version == 158 :
first_ends = self . _c . execute ( ' SELECT service_id, end FROM update_cache WHERE begin = ?; ' , ( 0 , ) ) . fetchall ( )
self . _c . execute ( ' DELETE FROM update_cache; ' )
2015-08-19 21:48:21 +00:00
for filename in os . listdir ( HC . SERVER_UPDATES_DIR ) :
2015-06-03 21:05:13 +00:00
2015-11-04 22:30:28 +00:00
path = os . path . join ( HC . SERVER_UPDATES_DIR , filename )
2015-06-03 21:05:13 +00:00
2015-11-25 22:00:57 +00:00
HydrusPaths . RecyclePath ( path )
2015-06-03 21:05:13 +00:00
for ( service_id , end ) in first_ends :
service_key = self . _GetServiceKey ( service_id )
self . _CreateUpdate ( service_key , 0 , end )
2015-06-24 22:10:14 +00:00
if version == 161 :
2015-08-19 21:48:21 +00:00
for filename in os . listdir ( HC . SERVER_UPDATES_DIR ) :
2015-06-24 22:10:14 +00:00
2015-11-04 22:30:28 +00:00
path = os . path . join ( HC . SERVER_UPDATES_DIR , filename )
2015-06-24 22:10:14 +00:00
with open ( path , ' rb ' ) as f :
compressed_inefficient_string = f . read ( )
try :
inefficient_string = lz4 . loads ( compressed_inefficient_string )
( dump_type , dump_version , dump ) = json . loads ( inefficient_string )
2016-01-13 22:08:19 +00:00
if not isinstance ( dump , ( unicode , str ) ) :
2015-06-24 22:10:14 +00:00
continue
serialisable_info = json . loads ( dump )
except :
continue
better_string = json . dumps ( ( dump_type , dump_version , serialisable_info ) )
compressed_better_string = lz4 . dumps ( better_string )
with open ( path , ' wb ' ) as f :
f . write ( compressed_better_string )
2015-08-19 21:48:21 +00:00
if version == 169 :
bad_tag_ids = set ( )
for ( tag_id , tag ) in self . _c . execute ( ' SELECT tag_id, tag FROM tags; ' ) :
try :
HydrusTags . CheckTagNotEmpty ( tag )
except HydrusExceptions . SizeException :
bad_tag_ids . add ( tag_id )
self . _c . executemany ( ' DELETE FROM mappings WHERE tag_id = ?; ' , ( ( tag_id , ) for tag_id in bad_tag_ids ) )
2015-06-24 22:10:14 +00:00
2015-11-04 22:30:28 +00:00
if version == 179 :
2015-11-18 22:44:07 +00:00
HydrusData . Print ( ' moving updates about ' )
2015-11-04 22:30:28 +00:00
for filename in os . listdir ( HC . SERVER_UPDATES_DIR ) :
try :
( service_key_encoded , gumpf ) = filename . split ( ' _ ' , 1 )
except ValueError :
continue
dest_dir = os . path . join ( HC . SERVER_UPDATES_DIR , service_key_encoded )
if not os . path . exists ( dest_dir ) :
2016-01-06 21:17:20 +00:00
os . makedirs ( dest_dir )
2015-11-04 22:30:28 +00:00
source_path = os . path . join ( HC . SERVER_UPDATES_DIR , filename )
dest_path = os . path . join ( dest_dir , gumpf )
shutil . move ( source_path , dest_path )
2015-11-25 22:00:57 +00:00
if version == 182 :
HydrusData . Print ( ' generating swf thumbnails ' )
mimes = { HC . APPLICATION_FLASH }
mimes . update ( HC . VIDEO )
hash_ids = { hash_id for ( hash_id , ) in self . _c . execute ( ' SELECT hash_id FROM files_info WHERE mime IN ' + HydrusData . SplayListForDB ( mimes ) + ' ; ' ) }
for hash_id in hash_ids :
hash = self . _GetHash ( hash_id )
try :
file_path = ServerFiles . GetPath ( ' file ' , hash )
except HydrusExceptions . NotFoundException :
continue
thumbnail = HydrusFileHandling . GenerateThumbnail ( file_path )
thumbnail_path = ServerFiles . GetExpectedPath ( ' thumbnail ' , hash )
with open ( thumbnail_path , ' wb ' ) as f :
f . write ( thumbnail )
2015-12-09 23:16:41 +00:00
if version == 184 :
result = self . _c . execute ( ' SELECT tag_id FROM tags WHERE tag = ?; ' , ( ' ' , ) ) . fetchone ( )
if result is not None :
( tag_id , ) = result
self . _c . execute ( ' DELETE FROM mappings WHERE tag_id = ?; ' , ( tag_id , ) )
2016-01-06 21:17:20 +00:00
if version == 188 :
self . _c . execute ( ' CREATE TABLE analyze_timestamps ( name TEXT, timestamp INTEGER ); ' )
2016-03-30 22:56:50 +00:00
if version == 198 :
HydrusData . Print ( ' exporting mappings to external db ' )
2016-05-18 20:07:14 +00:00
self . _c . execute ( ' CREATE TABLE IF NOT EXISTS external_mappings.mappings ( service_id INTEGER, tag_id INTEGER, hash_id INTEGER, account_id INTEGER, timestamp INTEGER, PRIMARY KEY( service_id, tag_id, hash_id ) ); ' )
2016-04-14 01:54:29 +00:00
self . _c . execute ( ' INSERT INTO external_mappings.mappings SELECT * FROM main.mappings; ' )
2016-03-30 22:56:50 +00:00
self . _c . execute ( ' DROP TABLE main.mappings; ' )
2016-05-18 20:07:14 +00:00
self . _c . execute ( ' CREATE TABLE IF NOT EXISTS external_mappings.mapping_petitions ( service_id INTEGER, account_id INTEGER, tag_id INTEGER, hash_id INTEGER, reason_id INTEGER, timestamp INTEGER, status INTEGER, PRIMARY KEY( service_id, account_id, tag_id, hash_id, status ) ); ' )
2016-04-14 01:54:29 +00:00
self . _c . execute ( ' INSERT INTO external_mappings.mapping_petitions SELECT * FROM main.mapping_petitions; ' )
2016-03-30 22:56:50 +00:00
self . _c . execute ( ' DROP TABLE main.mapping_petitions; ' )
2016-04-14 01:54:29 +00:00
if version == 200 :
HydrusData . Print ( ' exporting hashes to external db ' )
2016-04-20 20:42:21 +00:00
self . _c . execute ( ' CREATE TABLE IF NOT EXISTS external_master.hashes ( hash_id INTEGER PRIMARY KEY, hash BLOB_BYTES UNIQUE ); ' )
2016-04-14 01:54:29 +00:00
self . _c . execute ( ' INSERT INTO external_master.hashes SELECT * FROM main.hashes; ' )
self . _c . execute ( ' DROP TABLE main.hashes; ' )
HydrusData . Print ( ' exporting tags to external db ' )
2016-04-20 20:42:21 +00:00
self . _c . execute ( ' CREATE TABLE IF NOT EXISTS external_master.tags ( tag_id INTEGER PRIMARY KEY, tag TEXT UNIQUE ); ' )
2016-04-14 01:54:29 +00:00
self . _c . execute ( ' INSERT INTO external_master.tags SELECT * FROM main.tags; ' )
self . _c . execute ( ' DROP TABLE main.tags; ' )
#
HydrusData . Print ( ' compacting mappings tables ' )
self . _c . execute ( ' DROP INDEX mapping_petitions_service_id_account_id_reason_id_tag_id_index; ' )
self . _c . execute ( ' DROP INDEX mapping_petitions_service_id_tag_id_hash_id_index; ' )
self . _c . execute ( ' DROP INDEX mapping_petitions_service_id_status_index; ' )
self . _c . execute ( ' DROP INDEX mapping_petitions_service_id_timestamp_index; ' )
self . _c . execute ( ' DROP INDEX mappings_account_id_index; ' )
self . _c . execute ( ' DROP INDEX mappings_timestamp_index; ' )
self . _c . execute ( ' ALTER TABLE mapping_petitions RENAME TO mapping_petitions_old; ' )
self . _c . execute ( ' ALTER TABLE mappings RENAME TO mappings_old; ' )
2016-04-20 20:42:21 +00:00
self . _c . execute ( ' CREATE TABLE IF NOT EXISTS external_mappings.mapping_petitions ( service_id INTEGER, account_id INTEGER, tag_id INTEGER, hash_id INTEGER, reason_id INTEGER, timestamp INTEGER, status INTEGER, PRIMARY KEY( service_id, account_id, tag_id, hash_id, status ) ) WITHOUT ROWID; ' )
2016-04-14 01:54:29 +00:00
self . _c . execute ( ' INSERT INTO mapping_petitions SELECT * FROM mapping_petitions_old; ' )
self . _c . execute ( ' DROP TABLE mapping_petitions_old; ' )
2016-04-20 20:42:21 +00:00
self . _c . execute ( ' CREATE TABLE IF NOT EXISTS external_mappings.mappings ( service_id INTEGER, tag_id INTEGER, hash_id INTEGER, account_id INTEGER, timestamp INTEGER, PRIMARY KEY( service_id, tag_id, hash_id ) ) WITHOUT ROWID; ' )
self . _c . execute ( ' INSERT INTO mappings SELECT * FROM mappings_old; ' )
2016-04-14 01:54:29 +00:00
self . _c . execute ( ' DROP TABLE mappings_old; ' )
2016-04-20 20:42:21 +00:00
self . _c . execute ( ' CREATE INDEX IF NOT EXISTS external_mappings.mapping_petitions_service_id_account_id_reason_id_tag_id_index ON mapping_petitions ( service_id, account_id, reason_id, tag_id ); ' )
self . _c . execute ( ' CREATE INDEX IF NOT EXISTS external_mappings.mapping_petitions_service_id_tag_id_hash_id_index ON mapping_petitions ( service_id, tag_id, hash_id ); ' )
self . _c . execute ( ' CREATE INDEX IF NOT EXISTS external_mappings.mapping_petitions_service_id_status_index ON mapping_petitions ( service_id, status ); ' )
self . _c . execute ( ' CREATE INDEX IF NOT EXISTS external_mappings.mapping_petitions_service_id_timestamp_index ON mapping_petitions ( service_id, timestamp ); ' )
2016-04-14 01:54:29 +00:00
2016-04-20 20:42:21 +00:00
self . _c . execute ( ' CREATE INDEX IF NOT EXISTS external_mappings.mappings_account_id_index ON mappings ( account_id ); ' )
self . _c . execute ( ' CREATE INDEX IF NOT EXISTS external_mappings.mappings_timestamp_index ON mappings ( timestamp ); ' )
2016-04-14 01:54:29 +00:00
#
self . _c . execute ( ' COMMIT; ' )
self . _CloseDBCursor ( )
for filename in self . _db_filenames . values ( ) :
HydrusData . Print ( ' vacuuming ' + filename )
db_path = os . path . join ( self . _db_dir , filename )
if HydrusDB . CanVacuum ( db_path ) :
HydrusDB . VacuumDB ( db_path )
self . _InitDBCursor ( )
2016-03-30 22:56:50 +00:00
2016-04-14 01:54:29 +00:00
self . _c . execute ( ' BEGIN IMMEDIATE; ' )
2016-03-30 22:56:50 +00:00
2016-04-27 19:20:37 +00:00
if version == 202 :
self . _c . execute ( ' DELETE FROM analyze_timestamps; ' )
2015-11-18 22:44:07 +00:00
HydrusData . Print ( ' The server has updated to version ' + str ( version + 1 ) )
2015-05-06 20:26:18 +00:00
2014-11-20 01:48:04 +00:00
self . _c . execute ( ' UPDATE version SET version = ?; ' , ( version + 1 , ) )
2013-02-19 00:11:43 +00:00
2015-07-01 22:02:07 +00:00
HydrusGlobals . is_db_updated = True
2015-11-11 21:20:41 +00:00
2015-04-22 22:57:25 +00:00
def _VerifyAccessKey ( self , service_key , access_key ) :
2013-02-19 00:11:43 +00:00
2015-04-22 22:57:25 +00:00
service_id = self . _GetServiceId ( service_key )
2013-10-02 22:06:06 +00:00
2015-04-22 22:57:25 +00:00
result = self . _c . execute ( ' SELECT 1 FROM accounts WHERE service_id = ? AND hashed_access_key = ?; ' , ( service_id , sqlite3 . Binary ( hashlib . sha256 ( access_key ) . digest ( ) ) ) ) . fetchone ( )
2013-02-19 00:11:43 +00:00
2015-04-22 22:57:25 +00:00
if result is None :
result = self . _c . execute ( ' SELECT 1 FROM registration_keys WHERE service_id = ? AND access_key = ?; ' , ( service_id , sqlite3 . Binary ( access_key ) ) ) . fetchone ( )
if result is None : return False
2013-02-19 00:11:43 +00:00
2015-04-22 22:57:25 +00:00
return True
2013-02-19 00:11:43 +00:00
2015-04-22 22:57:25 +00:00
def _Write ( self , action , * args , * * kwargs ) :
if action == ' account ' : result = self . _ModifyAccount ( * args , * * kwargs )
elif action == ' account_types ' : result = self . _ModifyAccountTypes ( * args , * * kwargs )
2016-01-06 21:17:20 +00:00
elif action == ' analyze ' : result = self . _Analyze ( * args , * * kwargs )
2016-04-14 01:54:29 +00:00
elif action == ' backup ' : result = self . _Backup ( * args , * * kwargs )
2015-04-22 22:57:25 +00:00
elif action == ' check_data_usage ' : result = self . _CheckDataUsage ( * args , * * kwargs )
elif action == ' check_monthly_data ' : result = self . _CheckMonthlyData ( * args , * * kwargs )
elif action == ' clear_bans ' : result = self . _ClearBans ( * args , * * kwargs )
elif action == ' create_update ' : result = self . _CreateUpdate ( * args , * * kwargs )
elif action == ' delete_orphans ' : result = self . _DeleteOrphans ( * args , * * kwargs )
elif action == ' file ' : result = self . _AddFile ( * args , * * kwargs )
elif action == ' flush_requests_made ' : result = self . _FlushRequestsMade ( * args , * * kwargs )
elif action == ' messaging_session ' : result = self . _AddMessagingSession ( * args , * * kwargs )
elif action == ' news ' : result = self . _AddNews ( * args , * * kwargs )
elif action == ' services ' : result = self . _ModifyServices ( * args , * * kwargs )
elif action == ' session ' : result = self . _AddSession ( * args , * * kwargs )
elif action == ' update ' : result = self . _ProcessUpdate ( * args , * * kwargs )
else : raise Exception ( ' db received an unknown write command: ' + action )
2013-02-19 00:11:43 +00:00
2015-04-22 22:57:25 +00:00
return result
2013-10-02 22:06:06 +00:00
2013-11-06 18:22:07 +00:00