2015-03-25 22:04:19 +00:00
|
|
|
import gc
|
2013-08-07 22:25:18 +00:00
|
|
|
import hashlib
|
2013-07-17 20:56:13 +00:00
|
|
|
import HydrusAudioHandling
|
|
|
|
import HydrusConstants as HC
|
2018-09-05 20:52:32 +00:00
|
|
|
import HydrusData
|
2013-07-17 20:56:13 +00:00
|
|
|
import HydrusDocumentHandling
|
2013-07-24 20:26:00 +00:00
|
|
|
import HydrusExceptions
|
2013-07-17 20:56:13 +00:00
|
|
|
import HydrusFlashHandling
|
|
|
|
import HydrusImageHandling
|
2015-11-04 22:30:28 +00:00
|
|
|
import HydrusPaths
|
2018-09-05 20:52:32 +00:00
|
|
|
import HydrusText
|
2013-07-17 20:56:13 +00:00
|
|
|
import HydrusVideoHandling
|
|
|
|
import os
|
2015-10-21 21:53:10 +00:00
|
|
|
import threading
|
2013-07-17 20:56:13 +00:00
|
|
|
import traceback
|
2014-05-21 21:37:35 +00:00
|
|
|
import cStringIO
|
2013-07-17 20:56:13 +00:00
|
|
|
|
|
|
|
# Mime
|
|
|
|
|
2015-03-25 22:04:19 +00:00
|
|
|
header_and_mime = [
|
|
|
|
( 0, '\xff\xd8', HC.IMAGE_JPEG ),
|
|
|
|
( 0, 'GIF87a', HC.IMAGE_GIF ),
|
|
|
|
( 0, 'GIF89a', HC.IMAGE_GIF ),
|
2016-02-24 21:42:54 +00:00
|
|
|
( 0, '\x89PNG', HC.UNDETERMINED_PNG ),
|
2015-03-25 22:04:19 +00:00
|
|
|
( 0, 'BM', HC.IMAGE_BMP ),
|
|
|
|
( 0, 'CWS', HC.APPLICATION_FLASH ),
|
|
|
|
( 0, 'FWS', HC.APPLICATION_FLASH ),
|
2017-11-22 21:03:07 +00:00
|
|
|
( 0, 'ZWS', HC.APPLICATION_FLASH ),
|
2015-03-25 22:04:19 +00:00
|
|
|
( 0, 'FLV', HC.VIDEO_FLV ),
|
|
|
|
( 0, '%PDF', HC.APPLICATION_PDF ),
|
|
|
|
( 0, 'PK\x03\x04', HC.APPLICATION_ZIP ),
|
2017-10-04 17:51:58 +00:00
|
|
|
( 0, 'PK\x05\x06', HC.APPLICATION_ZIP ),
|
|
|
|
( 0, 'PK\x07\x08', HC.APPLICATION_ZIP ),
|
|
|
|
( 0, '7z\xBC\xAF\x27\x1C', HC.APPLICATION_7Z ),
|
|
|
|
( 0, '\x52\x61\x72\x21\x1A\x07\x00', HC.APPLICATION_RAR ),
|
|
|
|
( 0, '\x52\x61\x72\x21\x1A\x07\x01\x00', HC.APPLICATION_RAR ),
|
2015-03-25 22:04:19 +00:00
|
|
|
( 0, 'hydrus encrypted zip', HC.APPLICATION_HYDRUS_ENCRYPTED_ZIP ),
|
|
|
|
( 4, 'ftypmp4', HC.VIDEO_MP4 ),
|
2015-07-01 22:02:07 +00:00
|
|
|
( 4, 'ftypisom', HC.VIDEO_MP4 ),
|
|
|
|
( 4, 'ftypM4V', HC.VIDEO_MP4 ),
|
2016-10-12 21:52:50 +00:00
|
|
|
( 4, 'ftypMSNV', HC.VIDEO_MP4 ),
|
2016-10-26 20:45:34 +00:00
|
|
|
( 4, 'ftypavc1', HC.VIDEO_MP4 ),
|
|
|
|
( 4, 'ftypFACE', HC.VIDEO_MP4 ),
|
|
|
|
( 4, 'ftypdash', HC.VIDEO_MP4 ),
|
2016-09-07 20:01:05 +00:00
|
|
|
( 4, 'ftypqt', HC.VIDEO_MOV ),
|
2015-03-25 22:04:19 +00:00
|
|
|
( 0, 'fLaC', HC.AUDIO_FLAC ),
|
2016-10-19 20:02:56 +00:00
|
|
|
( 8, 'AVI ', HC.VIDEO_AVI ),
|
2015-03-25 22:04:19 +00:00
|
|
|
( 0, '\x30\x26\xB2\x75\x8E\x66\xCF\x11\xA6\xD9\x00\xAA\x00\x62\xCE\x6C', HC.UNDETERMINED_WM )
|
|
|
|
]
|
|
|
|
|
2017-11-08 22:07:12 +00:00
|
|
|
def SaveThumbnailToStreamPIL( pil_image, dimensions, f ):
|
2015-11-25 22:00:57 +00:00
|
|
|
|
2016-06-29 19:55:46 +00:00
|
|
|
# when the palette is limited, the thumbnail antialias won't add new colours, so you get nearest-neighbour-like behaviour
|
|
|
|
|
2016-07-06 21:13:15 +00:00
|
|
|
original_file_was_png = pil_image.format == 'PNG'
|
2016-06-29 19:55:46 +00:00
|
|
|
|
|
|
|
pil_image = HydrusImageHandling.Dequantize( pil_image )
|
|
|
|
|
2015-11-25 22:00:57 +00:00
|
|
|
HydrusImageHandling.EfficientlyThumbnailPILImage( pil_image, dimensions )
|
|
|
|
|
2016-07-06 21:13:15 +00:00
|
|
|
if original_file_was_png or pil_image.mode == 'RGBA':
|
2015-11-25 22:00:57 +00:00
|
|
|
|
|
|
|
pil_image.save( f, 'PNG' )
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
pil_image.save( f, 'JPEG', quality = 92 )
|
|
|
|
|
|
|
|
|
2018-05-30 20:13:21 +00:00
|
|
|
def GenerateThumbnail( path, mime, dimensions = HC.UNSCALED_THUMBNAIL_DIMENSIONS, percentage_in = 35 ):
|
2014-05-21 21:37:35 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
if mime in ( HC.IMAGE_JPEG, HC.IMAGE_PNG, HC.IMAGE_GIF ):
|
2014-05-21 21:37:35 +00:00
|
|
|
|
2017-11-15 22:35:49 +00:00
|
|
|
thumbnail = GenerateThumbnailFromStaticImage( path, dimensions, mime )
|
2014-05-21 21:37:35 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
else:
|
2014-05-21 21:37:35 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
f = cStringIO.StringIO()
|
2015-11-25 22:00:57 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
if mime == HC.APPLICATION_FLASH:
|
2014-05-21 21:37:35 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
( os_file_handle, temp_path ) = HydrusPaths.GetTempPath()
|
2014-05-21 21:37:35 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
try:
|
|
|
|
|
|
|
|
HydrusFlashHandling.RenderPageToFile( path, temp_path, 1 )
|
|
|
|
|
|
|
|
pil_image = HydrusImageHandling.GeneratePILImage( temp_path )
|
|
|
|
|
2017-11-08 22:07:12 +00:00
|
|
|
SaveThumbnailToStreamPIL( pil_image, dimensions, f )
|
2017-06-28 20:23:21 +00:00
|
|
|
|
|
|
|
except:
|
|
|
|
|
|
|
|
flash_default_path = os.path.join( HC.STATIC_DIR, 'flash.png' )
|
|
|
|
|
|
|
|
pil_image = HydrusImageHandling.GeneratePILImage( flash_default_path )
|
|
|
|
|
2017-11-08 22:07:12 +00:00
|
|
|
SaveThumbnailToStreamPIL( pil_image, dimensions, f )
|
2017-06-28 20:23:21 +00:00
|
|
|
|
|
|
|
finally:
|
|
|
|
|
|
|
|
del pil_image
|
|
|
|
|
|
|
|
HydrusPaths.CleanUpTempPath( os_file_handle, temp_path )
|
|
|
|
|
2014-05-21 21:37:35 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
else:
|
2014-05-21 21:37:35 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
( size, mime, width, height, duration, num_frames, num_words ) = GetFileInfo( path )
|
2015-11-25 22:00:57 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
cropped_dimensions = HydrusImageHandling.GetThumbnailResolution( ( width, height ), dimensions )
|
2015-11-25 22:00:57 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
renderer = HydrusVideoHandling.VideoRendererFFMPEG( path, mime, duration, num_frames, cropped_dimensions )
|
2015-11-25 22:00:57 +00:00
|
|
|
|
2018-06-06 21:27:02 +00:00
|
|
|
renderer.read_frame() # this initialises the renderer and loads the first frame as a fallback
|
|
|
|
|
2018-05-30 20:13:21 +00:00
|
|
|
desired_thumb_frame = int( ( percentage_in / 100.0 ) * num_frames )
|
|
|
|
|
|
|
|
renderer.set_position( desired_thumb_frame )
|
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
numpy_image = renderer.read_frame()
|
2015-11-25 22:00:57 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
if numpy_image is None:
|
|
|
|
|
|
|
|
raise Exception( 'Could not create a thumbnail from that video!' )
|
|
|
|
|
2015-11-25 22:00:57 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
pil_image = HydrusImageHandling.GeneratePILImageFromNumpyImage( numpy_image )
|
2015-11-25 22:00:57 +00:00
|
|
|
|
2017-11-08 22:07:12 +00:00
|
|
|
SaveThumbnailToStreamPIL( pil_image, dimensions, f )
|
2014-05-21 21:37:35 +00:00
|
|
|
|
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
f.seek( 0 )
|
2016-10-12 21:52:50 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
thumbnail = f.read()
|
2014-05-21 21:37:35 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
f.close()
|
2014-06-25 20:37:06 +00:00
|
|
|
|
2014-05-21 21:37:35 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
return thumbnail
|
|
|
|
|
2017-11-15 22:35:49 +00:00
|
|
|
def GenerateThumbnailFromStaticImagePIL( path, dimensions = HC.UNSCALED_THUMBNAIL_DIMENSIONS, mime = None ):
|
2017-06-28 20:23:21 +00:00
|
|
|
|
|
|
|
f = cStringIO.StringIO()
|
|
|
|
|
|
|
|
pil_image = HydrusImageHandling.GeneratePILImage( path )
|
|
|
|
|
2017-11-08 22:07:12 +00:00
|
|
|
SaveThumbnailToStreamPIL( pil_image, dimensions, f )
|
2017-06-28 20:23:21 +00:00
|
|
|
|
2015-11-25 22:00:57 +00:00
|
|
|
f.seek( 0 )
|
|
|
|
|
|
|
|
thumbnail = f.read()
|
|
|
|
|
|
|
|
f.close()
|
|
|
|
|
2014-05-21 21:37:35 +00:00
|
|
|
return thumbnail
|
|
|
|
|
2017-11-08 22:07:12 +00:00
|
|
|
GenerateThumbnailFromStaticImage = GenerateThumbnailFromStaticImagePIL
|
|
|
|
|
2015-03-25 22:04:19 +00:00
|
|
|
def GetExtraHashesFromPath( path ):
|
|
|
|
|
|
|
|
h_md5 = hashlib.md5()
|
|
|
|
h_sha1 = hashlib.sha1()
|
|
|
|
h_sha512 = hashlib.sha512()
|
|
|
|
|
|
|
|
with open( path, 'rb' ) as f:
|
|
|
|
|
2015-11-04 22:30:28 +00:00
|
|
|
for block in HydrusPaths.ReadFileLikeAsBlocks( f ):
|
2015-03-25 22:04:19 +00:00
|
|
|
|
|
|
|
h_md5.update( block )
|
|
|
|
h_sha1.update( block )
|
|
|
|
h_sha512.update( block )
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
md5 = h_md5.digest()
|
|
|
|
sha1 = h_sha1.digest()
|
|
|
|
sha512 = h_sha512.digest()
|
|
|
|
|
|
|
|
return ( md5, sha1, sha512 )
|
|
|
|
|
2017-10-04 17:51:58 +00:00
|
|
|
def GetFileInfo( path, mime = None ):
|
2013-07-17 20:56:13 +00:00
|
|
|
|
2016-04-14 01:54:29 +00:00
|
|
|
size = os.path.getsize( path )
|
2013-07-17 20:56:13 +00:00
|
|
|
|
2017-07-19 21:21:41 +00:00
|
|
|
if size == 0:
|
|
|
|
|
|
|
|
raise HydrusExceptions.SizeException( 'File is of zero length!' )
|
|
|
|
|
2013-07-17 20:56:13 +00:00
|
|
|
|
2017-10-04 17:51:58 +00:00
|
|
|
if mime is None:
|
|
|
|
|
|
|
|
mime = GetMime( path )
|
|
|
|
|
2013-07-17 20:56:13 +00:00
|
|
|
|
2017-03-08 23:23:12 +00:00
|
|
|
if mime not in HC.ALLOWED_MIMES:
|
|
|
|
|
2018-09-05 20:52:32 +00:00
|
|
|
if mime == HC.TEXT_HTML:
|
|
|
|
|
|
|
|
raise HydrusExceptions.MimeException( 'Looks like HTML -- maybe the client needs to be taught how to parse this?' )
|
|
|
|
|
|
|
|
elif mime == HC.APPLICATION_UNKNOWN:
|
|
|
|
|
|
|
|
raise HydrusExceptions.MimeException( 'Unknown filetype!' )
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
raise HydrusExceptions.MimeException( 'Filetype is not permitted!' )
|
|
|
|
|
2017-03-08 23:23:12 +00:00
|
|
|
|
2013-07-17 20:56:13 +00:00
|
|
|
|
|
|
|
width = None
|
|
|
|
height = None
|
|
|
|
duration = None
|
|
|
|
num_frames = None
|
|
|
|
num_words = None
|
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
if mime in ( HC.IMAGE_JPEG, HC.IMAGE_PNG, HC.IMAGE_GIF ):
|
2013-07-17 20:56:13 +00:00
|
|
|
|
2014-05-14 20:46:38 +00:00
|
|
|
( ( width, height ), duration, num_frames ) = HydrusImageHandling.GetImageProperties( path )
|
2013-07-17 20:56:13 +00:00
|
|
|
|
|
|
|
elif mime == HC.APPLICATION_FLASH:
|
|
|
|
|
2013-08-07 22:25:18 +00:00
|
|
|
( ( width, height ), duration, num_frames ) = HydrusFlashHandling.GetFlashProperties( path )
|
2013-07-17 20:56:13 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
elif mime in ( HC.IMAGE_APNG, HC.VIDEO_AVI, HC.VIDEO_FLV, HC.VIDEO_WMV, HC.VIDEO_MOV, HC.VIDEO_MP4, HC.VIDEO_MKV, HC.VIDEO_WEBM, HC.VIDEO_MPEG ):
|
2014-04-30 21:31:40 +00:00
|
|
|
|
2014-06-25 20:37:06 +00:00
|
|
|
( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetFFMPEGVideoProperties( path )
|
2014-04-30 21:31:40 +00:00
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
elif mime == HC.APPLICATION_PDF:
|
|
|
|
|
|
|
|
num_words = HydrusDocumentHandling.GetPDFNumWords( path )
|
|
|
|
|
2017-12-06 22:06:56 +00:00
|
|
|
elif mime in HC.AUDIO:
|
2017-06-28 20:23:21 +00:00
|
|
|
|
2017-12-06 22:06:56 +00:00
|
|
|
ffmpeg_lines = HydrusVideoHandling.GetFFMPEGInfoLines( path )
|
2017-06-28 20:23:21 +00:00
|
|
|
|
2017-12-06 22:06:56 +00:00
|
|
|
duration_in_s = HydrusVideoHandling.ParseFFMPEGDuration( ffmpeg_lines )
|
2017-06-28 20:23:21 +00:00
|
|
|
|
2017-12-06 22:06:56 +00:00
|
|
|
duration = int( duration_in_s * 1000 )
|
2017-06-28 20:23:21 +00:00
|
|
|
|
2013-07-17 20:56:13 +00:00
|
|
|
|
2018-03-22 00:03:33 +00:00
|
|
|
if width is not None and width < 0:
|
|
|
|
|
|
|
|
width *= -1
|
|
|
|
|
|
|
|
|
|
|
|
if height is not None and height < 0:
|
|
|
|
|
|
|
|
width *= -1
|
|
|
|
|
|
|
|
|
|
|
|
if duration is not None and duration < 0:
|
|
|
|
|
|
|
|
duration *= -1
|
|
|
|
|
|
|
|
|
|
|
|
if num_frames is not None and num_frames < 0:
|
|
|
|
|
|
|
|
num_frames *= -1
|
|
|
|
|
|
|
|
|
|
|
|
if num_words is not None and num_words < 0:
|
|
|
|
|
|
|
|
num_words *= -1
|
|
|
|
|
|
|
|
|
2013-07-17 20:56:13 +00:00
|
|
|
return ( size, mime, width, height, duration, num_frames, num_words )
|
|
|
|
|
2013-08-07 22:25:18 +00:00
|
|
|
def GetHashFromPath( path ):
|
|
|
|
|
|
|
|
h = hashlib.sha256()
|
|
|
|
|
2013-08-14 20:21:49 +00:00
|
|
|
with open( path, 'rb' ) as f:
|
2013-08-07 22:25:18 +00:00
|
|
|
|
2017-07-19 21:21:41 +00:00
|
|
|
for block in HydrusPaths.ReadFileLikeAsBlocks( f ):
|
|
|
|
|
|
|
|
h.update( block )
|
|
|
|
|
2013-08-07 22:25:18 +00:00
|
|
|
|
|
|
|
|
2014-11-12 23:33:13 +00:00
|
|
|
return h.digest()
|
|
|
|
|
2013-08-07 22:25:18 +00:00
|
|
|
def GetMime( path ):
|
2013-07-17 20:56:13 +00:00
|
|
|
|
2017-10-04 17:51:58 +00:00
|
|
|
size = os.path.getsize( path )
|
|
|
|
|
|
|
|
if size == 0:
|
|
|
|
|
|
|
|
raise HydrusExceptions.SizeException( 'File is of zero length!' )
|
|
|
|
|
|
|
|
|
2013-08-14 20:21:49 +00:00
|
|
|
with open( path, 'rb' ) as f:
|
2013-07-17 20:56:13 +00:00
|
|
|
|
2013-08-07 22:25:18 +00:00
|
|
|
bit_to_check = f.read( 256 )
|
|
|
|
|
|
|
|
|
|
|
|
for ( offset, header, mime ) in header_and_mime:
|
|
|
|
|
|
|
|
offset_bit_to_check = bit_to_check[ offset: ]
|
|
|
|
|
2013-08-14 20:21:49 +00:00
|
|
|
if offset_bit_to_check.startswith( header ):
|
|
|
|
|
|
|
|
if mime == HC.UNDETERMINED_WM:
|
|
|
|
|
2015-11-18 22:44:07 +00:00
|
|
|
if HydrusVideoHandling.HasVideoStream( path ):
|
2013-08-14 20:21:49 +00:00
|
|
|
|
|
|
|
return HC.VIDEO_WMV
|
|
|
|
|
2015-11-18 22:44:07 +00:00
|
|
|
|
|
|
|
# we'll catch and verify wma later
|
2013-08-14 20:21:49 +00:00
|
|
|
|
2016-02-24 21:42:54 +00:00
|
|
|
elif mime == HC.UNDETERMINED_PNG:
|
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
if HydrusVideoHandling.HasVideoStream( path ):
|
|
|
|
|
|
|
|
return HC.IMAGE_APNG
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
return HC.IMAGE_PNG
|
|
|
|
|
2016-02-24 21:42:54 +00:00
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
return mime
|
|
|
|
|
2013-08-14 20:21:49 +00:00
|
|
|
|
2013-07-17 20:56:13 +00:00
|
|
|
|
|
|
|
|
2014-04-30 21:31:40 +00:00
|
|
|
try:
|
|
|
|
|
2017-06-28 20:23:21 +00:00
|
|
|
mime = HydrusVideoHandling.GetMime( path )
|
2014-04-30 21:31:40 +00:00
|
|
|
|
2017-01-04 22:48:23 +00:00
|
|
|
if mime != HC.APPLICATION_UNKNOWN:
|
|
|
|
|
|
|
|
return mime
|
|
|
|
|
2014-04-30 21:31:40 +00:00
|
|
|
|
2016-12-07 22:12:52 +00:00
|
|
|
except HydrusExceptions.MimeException:
|
2016-08-17 20:07:22 +00:00
|
|
|
|
2016-12-07 22:12:52 +00:00
|
|
|
HydrusData.Print( 'FFMPEG couldn\'t figure out the mime for: ' + path )
|
|
|
|
|
|
|
|
except Exception as e:
|
|
|
|
|
|
|
|
HydrusData.Print( 'FFMPEG couldn\'t figure out the mime for: ' + path )
|
2017-01-04 22:48:23 +00:00
|
|
|
HydrusData.PrintException( e, do_wait = False )
|
2016-08-17 20:07:22 +00:00
|
|
|
|
2014-04-30 21:31:40 +00:00
|
|
|
|
2018-09-05 20:52:32 +00:00
|
|
|
if HydrusText.LooksLikeHTML( bit_to_check ):
|
|
|
|
|
|
|
|
return HC.TEXT_HTML
|
|
|
|
|
|
|
|
|
2013-08-07 22:25:18 +00:00
|
|
|
return HC.APPLICATION_UNKNOWN
|
2016-12-07 22:12:52 +00:00
|
|
|
|