Version 120
This commit is contained in:
parent
babac9e405
commit
11f375509f
|
@ -8,6 +8,21 @@
|
||||||
<div class="content">
|
<div class="content">
|
||||||
<h3>changelog</h3>
|
<h3>changelog</h3>
|
||||||
<ul>
|
<ul>
|
||||||
|
<li><h3>version 120</h3></li>
|
||||||
|
<ul>
|
||||||
|
<li>improved quality of downsized animation rendering</li>
|
||||||
|
<li>sped up downsized animation rendering in this case</li>
|
||||||
|
<li>neatened animation code</li>
|
||||||
|
<li>fixed a mid-animation resize parameter bug</li>
|
||||||
|
<li>I think I fixed an animation scan bug that was sometimes giving the wrong frames</li>
|
||||||
|
<li>added thumbnails for all video formats</li>
|
||||||
|
<li>fixed an off-by-one framecount bug for certain videos, and retroactively fixed counts for existing videos</li>
|
||||||
|
<li>fixed a couple harmless width/height numpy.shape switcharounds</li>
|
||||||
|
<li>cleaned up some file parsing code</li>
|
||||||
|
<li>added a pixiv unit test</li>
|
||||||
|
<li>fixed a bug when servers were returning unusual relative redirect urls (gelbooru)</li>
|
||||||
|
<li>semi-fixed a bug when servers were returning unescaped redirect urls (gelbooru)</li>
|
||||||
|
</ul>
|
||||||
<li><h3>version 119</h3></li>
|
<li><h3>version 119</h3></li>
|
||||||
<ul>
|
<ul>
|
||||||
<li>fixed an overzealous error in v118 update code</li>
|
<li>fixed an overzealous error in v118 update code</li>
|
||||||
|
|
|
@ -3079,7 +3079,7 @@ class ServiceDB( FileDB, MessageDB, TagDB, RatingDB ):
|
||||||
|
|
||||||
if can_add:
|
if can_add:
|
||||||
|
|
||||||
( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path, hash )
|
( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path )
|
||||||
|
|
||||||
if width is not None and height is not None:
|
if width is not None and height is not None:
|
||||||
|
|
||||||
|
@ -4771,7 +4771,7 @@ class DB( ServiceDB ):
|
||||||
|
|
||||||
if resize_thumbs:
|
if resize_thumbs:
|
||||||
|
|
||||||
thumbnail_paths = ( path for path in CC.IterateAllThumbnailPaths() if path.endswith( '_resized' ) )
|
thumbnail_paths = [ path for path in CC.IterateAllThumbnailPaths() if path.endswith( '_resized' ) ]
|
||||||
|
|
||||||
for path in thumbnail_paths: os.remove( path )
|
for path in thumbnail_paths: os.remove( path )
|
||||||
|
|
||||||
|
@ -4907,10 +4907,52 @@ class DB( ServiceDB ):
|
||||||
|
|
||||||
i += 1
|
i += 1
|
||||||
|
|
||||||
|
if i % 100 == 0: HC.app.SetSplashText( 'reprocessing thumbs: ' + HC.ConvertIntToPrettyString( i ) )
|
||||||
|
|
||||||
except: print( 'When updating to v118, ' + path + '\'s phash could not be recalculated.' )
|
except: print( 'When updating to v118, ' + path + '\'s phash could not be recalculated.' )
|
||||||
|
|
||||||
if i % 100 == 0: HC.app.SetSplashText( 'reprocessing thumbs: ' + HC.ConvertIntToPrettyString( i ) )
|
|
||||||
|
|
||||||
|
|
||||||
|
if version == 119:
|
||||||
|
|
||||||
|
i = 0
|
||||||
|
|
||||||
|
for path in CC.IterateAllFilePaths():
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
filename = os.path.basename( path )
|
||||||
|
|
||||||
|
( hash_encoded, ext ) = filename.split( '.' )
|
||||||
|
|
||||||
|
hash = hash_encoded.decode( 'hex' )
|
||||||
|
|
||||||
|
hash_id = self._GetHashId( c, hash )
|
||||||
|
|
||||||
|
if ext not in ( 'flv', 'mp4', 'wmv', 'mkv', 'webm' ): continue
|
||||||
|
|
||||||
|
( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path )
|
||||||
|
|
||||||
|
c.execute( 'UPDATE files_info SET duration = ?, num_frames = ? WHERE hash_id = ?;', ( duration, num_frames, hash_id ) )
|
||||||
|
|
||||||
|
thumbnail = HydrusFileHandling.GenerateThumbnail( path )
|
||||||
|
|
||||||
|
thumbnail_path = CC.GetExpectedThumbnailPath( hash )
|
||||||
|
|
||||||
|
with open( thumbnail_path, 'wb' ) as f: f.write( thumbnail )
|
||||||
|
|
||||||
|
phash = HydrusImageHandling.GeneratePerceptualHash( thumbnail_path )
|
||||||
|
|
||||||
|
c.execute( 'INSERT OR REPLACE INTO perceptual_hashes ( hash_id, phash ) VALUES ( ?, ? );', ( hash_id, sqlite3.Binary( phash ) ) )
|
||||||
|
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
if i % 100 == 0: HC.app.SetSplashText( 'creating video thumbs: ' + HC.ConvertIntToPrettyString( i ) )
|
||||||
|
|
||||||
|
except:
|
||||||
|
print( traceback.format_exc())
|
||||||
|
print( 'When updating to v119, ' + path + '\'s thumbnail or phash could not be calculated.' )
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -64,7 +64,7 @@ options = {}
|
||||||
# Misc
|
# Misc
|
||||||
|
|
||||||
NETWORK_VERSION = 13
|
NETWORK_VERSION = 13
|
||||||
SOFTWARE_VERSION = 119
|
SOFTWARE_VERSION = 120
|
||||||
|
|
||||||
UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )
|
UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )
|
||||||
|
|
||||||
|
@ -303,7 +303,7 @@ NOISY_MIMES = tuple( [ APPLICATION_FLASH ] + list( AUDIO ) + list( VIDEO ) )
|
||||||
|
|
||||||
ARCHIVES = ( APPLICATION_ZIP, APPLICATION_HYDRUS_ENCRYPTED_ZIP )
|
ARCHIVES = ( APPLICATION_ZIP, APPLICATION_HYDRUS_ENCRYPTED_ZIP )
|
||||||
|
|
||||||
MIMES_WITH_THUMBNAILS = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP, VIDEO_WEBM )
|
MIMES_WITH_THUMBNAILS = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP, VIDEO_WEBM, VIDEO_FLV, VIDEO_MP4, VIDEO_WMV, VIDEO_MKV, VIDEO_WEBM )
|
||||||
|
|
||||||
# mp3 header is complicated
|
# mp3 header is complicated
|
||||||
|
|
||||||
|
|
|
@ -857,9 +857,7 @@ class DownloaderPixiv( Downloader ):
|
||||||
|
|
||||||
tag = self._query
|
tag = self._query
|
||||||
|
|
||||||
tag = urllib.quote( tag.encode( 'utf-8' ) )
|
gallery_url = 'http://www.pixiv.net/search.php?word=' + urllib.quote( tag.encode( 'utf-8' ) ) + '&s_mode=s_tag_full&order=date_d'
|
||||||
|
|
||||||
gallery_url = 'http://www.pixiv.net/search.php?word=' + urllib.quote( tag ) + '&s_mode=s_tag_full&order=date_d'
|
|
||||||
|
|
||||||
|
|
||||||
return gallery_url + '&p=' + HC.u( self._num_pages_done + 1 )
|
return gallery_url + '&p=' + HC.u( self._num_pages_done + 1 )
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
import cv2
|
#import cv2
|
||||||
import hashlib
|
import hashlib
|
||||||
import hsaudiotag
|
import hsaudiotag
|
||||||
import hsaudiotag.auto
|
import hsaudiotag.auto
|
||||||
|
@ -55,24 +55,36 @@ def GenerateThumbnail( path, dimensions = HC.UNSCALED_THUMBNAIL_DIMENSIONS ):
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
|
||||||
cv_video = cv2.VideoCapture( path )
|
( size, mime, width, height, duration, num_frames, num_words ) = GetFileInfo( path )
|
||||||
|
|
||||||
cv_video.set( cv2.cv.CV_CAP_PROP_CONVERT_RGB, True )
|
cropped_dimensions = HydrusImageHandling.GetThumbnailResolution( ( width, height ), dimensions )
|
||||||
|
|
||||||
( retval, cv_image ) = cv_video.read()
|
renderer = HydrusVideoHandling.VideoRendererFFMPEG( path, mime, duration, num_frames, cropped_dimensions )
|
||||||
|
|
||||||
if not retval: raise Exception( 'Could not read first frame of ' + HC.u( path ) + ' to create thumbnail!' )
|
numpy_image = renderer.read_frame()
|
||||||
|
|
||||||
cv_image = HydrusImageHandling.EfficientlyThumbnailCVImage( cv_image, dimensions )
|
pil_image = HydrusImageHandling.GeneratePILImageFromNumpyImage( numpy_image )
|
||||||
|
|
||||||
( retval, thumbnail ) = cv2.imencode( '.jpg', cv_image, ( cv2.cv.CV_IMWRITE_JPEG_QUALITY, 92 ) )
|
f = cStringIO.StringIO()
|
||||||
|
|
||||||
if not retval: raise Exception( 'Could not export thumbnail for ' + HC.u( path ) + '!' )
|
pil_image.save( f, 'JPEG', quality=92 )
|
||||||
|
|
||||||
|
f.seek( 0 )
|
||||||
|
|
||||||
|
thumbnail = f.read()
|
||||||
|
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
#numpy_image = cv2.cvtColor( numpy_image, cv2.COLOR_RGB2BGR )
|
||||||
|
|
||||||
|
#( retval, thumbnail ) = cv2.imencode( '.jpg', numpy_image, ( cv2.cv.CV_IMWRITE_JPEG_QUALITY, 92 ) )
|
||||||
|
|
||||||
|
#if not retval: raise Exception( 'Could not export thumbnail for ' + HC.u( path ) + '!' )
|
||||||
|
|
||||||
|
|
||||||
return thumbnail
|
return thumbnail
|
||||||
|
|
||||||
def GetFileInfo( path, hash ):
|
def GetFileInfo( path ):
|
||||||
|
|
||||||
info = os.lstat( path )
|
info = os.lstat( path )
|
||||||
|
|
||||||
|
@ -102,13 +114,9 @@ def GetFileInfo( path, hash ):
|
||||||
|
|
||||||
( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetFLVProperties( path )
|
( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetFLVProperties( path )
|
||||||
|
|
||||||
elif mime in ( HC.VIDEO_WMV, HC.VIDEO_MP4 ):
|
elif mime in ( HC.VIDEO_WMV, HC.VIDEO_MP4, HC.VIDEO_MKV, HC.VIDEO_WEBM ):
|
||||||
|
|
||||||
( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetCVVideoProperties( path )
|
( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetFFMPEGVideoProperties( path )
|
||||||
|
|
||||||
elif mime in ( HC.VIDEO_MKV, HC.VIDEO_WEBM ):
|
|
||||||
|
|
||||||
( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetCVVideoProperties( path )
|
|
||||||
|
|
||||||
elif mime == HC.APPLICATION_PDF: num_words = HydrusDocumentHandling.GetPDFNumWords( path )
|
elif mime == HC.APPLICATION_PDF: num_words = HydrusDocumentHandling.GetPDFNumWords( path )
|
||||||
elif mime == HC.AUDIO_MP3: duration = HydrusAudioHandling.GetMP3Duration( path )
|
elif mime == HC.AUDIO_MP3: duration = HydrusAudioHandling.GetMP3Duration( path )
|
||||||
|
|
|
@ -89,7 +89,7 @@ def GenerateCVImage( path ):
|
||||||
|
|
||||||
cv_image = cv2.imread( self._path, flags = -1 ) # flags = -1 loads alpha channel, if present
|
cv_image = cv2.imread( self._path, flags = -1 ) # flags = -1 loads alpha channel, if present
|
||||||
|
|
||||||
( x, y, depth ) = cv_image.shape
|
( y, x, depth ) = cv_image.shape
|
||||||
|
|
||||||
if depth == 4: raise Exception( 'CV is bad at alpha!' )
|
if depth == 4: raise Exception( 'CV is bad at alpha!' )
|
||||||
else: cv_image = cv2.cvtColor( cv_image, cv2.COLOR_BGR2RGB )
|
else: cv_image = cv2.cvtColor( cv_image, cv2.COLOR_BGR2RGB )
|
||||||
|
@ -102,7 +102,7 @@ def GenerateHydrusBitmap( path ):
|
||||||
|
|
||||||
cv_image = GenerateCVImage( path )
|
cv_image = GenerateCVImage( path )
|
||||||
|
|
||||||
return GenerateHydrusBitmapFromCVImage( cv_image )
|
return GenerateHydrusBitmapFromNumPyImage( cv_image )
|
||||||
|
|
||||||
except:
|
except:
|
||||||
|
|
||||||
|
@ -111,7 +111,7 @@ def GenerateHydrusBitmap( path ):
|
||||||
return GenerateHydrusBitmapFromPILImage( pil_image )
|
return GenerateHydrusBitmapFromPILImage( pil_image )
|
||||||
|
|
||||||
|
|
||||||
def GenerateHydrusBitmapFromCVImage( cv_image ):
|
def GenerateHydrusBitmapFromNumPyImage( cv_image ):
|
||||||
|
|
||||||
( y, x, depth ) = cv_image.shape
|
( y, x, depth ) = cv_image.shape
|
||||||
|
|
||||||
|
@ -137,7 +137,7 @@ def GeneratePerceptualHash( path ):
|
||||||
|
|
||||||
cv_image = cv2.imread( path, cv2.CV_LOAD_IMAGE_UNCHANGED )
|
cv_image = cv2.imread( path, cv2.CV_LOAD_IMAGE_UNCHANGED )
|
||||||
|
|
||||||
( x, y, depth ) = cv_image.shape
|
( y, x, depth ) = cv_image.shape
|
||||||
|
|
||||||
if depth == 4:
|
if depth == 4:
|
||||||
|
|
||||||
|
@ -153,15 +153,15 @@ def GeneratePerceptualHash( path ):
|
||||||
|
|
||||||
cv_image_gray = cv2.cvtColor( cv_image_bgr, cv2.COLOR_BGR2GRAY )
|
cv_image_gray = cv2.cvtColor( cv_image_bgr, cv2.COLOR_BGR2GRAY )
|
||||||
|
|
||||||
cv_image_result = numpy.empty( ( x, y ), numpy.float32 )
|
cv_image_result = numpy.empty( ( y, x ), numpy.float32 )
|
||||||
|
|
||||||
# paste greyscale onto the white
|
# paste greyscale onto the white
|
||||||
|
|
||||||
# can't think of a better way to do this!
|
# can't think of a better way to do this!
|
||||||
# cv2.addWeighted only takes a scalar for weight!
|
# cv2.addWeighted only takes a scalar for weight!
|
||||||
for i in range( x ):
|
for i in range( y ):
|
||||||
|
|
||||||
for j in range( y ):
|
for j in range( x ):
|
||||||
|
|
||||||
opacity = float( cv_alpha[ i, j ] ) / 255.0
|
opacity = float( cv_alpha[ i, j ] ) / 255.0
|
||||||
|
|
||||||
|
@ -321,6 +321,17 @@ def old_GeneratePerceptualHash( path ):
|
||||||
|
|
||||||
def GeneratePILImage( path ): return PILImage.open( path )
|
def GeneratePILImage( path ): return PILImage.open( path )
|
||||||
|
|
||||||
|
def GeneratePILImageFromNumpyImage( numpy_image ):
|
||||||
|
|
||||||
|
( h, w, depth ) = numpy_image.shape
|
||||||
|
|
||||||
|
if depth == 3: format = 'RGB'
|
||||||
|
elif depth == 4: format = 'RGBA'
|
||||||
|
|
||||||
|
pil_image = PILImage.fromstring( format, ( w, h ), numpy_image.data )
|
||||||
|
|
||||||
|
return pil_image
|
||||||
|
|
||||||
def GetGIFFrameDurations( path ):
|
def GetGIFFrameDurations( path ):
|
||||||
|
|
||||||
pil_image_for_duration = GeneratePILImage( path )
|
pil_image_for_duration = GeneratePILImage( path )
|
||||||
|
@ -568,7 +579,7 @@ class ImageContainer( RasterContainer ):
|
||||||
|
|
||||||
resized_cv_image = EfficientlyResizeCVImage( cv_image, self._target_resolution )
|
resized_cv_image = EfficientlyResizeCVImage( cv_image, self._target_resolution )
|
||||||
|
|
||||||
return GenerateHydrusBitmapFromCVImage( resized_cv_image )
|
return GenerateHydrusBitmapFromNumPyImage( resized_cv_image )
|
||||||
|
|
||||||
except:
|
except:
|
||||||
|
|
||||||
|
|
|
@ -3,6 +3,7 @@ import HydrusExceptions
|
||||||
import httplib
|
import httplib
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
import urllib
|
||||||
import urlparse
|
import urlparse
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
|
@ -132,12 +133,15 @@ class HTTPConnectionManager():
|
||||||
threading.Thread( target = self.DAEMONMaintainConnections, name = 'Maintain Connections' ).start()
|
threading.Thread( target = self.DAEMONMaintainConnections, name = 'Maintain Connections' ).start()
|
||||||
|
|
||||||
|
|
||||||
def _DoRequest( self, location, method, path_and_query, request_headers, body, follow_redirects = True, report_hooks = [], response_to_path = False, num_redirects_permitted = 4, long_timeout = False ):
|
def _DoRequest( self, method, location, path, query, request_headers, body, follow_redirects = True, report_hooks = [], response_to_path = False, num_redirects_permitted = 4, long_timeout = False ):
|
||||||
|
|
||||||
connection = self._GetConnection( location, long_timeout )
|
connection = self._GetConnection( location, long_timeout )
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
||||||
|
if query == '': path_and_query = path
|
||||||
|
else: path_and_query = path + '?' + query
|
||||||
|
|
||||||
with connection.lock:
|
with connection.lock:
|
||||||
|
|
||||||
( parsed_response, redirect_info, size_of_response, response_headers, cookies ) = connection.Request( method, path_and_query, request_headers, body, report_hooks = report_hooks, response_to_path = response_to_path )
|
( parsed_response, redirect_info, size_of_response, response_headers, cookies ) = connection.Request( method, path_and_query, request_headers, body, report_hooks = report_hooks, response_to_path = response_to_path )
|
||||||
|
@ -154,10 +158,7 @@ class HTTPConnectionManager():
|
||||||
|
|
||||||
if new_location is None: new_location = location
|
if new_location is None: new_location = location
|
||||||
|
|
||||||
if new_query != '': new_path_and_query = new_path + '?' + new_query
|
return self._DoRequest( new_method, new_location, new_path, new_query, request_headers, body, follow_redirects = follow_redirects, report_hooks = report_hooks, response_to_path = response_to_path, num_redirects_permitted = num_redirects_permitted - 1, long_timeout = long_timeout )
|
||||||
else: new_path_and_query = new_path
|
|
||||||
|
|
||||||
return self._DoRequest( new_location, new_method, new_path_and_query, request_headers, body, report_hooks = report_hooks, response_to_path = response_to_path, num_redirects_permitted = num_redirects_permitted - 1 )
|
|
||||||
|
|
||||||
|
|
||||||
except:
|
except:
|
||||||
|
@ -191,12 +192,9 @@ class HTTPConnectionManager():
|
||||||
|
|
||||||
( location, path, query ) = ParseURL( url )
|
( location, path, query ) = ParseURL( url )
|
||||||
|
|
||||||
if query != '': path_and_query = path + '?' + query
|
|
||||||
else: path_and_query = path
|
|
||||||
|
|
||||||
follow_redirects = not return_cookies
|
follow_redirects = not return_cookies
|
||||||
|
|
||||||
( response, size_of_response, response_headers, cookies ) = self._DoRequest( location, method, path_and_query, request_headers, body, follow_redirects = follow_redirects, report_hooks = report_hooks, response_to_path = response_to_path, long_timeout = long_timeout )
|
( response, size_of_response, response_headers, cookies ) = self._DoRequest( method, location, path, query, request_headers, body, follow_redirects = follow_redirects, report_hooks = report_hooks, response_to_path = response_to_path, long_timeout = long_timeout )
|
||||||
|
|
||||||
if return_everything: return ( response, size_of_response, response_headers, cookies )
|
if return_everything: return ( response, size_of_response, response_headers, cookies )
|
||||||
elif return_cookies: return ( response, cookies )
|
elif return_cookies: return ( response, cookies )
|
||||||
|
@ -453,6 +451,23 @@ class HTTPConnection():
|
||||||
|
|
||||||
url = location
|
url = location
|
||||||
|
|
||||||
|
if ' ' in url:
|
||||||
|
|
||||||
|
# some booru is giving daft redirect responses
|
||||||
|
print( url )
|
||||||
|
url = urllib.quote( url, safe = '/?=&' )
|
||||||
|
print( url )
|
||||||
|
|
||||||
|
if not url.startswith( self._scheme ):
|
||||||
|
|
||||||
|
# assume it is like 'index.php' or '/index.php', rather than 'http://blah.com/index.php'
|
||||||
|
|
||||||
|
if url.startswith( '/' ): slash_sep = ''
|
||||||
|
else: slash_sep = '/'
|
||||||
|
|
||||||
|
url = self._scheme + '://' + self._host + slash_sep + url
|
||||||
|
|
||||||
|
|
||||||
if response.status in ( 301, 307 ):
|
if response.status in ( 301, 307 ):
|
||||||
|
|
||||||
# 301: moved permanently, repeat request
|
# 301: moved permanently, repeat request
|
||||||
|
|
|
@ -238,7 +238,7 @@ def ParseFileArguments( path ):
|
||||||
|
|
||||||
hash = HydrusFileHandling.GetHashFromPath( path )
|
hash = HydrusFileHandling.GetHashFromPath( path )
|
||||||
|
|
||||||
try: ( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path, hash )
|
try: ( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path )
|
||||||
except HydrusExceptions.SizeException: raise HydrusExceptions.ForbiddenException( 'File is of zero length!' )
|
except HydrusExceptions.SizeException: raise HydrusExceptions.ForbiddenException( 'File is of zero length!' )
|
||||||
except HydrusExceptions.MimeException: raise HydrusExceptions.ForbiddenException( 'Filetype is not permitted!' )
|
except HydrusExceptions.MimeException: raise HydrusExceptions.ForbiddenException( 'Filetype is not permitted!' )
|
||||||
except Exception as e: raise HydrusExceptions.ForbiddenException( HC.u( e ) )
|
except Exception as e: raise HydrusExceptions.ForbiddenException( HC.u( e ) )
|
||||||
|
|
|
@ -40,6 +40,20 @@ def GetCVVideoProperties( path ):
|
||||||
|
|
||||||
return ( ( width, height ), duration, num_frames )
|
return ( ( width, height ), duration, num_frames )
|
||||||
|
|
||||||
|
def GetFFMPEGVideoProperties( path ):
|
||||||
|
|
||||||
|
info = Hydrusffmpeg_parse_infos( path )
|
||||||
|
|
||||||
|
( w, h ) = info[ 'video_size' ]
|
||||||
|
|
||||||
|
duration_in_s = info[ 'duration' ]
|
||||||
|
|
||||||
|
duration = int( duration_in_s * 1000 )
|
||||||
|
|
||||||
|
num_frames = info[ 'video_nframes' ]
|
||||||
|
|
||||||
|
return ( ( w, h ), duration, num_frames )
|
||||||
|
|
||||||
def GetFLVProperties( path ):
|
def GetFLVProperties( path ):
|
||||||
|
|
||||||
with open( path, 'rb' ) as f:
|
with open( path, 'rb' ) as f:
|
||||||
|
@ -127,144 +141,7 @@ def GetMatroskaOrWebMProperties( path ):
|
||||||
|
|
||||||
return ( ( width, height ), duration, num_frames )
|
return ( ( width, height ), duration, num_frames )
|
||||||
|
|
||||||
# This is cribbed from moviepy's FFMPEG_VideoReader
|
# this is cribbed from moviepy
|
||||||
class HydrusFFMPEG_VideoReader( object ):
|
|
||||||
|
|
||||||
def __init__(self, media, print_infos=False, bufsize = None, pix_fmt="rgb24"):
|
|
||||||
|
|
||||||
self._media = media
|
|
||||||
|
|
||||||
hash = self._media.GetHash()
|
|
||||||
mime = self._media.GetMime()
|
|
||||||
|
|
||||||
self._path = CC.GetFilePath( hash, mime )
|
|
||||||
|
|
||||||
self.size = self._media.GetResolution()
|
|
||||||
self.duration = float( self._media.GetDuration() ) / 1000.0
|
|
||||||
self.nframes = self._media.GetNumFrames()
|
|
||||||
|
|
||||||
self.fps = float( self.nframes ) / self.duration
|
|
||||||
|
|
||||||
self.pix_fmt = pix_fmt
|
|
||||||
|
|
||||||
if pix_fmt == 'rgba': self.depth = 4
|
|
||||||
else: self.depth = 3
|
|
||||||
|
|
||||||
if bufsize is None:
|
|
||||||
|
|
||||||
( x, y ) = self.size
|
|
||||||
bufsize = self.depth * x * y * 5
|
|
||||||
|
|
||||||
|
|
||||||
self.process = None
|
|
||||||
|
|
||||||
self.bufsize = bufsize
|
|
||||||
|
|
||||||
self.initialize()
|
|
||||||
|
|
||||||
|
|
||||||
def __del__( self ):
|
|
||||||
|
|
||||||
self.close()
|
|
||||||
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
|
|
||||||
if self.process is not None:
|
|
||||||
|
|
||||||
self.process.terminate()
|
|
||||||
|
|
||||||
self.process.stdout.close()
|
|
||||||
self.process.stderr.close()
|
|
||||||
|
|
||||||
self.process = None
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def initialize( self, starttime=0 ):
|
|
||||||
"""Opens the file, creates the pipe. """
|
|
||||||
|
|
||||||
self.close()
|
|
||||||
|
|
||||||
if starttime != 0:
|
|
||||||
|
|
||||||
offset = min( 1, starttime )
|
|
||||||
|
|
||||||
i_arg = [ '-ss', "%.03f" % ( starttime - offset ), '-i', '"' + self._path + '"' ]
|
|
||||||
|
|
||||||
else: i_arg = [ '-i', '"' + self._path + '"' ]
|
|
||||||
|
|
||||||
cmd = ([FFMPEG_PATH]+ i_arg +
|
|
||||||
['-loglevel', 'error',
|
|
||||||
'-f', 'image2pipe',
|
|
||||||
"-pix_fmt", self.pix_fmt,
|
|
||||||
'-vcodec', 'rawvideo', '-'])
|
|
||||||
|
|
||||||
self.process = subprocess.Popen( ' '.join( cmd ), shell = True, bufsize= self.bufsize, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
|
|
||||||
|
|
||||||
self.pos = int( round( self.fps * starttime ) )
|
|
||||||
|
|
||||||
|
|
||||||
def skip_frames(self, n=1):
|
|
||||||
|
|
||||||
"""Reads and throws away n frames """
|
|
||||||
|
|
||||||
( w, h ) = self.size
|
|
||||||
|
|
||||||
for i in range( n ):
|
|
||||||
|
|
||||||
self.process.stdout.read( self.depth * w * h )
|
|
||||||
self.process.stdout.flush()
|
|
||||||
|
|
||||||
|
|
||||||
self.pos += n
|
|
||||||
|
|
||||||
|
|
||||||
def read_frame( self ):
|
|
||||||
|
|
||||||
( w, h ) = self.size
|
|
||||||
|
|
||||||
nbytes = self.depth * w * h
|
|
||||||
|
|
||||||
s = self.process.stdout.read(nbytes)
|
|
||||||
|
|
||||||
self.pos += 1
|
|
||||||
|
|
||||||
if len(s) != nbytes:
|
|
||||||
|
|
||||||
print( "Warning: in file %s, "%(self._path)+
|
|
||||||
"%d bytes wanted but %d bytes read,"%(nbytes, len(s))+
|
|
||||||
"at frame %d/%d, at time %.02f/%.02f sec. "%(
|
|
||||||
self.pos,self.nframes,
|
|
||||||
1.0*self.pos/self.fps,
|
|
||||||
self.duration)+
|
|
||||||
"Using the last valid frame instead.")
|
|
||||||
result = self.lastread
|
|
||||||
|
|
||||||
self.close()
|
|
||||||
|
|
||||||
else:
|
|
||||||
|
|
||||||
result = numpy.fromstring( s, dtype = 'uint8' ).reshape( ( h, w, len( s ) // ( w * h ) ) )
|
|
||||||
|
|
||||||
self.lastread = result
|
|
||||||
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def set_position( self, pos ):
|
|
||||||
|
|
||||||
if ( pos < self.pos ) or ( pos > self.pos + 60 ):
|
|
||||||
|
|
||||||
starttime = float( pos ) / self.fps
|
|
||||||
|
|
||||||
self.initialize( starttime )
|
|
||||||
|
|
||||||
else: self.skip_frames( pos - self.pos )
|
|
||||||
|
|
||||||
|
|
||||||
# same thing; this is cribbed from moviepy
|
|
||||||
def Hydrusffmpeg_parse_infos(filename, print_infos=False):
|
def Hydrusffmpeg_parse_infos(filename, print_infos=False):
|
||||||
"""Get file infos using ffmpeg.
|
"""Get file infos using ffmpeg.
|
||||||
|
|
||||||
|
@ -306,9 +183,19 @@ def Hydrusffmpeg_parse_infos(filename, print_infos=False):
|
||||||
result = dict()
|
result = dict()
|
||||||
|
|
||||||
# get duration (in seconds)
|
# get duration (in seconds)
|
||||||
|
# Duration: 00:00:02.46, start: 0.033000, bitrate: 1069 kb/s
|
||||||
try:
|
try:
|
||||||
keyword = ('frame=' if is_GIF else 'Duration: ')
|
keyword = ('frame=' if is_GIF else 'Duration: ')
|
||||||
line = [l for l in lines if keyword in l][0]
|
line = [l for l in lines if keyword in l][0]
|
||||||
|
|
||||||
|
if 'start:' in line:
|
||||||
|
|
||||||
|
m = re.search( '(start\\: )' + '[0-9]\\.[0-9]*', line )
|
||||||
|
|
||||||
|
start_offset = float( line[ m.start() + 7 : m.end() ] )
|
||||||
|
|
||||||
|
else: start_offset = 0
|
||||||
|
|
||||||
match = re.search("[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9]", line)
|
match = re.search("[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9]", line)
|
||||||
hms = map(float, line[match.start()+1:match.end()].split(':'))
|
hms = map(float, line[match.start()+1:match.end()].split(':'))
|
||||||
|
|
||||||
|
@ -319,6 +206,8 @@ def Hydrusffmpeg_parse_infos(filename, print_infos=False):
|
||||||
elif len(hms) ==3:
|
elif len(hms) ==3:
|
||||||
result['duration'] = 3600*hms[0]+60*hms[1]+hms[2]
|
result['duration'] = 3600*hms[0]+60*hms[1]+hms[2]
|
||||||
|
|
||||||
|
result[ 'duration' ] -= start_offset
|
||||||
|
|
||||||
except:
|
except:
|
||||||
raise IOError("Error reading duration in file %s,"%(filename)+
|
raise IOError("Error reading duration in file %s,"%(filename)+
|
||||||
"Text parsed: %s"%infos)
|
"Text parsed: %s"%infos)
|
||||||
|
@ -345,9 +234,13 @@ def Hydrusffmpeg_parse_infos(filename, print_infos=False):
|
||||||
except:
|
except:
|
||||||
match = re.search("( [0-9]*.| )[0-9]* fps", line)
|
match = re.search("( [0-9]*.| )[0-9]* fps", line)
|
||||||
result['video_fps'] = float(line[match.start():match.end()].split(' ')[1])
|
result['video_fps'] = float(line[match.start():match.end()].split(' ')[1])
|
||||||
|
|
||||||
result['video_nframes'] = int(result['duration']*result['video_fps'])+1
|
num_frames = result['duration'] * result['video_fps']
|
||||||
|
|
||||||
|
if num_frames != int( num_frames ): num_frames += 1 # rounding up
|
||||||
|
|
||||||
|
result['video_nframes'] = int( num_frames )
|
||||||
|
|
||||||
result['video_duration'] = result['duration']
|
result['video_duration'] = result['duration']
|
||||||
# We could have also recomputed the duration from the number
|
# We could have also recomputed the duration from the number
|
||||||
# of frames, as follows:
|
# of frames, as follows:
|
||||||
|
@ -378,8 +271,8 @@ class VideoContainer( HydrusImageHandling.RasterContainer ):
|
||||||
|
|
||||||
self._frames = {}
|
self._frames = {}
|
||||||
self._last_index_asked_for = -1
|
self._last_index_asked_for = -1
|
||||||
self._minimum_frame_asked_for = 0
|
self._buffer_start_index = -1
|
||||||
self._maximum_frame_asked_for = 0
|
self._buffer_end_index = -1
|
||||||
|
|
||||||
( x, y ) = self._target_resolution
|
( x, y ) = self._target_resolution
|
||||||
|
|
||||||
|
@ -391,9 +284,21 @@ class VideoContainer( HydrusImageHandling.RasterContainer ):
|
||||||
if self._media.GetMime() == HC.IMAGE_GIF: self._durations = HydrusImageHandling.GetGIFFrameDurations( self._path )
|
if self._media.GetMime() == HC.IMAGE_GIF: self._durations = HydrusImageHandling.GetGIFFrameDurations( self._path )
|
||||||
else: self._frame_duration = GetVideoFrameDuration( self._path )
|
else: self._frame_duration = GetVideoFrameDuration( self._path )
|
||||||
|
|
||||||
self._renderer = VideoRendererMoviePy( self, self._media, self._target_resolution )
|
self._render_lock = threading.Lock()
|
||||||
|
|
||||||
num_frames = self.GetNumFrames()
|
hash = self._media.GetHash()
|
||||||
|
mime = self._media.GetMime()
|
||||||
|
|
||||||
|
path = CC.GetFilePath( hash, mime )
|
||||||
|
|
||||||
|
duration = self._media.GetDuration()
|
||||||
|
num_frames = self._media.GetNumFrames()
|
||||||
|
|
||||||
|
self._ffmpeg_reader = VideoRendererFFMPEG( path, mime, duration, num_frames, target_resolution )
|
||||||
|
|
||||||
|
self._next_render_index = 0
|
||||||
|
self._render_to_index = -1
|
||||||
|
self._rendered_first_frame = False
|
||||||
|
|
||||||
self.SetPosition( init_position )
|
self.SetPosition( init_position )
|
||||||
|
|
||||||
|
@ -404,19 +309,81 @@ class VideoContainer( HydrusImageHandling.RasterContainer ):
|
||||||
|
|
||||||
for index in self._frames.keys():
|
for index in self._frames.keys():
|
||||||
|
|
||||||
if self._minimum_frame_asked_for < self._maximum_frame_asked_for:
|
if self._buffer_start_index < self._buffer_end_index:
|
||||||
|
|
||||||
if index < self._minimum_frame_asked_for or self._maximum_frame_asked_for < index: deletees.append( index )
|
if index < self._buffer_start_index or self._buffer_end_index < index: deletees.append( index )
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
|
||||||
if self._maximum_frame_asked_for < index and index < self._minimum_frame_asked_for: deletees.append( index )
|
if self._buffer_end_index < index and index < self._buffer_start_index: deletees.append( index )
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
for i in deletees: del self._frames[ i ]
|
for i in deletees: del self._frames[ i ]
|
||||||
|
|
||||||
|
|
||||||
|
def _RENDERERSetRenderToPosition( self, index ):
|
||||||
|
|
||||||
|
with self._render_lock:
|
||||||
|
|
||||||
|
if self._render_to_index != index:
|
||||||
|
|
||||||
|
self._render_to_index = index
|
||||||
|
|
||||||
|
HydrusThreading.CallToThread( self.THREADRender )
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def _RENDERERSetPosition( self, index ):
|
||||||
|
|
||||||
|
with self._render_lock:
|
||||||
|
|
||||||
|
if index == self._next_render_index: return
|
||||||
|
else:
|
||||||
|
|
||||||
|
self._ffmpeg_reader.set_position( index )
|
||||||
|
|
||||||
|
self._next_render_index = index
|
||||||
|
self._render_to_index = index
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def THREADRender( self ):
|
||||||
|
|
||||||
|
num_frames = self._media.GetNumFrames()
|
||||||
|
|
||||||
|
while True:
|
||||||
|
|
||||||
|
time.sleep( 0.00001 ) # thread yield
|
||||||
|
|
||||||
|
with self._render_lock:
|
||||||
|
|
||||||
|
if not self._rendered_first_frame or self._next_render_index != ( self._render_to_index + 1 ) % num_frames:
|
||||||
|
|
||||||
|
self._rendered_first_frame = True
|
||||||
|
|
||||||
|
frame_index = self._next_render_index # keep this before the get call, as it increments in a clock arithmetic way afterwards
|
||||||
|
|
||||||
|
try: numpy_image = self._ffmpeg_reader.read_frame()
|
||||||
|
except Exception as e:
|
||||||
|
|
||||||
|
HC.ShowException( e )
|
||||||
|
|
||||||
|
break
|
||||||
|
|
||||||
|
finally: self._next_render_index = ( self._next_render_index + 1 ) % num_frames
|
||||||
|
|
||||||
|
frame = HydrusImageHandling.GenerateHydrusBitmapFromNumPyImage( numpy_image )
|
||||||
|
|
||||||
|
wx.CallAfter( self.AddFrame, frame_index, frame )
|
||||||
|
|
||||||
|
else: break
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def AddFrame( self, index, frame ): self._frames[ index ] = frame
|
def AddFrame( self, index, frame ): self._frames[ index ] = frame
|
||||||
|
|
||||||
def GetDuration( self, index ):
|
def GetDuration( self, index ):
|
||||||
|
@ -464,61 +431,205 @@ class VideoContainer( HydrusImageHandling.RasterContainer ):
|
||||||
|
|
||||||
if num_frames > self._num_frames_backwards + 1 + self._num_frames_forwards:
|
if num_frames > self._num_frames_backwards + 1 + self._num_frames_forwards:
|
||||||
|
|
||||||
new_minimum_frame_to_ask_for = max( 0, index - self._num_frames_backwards ) % num_frames
|
new_buffer_start_index = max( 0, index - self._num_frames_backwards ) % num_frames
|
||||||
|
|
||||||
new_maximum_frame_to_ask_for = ( index + self._num_frames_forwards ) % num_frames
|
new_buffer_end_index = ( index + self._num_frames_forwards ) % num_frames
|
||||||
|
|
||||||
if index == self._last_index_asked_for: return
|
if index == self._last_index_asked_for: return
|
||||||
elif index < self._last_index_asked_for:
|
elif index < self._last_index_asked_for:
|
||||||
|
|
||||||
if index < self._minimum_frame_asked_for:
|
if index < self._buffer_start_index:
|
||||||
|
|
||||||
self._minimum_frame_asked_for = new_minimum_frame_to_ask_for
|
self._buffer_start_index = new_buffer_start_index
|
||||||
|
|
||||||
self._renderer.SetPosition( self._minimum_frame_asked_for )
|
self._RENDERERSetPosition( self._buffer_start_index )
|
||||||
|
|
||||||
self._maximum_frame_asked_for = new_maximum_frame_to_ask_for
|
self._buffer_end_index = new_buffer_end_index
|
||||||
|
|
||||||
self._renderer.SetRenderToPosition( self._maximum_frame_asked_for )
|
self._RENDERERSetRenderToPosition( self._buffer_end_index )
|
||||||
|
|
||||||
|
|
||||||
else: # index > self._last_index_asked_for
|
else: # index > self._last_index_asked_for
|
||||||
|
|
||||||
currently_no_wraparound = self._minimum_frame_asked_for < self._maximum_frame_asked_for
|
currently_no_wraparound = self._buffer_start_index < self._buffer_end_index
|
||||||
|
|
||||||
self._minimum_frame_asked_for = new_minimum_frame_to_ask_for
|
self._buffer_start_index = new_buffer_start_index
|
||||||
|
|
||||||
if currently_no_wraparound:
|
if currently_no_wraparound:
|
||||||
|
|
||||||
if index > self._maximum_frame_asked_for:
|
if index > self._buffer_end_index:
|
||||||
|
|
||||||
self._renderer.SetPosition( self._minimum_frame_asked_for )
|
self._RENDERERSetPosition( self._buffer_start_index )
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
self._maximum_frame_asked_for = new_maximum_frame_to_ask_for
|
self._buffer_end_index = new_buffer_end_index
|
||||||
|
|
||||||
self._renderer.SetRenderToPosition( self._maximum_frame_asked_for )
|
self._RENDERERSetRenderToPosition( self._buffer_end_index )
|
||||||
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
|
||||||
if self._maximum_frame_asked_for == 0:
|
if self._buffer_end_index == -1:
|
||||||
|
|
||||||
self._minimum_frame_asked_for = 0
|
self._buffer_start_index = 0
|
||||||
|
|
||||||
self._renderer.SetPosition( 0 )
|
self._RENDERERSetPosition( 0 )
|
||||||
|
|
||||||
self._maximum_frame_asked_for = num_frames - 1
|
self._buffer_end_index = num_frames - 1
|
||||||
|
|
||||||
self._renderer.SetRenderToPosition( self._maximum_frame_asked_for )
|
self._RENDERERSetRenderToPosition( self._buffer_end_index )
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
self._MaintainBuffer()
|
self._MaintainBuffer()
|
||||||
|
|
||||||
|
|
||||||
class VideoRendererCV():
|
# This was built from moviepy's FFMPEG_VideoReader
|
||||||
|
class VideoRendererFFMPEG( object ):
|
||||||
|
|
||||||
|
def __init__( self, path, mime, duration, num_frames, target_resolution, pix_fmt = "rgb24" ):
|
||||||
|
|
||||||
|
self._path = path
|
||||||
|
self._mime = mime
|
||||||
|
self._duration = float( duration ) / 1000.0
|
||||||
|
self._num_frames = num_frames
|
||||||
|
self._target_resolution = target_resolution
|
||||||
|
|
||||||
|
self.fps = float( self._num_frames ) / self._duration
|
||||||
|
|
||||||
|
if self.fps == 0: self.fps = 24
|
||||||
|
|
||||||
|
self.pix_fmt = pix_fmt
|
||||||
|
|
||||||
|
if pix_fmt == 'rgba': self.depth = 4
|
||||||
|
else: self.depth = 3
|
||||||
|
|
||||||
|
( x, y ) = self._target_resolution
|
||||||
|
bufsize = self.depth * x * y
|
||||||
|
|
||||||
|
self.process = None
|
||||||
|
|
||||||
|
self.bufsize = bufsize
|
||||||
|
|
||||||
|
self.initialize()
|
||||||
|
|
||||||
|
|
||||||
|
def __del__( self ):
|
||||||
|
|
||||||
|
self.close()
|
||||||
|
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
|
||||||
|
if self.process is not None:
|
||||||
|
|
||||||
|
self.process.terminate()
|
||||||
|
|
||||||
|
self.process.stdout.close()
|
||||||
|
self.process.stderr.close()
|
||||||
|
|
||||||
|
self.process = None
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def initialize( self, start_index = 0 ):
|
||||||
|
|
||||||
|
self.close()
|
||||||
|
|
||||||
|
if self._mime == HC.IMAGE_GIF:
|
||||||
|
|
||||||
|
ss = 0
|
||||||
|
self.pos = 0
|
||||||
|
skip_frames = start_index
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
ss = float( start_index ) / self.fps
|
||||||
|
self.pos = start_index
|
||||||
|
skip_frames = 0
|
||||||
|
|
||||||
|
|
||||||
|
( w, h ) = self._target_resolution
|
||||||
|
|
||||||
|
cmd = ( [ FFMPEG_PATH,
|
||||||
|
'-ss', "%.03f" % ss,
|
||||||
|
'-i', '"' + self._path + '"',
|
||||||
|
'-loglevel', 'quiet',
|
||||||
|
'-f', 'image2pipe',
|
||||||
|
"-pix_fmt", self.pix_fmt,
|
||||||
|
"-s", str( w ) + 'x' + str( h ),
|
||||||
|
'-vcodec', 'rawvideo', '-' ] )
|
||||||
|
|
||||||
|
self.process = subprocess.Popen( ' '.join( cmd ), shell = True, bufsize= self.bufsize, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
|
||||||
|
|
||||||
|
self.skip_frames( skip_frames )
|
||||||
|
|
||||||
|
|
||||||
|
def skip_frames( self, n ):
|
||||||
|
|
||||||
|
( w, h ) = self._target_resolution
|
||||||
|
|
||||||
|
for i in range( n ):
|
||||||
|
|
||||||
|
self.process.stdout.read( self.depth * w * h )
|
||||||
|
|
||||||
|
self.process.stdout.flush()
|
||||||
|
|
||||||
|
self.pos += 1
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def read_frame( self ):
|
||||||
|
|
||||||
|
if self.pos == self._num_frames: self.initialize()
|
||||||
|
|
||||||
|
if self.process is None: result = self.lastread
|
||||||
|
else:
|
||||||
|
|
||||||
|
( w, h ) = self._target_resolution
|
||||||
|
|
||||||
|
nbytes = self.depth * w * h
|
||||||
|
|
||||||
|
s = self.process.stdout.read(nbytes)
|
||||||
|
|
||||||
|
if len(s) != nbytes:
|
||||||
|
|
||||||
|
print( "Warning: in file %s, "%(self._path)+
|
||||||
|
"%d bytes wanted but %d bytes read,"%(nbytes, len(s))+
|
||||||
|
"at frame %d/%d, at time %.02f/%.02f sec. "%(
|
||||||
|
self.pos,self._num_frames,
|
||||||
|
1.0*self.pos/self.fps,
|
||||||
|
self._duration)+
|
||||||
|
"Using the last valid frame instead.")
|
||||||
|
|
||||||
|
result = self.lastread
|
||||||
|
|
||||||
|
self.close()
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
result = numpy.fromstring( s, dtype = 'uint8' ).reshape( ( h, w, len( s ) // ( w * h ) ) )
|
||||||
|
|
||||||
|
self.lastread = result
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
self.pos += 1
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def set_position( self, pos ):
|
||||||
|
|
||||||
|
rewind = pos < self.pos
|
||||||
|
jump_a_long_way_ahead = pos > self.pos + 60
|
||||||
|
|
||||||
|
if rewind or jump_a_long_way_ahead: self.initialize( pos )
|
||||||
|
else: self.skip_frames( pos - self.pos )
|
||||||
|
|
||||||
|
|
||||||
|
class OLDCODEVideoRendererCV():
|
||||||
|
|
||||||
def __init__( self, image_container, media, target_resolution ):
|
def __init__( self, image_container, media, target_resolution ):
|
||||||
|
|
||||||
|
@ -531,13 +642,13 @@ class VideoRendererCV():
|
||||||
|
|
||||||
self._path = CC.GetFilePath( hash, mime )
|
self._path = CC.GetFilePath( hash, mime )
|
||||||
|
|
||||||
self._lock = threading.Lock()
|
self._render_lock = threading.Lock()
|
||||||
|
|
||||||
self._cv_video = cv2.VideoCapture( self._path )
|
self._cv_video = cv2.VideoCapture( self._path )
|
||||||
|
|
||||||
self._cv_video.set( cv2.cv.CV_CAP_PROP_CONVERT_RGB, True )
|
self._cv_video.set( cv2.cv.CV_CAP_PROP_CONVERT_RGB, True )
|
||||||
|
|
||||||
self._current_index = 0
|
self._next_render_index = 0
|
||||||
self._last_index_rendered = -1
|
self._last_index_rendered = -1
|
||||||
self._last_frame = None
|
self._last_frame = None
|
||||||
self._render_to_index = -1
|
self._render_to_index = -1
|
||||||
|
@ -547,13 +658,13 @@ class VideoRendererCV():
|
||||||
|
|
||||||
( retval, cv_image ) = self._cv_video.read()
|
( retval, cv_image ) = self._cv_video.read()
|
||||||
|
|
||||||
self._last_index_rendered = self._current_index
|
self._last_index_rendered = self._next_render_index
|
||||||
|
|
||||||
num_frames = self._media.GetNumFrames()
|
num_frames = self._media.GetNumFrames()
|
||||||
|
|
||||||
self._current_index = ( self._current_index + 1 ) % num_frames
|
self._next_render_index = ( self._next_render_index + 1 ) % num_frames
|
||||||
|
|
||||||
if self._current_index == 0 and self._last_index_rendered != 0:
|
if self._next_render_index == 0 and self._last_index_rendered != 0:
|
||||||
|
|
||||||
if self._media.GetMime() == HC.IMAGE_GIF: self._RewindGIF()
|
if self._media.GetMime() == HC.IMAGE_GIF: self._RewindGIF()
|
||||||
else: self._cv_video.set( cv2.cv.CV_CAP_PROP_POS_FRAMES, 0.0 )
|
else: self._cv_video.set( cv2.cv.CV_CAP_PROP_POS_FRAMES, 0.0 )
|
||||||
|
@ -561,7 +672,7 @@ class VideoRendererCV():
|
||||||
|
|
||||||
if not retval:
|
if not retval:
|
||||||
|
|
||||||
raise HydrusExceptions.CantRenderWithCVException( 'CV could not render frame ' + HC.u( self._current_index ) + '.' )
|
raise HydrusExceptions.CantRenderWithCVException( 'CV could not render frame ' + HC.u( self._next_render_index ) + '.' )
|
||||||
|
|
||||||
|
|
||||||
return cv_image
|
return cv_image
|
||||||
|
@ -584,27 +695,7 @@ class VideoRendererCV():
|
||||||
cv_image = self._last_frame
|
cv_image = self._last_frame
|
||||||
|
|
||||||
|
|
||||||
return HydrusImageHandling.GenerateHydrusBitmapFromCVImage( cv_image )
|
return HydrusImageHandling.GenerateHydrusBitmapFromNumPyImage( cv_image )
|
||||||
|
|
||||||
|
|
||||||
def _RenderFrames( self ):
|
|
||||||
|
|
||||||
no_frames_yet = True
|
|
||||||
|
|
||||||
while True:
|
|
||||||
|
|
||||||
try:
|
|
||||||
|
|
||||||
yield self._RenderCurrentFrame()
|
|
||||||
|
|
||||||
no_frames_yet = False
|
|
||||||
|
|
||||||
except HydrusExceptions.CantRenderWithCVException:
|
|
||||||
|
|
||||||
if no_frames_yet: raise
|
|
||||||
else: break
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def _RewindGIF( self ):
|
def _RewindGIF( self ):
|
||||||
|
@ -612,12 +703,12 @@ class VideoRendererCV():
|
||||||
self._cv_video.release()
|
self._cv_video.release()
|
||||||
self._cv_video.open( self._path )
|
self._cv_video.open( self._path )
|
||||||
|
|
||||||
self._current_index = 0
|
self._next_render_index = 0
|
||||||
|
|
||||||
|
|
||||||
def SetRenderToPosition( self, index ):
|
def SetRenderToPosition( self, index ):
|
||||||
|
|
||||||
with self._lock:
|
with self._render_lock:
|
||||||
|
|
||||||
if self._render_to_index != index:
|
if self._render_to_index != index:
|
||||||
|
|
||||||
|
@ -630,14 +721,14 @@ class VideoRendererCV():
|
||||||
|
|
||||||
def SetPosition( self, index ):
|
def SetPosition( self, index ):
|
||||||
|
|
||||||
with self._lock:
|
with self._render_lock:
|
||||||
|
|
||||||
if self._media.GetMime() == HC.IMAGE_GIF:
|
if self._media.GetMime() == HC.IMAGE_GIF:
|
||||||
|
|
||||||
if index == self._current_index: return
|
if index == self._next_render_index: return
|
||||||
elif index < self._current_index: self._RewindGIF()
|
elif index < self._next_render_index: self._RewindGIF()
|
||||||
|
|
||||||
while self._current_index < index: self._GetCurrentFrame()
|
while self._next_render_index < index: self._GetCurrentFrame()
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
|
||||||
|
@ -654,149 +745,11 @@ class VideoRendererCV():
|
||||||
|
|
||||||
time.sleep( 0.00001 ) # thread yield
|
time.sleep( 0.00001 ) # thread yield
|
||||||
|
|
||||||
with self._lock:
|
with self._render_lock:
|
||||||
|
|
||||||
if self._last_index_rendered != self._render_to_index:
|
if self._last_index_rendered != self._render_to_index:
|
||||||
|
|
||||||
index = self._current_index
|
index = self._next_render_index
|
||||||
|
|
||||||
frame = self._RenderCurrentFrame()
|
|
||||||
|
|
||||||
wx.CallAfter( self._image_container.AddFrame, index, frame )
|
|
||||||
|
|
||||||
else: break
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class VideoRendererMoviePy():
|
|
||||||
|
|
||||||
def __init__( self, image_container, media, target_resolution ):
|
|
||||||
|
|
||||||
self._image_container = image_container
|
|
||||||
self._media = media
|
|
||||||
self._target_resolution = target_resolution
|
|
||||||
|
|
||||||
self._lock = threading.Lock()
|
|
||||||
|
|
||||||
( x, y ) = media.GetResolution()
|
|
||||||
|
|
||||||
self._ffmpeg_reader = HydrusFFMPEG_VideoReader( media, bufsize = x * y * 3 * 5 )
|
|
||||||
|
|
||||||
self._current_index = 0
|
|
||||||
self._last_index_rendered = -1
|
|
||||||
self._last_frame = None
|
|
||||||
self._render_to_index = -1
|
|
||||||
|
|
||||||
|
|
||||||
def _GetCurrentFrame( self ):
|
|
||||||
|
|
||||||
try: image = self._ffmpeg_reader.read_frame()
|
|
||||||
except Exception as e: raise HydrusExceptions.CantRenderWithCVException( 'FFMPEG could not render frame ' + HC.u( self._current_index ) + '.' + os.linesep * 2 + HC.u( e ) )
|
|
||||||
|
|
||||||
self._last_index_rendered = self._current_index
|
|
||||||
|
|
||||||
num_frames = self._media.GetNumFrames()
|
|
||||||
|
|
||||||
self._current_index = ( self._current_index + 1 ) % num_frames
|
|
||||||
|
|
||||||
if self._current_index == 0 and self._last_index_rendered != 0: self._ffmpeg_reader.initialize()
|
|
||||||
|
|
||||||
return image
|
|
||||||
|
|
||||||
|
|
||||||
def _RenderCurrentFrame( self ):
|
|
||||||
|
|
||||||
try:
|
|
||||||
|
|
||||||
image = self._GetCurrentFrame()
|
|
||||||
|
|
||||||
image = HydrusImageHandling.EfficientlyResizeCVImage( image, self._target_resolution )
|
|
||||||
|
|
||||||
self._last_frame = image
|
|
||||||
|
|
||||||
except HydrusExceptions.CantRenderWithCVException:
|
|
||||||
|
|
||||||
if self._last_frame is None: raise
|
|
||||||
|
|
||||||
image = self._last_frame
|
|
||||||
|
|
||||||
|
|
||||||
return HydrusImageHandling.GenerateHydrusBitmapFromCVImage( image )
|
|
||||||
|
|
||||||
|
|
||||||
def _RenderFrames( self ):
|
|
||||||
|
|
||||||
no_frames_yet = True
|
|
||||||
|
|
||||||
while True:
|
|
||||||
|
|
||||||
try:
|
|
||||||
|
|
||||||
yield self._RenderCurrentFrame()
|
|
||||||
|
|
||||||
no_frames_yet = False
|
|
||||||
|
|
||||||
except HydrusExceptions.CantRenderWithCVException:
|
|
||||||
|
|
||||||
if no_frames_yet: raise
|
|
||||||
else: break
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def SetRenderToPosition( self, index ):
|
|
||||||
|
|
||||||
with self._lock:
|
|
||||||
|
|
||||||
if self._render_to_index != index:
|
|
||||||
|
|
||||||
self._render_to_index = index
|
|
||||||
|
|
||||||
HydrusThreading.CallToThread( self.THREADDoWork )
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def SetPosition( self, index ):
|
|
||||||
|
|
||||||
with self._lock:
|
|
||||||
|
|
||||||
if index == self._current_index: return
|
|
||||||
else:
|
|
||||||
|
|
||||||
if self._media.GetMime() == HC.IMAGE_GIF:
|
|
||||||
|
|
||||||
if index < self._current_index:
|
|
||||||
|
|
||||||
self._ffmpeg_reader.initialize()
|
|
||||||
|
|
||||||
self._ffmpeg_reader.skip_frames( index )
|
|
||||||
|
|
||||||
|
|
||||||
else:
|
|
||||||
|
|
||||||
timecode = float( index ) / self._ffmpeg_reader.fps
|
|
||||||
|
|
||||||
self._ffmpeg_reader.set_position( timecode )
|
|
||||||
|
|
||||||
|
|
||||||
self._render_to_index = index
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def THREADDoWork( self ):
|
|
||||||
|
|
||||||
while True:
|
|
||||||
|
|
||||||
time.sleep( 0.00001 ) # thread yield
|
|
||||||
|
|
||||||
with self._lock:
|
|
||||||
|
|
||||||
if self._last_index_rendered != self._render_to_index:
|
|
||||||
|
|
||||||
index = self._current_index
|
|
||||||
|
|
||||||
frame = self._RenderCurrentFrame()
|
frame = self._RenderCurrentFrame()
|
||||||
|
|
||||||
|
|
|
@ -98,7 +98,46 @@ class TestDownloaders( unittest.TestCase ):
|
||||||
|
|
||||||
info = ( data, tags )
|
info = ( data, tags )
|
||||||
|
|
||||||
expected_info = ('swf file', set([u'chores', u'laser', u'silent', u'title:Cat Dust', u'creator:warlord-of-noodles', u'pointer']))
|
expected_info = ( 'swf file', set([u'chores', u'laser', u'silent', u'title:Cat Dust', u'creator:warlord-of-noodles', u'pointer']) )
|
||||||
|
|
||||||
|
self.assertEqual( info, expected_info )
|
||||||
|
|
||||||
|
|
||||||
|
def test_pixiv( self ):
|
||||||
|
|
||||||
|
with open( HC.STATIC_DIR + os.path.sep + 'testing' + os.path.sep + 'pixiv_gallery.html' ) as f: pixiv_gallery = f.read()
|
||||||
|
with open( HC.STATIC_DIR + os.path.sep + 'testing' + os.path.sep + 'pixiv_page.html' ) as f: pixiv_page = f.read()
|
||||||
|
|
||||||
|
HC.http.SetResponse( HC.GET, 'http://www.pixiv.net/search.php?word=naruto&s_mode=s_tag_full&order=date_d&p=1', pixiv_gallery )
|
||||||
|
HC.http.SetResponse( HC.GET, 'http://www.pixiv.net/member_illust.php?mode=medium&illust_id=43718605', pixiv_page )
|
||||||
|
|
||||||
|
HC.http.SetResponse( HC.GET, 'http://i1.pixiv.net/img59/img/dbhope/43718605.jpg', 'image file' )
|
||||||
|
|
||||||
|
#
|
||||||
|
|
||||||
|
downloader = HydrusDownloading.DownloaderPixiv( 'tags', 'naruto' )
|
||||||
|
|
||||||
|
#
|
||||||
|
|
||||||
|
gallery_urls = downloader.GetAnotherPage()
|
||||||
|
|
||||||
|
expected_gallery_urls = [ ( u'http://www.pixiv.net/member_illust.php?mode=medium&illust_id=43718605', ), 'a bunch of others' ]
|
||||||
|
|
||||||
|
self.assertEqual( gallery_urls[0], expected_gallery_urls[0] )
|
||||||
|
|
||||||
|
#
|
||||||
|
|
||||||
|
info = downloader.GetFileAndTags( 'http://www.pixiv.net/member_illust.php?mode=medium&illust_id=43718605' )
|
||||||
|
|
||||||
|
( temp_path, tags ) = info
|
||||||
|
|
||||||
|
with open( temp_path, 'rb' ) as f: data = f.read()
|
||||||
|
|
||||||
|
info = ( data, tags )
|
||||||
|
|
||||||
|
expected_tags = [ u'1P\u6f2b\u753b', u'\u7720\u305f\u3044', u'NARUTO', u'\u30ca\u30eb\u30c8', u'\u30b5\u30b9\u30b1', u'\u30b5\u30af\u30e9', u'\u30d2\u30ca\u30bf', u'creator:\u30df\u30c4\u30ad\u30e8\u3063\u3057\uff5e', u'title:\u7720\u305f\u3044', 'creator:dbhope' ]
|
||||||
|
|
||||||
|
expected_info = ( 'image file', expected_tags )
|
||||||
|
|
||||||
self.assertEqual( info, expected_info )
|
self.assertEqual( info, expected_info )
|
||||||
|
|
||||||
|
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue