Version 120
This commit is contained in:
Hydrus
parent
babac9e405
commit
11f375509f
|
|
@ -8,6 +8,21 @@
|
|||
<div class="content">
|
||||
<h3>changelog</h3>
|
||||
<ul>
|
||||
<li><h3>version 120</h3></li>
|
||||
<ul>
|
||||
<li>improved quality of downsized animation rendering</li>
|
||||
<li>sped up downsized animation rendering in this case</li>
|
||||
<li>neatened animation code</li>
|
||||
<li>fixed a mid-animation resize parameter bug</li>
|
||||
<li>I think I fixed an animation scan bug that was sometimes giving the wrong frames</li>
|
||||
<li>added thumbnails for all video formats</li>
|
||||
<li>fixed an off-by-one framecount bug for certain videos, and retroactively fixed counts for existing videos</li>
|
||||
<li>fixed a couple harmless width/height numpy.shape switcharounds</li>
|
||||
<li>cleaned up some file parsing code</li>
|
||||
<li>added a pixiv unit test</li>
|
||||
<li>fixed a bug when servers were returning unusual relative redirect urls (gelbooru)</li>
|
||||
<li>semi-fixed a bug when servers were returning unescaped redirect urls (gelbooru)</li>
|
||||
</ul>
|
||||
<li><h3>version 119</h3></li>
|
||||
<ul>
|
||||
<li>fixed an overzealous error in v118 update code</li>
|
||||
|
|
|
|||
|
|
@ -3079,7 +3079,7 @@ class ServiceDB( FileDB, MessageDB, TagDB, RatingDB ):
|
|||
|
||||
if can_add:
|
||||
|
||||
( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path, hash )
|
||||
( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path )
|
||||
|
||||
if width is not None and height is not None:
|
||||
|
||||
|
|
@ -4771,7 +4771,7 @@ class DB( ServiceDB ):
|
|||
|
||||
if resize_thumbs:
|
||||
|
||||
thumbnail_paths = ( path for path in CC.IterateAllThumbnailPaths() if path.endswith( '_resized' ) )
|
||||
thumbnail_paths = [ path for path in CC.IterateAllThumbnailPaths() if path.endswith( '_resized' ) ]
|
||||
|
||||
for path in thumbnail_paths: os.remove( path )
|
||||
|
||||
|
|
@ -4907,10 +4907,52 @@ class DB( ServiceDB ):
|
|||
|
||||
i += 1
|
||||
|
||||
if i % 100 == 0: HC.app.SetSplashText( 'reprocessing thumbs: ' + HC.ConvertIntToPrettyString( i ) )
|
||||
|
||||
except: print( 'When updating to v118, ' + path + '\'s phash could not be recalculated.' )
|
||||
|
||||
if i % 100 == 0: HC.app.SetSplashText( 'reprocessing thumbs: ' + HC.ConvertIntToPrettyString( i ) )
|
||||
|
||||
|
||||
|
||||
if version == 119:
|
||||
|
||||
i = 0
|
||||
|
||||
for path in CC.IterateAllFilePaths():
|
||||
|
||||
try:
|
||||
|
||||
filename = os.path.basename( path )
|
||||
|
||||
( hash_encoded, ext ) = filename.split( '.' )
|
||||
|
||||
hash = hash_encoded.decode( 'hex' )
|
||||
|
||||
hash_id = self._GetHashId( c, hash )
|
||||
|
||||
if ext not in ( 'flv', 'mp4', 'wmv', 'mkv', 'webm' ): continue
|
||||
|
||||
( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path )
|
||||
|
||||
c.execute( 'UPDATE files_info SET duration = ?, num_frames = ? WHERE hash_id = ?;', ( duration, num_frames, hash_id ) )
|
||||
|
||||
thumbnail = HydrusFileHandling.GenerateThumbnail( path )
|
||||
|
||||
thumbnail_path = CC.GetExpectedThumbnailPath( hash )
|
||||
|
||||
with open( thumbnail_path, 'wb' ) as f: f.write( thumbnail )
|
||||
|
||||
phash = HydrusImageHandling.GeneratePerceptualHash( thumbnail_path )
|
||||
|
||||
c.execute( 'INSERT OR REPLACE INTO perceptual_hashes ( hash_id, phash ) VALUES ( ?, ? );', ( hash_id, sqlite3.Binary( phash ) ) )
|
||||
|
||||
i += 1
|
||||
|
||||
if i % 100 == 0: HC.app.SetSplashText( 'creating video thumbs: ' + HC.ConvertIntToPrettyString( i ) )
|
||||
|
||||
except:
|
||||
print( traceback.format_exc())
|
||||
print( 'When updating to v119, ' + path + '\'s thumbnail or phash could not be calculated.' )
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ options = {}
|
|||
# Misc
|
||||
|
||||
NETWORK_VERSION = 13
|
||||
SOFTWARE_VERSION = 119
|
||||
SOFTWARE_VERSION = 120
|
||||
|
||||
UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )
|
||||
|
||||
|
|
@ -303,7 +303,7 @@ NOISY_MIMES = tuple( [ APPLICATION_FLASH ] + list( AUDIO ) + list( VIDEO ) )
|
|||
|
||||
ARCHIVES = ( APPLICATION_ZIP, APPLICATION_HYDRUS_ENCRYPTED_ZIP )
|
||||
|
||||
MIMES_WITH_THUMBNAILS = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP, VIDEO_WEBM )
|
||||
MIMES_WITH_THUMBNAILS = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP, VIDEO_WEBM, VIDEO_FLV, VIDEO_MP4, VIDEO_WMV, VIDEO_MKV, VIDEO_WEBM )
|
||||
|
||||
# mp3 header is complicated
|
||||
|
||||
|
|
|
|||
|
|
@ -857,9 +857,7 @@ class DownloaderPixiv( Downloader ):
|
|||
|
||||
tag = self._query
|
||||
|
||||
tag = urllib.quote( tag.encode( 'utf-8' ) )
|
||||
|
||||
gallery_url = 'http://www.pixiv.net/search.php?word=' + urllib.quote( tag ) + '&s_mode=s_tag_full&order=date_d'
|
||||
gallery_url = 'http://www.pixiv.net/search.php?word=' + urllib.quote( tag.encode( 'utf-8' ) ) + '&s_mode=s_tag_full&order=date_d'
|
||||
|
||||
|
||||
return gallery_url + '&p=' + HC.u( self._num_pages_done + 1 )
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import cv2
|
||||
#import cv2
|
||||
import hashlib
|
||||
import hsaudiotag
|
||||
import hsaudiotag.auto
|
||||
|
|
@ -55,24 +55,36 @@ def GenerateThumbnail( path, dimensions = HC.UNSCALED_THUMBNAIL_DIMENSIONS ):
|
|||
|
||||
else:
|
||||
|
||||
cv_video = cv2.VideoCapture( path )
|
||||
( size, mime, width, height, duration, num_frames, num_words ) = GetFileInfo( path )
|
||||
|
||||
cv_video.set( cv2.cv.CV_CAP_PROP_CONVERT_RGB, True )
|
||||
cropped_dimensions = HydrusImageHandling.GetThumbnailResolution( ( width, height ), dimensions )
|
||||
|
||||
( retval, cv_image ) = cv_video.read()
|
||||
renderer = HydrusVideoHandling.VideoRendererFFMPEG( path, mime, duration, num_frames, cropped_dimensions )
|
||||
|
||||
if not retval: raise Exception( 'Could not read first frame of ' + HC.u( path ) + ' to create thumbnail!' )
|
||||
numpy_image = renderer.read_frame()
|
||||
|
||||
cv_image = HydrusImageHandling.EfficientlyThumbnailCVImage( cv_image, dimensions )
|
||||
pil_image = HydrusImageHandling.GeneratePILImageFromNumpyImage( numpy_image )
|
||||
|
||||
( retval, thumbnail ) = cv2.imencode( '.jpg', cv_image, ( cv2.cv.CV_IMWRITE_JPEG_QUALITY, 92 ) )
|
||||
f = cStringIO.StringIO()
|
||||
|
||||
if not retval: raise Exception( 'Could not export thumbnail for ' + HC.u( path ) + '!' )
|
||||
pil_image.save( f, 'JPEG', quality=92 )
|
||||
|
||||
f.seek( 0 )
|
||||
|
||||
thumbnail = f.read()
|
||||
|
||||
f.close()
|
||||
|
||||
#numpy_image = cv2.cvtColor( numpy_image, cv2.COLOR_RGB2BGR )
|
||||
|
||||
#( retval, thumbnail ) = cv2.imencode( '.jpg', numpy_image, ( cv2.cv.CV_IMWRITE_JPEG_QUALITY, 92 ) )
|
||||
|
||||
#if not retval: raise Exception( 'Could not export thumbnail for ' + HC.u( path ) + '!' )
|
||||
|
||||
|
||||
return thumbnail
|
||||
|
||||
def GetFileInfo( path, hash ):
|
||||
def GetFileInfo( path ):
|
||||
|
||||
info = os.lstat( path )
|
||||
|
||||
|
|
@ -102,13 +114,9 @@ def GetFileInfo( path, hash ):
|
|||
|
||||
( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetFLVProperties( path )
|
||||
|
||||
elif mime in ( HC.VIDEO_WMV, HC.VIDEO_MP4 ):
|
||||
elif mime in ( HC.VIDEO_WMV, HC.VIDEO_MP4, HC.VIDEO_MKV, HC.VIDEO_WEBM ):
|
||||
|
||||
( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetCVVideoProperties( path )
|
||||
|
||||
elif mime in ( HC.VIDEO_MKV, HC.VIDEO_WEBM ):
|
||||
|
||||
( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetCVVideoProperties( path )
|
||||
( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetFFMPEGVideoProperties( path )
|
||||
|
||||
elif mime == HC.APPLICATION_PDF: num_words = HydrusDocumentHandling.GetPDFNumWords( path )
|
||||
elif mime == HC.AUDIO_MP3: duration = HydrusAudioHandling.GetMP3Duration( path )
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ def GenerateCVImage( path ):
|
|||
|
||||
cv_image = cv2.imread( self._path, flags = -1 ) # flags = -1 loads alpha channel, if present
|
||||
|
||||
( x, y, depth ) = cv_image.shape
|
||||
( y, x, depth ) = cv_image.shape
|
||||
|
||||
if depth == 4: raise Exception( 'CV is bad at alpha!' )
|
||||
else: cv_image = cv2.cvtColor( cv_image, cv2.COLOR_BGR2RGB )
|
||||
|
|
@ -102,7 +102,7 @@ def GenerateHydrusBitmap( path ):
|
|||
|
||||
cv_image = GenerateCVImage( path )
|
||||
|
||||
return GenerateHydrusBitmapFromCVImage( cv_image )
|
||||
return GenerateHydrusBitmapFromNumPyImage( cv_image )
|
||||
|
||||
except:
|
||||
|
||||
|
|
@ -111,7 +111,7 @@ def GenerateHydrusBitmap( path ):
|
|||
return GenerateHydrusBitmapFromPILImage( pil_image )
|
||||
|
||||
|
||||
def GenerateHydrusBitmapFromCVImage( cv_image ):
|
||||
def GenerateHydrusBitmapFromNumPyImage( cv_image ):
|
||||
|
||||
( y, x, depth ) = cv_image.shape
|
||||
|
||||
|
|
@ -137,7 +137,7 @@ def GeneratePerceptualHash( path ):
|
|||
|
||||
cv_image = cv2.imread( path, cv2.CV_LOAD_IMAGE_UNCHANGED )
|
||||
|
||||
( x, y, depth ) = cv_image.shape
|
||||
( y, x, depth ) = cv_image.shape
|
||||
|
||||
if depth == 4:
|
||||
|
||||
|
|
@ -153,15 +153,15 @@ def GeneratePerceptualHash( path ):
|
|||
|
||||
cv_image_gray = cv2.cvtColor( cv_image_bgr, cv2.COLOR_BGR2GRAY )
|
||||
|
||||
cv_image_result = numpy.empty( ( x, y ), numpy.float32 )
|
||||
cv_image_result = numpy.empty( ( y, x ), numpy.float32 )
|
||||
|
||||
# paste greyscale onto the white
|
||||
|
||||
# can't think of a better way to do this!
|
||||
# cv2.addWeighted only takes a scalar for weight!
|
||||
for i in range( x ):
|
||||
for i in range( y ):
|
||||
|
||||
for j in range( y ):
|
||||
for j in range( x ):
|
||||
|
||||
opacity = float( cv_alpha[ i, j ] ) / 255.0
|
||||
|
||||
|
|
@ -321,6 +321,17 @@ def old_GeneratePerceptualHash( path ):
|
|||
|
||||
def GeneratePILImage( path ): return PILImage.open( path )
|
||||
|
||||
def GeneratePILImageFromNumpyImage( numpy_image ):
|
||||
|
||||
( h, w, depth ) = numpy_image.shape
|
||||
|
||||
if depth == 3: format = 'RGB'
|
||||
elif depth == 4: format = 'RGBA'
|
||||
|
||||
pil_image = PILImage.fromstring( format, ( w, h ), numpy_image.data )
|
||||
|
||||
return pil_image
|
||||
|
||||
def GetGIFFrameDurations( path ):
|
||||
|
||||
pil_image_for_duration = GeneratePILImage( path )
|
||||
|
|
@ -568,7 +579,7 @@ class ImageContainer( RasterContainer ):
|
|||
|
||||
resized_cv_image = EfficientlyResizeCVImage( cv_image, self._target_resolution )
|
||||
|
||||
return GenerateHydrusBitmapFromCVImage( resized_cv_image )
|
||||
return GenerateHydrusBitmapFromNumPyImage( resized_cv_image )
|
||||
|
||||
except:
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import HydrusExceptions
|
|||
import httplib
|
||||
import threading
|
||||
import time
|
||||
import urllib
|
||||
import urlparse
|
||||
import yaml
|
||||
|
||||
|
|
@ -132,12 +133,15 @@ class HTTPConnectionManager():
|
|||
threading.Thread( target = self.DAEMONMaintainConnections, name = 'Maintain Connections' ).start()
|
||||
|
||||
|
||||
def _DoRequest( self, location, method, path_and_query, request_headers, body, follow_redirects = True, report_hooks = [], response_to_path = False, num_redirects_permitted = 4, long_timeout = False ):
|
||||
def _DoRequest( self, method, location, path, query, request_headers, body, follow_redirects = True, report_hooks = [], response_to_path = False, num_redirects_permitted = 4, long_timeout = False ):
|
||||
|
||||
connection = self._GetConnection( location, long_timeout )
|
||||
|
||||
try:
|
||||
|
||||
if query == '': path_and_query = path
|
||||
else: path_and_query = path + '?' + query
|
||||
|
||||
with connection.lock:
|
||||
|
||||
( parsed_response, redirect_info, size_of_response, response_headers, cookies ) = connection.Request( method, path_and_query, request_headers, body, report_hooks = report_hooks, response_to_path = response_to_path )
|
||||
|
|
@ -154,10 +158,7 @@ class HTTPConnectionManager():
|
|||
|
||||
if new_location is None: new_location = location
|
||||
|
||||
if new_query != '': new_path_and_query = new_path + '?' + new_query
|
||||
else: new_path_and_query = new_path
|
||||
|
||||
return self._DoRequest( new_location, new_method, new_path_and_query, request_headers, body, report_hooks = report_hooks, response_to_path = response_to_path, num_redirects_permitted = num_redirects_permitted - 1 )
|
||||
return self._DoRequest( new_method, new_location, new_path, new_query, request_headers, body, follow_redirects = follow_redirects, report_hooks = report_hooks, response_to_path = response_to_path, num_redirects_permitted = num_redirects_permitted - 1, long_timeout = long_timeout )
|
||||
|
||||
|
||||
except:
|
||||
|
|
@ -191,12 +192,9 @@ class HTTPConnectionManager():
|
|||
|
||||
( location, path, query ) = ParseURL( url )
|
||||
|
||||
if query != '': path_and_query = path + '?' + query
|
||||
else: path_and_query = path
|
||||
|
||||
follow_redirects = not return_cookies
|
||||
|
||||
( response, size_of_response, response_headers, cookies ) = self._DoRequest( location, method, path_and_query, request_headers, body, follow_redirects = follow_redirects, report_hooks = report_hooks, response_to_path = response_to_path, long_timeout = long_timeout )
|
||||
( response, size_of_response, response_headers, cookies ) = self._DoRequest( method, location, path, query, request_headers, body, follow_redirects = follow_redirects, report_hooks = report_hooks, response_to_path = response_to_path, long_timeout = long_timeout )
|
||||
|
||||
if return_everything: return ( response, size_of_response, response_headers, cookies )
|
||||
elif return_cookies: return ( response, cookies )
|
||||
|
|
@ -453,6 +451,23 @@ class HTTPConnection():
|
|||
|
||||
url = location
|
||||
|
||||
if ' ' in url:
|
||||
|
||||
# some booru is giving daft redirect responses
|
||||
print( url )
|
||||
url = urllib.quote( url, safe = '/?=&' )
|
||||
print( url )
|
||||
|
||||
if not url.startswith( self._scheme ):
|
||||
|
||||
# assume it is like 'index.php' or '/index.php', rather than 'http://blah.com/index.php'
|
||||
|
||||
if url.startswith( '/' ): slash_sep = ''
|
||||
else: slash_sep = '/'
|
||||
|
||||
url = self._scheme + '://' + self._host + slash_sep + url
|
||||
|
||||
|
||||
if response.status in ( 301, 307 ):
|
||||
|
||||
# 301: moved permanently, repeat request
|
||||
|
|
|
|||
|
|
@ -238,7 +238,7 @@ def ParseFileArguments( path ):
|
|||
|
||||
hash = HydrusFileHandling.GetHashFromPath( path )
|
||||
|
||||
try: ( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path, hash )
|
||||
try: ( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path )
|
||||
except HydrusExceptions.SizeException: raise HydrusExceptions.ForbiddenException( 'File is of zero length!' )
|
||||
except HydrusExceptions.MimeException: raise HydrusExceptions.ForbiddenException( 'Filetype is not permitted!' )
|
||||
except Exception as e: raise HydrusExceptions.ForbiddenException( HC.u( e ) )
|
||||
|
|
|
|||
|
|
@ -40,6 +40,20 @@ def GetCVVideoProperties( path ):
|
|||
|
||||
return ( ( width, height ), duration, num_frames )
|
||||
|
||||
def GetFFMPEGVideoProperties( path ):
|
||||
|
||||
info = Hydrusffmpeg_parse_infos( path )
|
||||
|
||||
( w, h ) = info[ 'video_size' ]
|
||||
|
||||
duration_in_s = info[ 'duration' ]
|
||||
|
||||
duration = int( duration_in_s * 1000 )
|
||||
|
||||
num_frames = info[ 'video_nframes' ]
|
||||
|
||||
return ( ( w, h ), duration, num_frames )
|
||||
|
||||
def GetFLVProperties( path ):
|
||||
|
||||
with open( path, 'rb' ) as f:
|
||||
|
|
@ -127,144 +141,7 @@ def GetMatroskaOrWebMProperties( path ):
|
|||
|
||||
return ( ( width, height ), duration, num_frames )
|
||||
|
||||
# This is cribbed from moviepy's FFMPEG_VideoReader
|
||||
class HydrusFFMPEG_VideoReader( object ):
|
||||
|
||||
def __init__(self, media, print_infos=False, bufsize = None, pix_fmt="rgb24"):
|
||||
|
||||
self._media = media
|
||||
|
||||
hash = self._media.GetHash()
|
||||
mime = self._media.GetMime()
|
||||
|
||||
self._path = CC.GetFilePath( hash, mime )
|
||||
|
||||
self.size = self._media.GetResolution()
|
||||
self.duration = float( self._media.GetDuration() ) / 1000.0
|
||||
self.nframes = self._media.GetNumFrames()
|
||||
|
||||
self.fps = float( self.nframes ) / self.duration
|
||||
|
||||
self.pix_fmt = pix_fmt
|
||||
|
||||
if pix_fmt == 'rgba': self.depth = 4
|
||||
else: self.depth = 3
|
||||
|
||||
if bufsize is None:
|
||||
|
||||
( x, y ) = self.size
|
||||
bufsize = self.depth * x * y * 5
|
||||
|
||||
|
||||
self.process = None
|
||||
|
||||
self.bufsize = bufsize
|
||||
|
||||
self.initialize()
|
||||
|
||||
|
||||
def __del__( self ):
|
||||
|
||||
self.close()
|
||||
|
||||
|
||||
def close(self):
|
||||
|
||||
if self.process is not None:
|
||||
|
||||
self.process.terminate()
|
||||
|
||||
self.process.stdout.close()
|
||||
self.process.stderr.close()
|
||||
|
||||
self.process = None
|
||||
|
||||
|
||||
|
||||
def initialize( self, starttime=0 ):
|
||||
"""Opens the file, creates the pipe. """
|
||||
|
||||
self.close()
|
||||
|
||||
if starttime != 0:
|
||||
|
||||
offset = min( 1, starttime )
|
||||
|
||||
i_arg = [ '-ss', "%.03f" % ( starttime - offset ), '-i', '"' + self._path + '"' ]
|
||||
|
||||
else: i_arg = [ '-i', '"' + self._path + '"' ]
|
||||
|
||||
cmd = ([FFMPEG_PATH]+ i_arg +
|
||||
['-loglevel', 'error',
|
||||
'-f', 'image2pipe',
|
||||
"-pix_fmt", self.pix_fmt,
|
||||
'-vcodec', 'rawvideo', '-'])
|
||||
|
||||
self.process = subprocess.Popen( ' '.join( cmd ), shell = True, bufsize= self.bufsize, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
|
||||
|
||||
self.pos = int( round( self.fps * starttime ) )
|
||||
|
||||
|
||||
def skip_frames(self, n=1):
|
||||
|
||||
"""Reads and throws away n frames """
|
||||
|
||||
( w, h ) = self.size
|
||||
|
||||
for i in range( n ):
|
||||
|
||||
self.process.stdout.read( self.depth * w * h )
|
||||
self.process.stdout.flush()
|
||||
|
||||
|
||||
self.pos += n
|
||||
|
||||
|
||||
def read_frame( self ):
|
||||
|
||||
( w, h ) = self.size
|
||||
|
||||
nbytes = self.depth * w * h
|
||||
|
||||
s = self.process.stdout.read(nbytes)
|
||||
|
||||
self.pos += 1
|
||||
|
||||
if len(s) != nbytes:
|
||||
|
||||
print( "Warning: in file %s, "%(self._path)+
|
||||
"%d bytes wanted but %d bytes read,"%(nbytes, len(s))+
|
||||
"at frame %d/%d, at time %.02f/%.02f sec. "%(
|
||||
self.pos,self.nframes,
|
||||
1.0*self.pos/self.fps,
|
||||
self.duration)+
|
||||
"Using the last valid frame instead.")
|
||||
result = self.lastread
|
||||
|
||||
self.close()
|
||||
|
||||
else:
|
||||
|
||||
result = numpy.fromstring( s, dtype = 'uint8' ).reshape( ( h, w, len( s ) // ( w * h ) ) )
|
||||
|
||||
self.lastread = result
|
||||
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def set_position( self, pos ):
|
||||
|
||||
if ( pos < self.pos ) or ( pos > self.pos + 60 ):
|
||||
|
||||
starttime = float( pos ) / self.fps
|
||||
|
||||
self.initialize( starttime )
|
||||
|
||||
else: self.skip_frames( pos - self.pos )
|
||||
|
||||
|
||||
# same thing; this is cribbed from moviepy
|
||||
# this is cribbed from moviepy
|
||||
def Hydrusffmpeg_parse_infos(filename, print_infos=False):
|
||||
"""Get file infos using ffmpeg.
|
||||
|
||||
|
|
@ -306,9 +183,19 @@ def Hydrusffmpeg_parse_infos(filename, print_infos=False):
|
|||
result = dict()
|
||||
|
||||
# get duration (in seconds)
|
||||
# Duration: 00:00:02.46, start: 0.033000, bitrate: 1069 kb/s
|
||||
try:
|
||||
keyword = ('frame=' if is_GIF else 'Duration: ')
|
||||
line = [l for l in lines if keyword in l][0]
|
||||
|
||||
if 'start:' in line:
|
||||
|
||||
m = re.search( '(start\\: )' + '[0-9]\\.[0-9]*', line )
|
||||
|
||||
start_offset = float( line[ m.start() + 7 : m.end() ] )
|
||||
|
||||
else: start_offset = 0
|
||||
|
||||
match = re.search("[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9]", line)
|
||||
hms = map(float, line[match.start()+1:match.end()].split(':'))
|
||||
|
||||
|
|
@ -319,6 +206,8 @@ def Hydrusffmpeg_parse_infos(filename, print_infos=False):
|
|||
elif len(hms) ==3:
|
||||
result['duration'] = 3600*hms[0]+60*hms[1]+hms[2]
|
||||
|
||||
result[ 'duration' ] -= start_offset
|
||||
|
||||
except:
|
||||
raise IOError("Error reading duration in file %s,"%(filename)+
|
||||
"Text parsed: %s"%infos)
|
||||
|
|
@ -345,9 +234,13 @@ def Hydrusffmpeg_parse_infos(filename, print_infos=False):
|
|||
except:
|
||||
match = re.search("( [0-9]*.| )[0-9]* fps", line)
|
||||
result['video_fps'] = float(line[match.start():match.end()].split(' ')[1])
|
||||
|
||||
result['video_nframes'] = int(result['duration']*result['video_fps'])+1
|
||||
|
||||
|
||||
num_frames = result['duration'] * result['video_fps']
|
||||
|
||||
if num_frames != int( num_frames ): num_frames += 1 # rounding up
|
||||
|
||||
result['video_nframes'] = int( num_frames )
|
||||
|
||||
result['video_duration'] = result['duration']
|
||||
# We could have also recomputed the duration from the number
|
||||
# of frames, as follows:
|
||||
|
|
@ -378,8 +271,8 @@ class VideoContainer( HydrusImageHandling.RasterContainer ):
|
|||
|
||||
self._frames = {}
|
||||
self._last_index_asked_for = -1
|
||||
self._minimum_frame_asked_for = 0
|
||||
self._maximum_frame_asked_for = 0
|
||||
self._buffer_start_index = -1
|
||||
self._buffer_end_index = -1
|
||||
|
||||
( x, y ) = self._target_resolution
|
||||
|
||||
|
|
@ -391,9 +284,21 @@ class VideoContainer( HydrusImageHandling.RasterContainer ):
|
|||
if self._media.GetMime() == HC.IMAGE_GIF: self._durations = HydrusImageHandling.GetGIFFrameDurations( self._path )
|
||||
else: self._frame_duration = GetVideoFrameDuration( self._path )
|
||||
|
||||
self._renderer = VideoRendererMoviePy( self, self._media, self._target_resolution )
|
||||
self._render_lock = threading.Lock()
|
||||
|
||||
num_frames = self.GetNumFrames()
|
||||
hash = self._media.GetHash()
|
||||
mime = self._media.GetMime()
|
||||
|
||||
path = CC.GetFilePath( hash, mime )
|
||||
|
||||
duration = self._media.GetDuration()
|
||||
num_frames = self._media.GetNumFrames()
|
||||
|
||||
self._ffmpeg_reader = VideoRendererFFMPEG( path, mime, duration, num_frames, target_resolution )
|
||||
|
||||
self._next_render_index = 0
|
||||
self._render_to_index = -1
|
||||
self._rendered_first_frame = False
|
||||
|
||||
self.SetPosition( init_position )
|
||||
|
||||
|
|
@ -404,19 +309,81 @@ class VideoContainer( HydrusImageHandling.RasterContainer ):
|
|||
|
||||
for index in self._frames.keys():
|
||||
|
||||
if self._minimum_frame_asked_for < self._maximum_frame_asked_for:
|
||||
if self._buffer_start_index < self._buffer_end_index:
|
||||
|
||||
if index < self._minimum_frame_asked_for or self._maximum_frame_asked_for < index: deletees.append( index )
|
||||
if index < self._buffer_start_index or self._buffer_end_index < index: deletees.append( index )
|
||||
|
||||
else:
|
||||
|
||||
if self._maximum_frame_asked_for < index and index < self._minimum_frame_asked_for: deletees.append( index )
|
||||
if self._buffer_end_index < index and index < self._buffer_start_index: deletees.append( index )
|
||||
|
||||
|
||||
|
||||
for i in deletees: del self._frames[ i ]
|
||||
|
||||
|
||||
def _RENDERERSetRenderToPosition( self, index ):
|
||||
|
||||
with self._render_lock:
|
||||
|
||||
if self._render_to_index != index:
|
||||
|
||||
self._render_to_index = index
|
||||
|
||||
HydrusThreading.CallToThread( self.THREADRender )
|
||||
|
||||
|
||||
|
||||
|
||||
def _RENDERERSetPosition( self, index ):
|
||||
|
||||
with self._render_lock:
|
||||
|
||||
if index == self._next_render_index: return
|
||||
else:
|
||||
|
||||
self._ffmpeg_reader.set_position( index )
|
||||
|
||||
self._next_render_index = index
|
||||
self._render_to_index = index
|
||||
|
||||
|
||||
|
||||
|
||||
def THREADRender( self ):
|
||||
|
||||
num_frames = self._media.GetNumFrames()
|
||||
|
||||
while True:
|
||||
|
||||
time.sleep( 0.00001 ) # thread yield
|
||||
|
||||
with self._render_lock:
|
||||
|
||||
if not self._rendered_first_frame or self._next_render_index != ( self._render_to_index + 1 ) % num_frames:
|
||||
|
||||
self._rendered_first_frame = True
|
||||
|
||||
frame_index = self._next_render_index # keep this before the get call, as it increments in a clock arithmetic way afterwards
|
||||
|
||||
try: numpy_image = self._ffmpeg_reader.read_frame()
|
||||
except Exception as e:
|
||||
|
||||
HC.ShowException( e )
|
||||
|
||||
break
|
||||
|
||||
finally: self._next_render_index = ( self._next_render_index + 1 ) % num_frames
|
||||
|
||||
frame = HydrusImageHandling.GenerateHydrusBitmapFromNumPyImage( numpy_image )
|
||||
|
||||
wx.CallAfter( self.AddFrame, frame_index, frame )
|
||||
|
||||
else: break
|
||||
|
||||
|
||||
|
||||
|
||||
def AddFrame( self, index, frame ): self._frames[ index ] = frame
|
||||
|
||||
def GetDuration( self, index ):
|
||||
|
|
@ -464,61 +431,205 @@ class VideoContainer( HydrusImageHandling.RasterContainer ):
|
|||
|
||||
if num_frames > self._num_frames_backwards + 1 + self._num_frames_forwards:
|
||||
|
||||
new_minimum_frame_to_ask_for = max( 0, index - self._num_frames_backwards ) % num_frames
|
||||
new_buffer_start_index = max( 0, index - self._num_frames_backwards ) % num_frames
|
||||
|
||||
new_maximum_frame_to_ask_for = ( index + self._num_frames_forwards ) % num_frames
|
||||
new_buffer_end_index = ( index + self._num_frames_forwards ) % num_frames
|
||||
|
||||
if index == self._last_index_asked_for: return
|
||||
elif index < self._last_index_asked_for:
|
||||
|
||||
if index < self._minimum_frame_asked_for:
|
||||
if index < self._buffer_start_index:
|
||||
|
||||
self._minimum_frame_asked_for = new_minimum_frame_to_ask_for
|
||||
self._buffer_start_index = new_buffer_start_index
|
||||
|
||||
self._renderer.SetPosition( self._minimum_frame_asked_for )
|
||||
self._RENDERERSetPosition( self._buffer_start_index )
|
||||
|
||||
self._maximum_frame_asked_for = new_maximum_frame_to_ask_for
|
||||
self._buffer_end_index = new_buffer_end_index
|
||||
|
||||
self._renderer.SetRenderToPosition( self._maximum_frame_asked_for )
|
||||
self._RENDERERSetRenderToPosition( self._buffer_end_index )
|
||||
|
||||
|
||||
else: # index > self._last_index_asked_for
|
||||
|
||||
currently_no_wraparound = self._minimum_frame_asked_for < self._maximum_frame_asked_for
|
||||
currently_no_wraparound = self._buffer_start_index < self._buffer_end_index
|
||||
|
||||
self._minimum_frame_asked_for = new_minimum_frame_to_ask_for
|
||||
self._buffer_start_index = new_buffer_start_index
|
||||
|
||||
if currently_no_wraparound:
|
||||
|
||||
if index > self._maximum_frame_asked_for:
|
||||
if index > self._buffer_end_index:
|
||||
|
||||
self._renderer.SetPosition( self._minimum_frame_asked_for )
|
||||
self._RENDERERSetPosition( self._buffer_start_index )
|
||||
|
||||
|
||||
|
||||
self._maximum_frame_asked_for = new_maximum_frame_to_ask_for
|
||||
self._buffer_end_index = new_buffer_end_index
|
||||
|
||||
self._renderer.SetRenderToPosition( self._maximum_frame_asked_for )
|
||||
self._RENDERERSetRenderToPosition( self._buffer_end_index )
|
||||
|
||||
|
||||
else:
|
||||
|
||||
if self._maximum_frame_asked_for == 0:
|
||||
if self._buffer_end_index == -1:
|
||||
|
||||
self._minimum_frame_asked_for = 0
|
||||
self._buffer_start_index = 0
|
||||
|
||||
self._renderer.SetPosition( 0 )
|
||||
self._RENDERERSetPosition( 0 )
|
||||
|
||||
self._maximum_frame_asked_for = num_frames - 1
|
||||
self._buffer_end_index = num_frames - 1
|
||||
|
||||
self._renderer.SetRenderToPosition( self._maximum_frame_asked_for )
|
||||
self._RENDERERSetRenderToPosition( self._buffer_end_index )
|
||||
|
||||
|
||||
|
||||
self._MaintainBuffer()
|
||||
|
||||
|
||||
class VideoRendererCV():
|
||||
# This was built from moviepy's FFMPEG_VideoReader
|
||||
class VideoRendererFFMPEG( object ):
|
||||
|
||||
def __init__( self, path, mime, duration, num_frames, target_resolution, pix_fmt = "rgb24" ):
|
||||
|
||||
self._path = path
|
||||
self._mime = mime
|
||||
self._duration = float( duration ) / 1000.0
|
||||
self._num_frames = num_frames
|
||||
self._target_resolution = target_resolution
|
||||
|
||||
self.fps = float( self._num_frames ) / self._duration
|
||||
|
||||
if self.fps == 0: self.fps = 24
|
||||
|
||||
self.pix_fmt = pix_fmt
|
||||
|
||||
if pix_fmt == 'rgba': self.depth = 4
|
||||
else: self.depth = 3
|
||||
|
||||
( x, y ) = self._target_resolution
|
||||
bufsize = self.depth * x * y
|
||||
|
||||
self.process = None
|
||||
|
||||
self.bufsize = bufsize
|
||||
|
||||
self.initialize()
|
||||
|
||||
|
||||
def __del__( self ):
|
||||
|
||||
self.close()
|
||||
|
||||
|
||||
def close(self):
|
||||
|
||||
if self.process is not None:
|
||||
|
||||
self.process.terminate()
|
||||
|
||||
self.process.stdout.close()
|
||||
self.process.stderr.close()
|
||||
|
||||
self.process = None
|
||||
|
||||
|
||||
|
||||
def initialize( self, start_index = 0 ):
|
||||
|
||||
self.close()
|
||||
|
||||
if self._mime == HC.IMAGE_GIF:
|
||||
|
||||
ss = 0
|
||||
self.pos = 0
|
||||
skip_frames = start_index
|
||||
|
||||
else:
|
||||
|
||||
ss = float( start_index ) / self.fps
|
||||
self.pos = start_index
|
||||
skip_frames = 0
|
||||
|
||||
|
||||
( w, h ) = self._target_resolution
|
||||
|
||||
cmd = ( [ FFMPEG_PATH,
|
||||
'-ss', "%.03f" % ss,
|
||||
'-i', '"' + self._path + '"',
|
||||
'-loglevel', 'quiet',
|
||||
'-f', 'image2pipe',
|
||||
"-pix_fmt", self.pix_fmt,
|
||||
"-s", str( w ) + 'x' + str( h ),
|
||||
'-vcodec', 'rawvideo', '-' ] )
|
||||
|
||||
self.process = subprocess.Popen( ' '.join( cmd ), shell = True, bufsize= self.bufsize, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
|
||||
|
||||
self.skip_frames( skip_frames )
|
||||
|
||||
|
||||
def skip_frames( self, n ):
|
||||
|
||||
( w, h ) = self._target_resolution
|
||||
|
||||
for i in range( n ):
|
||||
|
||||
self.process.stdout.read( self.depth * w * h )
|
||||
|
||||
self.process.stdout.flush()
|
||||
|
||||
self.pos += 1
|
||||
|
||||
|
||||
|
||||
def read_frame( self ):
|
||||
|
||||
if self.pos == self._num_frames: self.initialize()
|
||||
|
||||
if self.process is None: result = self.lastread
|
||||
else:
|
||||
|
||||
( w, h ) = self._target_resolution
|
||||
|
||||
nbytes = self.depth * w * h
|
||||
|
||||
s = self.process.stdout.read(nbytes)
|
||||
|
||||
if len(s) != nbytes:
|
||||
|
||||
print( "Warning: in file %s, "%(self._path)+
|
||||
"%d bytes wanted but %d bytes read,"%(nbytes, len(s))+
|
||||
"at frame %d/%d, at time %.02f/%.02f sec. "%(
|
||||
self.pos,self._num_frames,
|
||||
1.0*self.pos/self.fps,
|
||||
self._duration)+
|
||||
"Using the last valid frame instead.")
|
||||
|
||||
result = self.lastread
|
||||
|
||||
self.close()
|
||||
|
||||
else:
|
||||
|
||||
result = numpy.fromstring( s, dtype = 'uint8' ).reshape( ( h, w, len( s ) // ( w * h ) ) )
|
||||
|
||||
self.lastread = result
|
||||
|
||||
|
||||
|
||||
self.pos += 1
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def set_position( self, pos ):
|
||||
|
||||
rewind = pos < self.pos
|
||||
jump_a_long_way_ahead = pos > self.pos + 60
|
||||
|
||||
if rewind or jump_a_long_way_ahead: self.initialize( pos )
|
||||
else: self.skip_frames( pos - self.pos )
|
||||
|
||||
|
||||
class OLDCODEVideoRendererCV():
|
||||
|
||||
def __init__( self, image_container, media, target_resolution ):
|
||||
|
||||
|
|
@ -531,13 +642,13 @@ class VideoRendererCV():
|
|||
|
||||
self._path = CC.GetFilePath( hash, mime )
|
||||
|
||||
self._lock = threading.Lock()
|
||||
self._render_lock = threading.Lock()
|
||||
|
||||
self._cv_video = cv2.VideoCapture( self._path )
|
||||
|
||||
self._cv_video.set( cv2.cv.CV_CAP_PROP_CONVERT_RGB, True )
|
||||
|
||||
self._current_index = 0
|
||||
self._next_render_index = 0
|
||||
self._last_index_rendered = -1
|
||||
self._last_frame = None
|
||||
self._render_to_index = -1
|
||||
|
|
@ -547,13 +658,13 @@ class VideoRendererCV():
|
|||
|
||||
( retval, cv_image ) = self._cv_video.read()
|
||||
|
||||
self._last_index_rendered = self._current_index
|
||||
self._last_index_rendered = self._next_render_index
|
||||
|
||||
num_frames = self._media.GetNumFrames()
|
||||
|
||||
self._current_index = ( self._current_index + 1 ) % num_frames
|
||||
self._next_render_index = ( self._next_render_index + 1 ) % num_frames
|
||||
|
||||
if self._current_index == 0 and self._last_index_rendered != 0:
|
||||
if self._next_render_index == 0 and self._last_index_rendered != 0:
|
||||
|
||||
if self._media.GetMime() == HC.IMAGE_GIF: self._RewindGIF()
|
||||
else: self._cv_video.set( cv2.cv.CV_CAP_PROP_POS_FRAMES, 0.0 )
|
||||
|
|
@ -561,7 +672,7 @@ class VideoRendererCV():
|
|||
|
||||
if not retval:
|
||||
|
||||
raise HydrusExceptions.CantRenderWithCVException( 'CV could not render frame ' + HC.u( self._current_index ) + '.' )
|
||||
raise HydrusExceptions.CantRenderWithCVException( 'CV could not render frame ' + HC.u( self._next_render_index ) + '.' )
|
||||
|
||||
|
||||
return cv_image
|
||||
|
|
@ -584,27 +695,7 @@ class VideoRendererCV():
|
|||
cv_image = self._last_frame
|
||||
|
||||
|
||||
return HydrusImageHandling.GenerateHydrusBitmapFromCVImage( cv_image )
|
||||
|
||||
|
||||
def _RenderFrames( self ):
|
||||
|
||||
no_frames_yet = True
|
||||
|
||||
while True:
|
||||
|
||||
try:
|
||||
|
||||
yield self._RenderCurrentFrame()
|
||||
|
||||
no_frames_yet = False
|
||||
|
||||
except HydrusExceptions.CantRenderWithCVException:
|
||||
|
||||
if no_frames_yet: raise
|
||||
else: break
|
||||
|
||||
|
||||
return HydrusImageHandling.GenerateHydrusBitmapFromNumPyImage( cv_image )
|
||||
|
||||
|
||||
def _RewindGIF( self ):
|
||||
|
|
@ -612,12 +703,12 @@ class VideoRendererCV():
|
|||
self._cv_video.release()
|
||||
self._cv_video.open( self._path )
|
||||
|
||||
self._current_index = 0
|
||||
self._next_render_index = 0
|
||||
|
||||
|
||||
def SetRenderToPosition( self, index ):
|
||||
|
||||
with self._lock:
|
||||
with self._render_lock:
|
||||
|
||||
if self._render_to_index != index:
|
||||
|
||||
|
|
@ -630,14 +721,14 @@ class VideoRendererCV():
|
|||
|
||||
def SetPosition( self, index ):
|
||||
|
||||
with self._lock:
|
||||
with self._render_lock:
|
||||
|
||||
if self._media.GetMime() == HC.IMAGE_GIF:
|
||||
|
||||
if index == self._current_index: return
|
||||
elif index < self._current_index: self._RewindGIF()
|
||||
if index == self._next_render_index: return
|
||||
elif index < self._next_render_index: self._RewindGIF()
|
||||
|
||||
while self._current_index < index: self._GetCurrentFrame()
|
||||
while self._next_render_index < index: self._GetCurrentFrame()
|
||||
|
||||
else:
|
||||
|
||||
|
|
@ -654,149 +745,11 @@ class VideoRendererCV():
|
|||
|
||||
time.sleep( 0.00001 ) # thread yield
|
||||
|
||||
with self._lock:
|
||||
with self._render_lock:
|
||||
|
||||
if self._last_index_rendered != self._render_to_index:
|
||||
|
||||
index = self._current_index
|
||||
|
||||
frame = self._RenderCurrentFrame()
|
||||
|
||||
wx.CallAfter( self._image_container.AddFrame, index, frame )
|
||||
|
||||
else: break
|
||||
|
||||
|
||||
|
||||
|
||||
class VideoRendererMoviePy():
|
||||
|
||||
def __init__( self, image_container, media, target_resolution ):
|
||||
|
||||
self._image_container = image_container
|
||||
self._media = media
|
||||
self._target_resolution = target_resolution
|
||||
|
||||
self._lock = threading.Lock()
|
||||
|
||||
( x, y ) = media.GetResolution()
|
||||
|
||||
self._ffmpeg_reader = HydrusFFMPEG_VideoReader( media, bufsize = x * y * 3 * 5 )
|
||||
|
||||
self._current_index = 0
|
||||
self._last_index_rendered = -1
|
||||
self._last_frame = None
|
||||
self._render_to_index = -1
|
||||
|
||||
|
||||
def _GetCurrentFrame( self ):
|
||||
|
||||
try: image = self._ffmpeg_reader.read_frame()
|
||||
except Exception as e: raise HydrusExceptions.CantRenderWithCVException( 'FFMPEG could not render frame ' + HC.u( self._current_index ) + '.' + os.linesep * 2 + HC.u( e ) )
|
||||
|
||||
self._last_index_rendered = self._current_index
|
||||
|
||||
num_frames = self._media.GetNumFrames()
|
||||
|
||||
self._current_index = ( self._current_index + 1 ) % num_frames
|
||||
|
||||
if self._current_index == 0 and self._last_index_rendered != 0: self._ffmpeg_reader.initialize()
|
||||
|
||||
return image
|
||||
|
||||
|
||||
def _RenderCurrentFrame( self ):
|
||||
|
||||
try:
|
||||
|
||||
image = self._GetCurrentFrame()
|
||||
|
||||
image = HydrusImageHandling.EfficientlyResizeCVImage( image, self._target_resolution )
|
||||
|
||||
self._last_frame = image
|
||||
|
||||
except HydrusExceptions.CantRenderWithCVException:
|
||||
|
||||
if self._last_frame is None: raise
|
||||
|
||||
image = self._last_frame
|
||||
|
||||
|
||||
return HydrusImageHandling.GenerateHydrusBitmapFromCVImage( image )
|
||||
|
||||
|
||||
def _RenderFrames( self ):
|
||||
|
||||
no_frames_yet = True
|
||||
|
||||
while True:
|
||||
|
||||
try:
|
||||
|
||||
yield self._RenderCurrentFrame()
|
||||
|
||||
no_frames_yet = False
|
||||
|
||||
except HydrusExceptions.CantRenderWithCVException:
|
||||
|
||||
if no_frames_yet: raise
|
||||
else: break
|
||||
|
||||
|
||||
|
||||
|
||||
def SetRenderToPosition( self, index ):
|
||||
|
||||
with self._lock:
|
||||
|
||||
if self._render_to_index != index:
|
||||
|
||||
self._render_to_index = index
|
||||
|
||||
HydrusThreading.CallToThread( self.THREADDoWork )
|
||||
|
||||
|
||||
|
||||
|
||||
def SetPosition( self, index ):
|
||||
|
||||
with self._lock:
|
||||
|
||||
if index == self._current_index: return
|
||||
else:
|
||||
|
||||
if self._media.GetMime() == HC.IMAGE_GIF:
|
||||
|
||||
if index < self._current_index:
|
||||
|
||||
self._ffmpeg_reader.initialize()
|
||||
|
||||
self._ffmpeg_reader.skip_frames( index )
|
||||
|
||||
|
||||
else:
|
||||
|
||||
timecode = float( index ) / self._ffmpeg_reader.fps
|
||||
|
||||
self._ffmpeg_reader.set_position( timecode )
|
||||
|
||||
|
||||
self._render_to_index = index
|
||||
|
||||
|
||||
|
||||
|
||||
def THREADDoWork( self ):
|
||||
|
||||
while True:
|
||||
|
||||
time.sleep( 0.00001 ) # thread yield
|
||||
|
||||
with self._lock:
|
||||
|
||||
if self._last_index_rendered != self._render_to_index:
|
||||
|
||||
index = self._current_index
|
||||
index = self._next_render_index
|
||||
|
||||
frame = self._RenderCurrentFrame()
|
||||
|
||||
|
|
|
|||
|
|
@ -98,7 +98,46 @@ class TestDownloaders( unittest.TestCase ):
|
|||
|
||||
info = ( data, tags )
|
||||
|
||||
expected_info = ('swf file', set([u'chores', u'laser', u'silent', u'title:Cat Dust', u'creator:warlord-of-noodles', u'pointer']))
|
||||
expected_info = ( 'swf file', set([u'chores', u'laser', u'silent', u'title:Cat Dust', u'creator:warlord-of-noodles', u'pointer']) )
|
||||
|
||||
self.assertEqual( info, expected_info )
|
||||
|
||||
|
||||
def test_pixiv( self ):
|
||||
|
||||
with open( HC.STATIC_DIR + os.path.sep + 'testing' + os.path.sep + 'pixiv_gallery.html' ) as f: pixiv_gallery = f.read()
|
||||
with open( HC.STATIC_DIR + os.path.sep + 'testing' + os.path.sep + 'pixiv_page.html' ) as f: pixiv_page = f.read()
|
||||
|
||||
HC.http.SetResponse( HC.GET, 'http://www.pixiv.net/search.php?word=naruto&s_mode=s_tag_full&order=date_d&p=1', pixiv_gallery )
|
||||
HC.http.SetResponse( HC.GET, 'http://www.pixiv.net/member_illust.php?mode=medium&illust_id=43718605', pixiv_page )
|
||||
|
||||
HC.http.SetResponse( HC.GET, 'http://i1.pixiv.net/img59/img/dbhope/43718605.jpg', 'image file' )
|
||||
|
||||
#
|
||||
|
||||
downloader = HydrusDownloading.DownloaderPixiv( 'tags', 'naruto' )
|
||||
|
||||
#
|
||||
|
||||
gallery_urls = downloader.GetAnotherPage()
|
||||
|
||||
expected_gallery_urls = [ ( u'http://www.pixiv.net/member_illust.php?mode=medium&illust_id=43718605', ), 'a bunch of others' ]
|
||||
|
||||
self.assertEqual( gallery_urls[0], expected_gallery_urls[0] )
|
||||
|
||||
#
|
||||
|
||||
info = downloader.GetFileAndTags( 'http://www.pixiv.net/member_illust.php?mode=medium&illust_id=43718605' )
|
||||
|
||||
( temp_path, tags ) = info
|
||||
|
||||
with open( temp_path, 'rb' ) as f: data = f.read()
|
||||
|
||||
info = ( data, tags )
|
||||
|
||||
expected_tags = [ u'1P\u6f2b\u753b', u'\u7720\u305f\u3044', u'NARUTO', u'\u30ca\u30eb\u30c8', u'\u30b5\u30b9\u30b1', u'\u30b5\u30af\u30e9', u'\u30d2\u30ca\u30bf', u'creator:\u30df\u30c4\u30ad\u30e8\u3063\u3057\uff5e', u'title:\u7720\u305f\u3044', 'creator:dbhope' ]
|
||||
|
||||
expected_info = ( 'image file', expected_tags )
|
||||
|
||||
self.assertEqual( info, expected_info )
|
||||
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue