changelog
-
+
version 120
+ - improved quality of downsized animation rendering +
- sped up downsized animation rendering in this case +
- neatened animation code +
- fixed a mid-animation resize parameter bug +
- I think I fixed an animation scan bug that was sometimes giving the wrong frames +
- added thumbnails for all video formats +
- fixed an off-by-one framecount bug for certain videos, and retroactively fixed counts for existing videos +
- fixed a couple harmless width/height numpy.shape switcharounds +
- cleaned up some file parsing code +
- added a pixiv unit test +
- fixed a bug when servers were returning unusual relative redirect urls (gelbooru) +
- semi-fixed a bug when servers were returning unescaped redirect urls (gelbooru) +
version 119
- fixed an overzealous error in v118 update code diff --git a/include/ClientDB.py b/include/ClientDB.py index 7bec24ad..fec4af45 100755 --- a/include/ClientDB.py +++ b/include/ClientDB.py @@ -3079,7 +3079,7 @@ class ServiceDB( FileDB, MessageDB, TagDB, RatingDB ): if can_add: - ( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path, hash ) + ( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path ) if width is not None and height is not None: @@ -4771,7 +4771,7 @@ class DB( ServiceDB ): if resize_thumbs: - thumbnail_paths = ( path for path in CC.IterateAllThumbnailPaths() if path.endswith( '_resized' ) ) + thumbnail_paths = [ path for path in CC.IterateAllThumbnailPaths() if path.endswith( '_resized' ) ] for path in thumbnail_paths: os.remove( path ) @@ -4907,10 +4907,52 @@ class DB( ServiceDB ): i += 1 + if i % 100 == 0: HC.app.SetSplashText( 'reprocessing thumbs: ' + HC.ConvertIntToPrettyString( i ) ) + except: print( 'When updating to v118, ' + path + '\'s phash could not be recalculated.' ) - if i % 100 == 0: HC.app.SetSplashText( 'reprocessing thumbs: ' + HC.ConvertIntToPrettyString( i ) ) + + + + if version == 119: + + i = 0 + + for path in CC.IterateAllFilePaths(): + + try: + filename = os.path.basename( path ) + + ( hash_encoded, ext ) = filename.split( '.' ) + + hash = hash_encoded.decode( 'hex' ) + + hash_id = self._GetHashId( c, hash ) + + if ext not in ( 'flv', 'mp4', 'wmv', 'mkv', 'webm' ): continue + + ( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path ) + + c.execute( 'UPDATE files_info SET duration = ?, num_frames = ? WHERE hash_id = ?;', ( duration, num_frames, hash_id ) ) + + thumbnail = HydrusFileHandling.GenerateThumbnail( path ) + + thumbnail_path = CC.GetExpectedThumbnailPath( hash ) + + with open( thumbnail_path, 'wb' ) as f: f.write( thumbnail ) + + phash = HydrusImageHandling.GeneratePerceptualHash( thumbnail_path ) + + c.execute( 'INSERT OR REPLACE INTO perceptual_hashes ( hash_id, phash ) VALUES ( ?, ? );', ( hash_id, sqlite3.Binary( phash ) ) ) + + i += 1 + + if i % 100 == 0: HC.app.SetSplashText( 'creating video thumbs: ' + HC.ConvertIntToPrettyString( i ) ) + + except: + print( traceback.format_exc()) + print( 'When updating to v119, ' + path + '\'s thumbnail or phash could not be calculated.' ) diff --git a/include/HydrusConstants.py b/include/HydrusConstants.py index 20193671..fabbd3e3 100755 --- a/include/HydrusConstants.py +++ b/include/HydrusConstants.py @@ -64,7 +64,7 @@ options = {} # Misc NETWORK_VERSION = 13 -SOFTWARE_VERSION = 119 +SOFTWARE_VERSION = 120 UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 ) @@ -303,7 +303,7 @@ NOISY_MIMES = tuple( [ APPLICATION_FLASH ] + list( AUDIO ) + list( VIDEO ) ) ARCHIVES = ( APPLICATION_ZIP, APPLICATION_HYDRUS_ENCRYPTED_ZIP ) -MIMES_WITH_THUMBNAILS = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP, VIDEO_WEBM ) +MIMES_WITH_THUMBNAILS = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP, VIDEO_WEBM, VIDEO_FLV, VIDEO_MP4, VIDEO_WMV, VIDEO_MKV, VIDEO_WEBM ) # mp3 header is complicated diff --git a/include/HydrusDownloading.py b/include/HydrusDownloading.py index 51a2964b..2c2d553c 100644 --- a/include/HydrusDownloading.py +++ b/include/HydrusDownloading.py @@ -857,9 +857,7 @@ class DownloaderPixiv( Downloader ): tag = self._query - tag = urllib.quote( tag.encode( 'utf-8' ) ) - - gallery_url = 'http://www.pixiv.net/search.php?word=' + urllib.quote( tag ) + '&s_mode=s_tag_full&order=date_d' + gallery_url = 'http://www.pixiv.net/search.php?word=' + urllib.quote( tag.encode( 'utf-8' ) ) + '&s_mode=s_tag_full&order=date_d' return gallery_url + '&p=' + HC.u( self._num_pages_done + 1 ) diff --git a/include/HydrusFileHandling.py b/include/HydrusFileHandling.py index 5cd2ffb9..2c4eef3f 100644 --- a/include/HydrusFileHandling.py +++ b/include/HydrusFileHandling.py @@ -1,4 +1,4 @@ -import cv2 +#import cv2 import hashlib import hsaudiotag import hsaudiotag.auto @@ -55,24 +55,36 @@ def GenerateThumbnail( path, dimensions = HC.UNSCALED_THUMBNAIL_DIMENSIONS ): else: - cv_video = cv2.VideoCapture( path ) + ( size, mime, width, height, duration, num_frames, num_words ) = GetFileInfo( path ) - cv_video.set( cv2.cv.CV_CAP_PROP_CONVERT_RGB, True ) + cropped_dimensions = HydrusImageHandling.GetThumbnailResolution( ( width, height ), dimensions ) - ( retval, cv_image ) = cv_video.read() + renderer = HydrusVideoHandling.VideoRendererFFMPEG( path, mime, duration, num_frames, cropped_dimensions ) - if not retval: raise Exception( 'Could not read first frame of ' + HC.u( path ) + ' to create thumbnail!' ) + numpy_image = renderer.read_frame() - cv_image = HydrusImageHandling.EfficientlyThumbnailCVImage( cv_image, dimensions ) + pil_image = HydrusImageHandling.GeneratePILImageFromNumpyImage( numpy_image ) - ( retval, thumbnail ) = cv2.imencode( '.jpg', cv_image, ( cv2.cv.CV_IMWRITE_JPEG_QUALITY, 92 ) ) + f = cStringIO.StringIO() - if not retval: raise Exception( 'Could not export thumbnail for ' + HC.u( path ) + '!' ) + pil_image.save( f, 'JPEG', quality=92 ) + + f.seek( 0 ) + + thumbnail = f.read() + + f.close() + + #numpy_image = cv2.cvtColor( numpy_image, cv2.COLOR_RGB2BGR ) + + #( retval, thumbnail ) = cv2.imencode( '.jpg', numpy_image, ( cv2.cv.CV_IMWRITE_JPEG_QUALITY, 92 ) ) + + #if not retval: raise Exception( 'Could not export thumbnail for ' + HC.u( path ) + '!' ) return thumbnail -def GetFileInfo( path, hash ): +def GetFileInfo( path ): info = os.lstat( path ) @@ -102,13 +114,9 @@ def GetFileInfo( path, hash ): ( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetFLVProperties( path ) - elif mime in ( HC.VIDEO_WMV, HC.VIDEO_MP4 ): + elif mime in ( HC.VIDEO_WMV, HC.VIDEO_MP4, HC.VIDEO_MKV, HC.VIDEO_WEBM ): - ( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetCVVideoProperties( path ) - - elif mime in ( HC.VIDEO_MKV, HC.VIDEO_WEBM ): - - ( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetCVVideoProperties( path ) + ( ( width, height ), duration, num_frames ) = HydrusVideoHandling.GetFFMPEGVideoProperties( path ) elif mime == HC.APPLICATION_PDF: num_words = HydrusDocumentHandling.GetPDFNumWords( path ) elif mime == HC.AUDIO_MP3: duration = HydrusAudioHandling.GetMP3Duration( path ) diff --git a/include/HydrusImageHandling.py b/include/HydrusImageHandling.py index 4a555432..3104cdfe 100755 --- a/include/HydrusImageHandling.py +++ b/include/HydrusImageHandling.py @@ -89,7 +89,7 @@ def GenerateCVImage( path ): cv_image = cv2.imread( self._path, flags = -1 ) # flags = -1 loads alpha channel, if present - ( x, y, depth ) = cv_image.shape + ( y, x, depth ) = cv_image.shape if depth == 4: raise Exception( 'CV is bad at alpha!' ) else: cv_image = cv2.cvtColor( cv_image, cv2.COLOR_BGR2RGB ) @@ -102,7 +102,7 @@ def GenerateHydrusBitmap( path ): cv_image = GenerateCVImage( path ) - return GenerateHydrusBitmapFromCVImage( cv_image ) + return GenerateHydrusBitmapFromNumPyImage( cv_image ) except: @@ -111,7 +111,7 @@ def GenerateHydrusBitmap( path ): return GenerateHydrusBitmapFromPILImage( pil_image ) -def GenerateHydrusBitmapFromCVImage( cv_image ): +def GenerateHydrusBitmapFromNumPyImage( cv_image ): ( y, x, depth ) = cv_image.shape @@ -137,7 +137,7 @@ def GeneratePerceptualHash( path ): cv_image = cv2.imread( path, cv2.CV_LOAD_IMAGE_UNCHANGED ) - ( x, y, depth ) = cv_image.shape + ( y, x, depth ) = cv_image.shape if depth == 4: @@ -153,15 +153,15 @@ def GeneratePerceptualHash( path ): cv_image_gray = cv2.cvtColor( cv_image_bgr, cv2.COLOR_BGR2GRAY ) - cv_image_result = numpy.empty( ( x, y ), numpy.float32 ) + cv_image_result = numpy.empty( ( y, x ), numpy.float32 ) # paste greyscale onto the white # can't think of a better way to do this! # cv2.addWeighted only takes a scalar for weight! - for i in range( x ): + for i in range( y ): - for j in range( y ): + for j in range( x ): opacity = float( cv_alpha[ i, j ] ) / 255.0 @@ -321,6 +321,17 @@ def old_GeneratePerceptualHash( path ): def GeneratePILImage( path ): return PILImage.open( path ) +def GeneratePILImageFromNumpyImage( numpy_image ): + + ( h, w, depth ) = numpy_image.shape + + if depth == 3: format = 'RGB' + elif depth == 4: format = 'RGBA' + + pil_image = PILImage.fromstring( format, ( w, h ), numpy_image.data ) + + return pil_image + def GetGIFFrameDurations( path ): pil_image_for_duration = GeneratePILImage( path ) @@ -568,7 +579,7 @@ class ImageContainer( RasterContainer ): resized_cv_image = EfficientlyResizeCVImage( cv_image, self._target_resolution ) - return GenerateHydrusBitmapFromCVImage( resized_cv_image ) + return GenerateHydrusBitmapFromNumPyImage( resized_cv_image ) except: diff --git a/include/HydrusNetworking.py b/include/HydrusNetworking.py index 3a43ebb9..40303894 100644 --- a/include/HydrusNetworking.py +++ b/include/HydrusNetworking.py @@ -3,6 +3,7 @@ import HydrusExceptions import httplib import threading import time +import urllib import urlparse import yaml @@ -132,12 +133,15 @@ class HTTPConnectionManager(): threading.Thread( target = self.DAEMONMaintainConnections, name = 'Maintain Connections' ).start() - def _DoRequest( self, location, method, path_and_query, request_headers, body, follow_redirects = True, report_hooks = [], response_to_path = False, num_redirects_permitted = 4, long_timeout = False ): + def _DoRequest( self, method, location, path, query, request_headers, body, follow_redirects = True, report_hooks = [], response_to_path = False, num_redirects_permitted = 4, long_timeout = False ): connection = self._GetConnection( location, long_timeout ) try: + if query == '': path_and_query = path + else: path_and_query = path + '?' + query + with connection.lock: ( parsed_response, redirect_info, size_of_response, response_headers, cookies ) = connection.Request( method, path_and_query, request_headers, body, report_hooks = report_hooks, response_to_path = response_to_path ) @@ -154,10 +158,7 @@ class HTTPConnectionManager(): if new_location is None: new_location = location - if new_query != '': new_path_and_query = new_path + '?' + new_query - else: new_path_and_query = new_path - - return self._DoRequest( new_location, new_method, new_path_and_query, request_headers, body, report_hooks = report_hooks, response_to_path = response_to_path, num_redirects_permitted = num_redirects_permitted - 1 ) + return self._DoRequest( new_method, new_location, new_path, new_query, request_headers, body, follow_redirects = follow_redirects, report_hooks = report_hooks, response_to_path = response_to_path, num_redirects_permitted = num_redirects_permitted - 1, long_timeout = long_timeout ) except: @@ -191,12 +192,9 @@ class HTTPConnectionManager(): ( location, path, query ) = ParseURL( url ) - if query != '': path_and_query = path + '?' + query - else: path_and_query = path - follow_redirects = not return_cookies - ( response, size_of_response, response_headers, cookies ) = self._DoRequest( location, method, path_and_query, request_headers, body, follow_redirects = follow_redirects, report_hooks = report_hooks, response_to_path = response_to_path, long_timeout = long_timeout ) + ( response, size_of_response, response_headers, cookies ) = self._DoRequest( method, location, path, query, request_headers, body, follow_redirects = follow_redirects, report_hooks = report_hooks, response_to_path = response_to_path, long_timeout = long_timeout ) if return_everything: return ( response, size_of_response, response_headers, cookies ) elif return_cookies: return ( response, cookies ) @@ -453,6 +451,23 @@ class HTTPConnection(): url = location + if ' ' in url: + + # some booru is giving daft redirect responses + print( url ) + url = urllib.quote( url, safe = '/?=&' ) + print( url ) + + if not url.startswith( self._scheme ): + + # assume it is like 'index.php' or '/index.php', rather than 'http://blah.com/index.php' + + if url.startswith( '/' ): slash_sep = '' + else: slash_sep = '/' + + url = self._scheme + '://' + self._host + slash_sep + url + + if response.status in ( 301, 307 ): # 301: moved permanently, repeat request diff --git a/include/HydrusServerResources.py b/include/HydrusServerResources.py index a16e6b7f..20613bb4 100644 --- a/include/HydrusServerResources.py +++ b/include/HydrusServerResources.py @@ -238,7 +238,7 @@ def ParseFileArguments( path ): hash = HydrusFileHandling.GetHashFromPath( path ) - try: ( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path, hash ) + try: ( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path ) except HydrusExceptions.SizeException: raise HydrusExceptions.ForbiddenException( 'File is of zero length!' ) except HydrusExceptions.MimeException: raise HydrusExceptions.ForbiddenException( 'Filetype is not permitted!' ) except Exception as e: raise HydrusExceptions.ForbiddenException( HC.u( e ) ) diff --git a/include/HydrusVideoHandling.py b/include/HydrusVideoHandling.py index b87e934b..9b0b26ec 100755 --- a/include/HydrusVideoHandling.py +++ b/include/HydrusVideoHandling.py @@ -40,6 +40,20 @@ def GetCVVideoProperties( path ): return ( ( width, height ), duration, num_frames ) +def GetFFMPEGVideoProperties( path ): + + info = Hydrusffmpeg_parse_infos( path ) + + ( w, h ) = info[ 'video_size' ] + + duration_in_s = info[ 'duration' ] + + duration = int( duration_in_s * 1000 ) + + num_frames = info[ 'video_nframes' ] + + return ( ( w, h ), duration, num_frames ) + def GetFLVProperties( path ): with open( path, 'rb' ) as f: @@ -127,144 +141,7 @@ def GetMatroskaOrWebMProperties( path ): return ( ( width, height ), duration, num_frames ) -# This is cribbed from moviepy's FFMPEG_VideoReader -class HydrusFFMPEG_VideoReader( object ): - - def __init__(self, media, print_infos=False, bufsize = None, pix_fmt="rgb24"): - - self._media = media - - hash = self._media.GetHash() - mime = self._media.GetMime() - - self._path = CC.GetFilePath( hash, mime ) - - self.size = self._media.GetResolution() - self.duration = float( self._media.GetDuration() ) / 1000.0 - self.nframes = self._media.GetNumFrames() - - self.fps = float( self.nframes ) / self.duration - - self.pix_fmt = pix_fmt - - if pix_fmt == 'rgba': self.depth = 4 - else: self.depth = 3 - - if bufsize is None: - - ( x, y ) = self.size - bufsize = self.depth * x * y * 5 - - - self.process = None - - self.bufsize = bufsize - - self.initialize() - - - def __del__( self ): - - self.close() - - - def close(self): - - if self.process is not None: - - self.process.terminate() - - self.process.stdout.close() - self.process.stderr.close() - - self.process = None - - - - def initialize( self, starttime=0 ): - """Opens the file, creates the pipe. """ - - self.close() - - if starttime != 0: - - offset = min( 1, starttime ) - - i_arg = [ '-ss', "%.03f" % ( starttime - offset ), '-i', '"' + self._path + '"' ] - - else: i_arg = [ '-i', '"' + self._path + '"' ] - - cmd = ([FFMPEG_PATH]+ i_arg + - ['-loglevel', 'error', - '-f', 'image2pipe', - "-pix_fmt", self.pix_fmt, - '-vcodec', 'rawvideo', '-']) - - self.process = subprocess.Popen( ' '.join( cmd ), shell = True, bufsize= self.bufsize, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - - self.pos = int( round( self.fps * starttime ) ) - - - def skip_frames(self, n=1): - - """Reads and throws away n frames """ - - ( w, h ) = self.size - - for i in range( n ): - - self.process.stdout.read( self.depth * w * h ) - self.process.stdout.flush() - - - self.pos += n - - - def read_frame( self ): - - ( w, h ) = self.size - - nbytes = self.depth * w * h - - s = self.process.stdout.read(nbytes) - - self.pos += 1 - - if len(s) != nbytes: - - print( "Warning: in file %s, "%(self._path)+ - "%d bytes wanted but %d bytes read,"%(nbytes, len(s))+ - "at frame %d/%d, at time %.02f/%.02f sec. "%( - self.pos,self.nframes, - 1.0*self.pos/self.fps, - self.duration)+ - "Using the last valid frame instead.") - result = self.lastread - - self.close() - - else: - - result = numpy.fromstring( s, dtype = 'uint8' ).reshape( ( h, w, len( s ) // ( w * h ) ) ) - - self.lastread = result - - - return result - - - def set_position( self, pos ): - - if ( pos < self.pos ) or ( pos > self.pos + 60 ): - - starttime = float( pos ) / self.fps - - self.initialize( starttime ) - - else: self.skip_frames( pos - self.pos ) - - -# same thing; this is cribbed from moviepy +# this is cribbed from moviepy def Hydrusffmpeg_parse_infos(filename, print_infos=False): """Get file infos using ffmpeg. @@ -306,9 +183,19 @@ def Hydrusffmpeg_parse_infos(filename, print_infos=False): result = dict() # get duration (in seconds) + # Duration: 00:00:02.46, start: 0.033000, bitrate: 1069 kb/s try: keyword = ('frame=' if is_GIF else 'Duration: ') line = [l for l in lines if keyword in l][0] + + if 'start:' in line: + + m = re.search( '(start\\: )' + '[0-9]\\.[0-9]*', line ) + + start_offset = float( line[ m.start() + 7 : m.end() ] ) + + else: start_offset = 0 + match = re.search("[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9]", line) hms = map(float, line[match.start()+1:match.end()].split(':')) @@ -319,6 +206,8 @@ def Hydrusffmpeg_parse_infos(filename, print_infos=False): elif len(hms) ==3: result['duration'] = 3600*hms[0]+60*hms[1]+hms[2] + result[ 'duration' ] -= start_offset + except: raise IOError("Error reading duration in file %s,"%(filename)+ "Text parsed: %s"%infos) @@ -345,9 +234,13 @@ def Hydrusffmpeg_parse_infos(filename, print_infos=False): except: match = re.search("( [0-9]*.| )[0-9]* fps", line) result['video_fps'] = float(line[match.start():match.end()].split(' ')[1]) - - result['video_nframes'] = int(result['duration']*result['video_fps'])+1 - + + num_frames = result['duration'] * result['video_fps'] + + if num_frames != int( num_frames ): num_frames += 1 # rounding up + + result['video_nframes'] = int( num_frames ) + result['video_duration'] = result['duration'] # We could have also recomputed the duration from the number # of frames, as follows: @@ -378,8 +271,8 @@ class VideoContainer( HydrusImageHandling.RasterContainer ): self._frames = {} self._last_index_asked_for = -1 - self._minimum_frame_asked_for = 0 - self._maximum_frame_asked_for = 0 + self._buffer_start_index = -1 + self._buffer_end_index = -1 ( x, y ) = self._target_resolution @@ -391,9 +284,21 @@ class VideoContainer( HydrusImageHandling.RasterContainer ): if self._media.GetMime() == HC.IMAGE_GIF: self._durations = HydrusImageHandling.GetGIFFrameDurations( self._path ) else: self._frame_duration = GetVideoFrameDuration( self._path ) - self._renderer = VideoRendererMoviePy( self, self._media, self._target_resolution ) + self._render_lock = threading.Lock() - num_frames = self.GetNumFrames() + hash = self._media.GetHash() + mime = self._media.GetMime() + + path = CC.GetFilePath( hash, mime ) + + duration = self._media.GetDuration() + num_frames = self._media.GetNumFrames() + + self._ffmpeg_reader = VideoRendererFFMPEG( path, mime, duration, num_frames, target_resolution ) + + self._next_render_index = 0 + self._render_to_index = -1 + self._rendered_first_frame = False self.SetPosition( init_position ) @@ -404,19 +309,81 @@ class VideoContainer( HydrusImageHandling.RasterContainer ): for index in self._frames.keys(): - if self._minimum_frame_asked_for < self._maximum_frame_asked_for: + if self._buffer_start_index < self._buffer_end_index: - if index < self._minimum_frame_asked_for or self._maximum_frame_asked_for < index: deletees.append( index ) + if index < self._buffer_start_index or self._buffer_end_index < index: deletees.append( index ) else: - if self._maximum_frame_asked_for < index and index < self._minimum_frame_asked_for: deletees.append( index ) + if self._buffer_end_index < index and index < self._buffer_start_index: deletees.append( index ) for i in deletees: del self._frames[ i ] + def _RENDERERSetRenderToPosition( self, index ): + + with self._render_lock: + + if self._render_to_index != index: + + self._render_to_index = index + + HydrusThreading.CallToThread( self.THREADRender ) + + + + + def _RENDERERSetPosition( self, index ): + + with self._render_lock: + + if index == self._next_render_index: return + else: + + self._ffmpeg_reader.set_position( index ) + + self._next_render_index = index + self._render_to_index = index + + + + + def THREADRender( self ): + + num_frames = self._media.GetNumFrames() + + while True: + + time.sleep( 0.00001 ) # thread yield + + with self._render_lock: + + if not self._rendered_first_frame or self._next_render_index != ( self._render_to_index + 1 ) % num_frames: + + self._rendered_first_frame = True + + frame_index = self._next_render_index # keep this before the get call, as it increments in a clock arithmetic way afterwards + + try: numpy_image = self._ffmpeg_reader.read_frame() + except Exception as e: + + HC.ShowException( e ) + + break + + finally: self._next_render_index = ( self._next_render_index + 1 ) % num_frames + + frame = HydrusImageHandling.GenerateHydrusBitmapFromNumPyImage( numpy_image ) + + wx.CallAfter( self.AddFrame, frame_index, frame ) + + else: break + + + + def AddFrame( self, index, frame ): self._frames[ index ] = frame def GetDuration( self, index ): @@ -464,61 +431,205 @@ class VideoContainer( HydrusImageHandling.RasterContainer ): if num_frames > self._num_frames_backwards + 1 + self._num_frames_forwards: - new_minimum_frame_to_ask_for = max( 0, index - self._num_frames_backwards ) % num_frames + new_buffer_start_index = max( 0, index - self._num_frames_backwards ) % num_frames - new_maximum_frame_to_ask_for = ( index + self._num_frames_forwards ) % num_frames + new_buffer_end_index = ( index + self._num_frames_forwards ) % num_frames if index == self._last_index_asked_for: return elif index < self._last_index_asked_for: - if index < self._minimum_frame_asked_for: + if index < self._buffer_start_index: - self._minimum_frame_asked_for = new_minimum_frame_to_ask_for + self._buffer_start_index = new_buffer_start_index - self._renderer.SetPosition( self._minimum_frame_asked_for ) + self._RENDERERSetPosition( self._buffer_start_index ) - self._maximum_frame_asked_for = new_maximum_frame_to_ask_for + self._buffer_end_index = new_buffer_end_index - self._renderer.SetRenderToPosition( self._maximum_frame_asked_for ) + self._RENDERERSetRenderToPosition( self._buffer_end_index ) else: # index > self._last_index_asked_for - currently_no_wraparound = self._minimum_frame_asked_for < self._maximum_frame_asked_for + currently_no_wraparound = self._buffer_start_index < self._buffer_end_index - self._minimum_frame_asked_for = new_minimum_frame_to_ask_for + self._buffer_start_index = new_buffer_start_index if currently_no_wraparound: - if index > self._maximum_frame_asked_for: + if index > self._buffer_end_index: - self._renderer.SetPosition( self._minimum_frame_asked_for ) + self._RENDERERSetPosition( self._buffer_start_index ) - self._maximum_frame_asked_for = new_maximum_frame_to_ask_for + self._buffer_end_index = new_buffer_end_index - self._renderer.SetRenderToPosition( self._maximum_frame_asked_for ) + self._RENDERERSetRenderToPosition( self._buffer_end_index ) else: - if self._maximum_frame_asked_for == 0: + if self._buffer_end_index == -1: - self._minimum_frame_asked_for = 0 + self._buffer_start_index = 0 - self._renderer.SetPosition( 0 ) + self._RENDERERSetPosition( 0 ) - self._maximum_frame_asked_for = num_frames - 1 + self._buffer_end_index = num_frames - 1 - self._renderer.SetRenderToPosition( self._maximum_frame_asked_for ) + self._RENDERERSetRenderToPosition( self._buffer_end_index ) self._MaintainBuffer() -class VideoRendererCV(): +# This was built from moviepy's FFMPEG_VideoReader +class VideoRendererFFMPEG( object ): + + def __init__( self, path, mime, duration, num_frames, target_resolution, pix_fmt = "rgb24" ): + + self._path = path + self._mime = mime + self._duration = float( duration ) / 1000.0 + self._num_frames = num_frames + self._target_resolution = target_resolution + + self.fps = float( self._num_frames ) / self._duration + + if self.fps == 0: self.fps = 24 + + self.pix_fmt = pix_fmt + + if pix_fmt == 'rgba': self.depth = 4 + else: self.depth = 3 + + ( x, y ) = self._target_resolution + bufsize = self.depth * x * y + + self.process = None + + self.bufsize = bufsize + + self.initialize() + + + def __del__( self ): + + self.close() + + + def close(self): + + if self.process is not None: + + self.process.terminate() + + self.process.stdout.close() + self.process.stderr.close() + + self.process = None + + + + def initialize( self, start_index = 0 ): + + self.close() + + if self._mime == HC.IMAGE_GIF: + + ss = 0 + self.pos = 0 + skip_frames = start_index + + else: + + ss = float( start_index ) / self.fps + self.pos = start_index + skip_frames = 0 + + + ( w, h ) = self._target_resolution + + cmd = ( [ FFMPEG_PATH, + '-ss', "%.03f" % ss, + '-i', '"' + self._path + '"', + '-loglevel', 'quiet', + '-f', 'image2pipe', + "-pix_fmt", self.pix_fmt, + "-s", str( w ) + 'x' + str( h ), + '-vcodec', 'rawvideo', '-' ] ) + + self.process = subprocess.Popen( ' '.join( cmd ), shell = True, bufsize= self.bufsize, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + + self.skip_frames( skip_frames ) + + + def skip_frames( self, n ): + + ( w, h ) = self._target_resolution + + for i in range( n ): + + self.process.stdout.read( self.depth * w * h ) + + self.process.stdout.flush() + + self.pos += 1 + + + + def read_frame( self ): + + if self.pos == self._num_frames: self.initialize() + + if self.process is None: result = self.lastread + else: + + ( w, h ) = self._target_resolution + + nbytes = self.depth * w * h + + s = self.process.stdout.read(nbytes) + + if len(s) != nbytes: + + print( "Warning: in file %s, "%(self._path)+ + "%d bytes wanted but %d bytes read,"%(nbytes, len(s))+ + "at frame %d/%d, at time %.02f/%.02f sec. "%( + self.pos,self._num_frames, + 1.0*self.pos/self.fps, + self._duration)+ + "Using the last valid frame instead.") + + result = self.lastread + + self.close() + + else: + + result = numpy.fromstring( s, dtype = 'uint8' ).reshape( ( h, w, len( s ) // ( w * h ) ) ) + + self.lastread = result + + + + self.pos += 1 + + return result + + + def set_position( self, pos ): + + rewind = pos < self.pos + jump_a_long_way_ahead = pos > self.pos + 60 + + if rewind or jump_a_long_way_ahead: self.initialize( pos ) + else: self.skip_frames( pos - self.pos ) + + +class OLDCODEVideoRendererCV(): def __init__( self, image_container, media, target_resolution ): @@ -531,13 +642,13 @@ class VideoRendererCV(): self._path = CC.GetFilePath( hash, mime ) - self._lock = threading.Lock() + self._render_lock = threading.Lock() self._cv_video = cv2.VideoCapture( self._path ) self._cv_video.set( cv2.cv.CV_CAP_PROP_CONVERT_RGB, True ) - self._current_index = 0 + self._next_render_index = 0 self._last_index_rendered = -1 self._last_frame = None self._render_to_index = -1 @@ -547,13 +658,13 @@ class VideoRendererCV(): ( retval, cv_image ) = self._cv_video.read() - self._last_index_rendered = self._current_index + self._last_index_rendered = self._next_render_index num_frames = self._media.GetNumFrames() - self._current_index = ( self._current_index + 1 ) % num_frames + self._next_render_index = ( self._next_render_index + 1 ) % num_frames - if self._current_index == 0 and self._last_index_rendered != 0: + if self._next_render_index == 0 and self._last_index_rendered != 0: if self._media.GetMime() == HC.IMAGE_GIF: self._RewindGIF() else: self._cv_video.set( cv2.cv.CV_CAP_PROP_POS_FRAMES, 0.0 ) @@ -561,7 +672,7 @@ class VideoRendererCV(): if not retval: - raise HydrusExceptions.CantRenderWithCVException( 'CV could not render frame ' + HC.u( self._current_index ) + '.' ) + raise HydrusExceptions.CantRenderWithCVException( 'CV could not render frame ' + HC.u( self._next_render_index ) + '.' ) return cv_image @@ -584,27 +695,7 @@ class VideoRendererCV(): cv_image = self._last_frame - return HydrusImageHandling.GenerateHydrusBitmapFromCVImage( cv_image ) - - - def _RenderFrames( self ): - - no_frames_yet = True - - while True: - - try: - - yield self._RenderCurrentFrame() - - no_frames_yet = False - - except HydrusExceptions.CantRenderWithCVException: - - if no_frames_yet: raise - else: break - - + return HydrusImageHandling.GenerateHydrusBitmapFromNumPyImage( cv_image ) def _RewindGIF( self ): @@ -612,12 +703,12 @@ class VideoRendererCV(): self._cv_video.release() self._cv_video.open( self._path ) - self._current_index = 0 + self._next_render_index = 0 def SetRenderToPosition( self, index ): - with self._lock: + with self._render_lock: if self._render_to_index != index: @@ -630,14 +721,14 @@ class VideoRendererCV(): def SetPosition( self, index ): - with self._lock: + with self._render_lock: if self._media.GetMime() == HC.IMAGE_GIF: - if index == self._current_index: return - elif index < self._current_index: self._RewindGIF() + if index == self._next_render_index: return + elif index < self._next_render_index: self._RewindGIF() - while self._current_index < index: self._GetCurrentFrame() + while self._next_render_index < index: self._GetCurrentFrame() else: @@ -654,149 +745,11 @@ class VideoRendererCV(): time.sleep( 0.00001 ) # thread yield - with self._lock: + with self._render_lock: if self._last_index_rendered != self._render_to_index: - index = self._current_index - - frame = self._RenderCurrentFrame() - - wx.CallAfter( self._image_container.AddFrame, index, frame ) - - else: break - - - - -class VideoRendererMoviePy(): - - def __init__( self, image_container, media, target_resolution ): - - self._image_container = image_container - self._media = media - self._target_resolution = target_resolution - - self._lock = threading.Lock() - - ( x, y ) = media.GetResolution() - - self._ffmpeg_reader = HydrusFFMPEG_VideoReader( media, bufsize = x * y * 3 * 5 ) - - self._current_index = 0 - self._last_index_rendered = -1 - self._last_frame = None - self._render_to_index = -1 - - - def _GetCurrentFrame( self ): - - try: image = self._ffmpeg_reader.read_frame() - except Exception as e: raise HydrusExceptions.CantRenderWithCVException( 'FFMPEG could not render frame ' + HC.u( self._current_index ) + '.' + os.linesep * 2 + HC.u( e ) ) - - self._last_index_rendered = self._current_index - - num_frames = self._media.GetNumFrames() - - self._current_index = ( self._current_index + 1 ) % num_frames - - if self._current_index == 0 and self._last_index_rendered != 0: self._ffmpeg_reader.initialize() - - return image - - - def _RenderCurrentFrame( self ): - - try: - - image = self._GetCurrentFrame() - - image = HydrusImageHandling.EfficientlyResizeCVImage( image, self._target_resolution ) - - self._last_frame = image - - except HydrusExceptions.CantRenderWithCVException: - - if self._last_frame is None: raise - - image = self._last_frame - - - return HydrusImageHandling.GenerateHydrusBitmapFromCVImage( image ) - - - def _RenderFrames( self ): - - no_frames_yet = True - - while True: - - try: - - yield self._RenderCurrentFrame() - - no_frames_yet = False - - except HydrusExceptions.CantRenderWithCVException: - - if no_frames_yet: raise - else: break - - - - - def SetRenderToPosition( self, index ): - - with self._lock: - - if self._render_to_index != index: - - self._render_to_index = index - - HydrusThreading.CallToThread( self.THREADDoWork ) - - - - - def SetPosition( self, index ): - - with self._lock: - - if index == self._current_index: return - else: - - if self._media.GetMime() == HC.IMAGE_GIF: - - if index < self._current_index: - - self._ffmpeg_reader.initialize() - - self._ffmpeg_reader.skip_frames( index ) - - - else: - - timecode = float( index ) / self._ffmpeg_reader.fps - - self._ffmpeg_reader.set_position( timecode ) - - - self._render_to_index = index - - - - - def THREADDoWork( self ): - - while True: - - time.sleep( 0.00001 ) # thread yield - - with self._lock: - - if self._last_index_rendered != self._render_to_index: - - index = self._current_index + index = self._next_render_index frame = self._RenderCurrentFrame() diff --git a/include/TestHydrusDownloading.py b/include/TestHydrusDownloading.py index ddc86dba..f303fac0 100644 --- a/include/TestHydrusDownloading.py +++ b/include/TestHydrusDownloading.py @@ -98,7 +98,46 @@ class TestDownloaders( unittest.TestCase ): info = ( data, tags ) - expected_info = ('swf file', set([u'chores', u'laser', u'silent', u'title:Cat Dust', u'creator:warlord-of-noodles', u'pointer'])) + expected_info = ( 'swf file', set([u'chores', u'laser', u'silent', u'title:Cat Dust', u'creator:warlord-of-noodles', u'pointer']) ) + + self.assertEqual( info, expected_info ) + + + def test_pixiv( self ): + + with open( HC.STATIC_DIR + os.path.sep + 'testing' + os.path.sep + 'pixiv_gallery.html' ) as f: pixiv_gallery = f.read() + with open( HC.STATIC_DIR + os.path.sep + 'testing' + os.path.sep + 'pixiv_page.html' ) as f: pixiv_page = f.read() + + HC.http.SetResponse( HC.GET, 'http://www.pixiv.net/search.php?word=naruto&s_mode=s_tag_full&order=date_d&p=1', pixiv_gallery ) + HC.http.SetResponse( HC.GET, 'http://www.pixiv.net/member_illust.php?mode=medium&illust_id=43718605', pixiv_page ) + + HC.http.SetResponse( HC.GET, 'http://i1.pixiv.net/img59/img/dbhope/43718605.jpg', 'image file' ) + + # + + downloader = HydrusDownloading.DownloaderPixiv( 'tags', 'naruto' ) + + # + + gallery_urls = downloader.GetAnotherPage() + + expected_gallery_urls = [ ( u'http://www.pixiv.net/member_illust.php?mode=medium&illust_id=43718605', ), 'a bunch of others' ] + + self.assertEqual( gallery_urls[0], expected_gallery_urls[0] ) + + # + + info = downloader.GetFileAndTags( 'http://www.pixiv.net/member_illust.php?mode=medium&illust_id=43718605' ) + + ( temp_path, tags ) = info + + with open( temp_path, 'rb' ) as f: data = f.read() + + info = ( data, tags ) + + expected_tags = [ u'1P\u6f2b\u753b', u'\u7720\u305f\u3044', u'NARUTO', u'\u30ca\u30eb\u30c8', u'\u30b5\u30b9\u30b1', u'\u30b5\u30af\u30e9', u'\u30d2\u30ca\u30bf', u'creator:\u30df\u30c4\u30ad\u30e8\u3063\u3057\uff5e', u'title:\u7720\u305f\u3044', 'creator:dbhope' ] + + expected_info = ( 'image file', expected_tags ) self.assertEqual( info, expected_info ) diff --git a/static/testing/pixiv_gallery.html b/static/testing/pixiv_gallery.html new file mode 100644 index 00000000..522533f4 --- /dev/null +++ b/static/testing/pixiv_gallery.html @@ -0,0 +1,717 @@ + + + + + + + + + + + + + + + + + + + + + +
- Designating tags such as "Chobi" will display partial tag matches. +
- Designating tags such as "" Chobi "" or ""11"" will display exact tag matches. +
- Designating a number, such as ""11"" will check user IDs or tags by partial matches. For a more specific filter, please add more details. +
- Designating tags such as "Chobi" will display partial tag matches. +
- Designating tags such as "" Chobi "" or ""11"" will display exact tag matches. +
- Designating a number, such as ""11"" will check user IDs or tags by partial matches. For a more specific filter, please add more details. +
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+How to designate tags or user IDs
+-
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/static/testing/pixiv_page.html b/static/testing/pixiv_page.html
new file mode 100644
index 00000000..2d6d4611
--- /dev/null
+++ b/static/testing/pixiv_page.html
@@ -0,0 +1,1089 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Advanced Search
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+How to designate tags or user IDs
+-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+There are no comments.
+
+
Share
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
