libvmaf: exit gracefully if the library fails.

Fixes trac issue #6884 and Netflix/vmaf issue #124.

libavfilter/vf_libvmaf.c

@@ -61,6 +61,7 @@ typedef struct LIBVMAFContext {
     int ssim;
     int ms_ssim;
     char *pool;
+    int error;
 } LIBVMAFContext;
 
 #define OFFSET(x) offsetof(LIBVMAFContext, x)
@@ -158,18 +159,26 @@ static void compute_vmaf_score(LIBVMAFContext *s)
 
     format = (char *) s->desc->name;
 
-    s->vmaf_score = compute_vmaf(format, s->width, s->height, read_frame, s,
-                                 s->model_path, s->log_path, s->log_fmt, 0, 0,
-                                 s->enable_transform, s->phone_model, s->psnr,
-                                 s->ssim, s->ms_ssim, s->pool);
+    s->error = compute_vmaf(&s->vmaf_score, format, s->width, s->height,
+                            read_frame, s, s->model_path, s->log_path,
+                            s->log_fmt, 0, 0, s->enable_transform,
+                            s->phone_model, s->psnr, s->ssim,
+                            s->ms_ssim, s->pool);
 }
 
 static void *call_vmaf(void *ctx)
 {
     LIBVMAFContext *s = (LIBVMAFContext *) ctx;
     compute_vmaf_score(s);
-    av_log(ctx, AV_LOG_INFO, "VMAF score: %f\n",s->vmaf_score);
+    if (!s->error) {
+        av_log(ctx, AV_LOG_INFO, "VMAF score: %f\n",s->vmaf_score);
+    } else {
+        pthread_mutex_lock(&s->lock);
+        pthread_cond_signal(&s->cond);
+        pthread_mutex_unlock(&s->lock);
+    }
     pthread_exit(NULL);
+    return NULL;
 }
 
 static int do_vmaf(FFFrameSync *fs)
@@ -187,10 +196,17 @@ static int do_vmaf(FFFrameSync *fs)
 
     pthread_mutex_lock(&s->lock);
 
-    while (s->frame_set != 0) {
+    while (s->frame_set && !s->error) {
         pthread_cond_wait(&s->cond, &s->lock);
     }
 
+    if (s->error) {
+        av_log(ctx, AV_LOG_ERROR,
+               "libvmaf encountered an error, check log for details\n");
+        pthread_mutex_unlock(&s->lock);
+        return AVERROR(EINVAL);
+    }
+
     av_frame_ref(s->gref, ref);
     av_frame_ref(s->gmain, master);
 
@@ -208,6 +224,7 @@ static av_cold int init(AVFilterContext *ctx)
 
     s->gref = av_frame_alloc();
     s->gmain = av_frame_alloc();
+    s->error = 0;
 
     pthread_mutex_init(&s->lock, NULL);
     pthread_cond_init (&s->cond, NULL);