audio/format: add heuristic to estimate loss on format conversion

The added function af_format_conversion_score() can be used to select
the best sample format to convert to in order to reduce loss and extra
conversion work.

It calculates a "loss" score when going from one format to another, and
for each conversion that needs to be done a certain score is subtracted.
Thus, if you have to convert from one format to a set of other formats,
you can calculate the score for each conversion, and pick the one with
the highest score.

Conversion between int and float is considered the worst case. One odd
consequence is that when converting from s32 to u8 or float, u8 will be
picked.

Test program used to develop this follows:

#define MAX_FMT 200
struct entry {
    const char *name;
    int score;
};

static int compentry(const void *px1, const void *px2)
{
    const struct entry *x1 = px1;
    const struct entry *x2 = px2;
    if (x1->score > x2->score)
        return 1;
    if (x1->score < x2->score)
        return -1;
    return 0;
}

int main(int argc, char *argv[])
{
    for (int n = 0; af_fmtstr_table[n].name; n++) {
        struct entry entry[MAX_FMT];
        int entries = 0;
        for (int i = 0; af_fmtstr_table[i].name; i++) {
            assert(i < MAX_FMT);
            entry[entries].name = af_fmtstr_table[i].name;
            entry[entries].score =
                af_format_conversion_score(af_fmtstr_table[i].format,
                                           af_fmtstr_table[n].format);
            entries++;
        }
        qsort(&entry[0], entries, sizeof(entry[0]), compentry);
        for (int i = 0; i < entries; i++) {
            printf("%s -> %s: %d \n", af_fmtstr_table[n].name,
                   entry[i].name, entry[i].score);
        }
    }
}

audio/format.c

@@ -181,3 +181,51 @@ void af_fill_silence(void *dst, size_t bytes, int format)
     bool us = (format & AF_FORMAT_SIGN_MASK) == AF_FORMAT_US;
     memset(dst, us ? 0x80 : 0, bytes);
 }
+
+#define FMT_DIFF(type, a, b) (((a) & type) - ((b) & type))
+
+// Returns a "score" that serves as heuristic how lossy or hard a conversion is.
+// If the formats are equal, 1024 is returned. If they are gravely incompatible
+// (like s16<->ac3), INT_MIN is returned. If there is implied loss of precision
+// (like s16->s8), a value <0 is returned.
+int af_format_conversion_score(int dst_format, int src_format)
+{
+    if (dst_format == AF_FORMAT_UNKNOWN || src_format == AF_FORMAT_UNKNOWN)
+        return INT_MIN;
+    if (dst_format == src_format)
+        return 1024;
+    // Just endian swapping (separate because it works for special formats)
+    if ((dst_format & ~AF_FORMAT_END_MASK) == (src_format & ~AF_FORMAT_END_MASK))
+        return 1024 - 2;
+    // Can't be normally converted
+    if (AF_FORMAT_IS_SPECIAL(dst_format) || AF_FORMAT_IS_SPECIAL(src_format))
+        return INT_MIN;
+    int score = 1024;
+    if (FMT_DIFF(AF_FORMAT_INTERLEAVING_MASK, dst_format, src_format))
+        score -= 1;     // has to (de-)planarize
+    if (FMT_DIFF(AF_FORMAT_END_MASK, dst_format, src_format))
+        score -= 2;     // has to swap endian
+    if (FMT_DIFF(AF_FORMAT_SIGN_MASK, dst_format, src_format))
+        score -= 4;     // has to swap sign
+    if (FMT_DIFF(AF_FORMAT_POINT_MASK, dst_format, src_format)) {
+        int dst_bits = dst_format & AF_FORMAT_BITS_MASK;
+        if ((dst_format & AF_FORMAT_POINT_MASK) == AF_FORMAT_F) {
+            // For int->float, always prefer 32 bit float.
+            score -= dst_bits == AF_FORMAT_32BIT ? 8 : 0;
+        } else {
+            // For float->int, always prefer highest bit depth int
+            score -= 8 * (AF_FORMAT_64BIT - dst_bits);
+        }
+    } else {
+        int bits = FMT_DIFF(AF_FORMAT_BITS_MASK, dst_format, src_format);
+        if (bits > 0) {
+            score -= 8 * bits;          // has to add padding
+        } else if (bits < 0) {
+            score -= 1024 - 8 * bits;   // has to reduce bit depth
+        }
+    }
+    // Consider this the worst case.
+    if (FMT_DIFF(AF_FORMAT_POINT_MASK, dst_format, src_format))
+        score -= 2048;  // has to convert float<->int
+    return score;
+}

audio/format.h

@@ -47,6 +47,7 @@
 #define AF_FORMAT_SIGN_MASK     (1<<1)
 
 // Bits used
+// Some code assumes they're sorted by size.
 #define AF_FORMAT_8BIT          (0<<3)
 #define AF_FORMAT_16BIT         (1<<3)
 #define AF_FORMAT_24BIT         (2<<3)
@@ -164,4 +165,6 @@ bool af_fmt_is_valid(int format);
 
 void af_fill_silence(void *dst, size_t bytes, int format);
 
+int af_format_conversion_score(int dst_format, int src_format);
+
 #endif /* MPLAYER_AF_FORMAT_H */