audio/out: add helper code to do 24 bit conversion in AO

I plan to remove the S24 sample formats in mpv. It seems like we should
still support this _somehow_ in AOs though. So the idea is to convert
the data to more obscure representations (that would not be useful for
filtering etc. anyway) within the AO.

This commit adds helper to enable this. ao_convert_fmt is meant to
provide mechanisms for this, rather than a generic audio format
description (as the latter leads only to overly generic misery). The
conversion also supports only cases which we think will be needed at
all.

The main advantage of this approach is that we get S24 out of sight,
and that we could support other crazy formats (like S20). The main
disadvantage is that usually S32 will be selected (if both S32 and S24
are available), and there's no user control to force S24. That doesn't
really matter though, and at worst makes testing harder or will lead
to unpleasant arguments with audiophiles (they'd be wrong anyway).

ao_convert_fmt.pad_lsb is ignored, although if we ever find a case in
which playing S32 with data in the LSBs breaks when playing it as padded
24 bit format. (For example, WAVEFORMATEXTENSIBLE recommends setting the
unused bits to 0 if wValidBitsPerSample implies LSB padding.)

audio/out/ao.c

@@ -30,6 +30,7 @@
 
 #include "options/options.h"
 #include "options/m_config.h"
+#include "osdep/endian.h"
 #include "common/msg.h"
 #include "common/common.h"
 #include "common/global.h"
@@ -644,3 +645,71 @@ void ao_print_devices(struct mpv_global *global, struct mp_log *log)
     }
     ao_hotplug_destroy(hp);
 }
+
+static int get_conv_type(struct ao_convert_fmt *fmt)
+{
+    if (af_fmt_to_bytes(fmt->src_fmt) * 8 == fmt->dst_bits && !fmt->pad_msb)
+        return 0; // passthrough
+    if (fmt->src_fmt == AF_FORMAT_S32 && fmt->dst_bits == 24 && !fmt->pad_msb)
+        return 1; // simple 32->24 bit conversion
+    if (fmt->src_fmt == AF_FORMAT_S32 && fmt->dst_bits == 32 && fmt->pad_msb == 8)
+        return 2; // simple 32->24 bit conversion, with MSB padding
+    return -1; // unsupported
+}
+
+// Check whether ao_convert_inplace() can be called. As an exception, the
+// planar-ness of the sample format and the number of channels is ignored.
+// All other parameters must be as passed to ao_convert_inplace().
+bool ao_can_convert_inplace(struct ao_convert_fmt *fmt)
+{
+    return get_conv_type(fmt) >= 0;
+}
+
+bool ao_need_conversion(struct ao_convert_fmt *fmt)
+{
+    return get_conv_type(fmt) != 0;
+}
+
+// The LSB is always ignored.
+#if BYTE_ORDER == BIG_ENDIAN
+#define SHIFT24(x) ((3-(x))*8)
+#else
+#define SHIFT24(x) (((x)+1)*8)
+#endif
+
+static void convert_plane(int type, void *data, int num_samples)
+{
+    switch (type) {
+    case 0:
+        break;
+    case 1: /* fall through */
+    case 2: {
+        int bytes = type == 1 ? 3 : 4;
+        for (int s = 0; s < num_samples; s++) {
+            uint32_t val = *((uint32_t *)data + s);
+            uint8_t *ptr = (uint8_t *)data + s * bytes;
+            ptr[0] = val >> SHIFT24(0);
+            ptr[1] = val >> SHIFT24(1);
+            ptr[2] = val >> SHIFT24(2);
+            if (type == 2)
+                ptr[3] = 0;
+        }
+        break;
+    }
+    default:
+        abort();
+    }
+}
+
+// data[n] contains the pointer to the first sample of the n-th plane, in the
+// format implied by fmt->src_fmt. src_fmt also controls whether the data is
+// all in one plane, or of there is a plane per channel.
+void ao_convert_inplace(struct ao_convert_fmt *fmt, void **data, int num_samples)
+{
+    int type = get_conv_type(fmt);
+    bool planar = af_fmt_is_planar(fmt->src_fmt);
+    int planes = planar ? fmt->channels : 1;
+    int plane_samples = num_samples * (planar ? 1: fmt->channels);
+    for (int n = 0; n < planes; n++)
+        convert_plane(type, data[n], plane_samples);
+}

audio/out/internal.h

@@ -215,4 +215,19 @@ bool ao_chmap_sel_get_def(struct ao *ao, const struct mp_chmap_sel *s,
 void ao_device_list_add(struct ao_device_list *list, struct ao *ao,
                         struct ao_device_desc *e);
 
+struct ao_convert_fmt {
+    int src_fmt;        // source AF_FORMAT_*
+    int channels;       // number of channels
+    int dst_bits;       // total target data sample size
+    int pad_msb;        // padding in the MSB (i.e. required shifting)
+    int pad_lsb;        // padding in LSB (required 0 bits) (ignored)
+};
+
+bool ao_can_convert_inplace(struct ao_convert_fmt *fmt);
+bool ao_need_conversion(struct ao_convert_fmt *fmt);
+void ao_convert_inplace(struct ao_convert_fmt *fmt, void **data, int num_samples);
+
+int ao_read_data_converted(struct ao *ao, struct ao_convert_fmt *fmt,
+                           void **data, int samples, int64_t out_time_us);
+
 #endif

audio/out/pull.c

@@ -64,6 +64,8 @@ struct ao_pull_state {
 
     // Device delay of the last written sample, in realtime.
     atomic_llong end_time_us;
+
+    char *convert_buffer;
 };
 
 static void set_state(struct ao *ao, int new_state)
@@ -180,6 +182,45 @@ end:
     return bytes / ao->sstride;
 }
 
+// Same as ao_read_data(), but read pre-converted data according to *fmt.
+// fmt->src_fmt and fmt->channels must be the same as the AO parameters.
+int ao_read_data_converted(struct ao *ao, struct ao_convert_fmt *fmt,
+                           void **data, int samples, int64_t out_time_us)
+{
+    assert(ao->api == &ao_api_pull);
+
+    struct ao_pull_state *p = ao->api_priv;
+    void *ndata[MP_NUM_CHANNELS];
+
+    if (!ao_need_conversion(fmt))
+        return ao_read_data(ao, data, samples, out_time_us);
+
+    assert(ao->format == fmt->src_fmt);
+    assert(ao->channels.num == fmt->channels);
+
+    bool planar = af_fmt_is_planar(fmt->src_fmt);
+    int planes = planar ? fmt->channels : 1;
+    int plane_size = af_fmt_to_bytes(fmt->src_fmt) * samples *
+                     (planar ? 1: fmt->channels);
+
+    int needed = plane_size * planes * fmt->channels * samples;
+    if (needed > talloc_get_size(p->convert_buffer) || !p->convert_buffer) {
+        talloc_free(p->convert_buffer);
+        p->convert_buffer = talloc_size(NULL, needed);
+    }
+
+    for (int n = 0; n < planes; n++)
+        ndata[n] = p->convert_buffer + n * plane_size;
+
+    int res = ao_read_data(ao, ndata, samples, out_time_us);
+
+    ao_convert_inplace(fmt, ndata, samples);
+    for (int n = 0; n < planes; n++)
+        memcpy(data[n], ndata[n], plane_size);
+
+    return res;
+}
+
 static int control(struct ao *ao, enum aocontrol cmd, void *arg)
 {
     if (ao->driver->control)
@@ -256,7 +297,11 @@ static void drain(struct ao *ao)
 
 static void uninit(struct ao *ao)
 {
+    struct ao_pull_state *p = ao->api_priv;
+
     ao->driver->uninit(ao);
+
+    talloc_free(p->convert_buffer);
 }
 
 static int init(struct ao *ao)