ao_coreaudio: insane hack for passing through AC3 as float PCM

This uses the same hack as Kodi uses, and I suspect MPlayer/ancient mpv
also did this (but didn't research that).

DOCS/man/ao.rst

@@ -95,6 +95,15 @@ Available audio output drivers are:
         setting in the ``Audio Devices`` dialog in the ``Audio MIDI Setup``
         utility. Note that this does not affect the selected speaker setup.
 
+    ``--coreaudio-spdif-hack=<yes|no>``
+        Try to pass through AC3/DTS data as PCM. This is useful for drivers
+        which do not report AC3 support. It converts the AC3 data to float,
+        and assumes the driver will do the inverse conversion, which means
+        a typical A/V receiver will pick it up as compressed IEC framed AC3
+        stream, ignoring that it's marked as PCM. This disables normal AC3
+        passthrough (even if the device reports it as supported). Use with
+        extreme care.
+
 
 ``coreaudio_exclusive`` (Mac OS X only)
     Native Mac OS X audio output driver using direct device access and

audio/out/ao_coreaudio_exclusive.c

@@ -36,6 +36,9 @@
 
 #include <CoreAudio/HostTime.h>
 
+#include <libavutil/intreadwrite.h>
+#include <libavutil/intfloat.h>
+
 #include "config.h"
 #include "ao.h"
 #include "internal.h"
@@ -69,6 +72,9 @@ struct priv {
     AudioStreamBasicDescription stream_asbd;
     AudioStreamBasicDescription original_asbd;
 
+    // Output s16 physical format, float32 virtual format, ac3/dts mpv format
+    int spdif_hack;
+
     bool changed_mixing;
 
     atomic_bool reload_requested;
@@ -134,6 +140,24 @@ static OSStatus enable_property_listener(struct ao *ao, bool enabled)
     return status;
 }
 
+// This is a hack for passing through AC3/DTS on drivers which don't support it.
+// The goal is to have the driver output the AC3 data bitexact, so basically we
+// feed it float data by converting the AC3 data to float in the reverse way we
+// assume the driver outputs it.
+// Input: data_as_int16[0..samples]
+// Output: data_as_float[0..samples]
+// The conversion is done in-place.
+static void bad_hack_mygodwhy(char *data, int samples)
+{
+    // In reverse, so we can do it in-place.
+    for (int n = samples - 1; n >= 0; n--) {
+        int16_t val = AV_RN16(data + n * 2);
+        float fval = val / (float)(1 << 15);
+        uint32_t ival = av_float2int(fval);
+        AV_WN32(data + n * 4, ival);
+    }
+}
+
 static OSStatus render_cb_compressed(
         AudioDeviceID device, const AudioTimeStamp *ts,
         const void *in_data, const AudioTimeStamp *in_ts,
@@ -143,11 +167,12 @@ static OSStatus render_cb_compressed(
     struct priv *p   = ao->priv;
     AudioBuffer buf  = out_data->mBuffers[p->stream_idx];
     int requested    = buf.mDataByteSize;
+    int sstride      = p->spdif_hack ? 4 * ao->channels.num : ao->sstride;
 
-    int pseudo_frames = requested / ao->sstride;
+    int pseudo_frames = requested / sstride;
 
     // we expect the callback to read full frames, which are aligned accordingly
-    if (pseudo_frames * ao->sstride != requested) {
+    if (pseudo_frames * sstride != requested) {
         MP_ERR(ao, "Unsupported unaligned read of %d bytes.\n", requested);
         return kAudioHardwareUnspecifiedError;
     }
@@ -158,6 +183,9 @@ static OSStatus render_cb_compressed(
 
     ao_read_data(ao, &buf.mData, pseudo_frames, end);
 
+    if (p->spdif_hack)
+        bad_hack_mygodwhy(buf.mData, pseudo_frames * ao->channels.num);
+
     return noErr;
 }
 
@@ -187,8 +215,8 @@ static int select_stream(struct ao *ao)
             continue;
         }
 
-        if (af_fmt_is_pcm(ao->format) || ca_stream_supports_compressed(ao,
-                                                                   streams[i]))
+        if (af_fmt_is_pcm(ao->format) || p->spdif_hack ||
+            ca_stream_supports_compressed(ao, streams[i]))
         {
             MP_VERBOSE(ao, "Using substream %d/%zd.\n", i, n_streams);
             p->stream = streams[i];
@@ -253,6 +281,7 @@ coreaudio_error:
 static int init(struct ao *ao)
 {
     struct priv *p = ao->priv;
+    int original_format = ao->format;
 
     OSStatus err = ca_select_device(ao, ao->device, &p->device);
     CHECK_CA_ERROR_L(coreaudio_error_nounlock, "failed to select device");
@@ -264,12 +293,24 @@ static int init(struct ao *ao)
         goto coreaudio_error_nounlock;
     }
 
+    if (af_fmt_is_pcm(ao->format))
+        p->spdif_hack = false;
+
+    if (p->spdif_hack) {
+        if (af_fmt_to_bytes(ao->format) != 2) {
+            MP_ERR(ao, "HD formats not supported with spdif hack.\n");
+            goto coreaudio_error_nounlock;
+        }
+        // Let the pure evil begin!
+        ao->format = AF_FORMAT_S16;
+    }
+
     uint32_t is_alive = 1;
     err = CA_GET(p->device, kAudioDevicePropertyDeviceIsAlive, &is_alive);
     CHECK_CA_WARN("could not check whether device is alive");
 
     if (!is_alive)
-        MP_WARN(ao , "device is not alive\n");
+        MP_WARN(ao, "device is not alive\n");
 
     err = ca_lock_device(p->device, &p->hog_pid);
     CHECK_CA_WARN("failed to set hogmode");
@@ -329,6 +370,20 @@ static int init(struct ao *ao)
         goto coreaudio_error;
     }
 
+    if (p->spdif_hack) {
+        AudioStreamBasicDescription physical_format = {0};
+        err = CA_GET(p->stream, kAudioStreamPropertyPhysicalFormat,
+                     &physical_format);
+        CHECK_CA_ERROR("could not get stream's physical format");
+        int ph_format = ca_asbd_to_mp_format(&physical_format);
+        if (ao->format != AF_FORMAT_FLOAT || ph_format != AF_FORMAT_S16) {
+            MP_ERR(ao, "Wrong parameters for spdif hack (%d / %d)\n",
+                   ao->format, ph_format);
+        }
+        ao->format = original_format; // pretend AC3 or DTS *evil laughter*
+        MP_WARN(ao, "Using spdif passthrough hack. This could produce noise.\n");
+    }
+
     p->hw_latency_us = ca_get_device_latency_us(ao, p->device);
     MP_VERBOSE(ao, "base latency: %d microseconds\n", (int)p->hw_latency_us);
 
@@ -409,4 +464,9 @@ const struct ao_driver audio_out_coreaudio_exclusive = {
         .stream_idx = -1,
         .changed_mixing = false,
     },
+    .options = (const struct m_option[]){
+        OPT_FLAG("spdif-hack", spdif_hack, 0),
+        {0}
+    },
+    .options_prefix = "coreaudio",
 };