From 9aacbfb6caab2517e31b3ce4eea45f92efeadf64 Mon Sep 17 00:00:00 2001 From: Zhao Zhili Date: Wed, 6 Nov 2024 20:31:28 +0800 Subject: [PATCH] avcodec/mediacodec_wrapper: add async mode support Signed-off-by: Zhao Zhili --- libavcodec/mediacodec_wrapper.c | 130 ++++++++++++++++++++++++++++++++ libavcodec/mediacodec_wrapper.h | 28 +++++++ 2 files changed, 158 insertions(+) diff --git a/libavcodec/mediacodec_wrapper.c b/libavcodec/mediacodec_wrapper.c index 96c886666a..283bbe72d6 100644 --- a/libavcodec/mediacodec_wrapper.c +++ b/libavcodec/mediacodec_wrapper.c @@ -1762,6 +1762,14 @@ static int mediacodec_jni_signalEndOfInputStream(FFAMediaCodec *ctx) return 0; } +static int mediacodec_jni_setAsyncNotifyCallback(FFAMediaCodec *codec, + const FFAMediaCodecOnAsyncNotifyCallback *callback, + void *userdata) +{ + av_log(codec, AV_LOG_ERROR, "Doesn't support aync mode with JNI, please try ndk_codec=1\n"); + return AVERROR(ENOSYS); +} + static const FFAMediaFormat media_format_jni = { .class = &amediaformat_class, @@ -1821,6 +1829,7 @@ static const FFAMediaCodec media_codec_jni = { .getConfigureFlagEncode = mediacodec_jni_getConfigureFlagEncode, .cleanOutputBuffers = mediacodec_jni_cleanOutputBuffers, .signalEndOfInputStream = mediacodec_jni_signalEndOfInputStream, + .setAsyncNotifyCallback = mediacodec_jni_setAsyncNotifyCallback, }; typedef struct FFAMediaFormatNdk { @@ -1842,6 +1851,9 @@ typedef struct FFAMediaCodecNdk { AMediaCodec *impl; ANativeWindow *window; + FFAMediaCodecOnAsyncNotifyCallback async_cb; + void *async_userdata; + // Available since API level 28. media_status_t (*getName)(AMediaCodec*, char** out_name); void (*releaseName)(AMediaCodec*, char* name); @@ -1849,6 +1861,8 @@ typedef struct FFAMediaCodecNdk { // Available since API level 26. media_status_t (*setInputSurface)(AMediaCodec*, ANativeWindow *); media_status_t (*signalEndOfInputStream)(AMediaCodec *); + media_status_t (*setAsyncNotifyCallback)(AMediaCodec *, + struct AMediaCodecOnAsyncNotifyCallback callback, void *userdata); } FFAMediaCodecNdk; static const FFAMediaFormat media_format_ndk; @@ -1866,6 +1880,32 @@ static const AVClass amediacodec_ndk_class = { .version = LIBAVUTIL_VERSION_INT, }; +static int media_status_to_error(media_status_t status) +{ + switch (status) { + case AMEDIA_OK: + return 0; + case AMEDIACODEC_ERROR_INSUFFICIENT_RESOURCE: + return AVERROR(ENOMEM); + case AMEDIA_ERROR_MALFORMED: + return AVERROR_INVALIDDATA; + case AMEDIA_ERROR_UNSUPPORTED: + return AVERROR(ENOTSUP); + case AMEDIA_ERROR_INVALID_PARAMETER: + return AVERROR(EINVAL); + case AMEDIA_ERROR_INVALID_OPERATION: + return AVERROR(EOPNOTSUPP); + case AMEDIA_ERROR_END_OF_STREAM: + return AVERROR_EOF; + case AMEDIA_ERROR_IO: + return AVERROR(EIO); + case AMEDIA_ERROR_WOULD_BLOCK: + return AVERROR(EWOULDBLOCK); + default: + return AVERROR_EXTERNAL; + } +} + static FFAMediaFormat *mediaformat_ndk_create(AMediaFormat *impl) { FFAMediaFormatNdk *format = av_mallocz(sizeof(*format)); @@ -2060,6 +2100,7 @@ static inline FFAMediaCodec *ndk_codec_create(int method, const char *arg) { GET_SYMBOL(setInputSurface) GET_SYMBOL(signalEndOfInputStream) + GET_SYMBOL(setAsyncNotifyCallback) #undef GET_SYMBOL @@ -2335,6 +2376,94 @@ static int mediacodec_ndk_signalEndOfInputStream(FFAMediaCodec *ctx) return 0; } +static void mediacodec_ndk_onInputAvailable(AMediaCodec *impl, void *userdata, + int32_t index) +{ + FFAMediaCodecNdk *codec = userdata; + codec->async_cb.onAsyncInputAvailable((FFAMediaCodec *) codec, + codec->async_userdata, index); +} + +static void mediacodec_ndk_onOutputAvailable(AMediaCodec *impl, + void *userdata, + int32_t index, + AMediaCodecBufferInfo *buffer_info) +{ + FFAMediaCodecNdk *codec = userdata; + FFAMediaCodecBufferInfo info = { + .offset = buffer_info->offset, + .size = buffer_info->size, + .presentationTimeUs = buffer_info->presentationTimeUs, + .flags = buffer_info->flags, + }; + + codec->async_cb.onAsyncOutputAvailable(&codec->api, codec->async_userdata, + index, &info); +} + +static void mediacodec_ndk_onFormatChanged(AMediaCodec *impl, void *userdata, + AMediaFormat *format) +{ + FFAMediaCodecNdk *codec = userdata; + FFAMediaFormat *media_format = mediaformat_ndk_create(format); + if (!media_format) + return; + + codec->async_cb.onAsyncFormatChanged(&codec->api, codec->async_userdata, + media_format); + ff_AMediaFormat_delete(media_format); +} + +static void mediacodec_ndk_onError(AMediaCodec *impl, void *userdata, + media_status_t status, + int32_t actionCode, + const char *detail) +{ + FFAMediaCodecNdk *codec = userdata; + int error = media_status_to_error(status); + + codec->async_cb.onAsyncError(&codec->api, codec->async_userdata, error, + detail); +} + +static int mediacodec_ndk_setAsyncNotifyCallback(FFAMediaCodec *ctx, + const FFAMediaCodecOnAsyncNotifyCallback *callback, + void *userdata) +{ + FFAMediaCodecNdk *codec = (FFAMediaCodecNdk *)ctx; + struct AMediaCodecOnAsyncNotifyCallback cb = { + .onAsyncInputAvailable = mediacodec_ndk_onInputAvailable, + .onAsyncOutputAvailable = mediacodec_ndk_onOutputAvailable, + .onAsyncFormatChanged = mediacodec_ndk_onFormatChanged, + .onAsyncError = mediacodec_ndk_onError, + }; + media_status_t status; + + if (!codec->setAsyncNotifyCallback) { + av_log(codec, AV_LOG_ERROR, "setAsyncNotifyCallback unavailable\n"); + return AVERROR(ENOSYS); + } + + if (!callback || + !callback->onAsyncInputAvailable || + !callback->onAsyncOutputAvailable || + !callback->onAsyncFormatChanged || + !callback->onAsyncError) + return AVERROR(EINVAL); + + codec->async_cb = *callback; + codec->async_userdata = userdata; + + status = codec->setAsyncNotifyCallback(codec->impl, cb, codec); + if (status != AMEDIA_OK) { + av_log(codec, AV_LOG_ERROR, "setAsyncNotifyCallback failed, %d\n", + status); + return AVERROR_EXTERNAL; + } + + return 0; +} + static const FFAMediaFormat media_format_ndk = { .class = &amediaformat_ndk_class, @@ -2396,6 +2525,7 @@ static const FFAMediaCodec media_codec_ndk = { .getConfigureFlagEncode = mediacodec_ndk_getConfigureFlagEncode, .cleanOutputBuffers = mediacodec_ndk_cleanOutputBuffers, .signalEndOfInputStream = mediacodec_ndk_signalEndOfInputStream, + .setAsyncNotifyCallback = mediacodec_ndk_setAsyncNotifyCallback, }; FFAMediaFormat *ff_AMediaFormat_new(int ndk) diff --git a/libavcodec/mediacodec_wrapper.h b/libavcodec/mediacodec_wrapper.h index 11a4260497..18d0796445 100644 --- a/libavcodec/mediacodec_wrapper.h +++ b/libavcodec/mediacodec_wrapper.h @@ -178,6 +178,22 @@ struct FFAMediaCodecBufferInfo { typedef struct FFAMediaCodecBufferInfo FFAMediaCodecBufferInfo; typedef struct FFAMediaCodec FFAMediaCodec; + +typedef struct FFAMediaCodecOnAsyncNotifyCallback { + void (*onAsyncInputAvailable)(FFAMediaCodec *codec, void *userdata, + int32_t index); + + void (*onAsyncOutputAvailable)(FFAMediaCodec *codec, void *userdata, + int32_t index, + FFAMediaCodecBufferInfo *buffer_info); + + void (*onAsyncFormatChanged)(FFAMediaCodec *codec, void *userdata, + FFAMediaFormat *format); + + void (*onAsyncError)(FFAMediaCodec *codec, void *userdata, int error, + const char *detail); +} FFAMediaCodecOnAsyncNotifyCallback; + struct FFAMediaCodec { const AVClass *class; @@ -219,6 +235,11 @@ struct FFAMediaCodec { // For encoder with FFANativeWindow as input. int (*signalEndOfInputStream)(FFAMediaCodec *); + + // Introduced in Android API 28 + int (*setAsyncNotifyCallback)(FFAMediaCodec *codec, + const FFAMediaCodecOnAsyncNotifyCallback *callback, + void *userdata); }; static inline char *ff_AMediaCodec_getName(FFAMediaCodec *codec) @@ -343,6 +364,13 @@ static inline int ff_AMediaCodec_signalEndOfInputStream(FFAMediaCodec *codec) return codec->signalEndOfInputStream(codec); } +static inline int ff_AMediaCodec_setAsyncNotifyCallback(FFAMediaCodec *codec, + const FFAMediaCodecOnAsyncNotifyCallback *callback, + void *userdata) +{ + return codec->setAsyncNotifyCallback(codec, callback, userdata); +} + int ff_Build_SDK_INT(AVCodecContext *avctx); enum FFAMediaFormatColorRange {