tdesktop/Telegram/SourceFiles/media/audio/media_audio_ffmpeg_loader.cpp

654 lines
17 KiB
C++

/*
This file is part of Telegram Desktop,
the official desktop application for the Telegram messaging service.
For license and copyright information please follow this link:
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#include "media/audio/media_audio_ffmpeg_loader.h"
#include "core/file_location.h"
#include "ffmpeg/ffmpeg_utility.h"
#include "base/bytes.h"
namespace Media {
uint64_t AbstractFFMpegLoader::ComputeChannelLayout(
uint64_t channel_layout,
int channels) {
if (channel_layout) {
if (av_get_channel_layout_nb_channels(channel_layout) == channels) {
return channel_layout;
}
}
return av_get_default_channel_layout(channels);
}
int64 AbstractFFMpegLoader::Mul(int64 value, AVRational rational) {
return value * rational.num / rational.den;
}
bool AbstractFFMpegLoader::open(crl::time positionMs) {
if (!AudioPlayerLoader::openFile()) {
return false;
}
int res = 0;
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
ioBuffer = (uchar *)av_malloc(FFmpeg::kAVBlockSize);
if (!_data.isEmpty()) {
ioContext = avio_alloc_context(ioBuffer, FFmpeg::kAVBlockSize, 0, reinterpret_cast<void *>(this), &AbstractFFMpegLoader::ReadData, 0, &AbstractFFMpegLoader::SeekData);
} else if (!_bytes.empty()) {
ioContext = avio_alloc_context(ioBuffer, FFmpeg::kAVBlockSize, 0, reinterpret_cast<void *>(this), &AbstractFFMpegLoader::ReadBytes, 0, &AbstractFFMpegLoader::SeekBytes);
} else {
ioContext = avio_alloc_context(ioBuffer, FFmpeg::kAVBlockSize, 0, reinterpret_cast<void *>(this), &AbstractFFMpegLoader::ReadFile, 0, &AbstractFFMpegLoader::SeekFile);
}
fmtContext = avformat_alloc_context();
if (!fmtContext) {
DEBUG_LOG(("Audio Read Error: Unable to avformat_alloc_context for file '%1', data size '%2'").arg(_file.name()).arg(_data.size()));
return false;
}
fmtContext->pb = ioContext;
if ((res = avformat_open_input(&fmtContext, 0, 0, 0)) < 0) {
ioBuffer = 0;
DEBUG_LOG(("Audio Read Error: Unable to avformat_open_input for file '%1', data size '%2', error %3, %4").arg(_file.name()).arg(_data.size()).arg(res).arg(av_make_error_string(err, sizeof(err), res)));
return false;
}
_opened = true;
if ((res = avformat_find_stream_info(fmtContext, 0)) < 0) {
DEBUG_LOG(("Audio Read Error: Unable to avformat_find_stream_info for file '%1', data size '%2', error %3, %4").arg(_file.name()).arg(_data.size()).arg(res).arg(av_make_error_string(err, sizeof(err), res)));
return false;
}
streamId = av_find_best_stream(fmtContext, AVMEDIA_TYPE_AUDIO, -1, -1, &codec, 0);
if (streamId < 0) {
LOG(("Audio Error: Unable to av_find_best_stream for file '%1', data size '%2', error %3, %4").arg(_file.name()).arg(_data.size()).arg(streamId).arg(av_make_error_string(err, sizeof(err), streamId)));
return false;
}
const auto stream = fmtContext->streams[streamId];
const auto params = stream->codecpar;
_samplesFrequency = params->sample_rate;
if (stream->duration != AV_NOPTS_VALUE) {
_samplesCount = Mul(
stream->duration * _samplesFrequency,
stream->time_base);
} else {
_samplesCount = Mul(
fmtContext->duration * _samplesFrequency,
{ 1, AV_TIME_BASE });
}
return true;
}
AbstractFFMpegLoader::~AbstractFFMpegLoader() {
if (_opened) {
avformat_close_input(&fmtContext);
}
if (ioContext) {
av_freep(&ioContext->buffer);
av_freep(&ioContext);
} else if (ioBuffer) {
av_freep(&ioBuffer);
}
if (fmtContext) avformat_free_context(fmtContext);
}
int AbstractFFMpegLoader::ReadData(void *opaque, uint8_t *buf, int buf_size) {
auto l = reinterpret_cast<AbstractFFMpegLoader *>(opaque);
auto nbytes = qMin(l->_data.size() - l->_dataPos, int32(buf_size));
if (nbytes <= 0) {
return AVERROR_EOF;
}
memcpy(buf, l->_data.constData() + l->_dataPos, nbytes);
l->_dataPos += nbytes;
return nbytes;
}
int64_t AbstractFFMpegLoader::SeekData(void *opaque, int64_t offset, int whence) {
auto l = reinterpret_cast<AbstractFFMpegLoader *>(opaque);
int32 newPos = -1;
switch (whence) {
case SEEK_SET: newPos = offset; break;
case SEEK_CUR: newPos = l->_dataPos + offset; break;
case SEEK_END: newPos = l->_data.size() + offset; break;
case AVSEEK_SIZE: {
// Special whence for determining filesize without any seek.
return l->_data.size();
} break;
}
if (newPos < 0 || newPos > l->_data.size()) {
return -1;
}
l->_dataPos = newPos;
return l->_dataPos;
}
int AbstractFFMpegLoader::ReadBytes(void *opaque, uint8_t *buf, int buf_size) {
auto l = reinterpret_cast<AbstractFFMpegLoader *>(opaque);
auto nbytes = qMin(static_cast<int>(l->_bytes.size()) - l->_dataPos, buf_size);
if (nbytes <= 0) {
return AVERROR_EOF;
}
memcpy(buf, l->_bytes.data() + l->_dataPos, nbytes);
l->_dataPos += nbytes;
return nbytes;
}
int64_t AbstractFFMpegLoader::SeekBytes(void *opaque, int64_t offset, int whence) {
auto l = reinterpret_cast<AbstractFFMpegLoader *>(opaque);
int32 newPos = -1;
switch (whence) {
case SEEK_SET: newPos = offset; break;
case SEEK_CUR: newPos = l->_dataPos + offset; break;
case SEEK_END: newPos = static_cast<int>(l->_bytes.size()) + offset; break;
case AVSEEK_SIZE:
{
// Special whence for determining filesize without any seek.
return l->_bytes.size();
} break;
}
if (newPos < 0 || newPos > l->_bytes.size()) {
return -1;
}
l->_dataPos = newPos;
return l->_dataPos;
}
int AbstractFFMpegLoader::ReadFile(void *opaque, uint8_t *buf, int buf_size) {
auto l = reinterpret_cast<AbstractFFMpegLoader *>(opaque);
int ret = l->_f.read((char *)(buf), buf_size);
switch (ret) {
case -1: return AVERROR_EXTERNAL;
case 0: return AVERROR_EOF;
default: return ret;
}
}
int64_t AbstractFFMpegLoader::SeekFile(void *opaque, int64_t offset, int whence) {
auto l = reinterpret_cast<AbstractFFMpegLoader *>(opaque);
switch (whence) {
case SEEK_SET: return l->_f.seek(offset) ? l->_f.pos() : -1;
case SEEK_CUR: return l->_f.seek(l->_f.pos() + offset) ? l->_f.pos() : -1;
case SEEK_END: return l->_f.seek(l->_f.size() + offset) ? l->_f.pos() : -1;
case AVSEEK_SIZE:
{
// Special whence for determining filesize without any seek.
return l->_f.size();
} break;
}
return -1;
}
AbstractAudioFFMpegLoader::AbstractAudioFFMpegLoader(
const Core::FileLocation &file,
const QByteArray &data,
bytes::vector &&buffer)
: AbstractFFMpegLoader(file, data, std::move(buffer))
, _frame(FFmpeg::MakeFramePointer()) {
}
bool AbstractAudioFFMpegLoader::initUsingContext(
not_null<AVCodecContext*> context,
int64 initialCount,
int initialFrequency) {
const auto layout = ComputeChannelLayout(
context->channel_layout,
context->channels);
if (!layout) {
LOG(("Audio Error: Unknown channel layout %1 for %2 channels."
).arg(context->channel_layout
).arg(context->channels
));
return false;
}
_swrSrcSampleFormat = context->sample_fmt;
switch (layout) {
case AV_CH_LAYOUT_MONO:
switch (_swrSrcSampleFormat) {
case AV_SAMPLE_FMT_U8:
case AV_SAMPLE_FMT_U8P:
_swrDstSampleFormat = _swrSrcSampleFormat;
_swrDstChannelLayout = layout;
_outputChannels = 1;
_outputSampleSize = 1;
_outputFormat = AL_FORMAT_MONO8;
break;
case AV_SAMPLE_FMT_S16:
case AV_SAMPLE_FMT_S16P:
_swrDstSampleFormat = _swrSrcSampleFormat;
_swrDstChannelLayout = layout;
_outputChannels = 1;
_outputSampleSize = sizeof(uint16);
_outputFormat = AL_FORMAT_MONO16;
break;
}
break;
case AV_CH_LAYOUT_STEREO:
switch (_swrSrcSampleFormat) {
case AV_SAMPLE_FMT_U8:
_swrDstSampleFormat = _swrSrcSampleFormat;
_swrDstChannelLayout = layout;
_outputChannels = 2;
_outputSampleSize = 2;
_outputFormat = AL_FORMAT_STEREO8;
break;
case AV_SAMPLE_FMT_S16:
_swrDstSampleFormat = _swrSrcSampleFormat;
_swrDstChannelLayout = layout;
_outputChannels = 2;
_outputSampleSize = 2 * sizeof(uint16);
_outputFormat = AL_FORMAT_STEREO16;
break;
}
break;
}
if (_swrDstRate == initialFrequency) {
_outputSamplesCount = initialCount;
} else {
_outputSamplesCount = av_rescale_rnd(
initialCount,
_swrDstRate,
initialFrequency,
AV_ROUND_UP);
}
return true;
}
auto AbstractAudioFFMpegLoader::replaceFrameAndRead(
FFmpeg::FramePointer frame,
QByteArray &result,
int64 &samplesAdded)
-> ReadResult {
_frame = std::move(frame);
return readFromReadyFrame(result, samplesAdded);
}
auto AbstractAudioFFMpegLoader::readFromReadyContext(
not_null<AVCodecContext *> context,
QByteArray &result,
int64 &samplesAdded)
-> ReadResult {
const auto res = avcodec_receive_frame(context, _frame.get());
if (res >= 0) {
return readFromReadyFrame(result, samplesAdded);
}
if (res == AVERROR_EOF) {
return ReadResult::EndOfFile;
} else if (res != AVERROR(EAGAIN)) {
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
LOG(("Audio Error: "
"Unable to avcodec_receive_frame() file '%1', data size '%2', "
"error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)
));
return ReadResult::Error;
}
return ReadResult::Wait;
}
bool AbstractAudioFFMpegLoader::frameHasDesiredFormat() const {
const auto frameChannelLayout = ComputeChannelLayout(
_frame->channel_layout,
_frame->channels);
return true
&& (_frame->format == _swrDstSampleFormat)
&& (frameChannelLayout == _swrDstChannelLayout)
&& (_frame->sample_rate == _swrDstRate);
}
bool AbstractAudioFFMpegLoader::initResampleForFrame() {
const auto frameChannelLayout = ComputeChannelLayout(
_frame->channel_layout,
_frame->channels);
if (!frameChannelLayout) {
LOG(("Audio Error: "
"Unable to compute channel layout for frame in file '%1', "
"data size '%2', channel_layout %3, channels %4"
).arg(_file.name()
).arg(_data.size()
).arg(_frame->channel_layout
).arg(_frame->channels
));
return false;
} else if (_frame->format == -1) {
LOG(("Audio Error: "
"Unknown frame format in file '%1', data size '%2'"
).arg(_file.name()
).arg(_data.size()
));
return false;
} else if (_swrContext) {
if (true
&& (_frame->format == _swrSrcSampleFormat)
&& (frameChannelLayout == _swrSrcChannelLayout)
&& (_frame->sample_rate == _swrSrcRate)) {
return true;
}
swr_close(_swrContext);
}
_swrSrcSampleFormat = static_cast<AVSampleFormat>(_frame->format);
_swrSrcChannelLayout = frameChannelLayout;
_swrSrcRate = _frame->sample_rate;
return initResampleUsingFormat();
}
bool AbstractAudioFFMpegLoader::initResampleUsingFormat() {
int res = 0;
_swrContext = swr_alloc_set_opts(
_swrContext,
_swrDstChannelLayout,
_swrDstSampleFormat,
_swrDstRate,
_swrSrcChannelLayout,
_swrSrcSampleFormat,
_swrSrcRate,
0,
nullptr);
if (!_swrContext) {
LOG(("Audio Error: "
"Unable to swr_alloc for file '%1', data size '%2'"
).arg(_file.name()
).arg(_data.size()));
return false;
} else if ((res = swr_init(_swrContext)) < 0) {
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
LOG(("Audio Error: "
"Unable to swr_init for file '%1', data size '%2', "
"error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)
));
return false;
}
if (_swrDstData) {
av_freep(&_swrDstData[0]);
_swrDstDataCapacity = -1;
}
return true;
}
bool AbstractAudioFFMpegLoader::ensureResampleSpaceAvailable(int samples) {
if (_swrDstData != nullptr && _swrDstDataCapacity >= samples) {
return true;
}
const auto allocate = std::max(samples, int(av_rescale_rnd(
FFmpeg::kAVBlockSize / _outputSampleSize,
_swrDstRate,
_swrSrcRate,
AV_ROUND_UP)));
if (_swrDstData) {
av_freep(&_swrDstData[0]);
}
const auto res = _swrDstData
? av_samples_alloc(
_swrDstData,
nullptr,
_outputChannels,
allocate,
_swrDstSampleFormat,
0)
: av_samples_alloc_array_and_samples(
&_swrDstData,
nullptr,
_outputChannels,
allocate,
_swrDstSampleFormat,
0);
if (res < 0) {
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
LOG(("Audio Error: "
"Unable to av_samples_alloc for file '%1', data size '%2', "
"error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)
));
return false;
}
_swrDstDataCapacity = allocate;
return true;
}
void AbstractAudioFFMpegLoader::appendSamples(
QByteArray & result,
int64 & samplesAdded,
uint8_t * *data,
int count) const {
result.append(
reinterpret_cast<const char *>(data[0]),
count * _outputSampleSize);
samplesAdded += count;
}
AudioPlayerLoader::ReadResult AbstractAudioFFMpegLoader::readFromReadyFrame(
QByteArray & result,
int64 & samplesAdded) {
if (frameHasDesiredFormat()) {
appendSamples(
result,
samplesAdded,
_frame->extended_data,
_frame->nb_samples);
return ReadResult::Ok;
} else if (!initResampleForFrame()) {
return ReadResult::Error;
}
const auto maxSamples = av_rescale_rnd(
swr_get_delay(_swrContext, _swrSrcRate) + _frame->nb_samples,
_swrDstRate,
_swrSrcRate,
AV_ROUND_UP);
if (!ensureResampleSpaceAvailable(maxSamples)) {
return ReadResult::Error;
}
const auto samples = swr_convert(
_swrContext,
_swrDstData,
maxSamples,
(const uint8_t * *)_frame->extended_data,
_frame->nb_samples);
if (samples < 0) {
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
LOG(("Audio Error: "
"Unable to swr_convert for file '%1', data size '%2', "
"error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(samples
).arg(av_make_error_string(err, sizeof(err), samples)
));
return ReadResult::Error;
}
appendSamples(
result,
samplesAdded,
_swrDstData,
samples);
return ReadResult::Ok;
}
AbstractAudioFFMpegLoader::~AbstractAudioFFMpegLoader() {
if (_swrContext) {
swr_free(&_swrContext);
}
if (_swrDstData) {
if (_swrDstData[0]) {
av_freep(&_swrDstData[0]);
}
av_freep(&_swrDstData);
}
}
FFMpegLoader::FFMpegLoader(
const Core::FileLocation & file,
const QByteArray & data,
bytes::vector && buffer)
: AbstractAudioFFMpegLoader(file, data, std::move(buffer)) {
}
bool FFMpegLoader::open(crl::time positionMs) {
if (!AbstractFFMpegLoader::open(positionMs)) {
return false;
}
if (!openCodecContext()) {
return false;
}
if (!initUsingContext(_codecContext, _samplesCount, _samplesFrequency)) {
return false;
}
return seekTo(positionMs);
}
bool FFMpegLoader::openCodecContext() {
int res = 0;
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
_codecContext = avcodec_alloc_context3(nullptr);
if (!_codecContext) {
LOG(("Audio Error: "
"Unable to avcodec_alloc_context3 for file '%1', data size '%2'"
).arg(_file.name()
).arg(_data.size()
));
return false;
}
const auto stream = fmtContext->streams[streamId];
if ((res = avcodec_parameters_to_context(
_codecContext,
stream->codecpar)) < 0) {
LOG(("Audio Error: "
"Unable to avcodec_parameters_to_context for file '%1', "
"data size '%2', error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)
));
return false;
}
_codecContext->pkt_timebase = stream->time_base;
av_opt_set_int(_codecContext, "refcounted_frames", 1, 0);
if ((res = avcodec_open2(_codecContext, codec, 0)) < 0) {
LOG(("Audio Error: "
"Unable to avcodec_open2 for file '%1', data size '%2', "
"error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)
));
return false;
}
return true;
}
bool FFMpegLoader::seekTo(crl::time positionMs) {
if (positionMs) {
const auto stream = fmtContext->streams[streamId];
const auto timeBase = stream->time_base;
const auto timeStamp = (positionMs * timeBase.den)
/ (1000LL * timeBase.num);
const auto flags1 = AVSEEK_FLAG_ANY;
if (av_seek_frame(fmtContext, streamId, timeStamp, flags1) < 0) {
const auto flags2 = 0;
if (av_seek_frame(fmtContext, streamId, timeStamp, flags2) < 0) {
}
}
}
return true;
}
AudioPlayerLoader::ReadResult FFMpegLoader::readMore(
QByteArray &result,
int64 &samplesAdded) {
const auto readResult = readFromReadyContext(
_codecContext,
result,
samplesAdded);
if (readResult != ReadResult::Wait) {
return readResult;
}
auto res = 0;
if ((res = av_read_frame(fmtContext, &_packet)) < 0) {
if (res != AVERROR_EOF) {
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
LOG(("Audio Error: "
"Unable to av_read_frame() file '%1', data size '%2', "
"error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)
));
return ReadResult::Error;
}
avcodec_send_packet(_codecContext, nullptr); // drain
return ReadResult::Ok;
}
if (_packet.stream_index == streamId) {
res = avcodec_send_packet(_codecContext, &_packet);
if (res < 0) {
av_packet_unref(&_packet);
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
LOG(("Audio Error: "
"Unable to avcodec_send_packet() file '%1', data size '%2', "
"error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)
));
// There is a sample voice message where skipping such packet
// results in a crash (read_access to nullptr) in swr_convert().
//if (res == AVERROR_INVALIDDATA) {
// return ReadResult::NotYet; // try to skip bad packet
//}
return ReadResult::Error;
}
}
av_packet_unref(&_packet);
return ReadResult::Ok;
}
FFMpegLoader::~FFMpegLoader() {
if (_codecContext) {
avcodec_free_context(&_codecContext);
}
}
} // namespace Media