/* This file is part of Telegram Desktop, the official desktop application for the Telegram messaging service. For license and copyright information please follow this link: https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL */ #include "media/streaming/media_streaming_utility.h" #include "media/streaming/media_streaming_common.h" #include "ui/image/image_prepare.h" extern "C" { #include } // extern "C" namespace Media { namespace Streaming { namespace { constexpr auto kSkipInvalidDataPackets = 10; constexpr auto kAlignImageBy = 16; constexpr auto kPixelBytesSize = 4; constexpr auto kImageFormat = QImage::Format_ARGB32_Premultiplied; constexpr auto kAvioBlockSize = 4096; constexpr auto kMaxScaleByAspectRatio = 16; void AlignedImageBufferCleanupHandler(void* data) { const auto buffer = static_cast(data); delete[] buffer; } [[nodiscard]] bool IsAlignedImage(const QImage &image) { return !(reinterpret_cast(image.bits()) % kAlignImageBy) && !(image.bytesPerLine() % kAlignImageBy); } [[nodiscard]] bool IsValidAspectRatio(AVRational aspect) { return (aspect.num > 0) && (aspect.den > 0) && (aspect.num <= aspect.den * kMaxScaleByAspectRatio) && (aspect.den <= aspect.num * kMaxScaleByAspectRatio); } } // namespace bool GoodStorageForFrame(const QImage &storage, QSize size) { return !storage.isNull() && (storage.format() == kImageFormat) && (storage.size() == size) && storage.isDetached() && IsAlignedImage(storage); } // Create a QImage of desired size where all the data is properly aligned. QImage CreateFrameStorage(QSize size) { const auto width = size.width(); const auto height = size.height(); const auto widthAlign = kAlignImageBy / kPixelBytesSize; const auto neededWidth = width + ((width % widthAlign) ? (widthAlign - (width % widthAlign)) : 0); const auto perLine = neededWidth * kPixelBytesSize; const auto buffer = new uchar[perLine * height + kAlignImageBy]; const auto cleanupData = static_cast(buffer); const auto address = reinterpret_cast(buffer); const auto alignedBuffer = buffer + ((address % kAlignImageBy) ? (kAlignImageBy - (address % kAlignImageBy)) : 0); return QImage( alignedBuffer, width, height, perLine, kImageFormat, AlignedImageBufferCleanupHandler, cleanupData); } IOPointer MakeIOPointer( void *opaque, int(*read)(void *opaque, uint8_t *buffer, int bufferSize), int(*write)(void *opaque, uint8_t *buffer, int bufferSize), int64_t(*seek)(void *opaque, int64_t offset, int whence)) { auto buffer = reinterpret_cast(av_malloc(kAvioBlockSize)); if (!buffer) { LogError(qstr("av_malloc")); return {}; } auto result = IOPointer(avio_alloc_context( buffer, kAvioBlockSize, write ? 1 : 0, opaque, read, write, seek)); if (!result) { av_freep(&buffer); LogError(qstr("avio_alloc_context")); return {}; } return result; } void IODeleter::operator()(AVIOContext *value) { if (value) { av_freep(&value->buffer); avio_context_free(&value); } } FormatPointer MakeFormatPointer( void *opaque, int(*read)(void *opaque, uint8_t *buffer, int bufferSize), int(*write)(void *opaque, uint8_t *buffer, int bufferSize), int64_t(*seek)(void *opaque, int64_t offset, int whence)) { auto io = MakeIOPointer(opaque, read, write, seek); if (!io) { return {}; } auto result = avformat_alloc_context(); if (!result) { LogError(qstr("avformat_alloc_context")); return {}; } result->pb = io.get(); auto options = (AVDictionary*)nullptr; const auto guard = gsl::finally([&] { av_dict_free(&options); }); av_dict_set(&options, "usetoc", "1", 0); const auto error = AvErrorWrap(avformat_open_input( &result, nullptr, nullptr, &options)); if (error) { // avformat_open_input freed 'result' in case an error happened. LogError(qstr("avformat_open_input"), error); return {}; } result->flags |= AVFMT_FLAG_FAST_SEEK; // Now FormatPointer will own and free the IO context. io.release(); return FormatPointer(result); } void FormatDeleter::operator()(AVFormatContext *value) { if (value) { const auto deleter = IOPointer(value->pb); avformat_close_input(&value); } } CodecPointer MakeCodecPointer(not_null stream) { auto error = AvErrorWrap(); auto result = CodecPointer(avcodec_alloc_context3(nullptr)); const auto context = result.get(); if (!context) { LogError(qstr("avcodec_alloc_context3")); return {}; } error = avcodec_parameters_to_context(context, stream->codecpar); if (error) { LogError(qstr("avcodec_parameters_to_context"), error); return {}; } av_codec_set_pkt_timebase(context, stream->time_base); av_opt_set_int(context, "refcounted_frames", 1, 0); const auto codec = avcodec_find_decoder(context->codec_id); if (!codec) { LogError(qstr("avcodec_find_decoder"), context->codec_id); return {}; } else if ((error = avcodec_open2(context, codec, nullptr))) { LogError(qstr("avcodec_open2"), error); return {}; } return result; } void CodecDeleter::operator()(AVCodecContext *value) { if (value) { avcodec_free_context(&value); } } FramePointer MakeFramePointer() { return FramePointer(av_frame_alloc()); } bool FrameHasData(AVFrame *frame) { return (frame && frame->data[0] != nullptr); } void ClearFrameMemory(AVFrame *frame) { if (FrameHasData(frame)) { av_frame_unref(frame); } } void FrameDeleter::operator()(AVFrame *value) { av_frame_free(&value); } SwscalePointer MakeSwscalePointer( not_null frame, QSize resize, SwscalePointer *existing) { // We have to use custom caching for SwsContext, because // sws_getCachedContext checks passed flags with existing context flags, // and re-creates context if they're different, but in the process of // context creation the passed flags are modified before being written // to the resulting context, so the caching doesn't work. if (existing && (*existing) != nullptr) { const auto &deleter = existing->get_deleter(); if (deleter.resize == resize && deleter.frameSize == QSize(frame->width, frame->height) && deleter.frameFormat == frame->format) { return std::move(*existing); } } if (frame->format <= AV_PIX_FMT_NONE || frame->format >= AV_PIX_FMT_NB) { LogError(qstr("frame->format")); return SwscalePointer(); } const auto result = sws_getCachedContext( existing ? existing->release() : nullptr, frame->width, frame->height, AVPixelFormat(frame->format), resize.width(), resize.height(), AV_PIX_FMT_BGRA, 0, nullptr, nullptr, nullptr); if (!result) { LogError(qstr("sws_getCachedContext")); } return SwscalePointer( result, { resize, QSize{ frame->width, frame->height }, frame->format }); } void SwscaleDeleter::operator()(SwsContext *value) { if (value) { sws_freeContext(value); } } void LogError(QLatin1String method) { LOG(("Streaming Error: Error in %1.").arg(method)); } void LogError(QLatin1String method, AvErrorWrap error) { LOG(("Streaming Error: Error in %1 (code: %2, text: %3)." ).arg(method ).arg(error.code() ).arg(error.text())); } crl::time PtsToTime(int64_t pts, AVRational timeBase) { return (pts == AV_NOPTS_VALUE || !timeBase.den) ? kTimeUnknown : ((pts * 1000LL * timeBase.num) / timeBase.den); } crl::time PtsToTimeCeil(int64_t pts, AVRational timeBase) { return (pts == AV_NOPTS_VALUE || !timeBase.den) ? kTimeUnknown : ((pts * 1000LL * timeBase.num + timeBase.den - 1) / timeBase.den); } int64_t TimeToPts(crl::time time, AVRational timeBase) { return (time == kTimeUnknown || !timeBase.num) ? AV_NOPTS_VALUE : (time * timeBase.den) / (1000LL * timeBase.num); } crl::time PacketPosition(const Packet &packet, AVRational timeBase) { const auto &native = packet.fields(); return PtsToTime( (native.pts == AV_NOPTS_VALUE) ? native.dts : native.pts, timeBase); } crl::time PacketDuration(const Packet &packet, AVRational timeBase) { return PtsToTime(packet.fields().duration, timeBase); } int DurationByPacket(const Packet &packet, AVRational timeBase) { const auto position = PacketPosition(packet, timeBase); const auto duration = std::max( PacketDuration(packet, timeBase), crl::time(1)); const auto bad = [](crl::time time) { return (time < 0) || (time > kDurationMax); }; if (bad(position) || bad(duration) || bad(position + duration + 1)) { LOG(("Streaming Error: Wrong duration by packet: %1 + %2" ).arg(position ).arg(duration)); return -1; } return int(position + duration + 1); } crl::time FramePosition(const Stream &stream) { const auto pts = !stream.frame ? AV_NOPTS_VALUE : (stream.frame->best_effort_timestamp != AV_NOPTS_VALUE) ? stream.frame->best_effort_timestamp : (stream.frame->pts != AV_NOPTS_VALUE) ? stream.frame->pts : stream.frame->pkt_dts; return PtsToTime(pts, stream.timeBase); } int ReadRotationFromMetadata(not_null stream) { const auto tag = av_dict_get(stream->metadata, "rotate", nullptr, 0); if (tag && *tag->value) { const auto string = QString::fromUtf8(tag->value); auto ok = false; const auto degrees = string.toInt(&ok); if (ok && (degrees == 90 || degrees == 180 || degrees == 270)) { return degrees; } } return 0; } AVRational ValidateAspectRatio(AVRational aspect) { return IsValidAspectRatio(aspect) ? aspect : kNormalAspect; } QSize CorrectByAspect(QSize size, AVRational aspect) { Expects(IsValidAspectRatio(aspect)); return QSize(size.width() * aspect.num / aspect.den, size.height()); } bool RotationSwapWidthHeight(int rotation) { return (rotation == 90 || rotation == 270); } AvErrorWrap ProcessPacket(Stream &stream, Packet &&packet) { Expects(stream.codec != nullptr); auto error = AvErrorWrap(); const auto native = &packet.fields(); const auto guard = gsl::finally([ &, size = native->size, data = native->data ] { native->size = size; native->data = data; packet = Packet(); }); error = avcodec_send_packet( stream.codec.get(), native->data ? native : nullptr); // Drain on eof. if (error) { LogError(qstr("avcodec_send_packet"), error); if (error.code() == AVERROR_INVALIDDATA // There is a sample voice message where skipping such packet // results in a crash (read_access to nullptr) in swr_convert(). && stream.codec->codec_id != AV_CODEC_ID_OPUS) { if (++stream.invalidDataPackets < kSkipInvalidDataPackets) { return AvErrorWrap(); // Try to skip a bad packet. } } } return error; } AvErrorWrap ReadNextFrame(Stream &stream) { Expects(stream.frame != nullptr); auto error = AvErrorWrap(); do { error = avcodec_receive_frame( stream.codec.get(), stream.frame.get()); if (!error || error.code() != AVERROR(EAGAIN) || stream.queue.empty()) { return error; } error = ProcessPacket(stream, std::move(stream.queue.front())); stream.queue.pop_front(); } while (!error); return error; } bool GoodForRequest(const QImage &image, const FrameRequest &request) { if (request.resize.isEmpty()) { return true; } else if ((request.radius != ImageRoundRadius::None) && ((request.corners & RectPart::AllCorners) != 0)) { return false; } return (request.resize == request.outer) && (request.resize == image.size()); } QImage ConvertFrame( Stream &stream, AVFrame *frame, QSize resize, QImage storage) { Expects(frame != nullptr); const auto frameSize = QSize(frame->width, frame->height); if (frameSize.isEmpty()) { LOG(("Streaming Error: Bad frame size %1,%2" ).arg(frameSize.width() ).arg(frameSize.height())); return QImage(); } else if (!FrameHasData(frame)) { LOG(("Streaming Error: Bad frame data.")); return QImage(); } if (resize.isEmpty()) { resize = frameSize; } else if (RotationSwapWidthHeight(stream.rotation)) { resize.transpose(); } if (!GoodStorageForFrame(storage, resize)) { storage = CreateFrameStorage(resize); } const auto format = AV_PIX_FMT_BGRA; const auto hasDesiredFormat = (frame->format == format); if (frameSize == storage.size() && hasDesiredFormat) { static_assert(sizeof(uint32) == kPixelBytesSize); auto to = reinterpret_cast(storage.bits()); auto from = reinterpret_cast(frame->data[0]); const auto deltaTo = (storage.bytesPerLine() / kPixelBytesSize) - storage.width(); const auto deltaFrom = (frame->linesize[0] / kPixelBytesSize) - frame->width; for (const auto y : ranges::view::ints(0, frame->height)) { for (const auto x : ranges::view::ints(0, frame->width)) { // Wipe out possible alpha values. *to++ = 0xFF000000U | *from++; } to += deltaTo; from += deltaFrom; } } else { stream.swscale = MakeSwscalePointer( frame, resize, &stream.swscale); if (!stream.swscale) { return QImage(); } // AV_NUM_DATA_POINTERS defined in AVFrame struct uint8_t *data[AV_NUM_DATA_POINTERS] = { storage.bits(), nullptr }; int linesize[AV_NUM_DATA_POINTERS] = { storage.bytesPerLine(), 0 }; const auto lines = sws_scale( stream.swscale.get(), frame->data, frame->linesize, 0, frame->height, data, linesize); if (lines != resize.height()) { LOG(("Streaming Error: " "Unable to sws_scale to good size %1, got %2." ).arg(resize.height() ).arg(lines)); return QImage(); } } ClearFrameMemory(frame); return storage; } QImage PrepareByRequest( const QImage &original, const FrameRequest &request, QImage storage) { Expects(!request.outer.isEmpty()); if (!GoodStorageForFrame(storage, request.outer)) { storage = CreateFrameStorage(request.outer); } { Painter p(&storage); PainterHighQualityEnabler hq(p); p.drawImage(QRect(QPoint(), request.outer), original); } if ((request.corners & RectPart::AllCorners) && (request.radius != ImageRoundRadius::None)) { Images::prepareRound(storage, request.radius, request.corners); } // #TODO streaming later full prepare support. return storage; } } // namespace Streaming } // namespace Media