Trivial in-memory frame caching.

This commit is contained in:
John Preston 2019-06-26 19:14:46 +02:00
parent 56e137b20f
commit a026aec786
9 changed files with 360 additions and 73 deletions

View File

@ -1099,7 +1099,7 @@ std::unique_ptr<Lottie::Animation> LottieFromDocument(
Storage::Cache::Key{ key->high, key->low + int(sizeTag) },
data,
filepath,
box);
Lottie::FrameRequest{ box });
}
return Lottie::FromContent(data, filepath);
}

View File

@ -1368,9 +1368,10 @@ void StickersListWidget::setupLottie(Set &set, int section, int index) {
auto &sticker = set.stickers[index];
const auto document = sticker.document;
sticker.animated = Lottie::FromContent(
document->data(),
document->filepath());
sticker.animated = Stickers::LottieFromDocument(
document,
Stickers::LottieSize::StickersColumn, // #TODO stickers
boundingBoxSize() * cIntRetinaFactor());
const auto animation = sticker.animated.get();
animation->updates(

View File

@ -97,7 +97,10 @@ QSize HistorySticker::countCurrentSize(int newWidth) {
}
void HistorySticker::setupLottie() {
_lottie = Lottie::FromContent(_data->data(), _data->filepath());
_lottie = Stickers::LottieFromDocument(
_data,
Stickers::LottieSize::MessageHistory,
QSize(st::maxStickerSize, st::maxStickerSize) * cIntRetinaFactor());
_parent->data()->history()->owner().registerHeavyViewPart(_parent);
_lottie->updates(

View File

@ -76,18 +76,6 @@ std::optional<Error> ContentError(const QByteArray &content) {
return std::nullopt;
}
std::unique_ptr<rlottie::Animation> CreateImplementation(
const QByteArray &content) {
const auto string = UnpackGzip(content);
Assert(string.size() <= kMaxFileSize);
auto result = rlottie::Animation::loadFromData(string, std::string());
if (!result) {
qWarning() << "Lottie Error: Parse failed.";
}
return result;
}
details::InitData CheckSharedState(std::unique_ptr<SharedState> state) {
Expects(state != nullptr);
@ -104,7 +92,7 @@ details::InitData Init(const QByteArray &content) {
if (const auto error = ContentError(content)) {
return *error;
}
auto animation = CreateImplementation(content);
auto animation = details::CreateFromContent(content);
return animation
? CheckSharedState(std::make_unique<SharedState>(
std::move(animation)))
@ -116,22 +104,43 @@ details::InitData Init(
not_null<Storage::Cache::Database*> cache,
Storage::Cache::Key key,
const QByteArray &cached,
QSize box) {
const FrameRequest &request) {
if (const auto error = ContentError(content)) {
return *error;
}
auto state = CacheState(cached, box);
auto state = CacheState(cached, request);
const auto prepare = !state.framesCount()
|| (state.framesReady() < state.framesCount());
auto animation = prepare ? CreateImplementation(content) : nullptr;
auto animation = prepare ? details::CreateFromContent(content) : nullptr;
return (!prepare || animation)
? CheckSharedState(std::make_unique<SharedState>(
std::move(animation)))
content,
std::move(animation),
std::move(state),
cache,
key,
request))
: Error::ParseFailed;
}
} // namespace
namespace details {
std::unique_ptr<rlottie::Animation> CreateFromContent(
const QByteArray &content) {
const auto string = UnpackGzip(content);
Assert(string.size() <= kMaxFileSize);
auto result = rlottie::Animation::loadFromData(string, std::string());
if (!result) {
qWarning() << "Lottie Error: Parse failed.";
}
return result;
}
} // namespace details
std::unique_ptr<Animation> FromContent(
const QByteArray &data,
const QString &filepath) {
@ -143,12 +152,12 @@ std::unique_ptr<Animation> FromCached(
Storage::Cache::Key key,
const QByteArray &data,
const QString &filepath,
QSize box) {
const FrameRequest &request) {
return std::make_unique<Animation>(
cache,
key,
ReadContent(data, filepath),
box);
request);
}
QImage ReadThumbnail(const QByteArray &content) {
@ -174,12 +183,12 @@ Animation::Animation(
not_null<Storage::Cache::Database*> cache,
Storage::Cache::Key key,
const QByteArray &content,
QSize box)
const FrameRequest &request)
: _timer([=] { checkNextFrameRender(); }) {
const auto weak = base::make_weak(this);
cache->get(key, [=](QByteArray &&cached) mutable {
crl::async([=] {
auto result = Init(content, cache, key, cached, box);
auto result = Init(content, cache, key, cached, request);
crl::on_main(weak, [=, data = std::move(result)]() mutable {
initDone(std::move(data));
});

View File

@ -30,6 +30,10 @@ struct Key;
} // namespace Cache
} // namespace Storage
namespace rlottie {
class Animation;
} // namespace rlottie
namespace Lottie {
inline constexpr auto kMaxFileSize = 1024 * 1024;
@ -46,7 +50,7 @@ std::unique_ptr<Animation> FromCached(
Storage::Cache::Key key,
const QByteArray &data,
const QString &filepath,
QSize box);
const FrameRequest &request);
QImage ReadThumbnail(const QByteArray &content);
@ -54,6 +58,9 @@ namespace details {
using InitData = base::variant<std::unique_ptr<SharedState>, Error>;
std::unique_ptr<rlottie::Animation> CreateFromContent(
const QByteArray &content);
} // namespace details
class Animation final : public base::has_weak_ptr {
@ -63,7 +70,7 @@ public:
not_null<Storage::Cache::Database*> cache,
Storage::Cache::Key key,
const QByteArray &content,
QSize box);
const FrameRequest &request);
~Animation();
//void play(const PlaybackOptions &options);

View File

@ -30,7 +30,15 @@ bool UncompressToRaw(AlignedStorage &to, bytes::const_span from) {
}
}
void Decode(QImage &to, const AlignedStorage &from, const QSize &fromSize) {
void CompressFromRaw(QByteArray &to, const AlignedStorage &from) {
const auto size = to.size();
to.resize(size + from.rawSize());
memcpy(to.data() + size, from.raw(), from.rawSize());
// #TODO stickers
}
void Decode(QImage &to, AlignedStorage &from, const QSize &fromSize) {
from.copyRawToAligned();
if (!FFmpeg::GoodStorageForFrame(to, fromSize)) {
to = FFmpeg::CreateFrameStorage(fromSize);
}
@ -45,6 +53,46 @@ void Decode(QImage &to, const AlignedStorage &from, const QSize &fromSize) {
}
}
void Encode(AlignedStorage &to, const QImage &from, const QSize &toSize) {
auto fromBytes = from.bits();
auto toBytes = static_cast<char*>(to.aligned());
const auto fromPerLine = from.bytesPerLine();
const auto toPerLine = to.bytesPerLine();
for (auto i = 0; i != to.lines(); ++i) {
memcpy(toBytes, fromBytes, from.width() * 4);
fromBytes += fromPerLine;
toBytes += toPerLine;
}
to.copyAlignedToRaw();
}
void Xor(AlignedStorage &to, const AlignedStorage &from) {
Expects(to.rawSize() == from.rawSize());
using Block = std::conditional_t<
sizeof(void*) == sizeof(uint64),
uint64,
uint32>;
constexpr auto kBlockSize = sizeof(Block);
const auto amount = from.rawSize();
const auto fromBytes = reinterpret_cast<const uchar*>(from.raw());
const auto toBytes = reinterpret_cast<uchar*>(to.raw());
const auto skip = reinterpret_cast<quintptr>(toBytes) % kBlockSize;
const auto blocks = (amount - skip) / kBlockSize;
for (auto i = 0; i != skip; ++i) {
toBytes[i] ^= fromBytes[i];
}
const auto fromBlocks = reinterpret_cast<const Block*>(fromBytes + skip);
const auto toBlocks = reinterpret_cast<Block*>(toBytes + skip);
for (auto i = 0; i != blocks; ++i) {
toBlocks[i] ^= fromBlocks[i];
}
const auto left = amount - skip - (blocks * kBlockSize);
for (auto i = amount - left; i != amount; ++i) {
toBytes[i] ^= fromBytes[i];
}
}
} // namespace
void AlignedStorage::allocate(int packedBytesPerLine, int lines) {
@ -104,10 +152,10 @@ void AlignedStorage::copyRawToAligned() {
if (fromPerLine == toPerLine) {
return;
}
auto from = static_cast<char*>(raw());
auto from = static_cast<const char*>(raw());
auto to = static_cast<char*>(aligned());
for (auto i = 0; i != _lines; ++i) {
memcpy(from, to, fromPerLine);
memcpy(to, from, fromPerLine);
from += fromPerLine;
to += toPerLine;
}
@ -119,22 +167,36 @@ void AlignedStorage::copyAlignedToRaw() {
if (fromPerLine == toPerLine) {
return;
}
auto from = static_cast<char*>(aligned());
auto from = static_cast<const char*>(aligned());
auto to = static_cast<char*>(raw());
for (auto i = 0; i != _lines; ++i) {
memcpy(from, to, toPerLine);
memcpy(to, from, toPerLine);
from += fromPerLine;
to += toPerLine;
}
}
CacheState::CacheState(const QByteArray &data, QSize box)
CacheState::CacheState(const QByteArray &data, const FrameRequest &request)
: _data(data) {
if (!readHeader(box)) {
if (!readHeader(request)) {
_framesReady = 0;
_data = QByteArray();
}
}
void CacheState::init(
QSize original,
int frameRate,
int framesCount,
const FrameRequest &request) {
_size = request.size(original);
_original = original;
_frameRate = frameRate;
_framesCount = framesCount;
_framesReady = 0;
prepareBuffers();
}
int CacheState::frameRate() const {
return _frameRate;
}
@ -147,14 +209,18 @@ int CacheState::framesCount() const {
return _framesCount;
}
bool CacheState::readHeader(QSize box) {
QSize CacheState::originalSize() const {
return _original;
}
bool CacheState::readHeader(const FrameRequest &request) {
if (_data.isEmpty()) {
return false;
}
QDataStream stream(&_data, QIODevice::ReadOnly);
auto encoder = uchar(0);
auto encoder = quint8(0);
stream >> encoder;
if (static_cast<Encoder>(encoder) != Encoder::YUV420A4_LZ4) {
return false;
@ -180,47 +246,137 @@ bool CacheState::readHeader(QSize box) {
|| (framesCount > kMaxFramesCount)
|| (framesReady <= 0)
|| (framesReady > framesCount)
|| FrameRequest{ box }.size(original) != size) {
|| request.size(original) != size) {
return false;
}
_headerSize = stream.device()->pos();
_size = size;
_original = original;
_frameRate = frameRate;
_framesCount = framesCount;
_framesReady = framesReady;
prepareBuffers();
if (!readCompressedDelta(stream.device()->pos())) {
return false;
}
_uncompressed.copyRawToAligned();
std::swap(_uncompressed, _previous);
Decode(_firstFrame, _previous, _size);
return true;
return renderFrame(_firstFrame, request, 0);
}
QImage CacheState::takeFirstFrame() {
return std::move(_firstFrame);
}
bool CacheState::renderFrame(
QImage &to,
const FrameRequest &request,
int index) {
Expects(index >= _framesReady
|| index == _offsetFrameIndex
|| index == 0);
if (index >= _framesReady) {
return false;
} else if (request.size(_original) != _size) {
return false;
} else if (index == 0) {
_offset = _headerSize;
_offsetFrameIndex = 0;
}
if (!readCompressedDelta()) {
_framesReady = 0;
_data = QByteArray();
return false;
}
if (index == 0) {
std::swap(_uncompressed, _previous);
} else {
Xor(_previous, _uncompressed);
}
Decode(to, _previous, _size);
return true;
}
void CacheState::appendFrame(
const QImage &frame,
const FrameRequest &request,
int index) {
if (request.size(_original) != _size) {
_framesReady = 0;
_data = QByteArray();
}
if (index != _framesReady) {
return;
}
if (index == 0) {
_size = request.size(_original);
writeHeader();
prepareBuffers();
} else {
incrementFramesReady();
}
Encode(_uncompressed, frame, _size);
if (index == 0) {
writeCompressedDelta();
std::swap(_uncompressed, _previous);
} else {
std::swap(_uncompressed, _previous);
Xor(_uncompressed, _previous);
writeCompressedDelta();
}
}
void CacheState::writeHeader() {
Expects(_framesReady == 0);
Expects(_data.isEmpty());
QDataStream stream(&_data, QIODevice::WriteOnly);
stream
<< static_cast<quint8>(Encoder::YUV420A4_LZ4)
<< _size
<< _original
<< qint32(_frameRate)
<< qint32(_framesCount)
<< qint32(++_framesReady);
_headerSize = stream.device()->pos();
}
void CacheState::incrementFramesReady() {
Expects(_headerSize > sizeof(qint32) && _data.size() > _headerSize);
const auto framesReady = qint32(++_framesReady);
bytes::copy(
bytes::make_detached_span(_data).subspan(
_headerSize - sizeof(qint32)),
bytes::object_as_span(&framesReady));
}
void CacheState::writeCompressedDelta() {
auto length = qint32(0);
const auto size = _data.size();
_data.resize(size + sizeof(length));
CompressFromRaw(_data, _uncompressed);
length = _data.size() - size - sizeof(length);
bytes::copy(
bytes::make_detached_span(_data).subspan(size),
bytes::object_as_span(&length));
}
void CacheState::prepareBuffers() {
_uncompressed.allocate(_size.width() * 4, _size.height());
_previous.allocate(_size.width() * 4, _size.height());
}
int CacheState::uncompressedDeltaSize() const {
return _size.width() * _size.height() * 4; // #TODO stickers
}
bool CacheState::readCompressedDelta(int offset) {
bool CacheState::readCompressedDelta() {
auto length = qint32(0);
const auto part = bytes::make_span(_data).subspan(offset);
const auto part = bytes::make_span(_data).subspan(_offset);
if (part.size() < sizeof(length)) {
return false;
}
bytes::copy(bytes::object_as_span(&length), part);
bytes::copy(
bytes::object_as_span(&length),
part.subspan(0, sizeof(length)));
const auto bytes = part.subspan(sizeof(length));
const auto uncompressedSize = uncompressedDeltaSize();
_offset = offset + length;
_offset += sizeof(length) + length;
++_offsetFrameIndex;
return (length <= bytes.size())
? UncompressToRaw(_uncompressed, bytes.subspan(0, length))
: false;

View File

@ -13,6 +13,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
namespace Lottie {
struct FrameRequest;
class AlignedStorage {
public:
void allocate(int packedBytesPerLine, int lines);
@ -46,22 +48,40 @@ private:
class CacheState {
public:
enum class Encoder : uchar {
enum class Encoder : quint8 {
YUV420A4_LZ4,
};
CacheState(const QByteArray &data, QSize box);
CacheState(const QByteArray &data, const FrameRequest &request);
void init(
QSize original,
int frameRate,
int framesCount,
const FrameRequest &request);
[[nodiscard]] int frameRate() const;
[[nodiscard]] int framesReady() const;
[[nodiscard]] int framesCount() const;
[[nodiscard]] QSize originalSize() const;
[[nodiscard]] QImage takeFirstFrame();
[[nodiscard]] bool renderFrame(
QImage &to,
const FrameRequest &request,
int index);
void appendFrame(
const QImage &frame,
const FrameRequest &request,
int index);
private:
[[nodiscard]] bool readHeader(QSize box);
void prepareBuffers();
[[nodiscard]] bool readCompressedDelta(int offset);
[[nodiscard]] int uncompressedDeltaSize() const;
void writeHeader();
void incrementFramesReady();
[[nodiscard]] bool readHeader(const FrameRequest &request);
void writeCompressedDelta();
[[nodiscard]] bool readCompressedDelta();
QByteArray _data;
QSize _size;
@ -72,7 +92,9 @@ private:
int _frameRate = 0;
int _framesCount = 0;
int _framesReady = 0;
int _headerSize = 0;
int _offset = 0;
int _offsetFrameIndex = 0;
Encoder _encoder = Encoder::YUV420A4_LZ4;
};

View File

@ -8,6 +8,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include "lottie/lottie_frame_renderer.h"
#include "lottie/lottie_animation.h"
#include "lottie/lottie_cache.h"
#include "storage/cache/storage_cache_database.h"
#include "logs.h"
#include "rlottie.h"
@ -72,6 +74,26 @@ private:
};
struct SharedState::Cache {
Cache(
CacheState &&state,
not_null<Storage::Cache::Database*> storage,
Storage::Cache::Key key);
CacheState state;
not_null<Storage::Cache::Database*> storage;
Storage::Cache::Key key;
};
SharedState::Cache::Cache(
CacheState &&state,
not_null<Storage::Cache::Database*> storage,
Storage::Cache::Key key)
: state(std::move(state))
, storage(storage)
, key(key) {
}
[[nodiscard]] bool GoodForRequest(
const QImage &image,
const FrameRequest &request) {
@ -133,6 +155,8 @@ FrameRendererObject::FrameRendererObject(
void FrameRendererObject::append(std::unique_ptr<SharedState> state) {
_entries.push_back({ std::move(state) });
auto &entry = _entries.back();
entry.request = entry.state->frameForPaint()->request;
queueGenerateFrames();
}
@ -176,22 +200,56 @@ void FrameRendererObject::queueGenerateFrames() {
SharedState::SharedState(std::unique_ptr<rlottie::Animation> animation)
: _animation(std::move(animation)) {
Expects(_animation != nullptr);
construct(FrameRequest());
}
SharedState::SharedState(
const QByteArray &content,
std::unique_ptr<rlottie::Animation> animation,
CacheState &&state,
not_null<Storage::Cache::Database*> cache,
Storage::Cache::Key key,
const FrameRequest &request)
: _content(content)
, _animation(std::move(animation))
, _cache(std::make_unique<Cache>(std::move(state), cache, key)) {
construct(request);
}
void SharedState::construct(const FrameRequest &request) {
calculateProperties();
if (isValid()) {
auto cover = QImage();
renderFrame(cover, FrameRequest(), 0);
init(std::move(cover));
if (!isValid()) {
return;
}
auto cover = _cache ? _cache->state.takeFirstFrame() : QImage();
if (!cover.isNull()) {
init(std::move(cover), request);
return;
}
if (_cache) {
_cache->state.init(_size, _frameRate, _framesCount, request);
}
renderFrame(cover, request, 0);
init(std::move(cover), request);
}
void SharedState::calculateProperties() {
Expects(_animation != nullptr || _cache != nullptr);
auto width = size_t(0);
auto height = size_t(0);
_animation->size(width, height);
const auto rate = _animation->frameRate();
const auto count = _animation->totalFrame();
if (_animation) {
_animation->size(width, height);
} else {
width = _cache->state.originalSize().width();
height = _cache->state.originalSize().height();
}
const auto rate = _animation
? _animation->frameRate()
: _cache->state.frameRate();
const auto count = _animation
? _animation->totalFrame()
: _cache->state.framesCount();
_size = QSize(
(width > 0 && width < kMaxSize) ? int(width) : 0,
@ -216,21 +274,33 @@ void SharedState::renderFrame(
if (!GoodStorageForFrame(image, size)) {
image = CreateFrameStorage(size);
}
image.fill(Qt::transparent);
if (_cache && _cache->state.renderFrame(image, request, index)) {
return;
} else if (!_animation) {
_animation = details::CreateFromContent(_content);
}
image.fill(Qt::transparent);
auto surface = rlottie::Surface(
reinterpret_cast<uint32_t*>(image.bits()),
image.width(),
image.height(),
image.bytesPerLine());
_animation->renderSync(index, surface);
if (_cache) {
_cache->state.appendFrame(image, request, index);
if (_cache->state.framesReady() == _cache->state.framesCount()) {
_animation = nullptr;
}
}
}
void SharedState::init(QImage cover) {
void SharedState::init(QImage cover, const FrameRequest &request) {
Expects(!initialized());
_duration = crl::time(1000) * _framesCount / _frameRate;
_frames[0].request = request;
_frames[0].original = std::move(cover);
_frames[0].position = 0;
@ -302,8 +372,7 @@ bool SharedState::renderNextFrame(const FrameRequest &request) {
case 6: return present(6, 0);
case 7: return prerender(1);
}
Unexpected("Counter value in VideoTrack::Shared::prepareState.");
Unexpected("Counter value in Lottie::SharedState::renderNextFrame.");
}
int SharedState::counter() const {

View File

@ -21,6 +21,13 @@ namespace rlottie {
class Animation;
} // namespace rlottie
namespace Storage {
namespace Cache {
class Database;
struct Key;
} // namespace Cache
} // namespace Storage
namespace Lottie {
inline constexpr auto kMaxFrameRate = 120;
@ -28,7 +35,7 @@ inline constexpr auto kMaxSize = 3096;
inline constexpr auto kMaxFramesCount = 600;
class Animation;
class JsonObject;
class CacheState;
struct Frame {
QImage original;
@ -47,6 +54,13 @@ QImage PrepareFrameByRequest(
class SharedState {
public:
explicit SharedState(std::unique_ptr<rlottie::Animation> animation);
SharedState(
const QByteArray &content,
std::unique_ptr<rlottie::Animation> animation,
CacheState &&state,
not_null<Storage::Cache::Database*> cache,
Storage::Cache::Key key,
const FrameRequest &request);
void start(not_null<Animation*> owner, crl::time now);
@ -64,9 +78,12 @@ public:
~SharedState();
private:
struct Cache;
void construct(const FrameRequest &request);
void calculateProperties();
bool isValid() const;
void init(QImage cover);
void init(QImage cover, const FrameRequest &request);
void renderNextFrame(
not_null<Frame*> frame,
const FrameRequest &request);
@ -74,6 +91,7 @@ private:
[[nodiscard]] not_null<const Frame*> getFrame(int index) const;
[[nodiscard]] int counter() const;
QByteArray _content;
std::unique_ptr<rlottie::Animation> _animation;
static constexpr auto kCounterUninitialized = -1;
@ -91,6 +109,8 @@ private:
QSize _size;
std::atomic<int> _accumulatedDelayMs = 0;
std::unique_ptr<Cache> _cache;
};
class FrameRendererObject;