diff --git a/Telegram/SourceFiles/base/match_method.h b/Telegram/SourceFiles/base/match_method.h index 9553ed116a..209494939f 100644 --- a/Telegram/SourceFiles/base/match_method.h +++ b/Telegram/SourceFiles/base/match_method.h @@ -16,7 +16,8 @@ inline decltype(auto) match_method( Data &&data, Method &&method, Methods &&...methods) { - if constexpr (rpl::details::is_callable_plain_v) { + using namespace rpl::details; + if constexpr (is_callable_plain_v) { return std::forward(method)(std::forward(data)); } else { return match_method( @@ -25,4 +26,27 @@ inline decltype(auto) match_method( } } +template < + typename Data1, + typename Data2, + typename Method, + typename ...Methods> +inline decltype(auto) match_method2( + Data1 &&data1, + Data2 &&data2, + Method &&method, + Methods &&...methods) { + using namespace rpl::details; + if constexpr (is_callable_plain_v) { + return std::forward(method)( + std::forward(data1), + std::forward(data2)); + } else { + return match_method2( + std::forward(data1), + std::forward(data2), + std::forward(methods)...); + } +} + } // namespace base diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.cpp new file mode 100644 index 0000000000..492d5f9403 --- /dev/null +++ b/Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.cpp @@ -0,0 +1,89 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#include "storage/cache/storage_cache_binlog_reader.h" + +namespace Storage { +namespace Cache { +namespace details { + +BinlogWrapper::BinlogWrapper(File &binlog, const Settings &settings) +: _binlog(binlog) +, _settings(settings) +, _till(_binlog.size()) +, _data(_settings.readBlockSize) +, _full(_data) { +} + +bool BinlogWrapper::failed() const { + return _failed; +} + +bool BinlogWrapper::readPart() { + if (_finished) { + return false; + } + const auto no = [&] { + finish(); + return false; + }; + const auto offset = _binlog.offset(); + const auto left = (_till - offset); + if (!left) { + return no(); + } + + Assert(_notParsedBytes >= 0 && _notParsedBytes <= _part.size()); + if (_notParsedBytes > 0 && _notParsedBytes < _part.size()) { + bytes::move(_full, _part.subspan(_part.size() - _notParsedBytes)); + } + const auto amount = std::min( + left, + int64(_full.size() - _notParsedBytes)); + Assert(amount > 0); + const auto readBytes = _binlog.read( + _full.subspan(_notParsedBytes, amount)); + if (!readBytes) { + return no(); + } + _notParsedBytes += readBytes; + _part = _full.subspan(0, _notParsedBytes); + return true; +} + +bytes::const_span BinlogWrapper::readRecord(ReadRecordSize readRecordSize) { + if (_finished) { + return {}; + } + const auto size = readRecordSize(*this, _part); + if (size == kRecordSizeUnknown || size > _part.size()) { + return {}; + } else if (size == kRecordSizeInvalid) { + finish(); + _finished = _failed = true; + return {}; + } + Assert(size >= 0 && size <= _notParsedBytes); + const auto result = _part.subspan(0, size); + _part = _part.subspan(size); + _notParsedBytes -= size; + return result; +} + +void BinlogWrapper::finish(size_type rollback) { + Expects(rollback >= 0); + + if (rollback > 0) { + _failed = true; + _notParsedBytes += rollback; + } + _binlog.seek(_binlog.offset() - _notParsedBytes); +} + +} // namespace details +} // namespace Cache +} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.h b/Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.h new file mode 100644 index 0000000000..ef48d19e46 --- /dev/null +++ b/Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.h @@ -0,0 +1,261 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#pragma once + +#include "storage/cache/storage_cache_types.h" +#include "storage/storage_encrypted_file.h" +#include "base/bytes.h" +#include "base/match_method.h" + +namespace Storage { +namespace Cache { +namespace details { + +template +class BinlogReader; + +class BinlogWrapper { +public: + BinlogWrapper(File &binlog, const Settings &settings); + + bool failed() const; + +private: + template + friend class BinlogReader; + + bool readPart(); + void finish(size_type rollback = 0); + + using ReadRecordSize = size_type (*)( + const BinlogWrapper &that, + bytes::const_span data); + bytes::const_span readRecord(ReadRecordSize readRecordSize); + + File &_binlog; + Settings _settings; + + int64 _till = 0; + bytes::vector _data; + bytes::span _full; + bytes::span _part; + index_type _notParsedBytes = 0; + bool _finished = false; + bool _failed = false; + +}; + +template +class BinlogReader { +public: + explicit BinlogReader(BinlogWrapper &wrapper); + + template + bool readTillEnd(Handlers &&...handlers); + +private: + static size_type ReadRecordSize( + const BinlogWrapper &that, + bytes::const_span data); + + template + bool handleRecord(bytes::const_span data, Handlers &&...handlers) const; + + BinlogWrapper &_wrapper; + +}; + +template +struct MultiRecord { + using true_t = char; + using false_t = true_t(&)[2]; + static_assert(sizeof(true_t) != sizeof(false_t)); + + static false_t Check(...); + template + static true_t Check(const Test&); + + static constexpr bool Is = (sizeof(Check(std::declval())) + == sizeof(true_t)); +}; + +template +struct BinlogReaderRecursive { + static void CheckSettings(const Settings &settings) { + } + + static size_type ReadRecordSize( + RecordType type, + bytes::const_span data, + size_type partsLimit) { + return kRecordSizeInvalid; + } + + template + static bool HandleRecord( + RecordType type, + bytes::const_span data, + Handlers &&...handlers) { + Unexpected("Bad type in BinlogReaderRecursive::HandleRecord."); + } +}; + +template +struct BinlogReaderRecursive { + static void CheckSettings(const Settings &settings); + + static size_type ReadRecordSize( + RecordType type, + bytes::const_span data, + size_type partsLimit); + + template + static bool HandleRecord( + RecordType type, + bytes::const_span data, + Handlers &&...handlers); +}; + +template +inline void BinlogReaderRecursive::CheckSettings( + const Settings &settings) { + static_assert(GoodForEncryption); + if constexpr (MultiRecord::Is) { + using Head = Record; + using Part = typename Record::Part; + static_assert(GoodForEncryption); + Assert(settings.readBlockSize + >= (sizeof(Head) + + settings.maxBundledRecords * sizeof(Part))); + } else { + Assert(settings.readBlockSize >= sizeof(Record)); + } +} + +template +inline size_type BinlogReaderRecursive::ReadRecordSize( + RecordType type, + bytes::const_span data, + size_type partsLimit) { + if (type != Record::kType) { + return BinlogReaderRecursive::ReadRecordSize( + type, + data, + partsLimit); + } + if constexpr (MultiRecord::Is) { + using Head = Record; + using Part = typename Record::Part; + + if (data.size() < sizeof(Head)) { + return kRecordSizeUnknown; + } + const auto head = reinterpret_cast(data.data()); + const auto count = head->validateCount(); + return (count >= 0 && count < partsLimit) + ? (sizeof(Head) + count * sizeof(Part)) + : kRecordSizeInvalid; + } else { + return sizeof(Record); + } +} + +template +template +inline bool BinlogReaderRecursive::HandleRecord( + RecordType type, + bytes::const_span data, + Handlers &&...handlers) { + if (type != Record::kType) { + return BinlogReaderRecursive::HandleRecord( + type, + data, + std::forward(handlers)...); + } + if constexpr (MultiRecord::Is) { + using Head = Record; + using Part = typename Record::Part; + + Assert(data.size() >= sizeof(Head)); + const auto bytes = data.data(); + const auto head = reinterpret_cast(bytes); + const auto count = head->validateCount(); + Assert(data.size() == sizeof(Head) + count * sizeof(Part)); + const auto parts = gsl::make_span( + reinterpret_cast(bytes + sizeof(Head)), + count); + auto from = std::begin(parts); + const auto till = std::end(parts); + const auto element = [&] { + return (from == till) ? nullptr : &*from++; + }; + return base::match_method2( + *head, + element, + std::forward(handlers)...); + } else { + Assert(data.size() == sizeof(Record)); + return base::match_method( + *reinterpret_cast(data.data()), + std::forward(handlers)...); + } +} + +template +BinlogReader::BinlogReader(BinlogWrapper &wrapper) +: _wrapper(wrapper) { + BinlogReaderRecursive::CheckSettings(_wrapper._settings); +} + +template +template +bool BinlogReader::readTillEnd(Handlers &&...handlers) { + if (!_wrapper.readPart()) { + return true; + } + const auto readRecord = [&] { + return _wrapper.readRecord(&BinlogReader::ReadRecordSize); + }; + for (auto bytes = readRecord(); !bytes.empty(); bytes = readRecord()) { + if (!handleRecord(bytes, std::forward(handlers)...)) { + _wrapper.finish(bytes.size()); + return true; + } + } + return false; +} + +template +size_type BinlogReader::ReadRecordSize( + const BinlogWrapper &that, + bytes::const_span data) { + if (data.empty()) { + return kRecordSizeUnknown; + } + return BinlogReaderRecursive::ReadRecordSize( + static_cast(data[0]), + data, + that._settings.maxBundledRecords); +} + +template +template +bool BinlogReader::handleRecord( + bytes::const_span data, + Handlers &&...handlers) const { + Expects(!data.empty()); + + return BinlogReaderRecursive::HandleRecord( + static_cast(data[0]), + data, + std::forward(handlers)...); +} + +} // namespace details +} // namespace Cache +} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database.h b/Telegram/SourceFiles/storage/cache/storage_cache_database.h index df0df67ccf..31bcb41a21 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database.h +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database.h @@ -25,22 +25,7 @@ using Error = details::Error; class Database { public: - struct Settings { - size_type maxBundledRecords = 16 * 1024; - size_type readBlockSize = 8 * 1024 * 1024; - size_type maxDataSize = 10 * 1024 * 1024; - crl::time_type writeBundleDelay = 15 * 60 * crl::time_type(1000); - - int64 compactAfterExcess = 8 * 1024 * 1024; - int64 compactAfterFullSize = 0; - - bool trackEstimatedTime = true; - int64 totalSizeLimit = 1024 * 1024 * 1024; - size_type totalTimeLimit = 30 * 86400; // One month in seconds. - size_type maxTimeAdvancement = 365 * 86400; // One year in seconds. - crl::time_type pruneTimeout = 5 * crl::time_type(1000); - crl::time_type maxPruneCheckTimeout = 3600 * crl::time_type(1000); - }; + using Settings = details::Settings; Database(const QString &path, const Settings &settings); void open(EncryptionKey key, FnMut done); diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp index 41e2c6288b..654181c705 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp @@ -9,6 +9,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "storage/cache/storage_cache_cleaner.h" #include "storage/cache/storage_cache_compactor.h" +#include "storage/cache/storage_cache_binlog_reader.h" #include "storage/storage_encryption.h" #include "storage/storage_encrypted_file.h" #include "base/flat_map.h" @@ -193,30 +194,51 @@ bool DatabaseObject::writeHeader() { return _binlog.write(bytes::object_as_span(&header)); } -void DatabaseObject::readBinlog() { - auto data = bytes::vector(_settings.readBlockSize); - const auto full = bytes::make_span(data); - auto notParsedBytes = index_type(0); +template +void DatabaseObject::readBinlogHelper( + Reader &reader, + Handlers &&...handlers) { while (true) { - Assert(notParsedBytes < full.size()); - const auto readBytes = _binlog.read(full.subspan(notParsedBytes)); - if (!readBytes) { + const auto done = reader.readTillEnd( + std::forward(handlers)...); + if (done) { break; } - notParsedBytes += readBytes; - const auto bytes = full.subspan(0, notParsedBytes); - const auto parsedTill = readBinlogRecords(bytes); - if (parsedTill == kRecordSizeInvalid) { - break; - } - Assert(parsedTill >= 0 && parsedTill <= notParsedBytes); - notParsedBytes -= parsedTill; - if (parsedTill > 0 && parsedTill < bytes.size()) { - bytes::move(full, bytes.subspan(parsedTill)); - } } - _binlog.seek(_binlog.offset() - notParsedBytes); +} +void DatabaseObject::readBinlog() { + BinlogWrapper wrapper(_binlog, _settings); + if (_settings.trackEstimatedTime) { + BinlogReader< + StoreWithTime, + MultiStoreWithTime, + MultiRemove, + MultiAccess> reader(wrapper); + readBinlogHelper(reader, [&](const StoreWithTime &record) { + return processRecordStore( + &record, + std::is_class{}); + }, [&](const MultiStoreWithTime &header, const auto &element) { + return processRecordMultiStore(header, element); + }, [&](const MultiRemove &header, const auto &element) { + return processRecordMultiRemove(header, element); + }, [&](const MultiAccess &header, const auto &element) { + return processRecordMultiAccess(header, element); + }); + } else { + BinlogReader< + Store, + MultiStore, + MultiRemove> reader(wrapper); + readBinlogHelper(reader, [&](const Store &record) { + return processRecordStore(&record, std::is_class{}); + }, [&](const MultiStore &header, const auto &element) { + return processRecordMultiStore(header, element); + }, [&](const MultiRemove &header, const auto &element) { + return processRecordMultiRemove(header, element); + }); + } adjustRelativeTime(); optimize(); } @@ -367,107 +389,9 @@ void DatabaseObject::adjustRelativeTime() { } } -size_type DatabaseObject::readBinlogRecords(bytes::const_span data) { - auto result = 0; - while (true) { - const auto size = readBinlogRecordSize(data); - if (size == kRecordSizeUnknown || size > data.size()) { - return result; - } else if (size == kRecordSizeInvalid - || !readBinlogRecord(data.subspan(0, size))) { - return (result > 0) ? result : kRecordSizeInvalid; - } else { - result += size; - data = data.subspan(size); - } - } -} - -size_type DatabaseObject::readBinlogRecordSize(bytes::const_span data) const { - if (data.empty()) { - return kRecordSizeUnknown; - } - - switch (static_cast(data[0])) { - case Store::kType: - return storeRecordSize(); - - case MultiStoreHeader::kType: - if (data.size() >= sizeof(MultiStoreHeader)) { - const auto header = reinterpret_cast( - data.data()); - const auto count = ReadFrom(header->count); - const auto size = _settings.trackEstimatedTime - ? sizeof(MultiStoreWithTimePart) - : sizeof(MultiStorePart); - return (count > 0 && count < _settings.maxBundledRecords) - ? (sizeof(MultiStoreHeader) + count * size) - : kRecordSizeInvalid; - } - return kRecordSizeUnknown; - - case MultiRemoveHeader::kType: - if (data.size() >= sizeof(MultiRemoveHeader)) { - const auto header = reinterpret_cast( - data.data()); - const auto count = ReadFrom(header->count); - return (count > 0 && count < _settings.maxBundledRecords) - ? (sizeof(MultiRemoveHeader) - + count * sizeof(MultiRemovePart)) - : kRecordSizeInvalid; - } - return kRecordSizeUnknown; - - case MultiAccessHeader::kType: - if (!_settings.trackEstimatedTime) { - return kRecordSizeInvalid; - } else if (data.size() >= sizeof(MultiAccessHeader)) { - const auto header = reinterpret_cast( - data.data()); - const auto count = ReadFrom(header->count); - return (count >= 0 && count < _settings.maxBundledRecords) - ? (sizeof(MultiAccessHeader) - + count * sizeof(MultiAccessPart)) - : kRecordSizeInvalid; - } - return kRecordSizeUnknown; - - } - return kRecordSizeInvalid; -} - -bool DatabaseObject::readBinlogRecord(bytes::const_span data) { - Expects(!data.empty()); - - switch (static_cast(data[0])) { - case Store::kType: - return readRecordStore(data); - - case MultiStoreHeader::kType: - return readRecordMultiStore(data); - - case MultiRemoveHeader::kType: - return readRecordMultiRemove(data); - - case MultiAccessHeader::kType: - return readRecordMultiAccess(data); - - } - Unexpected("Bad type in DatabaseObject::readBinlogRecord."); -} - -template -bool DatabaseObject::readRecordStoreGeneric(bytes::const_span data) { - Expects(data.size() == sizeof(RecordStore)); - - return processRecordStore( - reinterpret_cast(data.data()), - std::is_class{}); -} - -template +template bool DatabaseObject::processRecordStoreGeneric( - const RecordStore *record, + const Record *record, Postprocess &&postprocess) { const auto size = ReadFrom(record->size); if (size <= 0 || size > _settings.maxDataSize) { @@ -506,52 +430,62 @@ bool DatabaseObject::processRecordStore( return processRecordStoreGeneric(record, postprocess); } -bool DatabaseObject::readRecordStore(bytes::const_span data) { - if (!_settings.trackEstimatedTime) { - return readRecordStoreGeneric(data); - } - return readRecordStoreGeneric(data); -} - -template -bool DatabaseObject::readRecordMultiStoreGeneric(bytes::const_span data) { - Expects(data.size() >= sizeof(MultiStoreHeader)); - - const auto bytes = data.data(); - const auto record = reinterpret_cast(bytes); - const auto count = ReadFrom(record->count); - Assert(data.size() == sizeof(MultiStoreHeader) - + count * sizeof(StorePart)); - const auto parts = gsl::make_span( - reinterpret_cast( - bytes + sizeof(MultiStoreHeader)), - count); - for (const auto &part : parts) { - if (!processRecordStore(&part, std::is_class{})) { +template +bool DatabaseObject::processRecordMultiStore( + const Record &header, + const GetElement &element) { + while (const auto entry = element()) { + if (!processRecordStore( + entry, + std::is_class{})) { return false; } } return true; } -bool DatabaseObject::readRecordMultiStore(bytes::const_span data) { - if (!_settings.trackEstimatedTime) { - return readRecordMultiStoreGeneric(data); +template +bool DatabaseObject::processRecordMultiRemove( + const MultiRemove &header, + const GetElement &element) { + _binlogExcessLength += sizeof(header); + while (const auto entry = element()) { + _binlogExcessLength += sizeof(*entry); + if (const auto i = _map.find(*entry); i != end(_map)) { + eraseMapEntry(i); + } } - return readRecordMultiStoreGeneric(data); + return true; } -size_type DatabaseObject::storeRecordSize() const { - return _settings.trackEstimatedTime - ? sizeof(StoreWithTime) - : sizeof(Store); +template +bool DatabaseObject::processRecordMultiAccess( + const MultiAccess &header, + const GetElement &element) { + Expects(_settings.trackEstimatedTime); + + if (header.time.relativeAdvancement > _settings.maxTimeAdvancement) { + return false; + } + applyTimePoint(header.time); + + _binlogExcessLength += sizeof(header); + while (const auto entry = element()) { + _binlogExcessLength += sizeof(*entry); + if (const auto i = _map.find(*entry); i != end(_map)) { + i->second.useTime = _relativeTime; + } + } + return true; } void DatabaseObject::setMapEntry(const Key &key, Entry &&entry) { auto &already = _map[key]; _totalSize += entry.size - already.size; if (already.size != 0) { - _binlogExcessLength += storeRecordSize(); + _binlogExcessLength += _settings.trackEstimatedTime + ? sizeof(StoreWithTime) + : sizeof(Store); } if (entry.useTime != 0 && (entry.useTime < _minimalEntryTime || !_minimalEntryTime)) { @@ -581,27 +515,6 @@ void DatabaseObject::eraseMapEntry(const Map::const_iterator &i) { } } -bool DatabaseObject::readRecordMultiRemove(bytes::const_span data) { - Expects(data.size() >= sizeof(MultiRemoveHeader)); - - const auto bytes = data.data(); - const auto record = reinterpret_cast(bytes); - const auto count = ReadFrom(record->count); - Assert(data.size() == sizeof(MultiRemoveHeader) - + count * sizeof(MultiRemovePart)); - const auto parts = gsl::make_span( - reinterpret_cast( - bytes + sizeof(MultiRemoveHeader)), - count); - for (const auto &part : parts) { - if (const auto i = _map.find(part.key); i != end(_map)) { - eraseMapEntry(i); - } - } - _binlogExcessLength += data.size(); - return true; -} - EstimatedTimePoint DatabaseObject::countTimePoint() const { const auto now = std::max(GetUnixtime(), 1); const auto delta = std::max(int64(now) - int64(_latestSystemTime), 0LL); @@ -618,32 +531,6 @@ void DatabaseObject::applyTimePoint(EstimatedTimePoint time) { _latestSystemTime = time.system; } -bool DatabaseObject::readRecordMultiAccess(bytes::const_span data) { - Expects(data.size() >= sizeof(MultiAccessHeader)); - Expects(_settings.trackEstimatedTime); - - const auto bytes = data.data(); - const auto record = reinterpret_cast(bytes); - if (record->time.relativeAdvancement > _settings.maxTimeAdvancement) { - return false; - } - applyTimePoint(record->time); - const auto count = ReadFrom(record->count); - Assert(data.size() == sizeof(MultiAccessHeader) - + count * sizeof(MultiAccessPart)); - const auto parts = gsl::make_span( - reinterpret_cast( - bytes + sizeof(MultiAccessHeader)), - count); - for (const auto &part : parts) { - if (const auto i = _map.find(part.key); i != end(_map)) { - i->second.useTime = _relativeTime; - } - } - _binlogExcessLength += data.size(); - return true; -} - void DatabaseObject::close(FnMut done) { writeBundles(); _cleaner = CleanerWrap(); @@ -740,7 +627,9 @@ base::optional DatabaseObject::writeKeyPlaceGeneric( } _binlog.flush(); - const auto applied = readRecordStore(bytes::object_as_span(&record)); + const auto applied = processRecordStore( + &record, + std::is_class{}); Assert(applied); return result; } @@ -850,11 +739,11 @@ void DatabaseObject::writeMultiRemove() { return; } const auto size = _removing.size(); - auto header = MultiRemoveHeader(size); - auto list = std::vector(); + auto header = MultiRemove(size); + auto list = std::vector(); list.reserve(size); for (const auto &key : base::take(_removing)) { - list.push_back({ key }); + list.push_back(key); } if (_binlog.write(bytes::object_as_span(&header))) { _binlog.write(bytes::make_span(list)); @@ -885,17 +774,17 @@ void DatabaseObject::writeMultiAccessBlock() { const auto time = countTimePoint(); const auto size = _accessed.size(); - auto header = MultiAccessHeader(time, size); - auto list = std::vector(); + auto header = MultiAccess(time, size); + auto list = std::vector(); if (size > 0) { list.reserve(size); for (const auto &key : base::take(_accessed)) { - list.push_back({ key }); + list.push_back(key); } } applyTimePoint(time); for (const auto &entry : list) { - if (const auto i = _map.find(entry.key); i != end(_map)) { + if (const auto i = _map.find(entry); i != end(_map)) { i->second.useTime = _relativeTime; } } diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h index 41eb4bd191..5c88c46368 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h @@ -79,26 +79,28 @@ private: bool writeHeader(); void readBinlog(); - size_type readBinlogRecords(bytes::const_span data); - size_type readBinlogRecordSize(bytes::const_span data) const; - bool readBinlogRecord(bytes::const_span data); - template - bool readRecordStoreGeneric(bytes::const_span data); - bool readRecordStore(bytes::const_span data); - template - bool readRecordMultiStoreGeneric(bytes::const_span data); - bool readRecordMultiStore(bytes::const_span data); - bool readRecordMultiRemove(bytes::const_span data); - bool readRecordMultiAccess(bytes::const_span data); - template + template + void readBinlogHelper(Reader &reader, Handlers &&...handlers); + template bool processRecordStoreGeneric( - const RecordStore *record, + const Record *record, Postprocess &&postprocess); bool processRecordStore(const Store *record, std::is_class); bool processRecordStore( const StoreWithTime *record, std::is_class); - size_type storeRecordSize() const; + template + bool processRecordMultiStore( + const Record &header, + const GetElement &element); + template + bool processRecordMultiRemove( + const MultiRemove &header, + const GetElement &element); + template + bool processRecordMultiAccess( + const MultiAccess &header, + const GetElement &element); void optimize(); void checkCompactor(); diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp index fa29b0c433..f7fca7ee74 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp +++ b/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp @@ -57,19 +57,27 @@ BasicHeader::BasicHeader() , flags(0) { } -MultiStoreHeader::MultiStoreHeader(size_type count) +MultiStore::MultiStore(size_type count) : type(kType) , count(ReadTo(count)) { Expects(count >= 0 && count < kBundledRecordsLimit); } -MultiRemoveHeader::MultiRemoveHeader(size_type count) +size_type MultiStore::validateCount() const { + return ValidateStrictCount(count); +} + +MultiRemove::MultiRemove(size_type count) : type(kType) , count(ReadTo(count)) { Expects(count >= 0 && count < kBundledRecordsLimit); } -MultiAccessHeader::MultiAccessHeader( +size_type MultiRemove::validateCount() const { + return ValidateStrictCount(count); +} + +MultiAccess::MultiAccess( EstimatedTimePoint time, size_type count) : type(kType) @@ -78,6 +86,10 @@ MultiAccessHeader::MultiAccessHeader( Expects(count >= 0 && count < kBundledRecordsLimit); } +size_type MultiAccess::validateCount() const { + return ReadFrom(count); +} + } // namespace details } // namespace Cache } // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_types.h b/Telegram/SourceFiles/storage/cache/storage_cache_types.h index e4339f34a6..4c86321799 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_types.h +++ b/Telegram/SourceFiles/storage/cache/storage_cache_types.h @@ -9,6 +9,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "base/basic_types.h" #include "base/optional.h" +#include namespace Storage { namespace Cache { @@ -31,6 +32,23 @@ inline bool operator<(const Key &a, const Key &b) { return std::tie(a.high, a.low) < std::tie(b.high, b.low); } +struct Settings { + size_type maxBundledRecords = 16 * 1024; + size_type readBlockSize = 8 * 1024 * 1024; + size_type maxDataSize = 10 * 1024 * 1024; + crl::time_type writeBundleDelay = 15 * 60 * crl::time_type(1000); + + int64 compactAfterExcess = 8 * 1024 * 1024; + int64 compactAfterFullSize = 0; + + bool trackEstimatedTime = true; + int64 totalSizeLimit = 1024 * 1024 * 1024; + size_type totalTimeLimit = 30 * 86400; // One month in seconds. + size_type maxTimeAdvancement = 365 * 86400; // One year in seconds. + crl::time_type pruneTimeout = 5 * crl::time_type(1000); + crl::time_type maxPruneCheckTimeout = 3600 * crl::time_type(1000); +}; + using Version = int32; QString ComputeBasePath(const QString &original); @@ -61,7 +79,7 @@ using EntrySize = std::array; using RecordsCount = std::array; template -Packed ReadTo(size_type count) { +inline Packed ReadTo(size_type count) { Expects(count >= 0 && count < (1 << (Packed().size() * 8))); auto result = Packed(); @@ -73,7 +91,7 @@ Packed ReadTo(size_type count) { } template -size_type ReadFrom(Packed count) { +inline size_type ReadFrom(const Packed &count) { auto result = size_type(); for (auto &element : (count | ranges::view::reverse)) { result <<= 8; @@ -82,6 +100,12 @@ size_type ReadFrom(Packed count) { return result; } +template +inline size_type ValidateStrictCount(const Packed &count) { + const auto result = ReadFrom(count); + return (result != 0) ? result : -1; +} + constexpr auto kRecordSizeUnknown = size_type(-1); constexpr auto kRecordSizeInvalid = size_type(-2); constexpr auto kBundledRecordsLimit = (1 << (RecordsCount().size() * 8)); @@ -105,7 +129,6 @@ struct BasicHeader { uint32 reserved1 = 0; uint32 reserved2 = 0; }; -static_assert(GoodForEncryption); struct EstimatedTimePoint { uint32 system = 0; @@ -129,44 +152,44 @@ struct StoreWithTime : Store { uint32 reserved1 = 0; uint32 reserved2 = 0; }; -static_assert(GoodForEncryption); -struct MultiStoreHeader { +struct MultiStore { static constexpr auto kType = RecordType(0x02); - explicit MultiStoreHeader(size_type count = 0); + explicit MultiStore(size_type count = 0); RecordType type = kType; RecordsCount count = { { 0 } }; uint32 reserved1 = 0; uint32 reserved2 = 0; uint32 reserved3 = 0; -}; -using MultiStorePart = Store; -using MultiStoreWithTimePart = StoreWithTime; -static_assert(GoodForEncryption); -struct MultiRemoveHeader { + using Part = Store; + size_type validateCount() const; +}; +struct MultiStoreWithTime : MultiStore { + using Part = StoreWithTime; +}; + +struct MultiRemove { static constexpr auto kType = RecordType(0x03); - explicit MultiRemoveHeader(size_type count = 0); + explicit MultiRemove(size_type count = 0); RecordType type = kType; RecordsCount count = { { 0 } }; uint32 reserved1 = 0; uint32 reserved2 = 0; uint32 reserved3 = 0; -}; -struct MultiRemovePart { - Key key; -}; -static_assert(GoodForEncryption); -static_assert(GoodForEncryption); -struct MultiAccessHeader { + using Part = Key; + size_type validateCount() const; +}; + +struct MultiAccess { static constexpr auto kType = RecordType(0x04); - explicit MultiAccessHeader( + explicit MultiAccess( EstimatedTimePoint time, size_type count = 0); @@ -174,12 +197,10 @@ struct MultiAccessHeader { RecordsCount count = { { 0 } }; EstimatedTimePoint time; uint32 reserved = 0; + + using Part = Key; + size_type validateCount() const; }; -struct MultiAccessPart { - Key key; -}; -static_assert(GoodForEncryption); -static_assert(GoodForEncryption); } // namespace details } // namespace Cache diff --git a/Telegram/SourceFiles/storage/storage_encrypted_file.cpp b/Telegram/SourceFiles/storage/storage_encrypted_file.cpp index 50d18b2f98..27af9e0372 100644 --- a/Telegram/SourceFiles/storage/storage_encrypted_file.cpp +++ b/Telegram/SourceFiles/storage/storage_encrypted_file.cpp @@ -208,7 +208,9 @@ bool File::write(bytes::span bytes) { encrypt(bytes); const auto count = writePlain(bytes); - if (count != bytes.size()) { + if (count == bytes.size()) { + _dataSize = std::max(_dataSize, offset()); + } else { decryptBack(bytes); if (count > 0) { _data.seek(_data.pos() - count); diff --git a/Telegram/SourceFiles/storage/storage_encrypted_file_tests.cpp b/Telegram/SourceFiles/storage/storage_encrypted_file_tests.cpp index 97ae69ed89..552066569f 100644 --- a/Telegram/SourceFiles/storage/storage_encrypted_file_tests.cpp +++ b/Telegram/SourceFiles/storage/storage_encrypted_file_tests.cpp @@ -105,7 +105,7 @@ TEST_CASE("simple encrypted file", "[storage_encrypted_file]") { Key); REQUIRE(result == Storage::File::Result::Success); - auto data = bytes::vector(16); + auto data = bytes::vector(Test1.size()); const auto read = file.read(data); REQUIRE(read == data.size()); REQUIRE(data == bytes::make_vector(Test1)); @@ -122,24 +122,29 @@ TEST_CASE("simple encrypted file", "[storage_encrypted_file]") { Key); REQUIRE(result == Storage::File::Result::Success); REQUIRE(file.offset() == 0); + REQUIRE(file.size() == Test1.size() + Test2.size()); - const auto success1 = file.seek(16); + const auto success1 = file.seek(Test1.size()); REQUIRE(success1); - REQUIRE(file.offset() == 16); + REQUIRE(file.offset() == Test1.size()); - auto data = bytes::vector(16); + auto data = bytes::vector(Test2.size()); const auto read = file.read(data); REQUIRE(read == data.size()); REQUIRE(data == bytes::make_vector(Test2)); - REQUIRE(file.offset() == 32); + REQUIRE(file.offset() == Test1.size() + Test2.size()); + REQUIRE(file.size() == Test1.size() + Test2.size()); - const auto success2 = file.seek(16); + const auto success2 = file.seek(Test1.size()); REQUIRE(success2); - REQUIRE(file.offset() == 16); + REQUIRE(file.offset() == Test1.size()); data = bytes::make_vector(Test1); - const auto success3 = file.write(data); + const auto success3 = file.write(data) && file.write(data); REQUIRE(success3); + + REQUIRE(file.offset() == 3 * Test1.size()); + REQUIRE(file.size() == 3 * Test1.size()); } SECTION("reading file") { Storage::File file; diff --git a/Telegram/gyp/lib_storage.gyp b/Telegram/gyp/lib_storage.gyp index 0e01e73f49..d00b7b7eca 100644 --- a/Telegram/gyp/lib_storage.gyp +++ b/Telegram/gyp/lib_storage.gyp @@ -57,6 +57,8 @@ '<(src_loc)/storage/storage_file_lock_posix.cpp', '<(src_loc)/storage/storage_file_lock_win.cpp', '<(src_loc)/storage/storage_file_lock.h', + '<(src_loc)/storage/cache/storage_cache_binlog_reader.cpp', + '<(src_loc)/storage/cache/storage_cache_binlog_reader.h', '<(src_loc)/storage/cache/storage_cache_cleaner.cpp', '<(src_loc)/storage/cache/storage_cache_cleaner.h', '<(src_loc)/storage/cache/storage_cache_compactor.cpp',