From a4c1d5fe9dd78fd516a4ba1e28b0942979e39659 Mon Sep 17 00:00:00 2001 From: John Preston Date: Sun, 19 Aug 2018 23:06:49 +0300 Subject: [PATCH] Move cache database to a separate module. Also start compactor code. --- .../storage/cache/storage_cache_cleaner.cpp | 4 +- .../storage/cache/storage_cache_cleaner.h | 3 +- .../storage/cache/storage_cache_compactor.cpp | 53 + .../storage/cache/storage_cache_compactor.h | 36 + .../storage/cache/storage_cache_database.cpp | 1266 +---------------- .../storage/cache/storage_cache_database.h | 25 +- .../cache/storage_cache_database_object.cpp | 1008 +++++++++++++ .../cache/storage_cache_database_object.h | 181 +++ .../storage/cache/storage_cache_types.cpp | 28 + .../storage/cache/storage_cache_types.h | 156 ++ Telegram/gyp/lib_storage.gyp | 4 + 11 files changed, 1477 insertions(+), 1287 deletions(-) create mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_compactor.cpp create mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_compactor.h create mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp create mode 100644 Telegram/SourceFiles/storage/cache/storage_cache_database_object.h diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_cleaner.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_cleaner.cpp index 2741857542..664ca5257f 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_cleaner.cpp +++ b/Telegram/SourceFiles/storage/cache/storage_cache_cleaner.cpp @@ -19,7 +19,6 @@ namespace details { class CleanerObject { public: - using Wrapper = Cache::Cleaner; CleanerObject( crl::weak_on_queue weak, const QString &base, @@ -98,8 +97,6 @@ void CleanerObject::done() { } } -} // namespace details - Cleaner::Cleaner( const QString &base, base::binary_guard &&guard, @@ -109,5 +106,6 @@ Cleaner::Cleaner( Cleaner::~Cleaner() = default; +} // namespace details } // namespace Cache } // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_cleaner.h b/Telegram/SourceFiles/storage/cache/storage_cache_cleaner.h index cc2d8441f8..26f6f28b84 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_cleaner.h +++ b/Telegram/SourceFiles/storage/cache/storage_cache_cleaner.h @@ -13,8 +13,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL namespace Storage { namespace Cache { namespace details { + class CleanerObject; -} // namespace details class Cleaner { public: @@ -31,5 +31,6 @@ private: }; +} // namespace details } // namespace Cache } // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_compactor.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_compactor.cpp new file mode 100644 index 0000000000..981ae015b1 --- /dev/null +++ b/Telegram/SourceFiles/storage/cache/storage_cache_compactor.cpp @@ -0,0 +1,53 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#include "storage/cache/storage_cache_compactor.h" + +namespace Storage { +namespace Cache { +namespace details { + +class CompactorObject { +public: + CompactorObject( + crl::weak_on_queue weak, + const QString &path, + crl::weak_on_queue database); + +private: + void start(); + + crl::weak_on_queue _weak; + crl::weak_on_queue _database; + QString _path; + +}; + +CompactorObject::CompactorObject( + crl::weak_on_queue weak, + const QString &path, + crl::weak_on_queue database) +: _weak(std::move(weak)) +, _database(std::move(database)) +, _path(path) { + start(); +} + +void CompactorObject::start() { +} + +Compactor::Compactor( + const QString &path, + crl::weak_on_queue database) +: _wrapped(path, std::move(database)) { +} + +Compactor::~Compactor() = default; + +} // namespace details +} // namespace Cache +} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_compactor.h b/Telegram/SourceFiles/storage/cache/storage_cache_compactor.h new file mode 100644 index 0000000000..8abf7ee397 --- /dev/null +++ b/Telegram/SourceFiles/storage/cache/storage_cache_compactor.h @@ -0,0 +1,36 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#pragma once + +#include "storage/cache/storage_cache_types.h" +#include + +namespace Storage { +namespace Cache { +namespace details { + +class CompactorObject; +class DatabaseObject; + +class Compactor { +public: + Compactor( + const QString &path, + crl::weak_on_queue database); + + ~Compactor(); + +private: + using Implementation = CompactorObject; + crl::object_on_queue _wrapped; + +}; + +} // namespace details +} // namespace Cache +} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp index e7762a66a7..98583f1883 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp @@ -7,1274 +7,10 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL */ #include "storage/cache/storage_cache_database.h" -#include "storage/cache/storage_cache_cleaner.h" -#include "storage/storage_encryption.h" -#include "storage/storage_encrypted_file.h" -#include "base/flat_set.h" -#include "base/flat_map.h" -#include "base/algorithm.h" -#include "base/concurrent_timer.h" -#include -#include -#include -#include -#include - -namespace std { - -template <> -struct hash { - size_t operator()(const Storage::Cache::Key &key) const { - return (hash()(key.high) ^ hash()(key.low)); - } -}; - -} // namespace std +#include "storage/cache/storage_cache_database_object.h" namespace Storage { namespace Cache { -namespace details { -namespace { - -using RecordType = uint8; -using PlaceId = std::array; -using EntrySize = std::array; -using RecordsCount = std::array; - -constexpr auto kRecordSizeUnknown = size_type(-1); -constexpr auto kRecordSizeInvalid = size_type(-2); -constexpr auto kBundledRecordsLimit = (1 << (RecordsCount().size() * 8)); -constexpr auto kDataSizeLimit = (1 << (EntrySize().size() * 8)); - -template -constexpr auto GoodForEncryption = ((sizeof(Record) & 0x0F) == 0); - -template -Packed ReadTo(size_type count) { - Expects(count >= 0 && count < (1 << (Packed().size() * 8))); - - auto result = Packed(); - for (auto &element : result) { - element = uint8(count & 0xFF); - count >>= 8; - } - return result; -} - -template -size_type ReadFrom(Packed count) { - auto result = size_type(); - RecordsCount a; - for (auto &element : (count | ranges::view::reverse)) { - result <<= 8; - result |= size_type(element); - } - return result; -} - -uint32 CountChecksum(bytes::const_span data) { - const auto seed = uint32(0); - return XXH32(data.data(), data.size(), seed); -} - -QString PlaceFromId(PlaceId place) { - auto result = QString(); - result.reserve(15); - const auto pushDigit = [&](uint8 digit) { - const auto hex = (digit < 0x0A) - ? char('0' + digit) - : char('A' + (digit - 0x0A)); - result.push_back(hex); - }; - const auto push = [&](uint8 value) { - pushDigit(value & 0x0F); - pushDigit(value >> 4); - }; - for (auto i = 0; i != place.size(); ++i) { - push(place[i]); - if (!i) { - result.push_back('/'); - } - } - return result; -} - -int32 GetUnixtime() { - return std::max(int32(time(nullptr)), 1); -} - -enum class Format : uint32 { - Format_0, -}; - -struct BasicHeader { - BasicHeader(); - - static constexpr auto kTrackEstimatedTime = 0x01U; - - Format format : 8; - uint32 flags : 24; - uint32 systemTime = 0; - uint32 reserved1 = 0; - uint32 reserved2 = 0; -}; -static_assert(GoodForEncryption); - -BasicHeader::BasicHeader() -: format(Format::Format_0) -, flags(0) { -} - -struct EstimatedTimePoint { - uint32 system = 0; - uint32 relativeAdvancement = 0; -}; - -struct Store { - static constexpr auto kType = RecordType(0x01); - - RecordType type = kType; - uint8 tag = 0; - EntrySize size = { { 0 } }; - PlaceId place = { { 0 } }; - uint32 checksum = 0; - Key key; -}; -static_assert(GoodForEncryption); - -struct StoreWithTime : Store { - EstimatedTimePoint time; - uint32 reserved1 = 0; - uint32 reserved2 = 0; -}; -static_assert(GoodForEncryption); - -struct MultiStoreHeader { - static constexpr auto kType = RecordType(0x02); - - explicit MultiStoreHeader(size_type count = 0); - - RecordType type = kType; - RecordsCount count = { { 0 } }; - uint32 reserved1 = 0; - uint32 reserved2 = 0; - uint32 reserved3 = 0; -}; -using MultiStorePart = Store; -using MultiStoreWithTimePart = StoreWithTime; -static_assert(GoodForEncryption); - -MultiStoreHeader::MultiStoreHeader(size_type count) -: type(kType) -, count(ReadTo(count)) { - Expects(count >= 0 && count < kBundledRecordsLimit); -} - -struct MultiRemoveHeader { - static constexpr auto kType = RecordType(0x03); - - explicit MultiRemoveHeader(size_type count = 0); - - RecordType type = kType; - RecordsCount count = { { 0 } }; - uint32 reserved1 = 0; - uint32 reserved2 = 0; - uint32 reserved3 = 0; -}; -struct MultiRemovePart { - Key key; -}; -static_assert(GoodForEncryption); -static_assert(GoodForEncryption); - -MultiRemoveHeader::MultiRemoveHeader(size_type count) -: type(kType) -, count(ReadTo(count)) { - Expects(count >= 0 && count < kBundledRecordsLimit); -} - -struct MultiAccessHeader { - static constexpr auto kType = RecordType(0x04); - - explicit MultiAccessHeader( - EstimatedTimePoint time, - size_type count = 0); - - RecordType type = kType; - RecordsCount count = { { 0 } }; - EstimatedTimePoint time; - uint32 reserved = 0; -}; -struct MultiAccessPart { - Key key; -}; -static_assert(GoodForEncryption); -static_assert(GoodForEncryption); - -MultiAccessHeader::MultiAccessHeader( - EstimatedTimePoint time, - size_type count) -: type(kType) -, count(ReadTo(count)) -, time(time) { - Expects(count >= 0 && count < kBundledRecordsLimit); -} - -} // namespace - -class Database { -public: - using Wrapper = Cache::Database; - using Settings = Wrapper::Settings; - Database( - crl::weak_on_queue weak, - const QString &path, - const Settings &settings); - - void open(EncryptionKey key, FnMut done); - void close(FnMut done); - - void put(const Key &key, QByteArray value, FnMut done); - void get(const Key &key, FnMut done); - void remove(const Key &key, FnMut done = nullptr); - - void clear(FnMut done); - - ~Database(); - -private: - struct Entry { - Entry() = default; - Entry( - PlaceId place, - uint8 tag, - uint32 checksum, - size_type size, - int64 useTime); - - int64 useTime = 0; - size_type size = 0; - uint32 checksum = 0; - PlaceId place = { { 0 } }; - uint8 tag = 0; - }; - struct CleanerWrap { - std::unique_ptr object; - base::binary_guard guard; - }; - using Map = std::unordered_map; - - template - void invokeCallback(Callback &&callback, Args &&...args); - - Error ioError(const QString &path) const; - - QString computePath(Version version) const; - QString binlogPath(Version version) const; - QString binlogPath() const; - QString binlogFilename() const; - File::Result openBinlog( - Version version, - File::Mode mode, - EncryptionKey &key); - bool readHeader(); - bool writeHeader(); - - void readBinlog(); - size_type readBinlogRecords(bytes::const_span data); - size_type readBinlogRecordSize(bytes::const_span data) const; - bool readBinlogRecord(bytes::const_span data); - template - bool readRecordStoreGeneric(bytes::const_span data); - bool readRecordStore(bytes::const_span data); - template - bool readRecordMultiStoreGeneric(bytes::const_span data); - bool readRecordMultiStore(bytes::const_span data); - bool readRecordMultiRemove(bytes::const_span data); - bool readRecordMultiAccess(bytes::const_span data); - template - bool processRecordStoreGeneric( - const RecordStore *record, - Postprocess &&postprocess); - bool processRecordStore(const Store *record, std::is_class); - bool processRecordStore( - const StoreWithTime *record, - std::is_class); - - void adjustRelativeTime(); - void startDelayedPruning(); - int64 countRelativeTime() const; - EstimatedTimePoint countTimePoint() const; - void applyTimePoint(EstimatedTimePoint time); - int64 pruneBeforeTime() const; - void prune(); - void collectTimePrune( - base::flat_set &stale, - int64 &staleTotalSize); - void collectSizePrune( - base::flat_set &stale, - int64 &staleTotalSize); - - void setMapEntry(const Key &key, Entry &&entry); - void eraseMapEntry(const Map::const_iterator &i); - void recordEntryAccess(const Key &key); - QByteArray readValueData(PlaceId place, size_type size) const; - - Version findAvailableVersion() const; - QString versionPath() const; - bool writeVersion(Version version); - Version readVersion() const; - - QString placePath(PlaceId place) const; - bool isFreePlace(PlaceId place) const; - - template - base::optional writeKeyPlaceGeneric( - StoreRecord &&record, - const Key &key, - const QByteArray &value, - uint32 checksum); - base::optional writeKeyPlace( - const Key &key, - const QByteArray &value, - uint32 checksum); - void writeMultiRemoveLazy(); - void writeMultiRemove(); - void writeMultiAccessLazy(); - void writeMultiAccess(); - void writeMultiAccessBlock(); - void writeBundlesLazy(); - void writeBundles(); - - void createCleaner(); - void cleanerDone(Error error); - - crl::weak_on_queue _weak; - QString _base, _path; - const Settings _settings; - EncryptionKey _key; - File _binlog; - Map _map; - std::set _removing; - std::set _accessed; - - int64 _relativeTime = 0; - int64 _timeCorrection = 0; - uint32 _latestSystemTime = 0; - - int64 _totalSize = 0; - int64 _minimalEntryTime = 0; - size_type _entriesWithMinimalTimeCount = 0; - - base::ConcurrentTimer _writeBundlesTimer; - base::ConcurrentTimer _pruneTimer; - - CleanerWrap _cleaner; - -}; - -Database::Entry::Entry( - PlaceId place, - uint8 tag, - uint32 checksum, - size_type size, - int64 useTime) -: useTime(useTime) -, size(size) -, checksum(checksum) -, tag(tag) -, place(place) { -} - -Database::Database( - crl::weak_on_queue weak, - const QString &path, - const Settings &settings) -: _weak(std::move(weak)) -, _base(ComputeBasePath(path)) -, _settings(settings) -, _writeBundlesTimer(_weak, [=] { writeBundles(); }) -, _pruneTimer(_weak, [=] { prune(); }) { - Expects(_settings.maxDataSize < kDataSizeLimit); - Expects(_settings.maxBundledRecords < kBundledRecordsLimit); - Expects(!_settings.totalTimeLimit - || _settings.totalTimeLimit > 0); - Expects(!_settings.totalSizeLimit - || _settings.totalSizeLimit > _settings.maxDataSize); -} - -template -void Database::invokeCallback(Callback &&callback, Args &&...args) { - if (callback) { - callback(std::move(args)...); - } -} - -Error Database::ioError(const QString &path) const { - return { Error::Type::IO, path }; -} - -void Database::open(EncryptionKey key, FnMut done) { - const auto version = readVersion(); - const auto result = openBinlog(version, File::Mode::ReadAppend, key); - switch (result) { - case File::Result::Success: - invokeCallback(done, Error::NoError()); - break; - case File::Result::LockFailed: - invokeCallback( - done, - Error{ Error::Type::LockFailed, binlogPath(version) }); - break; - case File::Result::WrongKey: - invokeCallback( - done, - Error{ Error::Type::WrongKey, binlogPath(version) }); - break; - case File::Result::Failed: { - const auto available = findAvailableVersion(); - if (writeVersion(available)) { - const auto open = openBinlog(available, File::Mode::Write, key); - if (open == File::Result::Success) { - invokeCallback(done, Error::NoError()); - } else { - invokeCallback(done, ioError(binlogPath(available))); - } - } else { - invokeCallback(done, ioError(versionPath())); - } - } break; - default: Unexpected("Result from Database::openBinlog."); - } -} - -QString Database::computePath(Version version) const { - return _base + QString::number(version) + '/'; -} - -QString Database::binlogFilename() const { - return QStringLiteral("binlog"); -} - -QString Database::binlogPath(Version version) const { - return computePath(version) + binlogFilename(); -} - -QString Database::binlogPath() const { - return _path + binlogFilename(); -} - -File::Result Database::openBinlog( - Version version, - File::Mode mode, - EncryptionKey &key) { - const auto path = binlogPath(version); - const auto result = _binlog.open(path, mode, key); - if (result == File::Result::Success) { - const auto headerRequired = (mode == File::Mode::Read) - || (mode == File::Mode::ReadAppend && _binlog.size() > 0); - if (headerRequired ? readHeader() : writeHeader()) { - _path = computePath(version); - _key = std::move(key); - createCleaner(); - readBinlog(); - } else { - return File::Result::Failed; - } - } - return result; -} - -bool Database::readHeader() { - auto header = BasicHeader(); - if (_binlog.read(bytes::object_as_span(&header)) != sizeof(header)) { - return false; - } else if (header.format != Format::Format_0) { - return false; - } else if (_settings.trackEstimatedTime - != !!(header.flags & header.kTrackEstimatedTime)) { - return false; - } - _relativeTime = _latestSystemTime = header.systemTime; - return true; -} - -bool Database::writeHeader() { - auto header = BasicHeader(); - const auto now = _settings.trackEstimatedTime ? GetUnixtime() : 0; - _relativeTime = _latestSystemTime = header.systemTime = now; - if (_settings.trackEstimatedTime) { - header.flags |= header.kTrackEstimatedTime; - } - return _binlog.write(bytes::object_as_span(&header)); -} - -void Database::readBinlog() { - auto data = bytes::vector(_settings.readBlockSize); - const auto full = bytes::make_span(data); - auto notParsedBytes = index_type(0); - while (true) { - Assert(notParsedBytes < full.size()); - const auto readBytes = _binlog.read(full.subspan(notParsedBytes)); - if (!readBytes) { - break; - } - notParsedBytes += readBytes; - const auto bytes = full.subspan(0, notParsedBytes); - const auto parsedTill = readBinlogRecords(bytes); - if (parsedTill == kRecordSizeInvalid) { - break; - } - Assert(parsedTill >= 0 && parsedTill <= notParsedBytes); - notParsedBytes -= parsedTill; - if (parsedTill > 0 && parsedTill < bytes.size()) { - bytes::move(full, bytes.subspan(parsedTill)); - } - } - _binlog.seek(_binlog.offset() - notParsedBytes); - - adjustRelativeTime(); - startDelayedPruning(); -} - -int64 Database::countRelativeTime() const { - const auto now = GetUnixtime(); - const auto delta = std::max(int64(now) - int64(_latestSystemTime), 0LL); - return _relativeTime + delta; -} - -int64 Database::pruneBeforeTime() const { - return _settings.totalTimeLimit - ? (countRelativeTime() - _settings.totalTimeLimit) - : 0LL; -} - -void Database::startDelayedPruning() { - if (!_settings.trackEstimatedTime || _map.empty()) { - return; - } - const auto pruning = [&] { - if (_settings.totalSizeLimit > 0 - && _totalSize > _settings.totalSizeLimit) { - return true; - } else if (_minimalEntryTime != 0 - && _minimalEntryTime <= pruneBeforeTime()) { - return true; - } - return false; - }(); - if (pruning) { - if (!_pruneTimer.isActive() - || _pruneTimer.remainingTime() > _settings.pruneTimeout) { - _pruneTimer.callOnce(_settings.pruneTimeout); - } - } else if (_minimalEntryTime != 0) { - const auto before = pruneBeforeTime(); - const auto seconds = (_minimalEntryTime - before); - if (!_pruneTimer.isActive()) { - _pruneTimer.callOnce(std::min( - seconds * crl::time_type(1000), - _settings.maxPruneCheckTimeout)); - } - } -} - -void Database::prune() { - auto stale = base::flat_set(); - auto staleTotalSize = int64(); - collectTimePrune(stale, staleTotalSize); - collectSizePrune(stale, staleTotalSize); - for (const auto &key : stale) { - remove(key); - } - startDelayedPruning(); -} - -void Database::collectTimePrune( - base::flat_set &stale, - int64 &staleTotalSize) { - if (!_settings.totalTimeLimit) { - return; - } - const auto before = pruneBeforeTime(); - if (!_minimalEntryTime || _minimalEntryTime > before) { - return; - } - _minimalEntryTime = 0; - _entriesWithMinimalTimeCount = 0; - for (const auto &[key, entry] : _map) { - if (entry.useTime <= before) { - stale.emplace(key); - staleTotalSize += entry.size; - } else if (!_minimalEntryTime - || _minimalEntryTime > entry.useTime) { - _minimalEntryTime = entry.useTime; - _entriesWithMinimalTimeCount = 1; - } else if (_minimalEntryTime == entry.useTime) { - ++_entriesWithMinimalTimeCount; - } - } -} - -void Database::collectSizePrune( - base::flat_set &stale, - int64 &staleTotalSize) { - const auto removeSize = (_settings.totalSizeLimit > 0) - ? (_totalSize - staleTotalSize - _settings.totalSizeLimit) - : 0; - if (removeSize <= 0) { - return; - } - - using Bucket = std::pair; - auto oldest = base::flat_multi_map< - int64, - const Bucket*, - std::greater<>>(); - auto oldestTotalSize = int64(); - - const auto canRemoveFirst = [&](const Entry &adding) { - const auto totalSizeAfterAdd = oldestTotalSize + adding.size; - const auto &first = oldest.begin()->second->second; - return (adding.useTime <= first.useTime - && (totalSizeAfterAdd - removeSize >= first.size)); - }; - - for (const auto &bucket : _map) { - const auto &entry = bucket.second; - if (stale.contains(bucket.first)) { - continue; - } - const auto add = (oldestTotalSize < removeSize) - ? true - : (entry.useTime < oldest.begin()->second->second.useTime); - if (!add) { - continue; - } - while (!oldest.empty() && canRemoveFirst(entry)) { - oldestTotalSize -= oldest.begin()->second->second.size; - oldest.erase(oldest.begin()); - } - oldestTotalSize += entry.size; - oldest.emplace(entry.useTime, &bucket); - } - - for (const auto &pair : oldest) { - stale.emplace(pair.second->first); - } - staleTotalSize += oldestTotalSize; -} - -void Database::adjustRelativeTime() { - if (!_settings.trackEstimatedTime) { - return; - } - const auto now = GetUnixtime(); - if (now < _latestSystemTime) { - writeMultiAccessBlock(); - } -} - -size_type Database::readBinlogRecords(bytes::const_span data) { - auto result = 0; - while (true) { - const auto size = readBinlogRecordSize(data); - if (size == kRecordSizeUnknown || size > data.size()) { - return result; - } else if (size == kRecordSizeInvalid || !readBinlogRecord(data)) { - return (result > 0) ? result : kRecordSizeInvalid; - } else { - result += size; - data = data.subspan(size); - } - } -} - -size_type Database::readBinlogRecordSize(bytes::const_span data) const { - if (data.empty()) { - return kRecordSizeUnknown; - } - - switch (static_cast(data[0])) { - case Store::kType: - return _settings.trackEstimatedTime - ? sizeof(StoreWithTime) - : sizeof(Store); - - case MultiStoreHeader::kType: - if (data.size() >= sizeof(MultiStoreHeader)) { - const auto header = reinterpret_cast( - data.data()); - const auto count = ReadFrom(header->count); - const auto size = _settings.trackEstimatedTime - ? sizeof(MultiStoreWithTimePart) - : sizeof(MultiStorePart); - return (count > 0 && count < _settings.maxBundledRecords) - ? (sizeof(MultiStoreHeader) + count * size) - : kRecordSizeInvalid; - } - return kRecordSizeUnknown; - - case MultiRemoveHeader::kType: - if (data.size() >= sizeof(MultiRemoveHeader)) { - const auto header = reinterpret_cast( - data.data()); - const auto count = ReadFrom(header->count); - return (count > 0 && count < _settings.maxBundledRecords) - ? (sizeof(MultiRemoveHeader) - + count * sizeof(MultiRemovePart)) - : kRecordSizeInvalid; - } - return kRecordSizeUnknown; - - case MultiAccessHeader::kType: - if (!_settings.trackEstimatedTime) { - return kRecordSizeInvalid; - } else if (data.size() >= sizeof(MultiAccessHeader)) { - const auto header = reinterpret_cast( - data.data()); - const auto count = ReadFrom(header->count); - return (count >= 0 && count < _settings.maxBundledRecords) - ? (sizeof(MultiAccessHeader) - + count * sizeof(MultiAccessPart)) - : kRecordSizeInvalid; - } - return kRecordSizeUnknown; - - } - return kRecordSizeInvalid; -} - -bool Database::readBinlogRecord(bytes::const_span data) { - Expects(!data.empty()); - - switch (static_cast(data[0])) { - case Store::kType: - return readRecordStore(data); - - case MultiStoreHeader::kType: - return readRecordMultiStore(data); - - case MultiRemoveHeader::kType: - return readRecordMultiRemove(data); - - case MultiAccessHeader::kType: - return readRecordMultiAccess(data); - - } - Unexpected("Bad type in Database::readBinlogRecord."); -} - -template -bool Database::readRecordStoreGeneric(bytes::const_span data) { - Expects(data.size() >= sizeof(RecordStore)); - - return processRecordStore( - reinterpret_cast(data.data()), - std::is_class{}); -} - -template -bool Database::processRecordStoreGeneric( - const RecordStore *record, - Postprocess &&postprocess) { - const auto size = ReadFrom(record->size); - if (size <= 0 || size > _settings.maxDataSize) { - return false; - } - auto entry = Entry( - record->place, - record->tag, - record->checksum, - size, - _relativeTime); - if (!postprocess(entry, record)) { - return false; - } - setMapEntry(record->key, std::move(entry)); - return true; -} - -bool Database::processRecordStore( - const Store *record, - std::is_class) { - const auto postprocess = [](auto&&...) { return true; }; - return processRecordStoreGeneric(record, postprocess); -} - -bool Database::processRecordStore( - const StoreWithTime *record, - std::is_class) { - const auto postprocess = [&]( - Entry &entry, - not_null record) { - applyTimePoint(record->time); - entry.useTime = _relativeTime; - return true; - }; - return processRecordStoreGeneric(record, postprocess); -} - -bool Database::readRecordStore(bytes::const_span data) { - if (!_settings.trackEstimatedTime) { - return readRecordStoreGeneric(data); - } - return readRecordStoreGeneric(data); -} - -template -bool Database::readRecordMultiStoreGeneric(bytes::const_span data) { - Expects(data.size() >= sizeof(MultiStoreHeader)); - - const auto bytes = data.data(); - const auto record = reinterpret_cast(bytes); - const auto count = ReadFrom(record->count); - Assert(data.size() >= sizeof(MultiStoreHeader) - + count * sizeof(StorePart)); - const auto parts = gsl::make_span( - reinterpret_cast( - bytes + sizeof(MultiStoreHeader)), - count); - for (const auto &part : parts) { - if (!processRecordStore(&part, std::is_class{})) { - return false; - } - } - return true; -} - -bool Database::readRecordMultiStore(bytes::const_span data) { - if (!_settings.trackEstimatedTime) { - return readRecordMultiStoreGeneric(data); - } - return readRecordMultiStoreGeneric(data); -} - -void Database::setMapEntry(const Key &key, Entry &&entry) { - auto &already = _map[key]; - _totalSize += entry.size - already.size; - if (entry.useTime != 0 - && (entry.useTime < _minimalEntryTime || !_minimalEntryTime)) { - _minimalEntryTime = entry.useTime; - _entriesWithMinimalTimeCount = 1; - } else if (_minimalEntryTime != 0 && already.useTime != entry.useTime) { - if (entry.useTime == _minimalEntryTime) { - Assert(_entriesWithMinimalTimeCount > 0); - ++_entriesWithMinimalTimeCount; - } else if (already.useTime == _minimalEntryTime) { - Assert(_entriesWithMinimalTimeCount > 0); - --_entriesWithMinimalTimeCount; - } - } - already = std::move(entry); -} - -void Database::eraseMapEntry(const Map::const_iterator &i) { - if (i != end(_map)) { - const auto &entry = i->second; - _totalSize -= entry.size; - if (_minimalEntryTime != 0 && entry.useTime == _minimalEntryTime) { - Assert(_entriesWithMinimalTimeCount > 0); - --_entriesWithMinimalTimeCount; - } - _map.erase(i); - } -} - -bool Database::readRecordMultiRemove(bytes::const_span data) { - Expects(data.size() >= sizeof(MultiRemoveHeader)); - - const auto bytes = data.data(); - const auto record = reinterpret_cast(bytes); - const auto count = ReadFrom(record->count); - Assert(data.size() >= sizeof(MultiRemoveHeader) - + count * sizeof(MultiRemovePart)); - const auto parts = gsl::make_span( - reinterpret_cast( - bytes + sizeof(MultiRemoveHeader)), - count); - for (const auto &part : parts) { - eraseMapEntry(_map.find(part.key)); - } - return true; -} - -EstimatedTimePoint Database::countTimePoint() const { - const auto now = std::max(GetUnixtime(), 1); - const auto delta = std::max(int64(now) - int64(_latestSystemTime), 0LL); - auto result = EstimatedTimePoint(); - result.system = now; - result.relativeAdvancement = std::min( - delta, - int64(_settings.maxTimeAdvancement)); - return result; -} - -void Database::applyTimePoint(EstimatedTimePoint time) { - _relativeTime += time.relativeAdvancement; - _latestSystemTime = time.system; -} - -bool Database::readRecordMultiAccess(bytes::const_span data) { - Expects(data.size() >= sizeof(MultiAccessHeader)); - Expects(_settings.trackEstimatedTime); - - const auto bytes = data.data(); - const auto record = reinterpret_cast(bytes); - if (record->time.relativeAdvancement > _settings.maxTimeAdvancement) { - return false; - } - applyTimePoint(record->time); - const auto count = ReadFrom(record->count); - Assert(data.size() >= sizeof(MultiAccessHeader) - + count * sizeof(MultiAccessPart)); - const auto parts = gsl::make_span( - reinterpret_cast( - bytes + sizeof(MultiAccessHeader)), - count); - for (const auto &part : parts) { - if (const auto i = _map.find(part.key); i != end(_map)) { - i->second.useTime = _relativeTime; - } - } - return true; -} - -void Database::close(FnMut done) { - writeBundles(); - _cleaner = CleanerWrap(); - _binlog.close(); - invokeCallback(done); -} - -void Database::put( - const Key &key, - QByteArray value, - FnMut done) { - if (value.isEmpty()) { - remove(key, [done = std::move(done)]() mutable { - done(Error::NoError()); - }); - return; - } - _removing.erase(key); - - const auto checksum = CountChecksum(bytes::make_span(value)); - const auto maybepath = writeKeyPlace(key, value, checksum); - if (!maybepath) { - invokeCallback(done, ioError(binlogPath())); - return; - } else if (maybepath->isEmpty()) { - // Nothing changed. - invokeCallback(done, Error::NoError()); - recordEntryAccess(key); - return; - } - const auto path = *maybepath; - File data; - const auto result = data.open(path, File::Mode::Write, _key); - switch (result) { - case File::Result::Failed: - invokeCallback(done, ioError(path)); - break; - - case File::Result::LockFailed: - invokeCallback(done, Error{ Error::Type::LockFailed, path }); - break; - - case File::Result::Success: { - const auto success = data.writeWithPadding(bytes::make_span(value)); - if (!success) { - data.close(); - remove(key, nullptr); - invokeCallback(done, ioError(path)); - } else { - data.flush(); - invokeCallback(done, Error::NoError()); - startDelayedPruning(); - } - } break; - - default: Unexpected("Result in Database::put."); - } -} - -template -base::optional Database::writeKeyPlaceGeneric( - StoreRecord &&record, - const Key &key, - const QByteArray &value, - uint32 checksum) { - Expects(value.size() <= _settings.maxDataSize); - - const auto size = size_type(value.size()); - record.key = key; - record.size = ReadTo(size); - record.checksum = checksum; - if (const auto i = _map.find(key); i != end(_map)) { - const auto &already = i->second; - if (already.tag == record.tag - && already.size == size - && already.checksum == checksum - && readValueData(already.place, size) == value) { - return QString(); - } - record.place = already.place; - } else { - do { - bytes::set_random(bytes::object_as_span(&record.place)); - } while (!isFreePlace(record.place)); - } - const auto result = placePath(record.place); - auto writeable = record; - const auto success = _binlog.write(bytes::object_as_span(&writeable)); - if (!success) { - return QString(); - } - _binlog.flush(); - readRecordStore(bytes::object_as_span(&record)); - return result; -} - -base::optional Database::writeKeyPlace( - const Key &key, - const QByteArray &data, - uint32 checksum) { - if (!_settings.trackEstimatedTime) { - return writeKeyPlaceGeneric(Store(), key, data, checksum); - } - auto record = StoreWithTime(); - record.time = countTimePoint(); - if (record.time.relativeAdvancement * crl::time_type(1000) - < _settings.writeBundleDelay) { - // We don't want to produce a lot of unique relativeTime values. - // So if change in it is not large we stick to the old value. - record.time.system = _latestSystemTime; - record.time.relativeAdvancement = 0; - } - return writeKeyPlaceGeneric(std::move(record), key, data, checksum); -} - -void Database::get(const Key &key, FnMut done) { - if (_removing.find(key) != end(_removing)) { - invokeCallback(done, QByteArray()); - return; - } - const auto i = _map.find(key); - if (i == _map.end()) { - invokeCallback(done, QByteArray()); - return; - } - const auto &entry = i->second; - - auto result = readValueData(entry.place, entry.size); - if (result.isEmpty()) { - invokeCallback(done, QByteArray()); - } else if (CountChecksum(bytes::make_span(result)) != entry.checksum) { - invokeCallback(done, QByteArray()); - } else { - invokeCallback(done, std::move(result)); - recordEntryAccess(key); - } -} - -QByteArray Database::readValueData(PlaceId place, size_type size) const { - const auto path = placePath(place); - File data; - const auto result = data.open(path, File::Mode::Read, _key); - switch (result) { - case File::Result::Failed: - case File::Result::WrongKey: return QByteArray(); - case File::Result::Success: { - auto result = QByteArray(size, Qt::Uninitialized); - const auto bytes = bytes::make_span(result); - const auto read = data.readWithPadding(bytes); - if (read != size) { - return QByteArray(); - } - return result; - } break; - default: Unexpected("Result in Database::get."); - } -} - -void Database::recordEntryAccess(const Key &key) { - if (!_settings.trackEstimatedTime) { - return; - } - _accessed.emplace(key); - writeMultiAccessLazy(); - startDelayedPruning(); -} - -void Database::remove(const Key &key, FnMut done) { - const auto i = _map.find(key); - if (i != _map.end()) { - _removing.emplace(key); - writeMultiRemoveLazy(); - - const auto path = placePath(i->second.place); - eraseMapEntry(i); - QFile(path).remove(); - } - invokeCallback(done); -} - -void Database::writeBundlesLazy() { - if (!_writeBundlesTimer.isActive()) { - _writeBundlesTimer.callOnce(_settings.writeBundleDelay); - } -} - -void Database::writeMultiRemoveLazy() { - if (_removing.size() == _settings.maxBundledRecords) { - writeMultiRemove(); - } else { - writeBundlesLazy(); - } -} - -void Database::writeMultiRemove() { - Expects(_removing.size() <= _settings.maxBundledRecords); - - if (_removing.empty()) { - return; - } - const auto size = _removing.size(); - auto header = MultiRemoveHeader(size); - auto list = std::vector(); - list.reserve(size); - for (const auto &key : base::take(_removing)) { - list.push_back({ key }); - } - if (_binlog.write(bytes::object_as_span(&header))) { - _binlog.write(bytes::make_span(list)); - _binlog.flush(); - } -} - -void Database::writeMultiAccessLazy() { - if (_accessed.size() == _settings.maxBundledRecords) { - writeMultiAccess(); - } else { - writeBundlesLazy(); - } -} - -void Database::writeMultiAccess() { - if (!_accessed.empty()) { - writeMultiAccessBlock(); - } -} - -void Database::writeMultiAccessBlock() { - Expects(_settings.trackEstimatedTime); - Expects(_accessed.size() <= _settings.maxBundledRecords); - - const auto time = countTimePoint(); - const auto size = _accessed.size(); - auto header = MultiAccessHeader(time, size); - auto list = std::vector(); - if (size > 0) { - list.reserve(size); - for (const auto &key : base::take(_accessed)) { - list.push_back({ key }); - } - } - applyTimePoint(time); - for (const auto &entry : list) { - if (const auto i = _map.find(entry.key); i != end(_map)) { - i->second.useTime = _relativeTime; - } - } - - if (_binlog.write(bytes::object_as_span(&header))) { - if (size > 0) { - _binlog.write(bytes::make_span(list)); - } - _binlog.flush(); - } -} - -void Database::writeBundles() { - writeMultiRemove(); - if (_settings.trackEstimatedTime) { - writeMultiAccess(); - } -} - -void Database::createCleaner() { - auto [left, right] = base::make_binary_guard(); - _cleaner.guard = std::move(left); - auto done = [weak = _weak](Error error) { - weak.with([=](Database &that) { - that.cleanerDone(error); - }); - }; - _cleaner.object = std::make_unique( - _base, - std::move(right), - std::move(done)); -} - -void Database::cleanerDone(Error error) { - _cleaner = CleanerWrap(); -} - -void Database::clear(FnMut done) { - Expects(_key.empty()); - - const auto version = findAvailableVersion(); - invokeCallback( - done, - writeVersion(version) ? Error::NoError() : ioError(versionPath())); -} - -Database::~Database() { - close(nullptr); -} - -auto Database::findAvailableVersion() const -> Version { - const auto entries = QDir(_base).entryList( - QDir::Dirs | QDir::NoDotAndDotDot); - auto versions = base::flat_set(); - for (const auto entry : entries) { - versions.insert(entry.toInt()); - } - auto result = Version(); - for (const auto version : versions) { - if (result != version) { - break; - } - ++result; - } - return result; -} - -QString Database::versionPath() const { - return VersionFilePath(_base); -} - -bool Database::writeVersion(Version version) { - return WriteVersionValue(_base, version); -} - -auto Database::readVersion() const -> Version { - if (const auto result = ReadVersionValue(_base)) { - return *result; - } - return Version(); -} - -QString Database::placePath(PlaceId place) const { - return _path + PlaceFromId(place); -} - -bool Database::isFreePlace(PlaceId place) const { - return !QFile(placePath(place)).exists(); -} - -} // namespace details Database::Database(const QString &path, const Settings &settings) : _wrapped(path, settings) { diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database.h b/Telegram/SourceFiles/storage/cache/storage_cache_database.h index 764a8f3a6d..df0df67ccf 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database.h +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database.h @@ -17,25 +17,11 @@ namespace Storage { class EncryptionKey; namespace Cache { namespace details { -class Database; +class DatabaseObject; } // namespace details -struct Key { - uint64 high = 0; - uint64 low = 0; -}; - -inline bool operator==(const Key &a, const Key &b) { - return (a.high == b.high) && (a.low == b.low); -} - -inline bool operator!=(const Key &a, const Key &b) { - return !(a == b); -} - -inline bool operator<(const Key &a, const Key &b) { - return std::tie(a.high, a.low) < std::tie(b.high, b.low); -} +using Key = details::Key; +using Error = details::Error; class Database { public: @@ -45,6 +31,9 @@ public: size_type maxDataSize = 10 * 1024 * 1024; crl::time_type writeBundleDelay = 15 * 60 * crl::time_type(1000); + int64 compactAfterExcess = 8 * 1024 * 1024; + int64 compactAfterFullSize = 0; + bool trackEstimatedTime = true; int64 totalSizeLimit = 1024 * 1024 * 1024; size_type totalTimeLimit = 30 * 86400; // One month in seconds. @@ -66,7 +55,7 @@ public: ~Database(); private: - using Implementation = details::Database; + using Implementation = details::DatabaseObject; crl::object_on_queue _wrapped; }; diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp new file mode 100644 index 0000000000..41e2c6288b --- /dev/null +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp @@ -0,0 +1,1008 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#include "storage/cache/storage_cache_database_object.h" + +#include "storage/cache/storage_cache_cleaner.h" +#include "storage/cache/storage_cache_compactor.h" +#include "storage/storage_encryption.h" +#include "storage/storage_encrypted_file.h" +#include "base/flat_map.h" +#include "base/algorithm.h" +#include +#include +#include +#include +#include + +namespace Storage { +namespace Cache { +namespace details { +namespace { + +uint32 CountChecksum(bytes::const_span data) { + const auto seed = uint32(0); + return XXH32(data.data(), data.size(), seed); +} + +QString PlaceFromId(PlaceId place) { + auto result = QString(); + result.reserve(15); + const auto pushDigit = [&](uint8 digit) { + const auto hex = (digit < 0x0A) + ? char('0' + digit) + : char('A' + (digit - 0x0A)); + result.push_back(hex); + }; + const auto push = [&](uint8 value) { + pushDigit(value & 0x0F); + pushDigit(value >> 4); + }; + for (auto i = 0; i != place.size(); ++i) { + push(place[i]); + if (!i) { + result.push_back('/'); + } + } + return result; +} + +int32 GetUnixtime() { + return std::max(int32(time(nullptr)), 1); +} + +} // namespace + +DatabaseObject::Entry::Entry( + PlaceId place, + uint8 tag, + uint32 checksum, + size_type size, + int64 useTime) +: useTime(useTime) +, size(size) +, checksum(checksum) +, tag(tag) +, place(place) { +} + +DatabaseObject::DatabaseObject( + crl::weak_on_queue weak, + const QString &path, + const Settings &settings) +: _weak(std::move(weak)) +, _base(ComputeBasePath(path)) +, _settings(settings) +, _writeBundlesTimer(_weak, [=] { writeBundles(); checkCompactor(); }) +, _pruneTimer(_weak, [=] { prune(); }) { + Expects(_settings.maxDataSize < kDataSizeLimit); + Expects(_settings.maxBundledRecords < kBundledRecordsLimit); + Expects(!_settings.totalTimeLimit + || _settings.totalTimeLimit > 0); + Expects(!_settings.totalSizeLimit + || _settings.totalSizeLimit > _settings.maxDataSize); +} + +template +void DatabaseObject::invokeCallback(Callback &&callback, Args &&...args) { + if (callback) { + callback(std::move(args)...); + } +} + +Error DatabaseObject::ioError(const QString &path) const { + return { Error::Type::IO, path }; +} + +void DatabaseObject::open(EncryptionKey key, FnMut done) { + const auto version = readVersion(); + const auto result = openBinlog(version, File::Mode::ReadAppend, key); + switch (result) { + case File::Result::Success: + invokeCallback(done, Error::NoError()); + break; + case File::Result::LockFailed: + invokeCallback( + done, + Error{ Error::Type::LockFailed, binlogPath(version) }); + break; + case File::Result::WrongKey: + invokeCallback( + done, + Error{ Error::Type::WrongKey, binlogPath(version) }); + break; + case File::Result::Failed: { + const auto available = findAvailableVersion(); + if (writeVersion(available)) { + const auto open = openBinlog(available, File::Mode::Write, key); + if (open == File::Result::Success) { + invokeCallback(done, Error::NoError()); + } else { + invokeCallback(done, ioError(binlogPath(available))); + } + } else { + invokeCallback(done, ioError(versionPath())); + } + } break; + default: Unexpected("Result from DatabaseObject::openBinlog."); + } +} + +QString DatabaseObject::computePath(Version version) const { + return _base + QString::number(version) + '/'; +} + +QString DatabaseObject::binlogFilename() const { + return QStringLiteral("binlog"); +} + +QString DatabaseObject::binlogPath(Version version) const { + return computePath(version) + binlogFilename(); +} + +QString DatabaseObject::binlogPath() const { + return _path + binlogFilename(); +} + +File::Result DatabaseObject::openBinlog( + Version version, + File::Mode mode, + EncryptionKey &key) { + const auto path = binlogPath(version); + const auto result = _binlog.open(path, mode, key); + if (result == File::Result::Success) { + const auto headerRequired = (mode == File::Mode::Read) + || (mode == File::Mode::ReadAppend && _binlog.size() > 0); + if (headerRequired ? readHeader() : writeHeader()) { + _path = computePath(version); + _key = std::move(key); + createCleaner(); + readBinlog(); + } else { + return File::Result::Failed; + } + } + return result; +} + +bool DatabaseObject::readHeader() { + auto header = BasicHeader(); + if (_binlog.read(bytes::object_as_span(&header)) != sizeof(header)) { + return false; + } else if (header.format != Format::Format_0) { + return false; + } else if (_settings.trackEstimatedTime + != !!(header.flags & header.kTrackEstimatedTime)) { + return false; + } + _relativeTime = _latestSystemTime = header.systemTime; + return true; +} + +bool DatabaseObject::writeHeader() { + auto header = BasicHeader(); + const auto now = _settings.trackEstimatedTime ? GetUnixtime() : 0; + _relativeTime = _latestSystemTime = header.systemTime = now; + if (_settings.trackEstimatedTime) { + header.flags |= header.kTrackEstimatedTime; + } + return _binlog.write(bytes::object_as_span(&header)); +} + +void DatabaseObject::readBinlog() { + auto data = bytes::vector(_settings.readBlockSize); + const auto full = bytes::make_span(data); + auto notParsedBytes = index_type(0); + while (true) { + Assert(notParsedBytes < full.size()); + const auto readBytes = _binlog.read(full.subspan(notParsedBytes)); + if (!readBytes) { + break; + } + notParsedBytes += readBytes; + const auto bytes = full.subspan(0, notParsedBytes); + const auto parsedTill = readBinlogRecords(bytes); + if (parsedTill == kRecordSizeInvalid) { + break; + } + Assert(parsedTill >= 0 && parsedTill <= notParsedBytes); + notParsedBytes -= parsedTill; + if (parsedTill > 0 && parsedTill < bytes.size()) { + bytes::move(full, bytes.subspan(parsedTill)); + } + } + _binlog.seek(_binlog.offset() - notParsedBytes); + + adjustRelativeTime(); + optimize(); +} + +int64 DatabaseObject::countRelativeTime() const { + const auto now = GetUnixtime(); + const auto delta = std::max(int64(now) - int64(_latestSystemTime), 0LL); + return _relativeTime + delta; +} + +int64 DatabaseObject::pruneBeforeTime() const { + return _settings.totalTimeLimit + ? (countRelativeTime() - _settings.totalTimeLimit) + : 0LL; +} + +void DatabaseObject::optimize() { + if (!startDelayedPruning()) { + checkCompactor(); + } +} + +bool DatabaseObject::startDelayedPruning() { + if (!_settings.trackEstimatedTime || _map.empty()) { + return false; + } + const auto pruning = [&] { + if (_settings.totalSizeLimit > 0 + && _totalSize > _settings.totalSizeLimit) { + return true; + } else if (_minimalEntryTime != 0 + && _minimalEntryTime <= pruneBeforeTime()) { + return true; + } + return false; + }(); + if (pruning) { + if (!_pruneTimer.isActive() + || _pruneTimer.remainingTime() > _settings.pruneTimeout) { + _pruneTimer.callOnce(_settings.pruneTimeout); + } + return true; + } else if (_minimalEntryTime != 0) { + const auto before = pruneBeforeTime(); + const auto seconds = (_minimalEntryTime - before); + if (!_pruneTimer.isActive()) { + _pruneTimer.callOnce(std::min( + seconds * crl::time_type(1000), + _settings.maxPruneCheckTimeout)); + } + } + return false; +} + +void DatabaseObject::prune() { + auto stale = base::flat_set(); + auto staleTotalSize = int64(); + collectTimePrune(stale, staleTotalSize); + collectSizePrune(stale, staleTotalSize); + for (const auto &key : stale) { + remove(key); + } + optimize(); +} + +void DatabaseObject::collectTimePrune( + base::flat_set &stale, + int64 &staleTotalSize) { + if (!_settings.totalTimeLimit) { + return; + } + const auto before = pruneBeforeTime(); + if (!_minimalEntryTime || _minimalEntryTime > before) { + return; + } + _minimalEntryTime = 0; + _entriesWithMinimalTimeCount = 0; + for (const auto &[key, entry] : _map) { + if (entry.useTime <= before) { + stale.emplace(key); + staleTotalSize += entry.size; + } else if (!_minimalEntryTime + || _minimalEntryTime > entry.useTime) { + _minimalEntryTime = entry.useTime; + _entriesWithMinimalTimeCount = 1; + } else if (_minimalEntryTime == entry.useTime) { + ++_entriesWithMinimalTimeCount; + } + } +} + +void DatabaseObject::collectSizePrune( + base::flat_set &stale, + int64 &staleTotalSize) { + const auto removeSize = (_settings.totalSizeLimit > 0) + ? (_totalSize - staleTotalSize - _settings.totalSizeLimit) + : 0; + if (removeSize <= 0) { + return; + } + + using Bucket = std::pair; + auto oldest = base::flat_multi_map< + int64, + const Bucket*, + std::greater<>>(); + auto oldestTotalSize = int64(); + + const auto canRemoveFirst = [&](const Entry &adding) { + const auto totalSizeAfterAdd = oldestTotalSize + adding.size; + const auto &first = oldest.begin()->second->second; + return (adding.useTime <= first.useTime + && (totalSizeAfterAdd - removeSize >= first.size)); + }; + + for (const auto &bucket : _map) { + const auto &entry = bucket.second; + if (stale.contains(bucket.first)) { + continue; + } + const auto add = (oldestTotalSize < removeSize) + ? true + : (entry.useTime < oldest.begin()->second->second.useTime); + if (!add) { + continue; + } + while (!oldest.empty() && canRemoveFirst(entry)) { + oldestTotalSize -= oldest.begin()->second->second.size; + oldest.erase(oldest.begin()); + } + oldestTotalSize += entry.size; + oldest.emplace(entry.useTime, &bucket); + } + + for (const auto &pair : oldest) { + stale.emplace(pair.second->first); + } + staleTotalSize += oldestTotalSize; +} + +void DatabaseObject::adjustRelativeTime() { + if (!_settings.trackEstimatedTime) { + return; + } + const auto now = GetUnixtime(); + if (now < _latestSystemTime) { + writeMultiAccessBlock(); + } +} + +size_type DatabaseObject::readBinlogRecords(bytes::const_span data) { + auto result = 0; + while (true) { + const auto size = readBinlogRecordSize(data); + if (size == kRecordSizeUnknown || size > data.size()) { + return result; + } else if (size == kRecordSizeInvalid + || !readBinlogRecord(data.subspan(0, size))) { + return (result > 0) ? result : kRecordSizeInvalid; + } else { + result += size; + data = data.subspan(size); + } + } +} + +size_type DatabaseObject::readBinlogRecordSize(bytes::const_span data) const { + if (data.empty()) { + return kRecordSizeUnknown; + } + + switch (static_cast(data[0])) { + case Store::kType: + return storeRecordSize(); + + case MultiStoreHeader::kType: + if (data.size() >= sizeof(MultiStoreHeader)) { + const auto header = reinterpret_cast( + data.data()); + const auto count = ReadFrom(header->count); + const auto size = _settings.trackEstimatedTime + ? sizeof(MultiStoreWithTimePart) + : sizeof(MultiStorePart); + return (count > 0 && count < _settings.maxBundledRecords) + ? (sizeof(MultiStoreHeader) + count * size) + : kRecordSizeInvalid; + } + return kRecordSizeUnknown; + + case MultiRemoveHeader::kType: + if (data.size() >= sizeof(MultiRemoveHeader)) { + const auto header = reinterpret_cast( + data.data()); + const auto count = ReadFrom(header->count); + return (count > 0 && count < _settings.maxBundledRecords) + ? (sizeof(MultiRemoveHeader) + + count * sizeof(MultiRemovePart)) + : kRecordSizeInvalid; + } + return kRecordSizeUnknown; + + case MultiAccessHeader::kType: + if (!_settings.trackEstimatedTime) { + return kRecordSizeInvalid; + } else if (data.size() >= sizeof(MultiAccessHeader)) { + const auto header = reinterpret_cast( + data.data()); + const auto count = ReadFrom(header->count); + return (count >= 0 && count < _settings.maxBundledRecords) + ? (sizeof(MultiAccessHeader) + + count * sizeof(MultiAccessPart)) + : kRecordSizeInvalid; + } + return kRecordSizeUnknown; + + } + return kRecordSizeInvalid; +} + +bool DatabaseObject::readBinlogRecord(bytes::const_span data) { + Expects(!data.empty()); + + switch (static_cast(data[0])) { + case Store::kType: + return readRecordStore(data); + + case MultiStoreHeader::kType: + return readRecordMultiStore(data); + + case MultiRemoveHeader::kType: + return readRecordMultiRemove(data); + + case MultiAccessHeader::kType: + return readRecordMultiAccess(data); + + } + Unexpected("Bad type in DatabaseObject::readBinlogRecord."); +} + +template +bool DatabaseObject::readRecordStoreGeneric(bytes::const_span data) { + Expects(data.size() == sizeof(RecordStore)); + + return processRecordStore( + reinterpret_cast(data.data()), + std::is_class{}); +} + +template +bool DatabaseObject::processRecordStoreGeneric( + const RecordStore *record, + Postprocess &&postprocess) { + const auto size = ReadFrom(record->size); + if (size <= 0 || size > _settings.maxDataSize) { + return false; + } + auto entry = Entry( + record->place, + record->tag, + record->checksum, + size, + _relativeTime); + if (!postprocess(entry, record)) { + return false; + } + setMapEntry(record->key, std::move(entry)); + return true; +} + +bool DatabaseObject::processRecordStore( + const Store *record, + std::is_class) { + const auto postprocess = [](auto&&...) { return true; }; + return processRecordStoreGeneric(record, postprocess); +} + +bool DatabaseObject::processRecordStore( + const StoreWithTime *record, + std::is_class) { + const auto postprocess = [&]( + Entry &entry, + not_null record) { + applyTimePoint(record->time); + entry.useTime = _relativeTime; + return true; + }; + return processRecordStoreGeneric(record, postprocess); +} + +bool DatabaseObject::readRecordStore(bytes::const_span data) { + if (!_settings.trackEstimatedTime) { + return readRecordStoreGeneric(data); + } + return readRecordStoreGeneric(data); +} + +template +bool DatabaseObject::readRecordMultiStoreGeneric(bytes::const_span data) { + Expects(data.size() >= sizeof(MultiStoreHeader)); + + const auto bytes = data.data(); + const auto record = reinterpret_cast(bytes); + const auto count = ReadFrom(record->count); + Assert(data.size() == sizeof(MultiStoreHeader) + + count * sizeof(StorePart)); + const auto parts = gsl::make_span( + reinterpret_cast( + bytes + sizeof(MultiStoreHeader)), + count); + for (const auto &part : parts) { + if (!processRecordStore(&part, std::is_class{})) { + return false; + } + } + return true; +} + +bool DatabaseObject::readRecordMultiStore(bytes::const_span data) { + if (!_settings.trackEstimatedTime) { + return readRecordMultiStoreGeneric(data); + } + return readRecordMultiStoreGeneric(data); +} + +size_type DatabaseObject::storeRecordSize() const { + return _settings.trackEstimatedTime + ? sizeof(StoreWithTime) + : sizeof(Store); +} + +void DatabaseObject::setMapEntry(const Key &key, Entry &&entry) { + auto &already = _map[key]; + _totalSize += entry.size - already.size; + if (already.size != 0) { + _binlogExcessLength += storeRecordSize(); + } + if (entry.useTime != 0 + && (entry.useTime < _minimalEntryTime || !_minimalEntryTime)) { + _minimalEntryTime = entry.useTime; + _entriesWithMinimalTimeCount = 1; + } else if (_minimalEntryTime != 0 && already.useTime != entry.useTime) { + if (entry.useTime == _minimalEntryTime) { + Assert(_entriesWithMinimalTimeCount > 0); + ++_entriesWithMinimalTimeCount; + } else if (already.useTime == _minimalEntryTime) { + Assert(_entriesWithMinimalTimeCount > 0); + --_entriesWithMinimalTimeCount; + } + } + already = std::move(entry); +} + +void DatabaseObject::eraseMapEntry(const Map::const_iterator &i) { + if (i != end(_map)) { + const auto &entry = i->second; + _totalSize -= entry.size; + if (_minimalEntryTime != 0 && entry.useTime == _minimalEntryTime) { + Assert(_entriesWithMinimalTimeCount > 0); + --_entriesWithMinimalTimeCount; + } + _map.erase(i); + } +} + +bool DatabaseObject::readRecordMultiRemove(bytes::const_span data) { + Expects(data.size() >= sizeof(MultiRemoveHeader)); + + const auto bytes = data.data(); + const auto record = reinterpret_cast(bytes); + const auto count = ReadFrom(record->count); + Assert(data.size() == sizeof(MultiRemoveHeader) + + count * sizeof(MultiRemovePart)); + const auto parts = gsl::make_span( + reinterpret_cast( + bytes + sizeof(MultiRemoveHeader)), + count); + for (const auto &part : parts) { + if (const auto i = _map.find(part.key); i != end(_map)) { + eraseMapEntry(i); + } + } + _binlogExcessLength += data.size(); + return true; +} + +EstimatedTimePoint DatabaseObject::countTimePoint() const { + const auto now = std::max(GetUnixtime(), 1); + const auto delta = std::max(int64(now) - int64(_latestSystemTime), 0LL); + auto result = EstimatedTimePoint(); + result.system = now; + result.relativeAdvancement = std::min( + delta, + int64(_settings.maxTimeAdvancement)); + return result; +} + +void DatabaseObject::applyTimePoint(EstimatedTimePoint time) { + _relativeTime += time.relativeAdvancement; + _latestSystemTime = time.system; +} + +bool DatabaseObject::readRecordMultiAccess(bytes::const_span data) { + Expects(data.size() >= sizeof(MultiAccessHeader)); + Expects(_settings.trackEstimatedTime); + + const auto bytes = data.data(); + const auto record = reinterpret_cast(bytes); + if (record->time.relativeAdvancement > _settings.maxTimeAdvancement) { + return false; + } + applyTimePoint(record->time); + const auto count = ReadFrom(record->count); + Assert(data.size() == sizeof(MultiAccessHeader) + + count * sizeof(MultiAccessPart)); + const auto parts = gsl::make_span( + reinterpret_cast( + bytes + sizeof(MultiAccessHeader)), + count); + for (const auto &part : parts) { + if (const auto i = _map.find(part.key); i != end(_map)) { + i->second.useTime = _relativeTime; + } + } + _binlogExcessLength += data.size(); + return true; +} + +void DatabaseObject::close(FnMut done) { + writeBundles(); + _cleaner = CleanerWrap(); + _compactor = nullptr; + _binlog.close(); + invokeCallback(done); + _map.clear(); + _binlogExcessLength = 0; +} + +void DatabaseObject::put( + const Key &key, + QByteArray value, + FnMut done) { + if (value.isEmpty()) { + remove(key, [done = std::move(done)]() mutable { + done(Error::NoError()); + }); + return; + } + _removing.erase(key); + + const auto checksum = CountChecksum(bytes::make_span(value)); + const auto maybepath = writeKeyPlace(key, value, checksum); + if (!maybepath) { + invokeCallback(done, ioError(binlogPath())); + return; + } else if (maybepath->isEmpty()) { + // Nothing changed. + invokeCallback(done, Error::NoError()); + recordEntryAccess(key); + return; + } + const auto path = *maybepath; + File data; + const auto result = data.open(path, File::Mode::Write, _key); + switch (result) { + case File::Result::Failed: + invokeCallback(done, ioError(path)); + break; + + case File::Result::LockFailed: + invokeCallback(done, Error{ Error::Type::LockFailed, path }); + break; + + case File::Result::Success: { + const auto success = data.writeWithPadding(bytes::make_span(value)); + if (!success) { + data.close(); + remove(key, nullptr); + invokeCallback(done, ioError(path)); + } else { + data.flush(); + invokeCallback(done, Error::NoError()); + optimize(); + } + } break; + + default: Unexpected("Result in DatabaseObject::put."); + } +} + +template +base::optional DatabaseObject::writeKeyPlaceGeneric( + StoreRecord &&record, + const Key &key, + const QByteArray &value, + uint32 checksum) { + Expects(value.size() <= _settings.maxDataSize); + + const auto size = size_type(value.size()); + record.key = key; + record.size = ReadTo(size); + record.checksum = checksum; + if (const auto i = _map.find(key); i != end(_map)) { + const auto &already = i->second; + if (already.tag == record.tag + && already.size == size + && already.checksum == checksum + && readValueData(already.place, size) == value) { + return QString(); + } + record.place = already.place; + } else { + do { + bytes::set_random(bytes::object_as_span(&record.place)); + } while (!isFreePlace(record.place)); + } + const auto result = placePath(record.place); + auto writeable = record; + const auto success = _binlog.write(bytes::object_as_span(&writeable)); + if (!success) { + return QString(); + } + _binlog.flush(); + + const auto applied = readRecordStore(bytes::object_as_span(&record)); + Assert(applied); + return result; +} + +base::optional DatabaseObject::writeKeyPlace( + const Key &key, + const QByteArray &data, + uint32 checksum) { + if (!_settings.trackEstimatedTime) { + return writeKeyPlaceGeneric(Store(), key, data, checksum); + } + auto record = StoreWithTime(); + record.time = countTimePoint(); + if (record.time.relativeAdvancement * crl::time_type(1000) + < _settings.writeBundleDelay) { + // We don't want to produce a lot of unique relativeTime values. + // So if change in it is not large we stick to the old value. + record.time.system = _latestSystemTime; + record.time.relativeAdvancement = 0; + } + return writeKeyPlaceGeneric(std::move(record), key, data, checksum); +} + +void DatabaseObject::get(const Key &key, FnMut done) { + if (_removing.find(key) != end(_removing)) { + invokeCallback(done, QByteArray()); + return; + } + const auto i = _map.find(key); + if (i == _map.end()) { + invokeCallback(done, QByteArray()); + return; + } + const auto &entry = i->second; + + auto result = readValueData(entry.place, entry.size); + if (result.isEmpty()) { + invokeCallback(done, QByteArray()); + } else if (CountChecksum(bytes::make_span(result)) != entry.checksum) { + invokeCallback(done, QByteArray()); + } else { + invokeCallback(done, std::move(result)); + recordEntryAccess(key); + } +} + +QByteArray DatabaseObject::readValueData(PlaceId place, size_type size) const { + const auto path = placePath(place); + File data; + const auto result = data.open(path, File::Mode::Read, _key); + switch (result) { + case File::Result::Failed: + case File::Result::WrongKey: return QByteArray(); + case File::Result::Success: { + auto result = QByteArray(size, Qt::Uninitialized); + const auto bytes = bytes::make_span(result); + const auto read = data.readWithPadding(bytes); + if (read != size) { + return QByteArray(); + } + return result; + } break; + default: Unexpected("Result in DatabaseObject::get."); + } +} + +void DatabaseObject::recordEntryAccess(const Key &key) { + if (!_settings.trackEstimatedTime) { + return; + } + _accessed.emplace(key); + writeMultiAccessLazy(); + optimize(); +} + +void DatabaseObject::remove(const Key &key, FnMut done) { + const auto i = _map.find(key); + if (i != _map.end()) { + _removing.emplace(key); + writeMultiRemoveLazy(); + + const auto path = placePath(i->second.place); + eraseMapEntry(i); + QFile(path).remove(); + } + invokeCallback(done); +} + +void DatabaseObject::writeBundlesLazy() { + if (!_writeBundlesTimer.isActive()) { + _writeBundlesTimer.callOnce(_settings.writeBundleDelay); + } +} + +void DatabaseObject::writeMultiRemoveLazy() { + if (_removing.size() == _settings.maxBundledRecords) { + writeMultiRemove(); + } else { + writeBundlesLazy(); + } +} + +void DatabaseObject::writeMultiRemove() { + Expects(_removing.size() <= _settings.maxBundledRecords); + + if (_removing.empty()) { + return; + } + const auto size = _removing.size(); + auto header = MultiRemoveHeader(size); + auto list = std::vector(); + list.reserve(size); + for (const auto &key : base::take(_removing)) { + list.push_back({ key }); + } + if (_binlog.write(bytes::object_as_span(&header))) { + _binlog.write(bytes::make_span(list)); + _binlog.flush(); + + _binlogExcessLength += bytes::object_as_span(&header).size() + + bytes::make_span(list).size(); + } +} + +void DatabaseObject::writeMultiAccessLazy() { + if (_accessed.size() == _settings.maxBundledRecords) { + writeMultiAccess(); + } else { + writeBundlesLazy(); + } +} + +void DatabaseObject::writeMultiAccess() { + if (!_accessed.empty()) { + writeMultiAccessBlock(); + } +} + +void DatabaseObject::writeMultiAccessBlock() { + Expects(_settings.trackEstimatedTime); + Expects(_accessed.size() <= _settings.maxBundledRecords); + + const auto time = countTimePoint(); + const auto size = _accessed.size(); + auto header = MultiAccessHeader(time, size); + auto list = std::vector(); + if (size > 0) { + list.reserve(size); + for (const auto &key : base::take(_accessed)) { + list.push_back({ key }); + } + } + applyTimePoint(time); + for (const auto &entry : list) { + if (const auto i = _map.find(entry.key); i != end(_map)) { + i->second.useTime = _relativeTime; + } + } + + if (_binlog.write(bytes::object_as_span(&header))) { + if (size > 0) { + _binlog.write(bytes::make_span(list)); + } + _binlog.flush(); + + _binlogExcessLength += bytes::object_as_span(&header).size() + + bytes::make_span(list).size(); + } +} + +void DatabaseObject::writeBundles() { + writeMultiRemove(); + if (_settings.trackEstimatedTime) { + writeMultiAccess(); + } +} + +void DatabaseObject::createCleaner() { + auto [left, right] = base::make_binary_guard(); + _cleaner.guard = std::move(left); + auto done = [weak = _weak](Error error) { + weak.with([=](DatabaseObject &that) { + that.cleanerDone(error); + }); + }; + _cleaner.object = std::make_unique( + _base, + std::move(right), + std::move(done)); +} + +void DatabaseObject::cleanerDone(Error error) { + _cleaner = CleanerWrap(); +} + +void DatabaseObject::checkCompactor() { + if (_compactor + || !_settings.compactAfterExcess + || _binlogExcessLength < _settings.compactAfterExcess) { + return; + } else if (_settings.compactAfterFullSize) { + if (_binlogExcessLength * _settings.compactAfterFullSize + < _settings.compactAfterExcess * _binlog.size()) { + return; + } + } + _compactor = std::make_unique(_path, _weak); +} + +void DatabaseObject::clear(FnMut done) { + Expects(_key.empty()); + + const auto version = findAvailableVersion(); + invokeCallback( + done, + writeVersion(version) ? Error::NoError() : ioError(versionPath())); +} + +DatabaseObject::~DatabaseObject() { + close(nullptr); +} + +auto DatabaseObject::findAvailableVersion() const -> Version { + const auto entries = QDir(_base).entryList( + QDir::Dirs | QDir::NoDotAndDotDot); + auto versions = base::flat_set(); + for (const auto entry : entries) { + versions.insert(entry.toInt()); + } + auto result = Version(); + for (const auto version : versions) { + if (result != version) { + break; + } + ++result; + } + return result; +} + +QString DatabaseObject::versionPath() const { + return VersionFilePath(_base); +} + +bool DatabaseObject::writeVersion(Version version) { + return WriteVersionValue(_base, version); +} + +auto DatabaseObject::readVersion() const -> Version { + if (const auto result = ReadVersionValue(_base)) { + return *result; + } + return Version(); +} + +QString DatabaseObject::placePath(PlaceId place) const { + return _path + PlaceFromId(place); +} + +bool DatabaseObject::isFreePlace(PlaceId place) const { + return !QFile(placePath(place)).exists(); +} + +} // namespace details +} // namespace Cache +} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h new file mode 100644 index 0000000000..41eb4bd191 --- /dev/null +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h @@ -0,0 +1,181 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#pragma once + +#include "storage/cache/storage_cache_database.h" +#include "storage/storage_encrypted_file.h" +#include "base/binary_guard.h" +#include "base/concurrent_timer.h" +#include "base/bytes.h" +#include "base/flat_set.h" + +namespace Storage { +namespace Cache { +namespace details { + +class Cleaner; +class Compactor; + +class DatabaseObject { +public: + using Settings = Cache::Database::Settings; + DatabaseObject( + crl::weak_on_queue weak, + const QString &path, + const Settings &settings); + + void open(EncryptionKey key, FnMut done); + void close(FnMut done); + + void put(const Key &key, QByteArray value, FnMut done); + void get(const Key &key, FnMut done); + void remove(const Key &key, FnMut done = nullptr); + + void clear(FnMut done); + + ~DatabaseObject(); + +private: + struct Entry { + Entry() = default; + Entry( + PlaceId place, + uint8 tag, + uint32 checksum, + size_type size, + int64 useTime); + + int64 useTime = 0; + size_type size = 0; + uint32 checksum = 0; + PlaceId place = { { 0 } }; + uint8 tag = 0; + }; + struct CleanerWrap { + std::unique_ptr object; + base::binary_guard guard; + }; + using Map = std::unordered_map; + + template + void invokeCallback(Callback &&callback, Args &&...args); + + Error ioError(const QString &path) const; + + QString computePath(Version version) const; + QString binlogPath(Version version) const; + QString binlogPath() const; + QString binlogFilename() const; + File::Result openBinlog( + Version version, + File::Mode mode, + EncryptionKey &key); + bool readHeader(); + bool writeHeader(); + + void readBinlog(); + size_type readBinlogRecords(bytes::const_span data); + size_type readBinlogRecordSize(bytes::const_span data) const; + bool readBinlogRecord(bytes::const_span data); + template + bool readRecordStoreGeneric(bytes::const_span data); + bool readRecordStore(bytes::const_span data); + template + bool readRecordMultiStoreGeneric(bytes::const_span data); + bool readRecordMultiStore(bytes::const_span data); + bool readRecordMultiRemove(bytes::const_span data); + bool readRecordMultiAccess(bytes::const_span data); + template + bool processRecordStoreGeneric( + const RecordStore *record, + Postprocess &&postprocess); + bool processRecordStore(const Store *record, std::is_class); + bool processRecordStore( + const StoreWithTime *record, + std::is_class); + size_type storeRecordSize() const; + + void optimize(); + void checkCompactor(); + void adjustRelativeTime(); + bool startDelayedPruning(); + int64 countRelativeTime() const; + EstimatedTimePoint countTimePoint() const; + void applyTimePoint(EstimatedTimePoint time); + int64 pruneBeforeTime() const; + void prune(); + void collectTimePrune( + base::flat_set &stale, + int64 &staleTotalSize); + void collectSizePrune( + base::flat_set &stale, + int64 &staleTotalSize); + + void setMapEntry(const Key &key, Entry &&entry); + void eraseMapEntry(const Map::const_iterator &i); + void recordEntryAccess(const Key &key); + QByteArray readValueData(PlaceId place, size_type size) const; + + Version findAvailableVersion() const; + QString versionPath() const; + bool writeVersion(Version version); + Version readVersion() const; + + QString placePath(PlaceId place) const; + bool isFreePlace(PlaceId place) const; + + template + base::optional writeKeyPlaceGeneric( + StoreRecord &&record, + const Key &key, + const QByteArray &value, + uint32 checksum); + base::optional writeKeyPlace( + const Key &key, + const QByteArray &value, + uint32 checksum); + void writeMultiRemoveLazy(); + void writeMultiRemove(); + void writeMultiAccessLazy(); + void writeMultiAccess(); + void writeMultiAccessBlock(); + void writeBundlesLazy(); + void writeBundles(); + + void createCleaner(); + void cleanerDone(Error error); + + crl::weak_on_queue _weak; + QString _base, _path; + const Settings _settings; + EncryptionKey _key; + File _binlog; + Map _map; + std::set _removing; + std::set _accessed; + + int64 _relativeTime = 0; + int64 _timeCorrection = 0; + uint32 _latestSystemTime = 0; + + int64 _binlogExcessLength = 0; + int64 _totalSize = 0; + int64 _minimalEntryTime = 0; + size_type _entriesWithMinimalTimeCount = 0; + + base::ConcurrentTimer _writeBundlesTimer; + base::ConcurrentTimer _pruneTimer; + + CleanerWrap _cleaner; + std::unique_ptr _compactor; + +}; + +} // namespace details +} // namespace Cache +} // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp index d4f06a37de..fa29b0c433 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp +++ b/Telegram/SourceFiles/storage/cache/storage_cache_types.cpp @@ -11,6 +11,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL namespace Storage { namespace Cache { +namespace details { QString ComputeBasePath(const QString &original) { const auto result = QDir(original).absolutePath(); @@ -51,5 +52,32 @@ bool WriteVersionValue(const QString &base, Version value) { return file.flush(); } +BasicHeader::BasicHeader() +: format(Format::Format_0) +, flags(0) { +} + +MultiStoreHeader::MultiStoreHeader(size_type count) +: type(kType) +, count(ReadTo(count)) { + Expects(count >= 0 && count < kBundledRecordsLimit); +} + +MultiRemoveHeader::MultiRemoveHeader(size_type count) +: type(kType) +, count(ReadTo(count)) { + Expects(count >= 0 && count < kBundledRecordsLimit); +} + +MultiAccessHeader::MultiAccessHeader( + EstimatedTimePoint time, + size_type count) +: type(kType) +, count(ReadTo(count)) +, time(time) { + Expects(count >= 0 && count < kBundledRecordsLimit); +} + +} // namespace details } // namespace Cache } // namespace Storage diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_types.h b/Telegram/SourceFiles/storage/cache/storage_cache_types.h index 6e7acebe91..e4339f34a6 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_types.h +++ b/Telegram/SourceFiles/storage/cache/storage_cache_types.h @@ -12,6 +12,24 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL namespace Storage { namespace Cache { +namespace details { + +struct Key { + uint64 high = 0; + uint64 low = 0; +}; + +inline bool operator==(const Key &a, const Key &b) { + return (a.high == b.high) && (a.low == b.low); +} + +inline bool operator!=(const Key &a, const Key &b) { + return !(a == b); +} + +inline bool operator<(const Key &a, const Key &b) { + return std::tie(a.high, a.low) < std::tie(b.high, b.low); +} using Version = int32; @@ -37,5 +55,143 @@ inline Error Error::NoError() { return Error(); } +using RecordType = uint8; +using PlaceId = std::array; +using EntrySize = std::array; +using RecordsCount = std::array; + +template +Packed ReadTo(size_type count) { + Expects(count >= 0 && count < (1 << (Packed().size() * 8))); + + auto result = Packed(); + for (auto &element : result) { + element = uint8(count & 0xFF); + count >>= 8; + } + return result; +} + +template +size_type ReadFrom(Packed count) { + auto result = size_type(); + for (auto &element : (count | ranges::view::reverse)) { + result <<= 8; + result |= size_type(element); + } + return result; +} + +constexpr auto kRecordSizeUnknown = size_type(-1); +constexpr auto kRecordSizeInvalid = size_type(-2); +constexpr auto kBundledRecordsLimit = (1 << (RecordsCount().size() * 8)); +constexpr auto kDataSizeLimit = (1 << (EntrySize().size() * 8)); + +template +constexpr auto GoodForEncryption = ((sizeof(Record) & 0x0F) == 0); + +enum class Format : uint32 { + Format_0, +}; + +struct BasicHeader { + BasicHeader(); + + static constexpr auto kTrackEstimatedTime = 0x01U; + + Format format : 8; + uint32 flags : 24; + uint32 systemTime = 0; + uint32 reserved1 = 0; + uint32 reserved2 = 0; +}; +static_assert(GoodForEncryption); + +struct EstimatedTimePoint { + uint32 system = 0; + uint32 relativeAdvancement = 0; +}; + +struct Store { + static constexpr auto kType = RecordType(0x01); + + RecordType type = kType; + uint8 tag = 0; + EntrySize size = { { 0 } }; + PlaceId place = { { 0 } }; + uint32 checksum = 0; + Key key; +}; +static_assert(GoodForEncryption); + +struct StoreWithTime : Store { + EstimatedTimePoint time; + uint32 reserved1 = 0; + uint32 reserved2 = 0; +}; +static_assert(GoodForEncryption); + +struct MultiStoreHeader { + static constexpr auto kType = RecordType(0x02); + + explicit MultiStoreHeader(size_type count = 0); + + RecordType type = kType; + RecordsCount count = { { 0 } }; + uint32 reserved1 = 0; + uint32 reserved2 = 0; + uint32 reserved3 = 0; +}; +using MultiStorePart = Store; +using MultiStoreWithTimePart = StoreWithTime; +static_assert(GoodForEncryption); + +struct MultiRemoveHeader { + static constexpr auto kType = RecordType(0x03); + + explicit MultiRemoveHeader(size_type count = 0); + + RecordType type = kType; + RecordsCount count = { { 0 } }; + uint32 reserved1 = 0; + uint32 reserved2 = 0; + uint32 reserved3 = 0; +}; +struct MultiRemovePart { + Key key; +}; +static_assert(GoodForEncryption); +static_assert(GoodForEncryption); + +struct MultiAccessHeader { + static constexpr auto kType = RecordType(0x04); + + explicit MultiAccessHeader( + EstimatedTimePoint time, + size_type count = 0); + + RecordType type = kType; + RecordsCount count = { { 0 } }; + EstimatedTimePoint time; + uint32 reserved = 0; +}; +struct MultiAccessPart { + Key key; +}; +static_assert(GoodForEncryption); +static_assert(GoodForEncryption); + +} // namespace details } // namespace Cache } // namespace Storage + +namespace std { + +template <> +struct hash { + size_t operator()(const Storage::Cache::details::Key &key) const { + return (hash()(key.high) ^ hash()(key.low)); + } +}; + +} // namespace std diff --git a/Telegram/gyp/lib_storage.gyp b/Telegram/gyp/lib_storage.gyp index 81bb4be52e..0e01e73f49 100644 --- a/Telegram/gyp/lib_storage.gyp +++ b/Telegram/gyp/lib_storage.gyp @@ -59,8 +59,12 @@ '<(src_loc)/storage/storage_file_lock.h', '<(src_loc)/storage/cache/storage_cache_cleaner.cpp', '<(src_loc)/storage/cache/storage_cache_cleaner.h', + '<(src_loc)/storage/cache/storage_cache_compactor.cpp', + '<(src_loc)/storage/cache/storage_cache_compactor.h', '<(src_loc)/storage/cache/storage_cache_database.cpp', '<(src_loc)/storage/cache/storage_cache_database.h', + '<(src_loc)/storage/cache/storage_cache_database_object.cpp', + '<(src_loc)/storage/cache/storage_cache_database_object.h', '<(src_loc)/storage/cache/storage_cache_types.cpp', '<(src_loc)/storage/cache/storage_cache_types.h', ],