Add a generic BinlogReader.

Telegram/SourceFiles/base/match_method.h

@@ -16,7 +16,8 @@ inline decltype(auto) match_method(
 		Data &&data,
 		Method &&method,
 		Methods &&...methods) {
-	if constexpr (rpl::details::is_callable_plain_v<Method, Data&&>) {
+	using namespace rpl::details;
+	if constexpr (is_callable_plain_v<Method, Data&&>) {
 		return std::forward<Method>(method)(std::forward<Data>(data));
 	} else {
 		return match_method(
@@ -25,4 +26,27 @@ inline decltype(auto) match_method(
 	}
 }
 
+template <
+	typename Data1,
+	typename Data2,
+	typename Method,
+	typename ...Methods>
+inline decltype(auto) match_method2(
+		Data1 &&data1,
+		Data2 &&data2,
+		Method &&method,
+		Methods &&...methods) {
+	using namespace rpl::details;
+	if constexpr (is_callable_plain_v<Method, Data1&&, Data2&&>) {
+		return std::forward<Method>(method)(
+			std::forward<Data1>(data1),
+			std::forward<Data2>(data2));
+	} else {
+		return match_method2(
+			std::forward<Data1>(data1),
+			std::forward<Data2>(data2),
+			std::forward<Methods>(methods)...);
+	}
+}
+
 } // namespace base

Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.cpp

@@ -0,0 +1,89 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/cache/storage_cache_binlog_reader.h"
+
+namespace Storage {
+namespace Cache {
+namespace details {
+
+BinlogWrapper::BinlogWrapper(File &binlog, const Settings &settings)
+: _binlog(binlog)
+, _settings(settings)
+, _till(_binlog.size())
+, _data(_settings.readBlockSize)
+, _full(_data) {
+}
+
+bool BinlogWrapper::failed() const {
+	return _failed;
+}
+
+bool BinlogWrapper::readPart() {
+	if (_finished) {
+		return false;
+	}
+	const auto no = [&] {
+		finish();
+		return false;
+	};
+	const auto offset = _binlog.offset();
+	const auto left = (_till - offset);
+	if (!left) {
+		return no();
+	}
+
+	Assert(_notParsedBytes >= 0 && _notParsedBytes <= _part.size());
+	if (_notParsedBytes > 0 && _notParsedBytes < _part.size()) {
+		bytes::move(_full, _part.subspan(_part.size() - _notParsedBytes));
+	}
+	const auto amount = std::min(
+		left,
+		int64(_full.size() - _notParsedBytes));
+	Assert(amount > 0);
+	const auto readBytes = _binlog.read(
+		_full.subspan(_notParsedBytes, amount));
+	if (!readBytes) {
+		return no();
+	}
+	_notParsedBytes += readBytes;
+	_part = _full.subspan(0, _notParsedBytes);
+	return true;
+}
+
+bytes::const_span BinlogWrapper::readRecord(ReadRecordSize readRecordSize) {
+	if (_finished) {
+		return {};
+	}
+	const auto size = readRecordSize(*this, _part);
+	if (size == kRecordSizeUnknown || size > _part.size()) {
+		return {};
+	} else if (size == kRecordSizeInvalid) {
+		finish();
+		_finished = _failed = true;
+		return {};
+	}
+	Assert(size >= 0 && size <= _notParsedBytes);
+	const auto result = _part.subspan(0, size);
+	_part = _part.subspan(size);
+	_notParsedBytes -= size;
+	return result;
+}
+
+void BinlogWrapper::finish(size_type rollback) {
+	Expects(rollback >= 0);
+
+	if (rollback > 0) {
+		_failed = true;
+		_notParsedBytes += rollback;
+	}
+	_binlog.seek(_binlog.offset() - _notParsedBytes);
+}
+
+} // namespace details
+} // namespace Cache
+} // namespace Storage

Telegram/SourceFiles/storage/cache/storage_cache_binlog_reader.h

@@ -0,0 +1,261 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "storage/cache/storage_cache_types.h"
+#include "storage/storage_encrypted_file.h"
+#include "base/bytes.h"
+#include "base/match_method.h"
+
+namespace Storage {
+namespace Cache {
+namespace details {
+
+template <typename ...Records>
+class BinlogReader;
+
+class BinlogWrapper {
+public:
+	BinlogWrapper(File &binlog, const Settings &settings);
+
+	bool failed() const;
+
+private:
+	template <typename ...Records>
+	friend class BinlogReader;
+
+	bool readPart();
+	void finish(size_type rollback = 0);
+
+	using ReadRecordSize = size_type (*)(
+		const BinlogWrapper &that,
+		bytes::const_span data);
+	bytes::const_span readRecord(ReadRecordSize readRecordSize);
+
+	File &_binlog;
+	Settings _settings;
+
+	int64 _till = 0;
+	bytes::vector _data;
+	bytes::span _full;
+	bytes::span _part;
+	index_type _notParsedBytes = 0;
+	bool _finished = false;
+	bool _failed = false;
+
+};
+
+template <typename ...Records>
+class BinlogReader {
+public:
+	explicit BinlogReader(BinlogWrapper &wrapper);
+
+	template <typename ...Handlers>
+	bool readTillEnd(Handlers &&...handlers);
+
+private:
+	static size_type ReadRecordSize(
+		const BinlogWrapper &that,
+		bytes::const_span data);
+
+	template <typename ...Handlers>
+	bool handleRecord(bytes::const_span data, Handlers &&...handlers) const;
+
+	BinlogWrapper &_wrapper;
+
+};
+
+template <typename Record>
+struct MultiRecord {
+	using true_t = char;
+	using false_t = true_t(&)[2];
+	static_assert(sizeof(true_t) != sizeof(false_t));
+
+	static false_t Check(...);
+	template <typename Test, typename = typename Test::Part>
+	static true_t Check(const Test&);
+
+	static constexpr bool Is = (sizeof(Check(std::declval<Record>()))
+		== sizeof(true_t));
+};
+
+template <typename ...Records>
+struct BinlogReaderRecursive {
+	static void CheckSettings(const Settings &settings) {
+	}
+
+	static size_type ReadRecordSize(
+			RecordType type,
+			bytes::const_span data,
+			size_type partsLimit) {
+		return kRecordSizeInvalid;
+	}
+
+	template <typename ...Handlers>
+	static bool HandleRecord(
+			RecordType type,
+			bytes::const_span data,
+			Handlers &&...handlers) {
+		Unexpected("Bad type in BinlogReaderRecursive::HandleRecord.");
+	}
+};
+
+template <typename Record, typename ...Other>
+struct BinlogReaderRecursive<Record, Other...> {
+	static void CheckSettings(const Settings &settings);
+
+	static size_type ReadRecordSize(
+		RecordType type,
+		bytes::const_span data,
+		size_type partsLimit);
+
+	template <typename ...Handlers>
+	static bool HandleRecord(
+		RecordType type,
+		bytes::const_span data,
+		Handlers &&...handlers);
+};
+
+template <typename Record, typename ...Other>
+inline void BinlogReaderRecursive<Record, Other...>::CheckSettings(
+		const Settings &settings) {
+	static_assert(GoodForEncryption<Record>);
+	if constexpr (MultiRecord<Record>::Is) {
+		using Head = Record;
+		using Part = typename Record::Part;
+		static_assert(GoodForEncryption<Part>);
+		Assert(settings.readBlockSize
+			>= (sizeof(Head)
+				+ settings.maxBundledRecords * sizeof(Part)));
+	} else {
+		Assert(settings.readBlockSize >= sizeof(Record));
+	}
+}
+
+template <typename Record, typename ...Other>
+inline size_type BinlogReaderRecursive<Record, Other...>::ReadRecordSize(
+		RecordType type,
+		bytes::const_span data,
+		size_type partsLimit) {
+	if (type != Record::kType) {
+		return BinlogReaderRecursive<Other...>::ReadRecordSize(
+			type,
+			data,
+			partsLimit);
+	}
+	if constexpr (MultiRecord<Record>::Is) {
+		using Head = Record;
+		using Part = typename Record::Part;
+
+		if (data.size() < sizeof(Head)) {
+			return kRecordSizeUnknown;
+		}
+		const auto head = reinterpret_cast<const Head*>(data.data());
+		const auto count = head->validateCount();
+		return (count >= 0 && count < partsLimit)
+				? (sizeof(Head) + count * sizeof(Part))
+				: kRecordSizeInvalid;
+	} else {
+		return sizeof(Record);
+	}
+}
+
+template <typename Record, typename ...Other>
+template <typename ...Handlers>
+inline bool BinlogReaderRecursive<Record, Other...>::HandleRecord(
+		RecordType type,
+		bytes::const_span data,
+		Handlers &&...handlers) {
+	if (type != Record::kType) {
+		return BinlogReaderRecursive<Other...>::HandleRecord(
+			type,
+			data,
+			std::forward<Handlers>(handlers)...);
+	}
+	if constexpr (MultiRecord<Record>::Is) {
+		using Head = Record;
+		using Part = typename Record::Part;
+
+		Assert(data.size() >= sizeof(Head));
+		const auto bytes = data.data();
+		const auto head = reinterpret_cast<const Head*>(bytes);
+		const auto count = head->validateCount();
+		Assert(data.size() == sizeof(Head) + count * sizeof(Part));
+		const auto parts = gsl::make_span(
+			reinterpret_cast<const Part*>(bytes + sizeof(Head)),
+			count);
+		auto from = std::begin(parts);
+		const auto till = std::end(parts);
+		const auto element = [&] {
+			return (from == till) ? nullptr : &*from++;
+		};
+		return base::match_method2(
+			*head,
+			element,
+			std::forward<Handlers>(handlers)...);
+	} else {
+		Assert(data.size() == sizeof(Record));
+		return base::match_method(
+			*reinterpret_cast<const Record*>(data.data()),
+			std::forward<Handlers>(handlers)...);
+	}
+}
+
+template <typename ...Records>
+BinlogReader<Records...>::BinlogReader(BinlogWrapper &wrapper)
+: _wrapper(wrapper) {
+	BinlogReaderRecursive<Records...>::CheckSettings(_wrapper._settings);
+}
+
+template <typename ...Records>
+template <typename ...Handlers>
+bool BinlogReader<Records...>::readTillEnd(Handlers &&...handlers) {
+	if (!_wrapper.readPart()) {
+		return true;
+	}
+	const auto readRecord = [&] {
+		return _wrapper.readRecord(&BinlogReader::ReadRecordSize);
+	};
+	for (auto bytes = readRecord(); !bytes.empty(); bytes = readRecord()) {
+		if (!handleRecord(bytes, std::forward<Handlers>(handlers)...)) {
+			_wrapper.finish(bytes.size());
+			return true;
+		}
+	}
+	return false;
+}
+
+template <typename ...Records>
+size_type BinlogReader<Records...>::ReadRecordSize(
+		const BinlogWrapper &that,
+		bytes::const_span data) {
+	if (data.empty()) {
+		return kRecordSizeUnknown;
+	}
+	return BinlogReaderRecursive<Records...>::ReadRecordSize(
+		static_cast<RecordType>(data[0]),
+		data,
+		that._settings.maxBundledRecords);
+}
+
+template <typename ...Records>
+template <typename ...Handlers>
+bool BinlogReader<Records...>::handleRecord(
+		bytes::const_span data,
+		Handlers &&...handlers) const {
+	Expects(!data.empty());
+
+	return BinlogReaderRecursive<Records...>::HandleRecord(
+		static_cast<RecordType>(data[0]),
+		data,
+		std::forward<Handlers>(handlers)...);
+}
+
+} // namespace details
+} // namespace Cache
+} // namespace Storage

Telegram/SourceFiles/storage/cache/storage_cache_database.h

@@ -25,22 +25,7 @@ using Error = details::Error;
 
 class Database {
 public:
-	struct Settings {
-		size_type maxBundledRecords = 16 * 1024;
-		size_type readBlockSize = 8 * 1024 * 1024;
-		size_type maxDataSize = 10 * 1024 * 1024;
-		crl::time_type writeBundleDelay = 15 * 60 * crl::time_type(1000);
-
-		int64 compactAfterExcess = 8 * 1024 * 1024;
-		int64 compactAfterFullSize = 0;
-
-		bool trackEstimatedTime = true;
-		int64 totalSizeLimit = 1024 * 1024 * 1024;
-		size_type totalTimeLimit = 30 * 86400; // One month in seconds.
-		size_type maxTimeAdvancement = 365 * 86400; // One year in seconds.
-		crl::time_type pruneTimeout = 5 * crl::time_type(1000);
-		crl::time_type maxPruneCheckTimeout = 3600 * crl::time_type(1000);
-	};
+	using Settings = details::Settings;
 	Database(const QString &path, const Settings &settings);
 
 	void open(EncryptionKey key, FnMut<void(Error)> done);

Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp

@@ -9,6 +9,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 
 #include "storage/cache/storage_cache_cleaner.h"
 #include "storage/cache/storage_cache_compactor.h"
+#include "storage/cache/storage_cache_binlog_reader.h"
 #include "storage/storage_encryption.h"
 #include "storage/storage_encrypted_file.h"
 #include "base/flat_map.h"
@@ -193,30 +194,51 @@ bool DatabaseObject::writeHeader() {
 	return _binlog.write(bytes::object_as_span(&header));
 }
 
-void DatabaseObject::readBinlog() {
-	auto data = bytes::vector(_settings.readBlockSize);
-	const auto full = bytes::make_span(data);
-	auto notParsedBytes = index_type(0);
+template <typename Reader, typename ...Handlers>
+void DatabaseObject::readBinlogHelper(
+		Reader &reader,
+		Handlers &&...handlers) {
 	while (true) {
-		Assert(notParsedBytes < full.size());
-		const auto readBytes = _binlog.read(full.subspan(notParsedBytes));
-		if (!readBytes) {
+		const auto done = reader.readTillEnd(
+			std::forward<Handlers>(handlers)...);
+		if (done) {
 			break;
 		}
-		notParsedBytes += readBytes;
-		const auto bytes = full.subspan(0, notParsedBytes);
-		const auto parsedTill = readBinlogRecords(bytes);
-		if (parsedTill == kRecordSizeInvalid) {
-			break;
-		}
-		Assert(parsedTill >= 0 && parsedTill <= notParsedBytes);
-		notParsedBytes -= parsedTill;
-		if (parsedTill > 0 && parsedTill < bytes.size()) {
-			bytes::move(full, bytes.subspan(parsedTill));
-		}
 	}
-	_binlog.seek(_binlog.offset() - notParsedBytes);
+}
 
+void DatabaseObject::readBinlog() {
+	BinlogWrapper wrapper(_binlog, _settings);
+	if (_settings.trackEstimatedTime) {
+		BinlogReader<
+			StoreWithTime,
+			MultiStoreWithTime,
+			MultiRemove,
+			MultiAccess> reader(wrapper);
+		readBinlogHelper(reader, [&](const StoreWithTime &record) {
+			return processRecordStore(
+				&record,
+				std::is_class<StoreWithTime>{});
+		}, [&](const MultiStoreWithTime &header, const auto &element) {
+			return processRecordMultiStore(header, element);
+		}, [&](const MultiRemove &header, const auto &element) {
+			return processRecordMultiRemove(header, element);
+		}, [&](const MultiAccess &header, const auto &element) {
+			return processRecordMultiAccess(header, element);
+		});
+	} else {
+		BinlogReader<
+			Store,
+			MultiStore,
+			MultiRemove> reader(wrapper);
+		readBinlogHelper(reader, [&](const Store &record) {
+			return processRecordStore(&record, std::is_class<Store>{});
+		}, [&](const MultiStore &header, const auto &element) {
+			return processRecordMultiStore(header, element);
+		}, [&](const MultiRemove &header, const auto &element) {
+			return processRecordMultiRemove(header, element);
+		});
+	}
 	adjustRelativeTime();
 	optimize();
 }
@@ -367,107 +389,9 @@ void DatabaseObject::adjustRelativeTime() {
 	}
 }
 
-size_type DatabaseObject::readBinlogRecords(bytes::const_span data) {
-	auto result = 0;
-	while (true) {
-		const auto size = readBinlogRecordSize(data);
-		if (size == kRecordSizeUnknown || size > data.size()) {
-			return result;
-		} else if (size == kRecordSizeInvalid
-			|| !readBinlogRecord(data.subspan(0, size))) {
-			return (result > 0) ? result : kRecordSizeInvalid;
-		} else {
-			result += size;
-			data = data.subspan(size);
-		}
-	}
-}
-
-size_type DatabaseObject::readBinlogRecordSize(bytes::const_span data) const {
-	if (data.empty()) {
-		return kRecordSizeUnknown;
-	}
-
-	switch (static_cast<RecordType>(data[0])) {
-	case Store::kType:
-		return storeRecordSize();
-
-	case MultiStoreHeader::kType:
-		if (data.size() >= sizeof(MultiStoreHeader)) {
-			const auto header = reinterpret_cast<const MultiStoreHeader*>(
-				data.data());
-			const auto count = ReadFrom(header->count);
-			const auto size = _settings.trackEstimatedTime
-				? sizeof(MultiStoreWithTimePart)
-				: sizeof(MultiStorePart);
-			return (count > 0 && count < _settings.maxBundledRecords)
-				? (sizeof(MultiStoreHeader) + count * size)
-				: kRecordSizeInvalid;
-		}
-		return kRecordSizeUnknown;
-
-	case MultiRemoveHeader::kType:
-		if (data.size() >= sizeof(MultiRemoveHeader)) {
-			const auto header = reinterpret_cast<const MultiRemoveHeader*>(
-				data.data());
-			const auto count = ReadFrom(header->count);
-			return (count > 0 && count < _settings.maxBundledRecords)
-				? (sizeof(MultiRemoveHeader)
-					+ count * sizeof(MultiRemovePart))
-				: kRecordSizeInvalid;
-		}
-		return kRecordSizeUnknown;
-
-	case MultiAccessHeader::kType:
-		if (!_settings.trackEstimatedTime) {
-			return kRecordSizeInvalid;
-		} else if (data.size() >= sizeof(MultiAccessHeader)) {
-			const auto header = reinterpret_cast<const MultiAccessHeader*>(
-				data.data());
-			const auto count = ReadFrom(header->count);
-			return (count >= 0 && count < _settings.maxBundledRecords)
-				? (sizeof(MultiAccessHeader)
-					+ count * sizeof(MultiAccessPart))
-				: kRecordSizeInvalid;
-		}
-		return kRecordSizeUnknown;
-
-	}
-	return kRecordSizeInvalid;
-}
-
-bool DatabaseObject::readBinlogRecord(bytes::const_span data) {
-	Expects(!data.empty());
-
-	switch (static_cast<RecordType>(data[0])) {
-	case Store::kType:
-		return readRecordStore(data);
-
-	case MultiStoreHeader::kType:
-		return readRecordMultiStore(data);
-
-	case MultiRemoveHeader::kType:
-		return readRecordMultiRemove(data);
-
-	case MultiAccessHeader::kType:
-		return readRecordMultiAccess(data);
-
-	}
-	Unexpected("Bad type in DatabaseObject::readBinlogRecord.");
-}
-
-template <typename RecordStore>
-bool DatabaseObject::readRecordStoreGeneric(bytes::const_span data) {
-	Expects(data.size() == sizeof(RecordStore));
-
-	return processRecordStore(
-		reinterpret_cast<const RecordStore*>(data.data()),
-		std::is_class<RecordStore>{});
-}
-
-template <typename RecordStore, typename Postprocess>
+template <typename Record, typename Postprocess>
 bool DatabaseObject::processRecordStoreGeneric(
-		const RecordStore *record,
+		const Record *record,
 		Postprocess &&postprocess) {
 	const auto size = ReadFrom(record->size);
 	if (size <= 0 || size > _settings.maxDataSize) {
@@ -506,52 +430,62 @@ bool DatabaseObject::processRecordStore(
 	return processRecordStoreGeneric(record, postprocess);
 }
 
-bool DatabaseObject::readRecordStore(bytes::const_span data) {
-	if (!_settings.trackEstimatedTime) {
-		return readRecordStoreGeneric<Store>(data);
-	}
-	return readRecordStoreGeneric<StoreWithTime>(data);
-}
-
-template <typename StorePart>
-bool DatabaseObject::readRecordMultiStoreGeneric(bytes::const_span data) {
-	Expects(data.size() >= sizeof(MultiStoreHeader));
-
-	const auto bytes = data.data();
-	const auto record = reinterpret_cast<const MultiStoreHeader*>(bytes);
-	const auto count = ReadFrom(record->count);
-	Assert(data.size() == sizeof(MultiStoreHeader)
-		+ count * sizeof(StorePart));
-	const auto parts = gsl::make_span(
-		reinterpret_cast<const StorePart*>(
-			bytes + sizeof(MultiStoreHeader)),
-		count);
-	for (const auto &part : parts) {
-		if (!processRecordStore(&part, std::is_class<StorePart>{})) {
+template <typename Record, typename GetElement>
+bool DatabaseObject::processRecordMultiStore(
+		const Record &header,
+		const GetElement &element) {
+	while (const auto entry = element()) {
+		if (!processRecordStore(
+				entry,
+				std::is_class<typename Record::Part>{})) {
 			return false;
 		}
 	}
 	return true;
 }
 
-bool DatabaseObject::readRecordMultiStore(bytes::const_span data) {
-	if (!_settings.trackEstimatedTime) {
-		return readRecordMultiStoreGeneric<MultiStorePart>(data);
+template <typename GetElement>
+bool DatabaseObject::processRecordMultiRemove(
+		const MultiRemove &header,
+		const GetElement &element) {
+	_binlogExcessLength += sizeof(header);
+	while (const auto entry = element()) {
+		_binlogExcessLength += sizeof(*entry);
+		if (const auto i = _map.find(*entry); i != end(_map)) {
+			eraseMapEntry(i);
+		}
 	}
-	return readRecordMultiStoreGeneric<MultiStoreWithTimePart>(data);
+	return true;
 }
 
-size_type DatabaseObject::storeRecordSize() const {
-	return _settings.trackEstimatedTime
-		? sizeof(StoreWithTime)
-		: sizeof(Store);
+template <typename GetElement>
+bool DatabaseObject::processRecordMultiAccess(
+		const MultiAccess &header,
+		const GetElement &element) {
+	Expects(_settings.trackEstimatedTime);
+
+	if (header.time.relativeAdvancement > _settings.maxTimeAdvancement) {
+		return false;
+	}
+	applyTimePoint(header.time);
+
+	_binlogExcessLength += sizeof(header);
+	while (const auto entry = element()) {
+		_binlogExcessLength += sizeof(*entry);
+		if (const auto i = _map.find(*entry); i != end(_map)) {
+			i->second.useTime = _relativeTime;
+		}
+	}
+	return true;
 }
 
 void DatabaseObject::setMapEntry(const Key &key, Entry &&entry) {
 	auto &already = _map[key];
 	_totalSize += entry.size - already.size;
 	if (already.size != 0) {
-		_binlogExcessLength += storeRecordSize();
+		_binlogExcessLength += _settings.trackEstimatedTime
+			? sizeof(StoreWithTime)
+			: sizeof(Store);
 	}
 	if (entry.useTime != 0
 		&& (entry.useTime < _minimalEntryTime || !_minimalEntryTime)) {
@@ -581,27 +515,6 @@ void DatabaseObject::eraseMapEntry(const Map::const_iterator &i) {
 	}
 }
 
-bool DatabaseObject::readRecordMultiRemove(bytes::const_span data) {
-	Expects(data.size() >= sizeof(MultiRemoveHeader));
-
-	const auto bytes = data.data();
-	const auto record = reinterpret_cast<const MultiRemoveHeader*>(bytes);
-	const auto count = ReadFrom(record->count);
-	Assert(data.size() == sizeof(MultiRemoveHeader)
-		+ count * sizeof(MultiRemovePart));
-	const auto parts = gsl::make_span(
-		reinterpret_cast<const MultiRemovePart*>(
-			bytes + sizeof(MultiRemoveHeader)),
-		count);
-	for (const auto &part : parts) {
-		if (const auto i = _map.find(part.key); i != end(_map)) {
-			eraseMapEntry(i);
-		}
-	}
-	_binlogExcessLength += data.size();
-	return true;
-}
-
 EstimatedTimePoint DatabaseObject::countTimePoint() const {
 	const auto now = std::max(GetUnixtime(), 1);
 	const auto delta = std::max(int64(now) - int64(_latestSystemTime), 0LL);
@@ -618,32 +531,6 @@ void DatabaseObject::applyTimePoint(EstimatedTimePoint time) {
 	_latestSystemTime = time.system;
 }
 
-bool DatabaseObject::readRecordMultiAccess(bytes::const_span data) {
-	Expects(data.size() >= sizeof(MultiAccessHeader));
-	Expects(_settings.trackEstimatedTime);
-
-	const auto bytes = data.data();
-	const auto record = reinterpret_cast<const MultiAccessHeader*>(bytes);
-	if (record->time.relativeAdvancement > _settings.maxTimeAdvancement) {
-		return false;
-	}
-	applyTimePoint(record->time);
-	const auto count = ReadFrom(record->count);
-	Assert(data.size() == sizeof(MultiAccessHeader)
-		+ count * sizeof(MultiAccessPart));
-	const auto parts = gsl::make_span(
-		reinterpret_cast<const MultiAccessPart*>(
-			bytes + sizeof(MultiAccessHeader)),
-		count);
-	for (const auto &part : parts) {
-		if (const auto i = _map.find(part.key); i != end(_map)) {
-			i->second.useTime = _relativeTime;
-		}
-	}
-	_binlogExcessLength += data.size();
-	return true;
-}
-
 void DatabaseObject::close(FnMut<void()> done) {
 	writeBundles();
 	_cleaner = CleanerWrap();
@@ -740,7 +627,9 @@ base::optional<QString> DatabaseObject::writeKeyPlaceGeneric(
 	}
 	_binlog.flush();
 
-	const auto applied = readRecordStore(bytes::object_as_span(&record));
+	const auto applied = processRecordStore(
+		&record,
+		std::is_class<StoreRecord>{});
 	Assert(applied);
 	return result;
 }
@@ -850,11 +739,11 @@ void DatabaseObject::writeMultiRemove() {
 		return;
 	}
 	const auto size = _removing.size();
-	auto header = MultiRemoveHeader(size);
-	auto list = std::vector<MultiRemovePart>();
+	auto header = MultiRemove(size);
+	auto list = std::vector<MultiRemove::Part>();
 	list.reserve(size);
 	for (const auto &key : base::take(_removing)) {
-		list.push_back({ key });
+		list.push_back(key);
 	}
 	if (_binlog.write(bytes::object_as_span(&header))) {
 		_binlog.write(bytes::make_span(list));
@@ -885,17 +774,17 @@ void DatabaseObject::writeMultiAccessBlock() {
 
 	const auto time = countTimePoint();
 	const auto size = _accessed.size();
-	auto header = MultiAccessHeader(time, size);
-	auto list = std::vector<MultiAccessPart>();
+	auto header = MultiAccess(time, size);
+	auto list = std::vector<MultiAccess::Part>();
 	if (size > 0) {
 		list.reserve(size);
 		for (const auto &key : base::take(_accessed)) {
-			list.push_back({ key });
+			list.push_back(key);
 		}
 	}
 	applyTimePoint(time);
 	for (const auto &entry : list) {
-		if (const auto i = _map.find(entry.key); i != end(_map)) {
+		if (const auto i = _map.find(entry); i != end(_map)) {
 			i->second.useTime = _relativeTime;
 		}
 	}

Telegram/SourceFiles/storage/cache/storage_cache_database_object.h

@@ -79,26 +79,28 @@ private:
 	bool writeHeader();
 
 	void readBinlog();
-	size_type readBinlogRecords(bytes::const_span data);
-	size_type readBinlogRecordSize(bytes::const_span data) const;
-	bool readBinlogRecord(bytes::const_span data);
-	template <typename RecordStore>
-	bool readRecordStoreGeneric(bytes::const_span data);
-	bool readRecordStore(bytes::const_span data);
-	template <typename StorePart>
-	bool readRecordMultiStoreGeneric(bytes::const_span data);
-	bool readRecordMultiStore(bytes::const_span data);
-	bool readRecordMultiRemove(bytes::const_span data);
-	bool readRecordMultiAccess(bytes::const_span data);
-	template <typename RecordStore, typename Postprocess>
+	template <typename Reader, typename ...Handlers>
+	void readBinlogHelper(Reader &reader, Handlers &&...handlers);
+	template <typename Record, typename Postprocess>
 	bool processRecordStoreGeneric(
-		const RecordStore *record,
+		const Record *record,
 		Postprocess &&postprocess);
 	bool processRecordStore(const Store *record, std::is_class<Store>);
 	bool processRecordStore(
 		const StoreWithTime *record,
 		std::is_class<StoreWithTime>);
-	size_type storeRecordSize() const;
+	template <typename Record, typename GetElement>
+	bool processRecordMultiStore(
+		const Record &header,
+		const GetElement &element);
+	template <typename GetElement>
+	bool processRecordMultiRemove(
+		const MultiRemove &header,
+		const GetElement &element);
+	template <typename GetElement>
+	bool processRecordMultiAccess(
+		const MultiAccess &header,
+		const GetElement &element);
 
 	void optimize();
 	void checkCompactor();

Telegram/SourceFiles/storage/cache/storage_cache_types.cpp

@@ -57,19 +57,27 @@ BasicHeader::BasicHeader()
 , flags(0) {
 }
 
-MultiStoreHeader::MultiStoreHeader(size_type count)
+MultiStore::MultiStore(size_type count)
 : type(kType)
 , count(ReadTo<RecordsCount>(count)) {
 	Expects(count >= 0 && count < kBundledRecordsLimit);
 }
 
-MultiRemoveHeader::MultiRemoveHeader(size_type count)
+size_type MultiStore::validateCount() const {
+	return ValidateStrictCount(count);
+}
+
+MultiRemove::MultiRemove(size_type count)
 : type(kType)
 , count(ReadTo<RecordsCount>(count)) {
 	Expects(count >= 0 && count < kBundledRecordsLimit);
 }
 
-MultiAccessHeader::MultiAccessHeader(
+size_type MultiRemove::validateCount() const {
+	return ValidateStrictCount(count);
+}
+
+MultiAccess::MultiAccess(
 	EstimatedTimePoint time,
 	size_type count)
 : type(kType)
@@ -78,6 +86,10 @@ MultiAccessHeader::MultiAccessHeader(
 	Expects(count >= 0 && count < kBundledRecordsLimit);
 }
 
+size_type MultiAccess::validateCount() const {
+	return ReadFrom(count);
+}
+
 } // namespace details
 } // namespace Cache
 } // namespace Storage

Telegram/SourceFiles/storage/cache/storage_cache_types.h

@@ -9,6 +9,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 
 #include "base/basic_types.h"
 #include "base/optional.h"
+#include <crl/crl_time.h>
 
 namespace Storage {
 namespace Cache {
@@ -31,6 +32,23 @@ inline bool operator<(const Key &a, const Key &b) {
 	return std::tie(a.high, a.low) < std::tie(b.high, b.low);
 }
 
+struct Settings {
+	size_type maxBundledRecords = 16 * 1024;
+	size_type readBlockSize = 8 * 1024 * 1024;
+	size_type maxDataSize = 10 * 1024 * 1024;
+	crl::time_type writeBundleDelay = 15 * 60 * crl::time_type(1000);
+
+	int64 compactAfterExcess = 8 * 1024 * 1024;
+	int64 compactAfterFullSize = 0;
+
+	bool trackEstimatedTime = true;
+	int64 totalSizeLimit = 1024 * 1024 * 1024;
+	size_type totalTimeLimit = 30 * 86400; // One month in seconds.
+	size_type maxTimeAdvancement = 365 * 86400; // One year in seconds.
+	crl::time_type pruneTimeout = 5 * crl::time_type(1000);
+	crl::time_type maxPruneCheckTimeout = 3600 * crl::time_type(1000);
+};
+
 using Version = int32;
 
 QString ComputeBasePath(const QString &original);
@@ -61,7 +79,7 @@ using EntrySize = std::array<uint8, 3>;
 using RecordsCount = std::array<uint8, 3>;
 
 template <typename Packed>
-Packed ReadTo(size_type count) {
+inline Packed ReadTo(size_type count) {
 	Expects(count >= 0 && count < (1 << (Packed().size() * 8)));
 
 	auto result = Packed();
@@ -73,7 +91,7 @@ Packed ReadTo(size_type count) {
 }
 
 template <typename Packed>
-size_type ReadFrom(Packed count) {
+inline size_type ReadFrom(const Packed &count) {
 	auto result = size_type();
 	for (auto &element : (count | ranges::view::reverse)) {
 		result <<= 8;
@@ -82,6 +100,12 @@ size_type ReadFrom(Packed count) {
 	return result;
 }
 
+template <typename Packed>
+inline size_type ValidateStrictCount(const Packed &count) {
+	const auto result = ReadFrom(count);
+	return (result != 0) ? result : -1;
+}
+
 constexpr auto kRecordSizeUnknown = size_type(-1);
 constexpr auto kRecordSizeInvalid = size_type(-2);
 constexpr auto kBundledRecordsLimit = (1 << (RecordsCount().size() * 8));
@@ -105,7 +129,6 @@ struct BasicHeader {
 	uint32 reserved1 = 0;
 	uint32 reserved2 = 0;
 };
-static_assert(GoodForEncryption<BasicHeader>);
 
 struct EstimatedTimePoint {
 	uint32 system = 0;
@@ -129,44 +152,44 @@ struct StoreWithTime : Store {
 	uint32 reserved1 = 0;
 	uint32 reserved2 = 0;
 };
-static_assert(GoodForEncryption<StoreWithTime>);
 
-struct MultiStoreHeader {
+struct MultiStore {
 	static constexpr auto kType = RecordType(0x02);
 
-	explicit MultiStoreHeader(size_type count = 0);
+	explicit MultiStore(size_type count = 0);
 
 	RecordType type = kType;
 	RecordsCount count = { { 0 } };
 	uint32 reserved1 = 0;
 	uint32 reserved2 = 0;
 	uint32 reserved3 = 0;
-};
-using MultiStorePart = Store;
-using MultiStoreWithTimePart = StoreWithTime;
-static_assert(GoodForEncryption<MultiStoreHeader>);
 
-struct MultiRemoveHeader {
+	using Part = Store;
+	size_type validateCount() const;
+};
+struct MultiStoreWithTime : MultiStore {
+	using Part = StoreWithTime;
+};
+
+struct MultiRemove {
 	static constexpr auto kType = RecordType(0x03);
 
-	explicit MultiRemoveHeader(size_type count = 0);
+	explicit MultiRemove(size_type count = 0);
 
 	RecordType type = kType;
 	RecordsCount count = { { 0 } };
 	uint32 reserved1 = 0;
 	uint32 reserved2 = 0;
 	uint32 reserved3 = 0;
-};
-struct MultiRemovePart {
-	Key key;
-};
-static_assert(GoodForEncryption<MultiRemoveHeader>);
-static_assert(GoodForEncryption<MultiRemovePart>);
 
-struct MultiAccessHeader {
+	using Part = Key;
+	size_type validateCount() const;
+};
+
+struct MultiAccess {
 	static constexpr auto kType = RecordType(0x04);
 
-	explicit MultiAccessHeader(
+	explicit MultiAccess(
 		EstimatedTimePoint time,
 		size_type count = 0);
 
@@ -174,12 +197,10 @@ struct MultiAccessHeader {
 	RecordsCount count = { { 0 } };
 	EstimatedTimePoint time;
 	uint32 reserved = 0;
+
+	using Part = Key;
+	size_type validateCount() const;
 };
-struct MultiAccessPart {
-	Key key;
-};
-static_assert(GoodForEncryption<MultiAccessHeader>);
-static_assert(GoodForEncryption<MultiAccessPart>);
 
 } // namespace details
 } // namespace Cache

Telegram/SourceFiles/storage/storage_encrypted_file.cpp

@@ -208,7 +208,9 @@ bool File::write(bytes::span bytes) {
 
 	encrypt(bytes);
 	const auto count = writePlain(bytes);
-	if (count != bytes.size()) {
+	if (count == bytes.size()) {
+		_dataSize = std::max(_dataSize, offset());
+	} else {
 		decryptBack(bytes);
 		if (count > 0) {
 			_data.seek(_data.pos() - count);

Telegram/SourceFiles/storage/storage_encrypted_file_tests.cpp

@@ -105,7 +105,7 @@ TEST_CASE("simple encrypted file", "[storage_encrypted_file]") {
 			Key);
 		REQUIRE(result == Storage::File::Result::Success);
 
-		auto data = bytes::vector(16);
+		auto data = bytes::vector(Test1.size());
 		const auto read = file.read(data);
 		REQUIRE(read == data.size());
 		REQUIRE(data == bytes::make_vector(Test1));
@@ -122,24 +122,29 @@ TEST_CASE("simple encrypted file", "[storage_encrypted_file]") {
 			Key);
 		REQUIRE(result == Storage::File::Result::Success);
 		REQUIRE(file.offset() == 0);
+		REQUIRE(file.size() == Test1.size() + Test2.size());
 
-		const auto success1 = file.seek(16);
+		const auto success1 = file.seek(Test1.size());
 		REQUIRE(success1);
-		REQUIRE(file.offset() == 16);
+		REQUIRE(file.offset() == Test1.size());
 
-		auto data = bytes::vector(16);
+		auto data = bytes::vector(Test2.size());
 		const auto read = file.read(data);
 		REQUIRE(read == data.size());
 		REQUIRE(data == bytes::make_vector(Test2));
-		REQUIRE(file.offset() == 32);
+		REQUIRE(file.offset() == Test1.size() + Test2.size());
+		REQUIRE(file.size() == Test1.size() + Test2.size());
 
-		const auto success2 = file.seek(16);
+		const auto success2 = file.seek(Test1.size());
 		REQUIRE(success2);
-		REQUIRE(file.offset() == 16);
+		REQUIRE(file.offset() == Test1.size());
 
 		data = bytes::make_vector(Test1);
-		const auto success3 = file.write(data);
+		const auto success3 = file.write(data) && file.write(data);
 		REQUIRE(success3);
+
+		REQUIRE(file.offset() == 3 * Test1.size());
+		REQUIRE(file.size() == 3 * Test1.size());
 	}
 	SECTION("reading file") {
 		Storage::File file;

Telegram/gyp/lib_storage.gyp

@@ -57,6 +57,8 @@
       '<(src_loc)/storage/storage_file_lock_posix.cpp',
       '<(src_loc)/storage/storage_file_lock_win.cpp',
       '<(src_loc)/storage/storage_file_lock.h',
+      '<(src_loc)/storage/cache/storage_cache_binlog_reader.cpp',
+      '<(src_loc)/storage/cache/storage_cache_binlog_reader.h',
       '<(src_loc)/storage/cache/storage_cache_cleaner.cpp',
       '<(src_loc)/storage/cache/storage_cache_cleaner.h',
       '<(src_loc)/storage/cache/storage_cache_compactor.cpp',