From 5824afa9415645603e7cee50f62464b91d8d92d7 Mon Sep 17 00:00:00 2001 From: John Preston Date: Mon, 27 Aug 2018 10:15:25 +0300 Subject: [PATCH] Add copy() and move() to Cache::Database. --- .../storage/cache/storage_cache_database.cpp | 28 +++- .../storage/cache/storage_cache_database.h | 25 ++-- .../cache/storage_cache_database_object.cpp | 133 +++++++++++++++--- .../cache/storage_cache_database_object.h | 19 ++- .../cache/storage_cache_database_tests.cpp | 2 +- .../storage/cache/storage_cache_types.h | 41 +++--- 6 files changed, 194 insertions(+), 54 deletions(-) diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp index 98583f1883..c0f33e012b 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database.cpp @@ -55,7 +55,7 @@ void Database::get(const Key &key, FnMut done) { }); } -void Database::remove(const Key &key, FnMut done) { +void Database::remove(const Key &key, FnMut done) { _wrapped.with([ key, done = std::move(done) @@ -64,6 +64,32 @@ void Database::remove(const Key &key, FnMut done) { }); } +void Database::copy( + const Key &from, + const Key &to, + FnMut done) { + _wrapped.with([ + from, + to, + done = std::move(done) + ](Implementation &unwrapped) mutable { + unwrapped.copy(from, to, std::move(done)); + }); +} + +void Database::move( + const Key &from, + const Key &to, + FnMut done) { + _wrapped.with([ + from, + to, + done = std::move(done) + ](Implementation &unwrapped) mutable { + unwrapped.move(from, to, std::move(done)); + }); +} + void Database::clear(FnMut done) { _wrapped.with([ done = std::move(done) diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database.h b/Telegram/SourceFiles/storage/cache/storage_cache_database.h index 31bcb41a21..eb4bd4d440 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database.h +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database.h @@ -20,22 +20,31 @@ namespace details { class DatabaseObject; } // namespace details -using Key = details::Key; -using Error = details::Error; - class Database { public: using Settings = details::Settings; Database(const QString &path, const Settings &settings); - void open(EncryptionKey key, FnMut done); - void close(FnMut done); + void open(EncryptionKey key, FnMut done = nullptr); + void close(FnMut done = nullptr); - void put(const Key &key, QByteArray value, FnMut done); + void put( + const Key &key, + QByteArray value, + FnMut done = nullptr); void get(const Key &key, FnMut done); - void remove(const Key &key, FnMut done); + void remove(const Key &key, FnMut done = nullptr); - void clear(FnMut done); + void copy( + const Key &from, + const Key &to, + FnMut done = nullptr); + void move( + const Key &from, + const Key &to, + FnMut done = nullptr); + + void clear(FnMut done = nullptr); ~Database(); diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp index 0f8d6a64e7..830cb0dbd0 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp @@ -320,7 +320,7 @@ void DatabaseObject::prune() { collectTimePrune(stale, staleTotalSize); collectSizePrune(stale, staleTotalSize); for (const auto &key : stale) { - remove(key); + remove(key, nullptr); } optimize(); } @@ -622,8 +622,8 @@ void DatabaseObject::put( QByteArray value, FnMut done) { if (value.isEmpty()) { - remove(key, [done = std::move(done)]() mutable { - done(Error::NoError()); + remove(key, [done = std::move(done)](Error error) mutable { + done(error); }); return; } @@ -732,6 +732,60 @@ base::optional DatabaseObject::writeKeyPlace( return writeKeyPlaceGeneric(std::move(record), key, data, checksum); } +template +Error DatabaseObject::writeExistingPlaceGeneric( + StoreRecord &&record, + const Key &key, + const Entry &entry) { + record.key = key; + record.size = ReadTo(entry.size); + record.checksum = entry.checksum; + if (const auto i = _map.find(key); i != end(_map)) { + const auto &already = i->second; + if (already.tag == record.tag + && already.size == entry.size + && already.checksum == entry.checksum + && (readValueData(already.place, already.size) + == readValueData(entry.place, entry.size))) { + return Error::NoError(); + } + } + record.place = entry.place; + auto writeable = record; + const auto success = _binlog.write(bytes::object_as_span(&writeable)); + if (!success) { + _binlog.close(); + return ioError(binlogPath()); + } + _binlog.flush(); + + const auto applied = processRecordStore( + &record, + std::is_class{}); + Assert(applied); + return Error::NoError(); +} + +Error DatabaseObject::writeExistingPlace( + const Key &key, + const Entry &entry) { + if (!_settings.trackEstimatedTime) { + return writeExistingPlaceGeneric(Store(), key, entry); + } + auto record = StoreWithTime(); + record.time = countTimePoint(); + const auto writing = record.time.getRelative(); + const auto current = _time.getRelative(); + Assert(writing >= current); + if ((writing - current) * crl::time_type(1000) + < _settings.writeBundleDelay) { + // We don't want to produce a lot of unique _time.relative values. + // So if change in it is not large we stick to the old value. + record.time = _time; + } + return writeExistingPlaceGeneric(std::move(record), key, entry); +} + void DatabaseObject::get(const Key &key, FnMut done) { if (_removing.find(key) != end(_removing)) { invokeCallback(done, QByteArray()); @@ -784,7 +838,7 @@ void DatabaseObject::recordEntryAccess(const Key &key) { optimize(); } -void DatabaseObject::remove(const Key &key, FnMut done) { +void DatabaseObject::remove(const Key &key, FnMut done) { const auto i = _map.find(key); if (i != _map.end()) { _removing.emplace(key); @@ -792,9 +846,45 @@ void DatabaseObject::remove(const Key &key, FnMut done) { const auto path = placePath(i->second.place); eraseMapEntry(i); - QFile(path).remove(); + if (QFile(path).remove() || !QFile(path).exists()) { + invokeCallback(done, Error::NoError()); + } else { + invokeCallback(done, ioError(path)); + } + } else { + invokeCallback(done, Error::NoError()); } - invokeCallback(done); +} + +void DatabaseObject::copy( + const Key &from, + const Key &to, + FnMut done) { + get(from, [&](QByteArray value) { + put(to, value, std::move(done)); + }); +} + +void DatabaseObject::move( + const Key &from, + const Key &to, + FnMut done) { + const auto i = _map.find(from); + if (i == _map.end()) { + put(to, QByteArray(), std::move(done)); + return; + } + _removing.emplace(from); + + const auto entry = i->second; + eraseMapEntry(i); + + const auto result = writeMultiRemove(); + if (result.type != Error::Type::None) { + invokeCallback(done, result); + return; + } + invokeCallback(done, writeExistingPlace(to, entry)); } void DatabaseObject::writeBundlesLazy() { @@ -811,11 +901,11 @@ void DatabaseObject::writeMultiRemoveLazy() { } } -void DatabaseObject::writeMultiRemove() { +Error DatabaseObject::writeMultiRemove() { Expects(_removing.size() <= _settings.maxBundledRecords); if (_removing.empty()) { - return; + return Error::NoError(); } const auto size = _removing.size(); auto header = MultiRemove(size); @@ -824,13 +914,15 @@ void DatabaseObject::writeMultiRemove() { for (const auto &key : base::take(_removing)) { list.push_back(key); } - if (_binlog.write(bytes::object_as_span(&header))) { - _binlog.write(bytes::make_span(list)); + if (_binlog.write(bytes::object_as_span(&header)) + && _binlog.write(bytes::make_span(list))) { _binlog.flush(); - _binlogExcessLength += bytes::object_as_span(&header).size() + bytes::make_span(list).size(); + return Error::NoError(); } + _binlog.close(); + return ioError(binlogPath()); } void DatabaseObject::writeMultiAccessLazy() { @@ -841,13 +933,14 @@ void DatabaseObject::writeMultiAccessLazy() { } } -void DatabaseObject::writeMultiAccess() { - if (!_accessed.empty()) { - writeMultiAccessBlock(); +Error DatabaseObject::writeMultiAccess() { + if (_accessed.empty()) { + return Error::NoError(); } + return writeMultiAccessBlock(); } -void DatabaseObject::writeMultiAccessBlock() { +Error DatabaseObject::writeMultiAccessBlock() { Expects(_settings.trackEstimatedTime); Expects(_accessed.size() <= _settings.maxBundledRecords); @@ -868,15 +961,15 @@ void DatabaseObject::writeMultiAccessBlock() { } } - if (_binlog.write(bytes::object_as_span(&header))) { - if (size > 0) { - _binlog.write(bytes::make_span(list)); - } + if (_binlog.write(bytes::object_as_span(&header)) + && (!size || _binlog.write(bytes::make_span(list)))) { _binlog.flush(); - _binlogExcessLength += bytes::object_as_span(&header).size() + bytes::make_span(list).size(); + return Error::NoError(); } + _binlog.close(); + return ioError(binlogPath()); } void DatabaseObject::writeBundles() { diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h index dd188f4827..7792f34de0 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database_object.h @@ -35,7 +35,10 @@ public: void put(const Key &key, QByteArray value, FnMut done); void get(const Key &key, FnMut done); - void remove(const Key &key, FnMut done = nullptr); + void remove(const Key &key, FnMut done); + + void copy(const Key &from, const Key &to, FnMut done); + void move(const Key &from, const Key &to, FnMut done); void clear(FnMut done); @@ -160,11 +163,19 @@ private: const Key &key, const QByteArray &value, uint32 checksum); + template + Error writeExistingPlaceGeneric( + StoreRecord &&record, + const Key &key, + const Entry &entry); + Error writeExistingPlace( + const Key &key, + const Entry &entry); void writeMultiRemoveLazy(); - void writeMultiRemove(); + Error writeMultiRemove(); void writeMultiAccessLazy(); - void writeMultiAccess(); - void writeMultiAccessBlock(); + Error writeMultiAccess(); + Error writeMultiAccessBlock(); void writeBundlesLazy(); void writeBundles(); diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp b/Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp index 86051ecc42..2cad84967a 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp +++ b/Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp @@ -106,7 +106,7 @@ Error Put(Database &db, const Key &key, const QByteArray &value) { } void Remove(Database &db, const Key &key) { - db.remove(key, [&] { Semaphore.release(); }); + db.remove(key, [&](Error) { Semaphore.release(); }); Semaphore.acquire(); } diff --git a/Telegram/SourceFiles/storage/cache/storage_cache_types.h b/Telegram/SourceFiles/storage/cache/storage_cache_types.h index 392051098d..7828262dad 100644 --- a/Telegram/SourceFiles/storage/cache/storage_cache_types.h +++ b/Telegram/SourceFiles/storage/cache/storage_cache_types.h @@ -13,7 +13,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL namespace Storage { namespace Cache { -namespace details { struct Key { uint64 high = 0; @@ -32,6 +31,25 @@ inline bool operator<(const Key &a, const Key &b) { return std::tie(a.high, a.low) < std::tie(b.high, b.low); } +struct Error { + enum class Type { + None, + IO, + WrongKey, + LockFailed, + }; + Type type = Type::None; + QString path; + + static Error NoError(); +}; + +inline Error Error::NoError() { + return Error(); +} + +namespace details { + struct Settings { size_type maxBundledRecords = 16 * 1024; size_type readBlockSize = 8 * 1024 * 1024; @@ -56,23 +74,6 @@ QString VersionFilePath(const QString &base); base::optional ReadVersionValue(const QString &base); bool WriteVersionValue(const QString &base, Version value); -struct Error { - enum class Type { - None, - IO, - WrongKey, - LockFailed, - }; - Type type = Type::None; - QString path; - - static Error NoError(); -}; - -inline Error Error::NoError() { - return Error(); -} - using RecordType = uint8; using PlaceId = std::array; using EntrySize = std::array; @@ -217,8 +218,8 @@ struct MultiAccess { namespace std { template <> -struct hash { - size_t operator()(const Storage::Cache::details::Key &key) const { +struct hash { + size_t operator()(const Storage::Cache::Key &key) const { return (hash()(key.high) ^ hash()(key.low)); } };