RepoMirrors/tdesktop
1
0
Fork 0
mirror of https://github.com/telegramdesktop/tdesktop synced 2025-02-21 23:57:38 +00:00
Code Issues Projects Releases Wiki Activity

Write uint8 tags to Database and count stats.

Browse Source 
Also pass rvalues to Database where copies are required anyway.
This commit is contained in:
John Preston 2018-08-28 23:49:16 +03:00
parent 2e7f4c2f21
commit e2f08d4161
10 changed files with 306 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
Telegram/SourceFiles/storage/cache/storage_cache_database.cpp vendored
View File

@ -16,16 +16,16 @@ Database::Database(const QString &path, const Settings &settings)
: _wrapped(path, settings) {
}
void Database::open(EncryptionKey key, FnMut<void(Error)> done) {
void Database::open(EncryptionKey &&key, FnMut<void(Error)> &&done) {
_wrapped.with([
key,
key = std::move(key),
done = std::move(done)
](Implementation &unwrapped) mutable {
unwrapped.open(key, std::move(done));
unwrapped.open(std::move(key), std::move(done));
});
}
void Database::close(FnMut<void()> done) {
void Database::close(FnMut<void()> &&done) {
_wrapped.with([
done = std::move(done)
](Implementation &unwrapped) mutable {
@ -35,27 +35,23 @@ void Database::close(FnMut<void()> done) {
void Database::put(
const Key &key,
QByteArray value,
FnMut<void(Error)> done) {
_wrapped.with([
key,
value = std::move(value),
done = std::move(done)
](Implementation &unwrapped) mutable {
unwrapped.put(key, std::move(value), std::move(done));
});
QByteArray &&value,
FnMut<void(Error)> &&done) {
return put(key, TaggedValue(std::move(value), 0), std::move(done));
}
void Database::get(const Key &key, FnMut<void(QByteArray)> done) {
_wrapped.with([
key,
done = std::move(done)
](Implementation &unwrapped) mutable {
unwrapped.get(key, std::move(done));
});
void Database::get(const Key &key, FnMut<void(QByteArray&&)> &&done) {
if (done) {
auto untag = [done = std::move(done)](TaggedValue &&value) mutable {
done(std::move(value.bytes));
};
getWithTag(key, std::move(untag));
} else {
getWithTag(key, nullptr);
}
}
void Database::remove(const Key &key, FnMut<void(Error)> done) {
void Database::remove(const Key &key, FnMut<void(Error)> &&done) {
_wrapped.with([
key,
done = std::move(done)
@ -66,21 +62,18 @@ void Database::remove(const Key &key, FnMut<void(Error)> done) {
void Database::putIfEmpty(
const Key &key,
QByteArray value,
FnMut<void(Error)> done) {
_wrapped.with([
QByteArray &&value,
FnMut<void(Error)> &&done) {
return putIfEmpty(
key,
value = std::move(value),
done = std::move(done)
](Implementation &unwrapped) mutable {
unwrapped.putIfEmpty(key, std::move(value), std::move(done));
});
TaggedValue(std::move(value), 0),
std::move(done));
}
void Database::copyIfEmpty(
const Key &from,
const Key &to,
FnMut<void(Error)> done) {
FnMut<void(Error)> &&done) {
_wrapped.with([
from,
to,
@ -93,7 +86,7 @@ void Database::copyIfEmpty(
void Database::moveIfEmpty(
const Key &from,
const Key &to,
FnMut<void(Error)> done) {
FnMut<void(Error)> &&done) {
_wrapped.with([
from,
to,
@ -103,7 +96,52 @@ void Database::moveIfEmpty(
});
}
void Database::clear(FnMut<void(Error)> done) {
void Database::put(
const Key &key,
TaggedValue &&value,
FnMut<void(Error)> &&done) {
_wrapped.with([
key,
value = std::move(value),
done = std::move(done)
](Implementation &unwrapped) mutable {
unwrapped.put(key, std::move(value), std::move(done));
});
}
void Database::putIfEmpty(
const Key &key,
TaggedValue &&value,
FnMut<void(Error)> &&done) {
_wrapped.with([
key,
value = std::move(value),
done = std::move(done)
](Implementation &unwrapped) mutable {
unwrapped.putIfEmpty(key, std::move(value), std::move(done));
});
}
void Database::getWithTag(
const Key &key,
FnMut<void(TaggedValue&&)> &&done) {
_wrapped.with([
key,
done = std::move(done)
](Implementation &unwrapped) mutable {
unwrapped.get(key, std::move(done));
});
}
void Database::stats(FnMut<void(Stats&&)> &&done) {
_wrapped.with([
done = std::move(done)
](Implementation &unwrapped) mutable {
unwrapped.stats(std::move(done));
});
}
void Database::clear(FnMut<void(Error)> &&done) {
_wrapped.with([
done = std::move(done)
](Implementation &unwrapped) mutable {

36
Telegram/SourceFiles/storage/cache/storage_cache_database.h vendored
View File

@ -25,30 +25,44 @@ public:
using Settings = details::Settings;
Database(const QString &path, const Settings &settings);
void open(EncryptionKey key, FnMut<void(Error)> done = nullptr);
void close(FnMut<void()> done = nullptr);
void open(EncryptionKey &&key, FnMut<void(Error)> &&done = nullptr);
void close(FnMut<void()> &&done = nullptr);
void put(
const Key &key,
QByteArray value,
FnMut<void(Error)> done = nullptr);
void get(const Key &key, FnMut<void(QByteArray)> done);
void remove(const Key &key, FnMut<void(Error)> done = nullptr);
QByteArray &&value,
FnMut<void(Error)> &&done = nullptr);
void get(const Key &key, FnMut<void(QByteArray&&)> &&done);
void remove(const Key &key, FnMut<void(Error)> &&done = nullptr);
void putIfEmpty(
const Key &key,
QByteArray value,
FnMut<void(Error)> done = nullptr);
QByteArray &&value,
FnMut<void(Error)> &&done = nullptr);
void copyIfEmpty(
const Key &from,
const Key &to,
FnMut<void(Error)> done = nullptr);
FnMut<void(Error)> &&done = nullptr);
void moveIfEmpty(
const Key &from,
const Key &to,
FnMut<void(Error)> done = nullptr);
FnMut<void(Error)> &&done = nullptr);
void clear(FnMut<void(Error)> done = nullptr);
using TaggedValue = details::TaggedValue;
void put(
const Key &key,
TaggedValue &&value,
FnMut<void(Error)> &&done = nullptr);
void putIfEmpty(
const Key &key,
TaggedValue &&value,
FnMut<void(Error)> &&done = nullptr);
void getWithTag(const Key &key, FnMut<void(TaggedValue&&)> &&done);
using Stats = details::Stats;
void stats(FnMut<void(Stats&&)> &&done);
void clear(FnMut<void(Error)> &&done = nullptr);
~Database();

109
Telegram/SourceFiles/storage/cache/storage_cache_database_object.cpp vendored
View File

@ -105,17 +105,17 @@ Error DatabaseObject::ioError(const QString &path) const {
return { Error::Type::IO, path };
}
void DatabaseObject::open(EncryptionKey key, FnMut<void(Error)> done) {
void DatabaseObject::open(EncryptionKey &&key, FnMut<void(Error)> &&done) {
close(nullptr);
const auto error = openSomeBinlog(key);
const auto error = openSomeBinlog(std::move(key));
if (error.type != Error::Type::None) {
close(nullptr);
}
invokeCallback(done, error);
}
Error DatabaseObject::openSomeBinlog(EncryptionKey &key) {
Error DatabaseObject::openSomeBinlog(EncryptionKey &&key) {
const auto version = readVersion();
const auto result = openBinlog(version, File::Mode::ReadAppend, key);
switch (result) {
@ -500,7 +500,7 @@ bool DatabaseObject::processRecordMultiAccess(
void DatabaseObject::setMapEntry(const Key &key, Entry &&entry) {
auto &already = _map[key];
_totalSize += entry.size - already.size;
updateStats(already, entry);
if (already.size != 0) {
_binlogExcessLength += _settings.trackEstimatedTime
? sizeof(StoreWithTime)
@ -522,10 +522,32 @@ void DatabaseObject::setMapEntry(const Key &key, Entry &&entry) {
already = std::move(entry);
}
void DatabaseObject::updateStats(const Entry &was, const Entry &now) {
_totalSize += now.size - was.size;
if (now.tag == was.tag) {
if (now.tag) {
auto &summary = _taggedStats[now.tag];
summary.count += (now.size ? 1 : 0) - (was.size ? 1 : 0);
summary.totalSize += now.size - was.size;
}
} else {
if (now.tag) {
auto &summary = _taggedStats[now.tag];
summary.count += (now.size ? 1 : 0);
summary.totalSize += now.size;
}
if (was.tag) {
auto &summary = _taggedStats[was.tag];
summary.count -= (was.size ? 1 : 0);
summary.totalSize -= was.size;
}
}
}
void DatabaseObject::eraseMapEntry(const Map::const_iterator &i) {
if (i != end(_map)) {
const auto &entry = i->second;
_totalSize -= entry.size;
updateStats(entry, Entry());
if (_minimalEntryTime != 0 && entry.useTime == _minimalEntryTime) {
Assert(_entriesWithMinimalTimeCount > 0);
--_entriesWithMinimalTimeCount;
@ -604,7 +626,7 @@ void DatabaseObject::compactorFail() {
QFile(compactReadyPath()).remove();
}
void DatabaseObject::close(FnMut<void()> done) {
void DatabaseObject::close(FnMut<void()> &&done) {
if (_binlog.isOpen()) {
writeBundles();
}
@ -619,15 +641,15 @@ void DatabaseObject::close(FnMut<void()> done) {
void DatabaseObject::put(
const Key &key,
QByteArray value,
FnMut<void(Error)> done) {
if (value.isEmpty()) {
TaggedValue &&value,
FnMut<void(Error)> &&done) {
if (value.bytes.isEmpty()) {
remove(key, std::move(done));
return;
}
_removing.erase(key);
const auto checksum = CountChecksum(bytes::make_span(value));
const auto checksum = CountChecksum(bytes::make_span(value.bytes));
const auto maybepath = writeKeyPlace(key, value, checksum);
if (!maybepath) {
invokeCallback(done, ioError(binlogPath()));
@ -653,7 +675,8 @@ void DatabaseObject::put(
break;
case File::Result::Success: {
const auto success = data.writeWithPadding(bytes::make_span(value));
const auto success = data.writeWithPadding(
bytes::make_span(value.bytes));
if (!success) {
data.close();
remove(key, nullptr);
@ -673,11 +696,12 @@ template <typename StoreRecord>
base::optional<QString> DatabaseObject::writeKeyPlaceGeneric(
StoreRecord &&record,
const Key &key,
const QByteArray &value,
const TaggedValue &value,
uint32 checksum) {
Expects(value.size() <= _settings.maxDataSize);
Expects(value.bytes.size() <= _settings.maxDataSize);
const auto size = size_type(value.size());
const auto size = size_type(value.bytes.size());
record.tag = value.tag;
record.key = key;
record.size = ReadTo<EntrySize>(size);
record.checksum = checksum;
@ -686,7 +710,7 @@ base::optional<QString> DatabaseObject::writeKeyPlaceGeneric(
if (already.tag == record.tag
&& already.size == size
&& already.checksum == checksum
&& readValueData(already.place, size) == value) {
&& readValueData(already.place, size) == value.bytes) {
return QString();
}
record.place = already.place;
@ -713,7 +737,7 @@ base::optional<QString> DatabaseObject::writeKeyPlaceGeneric(
base::optional<QString> DatabaseObject::writeKeyPlace(
const Key &key,
const QByteArray &data,
const TaggedValue &data,
uint32 checksum) {
if (!_settings.trackEstimatedTime) {
return writeKeyPlaceGeneric(Store(), key, data, checksum);
@ -786,27 +810,25 @@ Error DatabaseObject::writeExistingPlace(
return writeExistingPlaceGeneric(std::move(record), key, entry);
}
void DatabaseObject::get(const Key &key, FnMut<void(QByteArray)> done) {
if (_removing.find(key) != end(_removing)) {
invokeCallback(done, QByteArray());
return;
}
void DatabaseObject::get(
const Key &key,
FnMut<void(TaggedValue&&)> &&done) {
const auto i = _map.find(key);
if (i == _map.end()) {
invokeCallback(done, QByteArray());
invokeCallback(done, TaggedValue());
return;
}
const auto &entry = i->second;
auto result = readValueData(entry.place, entry.size);
if (result.isEmpty()) {
auto bytes = readValueData(entry.place, entry.size);
if (bytes.isEmpty()) {
remove(key, nullptr);
invokeCallback(done, QByteArray());
} else if (CountChecksum(bytes::make_span(result)) != entry.checksum) {
invokeCallback(done, TaggedValue());
} else if (CountChecksum(bytes::make_span(bytes)) != entry.checksum) {
remove(key, nullptr);
invokeCallback(done, QByteArray());
invokeCallback(done, TaggedValue());
} else {
invokeCallback(done, std::move(result));
invokeCallback(done, TaggedValue(std::move(bytes), entry.tag));
recordEntryAccess(key);
}
}
@ -840,7 +862,7 @@ void DatabaseObject::recordEntryAccess(const Key &key) {
optimize();
}
void DatabaseObject::remove(const Key &key, FnMut<void(Error)> done) {
void DatabaseObject::remove(const Key &key, FnMut<void(Error)> &&done) {
const auto i = _map.find(key);
if (i != _map.end()) {
_removing.emplace(key);
@ -860,8 +882,8 @@ void DatabaseObject::remove(const Key &key, FnMut<void(Error)> done) {
void DatabaseObject::putIfEmpty(
const Key &key,
QByteArray value,
FnMut<void(Error)> done) {
TaggedValue &&value,
FnMut<void(Error)> &&done) {
if (_map.find(key) != end(_map)) {
invokeCallback(done, Error::NoError());
return;
@ -872,20 +894,20 @@ void DatabaseObject::putIfEmpty(
void DatabaseObject::copyIfEmpty(
const Key &from,
const Key &to,
FnMut<void(Error)> done) {
FnMut<void(Error)> &&done) {
if (_map.find(to) != end(_map)) {
invokeCallback(done, Error::NoError());
return;
}
get(from, [&](QByteArray value) {
put(to, value, std::move(done));
get(from, [&](TaggedValue &&value) {
put(to, std::move(value), std::move(done));
});
}
void DatabaseObject::moveIfEmpty(
const Key &from,
const Key &to,
FnMut<void(Error)> done) {
FnMut<void(Error)> &&done) {
if (_map.find(to) != end(_map)) {
invokeCallback(done, Error::NoError());
return;
@ -908,6 +930,19 @@ void DatabaseObject::moveIfEmpty(
invokeCallback(done, writeExistingPlace(to, entry));
}
void DatabaseObject::stats(FnMut<void(Stats&&)> &&done) {
auto result = _taggedStats;
auto zero = TaggedSummary();
zero.count = _map.size();
zero.totalSize = _totalSize;
for (const auto &summary : result) {
zero.count -= summary.second.count;
zero.totalSize -= summary.second.totalSize;
}
result[0] = zero;
invokeCallback(done, std::move(result));
}
void DatabaseObject::writeBundlesLazy() {
if (!_writeBundlesTimer.isActive()) {
_writeBundlesTimer.callOnce(_settings.writeBundleDelay);
@ -1043,7 +1078,7 @@ void DatabaseObject::checkCompactor() {
_compactor.excessLength = _binlogExcessLength;
}
void DatabaseObject::clear(FnMut<void(Error)> done) {
void DatabaseObject::clear(FnMut<void(Error)> &&done) {
Expects(_key.empty());
const auto version = findAvailableVersion();
@ -1052,7 +1087,7 @@ void DatabaseObject::clear(FnMut<void(Error)> done) {
writeVersion(version) ? Error::NoError() : ioError(versionPath()));
}
auto DatabaseObject::getManyRaw(const std::vector<Key> keys) const
auto DatabaseObject::getManyRaw(const std::vector<Key> &keys) const
-> std::vector<Raw> {
auto result = std::vector<Raw>();
result.reserve(keys.size());

36
Telegram/SourceFiles/storage/cache/storage_cache_database_object.h vendored
View File

@ -30,27 +30,32 @@ public:
const QString &path,
const Settings &settings);
void open(EncryptionKey key, FnMut<void(Error)> done);
void close(FnMut<void()> done);
void open(EncryptionKey &&key, FnMut<void(Error)> &&done);
void close(FnMut<void()> &&done);
void put(const Key &key, QByteArray value, FnMut<void(Error)> done);
void get(const Key &key, FnMut<void(QByteArray)> done);
void remove(const Key &key, FnMut<void(Error)> done);
void put(
const Key &key,
TaggedValue &&value,
FnMut<void(Error)> &&done);
void get(const Key &key, FnMut<void(TaggedValue&&)> &&done);
void remove(const Key &key, FnMut<void(Error)> &&done);
void putIfEmpty(
const Key &key,
QByteArray value,
FnMut<void(Error)> done);
TaggedValue &&value,
FnMut<void(Error)> &&done);
void copyIfEmpty(
const Key &from,
const Key &to,
FnMut<void(Error)> done);
FnMut<void(Error)> &&done);
void moveIfEmpty(
const Key &from,
const Key &to,
FnMut<void(Error)> done);
FnMut<void(Error)> &&done);
void clear(FnMut<void(Error)> done);
void stats(FnMut<void(Stats&&)> &&done);
void clear(FnMut<void(Error)> &&done);
static QString BinlogFilename();
static QString CompactReadyFilename();
@ -74,7 +79,7 @@ public:
uint8 tag = 0;
};
using Raw = std::pair<Key, Entry>;
std::vector<Raw> getManyRaw(const std::vector<Key> keys) const;
std::vector<Raw> getManyRaw(const std::vector<Key> &keys) const;
~DatabaseObject();
@ -101,7 +106,7 @@ private:
QString binlogPath() const;
QString compactReadyPath(Version version) const;
QString compactReadyPath() const;
Error openSomeBinlog(EncryptionKey &key);
Error openSomeBinlog(EncryptionKey &&key);
Error openNewBinlog(EncryptionKey &key);
File::Result openBinlog(
Version version,
@ -149,6 +154,7 @@ private:
void collectSizePrune(
base::flat_set<Key> &stale,
int64 &staleTotalSize);
void updateStats(const Entry &was, const Entry &now);
void setMapEntry(const Key &key, Entry &&entry);
void eraseMapEntry(const Map::const_iterator &i);
@ -167,11 +173,11 @@ private:
base::optional<QString> writeKeyPlaceGeneric(
StoreRecord &&record,
const Key &key,
const QByteArray &value,
const TaggedValue &value,
uint32 checksum);
base::optional<QString> writeKeyPlace(
const Key &key,
const QByteArray &value,
const TaggedValue &value,
uint32 checksum);
template <typename StoreRecord>
Error writeExistingPlaceGeneric(
@ -208,6 +214,8 @@ private:
uint64 _minimalEntryTime = 0;
size_type _entriesWithMinimalTimeCount = 0;
Stats _taggedStats;
base::ConcurrentTimer _writeBundlesTimer;
base::ConcurrentTimer _pruneTimer;

152
Telegram/SourceFiles/storage/cache/storage_cache_database_tests.cpp vendored
View File

@ -59,8 +59,14 @@ QString GetBinlogPath() {
return name + '/' + QString::number(version) + "/binlog";
}
const auto TestValue1 = QByteArray("testbytetestbyt");
const auto TestValue2 = QByteArray("bytetestbytetestb");
const auto TestValue1 = [] {
static auto result = QByteArray("testbytetestbyt");
return result;
};
const auto TestValue2 = [] {
static auto result = QByteArray("bytetestbytetestb");
return result;
};
crl::semaphore Semaphore;
@ -77,7 +83,7 @@ const auto GetValue = [](QByteArray value) {
};
Error Open(Database &db, const Storage::EncryptionKey &key) {
db.open(key, GetResult);
db.open(base::duplicate(key), GetResult);
Semaphore.acquire();
return Result;
}
@ -99,8 +105,8 @@ QByteArray Get(Database &db, const Key &key) {
return Value;
}
Error Put(Database &db, const Key &key, const QByteArray &value) {
db.put(key, value, GetResult);
Error Put(Database &db, const Key &key, QByteArray &&value) {
db.put(key, std::move(value), GetResult);
Semaphore.acquire();
return Result;
}
@ -141,15 +147,15 @@ TEST_CASE("compacting db", "[storage_cache_database]") {
for (auto i = from; i != till; ++i) {
auto value = base;
value[0] = char('A') + i;
const auto result = Put(db, Key{ i, i + 1 }, value);
const auto result = Put(db, Key{ i, i + 1 }, std::move(value));
REQUIRE(result.type == Error::Type::None);
}
};
const auto put = [&](Database &db, uint32 from, uint32 till) {
write(db, from, till, TestValue1);
write(db, from, till, TestValue1());
};
const auto reput = [&](Database &db, uint32 from, uint32 till) {
write(db, from, till, TestValue2);
write(db, from, till, TestValue2());
};
const auto remove = [](Database &db, uint32 from, uint32 till) {
for (auto i = from; i != till; ++i) {
@ -201,11 +207,11 @@ TEST_CASE("compacting db", "[storage_cache_database]") {
const auto fullcheck = [&] {
check(db, 0, 15, {});
check(db, 15, 20, TestValue1);
check(db, 20, 30, TestValue2);
check(db, 15, 20, TestValue1());
check(db, 20, 30, TestValue2());
check(db, 30, 35, {});
check(db, 35, 37, TestValue2);
check(db, 37, 45, TestValue1);
check(db, 35, 37, TestValue2());
check(db, 37, 45, TestValue1());
};
fullcheck();
Close(db);
@ -241,11 +247,11 @@ TEST_CASE("compacting db", "[storage_cache_database]") {
const auto fullcheck = [&] {
check(db, 0, 15, {});
check(db, 15, 20, TestValue1);
check(db, 20, 30, TestValue2);
check(db, 15, 20, TestValue1());
check(db, 20, 30, TestValue2());
check(db, 30, 35, {});
check(db, 35, 37, TestValue2);
check(db, 37, 45, TestValue1);
check(db, 35, 37, TestValue2());
check(db, 37, 45, TestValue1());
};
fullcheck();
Close(db);
@ -285,9 +291,9 @@ TEST_CASE("compacting db", "[storage_cache_database]") {
REQUIRE(after < size2);
const auto fullcheck = [&] {
check(db, 0, 15, {});
check(db, 15, 20, TestValue1);
check(db, 15, 20, TestValue1());
check(db, 20, 35, {});
check(db, 35, 45, TestValue2);
check(db, 35, 45, TestValue2());
};
fullcheck();
Close(db);
@ -324,7 +330,7 @@ TEST_CASE("compacting db", "[storage_cache_database]") {
AdvanceTime(2);
REQUIRE(QFile(path).size() < size);
const auto fullcheck = [&] {
check(db, 15, 30, TestValue2);
check(db, 15, 30, TestValue2());
};
fullcheck();
Close(db);
@ -344,16 +350,16 @@ TEST_CASE("encrypted cache db", "[storage_cache_database]") {
REQUIRE(Clear(db).type == Error::Type::None);
REQUIRE(Open(db, key).type == Error::Type::None);
REQUIRE(Put(db, Key{ 0, 1 }, TestValue1).type == Error::Type::None);
REQUIRE(Put(db, Key{ 0, 1 }, TestValue1()).type == Error::Type::None);
Close(db);
}
SECTION("reading and writing db") {
Database db(name, Settings);
REQUIRE(Open(db, key).type == Error::Type::None);
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1));
REQUIRE(Put(db, Key{ 1, 0 }, TestValue2).type == Error::Type::None);
REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2));
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1()));
REQUIRE(Put(db, Key{ 1, 0 }, TestValue2()).type == Error::Type::None);
REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2()));
REQUIRE(Get(db, Key{ 1, 1 }).isEmpty());
Close(db);
}
@ -361,8 +367,8 @@ TEST_CASE("encrypted cache db", "[storage_cache_database]") {
Database db(name, Settings);
REQUIRE(Open(db, key).type == Error::Type::None);
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1));
REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2));
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1()));
REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2()));
Close(db);
}
SECTION("overwriting values") {
@ -370,13 +376,13 @@ TEST_CASE("encrypted cache db", "[storage_cache_database]") {
REQUIRE(Open(db, key).type == Error::Type::None);
const auto path = GetBinlogPath();
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1));
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1()));
const auto size = QFile(path).size();
REQUIRE(Put(db, Key{ 0, 1 }, TestValue2).type == Error::Type::None);
REQUIRE(Put(db, Key{ 0, 1 }, TestValue2()).type == Error::Type::None);
const auto next = QFile(path).size();
REQUIRE(next > size);
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue2));
REQUIRE(Put(db, Key{ 0, 1 }, TestValue2).type == Error::Type::None);
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue2()));
REQUIRE(Put(db, Key{ 0, 1 }, TestValue2()).type == Error::Type::None);
const auto same = QFile(path).size();
REQUIRE(same == next);
Close(db);
@ -393,16 +399,16 @@ TEST_CASE("encrypted cache db", "[storage_cache_database]") {
REQUIRE(Clear(db).type == Error::Type::None);
REQUIRE(Open(db, key).type == Error::Type::None);
for (auto i = 0U; i != count; ++i) {
auto value = TestValue1;
auto value = TestValue1();
value[0] = char('A') + i;
const auto result = Put(db, Key{ i, i * 2 }, value);
const auto result = Put(db, Key{ i, i * 2 }, std::move(value));
REQUIRE(result.type == Error::Type::None);
}
Close(db);
REQUIRE(Open(db, key).type == Error::Type::None);
for (auto i = 0U; i != count; ++i) {
auto value = TestValue1;
auto value = TestValue1();
value[0] = char('A') + i;
REQUIRE((Get(db, Key{ i, i * 2 }) == value));
}
@ -419,11 +425,11 @@ TEST_CASE("cache db remove", "[storage_cache_database]") {
REQUIRE(Clear(db).type == Error::Type::None);
REQUIRE(Open(db, key).type == Error::Type::None);
REQUIRE(Put(db, Key{ 0, 1 }, TestValue1).type == Error::Type::None);
REQUIRE(Put(db, Key{ 1, 0 }, TestValue2).type == Error::Type::None);
REQUIRE(Put(db, Key{ 0, 1 }, TestValue1()).type == Error::Type::None);
REQUIRE(Put(db, Key{ 1, 0 }, TestValue2()).type == Error::Type::None);
Remove(db, Key{ 0, 1 });
REQUIRE(Get(db, Key{ 0, 1 }).isEmpty());
REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2));
REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2()));
Close(db);
}
SECTION("db remove deletes value permanently") {
@ -431,7 +437,7 @@ TEST_CASE("cache db remove", "[storage_cache_database]") {
REQUIRE(Open(db, key).type == Error::Type::None);
REQUIRE(Get(db, Key{ 0, 1 }).isEmpty());
REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2));
REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2()));
Close(db);
}
}
@ -448,9 +454,9 @@ TEST_CASE("cache db bundled actions", "[storage_cache_database]") {
REQUIRE(Clear(db).type == Error::Type::None);
REQUIRE(Open(db, key).type == Error::Type::None);
const auto path = GetBinlogPath();
REQUIRE(Put(db, Key{ 0, 1 }, TestValue1).type == Error::Type::None);
REQUIRE(Put(db, Key{ 0, 1 }, TestValue1()).type == Error::Type::None);
const auto size = QFile(path).size();
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1));
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1()));
REQUIRE(QFile(path).size() == size);
AdvanceTime(2);
Get(db, Key{ 0, 1 });
@ -465,9 +471,9 @@ TEST_CASE("cache db bundled actions", "[storage_cache_database]") {
REQUIRE(Clear(db).type == Error::Type::None);
REQUIRE(Open(db, key).type == Error::Type::None);
const auto path = GetBinlogPath();
REQUIRE(Put(db, Key{ 0, 1 }, TestValue1).type == Error::Type::None);
REQUIRE(Put(db, Key{ 0, 1 }, TestValue1()).type == Error::Type::None);
const auto size = QFile(path).size();
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1));
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1()));
REQUIRE(QFile(path).size() == size);
Close(db);
REQUIRE(QFile(path).size() > size);
@ -478,7 +484,7 @@ TEST_CASE("cache db bundled actions", "[storage_cache_database]") {
REQUIRE(Clear(db).type == Error::Type::None);
REQUIRE(Open(db, key).type == Error::Type::None);
const auto path = GetBinlogPath();
REQUIRE(Put(db, Key{ 0, 1 }, TestValue1).type == Error::Type::None);
REQUIRE(Put(db, Key{ 0, 1 }, TestValue1()).type == Error::Type::None);
const auto size = QFile(path).size();
Remove(db, Key{ 0, 1 });
REQUIRE(QFile(path).size() == size);
@ -492,7 +498,7 @@ TEST_CASE("cache db bundled actions", "[storage_cache_database]") {
REQUIRE(Clear(db).type == Error::Type::None);
REQUIRE(Open(db, key).type == Error::Type::None);
const auto path = GetBinlogPath();
REQUIRE(Put(db, Key{ 0, 1 }, TestValue1).type == Error::Type::None);
REQUIRE(Put(db, Key{ 0, 1 }, TestValue1()).type == Error::Type::None);
const auto size = QFile(path).size();
Remove(db, Key{ 0, 1 });
REQUIRE(QFile(path).size() == size);
@ -513,21 +519,21 @@ TEST_CASE("cache db limits", "[storage_cache_database]") {
Database db(name, settings);
db.clear(nullptr);
db.open(key, nullptr);
db.put(Key{ 0, 1 }, TestValue1, nullptr);
db.put(Key{ 1, 0 }, TestValue2, nullptr);
db.open(base::duplicate(key), nullptr);
db.put(Key{ 0, 1 }, TestValue1(), nullptr);
db.put(Key{ 1, 0 }, TestValue2(), nullptr);
AdvanceTime(2);
db.get(Key{ 1, 0 }, nullptr);
AdvanceTime(3);
db.put(Key{ 1, 1 }, TestValue1, nullptr);
db.put(Key{ 2, 0 }, TestValue2, nullptr);
db.put(Key{ 0, 2 }, TestValue1, nullptr);
db.put(Key{ 1, 1 }, TestValue1(), nullptr);
db.put(Key{ 2, 0 }, TestValue2(), nullptr);
db.put(Key{ 0, 2 }, TestValue1(), nullptr);
AdvanceTime(2);
REQUIRE(Get(db, Key{ 0, 1 }).isEmpty());
REQUIRE(Get(db, Key{ 1, 0 }).isEmpty());
REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1));
REQUIRE((Get(db, Key{ 2, 0 }) == TestValue2));
REQUIRE((Get(db, Key{ 0, 2 }) == TestValue1));
REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1()));
REQUIRE((Get(db, Key{ 2, 0 }) == TestValue2()));
REQUIRE((Get(db, Key{ 0, 2 }) == TestValue1()));
Close(db);
}
SECTION("db size limit") {
@ -537,27 +543,27 @@ TEST_CASE("cache db limits", "[storage_cache_database]") {
Database db(name, settings);
db.clear(nullptr);
db.open(key, nullptr);
db.put(Key{ 0, 1 }, TestValue1, nullptr);
db.open(base::duplicate(key), nullptr);
db.put(Key{ 0, 1 }, TestValue1(), nullptr);
AdvanceTime(2);
db.put(Key{ 1, 0 }, TestValue2, nullptr);
db.put(Key{ 1, 0 }, TestValue2(), nullptr);
AdvanceTime(2);
db.put(Key{ 1, 1 }, TestValue1, nullptr);
db.put(Key{ 1, 1 }, TestValue1(), nullptr);
db.get(Key{ 0, 1 }, nullptr);
AdvanceTime(2);
db.put(Key{ 2, 0 }, TestValue2, nullptr);
db.put(Key{ 2, 0 }, TestValue2(), nullptr);
// Removing { 1, 0 } will be scheduled.
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1));
REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1));
REQUIRE((Get(db, Key{ 2, 0 }) == TestValue2));
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1()));
REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1()));
REQUIRE((Get(db, Key{ 2, 0 }) == TestValue2()));
AdvanceTime(2);
// Removing { 1, 0 } performed.
REQUIRE(Get(db, Key{ 1, 0 }).isEmpty());
REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1));
db.put(Key{ 0, 2 }, TestValue1, nullptr);
REQUIRE(Put(db, Key{ 2, 2 }, TestValue2).type == Error::Type::None);
REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1()));
db.put(Key{ 0, 2 }, TestValue1(), nullptr);
REQUIRE(Put(db, Key{ 2, 2 }, TestValue2()).type == Error::Type::None);
// Removing { 0, 1 } and { 2, 0 } will be scheduled.
AdvanceTime(2);
@ -565,9 +571,9 @@ TEST_CASE("cache db limits", "[storage_cache_database]") {
// Removing { 0, 1 } and { 2, 0 } performed.
REQUIRE(Get(db, Key{ 0, 1 }).isEmpty());
REQUIRE(Get(db, Key{ 2, 0 }).isEmpty());
REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1));
REQUIRE((Get(db, Key{ 0, 2 }) == TestValue1));
REQUIRE((Get(db, Key{ 2, 2 }) == TestValue2));
REQUIRE((Get(db, Key{ 1, 1 }) == TestValue1()));
REQUIRE((Get(db, Key{ 0, 2 }) == TestValue1()));
REQUIRE((Get(db, Key{ 2, 2 }) == TestValue2()));
Close(db);
}
SECTION("db time limit") {
@ -577,11 +583,11 @@ TEST_CASE("cache db limits", "[storage_cache_database]") {
Database db(name, settings);
db.clear(nullptr);
db.open(key, nullptr);
db.put(Key{ 0, 1 }, TestValue1, nullptr);
db.put(Key{ 1, 0 }, TestValue2, nullptr);
db.put(Key{ 1, 1 }, TestValue1, nullptr);
db.put(Key{ 2, 0 }, TestValue2, nullptr);
db.open(base::duplicate(key), nullptr);
db.put(Key{ 0, 1 }, TestValue1(), nullptr);
db.put(Key{ 1, 0 }, TestValue2(), nullptr);
db.put(Key{ 1, 1 }, TestValue1(), nullptr);
db.put(Key{ 2, 0 }, TestValue2(), nullptr);
AdvanceTime(1);
db.get(Key{ 1, 0 }, nullptr);
db.get(Key{ 1, 1 }, nullptr);
@ -594,8 +600,8 @@ TEST_CASE("cache db limits", "[storage_cache_database]") {
AdvanceTime(3);
REQUIRE(Get(db, Key{ 2, 0 }).isEmpty());
REQUIRE(Get(db, Key{ 1, 1 }).isEmpty());
REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2));
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1));
REQUIRE((Get(db, Key{ 1, 0 }) == TestValue2()));
REQUIRE((Get(db, Key{ 0, 1 }) == TestValue1()));
Close(db);
}
}
@ -623,7 +629,7 @@ TEST_CASE("large db", "[storage_cache_database]") {
};
const auto kWriteRecords = 100 * 1024;
for (auto i = 0; i != kWriteRecords; ++i) {
db.put(key(i), TestValue1, nullptr);
db.put(key(i), TestValue1(), nullptr);
const auto j = i ? (rand() % i) : 0;
if (i % 1024 == 1023) {
std::this_thread::sleep_for(std::chrono::milliseconds(100));

4
Telegram/SourceFiles/storage/cache/storage_cache_types.cpp vendored
View File

@ -13,6 +13,10 @@ namespace Storage {
namespace Cache {
namespace details {
TaggedValue::TaggedValue(QByteArray &&bytes, uint8 tag)
: bytes(std::move(bytes)), tag(tag) {
}
QString ComputeBasePath(const QString &original) {
const auto result = QDir(original).absolutePath();
return result.endsWith('/') ? result : (result + '/');

15
Telegram/SourceFiles/storage/cache/storage_cache_types.h vendored
View File

@ -8,6 +8,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#pragma once
#include "base/basic_types.h"
#include "base/flat_map.h"
#include "base/optional.h"
#include <crl/crl_time.h>
@ -67,6 +68,20 @@ struct Settings {
crl::time_type maxPruneCheckTimeout = 3600 * crl::time_type(1000);
};
struct TaggedValue {
TaggedValue() = default;
TaggedValue(QByteArray &&bytes, uint8 tag);
QByteArray bytes;
uint8 tag = 0;
};
struct TaggedSummary {
size_type count = 0;
size_type totalSize = 0;
};
using Stats = base::flat_map<uint8, TaggedSummary>;
using Version = int32;
QString ComputeBasePath(const QString &original);

21
Telegram/SourceFiles/storage/file_download.cpp
View File

@ -365,9 +365,9 @@ void FileLoader::loadLocal(const Storage::Cache::Key &key) {
auto [first, second] = base::make_binary_guard();
_localLoading = std::move(first);
auto done = [=, guard = std::move(second)](
QByteArray value,
QImage image,
QByteArray format) mutable {
QByteArray &&value,
QImage &&image,
QByteArray &&format) mutable {
crl::on_main([
=,
value = std::move(value),
@ -385,7 +385,7 @@ void FileLoader::loadLocal(const Storage::Cache::Key &key) {
});
};
Auth().data().cache().get(key, [=, callback = std::move(done)](
QByteArray value) mutable {
QByteArray &&value) mutable {
if (readImage) {
crl::async([
value = std::move(value),
@ -394,13 +394,16 @@ void FileLoader::loadLocal(const Storage::Cache::Key &key) {
auto format = QByteArray();
auto image = App::readImage(value, &format, false);
if (!image.isNull()) {
done(value, image, format);
done(
std::move(value),
std::move(image),
std::move(format));
} else {
done(value, {}, {});
done(std::move(value), {}, {});
}
});
} else {
callback(value, {}, {});
callback(std::move(value), {}, {});
}
});
}
@ -954,7 +957,7 @@ bool mtpFileLoader::feedPart(int offset, bytes::const_span buffer) {
|| _locationType == UnknownFileLocation
|| _toCache == LoadToCacheAsWell) {
if (const auto key = cacheKey()) {
Auth().data().cache().put(*key, _data);
Auth().data().cache().put(*key, base::duplicate(_data));
}
}
}
@ -1164,7 +1167,7 @@ void webFileLoader::onFinished(const QByteArray &data) {
if (_localStatus == LocalStatus::NotFound) {
if (const auto key = cacheKey()) {
Auth().data().cache().put(*key, _data);
Auth().data().cache().put(*key, base::duplicate(_data));
}
}
_downloader->taskFinished().notify();

6
Telegram/SourceFiles/storage/file_upload.cpp
View File

@ -127,7 +127,9 @@ void Uploader::uploadMedia(const FullMsgId &msgId, const SendMediaReady &media)
if (!media.data.isEmpty()) {
document->setData(media.data);
if (document->saveToCache()) {
Auth().data().cache().put(document->cacheKey(), media.data);
Auth().data().cache().put(
document->cacheKey(),
base::duplicate(media.data));
}
}
if (!media.file.isEmpty()) {
@ -154,7 +156,7 @@ void Uploader::upload(
if (document->saveToCache()) {
Auth().data().cache().put(
document->cacheKey(),
file->content);
base::duplicate(file->content));
}
}
if (!file->filepath.isEmpty()) {

4
Telegram/SourceFiles/ui/images.cpp
View File

@ -1022,7 +1022,9 @@ void RemoteImage::setImageBytes(
const auto location = this->location();
if (!location.isNull() && !bytes.isEmpty()) {
Auth().data().cache().putIfEmpty(Data::StorageCacheKey(location), bytes);
Auth().data().cache().putIfEmpty(
Data::StorageCacheKey(location),
base::duplicate(bytes));
}
}