From 9ba331693f2e809920912b884bcb2e69da0c63b5 Mon Sep 17 00:00:00 2001 From: John Preston Date: Tue, 4 Sep 2018 15:13:28 +0300 Subject: [PATCH] Async clear of legacy local storage. Sync call to QDir::entryList is a bad idea for the user data folder. Some users reported hanging on startup with 1.25M legacy cache files. Now we enumerate up to 10000 files at once asynchronously and clear. --- Telegram/SourceFiles/storage/localstorage.cpp | 51 ++++----- .../storage/storage_clear_legacy.cpp | 53 +++++++++ .../storage/storage_clear_legacy.h | 26 +++++ .../storage/storage_clear_legacy_posix.cpp | 101 ++++++++++++++++++ .../storage/storage_clear_legacy_win.cpp | 62 +++++++++++ Telegram/gyp/lib_storage.gyp | 20 ++-- 6 files changed, 275 insertions(+), 38 deletions(-) create mode 100644 Telegram/SourceFiles/storage/storage_clear_legacy.cpp create mode 100644 Telegram/SourceFiles/storage/storage_clear_legacy.h create mode 100644 Telegram/SourceFiles/storage/storage_clear_legacy_posix.cpp create mode 100644 Telegram/SourceFiles/storage/storage_clear_legacy_win.cpp diff --git a/Telegram/SourceFiles/storage/localstorage.cpp b/Telegram/SourceFiles/storage/localstorage.cpp index 2015bc3002..c4674c2421 100644 --- a/Telegram/SourceFiles/storage/localstorage.cpp +++ b/Telegram/SourceFiles/storage/localstorage.cpp @@ -10,6 +10,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "storage/serialize_document.h" #include "storage/serialize_common.h" #include "storage/storage_encrypted_file.h" +#include "storage/storage_clear_legacy.h" #include "chat_helpers/stickers.h" #include "data/data_drafts.h" #include "boxes/send_files_box.h" @@ -2657,11 +2658,7 @@ void setPasscode(const QByteArray &passcode) { Global::RefLocalPasscodeChanged().notify(); } -std::vector collectLeakedKeys() { - QDir user(_userBasePath); - if (!user.exists()) { - return {}; - } +base::flat_set CollectGoodNames() { const auto keys = { _locationsKey, _reportSpamStatusesKey, @@ -2680,40 +2677,32 @@ std::vector collectLeakedKeys() { _savedPeersKey, _trustedBotsKey }; - auto good = base::flat_set(); + auto result = base::flat_set{ "map0", "map1" }; + const auto push = [&](FileKey key) { + if (!key) { + return; + } + auto name = toFilePart(key) + '0'; + result.emplace(name); + name[name.size() - 1] = '1'; + result.emplace(name); + }; for (const auto &value : _draftsMap) { - good.emplace(toFilePart(value)); + push(value); } for (const auto &value : _draftCursorsMap) { - good.emplace(toFilePart(value)); + push(value); } for (const auto &value : keys) { - good.emplace(toFilePart(value)); - } - const auto list = user.entryList(QDir::Files); - auto result = std::vector(); - auto check = toFilePart(0); - for (const auto &name : list) { - if (name.size() != 17) { - continue; - } - memcpy(check.data(), name.data(), check.size() * sizeof(QChar)); - if (!good.contains(check)) { - result.push_back(name); - } + push(value); } return result; } -void clearLeakedFiles() { - auto leaked = collectLeakedKeys(); - if (!leaked.empty()) { - crl::async([base = _userBasePath, leaked = std::move(leaked)] { - for (const auto &name : leaked) { - QFile(base + name).remove(); - } - }); - } +void FilterLegacyFiles(FnMut&&)> then) { + crl::on_main([then = std::move(then)]() mutable { + then(CollectGoodNames()); + }); } ReadMapState readMap(const QByteArray &pass) { @@ -2723,7 +2712,7 @@ ReadMapState readMap(const QByteArray &pass) { _writeMap(WriteMapWhen::Now); } if (result != ReadMapPassNeeded) { - clearLeakedFiles(); + Storage::ClearLegacyFiles(_userBasePath, FilterLegacyFiles); } return result; } diff --git a/Telegram/SourceFiles/storage/storage_clear_legacy.cpp b/Telegram/SourceFiles/storage/storage_clear_legacy.cpp new file mode 100644 index 0000000000..993d79d639 --- /dev/null +++ b/Telegram/SourceFiles/storage/storage_clear_legacy.cpp @@ -0,0 +1,53 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#include "storage/storage_clear_legacy.h" + +#include + +namespace Storage { +namespace { + +constexpr auto kClearPartSize = size_type(10000); + +} // namespace + +void ClearLegacyFilesPart( + const QString &base, + CollectGoodFiles filter, + base::flat_set &&skip = {}) { + filter([ + =, + files = details::CollectFiles(base, kClearPartSize, skip) + ](base::flat_set &&skip) mutable { + crl::async([ + =, + files = std::move(files), + skip = std::move(skip) + ]() mutable { + for (const auto &name : files) { + if (!skip.contains(name) + && !details::RemoveLegacyFile(base + name)) { + skip.emplace(name); + } + } + if (files.size() == kClearPartSize) { + ClearLegacyFilesPart(base, filter, std::move(skip)); + } + }); + }); +} + +void ClearLegacyFiles(const QString &base, CollectGoodFiles filter) { + Expects(base.endsWith('/')); + + crl::async([=] { + ClearLegacyFilesPart(base, std::move(filter)); + }); +} + +} // namespace Storage diff --git a/Telegram/SourceFiles/storage/storage_clear_legacy.h b/Telegram/SourceFiles/storage/storage_clear_legacy.h new file mode 100644 index 0000000000..09ee4f9591 --- /dev/null +++ b/Telegram/SourceFiles/storage/storage_clear_legacy.h @@ -0,0 +1,26 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#pragma once + +namespace Storage { + +using CollectGoodFiles = Fn&&)>)>; + +void ClearLegacyFiles(const QString &base, CollectGoodFiles filter); + +namespace details { + +std::vector CollectFiles( + const QString &base, + size_type limit, + const base::flat_set &skip); + +bool RemoveLegacyFile(const QString &path); + +} // namespace details +} // namespace Storage diff --git a/Telegram/SourceFiles/storage/storage_clear_legacy_posix.cpp b/Telegram/SourceFiles/storage/storage_clear_legacy_posix.cpp new file mode 100644 index 0000000000..bb92db8f37 --- /dev/null +++ b/Telegram/SourceFiles/storage/storage_clear_legacy_posix.cpp @@ -0,0 +1,101 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#include "storage/storage_clear_legacy.h" + +#include +#include +#include + +namespace Storage { +namespace details { + +std::vector CollectFiles( + const QString &base, + size_type limit, + const base::flat_set &skip) { + Expects(base.endsWith('/')); + Expects(limit > 0); + + const auto path = QFile::encodeName(base); + const auto folder = path.mid(0, path.size() - 1); + const auto directory = opendir(folder.constData()); + if (!directory) { + return {}; + } + const auto guard = gsl::finally([&] { closedir(directory); }); + + auto result = std::vector(); + while (const auto entry = readdir(directory)) { + const auto local = entry->d_name; + if (!strcmp(local, ".") || !strcmp(local, "..")) { + continue; + } + + const auto full = path + QByteArray(local); + const auto data = full.constData(); + struct stat statbuf = { 0 }; + if (stat(full.constData(), &statbuf) != 0 || S_ISDIR(statbuf.st_mode)) { + continue; + } + + auto name = QFile::decodeName(local); + if (!skip.contains(name)) { + result.push_back(std::move(name)); + } + if (result.size() == limit) { + break; + } + } + return result; + +// // It looks like POSIX solution works fine on macOS so no need for Cocoa solution. +// +// NSString *native = [NSString stringWithUTF8String:utf8.constData()]; +// NSFileManager *manager = [NSFileManager defaultManager]; +// NSArray *properties = [NSArray arrayWithObject:NSURLIsDirectoryKey]; +// NSDirectoryEnumerator *enumerator = [manager +// enumeratorAtURL:[NSURL fileURLWithPath:native] +// includingPropertiesForKeys:properties +// options:0 +// errorHandler:^(NSURL *url, NSError *error) { +// return NO; +// }]; +// +// auto result = std::vector(); +// for (NSURL *url in enumerator) { +// NSNumber *isDirectory = nil; +// NSError *error = nil; +// if (![url getResourceValue:&isDirectory forKey:NSURLIsDirectoryKey error:&error]) { +// break; +// } else if ([isDirectory boolValue]) { +// continue; +// } +// NSString *full = [url path]; +// NSRange r = [full rangeOfString:native]; +// if (r.location != 0) { +// break; +// } +// NSString *file = [full substringFromIndex:r.length + 1]; +// auto name = QString::fromUtf8([file cStringUsingEncoding:NSUTF8StringEncoding]); +// if (!skip.contains(name)) { +// result.push_back(std::move(name)); +// } +// if (result.size() == limit) { +// break; +// } +// } +// return result; +} + +bool RemoveLegacyFile(const QString &path) { + const auto native = QFile::encodeName(path); + return unlink(native.constData()) == 0; +} + +} // namespace details +} // namespace Storage diff --git a/Telegram/SourceFiles/storage/storage_clear_legacy_win.cpp b/Telegram/SourceFiles/storage/storage_clear_legacy_win.cpp new file mode 100644 index 0000000000..112f545d3d --- /dev/null +++ b/Telegram/SourceFiles/storage/storage_clear_legacy_win.cpp @@ -0,0 +1,62 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#include "storage/storage_clear_legacy.h" + +#include + +namespace Storage { +namespace details { + +std::vector CollectFiles( + const QString &base, + size_type limit, + const base::flat_set &skip) { + Expects(base.endsWith('/')); + Expects(limit > 0); + + const auto native = QDir::toNativeSeparators(base).toStdWString(); + const auto search = native + L'*'; + + auto data = WIN32_FIND_DATA{ 0 }; + const auto handle = FindFirstFileEx( + search.c_str(), + FindExInfoBasic, + &data, + FindExSearchNameMatch, + nullptr, + 0); + if (handle == INVALID_HANDLE_VALUE) { + return {}; + } + const auto guard = gsl::finally([&] { FindClose(handle); }); + + auto result = std::vector(); + do { + const auto full = native + data.cFileName; + if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + continue; + } + const auto file = QString::fromWCharArray( + data.cFileName, + full.size() - native.size()); + auto name = QDir::fromNativeSeparators(file); + if (!skip.contains(name)) { + result.push_back(std::move(name)); + } + } while (result.size() != limit && FindNextFile(handle, &data)); + + return result; +} + +bool RemoveLegacyFile(const QString &path) { + const auto native = QDir::toNativeSeparators(path).toStdWString(); + return (::DeleteFile(native.c_str()) != 0); +} + +} // namespace details +} // namespace Storage diff --git a/Telegram/gyp/lib_storage.gyp b/Telegram/gyp/lib_storage.gyp index d24a105d04..527713546e 100644 --- a/Telegram/gyp/lib_storage.gyp +++ b/Telegram/gyp/lib_storage.gyp @@ -13,7 +13,7 @@ 'type': 'static_library', 'includes': [ 'common.gypi', - 'openssl.gypi', + 'openssl.gypi', 'qt.gypi', 'telegram_win.gypi', 'telegram_mac.gypi', @@ -50,6 +50,10 @@ '<(submodules_loc)/xxHash', ], 'sources': [ + '<(src_loc)/storage/storage_clear_legacy.cpp', + '<(src_loc)/storage/storage_clear_legacy_posix.cpp', + '<(src_loc)/storage/storage_clear_legacy_win.cpp', + '<(src_loc)/storage/storage_clear_legacy.h', '<(src_loc)/storage/storage_databases.cpp', '<(src_loc)/storage/storage_databases.h', '<(src_loc)/storage/storage_encryption.cpp', @@ -80,13 +84,15 @@ '/usr/local/macold/include/c++/v1', ], }], [ 'build_win', { - 'sources!': [ + 'sources!': [ + '<(src_loc)/storage/storage_clear_legacy_posix.cpp', '<(src_loc)/storage/storage_file_lock_posix.cpp', - ], - }, { - 'sources!': [ + ], + }, { + 'sources!': [ + '<(src_loc)/storage/storage_clear_legacy_win.cpp', '<(src_loc)/storage/storage_file_lock_win.cpp', - ], - }]], + ], + }]], }], }