Async clear of legacy local storage.

Sync call to QDir::entryList is a bad idea for the user data folder.
Some users reported hanging on startup with 1.25M legacy cache files.
Now we enumerate up to 10000 files at once asynchronously and clear.

Telegram/SourceFiles/storage/localstorage.cpp

@@ -10,6 +10,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #include "storage/serialize_document.h"
 #include "storage/serialize_common.h"
 #include "storage/storage_encrypted_file.h"
+#include "storage/storage_clear_legacy.h"
 #include "chat_helpers/stickers.h"
 #include "data/data_drafts.h"
 #include "boxes/send_files_box.h"
@@ -2657,11 +2658,7 @@ void setPasscode(const QByteArray &passcode) {
 	Global::RefLocalPasscodeChanged().notify();
 }
 
-std::vector<QString> collectLeakedKeys() {
-	QDir user(_userBasePath);
-	if (!user.exists()) {
-		return {};
-	}
+base::flat_set<QString> CollectGoodNames() {
 	const auto keys = {
 		_locationsKey,
 		_reportSpamStatusesKey,
@@ -2680,40 +2677,32 @@ std::vector<QString> collectLeakedKeys() {
 		_savedPeersKey,
 		_trustedBotsKey
 	};
-	auto good = base::flat_set<QString>();
+	auto result = base::flat_set<QString>{ "map0", "map1" };
+	const auto push = [&](FileKey key) {
+		if (!key) {
+			return;
+		}
+		auto name = toFilePart(key) + '0';
+		result.emplace(name);
+		name[name.size() - 1] = '1';
+		result.emplace(name);
+	};
 	for (const auto &value : _draftsMap) {
-		good.emplace(toFilePart(value));
+		push(value);
 	}
 	for (const auto &value : _draftCursorsMap) {
-		good.emplace(toFilePart(value));
+		push(value);
 	}
 	for (const auto &value : keys) {
-		good.emplace(toFilePart(value));
-	}
-	const auto list = user.entryList(QDir::Files);
-	auto result = std::vector<QString>();
-	auto check = toFilePart(0);
-	for (const auto &name : list) {
-		if (name.size() != 17) {
-			continue;
-		}
-		memcpy(check.data(), name.data(), check.size() * sizeof(QChar));
-		if (!good.contains(check)) {
-			result.push_back(name);
-		}
+		push(value);
 	}
 	return result;
 }
 
-void clearLeakedFiles() {
-	auto leaked = collectLeakedKeys();
-	if (!leaked.empty()) {
-		crl::async([base = _userBasePath, leaked = std::move(leaked)] {
-			for (const auto &name : leaked) {
-				QFile(base + name).remove();
-			}
-		});
-	}
+void FilterLegacyFiles(FnMut<void(base::flat_set<QString>&&)> then) {
+	crl::on_main([then = std::move(then)]() mutable {
+		then(CollectGoodNames());
+	});
 }
 
 ReadMapState readMap(const QByteArray &pass) {
@@ -2723,7 +2712,7 @@ ReadMapState readMap(const QByteArray &pass) {
 		_writeMap(WriteMapWhen::Now);
 	}
 	if (result != ReadMapPassNeeded) {
-		clearLeakedFiles();
+		Storage::ClearLegacyFiles(_userBasePath, FilterLegacyFiles);
 	}
 	return result;
 }

Telegram/SourceFiles/storage/storage_clear_legacy.cpp

@@ -0,0 +1,53 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/storage_clear_legacy.h"
+
+#include <crl/crl_async.h>
+
+namespace Storage {
+namespace {
+
+constexpr auto kClearPartSize = size_type(10000);
+
+} // namespace
+
+void ClearLegacyFilesPart(
+		const QString &base,
+		CollectGoodFiles filter,
+		base::flat_set<QString> &&skip = {}) {
+	filter([
+		=,
+		files = details::CollectFiles(base, kClearPartSize, skip)
+	](base::flat_set<QString> &&skip) mutable {
+		crl::async([
+			=,
+			files = std::move(files),
+			skip = std::move(skip)
+		]() mutable {
+			for (const auto &name : files) {
+				if (!skip.contains(name)
+					&& !details::RemoveLegacyFile(base + name)) {
+					skip.emplace(name);
+				}
+			}
+			if (files.size() == kClearPartSize) {
+				ClearLegacyFilesPart(base, filter, std::move(skip));
+			}
+		});
+	});
+}
+
+void ClearLegacyFiles(const QString &base, CollectGoodFiles filter) {
+	Expects(base.endsWith('/'));
+
+	crl::async([=] {
+		ClearLegacyFilesPart(base, std::move(filter));
+	});
+}
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_clear_legacy.h

@@ -0,0 +1,26 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+namespace Storage {
+
+using CollectGoodFiles = Fn<void(FnMut<void(base::flat_set<QString>&&)>)>;
+
+void ClearLegacyFiles(const QString &base, CollectGoodFiles filter);
+
+namespace details {
+
+std::vector<QString> CollectFiles(
+	const QString &base,
+	size_type limit,
+	const base::flat_set<QString> &skip);
+
+bool RemoveLegacyFile(const QString &path);
+
+} // namespace details
+} // namespace Storage

Telegram/SourceFiles/storage/storage_clear_legacy_posix.cpp

@@ -0,0 +1,101 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/storage_clear_legacy.h"
+
+#include <sys/stat.h>
+#include <dirent.h>
+#include <unistd.h>
+
+namespace Storage {
+namespace details {
+
+std::vector<QString> CollectFiles(
+		const QString &base,
+		size_type limit,
+		const base::flat_set<QString> &skip) {
+	Expects(base.endsWith('/'));
+	Expects(limit > 0);
+
+	const auto path = QFile::encodeName(base);
+	const auto folder = path.mid(0, path.size() - 1);
+	const auto directory = opendir(folder.constData());
+	if (!directory) {
+		return {};
+	}
+	const auto guard = gsl::finally([&] { closedir(directory); });
+
+	auto result = std::vector<QString>();
+	while (const auto entry = readdir(directory)) {
+		const auto local = entry->d_name;
+		if (!strcmp(local, ".") || !strcmp(local, "..")) {
+			continue;
+		}
+
+		const auto full = path + QByteArray(local);
+		const auto data = full.constData();
+		struct stat statbuf = { 0 };
+		if (stat(full.constData(), &statbuf) != 0 || S_ISDIR(statbuf.st_mode)) {
+			continue;
+		}
+
+		auto name = QFile::decodeName(local);
+		if (!skip.contains(name)) {
+			result.push_back(std::move(name));
+		}
+		if (result.size() == limit) {
+			break;
+		}
+	}
+	return result;
+
+// // It looks like POSIX solution works fine on macOS so no need for Cocoa solution.
+//
+//	NSString *native = [NSString stringWithUTF8String:utf8.constData()];
+//	NSFileManager *manager = [NSFileManager defaultManager];
+//	NSArray *properties = [NSArray arrayWithObject:NSURLIsDirectoryKey];
+//	NSDirectoryEnumerator *enumerator = [manager
+//		enumeratorAtURL:[NSURL fileURLWithPath:native]
+//		includingPropertiesForKeys:properties
+//		options:0
+//		errorHandler:^(NSURL *url, NSError *error) {
+//			return NO;
+//		}];
+//
+//	auto result = std::vector<QString>();
+//	for (NSURL *url in enumerator) {
+//		NSNumber *isDirectory = nil;
+//		NSError *error = nil;
+//		if (![url getResourceValue:&isDirectory forKey:NSURLIsDirectoryKey error:&error]) {
+//			break;
+//		} else if ([isDirectory boolValue]) {
+//			continue;
+//		}
+//		NSString *full = [url path];
+//		NSRange r = [full rangeOfString:native];
+//		if (r.location != 0) {
+//			break;
+//		}
+//		NSString *file = [full substringFromIndex:r.length + 1];
+//		auto name = QString::fromUtf8([file cStringUsingEncoding:NSUTF8StringEncoding]);
+//		if (!skip.contains(name)) {
+//			result.push_back(std::move(name));
+//		}
+//		if (result.size() == limit) {
+//			break;
+//		}
+//	}
+//	return result;
+}
+
+bool RemoveLegacyFile(const QString &path) {
+	const auto native = QFile::encodeName(path);
+	return unlink(native.constData()) == 0;
+}
+
+} // namespace details
+} // namespace Storage

Telegram/SourceFiles/storage/storage_clear_legacy_win.cpp

@@ -0,0 +1,62 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/storage_clear_legacy.h"
+
+#include <Windows.h>
+
+namespace Storage {
+namespace details {
+
+std::vector<QString> CollectFiles(
+		const QString &base,
+		size_type limit,
+		const base::flat_set<QString> &skip) {
+	Expects(base.endsWith('/'));
+	Expects(limit > 0);
+
+	const auto native = QDir::toNativeSeparators(base).toStdWString();
+	const auto search = native + L'*';
+
+	auto data = WIN32_FIND_DATA{ 0 };
+	const auto handle = FindFirstFileEx(
+		search.c_str(),
+		FindExInfoBasic,
+		&data,
+		FindExSearchNameMatch,
+		nullptr,
+		0);
+	if (handle == INVALID_HANDLE_VALUE) {
+		return {};
+	}
+	const auto guard = gsl::finally([&] { FindClose(handle); });
+
+	auto result = std::vector<QString>();
+	do {
+		const auto full = native + data.cFileName;
+		if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+			continue;
+		}
+		const auto file = QString::fromWCharArray(
+			data.cFileName,
+			full.size() - native.size());
+		auto name = QDir::fromNativeSeparators(file);
+		if (!skip.contains(name)) {
+			result.push_back(std::move(name));
+		}
+	} while (result.size() != limit && FindNextFile(handle, &data));
+
+	return result;
+}
+
+bool RemoveLegacyFile(const QString &path) {
+	const auto native = QDir::toNativeSeparators(path).toStdWString();
+	return (::DeleteFile(native.c_str()) != 0);
+}
+
+} // namespace details
+} // namespace Storage

Telegram/gyp/lib_storage.gyp

@@ -13,7 +13,7 @@
     'type': 'static_library',
     'includes': [
       'common.gypi',
-	  'openssl.gypi',
+      'openssl.gypi',
       'qt.gypi',
       'telegram_win.gypi',
       'telegram_mac.gypi',
@@ -50,6 +50,10 @@
       '<(submodules_loc)/xxHash',
     ],
     'sources': [
+      '<(src_loc)/storage/storage_clear_legacy.cpp',
+      '<(src_loc)/storage/storage_clear_legacy_posix.cpp',
+      '<(src_loc)/storage/storage_clear_legacy_win.cpp',
+      '<(src_loc)/storage/storage_clear_legacy.h',
       '<(src_loc)/storage/storage_databases.cpp',
       '<(src_loc)/storage/storage_databases.h',
       '<(src_loc)/storage/storage_encryption.cpp',
@@ -80,13 +84,15 @@
         '/usr/local/macold/include/c++/v1',
       ],
     }], [ 'build_win', {
-	  'sources!': [
+      'sources!': [
+        '<(src_loc)/storage/storage_clear_legacy_posix.cpp',
         '<(src_loc)/storage/storage_file_lock_posix.cpp',
-	  ],
-	}, {
-	  'sources!': [
+      ],
+    }, {
+      'sources!': [
+        '<(src_loc)/storage/storage_clear_legacy_win.cpp',
         '<(src_loc)/storage/storage_file_lock_win.cpp',
-	  ],
-	}]],
+      ],
+    }]],
   }],
 }