From 197b3c1cb533f7eacef4ea4e3f3e3d2ed6fe615d Mon Sep 17 00:00:00 2001 From: John Preston Date: Wed, 7 Oct 2020 12:00:31 +0300 Subject: [PATCH] Push all QSaveFile-s to background thread. --- Telegram/SourceFiles/intro/intro_step.cpp | 2 + .../details/storage_file_utilities.cpp | 306 ++++++++++++++---- .../storage/details/storage_file_utilities.h | 15 +- Telegram/SourceFiles/storage/localstorage.cpp | 5 + Telegram/SourceFiles/storage/localstorage.h | 1 + .../SourceFiles/storage/storage_account.cpp | 2 + 6 files changed, 255 insertions(+), 76 deletions(-) diff --git a/Telegram/SourceFiles/intro/intro_step.cpp b/Telegram/SourceFiles/intro/intro_step.cpp index 3b4535671e..7606c30efd 100644 --- a/Telegram/SourceFiles/intro/intro_step.cpp +++ b/Telegram/SourceFiles/intro/intro_step.cpp @@ -157,6 +157,7 @@ void Step::finish(const MTPUser &user, QImage &&photo) { _account->logOut(); crl::on_main(raw, [=] { Core::App().domain().activate(raw); + Local::sync(); }); return; } @@ -203,6 +204,7 @@ void Step::createSession( if (session.supportMode()) { PrepareSupportMode(&session); } + Local::sync(); } void Step::paintEvent(QPaintEvent *e) { diff --git a/Telegram/SourceFiles/storage/details/storage_file_utilities.cpp b/Telegram/SourceFiles/storage/details/storage_file_utilities.cpp index d859d3c3d1..a59d310a2f 100644 --- a/Telegram/SourceFiles/storage/details/storage_file_utilities.cpp +++ b/Telegram/SourceFiles/storage/details/storage_file_utilities.cpp @@ -11,6 +11,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "base/platform/base_platform_file_utilities.h" #include "base/openssl_help.h" +#include #include #include @@ -23,6 +24,217 @@ constexpr auto TdfMagicLen = int(sizeof(TdfMagic)); constexpr auto kStrongIterationsCount = 100'000; +struct WriteEntry { + QString basePath; + QString base; + QByteArray data; + QByteArray md5; +}; + +class WriteManager final { +public: + explicit WriteManager(crl::weak_on_thread weak); + + void write(WriteEntry &&entry); + void writeSync(WriteEntry &&entry); + void writeSyncAll(); + +private: + void scheduleWrite(); + void writeScheduled(); + bool writeOneScheduledNow(); + void writeNow(WriteEntry &&entry); + + template + [[nodiscard]] bool open(File &file, const WriteEntry &entry, char postfix); + + [[nodiscard]] QString path(const WriteEntry &entry, char postfix) const; + [[nodiscard]] bool writeHeader( + const QString &basePath, + QFileDevice &file); + + crl::weak_on_thread _weak; + std::deque _scheduled; + +}; + +class AsyncWriteManager final { +public: + void write(WriteEntry &&entry); + void writeSync(WriteEntry &&entry); + void sync(); + void stop(); + +private: + std::optional> _manager; + bool _finished = false; + +}; + +WriteManager::WriteManager(crl::weak_on_thread weak) +: _weak(std::move(weak)) { +} + +void WriteManager::write(WriteEntry &&entry) { + const auto i = ranges::find(_scheduled, entry.base, &WriteEntry::base); + if (i == end(_scheduled)) { + _scheduled.push_back(std::move(entry)); + } else { + *i = std::move(entry); + } + scheduleWrite(); +} + +void WriteManager::writeSync(WriteEntry &&entry) { + const auto i = ranges::find(_scheduled, entry.base, &WriteEntry::base); + if (i != end(_scheduled)) { + _scheduled.erase(i); + } + writeNow(std::move(entry)); +} + +void WriteManager::writeNow(WriteEntry &&entry) { + const auto path = [&](char postfix) { + return this->path(entry, postfix); + }; + const auto open = [&](auto &file, char postfix) { + return this->open(file, entry, postfix); + }; + const auto write = [&](auto &file) { + file.write(entry.data); + file.write(entry.md5); + }; + const auto safe = path('s'); + const auto simple = path('0'); + const auto backup = path('1'); + QSaveFile save; + if (open(save, 's')) { + write(save); + if (save.commit()) { + QFile::remove(simple); + QFile::remove(backup); + return; + } + LOG(("Storage Error: Could not commit '%1'.").arg(safe)); + } + QFile plain; + if (open(plain, '0')) { + write(plain); + base::Platform::FlushFileData(plain); + plain.close(); + + QFile::remove(backup); + if (base::Platform::RenameWithOverwrite(simple, safe)) { + return; + } + QFile::remove(safe); + LOG(("Storage Error: Could not rename '%1' to '%2', removing.").arg( + simple, + safe)); + } +} + +void WriteManager::writeSyncAll() { + while (writeOneScheduledNow()) { + } +} + +bool WriteManager::writeOneScheduledNow() { + if (_scheduled.empty()) { + return false; + } + + auto entry = std::move(_scheduled.front()); + _scheduled.pop_front(); + + writeNow(std::move(entry)); + return true; +} + +bool WriteManager::writeHeader(const QString &basePath, QFileDevice &file) { + if (!file.open(QIODevice::WriteOnly)) { + const auto dir = QDir(basePath); + if (dir.exists()) { + return false; + } else if (!QDir().mkpath(dir.absolutePath())) { + return false; + } else if (!file.open(QIODevice::WriteOnly)) { + return false; + } + } + file.write(TdfMagic, TdfMagicLen); + const auto version = qint32(AppVersion); + file.write((const char*)&version, sizeof(version)); + return true; +} + +QString WriteManager::path(const WriteEntry &entry, char postfix) const { + return entry.base + postfix; +} + +template +bool WriteManager::open(File &file, const WriteEntry &entry, char postfix) { + const auto name = path(entry, postfix); + file.setFileName(name); + if (!writeHeader(entry.basePath, file)) { + LOG(("Storage Error: Could not open '%1' for writing.").arg(name)); + return false; + } + return true; +} + +void WriteManager::scheduleWrite() { + _weak.with([](WriteManager &that) { + that.writeScheduled(); + }); +} + +void WriteManager::writeScheduled() { + if (writeOneScheduledNow() && !_scheduled.empty()) { + scheduleWrite(); + } +} + +void AsyncWriteManager::write(WriteEntry &&entry) { + Expects(!_finished); + + if (!_manager) { + _manager.emplace(); + } + _manager->with([entry = std::move(entry)](WriteManager &manager) mutable { + manager.write(std::move(entry)); + }); +} + +void AsyncWriteManager::writeSync(WriteEntry &&entry) { + Expects(!_finished); + + if (!_manager) { + _manager.emplace(); + } + _manager->with_sync([&](WriteManager &manager) { + manager.writeSync(std::move(entry)); + }); +} + +void AsyncWriteManager::sync() { + if (_manager) { + _manager->with_sync([](WriteManager &manager) { + manager.writeSyncAll(); + }); + } +} + +void AsyncWriteManager::stop() { + if (_manager) { + sync(); + _manager.reset(); + } + _finished = true; +} + +AsyncWriteManager Manager; + } // namespace QString ToFilePart(FileKey val) { @@ -165,14 +377,17 @@ void EncryptedDescriptor::finish() { FileWriteDescriptor::FileWriteDescriptor( const FileKey &key, - const QString &basePath) -: FileWriteDescriptor(ToFilePart(key), basePath) { + const QString &basePath, + bool sync) +: FileWriteDescriptor(ToFilePart(key), basePath, sync) { } FileWriteDescriptor::FileWriteDescriptor( const QString &name, - const QString &basePath) -: _basePath(basePath) { + const QString &basePath, + bool sync) +: _basePath(basePath) +, _sync(sync) { init(name); } @@ -180,42 +395,6 @@ FileWriteDescriptor::~FileWriteDescriptor() { finish(); } -QString FileWriteDescriptor::path(char postfix) const { - return _base + postfix; -} - -template -bool FileWriteDescriptor::open(File &file, char postfix) { - const auto name = path(postfix); - file.setFileName(name); - if (!writeHeader(file)) { - LOG(("Storage Error: Could not open '%1' for writing.").arg(name)); - return false; - } - return true; -} - -bool FileWriteDescriptor::writeHeader(QFileDevice &file) { - if (!file.open(QIODevice::WriteOnly)) { - const auto dir = QDir(_basePath); - if (dir.exists()) { - return false; - } else if (!QDir().mkpath(dir.absolutePath())) { - return false; - } else if (!file.open(QIODevice::WriteOnly)) { - return false; - } - } - file.write(TdfMagic, TdfMagicLen); - const auto version = qint32(AppVersion); - file.write((const char*)&version, sizeof(version)); - return true; -} - -void FileWriteDescriptor::writeFooter(QFileDevice &file) { - file.write((const char*)_md5.result(), 0x10); -} - void FileWriteDescriptor::init(const QString &name) { _base = _basePath + name; _buffer.setBuffer(&_safeData); @@ -257,35 +436,16 @@ void FileWriteDescriptor::finish() { _buffer.close(); - const auto safe = path('s'); - const auto simple = path('0'); - const auto backup = path('1'); - QSaveFile save; - if (open(save, 's')) { - save.write(_safeData); - writeFooter(save); - if (save.commit()) { - QFile::remove(simple); - QFile::remove(backup); - return; - } - LOG(("Storage Error: Could not commit '%1'.").arg(safe)); - } - QFile plain; - if (open(plain, '0')) { - plain.write(_safeData); - writeFooter(plain); - base::Platform::FlushFileData(plain); - plain.close(); - - QFile::remove(backup); - if (base::Platform::RenameWithOverwrite(simple, safe)) { - return; - } - QFile::remove(safe); - LOG(("Storage Error: Could not rename '%1' to '%2', removing.").arg( - simple, - safe)); + auto entry = WriteEntry{ + .basePath = _basePath, + .base = _base, + .data = _safeData, + .md5 = QByteArray((const char*)_md5.result(), 0x10) + }; + if (_sync) { + Manager.writeSync(std::move(entry)); + } else { + Manager.write(std::move(entry)); } } @@ -504,5 +664,13 @@ bool ReadEncryptedFile( return ReadEncryptedFile(result, ToFilePart(fkey), basePath, key); } +void Sync() { + Manager.sync(); +} + +void Finish() { + Manager.stop(); +} + } // namespace details } // namespace Storage diff --git a/Telegram/SourceFiles/storage/details/storage_file_utilities.h b/Telegram/SourceFiles/storage/details/storage_file_utilities.h index dcc12ac3b2..f854c48ac1 100644 --- a/Telegram/SourceFiles/storage/details/storage_file_utilities.h +++ b/Telegram/SourceFiles/storage/details/storage_file_utilities.h @@ -56,10 +56,12 @@ class FileWriteDescriptor final { public: FileWriteDescriptor( const FileKey &key, - const QString &basePath); + const QString &basePath, + bool sync = false); FileWriteDescriptor( const QString &name, - const QString &basePath); + const QString &basePath, + bool sync = false); ~FileWriteDescriptor(); void writeData(const QByteArray &data); @@ -69,11 +71,6 @@ public: private: void init(const QString &name); - [[nodiscard]] QString path(char postfix) const; - template - [[nodiscard]] bool open(File &file, char postfix); - [[nodiscard]] bool writeHeader(QFileDevice &file); - void writeFooter(QFileDevice &file); void finish(); const QString _basePath; @@ -83,6 +80,7 @@ private: QString _base; HashMd5 _md5; int _fullSize = 0; + bool _sync = false; }; @@ -108,5 +106,8 @@ bool ReadEncryptedFile( const QString &basePath, const MTP::AuthKeyPtr &key); +void Sync(); +void Finish(); + } // namespace details } // namespace Storage diff --git a/Telegram/SourceFiles/storage/localstorage.cpp b/Telegram/SourceFiles/storage/localstorage.cpp index 68ec904717..bdb62effbc 100644 --- a/Telegram/SourceFiles/storage/localstorage.cpp +++ b/Telegram/SourceFiles/storage/localstorage.cpp @@ -338,8 +338,13 @@ bool _readOldMtpData(bool remove, ReadSettingsContext &context) { } // namespace +void sync() { + Storage::details::Sync(); +} + void finish() { delete base::take(_localLoader); + Storage::details::Finish(); } void InitialLoadTheme(); diff --git a/Telegram/SourceFiles/storage/localstorage.h b/Telegram/SourceFiles/storage/localstorage.h index 87417776ca..d79314bcb4 100644 --- a/Telegram/SourceFiles/storage/localstorage.h +++ b/Telegram/SourceFiles/storage/localstorage.h @@ -49,6 +49,7 @@ using AuthKeyPtr = std::shared_ptr; namespace Local { void start(); +void sync(); void finish(); void writeSettings(); diff --git a/Telegram/SourceFiles/storage/storage_account.cpp b/Telegram/SourceFiles/storage/storage_account.cpp index 91ed60c167..f77f9b8a18 100644 --- a/Telegram/SourceFiles/storage/storage_account.cpp +++ b/Telegram/SourceFiles/storage/storage_account.cpp @@ -577,6 +577,8 @@ void Account::reset() { QDir(LegacyTempDirectory()).removeRecursively(); QDir(temp).removeRecursively(); }); + + Local::sync(); } void Account::writeLocations() {