From 4611727ab9006f1be4a3acb95567e1d5aeeccb1b Mon Sep 17 00:00:00 2001 From: John Preston Date: Wed, 4 Dec 2019 15:15:58 +0300 Subject: [PATCH] Rewrite webFileLoader without Storage::DownloadManager. --- Telegram/SourceFiles/apiwrap.cpp | 2 +- Telegram/SourceFiles/core/application.cpp | 1 - Telegram/SourceFiles/data/data_document.cpp | 2 + .../passport/passport_form_controller.cpp | 2 +- .../SourceFiles/storage/file_download.cpp | 1050 +---------------- Telegram/SourceFiles/storage/file_download.h | 253 +--- .../storage/file_download_mtproto.cpp | 590 +++++++++ .../storage/file_download_mtproto.h | 140 +++ .../SourceFiles/storage/file_download_web.cpp | 527 +++++++++ .../SourceFiles/storage/file_download_web.h | 45 + .../storage/streamed_file_downloader.cpp | 11 +- .../storage/streamed_file_downloader.h | 3 +- .../SourceFiles/ui/image/image_source.cpp | 5 +- Telegram/gyp/telegram/sources.txt | 4 + 14 files changed, 1334 insertions(+), 1301 deletions(-) create mode 100644 Telegram/SourceFiles/storage/file_download_mtproto.cpp create mode 100644 Telegram/SourceFiles/storage/file_download_mtproto.h create mode 100644 Telegram/SourceFiles/storage/file_download_web.cpp create mode 100644 Telegram/SourceFiles/storage/file_download_web.h diff --git a/Telegram/SourceFiles/apiwrap.cpp b/Telegram/SourceFiles/apiwrap.cpp index 9cb0a8f54e..b9272d3ec4 100644 --- a/Telegram/SourceFiles/apiwrap.cpp +++ b/Telegram/SourceFiles/apiwrap.cpp @@ -55,7 +55,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "ui/emoji_config.h" #include "support/support_helper.h" #include "storage/localimageloader.h" -#include "storage/file_download.h" +#include "storage/file_download_mtproto.h" #include "storage/file_upload.h" #include "storage/storage_facade.h" #include "storage/storage_shared_media.h" diff --git a/Telegram/SourceFiles/core/application.cpp b/Telegram/SourceFiles/core/application.cpp index 9954a1faaf..c13abe1d27 100644 --- a/Telegram/SourceFiles/core/application.cpp +++ b/Telegram/SourceFiles/core/application.cpp @@ -139,7 +139,6 @@ Application::~Application() { Ui::Emoji::Clear(); Media::Clip::Finish(); - stopWebLoadManager(); App::deinitMedia(); Window::Theme::Uninitialize(); diff --git a/Telegram/SourceFiles/data/data_document.cpp b/Telegram/SourceFiles/data/data_document.cpp index f23a484b00..f00a70b80b 100644 --- a/Telegram/SourceFiles/data/data_document.cpp +++ b/Telegram/SourceFiles/data/data_document.cpp @@ -21,6 +21,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "media/streaming/media_streaming_loader_local.h" #include "storage/localstorage.h" #include "storage/streamed_file_downloader.h" +#include "storage/file_download_mtproto.h" +#include "storage/file_download_web.h" #include "platform/platform_specific.h" #include "history/history.h" #include "history/history_item.h" diff --git a/Telegram/SourceFiles/passport/passport_form_controller.cpp b/Telegram/SourceFiles/passport/passport_form_controller.cpp index 6d4e2c0fb2..05b3c88b1e 100644 --- a/Telegram/SourceFiles/passport/passport_form_controller.cpp +++ b/Telegram/SourceFiles/passport/passport_form_controller.cpp @@ -28,7 +28,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "storage/localimageloader.h" #include "storage/localstorage.h" #include "storage/file_upload.h" -#include "storage/file_download.h" +#include "storage/file_download_mtproto.h" #include "app.h" #include "apiwrap.h" diff --git a/Telegram/SourceFiles/storage/file_download.cpp b/Telegram/SourceFiles/storage/file_download.cpp index ec759ef4d4..77f6f951aa 100644 --- a/Telegram/SourceFiles/storage/file_download.cpp +++ b/Telegram/SourceFiles/storage/file_download.cpp @@ -37,12 +37,6 @@ constexpr auto kMaxWaitedInConnection = 512 * 1024; // Max 8 http[s] files downloaded at the same time. constexpr auto kMaxWebFileQueries = 8; -// Different part sizes are not supported for now :( -// Because we start downloading with some part size -// and then we get a cdn-redirect where we support only -// fixed part size download for hash checking. -constexpr auto kPartSize = 128 * 1024; - constexpr auto kStartSessionsCount = 1; constexpr auto kMaxSessionsCount = 8; constexpr auto kResetDownloadPrioritiesTimeout = crl::time(200); @@ -85,7 +79,7 @@ bool DownloadManager::Queue::empty() const { Downloader *DownloadManager::Queue::nextLoader() const { auto &&all = ranges::view::concat(_loaders, _previousGeneration); - const auto i = ranges::find(all, true, &FileLoader::readyToRequest); + const auto i = ranges::find(all, true, &Downloader::readyToRequest); return (i != all.end()) ? i->get() : nullptr; } @@ -134,8 +128,8 @@ void DownloadManager::checkSendNext() { return 0; } const auto j = ranges::min_element(i->second); - const auto inConnection = *j; - return (inConnection + kPartSize <= kMaxWaitedInConnection) + const auto already = *j; + return (already + kDownloadPartSize <= kMaxWaitedInConnection) ? (j - begin(i->second)) : -1; }(); @@ -233,28 +227,15 @@ void DownloadManager::killDownloadSessions() { } // namespace Storage -namespace { - -QThread *_webLoadThread = nullptr; -WebLoadManager *_webLoadManager = nullptr; -WebLoadManager *webLoadManager() { - return (_webLoadManager && _webLoadManager != FinishedWebLoadManager) ? _webLoadManager : nullptr; -} -WebLoadMainManager *_webLoadMainManager = nullptr; - -} // namespace - FileLoader::FileLoader( const QString &toFile, - MTP::DcId dcId, int32 size, LocationType locationType, LoadToCacheSetting toCache, LoadFromCloudSetting fromCloud, bool autoLoading, uint8 cacheTag) -: _dcId(dcId) -, _downloader(&Auth().downloader()) +: _session(&Auth()) , _autoLoading(autoLoading) , _cacheTag(cacheTag) , _filename(toFile) @@ -266,12 +247,10 @@ FileLoader::FileLoader( Expects(!_filename.isEmpty() || (_size <= Storage::kMaxFileInMemory)); } -FileLoader::~FileLoader() { - _downloader->remove(this); -} +FileLoader::~FileLoader() = default; Main::Session &FileLoader::session() const { - return _downloader->api().session(); + return *_session; } void FileLoader::finishWithBytes(const QByteArray &data) { @@ -297,7 +276,7 @@ void FileLoader::finishWithBytes(const QByteArray &data) { Platform::File::PostprocessDownloaded( QFileInfo(_file).absoluteFilePath()); } - _downloader->taskFinished().notify(); + Auth().downloaderTaskFinished().notify(); } QByteArray FileLoader::imageFormat(const QSize &shrinkBox) const { @@ -390,7 +369,7 @@ void FileLoader::start() { return cancel(true); } } - _downloader->enqueue(this); + startLoading(); } void FileLoader::loadLocal(const Storage::Cache::Key &key) { @@ -576,8 +555,6 @@ bool FileLoader::finalizeResult() { Platform::File::PostprocessDownloaded( QFileInfo(_file).absoluteFilePath()); } - _downloader->remove(this); - if (_localStatus == LocalStatus::NotFound) { if (const auto key = fileLocationKey()) { if (!_filename.isEmpty()) { @@ -593,1015 +570,6 @@ bool FileLoader::finalizeResult() { _cacheTag)); } } - _downloader->taskFinished().notify(); + Auth().downloaderTaskFinished().notify(); return true; } - -mtpFileLoader::mtpFileLoader( - const StorageFileLocation &location, - Data::FileOrigin origin, - LocationType type, - const QString &to, - int32 size, - LoadToCacheSetting toCache, - LoadFromCloudSetting fromCloud, - bool autoLoading, - uint8 cacheTag) -: FileLoader( - to, - location.dcId(), - size, - type, - toCache, - fromCloud, - autoLoading, - cacheTag) -, _location(location) -, _origin(origin) { -} - -mtpFileLoader::mtpFileLoader( - const WebFileLocation &location, - int32 size, - LoadFromCloudSetting fromCloud, - bool autoLoading, - uint8 cacheTag) -: FileLoader( - QString(), - Global::WebFileDcId(), - size, - UnknownFileLocation, - LoadToCacheAsWell, - fromCloud, - autoLoading, - cacheTag) -, _location(location) { -} - -mtpFileLoader::mtpFileLoader( - const GeoPointLocation &location, - int32 size, - LoadFromCloudSetting fromCloud, - bool autoLoading, - uint8 cacheTag) -: FileLoader( - QString(), - Global::WebFileDcId(), - size, - UnknownFileLocation, - LoadToCacheAsWell, - fromCloud, - autoLoading, - cacheTag) -, _location(location) { -} - -mtpFileLoader::~mtpFileLoader() { - cancelRequests(); -} - -Data::FileOrigin mtpFileLoader::fileOrigin() const { - return _origin; -} - -uint64 mtpFileLoader::objId() const { - if (const auto storage = base::get_if(&_location)) { - return storage->objectId(); - } - return 0; -} - -void mtpFileLoader::refreshFileReferenceFrom( - const Data::UpdatedFileReferences &updates, - int requestId, - const QByteArray ¤t) { - if (const auto storage = base::get_if(&_location)) { - storage->refreshFileReference(updates); - if (storage->fileReference() == current) { - cancel(true); - return; - } - } else { - cancel(true); - return; - } - makeRequest(finishSentRequest(requestId)); -} - -bool mtpFileLoader::readyToRequest() const { - return !_finished - && !_lastComplete - && (_sentRequests.empty() || _size != 0) - && (!_size || _nextRequestOffset < _size); -} - -void mtpFileLoader::loadPart(int dcIndex) { - Expects(readyToRequest()); - - makeRequest({ _nextRequestOffset, dcIndex }); - _nextRequestOffset += Storage::kPartSize; -} - -mtpRequestId mtpFileLoader::sendRequest(const RequestData &requestData) { - const auto offset = requestData.offset; - const auto limit = Storage::kPartSize; - const auto shiftedDcId = MTP::downloadDcId( - _cdnDcId ? _cdnDcId : dcId(), - requestData.dcIndex); - if (_cdnDcId) { - return MTP::send( - MTPupload_GetCdnFile( - MTP_bytes(_cdnToken), - MTP_int(offset), - MTP_int(limit)), - rpcDone(&mtpFileLoader::cdnPartLoaded), - rpcFail(&mtpFileLoader::cdnPartFailed), - shiftedDcId, - 50); - } - return _location.match([&](const WebFileLocation &location) { - return MTP::send( - MTPupload_GetWebFile( - MTP_inputWebFileLocation( - MTP_bytes(location.url()), - MTP_long(location.accessHash())), - MTP_int(offset), - MTP_int(limit)), - rpcDone(&mtpFileLoader::webPartLoaded), - rpcFail(&mtpFileLoader::partFailed), - shiftedDcId, - 50); - }, [&](const GeoPointLocation &location) { - return MTP::send( - MTPupload_GetWebFile( - MTP_inputWebFileGeoPointLocation( - MTP_inputGeoPoint( - MTP_double(location.lat), - MTP_double(location.lon)), - MTP_long(location.access), - MTP_int(location.width), - MTP_int(location.height), - MTP_int(location.zoom), - MTP_int(location.scale)), - MTP_int(offset), - MTP_int(limit)), - rpcDone(&mtpFileLoader::webPartLoaded), - rpcFail(&mtpFileLoader::partFailed), - shiftedDcId, - 50); - }, [&](const StorageFileLocation &location) { - return MTP::send( - MTPupload_GetFile( - MTP_flags(0), - location.tl(session().userId()), - MTP_int(offset), - MTP_int(limit)), - rpcDone(&mtpFileLoader::normalPartLoaded), - rpcFail( - &mtpFileLoader::normalPartFailed, - location.fileReference()), - shiftedDcId, - 50); - }); -} - -void mtpFileLoader::makeRequest(const RequestData &requestData) { - Expects(!_finished); - - placeSentRequest(sendRequest(requestData), requestData); -} - -void mtpFileLoader::requestMoreCdnFileHashes() { - Expects(!_finished); - - if (_cdnHashesRequestId || _cdnUncheckedParts.empty()) { - return; - } - - const auto requestData = _cdnUncheckedParts.cbegin()->first; - const auto shiftedDcId = MTP::downloadDcId( - dcId(), - requestData.dcIndex); - const auto requestId = _cdnHashesRequestId = MTP::send( - MTPupload_GetCdnFileHashes( - MTP_bytes(_cdnToken), - MTP_int(requestData.offset)), - rpcDone(&mtpFileLoader::getCdnFileHashesDone), - rpcFail(&mtpFileLoader::cdnPartFailed), - shiftedDcId); - placeSentRequest(requestId, requestData); -} - -void mtpFileLoader::normalPartLoaded( - const MTPupload_File &result, - mtpRequestId requestId) { - Expects(!_finished); - - const auto requestData = finishSentRequest(requestId); - result.match([&](const MTPDupload_fileCdnRedirect &data) { - switchToCDN(requestData, data); - }, [&](const MTPDupload_file &data) { - partLoaded(requestData.offset, bytes::make_span(data.vbytes().v)); - }); -} - -void mtpFileLoader::webPartLoaded( - const MTPupload_WebFile &result, - mtpRequestId requestId) { - result.match([&](const MTPDupload_webFile &data) { - const auto requestData = finishSentRequest(requestId); - if (!_size) { - _size = data.vsize().v; - } else if (data.vsize().v != _size) { - LOG(("MTP Error: " - "Bad size provided by bot for webDocument: %1, real: %2" - ).arg(_size - ).arg(data.vsize().v)); - cancel(true); - return; - } - partLoaded(requestData.offset, bytes::make_span(data.vbytes().v)); - }); -} - -void mtpFileLoader::cdnPartLoaded(const MTPupload_CdnFile &result, mtpRequestId requestId) { - Expects(!_finished); - - const auto requestData = finishSentRequest(requestId); - result.match([&](const MTPDupload_cdnFileReuploadNeeded &data) { - const auto shiftedDcId = MTP::downloadDcId( - dcId(), - requestData.dcIndex); - const auto requestId = MTP::send( - MTPupload_ReuploadCdnFile( - MTP_bytes(_cdnToken), - data.vrequest_token()), - rpcDone(&mtpFileLoader::reuploadDone), - rpcFail(&mtpFileLoader::cdnPartFailed), - shiftedDcId); - placeSentRequest(requestId, requestData); - }, [&](const MTPDupload_cdnFile &data) { - auto key = bytes::make_span(_cdnEncryptionKey); - auto iv = bytes::make_span(_cdnEncryptionIV); - Expects(key.size() == MTP::CTRState::KeySize); - Expects(iv.size() == MTP::CTRState::IvecSize); - - auto state = MTP::CTRState(); - auto ivec = bytes::make_span(state.ivec); - std::copy(iv.begin(), iv.end(), ivec.begin()); - - auto counterOffset = static_cast(requestData.offset) >> 4; - state.ivec[15] = static_cast(counterOffset & 0xFF); - state.ivec[14] = static_cast((counterOffset >> 8) & 0xFF); - state.ivec[13] = static_cast((counterOffset >> 16) & 0xFF); - state.ivec[12] = static_cast((counterOffset >> 24) & 0xFF); - - auto decryptInPlace = data.vbytes().v; - auto buffer = bytes::make_detached_span(decryptInPlace); - MTP::aesCtrEncrypt(buffer, key.data(), &state); - - switch (checkCdnFileHash(requestData.offset, buffer)) { - case CheckCdnHashResult::NoHash: { - _cdnUncheckedParts.emplace(requestData, decryptInPlace); - requestMoreCdnFileHashes(); - } return; - - case CheckCdnHashResult::Invalid: { - LOG(("API Error: Wrong cdnFileHash for offset %1." - ).arg(requestData.offset)); - cancel(true); - } return; - - case CheckCdnHashResult::Good: { - partLoaded(requestData.offset, buffer); - } return; - } - Unexpected("Result of checkCdnFileHash()"); - }); -} - -mtpFileLoader::CheckCdnHashResult mtpFileLoader::checkCdnFileHash( - int offset, - bytes::const_span buffer) { - const auto cdnFileHashIt = _cdnFileHashes.find(offset); - if (cdnFileHashIt == _cdnFileHashes.cend()) { - return CheckCdnHashResult::NoHash; - } - const auto realHash = openssl::Sha256(buffer); - const auto receivedHash = bytes::make_span(cdnFileHashIt->second.hash); - if (bytes::compare(realHash, receivedHash)) { - return CheckCdnHashResult::Invalid; - } - return CheckCdnHashResult::Good; -} - -void mtpFileLoader::reuploadDone( - const MTPVector &result, - mtpRequestId requestId) { - const auto requestData = finishSentRequest(requestId); - addCdnHashes(result.v); - makeRequest(requestData); -} - -void mtpFileLoader::getCdnFileHashesDone( - const MTPVector &result, - mtpRequestId requestId) { - Expects(!_finished); - Expects(_cdnHashesRequestId == requestId); - - _cdnHashesRequestId = 0; - - const auto requestData = finishSentRequest(requestId); - addCdnHashes(result.v); - auto someMoreChecked = false; - for (auto i = _cdnUncheckedParts.begin(); i != _cdnUncheckedParts.cend();) { - const auto uncheckedData = i->first; - const auto uncheckedBytes = bytes::make_span(i->second); - - switch (checkCdnFileHash(uncheckedData.offset, uncheckedBytes)) { - case CheckCdnHashResult::NoHash: { - ++i; - } break; - - case CheckCdnHashResult::Invalid: { - LOG(("API Error: Wrong cdnFileHash for offset %1." - ).arg(uncheckedData.offset)); - cancel(true); - return; - } break; - - case CheckCdnHashResult::Good: { - someMoreChecked = true; - const auto goodOffset = uncheckedData.offset; - const auto goodBytes = std::move(i->second); - const auto weak = QPointer(this); - i = _cdnUncheckedParts.erase(i); - if (!feedPart(goodOffset, bytes::make_span(goodBytes)) - || !weak) { - return; - } else if (_finished) { - notifyAboutProgress(); - return; - } - } break; - - default: Unexpected("Result of checkCdnFileHash()"); - } - } - if (someMoreChecked) { - const auto weak = QPointer(this); - notifyAboutProgress(); - if (weak) { - requestMoreCdnFileHashes(); - } - return; - } - LOG(("API Error: " - "Could not find cdnFileHash for offset %1 " - "after getCdnFileHashes request." - ).arg(requestData.offset)); - cancel(true); -} - -void mtpFileLoader::placeSentRequest( - mtpRequestId requestId, - const RequestData &requestData) { - Expects(!_finished); - - _downloader->requestedAmountIncrement( - dcId(), - requestData.dcIndex, - Storage::kPartSize); - _sentRequests.emplace(requestId, requestData); -} - -auto mtpFileLoader::finishSentRequest(mtpRequestId requestId) --> RequestData { - auto it = _sentRequests.find(requestId); - Assert(it != _sentRequests.cend()); - - const auto result = it->second; - _downloader->requestedAmountIncrement( - dcId(), - result.dcIndex, - -Storage::kPartSize); - _sentRequests.erase(it); - - return result; -} - -bool mtpFileLoader::feedPart(int offset, bytes::const_span buffer) { - if (!writeResultPart(offset, buffer)) { - return false; - } - if (buffer.empty() || (buffer.size() % 1024)) { // bad next offset - _lastComplete = true; - } - const auto finished = _sentRequests.empty() - && _cdnUncheckedParts.empty() - && (_lastComplete || (_size && _nextRequestOffset >= _size)); - if (finished && !finalizeResult()) { - return false; - } - return true; -} - -void mtpFileLoader::partLoaded(int offset, bytes::const_span buffer) { - if (feedPart(offset, buffer)) { - notifyAboutProgress(); - } -} - -bool mtpFileLoader::normalPartFailed( - QByteArray fileReference, - const RPCError &error, - mtpRequestId requestId) { - if (MTP::isDefaultHandledError(error)) { - return false; - } - if (error.code() == 400 - && error.type().startsWith(qstr("FILE_REFERENCE_"))) { - session().api().refreshFileReference( - _origin, - this, - requestId, - fileReference); - return true; - } - return partFailed(error, requestId); -} - - -bool mtpFileLoader::partFailed( - const RPCError &error, - mtpRequestId requestId) { - if (MTP::isDefaultHandledError(error)) { - return false; - } - cancel(true); - return true; -} - -bool mtpFileLoader::cdnPartFailed( - const RPCError &error, - mtpRequestId requestId) { - if (MTP::isDefaultHandledError(error)) { - return false; - } - - if (requestId == _cdnHashesRequestId) { - _cdnHashesRequestId = 0; - } - if (error.type() == qstr("FILE_TOKEN_INVALID") - || error.type() == qstr("REQUEST_TOKEN_INVALID")) { - const auto requestData = finishSentRequest(requestId); - changeCDNParams( - requestData, - 0, - QByteArray(), - QByteArray(), - QByteArray(), - QVector()); - return true; - } - return partFailed(error, requestId); -} - -void mtpFileLoader::cancelRequests() { - while (!_sentRequests.empty()) { - auto requestId = _sentRequests.begin()->first; - MTP::cancel(requestId); - [[maybe_unused]] const auto data = finishSentRequest(requestId); - } -} - -void mtpFileLoader::switchToCDN( - const RequestData &requestData, - const MTPDupload_fileCdnRedirect &redirect) { - changeCDNParams( - requestData, - redirect.vdc_id().v, - redirect.vfile_token().v, - redirect.vencryption_key().v, - redirect.vencryption_iv().v, - redirect.vfile_hashes().v); -} - -void mtpFileLoader::addCdnHashes(const QVector &hashes) { - for (const auto &hash : hashes) { - hash.match([&](const MTPDfileHash &data) { - _cdnFileHashes.emplace( - data.voffset().v, - CdnFileHash{ data.vlimit().v, data.vhash().v }); - }); - } -} - -void mtpFileLoader::changeCDNParams( - const RequestData &requestData, - MTP::DcId dcId, - const QByteArray &token, - const QByteArray &encryptionKey, - const QByteArray &encryptionIV, - const QVector &hashes) { - if (dcId != 0 - && (encryptionKey.size() != MTP::CTRState::KeySize - || encryptionIV.size() != MTP::CTRState::IvecSize)) { - LOG(("Message Error: Wrong key (%1) / iv (%2) size in CDN params").arg(encryptionKey.size()).arg(encryptionIV.size())); - cancel(true); - return; - } - - auto resendAllRequests = (_cdnDcId != dcId - || _cdnToken != token - || _cdnEncryptionKey != encryptionKey - || _cdnEncryptionIV != encryptionIV); - _cdnDcId = dcId; - _cdnToken = token; - _cdnEncryptionKey = encryptionKey; - _cdnEncryptionIV = encryptionIV; - addCdnHashes(hashes); - - if (resendAllRequests && !_sentRequests.empty()) { - auto resendRequests = std::vector(); - resendRequests.reserve(_sentRequests.size()); - while (!_sentRequests.empty()) { - auto requestId = _sentRequests.begin()->first; - MTP::cancel(requestId); - resendRequests.push_back(finishSentRequest(requestId)); - } - for (const auto &requestData : resendRequests) { - makeRequest(requestData); - } - } - makeRequest(requestData); -} - -Storage::Cache::Key mtpFileLoader::cacheKey() const { - return _location.match([&](const WebFileLocation &location) { - return Data::WebDocumentCacheKey(location); - }, [&](const GeoPointLocation &location) { - return Data::GeoPointCacheKey(location); - }, [&](const StorageFileLocation &location) { - return location.cacheKey(); - }); -} - -std::optional mtpFileLoader::fileLocationKey() const { - if (_locationType != UnknownFileLocation) { - return mediaKey(_locationType, dcId(), objId()); - } - return std::nullopt; -} - -webFileLoader::webFileLoader( - const QString &url, - const QString &to, - LoadFromCloudSetting fromCloud, - bool autoLoading, - uint8 cacheTag) -: FileLoader( - QString(), - 0, - 0, - UnknownFileLocation, - LoadToCacheAsWell, - fromCloud, - autoLoading, - cacheTag) -, _url(url) { -} - -webFileLoader::~webFileLoader() { - markAsNotSent(); -} - -bool webFileLoader::readyToRequest() const { - return !_finished - && !_requestSent - && (_webLoadManager != FinishedWebLoadManager); -} - -void webFileLoader::loadPart(int dcIndex) { - Expects(readyToRequest()); - - if (!_webLoadManager) { - _webLoadMainManager = new WebLoadMainManager(); - - _webLoadThread = new QThread(); - _webLoadManager = new WebLoadManager(_webLoadThread); - - _webLoadThread->start(); - } - - markAsSent(); - _webLoadManager->append(this, _url); -} - -int webFileLoader::currentOffset() const { - return _already; -} - -void webFileLoader::loadProgress(qint64 already, qint64 size) { - _size = size; - _already = already; - notifyAboutProgress(); -} - -void webFileLoader::loadFinished(const QByteArray &data) { - markAsNotSent(); - if (writeResultPart(0, bytes::make_span(data))) { - if (finalizeResult()) { - notifyAboutProgress(); - } - } -} - -void webFileLoader::loadError() { - markAsNotSent(); - cancel(true); -} - -Storage::Cache::Key webFileLoader::cacheKey() const { - return Data::UrlCacheKey(_url); -} - -std::optional webFileLoader::fileLocationKey() const { - return std::nullopt; -} - -void webFileLoader::cancelRequests() { - if (!webLoadManager()) { - return; - } - webLoadManager()->stop(this); - markAsNotSent(); -} - -void webFileLoader::markAsSent() { - if (_requestSent) { - return; - } - _requestSent = true; - _downloader->requestedAmountIncrement(0, 0, 1); -} - -void webFileLoader::markAsNotSent() { - if (!_requestSent) { - return; - } - _requestSent = false; - _downloader->requestedAmountIncrement(0, 0, -1); -} - -class webFileLoaderPrivate { -public: - webFileLoaderPrivate(webFileLoader *loader, const QString &url) - : _interface(loader) - , _url(url) - , _redirectsLeft(kMaxHttpRedirects) { - } - - QNetworkReply *reply() { - return _reply; - } - - QNetworkReply *request(QNetworkAccessManager &manager, const QString &redirect) { - if (!redirect.isEmpty()) _url = redirect; - - QNetworkRequest req(_url); - QByteArray rangeHeaderValue = "bytes=" + QByteArray::number(_already) + "-"; - req.setRawHeader("Range", rangeHeaderValue); - _reply = manager.get(req); - return _reply; - } - - bool oneMoreRedirect() { - if (_redirectsLeft) { - --_redirectsLeft; - return true; - } - return false; - } - - void setData(const QByteArray &data) { - _data = data; - } - void addData(const QByteArray &data) { - _data.append(data); - } - const QByteArray &data() { - return _data; - } - void setProgress(qint64 already, qint64 size) { - _already = already; - _size = qMax(size, 0LL); - } - - qint64 size() const { - return _size; - } - qint64 already() const { - return _already; - } - -private: - static constexpr auto kMaxHttpRedirects = 5; - - webFileLoader *_interface = nullptr; - QUrl _url; - qint64 _already = 0; - qint64 _size = 0; - QNetworkReply *_reply = nullptr; - int32 _redirectsLeft = kMaxHttpRedirects; - QByteArray _data; - - friend class WebLoadManager; -}; - -void stopWebLoadManager() { - if (webLoadManager()) { - _webLoadThread->quit(); - DEBUG_LOG(("Waiting for webloadThread to finish")); - _webLoadThread->wait(); - delete _webLoadManager; - delete _webLoadMainManager; - delete _webLoadThread; - _webLoadThread = nullptr; - _webLoadMainManager = nullptr; - _webLoadManager = FinishedWebLoadManager; - } -} - -WebLoadManager::WebLoadManager(QThread *thread) { - moveToThread(thread); - _manager.moveToThread(thread); - connect(thread, SIGNAL(started()), this, SLOT(process())); - connect(thread, SIGNAL(finished()), this, SLOT(finish())); - connect(this, SIGNAL(processDelayed()), this, SLOT(process()), Qt::QueuedConnection); - - connect(this, SIGNAL(progress(webFileLoader*,qint64,qint64)), _webLoadMainManager, SLOT(progress(webFileLoader*,qint64,qint64))); - connect(this, SIGNAL(finished(webFileLoader*,QByteArray)), _webLoadMainManager, SLOT(finished(webFileLoader*,QByteArray))); - connect(this, SIGNAL(error(webFileLoader*)), _webLoadMainManager, SLOT(error(webFileLoader*))); - - connect(&_manager, SIGNAL(authenticationRequired(QNetworkReply*,QAuthenticator*)), this, SLOT(onFailed(QNetworkReply*))); -#ifndef OS_MAC_OLD - connect(&_manager, SIGNAL(sslErrors(QNetworkReply*,const QList&)), this, SLOT(onFailed(QNetworkReply*))); -#endif // OS_MAC_OLD -} - -WebLoadManager::~WebLoadManager() { - clear(); -} - -void WebLoadManager::append(webFileLoader *loader, const QString &url) { - loader->_private = new webFileLoaderPrivate(loader, url); - - QMutexLocker lock(&_loaderPointersMutex); - _loaderPointers.insert(loader, loader->_private); - emit processDelayed(); -} - -void WebLoadManager::stop(webFileLoader *loader) { - QMutexLocker lock(&_loaderPointersMutex); - _loaderPointers.remove(loader); - emit processDelayed(); -} - -bool WebLoadManager::carries(webFileLoader *loader) const { - QMutexLocker lock(&_loaderPointersMutex); - return _loaderPointers.contains(loader); -} - -bool WebLoadManager::handleReplyResult(webFileLoaderPrivate *loader, WebReplyProcessResult result) { - QMutexLocker lock(&_loaderPointersMutex); - LoaderPointers::iterator it = _loaderPointers.find(loader->_interface); - if (it != _loaderPointers.cend() && it.key()->_private != loader) { - it = _loaderPointers.end(); // it is a new loader which was realloced in the same address - } - if (it == _loaderPointers.cend()) { - return false; - } - - if (result == WebReplyProcessProgress) { - if (loader->size() > Storage::kMaxFileInMemory) { - LOG(("API Error: too large file is loaded to cache: %1").arg(loader->size())); - result = WebReplyProcessError; - } - } - if (result == WebReplyProcessError) { - if (it != _loaderPointers.cend()) { - emit error(it.key()); - } - return false; - } - if (loader->already() < loader->size() || !loader->size()) { - emit progress(it.key(), loader->already(), loader->size()); - return true; - } - emit finished(it.key(), loader->data()); - return false; -} - -void WebLoadManager::onFailed(QNetworkReply::NetworkError error) { - onFailed(qobject_cast(QObject::sender())); -} - -void WebLoadManager::onFailed(QNetworkReply *reply) { - if (!reply) return; - reply->deleteLater(); - - Replies::iterator j = _replies.find(reply); - if (j == _replies.cend()) { // handled already - return; - } - webFileLoaderPrivate *loader = j.value(); - _replies.erase(j); - - LOG(("Network Error: Failed to request '%1', error %2 (%3)").arg(QString::fromLatin1(loader->_url.toEncoded())).arg(int(reply->error())).arg(reply->errorString())); - - if (!handleReplyResult(loader, WebReplyProcessError)) { - _loaders.remove(loader); - delete loader; - } -} - -void WebLoadManager::onProgress(qint64 already, qint64 size) { - const auto reply = qobject_cast(QObject::sender()); - if (!reply) return; - - const auto j = _replies.find(reply); - if (j == _replies.cend()) { // handled already - return; - } - const auto loader = j.value(); - - auto result = WebReplyProcessProgress; - const auto statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute); - const auto status = statusCode.isValid() ? statusCode.toInt() : 200; - if (status != 200 && status != 206 && status != 416) { - if (status == 301 || status == 302) { - QString loc = reply->header(QNetworkRequest::LocationHeader).toString(); - if (!loc.isEmpty()) { - if (loader->oneMoreRedirect()) { - sendRequest(loader, loc); - return; - } else { - LOG(("Network Error: Too many HTTP redirects in onFinished() for web file loader: %1").arg(loc)); - result = WebReplyProcessError; - } - } - } else { - LOG(("Network Error: Bad HTTP status received in WebLoadManager::onProgress(): %1").arg(statusCode.toInt())); - result = WebReplyProcessError; - } - } else { - loader->setProgress(already, size); - QByteArray r = reply->readAll(); - if (!r.isEmpty()) { - loader->addData(r); - } - if (size == 0) { - LOG(("Network Error: Zero size received for HTTP download progress in WebLoadManager::onProgress(): %1 / %2").arg(already).arg(size)); - result = WebReplyProcessError; - } - } - if (!handleReplyResult(loader, result)) { - _replies.erase(j); - _loaders.remove(loader); - delete loader; - - reply->abort(); - reply->deleteLater(); - } -} - -void WebLoadManager::onMeta() { - const auto reply = qobject_cast(QObject::sender()); - if (!reply) return; - - const auto j = _replies.find(reply); - if (j == _replies.cend()) { // handled already - return; - } - const auto loader = j.value(); - - const auto pairs = reply->rawHeaderPairs(); - for (const auto &pair : pairs) { - if (QString::fromUtf8(pair.first).toLower() == "content-range") { - const auto m = QRegularExpression(qsl("/(\\d+)([^\\d]|$)")).match(QString::fromUtf8(pair.second)); - if (m.hasMatch()) { - loader->setProgress(qMax(qint64(loader->data().size()), loader->already()), m.captured(1).toLongLong()); - if (!handleReplyResult(loader, WebReplyProcessProgress)) { - _replies.erase(j); - _loaders.remove(loader); - delete loader; - - reply->abort(); - reply->deleteLater(); - } - } - } - } -} - -void WebLoadManager::process() { - Loaders newLoaders; - { - QMutexLocker lock(&_loaderPointersMutex); - for (LoaderPointers::iterator i = _loaderPointers.begin(), e = _loaderPointers.end(); i != e; ++i) { - Loaders::iterator it = _loaders.find(i.value()); - if (i.value()) { - if (it == _loaders.cend()) { - _loaders.insert(i.value()); - newLoaders.insert(i.value()); - } - i.value() = 0; - } - } - for (auto i = _loaders.begin(), e = _loaders.end(); i != e;) { - LoaderPointers::iterator it = _loaderPointers.find((*i)->_interface); - if (it != _loaderPointers.cend() && it.key()->_private != (*i)) { - it = _loaderPointers.end(); - } - if (it == _loaderPointers.cend()) { - if (QNetworkReply *reply = (*i)->reply()) { - _replies.remove(reply); - reply->abort(); - reply->deleteLater(); - } - delete (*i); - i = _loaders.erase(i); - } else { - ++i; - } - } - } - for_const (webFileLoaderPrivate *loader, newLoaders) { - if (_loaders.contains(loader)) { - sendRequest(loader); - } - } -} - -void WebLoadManager::sendRequest(webFileLoaderPrivate *loader, const QString &redirect) { - Replies::iterator j = _replies.find(loader->reply()); - if (j != _replies.cend()) { - QNetworkReply *r = j.key(); - _replies.erase(j); - - r->abort(); - r->deleteLater(); - } - - QNetworkReply *r = loader->request(_manager, redirect); - - // Those use QObject::sender, so don't just remove the receiver pointer! - connect(r, SIGNAL(downloadProgress(qint64, qint64)), this, SLOT(onProgress(qint64, qint64))); - connect(r, SIGNAL(error(QNetworkReply::NetworkError)), this, SLOT(onFailed(QNetworkReply::NetworkError))); - connect(r, SIGNAL(metaDataChanged()), this, SLOT(onMeta())); - - _replies.insert(r, loader); -} - -void WebLoadManager::finish() { - clear(); -} - -void WebLoadManager::clear() { - QMutexLocker lock(&_loaderPointersMutex); - for (auto i = _loaderPointers.begin(), e = _loaderPointers.end(); i != e; ++i) { - if (i.value()) { - i.key()->_private = nullptr; - } - } - _loaderPointers.clear(); - - for (const auto loader : _loaders) { - delete loader; - } - _loaders.clear(); - - for (auto i = _replies.begin(), e = _replies.end(); i != e; ++i) { - delete i.key(); - } - _replies.clear(); -} - -void WebLoadMainManager::progress(webFileLoader *loader, qint64 already, qint64 size) { - if (webLoadManager() && webLoadManager()->carries(loader)) { - loader->loadProgress(already, size); - } -} - -void WebLoadMainManager::finished(webFileLoader *loader, QByteArray data) { - if (webLoadManager() && webLoadManager()->carries(loader)) { - loader->loadFinished(data); - } -} - -void WebLoadMainManager::error(webFileLoader *loader) { - if (webLoadManager() && webLoadManager()->carries(loader)) { - loader->loadError(); - } -} diff --git a/Telegram/SourceFiles/storage/file_download.h b/Telegram/SourceFiles/storage/file_download.h index 520501de96..7f84ff8207 100644 --- a/Telegram/SourceFiles/storage/file_download.h +++ b/Telegram/SourceFiles/storage/file_download.h @@ -35,6 +35,12 @@ constexpr auto kMaxWallPaperInMemory = kMaxFileInMemory; constexpr auto kMaxAnimationInMemory = kMaxFileInMemory; // 10 MB gif and mp4 animations held in memory while playing constexpr auto kMaxWallPaperDimension = 4096; // 4096x4096 is max area. +// Different part sizes are not supported for now :( +// Because we start downloading with some part size +// and then we get a cdn-redirect where we support only +// fixed part size download for hash checking. +constexpr auto kDownloadPartSize = 128 * 1024; + class Downloader { public: virtual ~Downloader() = default; @@ -114,13 +120,12 @@ struct StorageImageSaved { }; -class FileLoader : public QObject, public Storage::Downloader { +class FileLoader : public QObject { Q_OBJECT public: FileLoader( const QString &toFile, - MTP::DcId dcId, int32 size, LocationType locationType, LoadToCacheSetting toCache, @@ -180,8 +185,6 @@ signals: void failed(FileLoader *loader, bool started); protected: - friend class Storage::DownloadManager; - enum class LocalStatus { NotTried, NotFound, @@ -189,10 +192,6 @@ protected: Loaded, }; - MTP::DcId dcId() const override { - return _dcId; - } - void readImage(const QSize &shrinkBox) const; bool tryLoadLocal(); @@ -200,6 +199,7 @@ protected: virtual Storage::Cache::Key cacheKey() const = 0; virtual std::optional fileLocationKey() const = 0; virtual void cancelRequests() = 0; + virtual void startLoading() = 0; void cancel(bool failed); @@ -209,8 +209,7 @@ protected: bool finalizeResult(); [[nodiscard]] QByteArray readLoadedPartBack(int offset, int size); - const MTP::DcId _dcId = 0; - const not_null _downloader; + const not_null _session; bool _autoLoading = false; uint8 _cacheTag = 0; @@ -236,237 +235,3 @@ protected: mutable QImage _imageData; }; - -class StorageImageLocation; -class WebFileLocation; -class mtpFileLoader final : public FileLoader, public RPCSender { -public: - mtpFileLoader( - const StorageFileLocation &location, - Data::FileOrigin origin, - LocationType type, - const QString &toFile, - int32 size, - LoadToCacheSetting toCache, - LoadFromCloudSetting fromCloud, - bool autoLoading, - uint8 cacheTag); - mtpFileLoader( - const WebFileLocation &location, - int32 size, - LoadFromCloudSetting fromCloud, - bool autoLoading, - uint8 cacheTag); - mtpFileLoader( - const GeoPointLocation &location, - int32 size, - LoadFromCloudSetting fromCloud, - bool autoLoading, - uint8 cacheTag); - - Data::FileOrigin fileOrigin() const override; - - uint64 objId() const override; - - void stop() override { - rpcInvalidate(); - } - void refreshFileReferenceFrom( - const Data::UpdatedFileReferences &updates, - int requestId, - const QByteArray ¤t); - - ~mtpFileLoader(); - -private: - friend class DownloadManager; - - struct RequestData { - int offset = 0; - int dcIndex = 0; - - inline bool operator<(const RequestData &other) const { - return offset < other.offset; - } - }; - struct CdnFileHash { - CdnFileHash(int limit, QByteArray hash) : limit(limit), hash(hash) { - } - int limit = 0; - QByteArray hash; - }; - Storage::Cache::Key cacheKey() const override; - std::optional fileLocationKey() const override; - void cancelRequests() override; - - void makeRequest(const RequestData &requestData); - - bool readyToRequest() const override; - void loadPart(int dcIndex) override; - void normalPartLoaded(const MTPupload_File &result, mtpRequestId requestId); - void webPartLoaded(const MTPupload_WebFile &result, mtpRequestId requestId); - void cdnPartLoaded(const MTPupload_CdnFile &result, mtpRequestId requestId); - void reuploadDone(const MTPVector &result, mtpRequestId requestId); - void requestMoreCdnFileHashes(); - void getCdnFileHashesDone(const MTPVector &result, mtpRequestId requestId); - - void partLoaded(int offset, bytes::const_span buffer); - bool feedPart(int offset, bytes::const_span buffer); - - bool partFailed(const RPCError &error, mtpRequestId requestId); - bool normalPartFailed(QByteArray fileReference, const RPCError &error, mtpRequestId requestId); - bool cdnPartFailed(const RPCError &error, mtpRequestId requestId); - - mtpRequestId sendRequest(const RequestData &requestData); - void placeSentRequest( - mtpRequestId requestId, - const RequestData &requestData); - [[nodiscard]] RequestData finishSentRequest(mtpRequestId requestId); - void switchToCDN( - const RequestData &requestData, - const MTPDupload_fileCdnRedirect &redirect); - void addCdnHashes(const QVector &hashes); - void changeCDNParams( - const RequestData &requestData, - MTP::DcId dcId, - const QByteArray &token, - const QByteArray &encryptionKey, - const QByteArray &encryptionIV, - const QVector &hashes); - - enum class CheckCdnHashResult { - NoHash, - Invalid, - Good, - }; - CheckCdnHashResult checkCdnFileHash(int offset, bytes::const_span buffer); - - std::map _sentRequests; - - bool _lastComplete = false; - int32 _nextRequestOffset = 0; - - base::variant< - StorageFileLocation, - WebFileLocation, - GeoPointLocation> _location; - Data::FileOrigin _origin; - - MTP::DcId _cdnDcId = 0; - QByteArray _cdnToken; - QByteArray _cdnEncryptionKey; - QByteArray _cdnEncryptionIV; - base::flat_map _cdnFileHashes; - base::flat_map _cdnUncheckedParts; - mtpRequestId _cdnHashesRequestId = 0; - -}; - -class webFileLoaderPrivate; - -class webFileLoader final : public FileLoader { -public: - webFileLoader( - const QString &url, - const QString &to, - LoadFromCloudSetting fromCloud, - bool autoLoading, - uint8 cacheTag); - - int currentOffset() const override; - - void loadProgress(qint64 already, qint64 size); - void loadFinished(const QByteArray &data); - void loadError(); - - void stop() override { - cancelRequests(); - } - - ~webFileLoader(); - -private: - void cancelRequests() override; - Storage::Cache::Key cacheKey() const override; - std::optional fileLocationKey() const override; - bool readyToRequest() const override; - void loadPart(int dcIndex) override; - - void markAsSent(); - void markAsNotSent(); - - QString _url; - - bool _requestSent = false; - int32 _already = 0; - - friend class WebLoadManager; - webFileLoaderPrivate *_private = nullptr; - -}; - -enum WebReplyProcessResult { - WebReplyProcessError, - WebReplyProcessProgress, - WebReplyProcessFinished, -}; - -class WebLoadManager : public QObject { - Q_OBJECT - -public: - WebLoadManager(QThread *thread); - - void append(webFileLoader *loader, const QString &url); - void stop(webFileLoader *reader); - bool carries(webFileLoader *reader) const; - - ~WebLoadManager(); - -signals: - void processDelayed(); - - void progress(webFileLoader *loader, qint64 already, qint64 size); - void finished(webFileLoader *loader, QByteArray data); - void error(webFileLoader *loader); - -public slots: - void onFailed(QNetworkReply *reply); - void onFailed(QNetworkReply::NetworkError error); - void onProgress(qint64 already, qint64 size); - void onMeta(); - - void process(); - void finish(); - -private: - void clear(); - void sendRequest(webFileLoaderPrivate *loader, const QString &redirect = QString()); - bool handleReplyResult(webFileLoaderPrivate *loader, WebReplyProcessResult result); - - QNetworkAccessManager _manager; - typedef QMap LoaderPointers; - LoaderPointers _loaderPointers; - mutable QMutex _loaderPointersMutex; - - typedef OrderedSet Loaders; - Loaders _loaders; - - typedef QMap Replies; - Replies _replies; - -}; - -class WebLoadMainManager : public QObject { - Q_OBJECT - -public slots: - void progress(webFileLoader *loader, qint64 already, qint64 size); - void finished(webFileLoader *loader, QByteArray data); - void error(webFileLoader *loader); - -}; - -static WebLoadManager * const FinishedWebLoadManager = SharedMemoryLocation(); - -void stopWebLoadManager(); diff --git a/Telegram/SourceFiles/storage/file_download_mtproto.cpp b/Telegram/SourceFiles/storage/file_download_mtproto.cpp new file mode 100644 index 0000000000..0cb04513af --- /dev/null +++ b/Telegram/SourceFiles/storage/file_download_mtproto.cpp @@ -0,0 +1,590 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#include "storage/file_download_mtproto.h" + +#include "data/data_document.h" +#include "data/data_file_origin.h" +#include "storage/cache/storage_cache_types.h" +#include "main/main_session.h" +#include "apiwrap.h" +#include "mtproto/mtp_instance.h" +#include "mtproto/mtproto_auth_key.h" +#include "base/openssl_help.h" +#include "facades.h" + +mtpFileLoader::mtpFileLoader( + const StorageFileLocation &location, + Data::FileOrigin origin, + LocationType type, + const QString &to, + int32 size, + LoadToCacheSetting toCache, + LoadFromCloudSetting fromCloud, + bool autoLoading, + uint8 cacheTag) +: FileLoader( + to, + size, + type, + toCache, + fromCloud, + autoLoading, + cacheTag) +, _downloader(&session().downloader()) +, _dcId(location.dcId()) +, _location(location) +, _origin(origin) { +} + +mtpFileLoader::mtpFileLoader( + const WebFileLocation &location, + int32 size, + LoadFromCloudSetting fromCloud, + bool autoLoading, + uint8 cacheTag) +: FileLoader( + QString(), + size, + UnknownFileLocation, + LoadToCacheAsWell, + fromCloud, + autoLoading, + cacheTag) +, _downloader(&session().downloader()) +, _dcId(Global::WebFileDcId()) +, _location(location) { +} + +mtpFileLoader::mtpFileLoader( + const GeoPointLocation &location, + int32 size, + LoadFromCloudSetting fromCloud, + bool autoLoading, + uint8 cacheTag) +: FileLoader( + QString(), + size, + UnknownFileLocation, + LoadToCacheAsWell, + fromCloud, + autoLoading, + cacheTag) +, _downloader(&session().downloader()) +, _dcId(Global::WebFileDcId()) +, _location(location) { +} + +mtpFileLoader::~mtpFileLoader() { + cancelRequests(); + _downloader->remove(this); +} + +Data::FileOrigin mtpFileLoader::fileOrigin() const { + return _origin; +} + +uint64 mtpFileLoader::objId() const { + if (const auto storage = base::get_if(&_location)) { + return storage->objectId(); + } + return 0; +} + +void mtpFileLoader::refreshFileReferenceFrom( + const Data::UpdatedFileReferences &updates, + int requestId, + const QByteArray ¤t) { + if (const auto storage = base::get_if(&_location)) { + storage->refreshFileReference(updates); + if (storage->fileReference() == current) { + cancel(true); + return; + } + } else { + cancel(true); + return; + } + makeRequest(finishSentRequest(requestId)); +} + +MTP::DcId mtpFileLoader::dcId() const { + return _dcId; +} + +bool mtpFileLoader::readyToRequest() const { + return !_finished + && !_lastComplete + && (_sentRequests.empty() || _size != 0) + && (!_size || _nextRequestOffset < _size); +} + +void mtpFileLoader::loadPart(int dcIndex) { + Expects(readyToRequest()); + + makeRequest({ _nextRequestOffset, dcIndex }); + _nextRequestOffset += Storage::kDownloadPartSize; +} + +mtpRequestId mtpFileLoader::sendRequest(const RequestData &requestData) { + const auto offset = requestData.offset; + const auto limit = Storage::kDownloadPartSize; + const auto shiftedDcId = MTP::downloadDcId( + _cdnDcId ? _cdnDcId : dcId(), + requestData.dcIndex); + if (_cdnDcId) { + return MTP::send( + MTPupload_GetCdnFile( + MTP_bytes(_cdnToken), + MTP_int(offset), + MTP_int(limit)), + rpcDone(&mtpFileLoader::cdnPartLoaded), + rpcFail(&mtpFileLoader::cdnPartFailed), + shiftedDcId, + 50); + } + return _location.match([&](const WebFileLocation &location) { + return MTP::send( + MTPupload_GetWebFile( + MTP_inputWebFileLocation( + MTP_bytes(location.url()), + MTP_long(location.accessHash())), + MTP_int(offset), + MTP_int(limit)), + rpcDone(&mtpFileLoader::webPartLoaded), + rpcFail(&mtpFileLoader::partFailed), + shiftedDcId, + 50); + }, [&](const GeoPointLocation &location) { + return MTP::send( + MTPupload_GetWebFile( + MTP_inputWebFileGeoPointLocation( + MTP_inputGeoPoint( + MTP_double(location.lat), + MTP_double(location.lon)), + MTP_long(location.access), + MTP_int(location.width), + MTP_int(location.height), + MTP_int(location.zoom), + MTP_int(location.scale)), + MTP_int(offset), + MTP_int(limit)), + rpcDone(&mtpFileLoader::webPartLoaded), + rpcFail(&mtpFileLoader::partFailed), + shiftedDcId, + 50); + }, [&](const StorageFileLocation &location) { + return MTP::send( + MTPupload_GetFile( + MTP_flags(0), + location.tl(session().userId()), + MTP_int(offset), + MTP_int(limit)), + rpcDone(&mtpFileLoader::normalPartLoaded), + rpcFail( + &mtpFileLoader::normalPartFailed, + location.fileReference()), + shiftedDcId, + 50); + }); +} + +void mtpFileLoader::makeRequest(const RequestData &requestData) { + Expects(!_finished); + + placeSentRequest(sendRequest(requestData), requestData); +} + +void mtpFileLoader::requestMoreCdnFileHashes() { + Expects(!_finished); + + if (_cdnHashesRequestId || _cdnUncheckedParts.empty()) { + return; + } + + const auto requestData = _cdnUncheckedParts.cbegin()->first; + const auto shiftedDcId = MTP::downloadDcId( + dcId(), + requestData.dcIndex); + const auto requestId = _cdnHashesRequestId = MTP::send( + MTPupload_GetCdnFileHashes( + MTP_bytes(_cdnToken), + MTP_int(requestData.offset)), + rpcDone(&mtpFileLoader::getCdnFileHashesDone), + rpcFail(&mtpFileLoader::cdnPartFailed), + shiftedDcId); + placeSentRequest(requestId, requestData); +} + +void mtpFileLoader::normalPartLoaded( + const MTPupload_File &result, + mtpRequestId requestId) { + Expects(!_finished); + + const auto requestData = finishSentRequest(requestId); + result.match([&](const MTPDupload_fileCdnRedirect &data) { + switchToCDN(requestData, data); + }, [&](const MTPDupload_file &data) { + partLoaded(requestData.offset, bytes::make_span(data.vbytes().v)); + }); +} + +void mtpFileLoader::webPartLoaded( + const MTPupload_WebFile &result, + mtpRequestId requestId) { + result.match([&](const MTPDupload_webFile &data) { + const auto requestData = finishSentRequest(requestId); + if (!_size) { + _size = data.vsize().v; + } else if (data.vsize().v != _size) { + LOG(("MTP Error: " + "Bad size provided by bot for webDocument: %1, real: %2" + ).arg(_size + ).arg(data.vsize().v)); + cancel(true); + return; + } + partLoaded(requestData.offset, bytes::make_span(data.vbytes().v)); + }); +} + +void mtpFileLoader::cdnPartLoaded(const MTPupload_CdnFile &result, mtpRequestId requestId) { + Expects(!_finished); + + const auto requestData = finishSentRequest(requestId); + result.match([&](const MTPDupload_cdnFileReuploadNeeded &data) { + const auto shiftedDcId = MTP::downloadDcId( + dcId(), + requestData.dcIndex); + const auto requestId = MTP::send( + MTPupload_ReuploadCdnFile( + MTP_bytes(_cdnToken), + data.vrequest_token()), + rpcDone(&mtpFileLoader::reuploadDone), + rpcFail(&mtpFileLoader::cdnPartFailed), + shiftedDcId); + placeSentRequest(requestId, requestData); + }, [&](const MTPDupload_cdnFile &data) { + auto key = bytes::make_span(_cdnEncryptionKey); + auto iv = bytes::make_span(_cdnEncryptionIV); + Expects(key.size() == MTP::CTRState::KeySize); + Expects(iv.size() == MTP::CTRState::IvecSize); + + auto state = MTP::CTRState(); + auto ivec = bytes::make_span(state.ivec); + std::copy(iv.begin(), iv.end(), ivec.begin()); + + auto counterOffset = static_cast(requestData.offset) >> 4; + state.ivec[15] = static_cast(counterOffset & 0xFF); + state.ivec[14] = static_cast((counterOffset >> 8) & 0xFF); + state.ivec[13] = static_cast((counterOffset >> 16) & 0xFF); + state.ivec[12] = static_cast((counterOffset >> 24) & 0xFF); + + auto decryptInPlace = data.vbytes().v; + auto buffer = bytes::make_detached_span(decryptInPlace); + MTP::aesCtrEncrypt(buffer, key.data(), &state); + + switch (checkCdnFileHash(requestData.offset, buffer)) { + case CheckCdnHashResult::NoHash: { + _cdnUncheckedParts.emplace(requestData, decryptInPlace); + requestMoreCdnFileHashes(); + } return; + + case CheckCdnHashResult::Invalid: { + LOG(("API Error: Wrong cdnFileHash for offset %1." + ).arg(requestData.offset)); + cancel(true); + } return; + + case CheckCdnHashResult::Good: { + partLoaded(requestData.offset, buffer); + } return; + } + Unexpected("Result of checkCdnFileHash()"); + }); +} + +mtpFileLoader::CheckCdnHashResult mtpFileLoader::checkCdnFileHash( + int offset, + bytes::const_span buffer) { + const auto cdnFileHashIt = _cdnFileHashes.find(offset); + if (cdnFileHashIt == _cdnFileHashes.cend()) { + return CheckCdnHashResult::NoHash; + } + const auto realHash = openssl::Sha256(buffer); + const auto receivedHash = bytes::make_span(cdnFileHashIt->second.hash); + if (bytes::compare(realHash, receivedHash)) { + return CheckCdnHashResult::Invalid; + } + return CheckCdnHashResult::Good; +} + +void mtpFileLoader::reuploadDone( + const MTPVector &result, + mtpRequestId requestId) { + const auto requestData = finishSentRequest(requestId); + addCdnHashes(result.v); + makeRequest(requestData); +} + +void mtpFileLoader::getCdnFileHashesDone( + const MTPVector &result, + mtpRequestId requestId) { + Expects(!_finished); + Expects(_cdnHashesRequestId == requestId); + + _cdnHashesRequestId = 0; + + const auto requestData = finishSentRequest(requestId); + addCdnHashes(result.v); + auto someMoreChecked = false; + for (auto i = _cdnUncheckedParts.begin(); i != _cdnUncheckedParts.cend();) { + const auto uncheckedData = i->first; + const auto uncheckedBytes = bytes::make_span(i->second); + + switch (checkCdnFileHash(uncheckedData.offset, uncheckedBytes)) { + case CheckCdnHashResult::NoHash: { + ++i; + } break; + + case CheckCdnHashResult::Invalid: { + LOG(("API Error: Wrong cdnFileHash for offset %1." + ).arg(uncheckedData.offset)); + cancel(true); + return; + } break; + + case CheckCdnHashResult::Good: { + someMoreChecked = true; + const auto goodOffset = uncheckedData.offset; + const auto goodBytes = std::move(i->second); + const auto weak = QPointer(this); + i = _cdnUncheckedParts.erase(i); + if (!feedPart(goodOffset, bytes::make_span(goodBytes)) + || !weak) { + return; + } else if (_finished) { + notifyAboutProgress(); + return; + } + } break; + + default: Unexpected("Result of checkCdnFileHash()"); + } + } + if (someMoreChecked) { + const auto weak = QPointer(this); + notifyAboutProgress(); + if (weak) { + requestMoreCdnFileHashes(); + } + return; + } + LOG(("API Error: " + "Could not find cdnFileHash for offset %1 " + "after getCdnFileHashes request." + ).arg(requestData.offset)); + cancel(true); +} + +void mtpFileLoader::placeSentRequest( + mtpRequestId requestId, + const RequestData &requestData) { + Expects(!_finished); + + _downloader->requestedAmountIncrement( + dcId(), + requestData.dcIndex, + Storage::kDownloadPartSize); + _sentRequests.emplace(requestId, requestData); +} + +auto mtpFileLoader::finishSentRequest(mtpRequestId requestId) +-> RequestData { + auto it = _sentRequests.find(requestId); + Assert(it != _sentRequests.cend()); + + const auto result = it->second; + _downloader->requestedAmountIncrement( + dcId(), + result.dcIndex, + -Storage::kDownloadPartSize); + _sentRequests.erase(it); + + return result; +} + +bool mtpFileLoader::feedPart(int offset, bytes::const_span buffer) { + if (!writeResultPart(offset, buffer)) { + return false; + } + if (buffer.empty() || (buffer.size() % 1024)) { // bad next offset + _lastComplete = true; + } + const auto finished = _sentRequests.empty() + && _cdnUncheckedParts.empty() + && (_lastComplete || (_size && _nextRequestOffset >= _size)); + if (finished) { + _downloader->remove(this); + if (!finalizeResult()) { + return false; + } + } + return true; +} + +void mtpFileLoader::partLoaded(int offset, bytes::const_span buffer) { + if (feedPart(offset, buffer)) { + notifyAboutProgress(); + } +} + +bool mtpFileLoader::normalPartFailed( + QByteArray fileReference, + const RPCError &error, + mtpRequestId requestId) { + if (MTP::isDefaultHandledError(error)) { + return false; + } + if (error.code() == 400 + && error.type().startsWith(qstr("FILE_REFERENCE_"))) { + session().api().refreshFileReference( + _origin, + this, + requestId, + fileReference); + return true; + } + return partFailed(error, requestId); +} + + +bool mtpFileLoader::partFailed( + const RPCError &error, + mtpRequestId requestId) { + if (MTP::isDefaultHandledError(error)) { + return false; + } + cancel(true); + return true; +} + +bool mtpFileLoader::cdnPartFailed( + const RPCError &error, + mtpRequestId requestId) { + if (MTP::isDefaultHandledError(error)) { + return false; + } + + if (requestId == _cdnHashesRequestId) { + _cdnHashesRequestId = 0; + } + if (error.type() == qstr("FILE_TOKEN_INVALID") + || error.type() == qstr("REQUEST_TOKEN_INVALID")) { + const auto requestData = finishSentRequest(requestId); + changeCDNParams( + requestData, + 0, + QByteArray(), + QByteArray(), + QByteArray(), + QVector()); + return true; + } + return partFailed(error, requestId); +} + +void mtpFileLoader::startLoading() { + _downloader->enqueue(this); +} + +void mtpFileLoader::cancelRequests() { + while (!_sentRequests.empty()) { + auto requestId = _sentRequests.begin()->first; + MTP::cancel(requestId); + [[maybe_unused]] const auto data = finishSentRequest(requestId); + } +} + +void mtpFileLoader::switchToCDN( + const RequestData &requestData, + const MTPDupload_fileCdnRedirect &redirect) { + changeCDNParams( + requestData, + redirect.vdc_id().v, + redirect.vfile_token().v, + redirect.vencryption_key().v, + redirect.vencryption_iv().v, + redirect.vfile_hashes().v); +} + +void mtpFileLoader::addCdnHashes(const QVector &hashes) { + for (const auto &hash : hashes) { + hash.match([&](const MTPDfileHash &data) { + _cdnFileHashes.emplace( + data.voffset().v, + CdnFileHash{ data.vlimit().v, data.vhash().v }); + }); + } +} + +void mtpFileLoader::changeCDNParams( + const RequestData &requestData, + MTP::DcId dcId, + const QByteArray &token, + const QByteArray &encryptionKey, + const QByteArray &encryptionIV, + const QVector &hashes) { + if (dcId != 0 + && (encryptionKey.size() != MTP::CTRState::KeySize + || encryptionIV.size() != MTP::CTRState::IvecSize)) { + LOG(("Message Error: Wrong key (%1) / iv (%2) size in CDN params").arg(encryptionKey.size()).arg(encryptionIV.size())); + cancel(true); + return; + } + + auto resendAllRequests = (_cdnDcId != dcId + || _cdnToken != token + || _cdnEncryptionKey != encryptionKey + || _cdnEncryptionIV != encryptionIV); + _cdnDcId = dcId; + _cdnToken = token; + _cdnEncryptionKey = encryptionKey; + _cdnEncryptionIV = encryptionIV; + addCdnHashes(hashes); + + if (resendAllRequests && !_sentRequests.empty()) { + auto resendRequests = std::vector(); + resendRequests.reserve(_sentRequests.size()); + while (!_sentRequests.empty()) { + auto requestId = _sentRequests.begin()->first; + MTP::cancel(requestId); + resendRequests.push_back(finishSentRequest(requestId)); + } + for (const auto &requestData : resendRequests) { + makeRequest(requestData); + } + } + makeRequest(requestData); +} + +Storage::Cache::Key mtpFileLoader::cacheKey() const { + return _location.match([&](const WebFileLocation &location) { + return Data::WebDocumentCacheKey(location); + }, [&](const GeoPointLocation &location) { + return Data::GeoPointCacheKey(location); + }, [&](const StorageFileLocation &location) { + return location.cacheKey(); + }); +} + +std::optional mtpFileLoader::fileLocationKey() const { + if (_locationType != UnknownFileLocation) { + return mediaKey(_locationType, dcId(), objId()); + } + return std::nullopt; +} diff --git a/Telegram/SourceFiles/storage/file_download_mtproto.h b/Telegram/SourceFiles/storage/file_download_mtproto.h new file mode 100644 index 0000000000..01cadd3b2a --- /dev/null +++ b/Telegram/SourceFiles/storage/file_download_mtproto.h @@ -0,0 +1,140 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#pragma once + +#include "storage/file_download.h" + +class StorageImageLocation; +class WebFileLocation; +class mtpFileLoader final + : public FileLoader + , public RPCSender + , public Storage::Downloader { +public: + mtpFileLoader( + const StorageFileLocation &location, + Data::FileOrigin origin, + LocationType type, + const QString &toFile, + int32 size, + LoadToCacheSetting toCache, + LoadFromCloudSetting fromCloud, + bool autoLoading, + uint8 cacheTag); + mtpFileLoader( + const WebFileLocation &location, + int32 size, + LoadFromCloudSetting fromCloud, + bool autoLoading, + uint8 cacheTag); + mtpFileLoader( + const GeoPointLocation &location, + int32 size, + LoadFromCloudSetting fromCloud, + bool autoLoading, + uint8 cacheTag); + + Data::FileOrigin fileOrigin() const override; + + uint64 objId() const override; + + void stop() override { + rpcInvalidate(); + } + void refreshFileReferenceFrom( + const Data::UpdatedFileReferences &updates, + int requestId, + const QByteArray ¤t); + + ~mtpFileLoader(); + +private: + struct RequestData { + int offset = 0; + int dcIndex = 0; + + inline bool operator<(const RequestData &other) const { + return offset < other.offset; + } + }; + struct CdnFileHash { + CdnFileHash(int limit, QByteArray hash) : limit(limit), hash(hash) { + } + int limit = 0; + QByteArray hash; + }; + Storage::Cache::Key cacheKey() const override; + std::optional fileLocationKey() const override; + void startLoading() override; + void cancelRequests() override; + + void makeRequest(const RequestData &requestData); + + MTP::DcId dcId() const override; + bool readyToRequest() const override; + void loadPart(int dcIndex) override; + void normalPartLoaded(const MTPupload_File &result, mtpRequestId requestId); + void webPartLoaded(const MTPupload_WebFile &result, mtpRequestId requestId); + void cdnPartLoaded(const MTPupload_CdnFile &result, mtpRequestId requestId); + void reuploadDone(const MTPVector &result, mtpRequestId requestId); + void requestMoreCdnFileHashes(); + void getCdnFileHashesDone(const MTPVector &result, mtpRequestId requestId); + + void partLoaded(int offset, bytes::const_span buffer); + bool feedPart(int offset, bytes::const_span buffer); + + bool partFailed(const RPCError &error, mtpRequestId requestId); + bool normalPartFailed(QByteArray fileReference, const RPCError &error, mtpRequestId requestId); + bool cdnPartFailed(const RPCError &error, mtpRequestId requestId); + + mtpRequestId sendRequest(const RequestData &requestData); + void placeSentRequest( + mtpRequestId requestId, + const RequestData &requestData); + [[nodiscard]] RequestData finishSentRequest(mtpRequestId requestId); + void switchToCDN( + const RequestData &requestData, + const MTPDupload_fileCdnRedirect &redirect); + void addCdnHashes(const QVector &hashes); + void changeCDNParams( + const RequestData &requestData, + MTP::DcId dcId, + const QByteArray &token, + const QByteArray &encryptionKey, + const QByteArray &encryptionIV, + const QVector &hashes); + + enum class CheckCdnHashResult { + NoHash, + Invalid, + Good, + }; + CheckCdnHashResult checkCdnFileHash(int offset, bytes::const_span buffer); + + const not_null _downloader; + const MTP::DcId _dcId = 0; + std::map _sentRequests; + + bool _lastComplete = false; + int32 _nextRequestOffset = 0; + + base::variant< + StorageFileLocation, + WebFileLocation, + GeoPointLocation> _location; + Data::FileOrigin _origin; + + MTP::DcId _cdnDcId = 0; + QByteArray _cdnToken; + QByteArray _cdnEncryptionKey; + QByteArray _cdnEncryptionIV; + base::flat_map _cdnFileHashes; + base::flat_map _cdnUncheckedParts; + mtpRequestId _cdnHashesRequestId = 0; + +}; diff --git a/Telegram/SourceFiles/storage/file_download_web.cpp b/Telegram/SourceFiles/storage/file_download_web.cpp new file mode 100644 index 0000000000..e9714236a7 --- /dev/null +++ b/Telegram/SourceFiles/storage/file_download_web.cpp @@ -0,0 +1,527 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#include "storage/file_download_web.h" + +#include "storage/cache/storage_cache_types.h" + +#include + +namespace { + +constexpr auto kMaxWebFileQueries = 8; +constexpr auto kMaxHttpRedirects = 5; +constexpr auto kResetDownloadPrioritiesTimeout = crl::time(200); + +std::weak_ptr GlobalLoadManager; + +using ErrorSignal = void(QNetworkReply::*)(QNetworkReply::NetworkError); +const auto QNetworkReply_error = ErrorSignal(&QNetworkReply::error); + +[[nodiscard]] std::shared_ptr GetManager() { + auto result = GlobalLoadManager.lock(); + if (!result) { + GlobalLoadManager = result = std::make_shared(); + } + return result; +} + +enum class ProcessResult { + Error, + Progress, + Finished, +}; + +enum class Error { +}; + +struct Progress { + qint64 ready = 0; + qint64 total = 0; +}; + +using Update = base::variant; + +struct UpdateForLoader { + not_null loader; + Update data; +}; + +} // namespace + +class WebLoadManager final : public QObject { +public: + WebLoadManager(); + ~WebLoadManager(); + + void enqueue(not_null loader); + void remove(not_null loader); + + [[nodiscard]] rpl::producer updates( + not_null loader) const; + +private: + struct Enqueued { + int id = 0; + QString url; + }; + struct Sent { + QString url; + not_null reply; + QByteArray data; + int64 ready = 0; + int64 total = 0; + int redirectsLeft = kMaxHttpRedirects; + }; + + // Constructor. + void handleNetworkErrors(); + + // Worker thread. + void enqueue(int id, const QString &url); + void remove(int id); + void resetGeneration(); + void checkSendNext(); + void send(const Enqueued &entry); + [[nodiscard]] not_null send(int id, const QString &url); + [[nodiscard]] Sent *findSent(int id, not_null reply); + void removeSent(int id); + void progress( + int id, + not_null reply, + int64 ready, + int64 total); + void failed( + int id, + not_null reply, + QNetworkReply::NetworkError error); + void redirect(int id, not_null reply); + void notify( + int id, + not_null reply, + int64 ready, + int64 total); + void failed(int id, not_null reply); + void finished(int id, not_null reply); + void deleteDeferred(not_null reply); + void queueProgressUpdate(int id, int64 ready, int64 total); + void queueFailedUpdate(int id); + void queueFinishedUpdate(int id, const QByteArray &data); + void clear(); + + // Main thread. + void sendUpdate(int id, Update &&data); + + QThread _thread; + QNetworkAccessManager _network; + base::Timer _resetGenerationTimer; + + // Main thread. + rpl::event_stream _updates; + int _autoincrement = 0; + base::flat_map, int> _ids; + + // Worker thread. + std::deque _queue; + std::deque _previousGeneration; + base::flat_map _sent; + std::vector> _repliesBeingDeleted; + +}; + +WebLoadManager::WebLoadManager() +: _resetGenerationTimer(&_thread, [=] { resetGeneration(); }) { + handleNetworkErrors(); + + const auto original = QThread::currentThread(); + moveToThread(&_thread); + _network.moveToThread(&_thread); + connect(&_thread, &QThread::finished, [=] { + clear(); + moveToThread(original); + _network.moveToThread(original); + }); + _thread.start(); +} + +void WebLoadManager::handleNetworkErrors() { + const auto fail = [=](QNetworkReply *reply) { + for (const auto &[id, sent] : _sent) { + if (sent.reply == reply) { + failed(id, reply); + return; + } + } + }; + connect(&_network, &QNetworkAccessManager::authenticationRequired, fail); + connect(&_network, &QNetworkAccessManager::sslErrors, fail); +} + +WebLoadManager::~WebLoadManager() { + _thread.quit(); + _thread.wait(); +} + +[[nodiscard]] rpl::producer WebLoadManager::updates( + not_null loader) const { + return _updates.events( + ) | rpl::filter([=](const UpdateForLoader &update) { + return (update.loader == loader); + }) | rpl::map([=](UpdateForLoader &&update) { + return std::move(update.data); + }); +} + +void WebLoadManager::enqueue(not_null loader) { + const auto id = [&] { + const auto i = _ids.find(loader); + return (i != end(_ids)) + ? i->second + : _ids.emplace(loader, ++_autoincrement).first->second; + }(); + const auto url = loader->url(); + InvokeQueued(this, [=] { + enqueue(id, url); + }); +} + +void WebLoadManager::remove(not_null loader) { + const auto i = _ids.find(loader); + if (i == end(_ids)) { + return; + } + const auto id = i->second; + _ids.erase(i); + InvokeQueued(this, [=] { + remove(id); + }); +} + +void WebLoadManager::enqueue(int id, const QString &url) { + const auto i = ranges::find(_queue, id, &Enqueued::id); + if (i != end(_queue)) { + return; + } + _previousGeneration.erase( + ranges::remove(_previousGeneration, id, &Enqueued::id), + end(_previousGeneration)); + _queue.push_back(Enqueued{ id, url }); + if (!_resetGenerationTimer.isActive()) { + _resetGenerationTimer.callOnce(kResetDownloadPrioritiesTimeout); + } + checkSendNext(); +} + +void WebLoadManager::remove(int id) { + _queue.erase(ranges::remove(_queue, id, &Enqueued::id), end(_queue)); + _previousGeneration.erase( + ranges::remove(_previousGeneration, id, &Enqueued::id), + end(_previousGeneration)); + removeSent(id); + checkSendNext(); +} + +void WebLoadManager::resetGeneration() { + if (!_previousGeneration.empty()) { + std::copy( + begin(_previousGeneration), + end(_previousGeneration), + std::back_inserter(_queue)); + _previousGeneration.clear(); + } + std::swap(_queue, _previousGeneration); +} + +void WebLoadManager::checkSendNext() { + if (_sent.size() >= kMaxWebFileQueries + || (_queue.empty() && _previousGeneration.empty())) { + return; + } + const auto entry = _queue.empty() + ? _previousGeneration.front() + : _queue.front(); + (_queue.empty() ? _previousGeneration : _queue).pop_front(); + send(entry); +} + +void WebLoadManager::send(const Enqueued &entry) { + const auto id = entry.id; + const auto url = entry.url; + _sent.emplace(id, Sent{ url, send(id, url) }); +} + +void WebLoadManager::removeSent(int id) { + if (const auto i = _sent.find(id); i != end(_sent)) { + deleteDeferred(i->second.reply); + _sent.erase(i); + } +} + +not_null WebLoadManager::send(int id, const QString &url) { + const auto result = _network.get(QNetworkRequest(url)); + const auto handleProgress = [=](qint64 ready, qint64 total) { + progress(id, result, ready, total); + checkSendNext(); + }; + const auto handleError = [=](QNetworkReply::NetworkError error) { + failed(id, result, error); + checkSendNext(); + }; + connect(result, &QNetworkReply::downloadProgress, handleProgress); + connect(result, QNetworkReply_error, handleError); + return result; +} + +WebLoadManager::Sent *WebLoadManager::findSent( + int id, + not_null reply) { + const auto i = _sent.find(id); + return (i != end(_sent) && i->second.reply == reply) + ? &i->second + : nullptr; +} + +void WebLoadManager::progress( + int id, + not_null reply, + int64 ready, + int64 total) { + const auto statusCode = reply->attribute( + QNetworkRequest::HttpStatusCodeAttribute); + const auto status = statusCode.isValid() ? statusCode.toInt() : 200; + if (status == 301 || status == 302) { + redirect(id, reply); + } else if (status != 200 && status != 206 && status != 416) { + LOG(("Network Error: " + "Bad HTTP status received in WebLoadManager::onProgress() %1" + ).arg(status)); + failed(id, reply); + } else { + notify(id, reply, ready, total); + } +} + +void WebLoadManager::redirect(int id, not_null reply) { + const auto header = reply->header(QNetworkRequest::LocationHeader); + const auto url = header.toString(); + if (url.isEmpty()) { + return; + } + + if (const auto sent = findSent(id, reply)) { + if (!sent->redirectsLeft--) { + LOG(("Network Error: " + "Too many HTTP redirects in onFinished() " + "for web file loader: %1").arg(url)); + failed(id, reply); + return; + } + deleteDeferred(reply); + sent->url = url; + sent->reply = send(id, url); + } +} + +void WebLoadManager::notify( + int id, + not_null reply, + int64 ready, + int64 total) { + if (const auto sent = findSent(id, reply)) { + sent->ready = ready; + sent->total = std::max(total, int64(0)); + sent->data.append(reply->readAll()); + if (total == 0 + || total > Storage::kMaxFileInMemory + || sent->data.size() > Storage::kMaxFileInMemory) { + LOG(("Network Error: " + "Bad size received for HTTP download progress " + "in WebLoadManager::onProgress(): %1 / %2 (bytes %3)" + ).arg(ready + ).arg(total + ).arg(sent->data.size())); + failed(id, reply); + } else if (total > 0 && ready >= total) { + finished(id, reply); + } else { + queueProgressUpdate(id, sent->ready, sent->total); + } + } +} + +void WebLoadManager::failed( + int id, + not_null reply, + QNetworkReply::NetworkError error) { + if (const auto sent = findSent(id, reply)) { + LOG(("Network Error: " + "Failed to request '%1', error %2 (%3)" + ).arg(sent->url + ).arg(int(error) + ).arg(reply->errorString())); + failed(id, reply); + } +} + +void WebLoadManager::failed(int id, not_null reply) { + if (const auto sent = findSent(id, reply)) { + removeSent(id); + queueFailedUpdate(id); + } +} + +void WebLoadManager::deleteDeferred(not_null reply) { + reply->deleteLater(); + _repliesBeingDeleted.erase( + ranges::remove(_repliesBeingDeleted, nullptr), + end(_repliesBeingDeleted)); + _repliesBeingDeleted.emplace_back(reply.get()); +} + +void WebLoadManager::finished(int id, not_null reply) { + if (const auto sent = findSent(id, reply)) { + const auto data = base::take(sent->data); + removeSent(id); + queueFinishedUpdate(id, data); + } +} + +void WebLoadManager::clear() { + for (const auto &[id, sent] : base::take(_sent)) { + sent.reply->abort(); + delete sent.reply; + } + for (const auto reply : base::take(_repliesBeingDeleted)) { + if (reply) { + delete reply; + } + } +} + +void WebLoadManager::queueProgressUpdate(int id, int64 ready, int64 total) { + crl::on_main(this, [=] { + sendUpdate(id, Progress{ ready, total }); + }); +} + +void WebLoadManager::queueFailedUpdate(int id) { + crl::on_main(this, [=] { + sendUpdate(id, Error{}); + }); +} + +void WebLoadManager::queueFinishedUpdate(int id, const QByteArray &data) { + crl::on_main(this, [=] { + LOG(("FINISHED UPDATE FOR: %1").arg(id)); + for (const auto &[loader, loaderId] : _ids) { + if (loaderId == id) { + LOG(("LOADER ID: %2").arg(quintptr(loader.get()))); + break; + } + } + LOG(("SENT")); + sendUpdate(id, QByteArray(data)); + }); +} + +void WebLoadManager::sendUpdate(int id, Update &&data) { + for (const auto &[loader, loaderId] : _ids) { + if (loaderId == id) { + _updates.fire(UpdateForLoader{ loader, std::move(data) }); + return; + } + } +} + +webFileLoader::webFileLoader( + const QString &url, + const QString &to, + LoadFromCloudSetting fromCloud, + bool autoLoading, + uint8 cacheTag) +: FileLoader( + QString(), + 0, + UnknownFileLocation, + LoadToCacheAsWell, + fromCloud, + autoLoading, + cacheTag) +, _url(url) { +} + +webFileLoader::~webFileLoader() { + cancelRequests(); +} + +QString webFileLoader::url() const { + return _url; +} + +void webFileLoader::startLoading() { + if (_finished) { + return; + } else if (!_manager) { + _manager = GetManager(); + _manager->updates( + this + ) | rpl::start_with_next([=](const Update &data) { + if (const auto progress = base::get_if(&data)) { + loadProgress(progress->ready, progress->total); + } else if (const auto bytes = base::get_if(&data)) { + loadFinished(*bytes); + } else { + loadFailed(); + } + }, _managerLifetime); + } + _manager->enqueue(this); +} + +int webFileLoader::currentOffset() const { + return _ready; +} + +void webFileLoader::loadProgress(qint64 ready, qint64 total) { + _size = total; + _ready = ready; + notifyAboutProgress(); +} + +void webFileLoader::loadFinished(const QByteArray &data) { + cancelRequests(); + if (writeResultPart(0, bytes::make_span(data))) { + if (finalizeResult()) { + notifyAboutProgress(); + } + } +} + +void webFileLoader::loadFailed() { + cancel(true); +} + +Storage::Cache::Key webFileLoader::cacheKey() const { + return Data::UrlCacheKey(_url); +} + +std::optional webFileLoader::fileLocationKey() const { + return std::nullopt; +} + +void webFileLoader::stop() { + cancelRequests(); +} + +void webFileLoader::cancelRequests() { + if (!_manager) { + return; + } + _managerLifetime.destroy(); + _manager->remove(this); + _manager = nullptr; +} diff --git a/Telegram/SourceFiles/storage/file_download_web.h b/Telegram/SourceFiles/storage/file_download_web.h new file mode 100644 index 0000000000..bfc1b8638b --- /dev/null +++ b/Telegram/SourceFiles/storage/file_download_web.h @@ -0,0 +1,45 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#pragma once + +#include "storage/file_download.h" + +class WebLoadManager; + +class webFileLoader final : public FileLoader { +public: + webFileLoader( + const QString &url, + const QString &to, + LoadFromCloudSetting fromCloud, + bool autoLoading, + uint8 cacheTag); + ~webFileLoader(); + + [[nodiscard]] QString url() const; + + int currentOffset() const override; + void stop() override; + void cancelRequests() override; + +private: + void startLoading() override; + Storage::Cache::Key cacheKey() const override; + std::optional fileLocationKey() const override; + + void loadProgress(qint64 ready, qint64 size); + void loadFinished(const QByteArray &data); + void loadFailed(); + + const QString _url; + int _ready = 0; + + std::shared_ptr _manager; + rpl::lifetime _managerLifetime; + +}; diff --git a/Telegram/SourceFiles/storage/streamed_file_downloader.cpp b/Telegram/SourceFiles/storage/streamed_file_downloader.cpp index 33b15bb54e..d5afb2a438 100644 --- a/Telegram/SourceFiles/storage/streamed_file_downloader.cpp +++ b/Telegram/SourceFiles/storage/streamed_file_downloader.cpp @@ -38,7 +38,6 @@ StreamedFileDownloader::StreamedFileDownloader( uint8 cacheTag) : FileLoader( toFile, - dcId, size, locationType, toCache, @@ -61,8 +60,6 @@ StreamedFileDownloader::StreamedFileDownloader( savePart(std::move(part)); } }, _lifetime); - - requestParts(); } StreamedFileDownloader::~StreamedFileDownloader() { @@ -131,12 +128,8 @@ void StreamedFileDownloader::cancelRequests() { _reader->cancelForDownloader(this); } -bool StreamedFileDownloader::readyToRequest() const { - return false; -} - -void StreamedFileDownloader::loadPart(int dcIndex) { - Unexpected("StreamedFileDownloader can't load parts."); +void StreamedFileDownloader::startLoading() { + requestParts(); } void StreamedFileDownloader::savePart(const LoadedPart &part) { diff --git a/Telegram/SourceFiles/storage/streamed_file_downloader.h b/Telegram/SourceFiles/storage/streamed_file_downloader.h index 90aea8c099..43aa5e575e 100644 --- a/Telegram/SourceFiles/storage/streamed_file_downloader.h +++ b/Telegram/SourceFiles/storage/streamed_file_downloader.h @@ -46,11 +46,10 @@ public: QByteArray readLoadedPart(int offset); private: + void startLoading() override; Cache::Key cacheKey() const override; std::optional fileLocationKey() const override; void cancelRequests() override; - bool readyToRequest() const override; - void loadPart(int dcIndex) override; void requestParts(); void requestPart(); diff --git a/Telegram/SourceFiles/ui/image/image_source.cpp b/Telegram/SourceFiles/ui/image/image_source.cpp index f8246ec058..d5ae427c7e 100644 --- a/Telegram/SourceFiles/ui/image/image_source.cpp +++ b/Telegram/SourceFiles/ui/image/image_source.cpp @@ -7,10 +7,11 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL */ #include "ui/image/image_source.h" -#include "storage/file_download.h" +#include "storage/cache/storage_cache_database.h" +#include "storage/file_download_mtproto.h" +#include "storage/file_download_web.h" #include "data/data_session.h" #include "data/data_file_origin.h" -#include "storage/cache/storage_cache_database.h" #include "history/history_item.h" #include "history/history.h" #include "main/main_session.h" diff --git a/Telegram/gyp/telegram/sources.txt b/Telegram/gyp/telegram/sources.txt index 6276efafc7..14afd962af 100644 --- a/Telegram/gyp/telegram/sources.txt +++ b/Telegram/gyp/telegram/sources.txt @@ -685,6 +685,10 @@ <(src_loc)/settings/settings_privacy_security.h <(src_loc)/storage/file_download.cpp <(src_loc)/storage/file_download.h +<(src_loc)/storage/file_download_mtproto.cpp +<(src_loc)/storage/file_download_mtproto.h +<(src_loc)/storage/file_download_web.cpp +<(src_loc)/storage/file_download_web.h <(src_loc)/storage/file_upload.cpp <(src_loc)/storage/file_upload.h <(src_loc)/storage/localimageloader.cpp