Rewrite webFileLoader without Storage::DownloadManager.

Telegram/SourceFiles/apiwrap.cpp

@@ -55,7 +55,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #include "ui/emoji_config.h"
 #include "support/support_helper.h"
 #include "storage/localimageloader.h"
-#include "storage/file_download.h"
+#include "storage/file_download_mtproto.h"
 #include "storage/file_upload.h"
 #include "storage/storage_facade.h"
 #include "storage/storage_shared_media.h"

Telegram/SourceFiles/core/application.cpp

@@ -139,7 +139,6 @@ Application::~Application() {
 	Ui::Emoji::Clear();
 	Media::Clip::Finish();
 
-	stopWebLoadManager();
 	App::deinitMedia();
 
 	Window::Theme::Uninitialize();

Telegram/SourceFiles/data/data_document.cpp

@@ -21,6 +21,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #include "media/streaming/media_streaming_loader_local.h"
 #include "storage/localstorage.h"
 #include "storage/streamed_file_downloader.h"
+#include "storage/file_download_mtproto.h"
+#include "storage/file_download_web.h"
 #include "platform/platform_specific.h"
 #include "history/history.h"
 #include "history/history_item.h"

Telegram/SourceFiles/passport/passport_form_controller.cpp

@@ -28,7 +28,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #include "storage/localimageloader.h"
 #include "storage/localstorage.h"
 #include "storage/file_upload.h"
-#include "storage/file_download.h"
+#include "storage/file_download_mtproto.h"
 #include "app.h"
 #include "apiwrap.h"
 

Telegram/SourceFiles/storage/file_download.h

@@ -35,6 +35,12 @@ constexpr auto kMaxWallPaperInMemory = kMaxFileInMemory;
 constexpr auto kMaxAnimationInMemory = kMaxFileInMemory; // 10 MB gif and mp4 animations held in memory while playing
 constexpr auto kMaxWallPaperDimension = 4096; // 4096x4096 is max area.
 
+// Different part sizes are not supported for now :(
+// Because we start downloading with some part size
+// and then we get a cdn-redirect where we support only
+// fixed part size download for hash checking.
+constexpr auto kDownloadPartSize = 128 * 1024;
+
 class Downloader {
 public:
 	virtual ~Downloader() = default;
@@ -114,13 +120,12 @@ struct StorageImageSaved {
 
 };
 
-class FileLoader : public QObject, public Storage::Downloader {
+class FileLoader : public QObject {
 	Q_OBJECT
 
 public:
 	FileLoader(
 		const QString &toFile,
-		MTP::DcId dcId,
 		int32 size,
 		LocationType locationType,
 		LoadToCacheSetting toCache,
@@ -180,8 +185,6 @@ signals:
 	void failed(FileLoader *loader, bool started);
 
 protected:
-	friend class Storage::DownloadManager;
-
 	enum class LocalStatus {
 		NotTried,
 		NotFound,
@@ -189,10 +192,6 @@ protected:
 		Loaded,
 	};
 
-	MTP::DcId dcId() const override {
-		return _dcId;
-	}
-
 	void readImage(const QSize &shrinkBox) const;
 
 	bool tryLoadLocal();
@@ -200,6 +199,7 @@ protected:
 	virtual Storage::Cache::Key cacheKey() const = 0;
 	virtual std::optional<MediaKey> fileLocationKey() const = 0;
 	virtual void cancelRequests() = 0;
+	virtual void startLoading() = 0;
 
 	void cancel(bool failed);
 
@@ -209,8 +209,7 @@ protected:
 	bool finalizeResult();
 	[[nodiscard]] QByteArray readLoadedPartBack(int offset, int size);
 
-	const MTP::DcId _dcId = 0;
-	const not_null<Storage::DownloadManager*> _downloader;
+	const not_null<Main::Session*> _session;
 
 	bool _autoLoading = false;
 	uint8 _cacheTag = 0;
@@ -236,237 +235,3 @@ protected:
 	mutable QImage _imageData;
 
 };
-
-class StorageImageLocation;
-class WebFileLocation;
-class mtpFileLoader final : public FileLoader, public RPCSender {
-public:
-	mtpFileLoader(
-		const StorageFileLocation &location,
-		Data::FileOrigin origin,
-		LocationType type,
-		const QString &toFile,
-		int32 size,
-		LoadToCacheSetting toCache,
-		LoadFromCloudSetting fromCloud,
-		bool autoLoading,
-		uint8 cacheTag);
-	mtpFileLoader(
-		const WebFileLocation &location,
-		int32 size,
-		LoadFromCloudSetting fromCloud,
-		bool autoLoading,
-		uint8 cacheTag);
-	mtpFileLoader(
-		const GeoPointLocation &location,
-		int32 size,
-		LoadFromCloudSetting fromCloud,
-		bool autoLoading,
-		uint8 cacheTag);
-
-	Data::FileOrigin fileOrigin() const override;
-
-	uint64 objId() const override;
-
-	void stop() override {
-		rpcInvalidate();
-	}
-	void refreshFileReferenceFrom(
-		const Data::UpdatedFileReferences &updates,
-		int requestId,
-		const QByteArray &current);
-
-	~mtpFileLoader();
-
-private:
-	friend class DownloadManager;
-
-	struct RequestData {
-		int offset = 0;
-		int dcIndex = 0;
-
-		inline bool operator<(const RequestData &other) const {
-			return offset < other.offset;
-		}
-	};
-	struct CdnFileHash {
-		CdnFileHash(int limit, QByteArray hash) : limit(limit), hash(hash) {
-		}
-		int limit = 0;
-		QByteArray hash;
-	};
-	Storage::Cache::Key cacheKey() const override;
-	std::optional<MediaKey> fileLocationKey() const override;
-	void cancelRequests() override;
-
-	void makeRequest(const RequestData &requestData);
-
-	bool readyToRequest() const override;
-	void loadPart(int dcIndex) override;
-	void normalPartLoaded(const MTPupload_File &result, mtpRequestId requestId);
-	void webPartLoaded(const MTPupload_WebFile &result, mtpRequestId requestId);
-	void cdnPartLoaded(const MTPupload_CdnFile &result, mtpRequestId requestId);
-	void reuploadDone(const MTPVector<MTPFileHash> &result, mtpRequestId requestId);
-	void requestMoreCdnFileHashes();
-	void getCdnFileHashesDone(const MTPVector<MTPFileHash> &result, mtpRequestId requestId);
-
-	void partLoaded(int offset, bytes::const_span buffer);
-	bool feedPart(int offset, bytes::const_span buffer);
-
-	bool partFailed(const RPCError &error, mtpRequestId requestId);
-	bool normalPartFailed(QByteArray fileReference, const RPCError &error, mtpRequestId requestId);
-	bool cdnPartFailed(const RPCError &error, mtpRequestId requestId);
-
-	mtpRequestId sendRequest(const RequestData &requestData);
-	void placeSentRequest(
-		mtpRequestId requestId,
-		const RequestData &requestData);
-	[[nodiscard]] RequestData finishSentRequest(mtpRequestId requestId);
-	void switchToCDN(
-		const RequestData &requestData,
-		const MTPDupload_fileCdnRedirect &redirect);
-	void addCdnHashes(const QVector<MTPFileHash> &hashes);
-	void changeCDNParams(
-		const RequestData &requestData,
-		MTP::DcId dcId,
-		const QByteArray &token,
-		const QByteArray &encryptionKey,
-		const QByteArray &encryptionIV,
-		const QVector<MTPFileHash> &hashes);
-
-	enum class CheckCdnHashResult {
-		NoHash,
-		Invalid,
-		Good,
-	};
-	CheckCdnHashResult checkCdnFileHash(int offset, bytes::const_span buffer);
-
-	std::map<mtpRequestId, RequestData> _sentRequests;
-
-	bool _lastComplete = false;
-	int32 _nextRequestOffset = 0;
-
-	base::variant<
-		StorageFileLocation,
-		WebFileLocation,
-		GeoPointLocation> _location;
-	Data::FileOrigin _origin;
-
-	MTP::DcId _cdnDcId = 0;
-	QByteArray _cdnToken;
-	QByteArray _cdnEncryptionKey;
-	QByteArray _cdnEncryptionIV;
-	base::flat_map<int, CdnFileHash> _cdnFileHashes;
-	base::flat_map<RequestData, QByteArray> _cdnUncheckedParts;
-	mtpRequestId _cdnHashesRequestId = 0;
-
-};
-
-class webFileLoaderPrivate;
-
-class webFileLoader final : public FileLoader {
-public:
-	webFileLoader(
-		const QString &url,
-		const QString &to,
-		LoadFromCloudSetting fromCloud,
-		bool autoLoading,
-		uint8 cacheTag);
-
-	int currentOffset() const override;
-
-	void loadProgress(qint64 already, qint64 size);
-	void loadFinished(const QByteArray &data);
-	void loadError();
-
-	void stop() override {
-		cancelRequests();
-	}
-
-	~webFileLoader();
-
-private:
-	void cancelRequests() override;
-	Storage::Cache::Key cacheKey() const override;
-	std::optional<MediaKey> fileLocationKey() const override;
-	bool readyToRequest() const override;
-	void loadPart(int dcIndex) override;
-
-	void markAsSent();
-	void markAsNotSent();
-
-	QString _url;
-
-	bool _requestSent = false;
-	int32 _already = 0;
-
-	friend class WebLoadManager;
-	webFileLoaderPrivate *_private = nullptr;
-
-};
-
-enum WebReplyProcessResult {
-	WebReplyProcessError,
-	WebReplyProcessProgress,
-	WebReplyProcessFinished,
-};
-
-class WebLoadManager : public QObject {
-	Q_OBJECT
-
-public:
-	WebLoadManager(QThread *thread);
-
-	void append(webFileLoader *loader, const QString &url);
-	void stop(webFileLoader *reader);
-	bool carries(webFileLoader *reader) const;
-
-	~WebLoadManager();
-
-signals:
-	void processDelayed();
-
-	void progress(webFileLoader *loader, qint64 already, qint64 size);
-	void finished(webFileLoader *loader, QByteArray data);
-	void error(webFileLoader *loader);
-
-public slots:
-	void onFailed(QNetworkReply *reply);
-	void onFailed(QNetworkReply::NetworkError error);
-	void onProgress(qint64 already, qint64 size);
-	void onMeta();
-
-	void process();
-	void finish();
-
-private:
-	void clear();
-	void sendRequest(webFileLoaderPrivate *loader, const QString &redirect = QString());
-	bool handleReplyResult(webFileLoaderPrivate *loader, WebReplyProcessResult result);
-
-	QNetworkAccessManager _manager;
-	typedef QMap<webFileLoader*, webFileLoaderPrivate*> LoaderPointers;
-	LoaderPointers _loaderPointers;
-	mutable QMutex _loaderPointersMutex;
-
-	typedef OrderedSet<webFileLoaderPrivate*> Loaders;
-	Loaders _loaders;
-
-	typedef QMap<QNetworkReply*, webFileLoaderPrivate*> Replies;
-	Replies _replies;
-
-};
-
-class WebLoadMainManager : public QObject {
-	Q_OBJECT
-
-public slots:
-	void progress(webFileLoader *loader, qint64 already, qint64 size);
-	void finished(webFileLoader *loader, QByteArray data);
-	void error(webFileLoader *loader);
-
-};
-
-static WebLoadManager * const FinishedWebLoadManager = SharedMemoryLocation<WebLoadManager, 0>();
-
-void stopWebLoadManager();

Telegram/SourceFiles/storage/file_download_mtproto.cpp

@@ -0,0 +1,590 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/file_download_mtproto.h"
+
+#include "data/data_document.h"
+#include "data/data_file_origin.h"
+#include "storage/cache/storage_cache_types.h"
+#include "main/main_session.h"
+#include "apiwrap.h"
+#include "mtproto/mtp_instance.h"
+#include "mtproto/mtproto_auth_key.h"
+#include "base/openssl_help.h"
+#include "facades.h"
+
+mtpFileLoader::mtpFileLoader(
+	const StorageFileLocation &location,
+	Data::FileOrigin origin,
+	LocationType type,
+	const QString &to,
+	int32 size,
+	LoadToCacheSetting toCache,
+	LoadFromCloudSetting fromCloud,
+	bool autoLoading,
+	uint8 cacheTag)
+: FileLoader(
+	to,
+	size,
+	type,
+	toCache,
+	fromCloud,
+	autoLoading,
+	cacheTag)
+, _downloader(&session().downloader())
+, _dcId(location.dcId())
+, _location(location)
+, _origin(origin) {
+}
+
+mtpFileLoader::mtpFileLoader(
+	const WebFileLocation &location,
+	int32 size,
+	LoadFromCloudSetting fromCloud,
+	bool autoLoading,
+	uint8 cacheTag)
+: FileLoader(
+	QString(),
+	size,
+	UnknownFileLocation,
+	LoadToCacheAsWell,
+	fromCloud,
+	autoLoading,
+	cacheTag)
+, _downloader(&session().downloader())
+, _dcId(Global::WebFileDcId())
+, _location(location) {
+}
+
+mtpFileLoader::mtpFileLoader(
+	const GeoPointLocation &location,
+	int32 size,
+	LoadFromCloudSetting fromCloud,
+	bool autoLoading,
+	uint8 cacheTag)
+: FileLoader(
+	QString(),
+	size,
+	UnknownFileLocation,
+	LoadToCacheAsWell,
+	fromCloud,
+	autoLoading,
+	cacheTag)
+, _downloader(&session().downloader())
+, _dcId(Global::WebFileDcId())
+, _location(location) {
+}
+
+mtpFileLoader::~mtpFileLoader() {
+	cancelRequests();
+	_downloader->remove(this);
+}
+
+Data::FileOrigin mtpFileLoader::fileOrigin() const {
+	return _origin;
+}
+
+uint64 mtpFileLoader::objId() const {
+	if (const auto storage = base::get_if<StorageFileLocation>(&_location)) {
+		return storage->objectId();
+	}
+	return 0;
+}
+
+void mtpFileLoader::refreshFileReferenceFrom(
+		const Data::UpdatedFileReferences &updates,
+		int requestId,
+		const QByteArray &current) {
+	if (const auto storage = base::get_if<StorageFileLocation>(&_location)) {
+		storage->refreshFileReference(updates);
+		if (storage->fileReference() == current) {
+			cancel(true);
+			return;
+		}
+	} else {
+		cancel(true);
+		return;
+	}
+	makeRequest(finishSentRequest(requestId));
+}
+
+MTP::DcId mtpFileLoader::dcId() const {
+	return _dcId;
+}
+
+bool mtpFileLoader::readyToRequest() const {
+	return !_finished
+		&& !_lastComplete
+		&& (_sentRequests.empty() || _size != 0)
+		&& (!_size || _nextRequestOffset < _size);
+}
+
+void mtpFileLoader::loadPart(int dcIndex) {
+	Expects(readyToRequest());
+
+	makeRequest({ _nextRequestOffset, dcIndex });
+	_nextRequestOffset += Storage::kDownloadPartSize;
+}
+
+mtpRequestId mtpFileLoader::sendRequest(const RequestData &requestData) {
+	const auto offset = requestData.offset;
+	const auto limit = Storage::kDownloadPartSize;
+	const auto shiftedDcId = MTP::downloadDcId(
+		_cdnDcId ? _cdnDcId : dcId(),
+		requestData.dcIndex);
+	if (_cdnDcId) {
+		return MTP::send(
+			MTPupload_GetCdnFile(
+				MTP_bytes(_cdnToken),
+				MTP_int(offset),
+				MTP_int(limit)),
+			rpcDone(&mtpFileLoader::cdnPartLoaded),
+			rpcFail(&mtpFileLoader::cdnPartFailed),
+			shiftedDcId,
+			50);
+	}
+	return _location.match([&](const WebFileLocation &location) {
+		return MTP::send(
+			MTPupload_GetWebFile(
+				MTP_inputWebFileLocation(
+					MTP_bytes(location.url()),
+					MTP_long(location.accessHash())),
+				MTP_int(offset),
+				MTP_int(limit)),
+			rpcDone(&mtpFileLoader::webPartLoaded),
+			rpcFail(&mtpFileLoader::partFailed),
+			shiftedDcId,
+			50);
+	}, [&](const GeoPointLocation &location) {
+		return MTP::send(
+			MTPupload_GetWebFile(
+				MTP_inputWebFileGeoPointLocation(
+					MTP_inputGeoPoint(
+						MTP_double(location.lat),
+						MTP_double(location.lon)),
+					MTP_long(location.access),
+					MTP_int(location.width),
+					MTP_int(location.height),
+					MTP_int(location.zoom),
+					MTP_int(location.scale)),
+				MTP_int(offset),
+				MTP_int(limit)),
+			rpcDone(&mtpFileLoader::webPartLoaded),
+			rpcFail(&mtpFileLoader::partFailed),
+			shiftedDcId,
+			50);
+	}, [&](const StorageFileLocation &location) {
+		return MTP::send(
+			MTPupload_GetFile(
+				MTP_flags(0),
+				location.tl(session().userId()),
+				MTP_int(offset),
+				MTP_int(limit)),
+			rpcDone(&mtpFileLoader::normalPartLoaded),
+			rpcFail(
+				&mtpFileLoader::normalPartFailed,
+				location.fileReference()),
+			shiftedDcId,
+			50);
+	});
+}
+
+void mtpFileLoader::makeRequest(const RequestData &requestData) {
+	Expects(!_finished);
+
+	placeSentRequest(sendRequest(requestData), requestData);
+}
+
+void mtpFileLoader::requestMoreCdnFileHashes() {
+	Expects(!_finished);
+
+	if (_cdnHashesRequestId || _cdnUncheckedParts.empty()) {
+		return;
+	}
+
+	const auto requestData = _cdnUncheckedParts.cbegin()->first;
+	const auto shiftedDcId = MTP::downloadDcId(
+		dcId(),
+		requestData.dcIndex);
+	const auto requestId = _cdnHashesRequestId = MTP::send(
+		MTPupload_GetCdnFileHashes(
+			MTP_bytes(_cdnToken),
+			MTP_int(requestData.offset)),
+		rpcDone(&mtpFileLoader::getCdnFileHashesDone),
+		rpcFail(&mtpFileLoader::cdnPartFailed),
+		shiftedDcId);
+	placeSentRequest(requestId, requestData);
+}
+
+void mtpFileLoader::normalPartLoaded(
+		const MTPupload_File &result,
+		mtpRequestId requestId) {
+	Expects(!_finished);
+
+	const auto requestData = finishSentRequest(requestId);
+	result.match([&](const MTPDupload_fileCdnRedirect &data) {
+		switchToCDN(requestData, data);
+	}, [&](const MTPDupload_file &data) {
+		partLoaded(requestData.offset, bytes::make_span(data.vbytes().v));
+	});
+}
+
+void mtpFileLoader::webPartLoaded(
+		const MTPupload_WebFile &result,
+		mtpRequestId requestId) {
+	result.match([&](const MTPDupload_webFile &data) {
+		const auto requestData = finishSentRequest(requestId);
+		if (!_size) {
+			_size = data.vsize().v;
+		} else if (data.vsize().v != _size) {
+			LOG(("MTP Error: "
+				"Bad size provided by bot for webDocument: %1, real: %2"
+				).arg(_size
+				).arg(data.vsize().v));
+			cancel(true);
+			return;
+		}
+		partLoaded(requestData.offset, bytes::make_span(data.vbytes().v));
+	});
+}
+
+void mtpFileLoader::cdnPartLoaded(const MTPupload_CdnFile &result, mtpRequestId requestId) {
+	Expects(!_finished);
+
+	const auto requestData = finishSentRequest(requestId);
+	result.match([&](const MTPDupload_cdnFileReuploadNeeded &data) {
+		const auto shiftedDcId = MTP::downloadDcId(
+			dcId(),
+			requestData.dcIndex);
+		const auto requestId = MTP::send(
+			MTPupload_ReuploadCdnFile(
+				MTP_bytes(_cdnToken),
+				data.vrequest_token()),
+			rpcDone(&mtpFileLoader::reuploadDone),
+			rpcFail(&mtpFileLoader::cdnPartFailed),
+			shiftedDcId);
+		placeSentRequest(requestId, requestData);
+	}, [&](const MTPDupload_cdnFile &data) {
+		auto key = bytes::make_span(_cdnEncryptionKey);
+		auto iv = bytes::make_span(_cdnEncryptionIV);
+		Expects(key.size() == MTP::CTRState::KeySize);
+		Expects(iv.size() == MTP::CTRState::IvecSize);
+
+		auto state = MTP::CTRState();
+		auto ivec = bytes::make_span(state.ivec);
+		std::copy(iv.begin(), iv.end(), ivec.begin());
+
+		auto counterOffset = static_cast<uint32>(requestData.offset) >> 4;
+		state.ivec[15] = static_cast<uchar>(counterOffset & 0xFF);
+		state.ivec[14] = static_cast<uchar>((counterOffset >> 8) & 0xFF);
+		state.ivec[13] = static_cast<uchar>((counterOffset >> 16) & 0xFF);
+		state.ivec[12] = static_cast<uchar>((counterOffset >> 24) & 0xFF);
+
+		auto decryptInPlace = data.vbytes().v;
+		auto buffer = bytes::make_detached_span(decryptInPlace);
+		MTP::aesCtrEncrypt(buffer, key.data(), &state);
+
+		switch (checkCdnFileHash(requestData.offset, buffer)) {
+		case CheckCdnHashResult::NoHash: {
+			_cdnUncheckedParts.emplace(requestData, decryptInPlace);
+			requestMoreCdnFileHashes();
+		} return;
+
+		case CheckCdnHashResult::Invalid: {
+			LOG(("API Error: Wrong cdnFileHash for offset %1."
+				).arg(requestData.offset));
+			cancel(true);
+		} return;
+
+		case CheckCdnHashResult::Good: {
+			partLoaded(requestData.offset, buffer);
+		} return;
+		}
+		Unexpected("Result of checkCdnFileHash()");
+	});
+}
+
+mtpFileLoader::CheckCdnHashResult mtpFileLoader::checkCdnFileHash(
+		int offset,
+		bytes::const_span buffer) {
+	const auto cdnFileHashIt = _cdnFileHashes.find(offset);
+	if (cdnFileHashIt == _cdnFileHashes.cend()) {
+		return CheckCdnHashResult::NoHash;
+	}
+	const auto realHash = openssl::Sha256(buffer);
+	const auto receivedHash = bytes::make_span(cdnFileHashIt->second.hash);
+	if (bytes::compare(realHash, receivedHash)) {
+		return CheckCdnHashResult::Invalid;
+	}
+	return CheckCdnHashResult::Good;
+}
+
+void mtpFileLoader::reuploadDone(
+		const MTPVector<MTPFileHash> &result,
+		mtpRequestId requestId) {
+	const auto requestData = finishSentRequest(requestId);
+	addCdnHashes(result.v);
+	makeRequest(requestData);
+}
+
+void mtpFileLoader::getCdnFileHashesDone(
+		const MTPVector<MTPFileHash> &result,
+		mtpRequestId requestId) {
+	Expects(!_finished);
+	Expects(_cdnHashesRequestId == requestId);
+
+	_cdnHashesRequestId = 0;
+
+	const auto requestData = finishSentRequest(requestId);
+	addCdnHashes(result.v);
+	auto someMoreChecked = false;
+	for (auto i = _cdnUncheckedParts.begin(); i != _cdnUncheckedParts.cend();) {
+		const auto uncheckedData = i->first;
+		const auto uncheckedBytes = bytes::make_span(i->second);
+
+		switch (checkCdnFileHash(uncheckedData.offset, uncheckedBytes)) {
+		case CheckCdnHashResult::NoHash: {
+			++i;
+		} break;
+
+		case CheckCdnHashResult::Invalid: {
+			LOG(("API Error: Wrong cdnFileHash for offset %1."
+				).arg(uncheckedData.offset));
+			cancel(true);
+			return;
+		} break;
+
+		case CheckCdnHashResult::Good: {
+			someMoreChecked = true;
+			const auto goodOffset = uncheckedData.offset;
+			const auto goodBytes = std::move(i->second);
+			const auto weak = QPointer<mtpFileLoader>(this);
+			i = _cdnUncheckedParts.erase(i);
+			if (!feedPart(goodOffset, bytes::make_span(goodBytes))
+				|| !weak) {
+				return;
+			} else if (_finished) {
+				notifyAboutProgress();
+				return;
+			}
+		} break;
+
+		default: Unexpected("Result of checkCdnFileHash()");
+		}
+	}
+	if (someMoreChecked) {
+		const auto weak = QPointer<mtpFileLoader>(this);
+		notifyAboutProgress();
+		if (weak) {
+			requestMoreCdnFileHashes();
+		}
+		return;
+	}
+	LOG(("API Error: "
+		"Could not find cdnFileHash for offset %1 "
+		"after getCdnFileHashes request."
+		).arg(requestData.offset));
+	cancel(true);
+}
+
+void mtpFileLoader::placeSentRequest(
+		mtpRequestId requestId,
+		const RequestData &requestData) {
+	Expects(!_finished);
+
+	_downloader->requestedAmountIncrement(
+		dcId(),
+		requestData.dcIndex,
+		Storage::kDownloadPartSize);
+	_sentRequests.emplace(requestId, requestData);
+}
+
+auto mtpFileLoader::finishSentRequest(mtpRequestId requestId)
+-> RequestData {
+	auto it = _sentRequests.find(requestId);
+	Assert(it != _sentRequests.cend());
+
+	const auto result = it->second;
+	_downloader->requestedAmountIncrement(
+		dcId(),
+		result.dcIndex,
+		-Storage::kDownloadPartSize);
+	_sentRequests.erase(it);
+
+	return result;
+}
+
+bool mtpFileLoader::feedPart(int offset, bytes::const_span buffer) {
+	if (!writeResultPart(offset, buffer)) {
+		return false;
+	}
+	if (buffer.empty() || (buffer.size() % 1024)) { // bad next offset
+		_lastComplete = true;
+	}
+	const auto finished = _sentRequests.empty()
+		&& _cdnUncheckedParts.empty()
+		&& (_lastComplete || (_size && _nextRequestOffset >= _size));
+	if (finished) {
+		_downloader->remove(this);
+		if (!finalizeResult()) {
+			return false;
+		}
+	}
+	return true;
+}
+
+void mtpFileLoader::partLoaded(int offset, bytes::const_span buffer) {
+	if (feedPart(offset, buffer)) {
+		notifyAboutProgress();
+	}
+}
+
+bool mtpFileLoader::normalPartFailed(
+		QByteArray fileReference,
+		const RPCError &error,
+		mtpRequestId requestId) {
+	if (MTP::isDefaultHandledError(error)) {
+		return false;
+	}
+	if (error.code() == 400
+		&& error.type().startsWith(qstr("FILE_REFERENCE_"))) {
+		session().api().refreshFileReference(
+			_origin,
+			this,
+			requestId,
+			fileReference);
+		return true;
+	}
+	return partFailed(error, requestId);
+}
+
+
+bool mtpFileLoader::partFailed(
+		const RPCError &error,
+		mtpRequestId requestId) {
+	if (MTP::isDefaultHandledError(error)) {
+		return false;
+	}
+	cancel(true);
+	return true;
+}
+
+bool mtpFileLoader::cdnPartFailed(
+		const RPCError &error,
+		mtpRequestId requestId) {
+	if (MTP::isDefaultHandledError(error)) {
+		return false;
+	}
+
+	if (requestId == _cdnHashesRequestId) {
+		_cdnHashesRequestId = 0;
+	}
+	if (error.type() == qstr("FILE_TOKEN_INVALID")
+		|| error.type() == qstr("REQUEST_TOKEN_INVALID")) {
+		const auto requestData = finishSentRequest(requestId);
+		changeCDNParams(
+			requestData,
+			0,
+			QByteArray(),
+			QByteArray(),
+			QByteArray(),
+			QVector<MTPFileHash>());
+		return true;
+	}
+	return partFailed(error, requestId);
+}
+
+void mtpFileLoader::startLoading() {
+	_downloader->enqueue(this);
+}
+
+void mtpFileLoader::cancelRequests() {
+	while (!_sentRequests.empty()) {
+		auto requestId = _sentRequests.begin()->first;
+		MTP::cancel(requestId);
+		[[maybe_unused]] const auto data = finishSentRequest(requestId);
+	}
+}
+
+void mtpFileLoader::switchToCDN(
+		const RequestData &requestData,
+		const MTPDupload_fileCdnRedirect &redirect) {
+	changeCDNParams(
+		requestData,
+		redirect.vdc_id().v,
+		redirect.vfile_token().v,
+		redirect.vencryption_key().v,
+		redirect.vencryption_iv().v,
+		redirect.vfile_hashes().v);
+}
+
+void mtpFileLoader::addCdnHashes(const QVector<MTPFileHash> &hashes) {
+	for (const auto &hash : hashes) {
+		hash.match([&](const MTPDfileHash &data) {
+			_cdnFileHashes.emplace(
+				data.voffset().v,
+				CdnFileHash{ data.vlimit().v, data.vhash().v });
+		});
+	}
+}
+
+void mtpFileLoader::changeCDNParams(
+		const RequestData &requestData,
+		MTP::DcId dcId,
+		const QByteArray &token,
+		const QByteArray &encryptionKey,
+		const QByteArray &encryptionIV,
+		const QVector<MTPFileHash> &hashes) {
+	if (dcId != 0
+		&& (encryptionKey.size() != MTP::CTRState::KeySize
+			|| encryptionIV.size() != MTP::CTRState::IvecSize)) {
+		LOG(("Message Error: Wrong key (%1) / iv (%2) size in CDN params").arg(encryptionKey.size()).arg(encryptionIV.size()));
+		cancel(true);
+		return;
+	}
+
+	auto resendAllRequests = (_cdnDcId != dcId
+		|| _cdnToken != token
+		|| _cdnEncryptionKey != encryptionKey
+		|| _cdnEncryptionIV != encryptionIV);
+	_cdnDcId = dcId;
+	_cdnToken = token;
+	_cdnEncryptionKey = encryptionKey;
+	_cdnEncryptionIV = encryptionIV;
+	addCdnHashes(hashes);
+
+	if (resendAllRequests && !_sentRequests.empty()) {
+		auto resendRequests = std::vector<RequestData>();
+		resendRequests.reserve(_sentRequests.size());
+		while (!_sentRequests.empty()) {
+			auto requestId = _sentRequests.begin()->first;
+			MTP::cancel(requestId);
+			resendRequests.push_back(finishSentRequest(requestId));
+		}
+		for (const auto &requestData : resendRequests) {
+			makeRequest(requestData);
+		}
+	}
+	makeRequest(requestData);
+}
+
+Storage::Cache::Key mtpFileLoader::cacheKey() const {
+	return _location.match([&](const WebFileLocation &location) {
+		return Data::WebDocumentCacheKey(location);
+	}, [&](const GeoPointLocation &location) {
+		return Data::GeoPointCacheKey(location);
+	}, [&](const StorageFileLocation &location) {
+		return location.cacheKey();
+	});
+}
+
+std::optional<MediaKey> mtpFileLoader::fileLocationKey() const {
+	if (_locationType != UnknownFileLocation) {
+		return mediaKey(_locationType, dcId(), objId());
+	}
+	return std::nullopt;
+}

Telegram/SourceFiles/storage/file_download_mtproto.h

@@ -0,0 +1,140 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "storage/file_download.h"
+
+class StorageImageLocation;
+class WebFileLocation;
+class mtpFileLoader final
+	: public FileLoader
+	, public RPCSender
+	, public Storage::Downloader {
+public:
+	mtpFileLoader(
+		const StorageFileLocation &location,
+		Data::FileOrigin origin,
+		LocationType type,
+		const QString &toFile,
+		int32 size,
+		LoadToCacheSetting toCache,
+		LoadFromCloudSetting fromCloud,
+		bool autoLoading,
+		uint8 cacheTag);
+	mtpFileLoader(
+		const WebFileLocation &location,
+		int32 size,
+		LoadFromCloudSetting fromCloud,
+		bool autoLoading,
+		uint8 cacheTag);
+	mtpFileLoader(
+		const GeoPointLocation &location,
+		int32 size,
+		LoadFromCloudSetting fromCloud,
+		bool autoLoading,
+		uint8 cacheTag);
+
+	Data::FileOrigin fileOrigin() const override;
+
+	uint64 objId() const override;
+
+	void stop() override {
+		rpcInvalidate();
+	}
+	void refreshFileReferenceFrom(
+		const Data::UpdatedFileReferences &updates,
+		int requestId,
+		const QByteArray &current);
+
+	~mtpFileLoader();
+
+private:
+	struct RequestData {
+		int offset = 0;
+		int dcIndex = 0;
+
+		inline bool operator<(const RequestData &other) const {
+			return offset < other.offset;
+		}
+	};
+	struct CdnFileHash {
+		CdnFileHash(int limit, QByteArray hash) : limit(limit), hash(hash) {
+		}
+		int limit = 0;
+		QByteArray hash;
+	};
+	Storage::Cache::Key cacheKey() const override;
+	std::optional<MediaKey> fileLocationKey() const override;
+	void startLoading() override;
+	void cancelRequests() override;
+
+	void makeRequest(const RequestData &requestData);
+
+	MTP::DcId dcId() const override;
+	bool readyToRequest() const override;
+	void loadPart(int dcIndex) override;
+	void normalPartLoaded(const MTPupload_File &result, mtpRequestId requestId);
+	void webPartLoaded(const MTPupload_WebFile &result, mtpRequestId requestId);
+	void cdnPartLoaded(const MTPupload_CdnFile &result, mtpRequestId requestId);
+	void reuploadDone(const MTPVector<MTPFileHash> &result, mtpRequestId requestId);
+	void requestMoreCdnFileHashes();
+	void getCdnFileHashesDone(const MTPVector<MTPFileHash> &result, mtpRequestId requestId);
+
+	void partLoaded(int offset, bytes::const_span buffer);
+	bool feedPart(int offset, bytes::const_span buffer);
+
+	bool partFailed(const RPCError &error, mtpRequestId requestId);
+	bool normalPartFailed(QByteArray fileReference, const RPCError &error, mtpRequestId requestId);
+	bool cdnPartFailed(const RPCError &error, mtpRequestId requestId);
+
+	mtpRequestId sendRequest(const RequestData &requestData);
+	void placeSentRequest(
+		mtpRequestId requestId,
+		const RequestData &requestData);
+	[[nodiscard]] RequestData finishSentRequest(mtpRequestId requestId);
+	void switchToCDN(
+		const RequestData &requestData,
+		const MTPDupload_fileCdnRedirect &redirect);
+	void addCdnHashes(const QVector<MTPFileHash> &hashes);
+	void changeCDNParams(
+		const RequestData &requestData,
+		MTP::DcId dcId,
+		const QByteArray &token,
+		const QByteArray &encryptionKey,
+		const QByteArray &encryptionIV,
+		const QVector<MTPFileHash> &hashes);
+
+	enum class CheckCdnHashResult {
+		NoHash,
+		Invalid,
+		Good,
+	};
+	CheckCdnHashResult checkCdnFileHash(int offset, bytes::const_span buffer);
+
+	const not_null<Storage::DownloadManager*> _downloader;
+	const MTP::DcId _dcId = 0;
+	std::map<mtpRequestId, RequestData> _sentRequests;
+
+	bool _lastComplete = false;
+	int32 _nextRequestOffset = 0;
+
+	base::variant<
+		StorageFileLocation,
+		WebFileLocation,
+		GeoPointLocation> _location;
+	Data::FileOrigin _origin;
+
+	MTP::DcId _cdnDcId = 0;
+	QByteArray _cdnToken;
+	QByteArray _cdnEncryptionKey;
+	QByteArray _cdnEncryptionIV;
+	base::flat_map<int, CdnFileHash> _cdnFileHashes;
+	base::flat_map<RequestData, QByteArray> _cdnUncheckedParts;
+	mtpRequestId _cdnHashesRequestId = 0;
+
+};

Telegram/SourceFiles/storage/file_download_web.cpp

@@ -0,0 +1,527 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/file_download_web.h"
+
+#include "storage/cache/storage_cache_types.h"
+
+#include <QtNetwork/QAuthenticator>
+
+namespace {
+
+constexpr auto kMaxWebFileQueries = 8;
+constexpr auto kMaxHttpRedirects = 5;
+constexpr auto kResetDownloadPrioritiesTimeout = crl::time(200);
+
+std::weak_ptr<WebLoadManager> GlobalLoadManager;
+
+using ErrorSignal = void(QNetworkReply::*)(QNetworkReply::NetworkError);
+const auto QNetworkReply_error = ErrorSignal(&QNetworkReply::error);
+
+[[nodiscard]] std::shared_ptr<WebLoadManager> GetManager() {
+	auto result = GlobalLoadManager.lock();
+	if (!result) {
+		GlobalLoadManager = result = std::make_shared<WebLoadManager>();
+	}
+	return result;
+}
+
+enum class ProcessResult {
+	Error,
+	Progress,
+	Finished,
+};
+
+enum class Error {
+};
+
+struct Progress {
+	qint64 ready = 0;
+	qint64 total = 0;
+};
+
+using Update = base::variant<Progress, QByteArray, Error>;
+
+struct UpdateForLoader {
+	not_null<webFileLoader*> loader;
+	Update data;
+};
+
+} // namespace
+
+class WebLoadManager final : public QObject {
+public:
+	WebLoadManager();
+	~WebLoadManager();
+
+	void enqueue(not_null<webFileLoader*> loader);
+	void remove(not_null<webFileLoader*> loader);
+
+	[[nodiscard]] rpl::producer<Update> updates(
+		not_null<webFileLoader*> loader) const;
+
+private:
+	struct Enqueued {
+		int id = 0;
+		QString url;
+	};
+	struct Sent {
+		QString url;
+		not_null<QNetworkReply*> reply;
+		QByteArray data;
+		int64 ready = 0;
+		int64 total = 0;
+		int redirectsLeft = kMaxHttpRedirects;
+	};
+
+	// Constructor.
+	void handleNetworkErrors();
+
+	// Worker thread.
+	void enqueue(int id, const QString &url);
+	void remove(int id);
+	void resetGeneration();
+	void checkSendNext();
+	void send(const Enqueued &entry);
+	[[nodiscard]] not_null<QNetworkReply*> send(int id, const QString &url);
+	[[nodiscard]] Sent *findSent(int id, not_null<QNetworkReply*> reply);
+	void removeSent(int id);
+	void progress(
+		int id,
+		not_null<QNetworkReply*> reply,
+		int64 ready,
+		int64 total);
+	void failed(
+		int id,
+		not_null<QNetworkReply*> reply,
+		QNetworkReply::NetworkError error);
+	void redirect(int id, not_null<QNetworkReply*> reply);
+	void notify(
+		int id,
+		not_null<QNetworkReply*> reply,
+		int64 ready,
+		int64 total);
+	void failed(int id, not_null<QNetworkReply*> reply);
+	void finished(int id, not_null<QNetworkReply*> reply);
+	void deleteDeferred(not_null<QNetworkReply*> reply);
+	void queueProgressUpdate(int id, int64 ready, int64 total);
+	void queueFailedUpdate(int id);
+	void queueFinishedUpdate(int id, const QByteArray &data);
+	void clear();
+
+	// Main thread.
+	void sendUpdate(int id, Update &&data);
+
+	QThread _thread;
+	QNetworkAccessManager _network;
+	base::Timer _resetGenerationTimer;
+
+	// Main thread.
+	rpl::event_stream<UpdateForLoader> _updates;
+	int _autoincrement = 0;
+	base::flat_map<not_null<webFileLoader*>, int> _ids;
+
+	// Worker thread.
+	std::deque<Enqueued> _queue;
+	std::deque<Enqueued> _previousGeneration;
+	base::flat_map<int, Sent> _sent;
+	std::vector<QPointer<QNetworkReply>> _repliesBeingDeleted;
+
+};
+
+WebLoadManager::WebLoadManager()
+: _resetGenerationTimer(&_thread, [=] { resetGeneration(); }) {
+	handleNetworkErrors();
+
+	const auto original = QThread::currentThread();
+	moveToThread(&_thread);
+	_network.moveToThread(&_thread);
+	connect(&_thread, &QThread::finished, [=] {
+		clear();
+		moveToThread(original);
+		_network.moveToThread(original);
+	});
+	_thread.start();
+}
+
+void WebLoadManager::handleNetworkErrors() {
+	const auto fail = [=](QNetworkReply *reply) {
+		for (const auto &[id, sent] : _sent) {
+			if (sent.reply == reply) {
+				failed(id, reply);
+				return;
+			}
+		}
+	};
+	connect(&_network, &QNetworkAccessManager::authenticationRequired, fail);
+	connect(&_network, &QNetworkAccessManager::sslErrors, fail);
+}
+
+WebLoadManager::~WebLoadManager() {
+	_thread.quit();
+	_thread.wait();
+}
+
+[[nodiscard]] rpl::producer<Update> WebLoadManager::updates(
+		not_null<webFileLoader*> loader) const {
+	return _updates.events(
+	) | rpl::filter([=](const UpdateForLoader &update) {
+		return (update.loader == loader);
+	}) | rpl::map([=](UpdateForLoader &&update) {
+		return std::move(update.data);
+	});
+}
+
+void WebLoadManager::enqueue(not_null<webFileLoader*> loader) {
+	const auto id = [&] {
+		const auto i = _ids.find(loader);
+		return (i != end(_ids))
+			? i->second
+			: _ids.emplace(loader, ++_autoincrement).first->second;
+	}();
+	const auto url = loader->url();
+	InvokeQueued(this, [=] {
+		enqueue(id, url);
+	});
+}
+
+void WebLoadManager::remove(not_null<webFileLoader*> loader) {
+	const auto i = _ids.find(loader);
+	if (i == end(_ids)) {
+		return;
+	}
+	const auto id = i->second;
+	_ids.erase(i);
+	InvokeQueued(this, [=] {
+		remove(id);
+	});
+}
+
+void WebLoadManager::enqueue(int id, const QString &url) {
+	const auto i = ranges::find(_queue, id, &Enqueued::id);
+	if (i != end(_queue)) {
+		return;
+	}
+	_previousGeneration.erase(
+		ranges::remove(_previousGeneration, id, &Enqueued::id),
+		end(_previousGeneration));
+	_queue.push_back(Enqueued{ id, url });
+	if (!_resetGenerationTimer.isActive()) {
+		_resetGenerationTimer.callOnce(kResetDownloadPrioritiesTimeout);
+	}
+	checkSendNext();
+}
+
+void WebLoadManager::remove(int id) {
+	_queue.erase(ranges::remove(_queue, id, &Enqueued::id), end(_queue));
+	_previousGeneration.erase(
+		ranges::remove(_previousGeneration, id, &Enqueued::id),
+		end(_previousGeneration));
+	removeSent(id);
+	checkSendNext();
+}
+
+void WebLoadManager::resetGeneration() {
+	if (!_previousGeneration.empty()) {
+		std::copy(
+			begin(_previousGeneration),
+			end(_previousGeneration),
+			std::back_inserter(_queue));
+		_previousGeneration.clear();
+	}
+	std::swap(_queue, _previousGeneration);
+}
+
+void WebLoadManager::checkSendNext() {
+	if (_sent.size() >= kMaxWebFileQueries
+		|| (_queue.empty() && _previousGeneration.empty())) {
+		return;
+	}
+	const auto entry = _queue.empty()
+		? _previousGeneration.front()
+		: _queue.front();
+	(_queue.empty() ? _previousGeneration : _queue).pop_front();
+	send(entry);
+}
+
+void WebLoadManager::send(const Enqueued &entry) {
+	const auto id = entry.id;
+	const auto url = entry.url;
+	_sent.emplace(id, Sent{ url, send(id, url) });
+}
+
+void WebLoadManager::removeSent(int id) {
+	if (const auto i = _sent.find(id); i != end(_sent)) {
+		deleteDeferred(i->second.reply);
+		_sent.erase(i);
+	}
+}
+
+not_null<QNetworkReply*> WebLoadManager::send(int id, const QString &url) {
+	const auto result = _network.get(QNetworkRequest(url));
+	const auto handleProgress = [=](qint64 ready, qint64 total) {
+		progress(id, result, ready, total);
+		checkSendNext();
+	};
+	const auto handleError = [=](QNetworkReply::NetworkError error) {
+		failed(id, result, error);
+		checkSendNext();
+	};
+	connect(result, &QNetworkReply::downloadProgress, handleProgress);
+	connect(result, QNetworkReply_error, handleError);
+	return result;
+}
+
+WebLoadManager::Sent *WebLoadManager::findSent(
+		int id,
+		not_null<QNetworkReply*> reply) {
+	const auto i = _sent.find(id);
+	return (i != end(_sent) && i->second.reply == reply)
+		? &i->second
+		: nullptr;
+}
+
+void WebLoadManager::progress(
+		int id,
+		not_null<QNetworkReply*> reply,
+		int64 ready,
+		int64 total) {
+	const auto statusCode = reply->attribute(
+		QNetworkRequest::HttpStatusCodeAttribute);
+	const auto status = statusCode.isValid() ? statusCode.toInt() : 200;
+	if (status == 301 || status == 302) {
+		redirect(id, reply);
+	} else if (status != 200 && status != 206 && status != 416) {
+		LOG(("Network Error: "
+			"Bad HTTP status received in WebLoadManager::onProgress() %1"
+			).arg(status));
+		failed(id, reply);
+	} else {
+		notify(id, reply, ready, total);
+	}
+}
+
+void WebLoadManager::redirect(int id, not_null<QNetworkReply*> reply) {
+	const auto header = reply->header(QNetworkRequest::LocationHeader);
+	const auto url = header.toString();
+	if (url.isEmpty()) {
+		return;
+	}
+
+	if (const auto sent = findSent(id, reply)) {
+		if (!sent->redirectsLeft--) {
+			LOG(("Network Error: "
+				"Too many HTTP redirects in onFinished() "
+				"for web file loader: %1").arg(url));
+			failed(id, reply);
+			return;
+		}
+		deleteDeferred(reply);
+		sent->url = url;
+		sent->reply = send(id, url);
+	}
+}
+
+void WebLoadManager::notify(
+		int id,
+		not_null<QNetworkReply*> reply,
+		int64 ready,
+		int64 total) {
+	if (const auto sent = findSent(id, reply)) {
+		sent->ready = ready;
+		sent->total = std::max(total, int64(0));
+		sent->data.append(reply->readAll());
+		if (total == 0
+			|| total > Storage::kMaxFileInMemory
+			|| sent->data.size() > Storage::kMaxFileInMemory) {
+			LOG(("Network Error: "
+				"Bad size received for HTTP download progress "
+				"in WebLoadManager::onProgress(): %1 / %2 (bytes %3)"
+				).arg(ready
+				).arg(total
+				).arg(sent->data.size()));
+			failed(id, reply);
+		} else if (total > 0 && ready >= total) {
+			finished(id, reply);
+		} else {
+			queueProgressUpdate(id, sent->ready, sent->total);
+		}
+	}
+}
+
+void WebLoadManager::failed(
+		int id,
+		not_null<QNetworkReply*> reply,
+		QNetworkReply::NetworkError error) {
+	if (const auto sent = findSent(id, reply)) {
+		LOG(("Network Error: "
+			"Failed to request '%1', error %2 (%3)"
+			).arg(sent->url
+			).arg(int(error)
+			).arg(reply->errorString()));
+		failed(id, reply);
+	}
+}
+
+void WebLoadManager::failed(int id, not_null<QNetworkReply*> reply) {
+	if (const auto sent = findSent(id, reply)) {
+		removeSent(id);
+		queueFailedUpdate(id);
+	}
+}
+
+void WebLoadManager::deleteDeferred(not_null<QNetworkReply*> reply) {
+	reply->deleteLater();
+	_repliesBeingDeleted.erase(
+		ranges::remove(_repliesBeingDeleted, nullptr),
+		end(_repliesBeingDeleted));
+	_repliesBeingDeleted.emplace_back(reply.get());
+}
+
+void WebLoadManager::finished(int id, not_null<QNetworkReply*> reply) {
+	if (const auto sent = findSent(id, reply)) {
+		const auto data = base::take(sent->data);
+		removeSent(id);
+		queueFinishedUpdate(id, data);
+	}
+}
+
+void WebLoadManager::clear() {
+	for (const auto &[id, sent] : base::take(_sent)) {
+		sent.reply->abort();
+		delete sent.reply;
+	}
+	for (const auto reply : base::take(_repliesBeingDeleted)) {
+		if (reply) {
+			delete reply;
+		}
+	}
+}
+
+void WebLoadManager::queueProgressUpdate(int id, int64 ready, int64 total) {
+	crl::on_main(this, [=] {
+		sendUpdate(id, Progress{ ready, total });
+	});
+}
+
+void WebLoadManager::queueFailedUpdate(int id) {
+	crl::on_main(this, [=] {
+		sendUpdate(id, Error{});
+	});
+}
+
+void WebLoadManager::queueFinishedUpdate(int id, const QByteArray &data) {
+	crl::on_main(this, [=] {
+		LOG(("FINISHED UPDATE FOR: %1").arg(id));
+		for (const auto &[loader, loaderId] : _ids) {
+			if (loaderId == id) {
+				LOG(("LOADER ID: %2").arg(quintptr(loader.get())));
+				break;
+			}
+		}
+		LOG(("SENT"));
+		sendUpdate(id, QByteArray(data));
+	});
+}
+
+void WebLoadManager::sendUpdate(int id, Update &&data) {
+	for (const auto &[loader, loaderId] : _ids) {
+		if (loaderId == id) {
+			_updates.fire(UpdateForLoader{ loader, std::move(data) });
+			return;
+		}
+	}
+}
+
+webFileLoader::webFileLoader(
+	const QString &url,
+	const QString &to,
+	LoadFromCloudSetting fromCloud,
+	bool autoLoading,
+	uint8 cacheTag)
+: FileLoader(
+	QString(),
+	0,
+	UnknownFileLocation,
+	LoadToCacheAsWell,
+	fromCloud,
+	autoLoading,
+	cacheTag)
+, _url(url) {
+}
+
+webFileLoader::~webFileLoader() {
+	cancelRequests();
+}
+
+QString webFileLoader::url() const {
+	return _url;
+}
+
+void webFileLoader::startLoading() {
+	if (_finished) {
+		return;
+	} else if (!_manager) {
+		_manager = GetManager();
+		_manager->updates(
+			this
+		) | rpl::start_with_next([=](const Update &data) {
+			if (const auto progress = base::get_if<Progress>(&data)) {
+				loadProgress(progress->ready, progress->total);
+			} else if (const auto bytes = base::get_if<QByteArray>(&data)) {
+				loadFinished(*bytes);
+			} else {
+				loadFailed();
+			}
+		}, _managerLifetime);
+	}
+	_manager->enqueue(this);
+}
+
+int webFileLoader::currentOffset() const {
+	return _ready;
+}
+
+void webFileLoader::loadProgress(qint64 ready, qint64 total) {
+	_size = total;
+	_ready = ready;
+	notifyAboutProgress();
+}
+
+void webFileLoader::loadFinished(const QByteArray &data) {
+	cancelRequests();
+	if (writeResultPart(0, bytes::make_span(data))) {
+		if (finalizeResult()) {
+			notifyAboutProgress();
+		}
+	}
+}
+
+void webFileLoader::loadFailed() {
+	cancel(true);
+}
+
+Storage::Cache::Key webFileLoader::cacheKey() const {
+	return Data::UrlCacheKey(_url);
+}
+
+std::optional<MediaKey> webFileLoader::fileLocationKey() const {
+	return std::nullopt;
+}
+
+void webFileLoader::stop() {
+	cancelRequests();
+}
+
+void webFileLoader::cancelRequests() {
+	if (!_manager) {
+		return;
+	}
+	_managerLifetime.destroy();
+	_manager->remove(this);
+	_manager = nullptr;
+}

Telegram/SourceFiles/storage/file_download_web.h

@@ -0,0 +1,45 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "storage/file_download.h"
+
+class WebLoadManager;
+
+class webFileLoader final : public FileLoader {
+public:
+	webFileLoader(
+		const QString &url,
+		const QString &to,
+		LoadFromCloudSetting fromCloud,
+		bool autoLoading,
+		uint8 cacheTag);
+	~webFileLoader();
+
+	[[nodiscard]] QString url() const;
+
+	int currentOffset() const override;
+	void stop() override;
+	void cancelRequests() override;
+
+private:
+	void startLoading() override;
+	Storage::Cache::Key cacheKey() const override;
+	std::optional<MediaKey> fileLocationKey() const override;
+
+	void loadProgress(qint64 ready, qint64 size);
+	void loadFinished(const QByteArray &data);
+	void loadFailed();
+
+	const QString _url;
+	int _ready = 0;
+
+	std::shared_ptr<WebLoadManager> _manager;
+	rpl::lifetime _managerLifetime;
+
+};

Telegram/SourceFiles/storage/streamed_file_downloader.cpp

@@ -38,7 +38,6 @@ StreamedFileDownloader::StreamedFileDownloader(
 	uint8 cacheTag)
 : FileLoader(
 	toFile,
-	dcId,
 	size,
 	locationType,
 	toCache,
@@ -61,8 +60,6 @@ StreamedFileDownloader::StreamedFileDownloader(
 			savePart(std::move(part));
 		}
 	}, _lifetime);
-
-	requestParts();
 }
 
 StreamedFileDownloader::~StreamedFileDownloader() {
@@ -131,12 +128,8 @@ void StreamedFileDownloader::cancelRequests() {
 	_reader->cancelForDownloader(this);
 }
 
-bool StreamedFileDownloader::readyToRequest() const {
-	return false;
-}
-
-void StreamedFileDownloader::loadPart(int dcIndex) {
-	Unexpected("StreamedFileDownloader can't load parts.");
+void StreamedFileDownloader::startLoading() {
+	requestParts();
 }
 
 void StreamedFileDownloader::savePart(const LoadedPart &part) {

Telegram/SourceFiles/storage/streamed_file_downloader.h

@@ -46,11 +46,10 @@ public:
 	QByteArray readLoadedPart(int offset);
 
 private:
+	void startLoading() override;
 	Cache::Key cacheKey() const override;
 	std::optional<MediaKey> fileLocationKey() const override;
 	void cancelRequests() override;
-	bool readyToRequest() const override;
-	void loadPart(int dcIndex) override;
 	void requestParts();
 	void requestPart();
 

Telegram/SourceFiles/ui/image/image_source.cpp

@@ -7,10 +7,11 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 */
 #include "ui/image/image_source.h"
 
-#include "storage/file_download.h"
+#include "storage/cache/storage_cache_database.h"
+#include "storage/file_download_mtproto.h"
+#include "storage/file_download_web.h"
 #include "data/data_session.h"
 #include "data/data_file_origin.h"
-#include "storage/cache/storage_cache_database.h"
 #include "history/history_item.h"
 #include "history/history.h"
 #include "main/main_session.h"

Telegram/gyp/telegram/sources.txt

@@ -685,6 +685,10 @@
 <(src_loc)/settings/settings_privacy_security.h
 <(src_loc)/storage/file_download.cpp
 <(src_loc)/storage/file_download.h
+<(src_loc)/storage/file_download_mtproto.cpp
+<(src_loc)/storage/file_download_mtproto.h
+<(src_loc)/storage/file_download_web.cpp
+<(src_loc)/storage/file_download_web.h
 <(src_loc)/storage/file_upload.cpp
 <(src_loc)/storage/file_upload.h
 <(src_loc)/storage/localimageloader.cpp