mirror of
https://github.com/telegramdesktop/tdesktop
synced 2025-03-25 04:38:23 +00:00
Use vector queues in mtproto file downloader.
This commit is contained in:
John Preston
parent
524d64a462
commit
12f5ccaaa5
@ -1287,7 +1287,6 @@ void PeerListContent::loadProfilePhotos() {
|
||||
|
||||
auto yFrom = _visibleTop;
|
||||
auto yTo = _visibleBottom + (_visibleBottom - _visibleTop) * PreloadHeightsCount;
|
||||
_controller->session().downloader().clearPriorities();
|
||||
|
||||
if (yTo < 0) return;
|
||||
if (yFrom < 0) yFrom = 0;
|
||||
|
||||
@ -722,7 +722,6 @@ void ShareBox::Inner::loadProfilePhotos(int yFrom) {
|
||||
yFrom *= _columnCount;
|
||||
yTo *= _columnCount;
|
||||
|
||||
_navigation->session().downloader().clearPriorities();
|
||||
if (_filter.isEmpty()) {
|
||||
if (!_chatsIndexed->empty()) {
|
||||
auto i = _chatsIndexed->cfind(yFrom, _rowHeight);
|
||||
|
||||
@ -2464,7 +2464,6 @@ void InnerWidget::loadPeerPhotos() {
|
||||
|
||||
auto yFrom = _visibleTop;
|
||||
auto yTo = _visibleTop + (_visibleBottom - _visibleTop) * (PreloadHeightsCount + 1);
|
||||
session().downloader().clearPriorities();
|
||||
if (_state == WidgetState::Default) {
|
||||
auto otherStart = shownDialogs()->size() * st::dialogsRowHeight;
|
||||
if (yFrom < otherStart) {
|
||||
|
||||
@ -1798,8 +1798,6 @@ void HistoryWidget::showHistory(
|
||||
_nonEmptySelection = false;
|
||||
|
||||
if (_peer) {
|
||||
session().downloader().clearPriorities();
|
||||
|
||||
_history = _peer->owner().history(_peer);
|
||||
_migrated = _history->migrateFrom();
|
||||
if (_migrated
|
||||
|
||||
@ -1803,7 +1803,6 @@ void OverlayWidget::displayPhoto(not_null<PhotoData*> photo, HistoryItem *item)
|
||||
|
||||
_zoom = 0;
|
||||
_zoomToScreen = 0;
|
||||
Auth().downloader().clearPriorities();
|
||||
_blurred = true;
|
||||
_current = QPixmap();
|
||||
_down = OverNone;
|
||||
|
||||
@ -32,6 +32,8 @@ constexpr auto kKillSessionTimeout = 15 * crl::time(1000);
|
||||
// Max 16 file parts downloaded at the same time, 128 KB each.
|
||||
constexpr auto kMaxFileQueries = 16;
|
||||
|
||||
constexpr auto kMaxWaitedInConnection = 512 * 1024;
|
||||
|
||||
// Max 8 http[s] files downloaded at the same time.
|
||||
constexpr auto kMaxWebFileQueries = 8;
|
||||
|
||||
@ -43,17 +45,108 @@ constexpr auto kPartSize = 128 * 1024;
|
||||
|
||||
} // namespace
|
||||
|
||||
void Downloader::Queue::enqueue(not_null<FileLoader*> loader) {
|
||||
const auto i = ranges::find(_loaders, loader);
|
||||
if (i != end(_loaders)) {
|
||||
return;
|
||||
}
|
||||
_loaders.push_back(loader);
|
||||
_previousGeneration.erase(
|
||||
ranges::remove(_previousGeneration, loader),
|
||||
end(_previousGeneration));
|
||||
}
|
||||
|
||||
void Downloader::Queue::remove(not_null<FileLoader*> loader) {
|
||||
_loaders.erase(ranges::remove(_loaders, loader), end(_loaders));
|
||||
_previousGeneration.erase(
|
||||
ranges::remove(_previousGeneration, loader),
|
||||
end(_previousGeneration));
|
||||
}
|
||||
|
||||
void Downloader::Queue::resetGeneration() {
|
||||
if (!_previousGeneration.empty()) {
|
||||
_loaders.reserve(_loaders.size() + _previousGeneration.size());
|
||||
std::copy(
|
||||
begin(_previousGeneration),
|
||||
end(_previousGeneration),
|
||||
std::back_inserter(_loaders));
|
||||
_previousGeneration.clear();
|
||||
}
|
||||
std::swap(_loaders, _previousGeneration);
|
||||
}
|
||||
|
||||
FileLoader *Downloader::Queue::nextLoader() const {
|
||||
auto &&all = ranges::view::concat(_loaders, _previousGeneration);
|
||||
const auto i = ranges::find(all, true, &FileLoader::readyToRequest);
|
||||
return (i != all.end()) ? i->get() : nullptr;
|
||||
}
|
||||
|
||||
Downloader::Downloader(not_null<ApiWrap*> api)
|
||||
: _api(api)
|
||||
, _killDownloadSessionsTimer([=] { killDownloadSessions(); })
|
||||
, _queueForWeb(kMaxWebFileQueries) {
|
||||
, _killDownloadSessionsTimer([=] { killDownloadSessions(); }) {
|
||||
}
|
||||
|
||||
void Downloader::clearPriorities() {
|
||||
++_priority;
|
||||
Downloader::~Downloader() {
|
||||
killDownloadSessions();
|
||||
}
|
||||
|
||||
void Downloader::requestedAmountIncrement(MTP::DcId dcId, int index, int amount) {
|
||||
void Downloader::enqueue(not_null<FileLoader*> loader) {
|
||||
const auto dcId = loader->dcId();
|
||||
(dcId ? _mtprotoLoaders[dcId] : _webLoaders).enqueue(loader);
|
||||
if (!_resettingGeneration) {
|
||||
_resettingGeneration = true;
|
||||
crl::on_main(this, [=] {
|
||||
resetGeneration();
|
||||
});
|
||||
}
|
||||
checkSendNext();
|
||||
}
|
||||
|
||||
void Downloader::remove(not_null<FileLoader*> loader) {
|
||||
const auto dcId = loader->dcId();
|
||||
(dcId ? _mtprotoLoaders[dcId] : _webLoaders).remove(loader);
|
||||
crl::on_main(&_api->session(), [=] { checkSendNext(); });
|
||||
}
|
||||
|
||||
void Downloader::resetGeneration() {
|
||||
_resettingGeneration = false;
|
||||
for (auto &[dcId, queue] : _mtprotoLoaders) {
|
||||
queue.resetGeneration();
|
||||
}
|
||||
_webLoaders.resetGeneration();
|
||||
}
|
||||
|
||||
void Downloader::checkSendNext() {
|
||||
for (auto &[dcId, queue] : _mtprotoLoaders) {
|
||||
const auto bestIndex = [&] {
|
||||
const auto i = _requestedBytesAmount.find(dcId);
|
||||
if (i == end(_requestedBytesAmount)) {
|
||||
return 0;
|
||||
}
|
||||
const auto j = ranges::min_element(i->second);
|
||||
const auto inConnection = *j;
|
||||
return (inConnection + kPartSize <= kMaxWaitedInConnection)
|
||||
? (j - begin(i->second))
|
||||
: -1;
|
||||
}();
|
||||
if (bestIndex < 0) {
|
||||
continue;
|
||||
}
|
||||
if (const auto loader = queue.nextLoader()) {
|
||||
loader->loadPart(bestIndex);
|
||||
}
|
||||
}
|
||||
if (_requestedBytesAmount[0][0] < kMaxWebFileQueries) {
|
||||
if (const auto loader = _webLoaders.nextLoader()) {
|
||||
loader->loadPart(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Downloader::requestedAmountIncrement(
|
||||
MTP::DcId dcId,
|
||||
int index,
|
||||
int amount) {
|
||||
Expects(index >= 0 && index < MTP::kDownloadSessionsCount);
|
||||
|
||||
using namespace rpl::mappers;
|
||||
@ -63,6 +156,9 @@ void Downloader::requestedAmountIncrement(MTP::DcId dcId, int index, int amount)
|
||||
it = _requestedBytesAmount.emplace(dcId, RequestedInDc { { 0 } }).first;
|
||||
}
|
||||
it->second[index] += amount;
|
||||
if (!dcId) {
|
||||
return; // webLoaders.
|
||||
}
|
||||
if (amount > 0) {
|
||||
killDownloadSessionsStop(dcId);
|
||||
} else if (ranges::find_if(it->second, _1 > 0) == end(it->second)) {
|
||||
@ -70,6 +166,13 @@ void Downloader::requestedAmountIncrement(MTP::DcId dcId, int index, int amount)
|
||||
}
|
||||
}
|
||||
|
||||
int Downloader::chooseDcIndexForRequest(MTP::DcId dcId) {
|
||||
const auto i = _requestedBytesAmount.find(dcId);
|
||||
return (i != end(_requestedBytesAmount))
|
||||
? (ranges::min_element(i->second) - begin(i->second))
|
||||
: 0;
|
||||
}
|
||||
|
||||
void Downloader::killDownloadSessionsStart(MTP::DcId dcId) {
|
||||
if (!_killDownloadSessionTimes.contains(dcId)) {
|
||||
_killDownloadSessionTimes.emplace(
|
||||
@ -110,35 +213,6 @@ void Downloader::killDownloadSessions() {
|
||||
}
|
||||
}
|
||||
|
||||
int Downloader::chooseDcIndexForRequest(MTP::DcId dcId) const {
|
||||
auto result = 0;
|
||||
auto it = _requestedBytesAmount.find(dcId);
|
||||
if (it != _requestedBytesAmount.cend()) {
|
||||
for (auto i = 1; i != MTP::kDownloadSessionsCount; ++i) {
|
||||
if (it->second[i] < it->second[result]) {
|
||||
result = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
not_null<Downloader::Queue*> Downloader::queueForDc(MTP::DcId dcId) {
|
||||
const auto i = _queuesForDc.find(dcId);
|
||||
const auto result = (i != end(_queuesForDc))
|
||||
? i
|
||||
: _queuesForDc.emplace(dcId, Queue(kMaxFileQueries)).first;
|
||||
return &result->second;
|
||||
}
|
||||
|
||||
not_null<Downloader::Queue*> Downloader::queueForWeb() {
|
||||
return &_queueForWeb;
|
||||
}
|
||||
|
||||
Downloader::~Downloader() {
|
||||
killDownloadSessions();
|
||||
}
|
||||
|
||||
} // namespace Storage
|
||||
|
||||
namespace {
|
||||
@ -154,13 +228,15 @@ WebLoadMainManager *_webLoadMainManager = nullptr;
|
||||
|
||||
FileLoader::FileLoader(
|
||||
const QString &toFile,
|
||||
MTP::DcId dcId,
|
||||
int32 size,
|
||||
LocationType locationType,
|
||||
LoadToCacheSetting toCache,
|
||||
LoadFromCloudSetting fromCloud,
|
||||
bool autoLoading,
|
||||
uint8 cacheTag)
|
||||
: _downloader(&Auth().downloader())
|
||||
: _dcId(dcId)
|
||||
, _downloader(&Auth().downloader())
|
||||
, _autoLoading(autoLoading)
|
||||
, _cacheTag(cacheTag)
|
||||
, _filename(toFile)
|
||||
@ -172,6 +248,10 @@ FileLoader::FileLoader(
|
||||
Expects(!_filename.isEmpty() || (_size <= Storage::kMaxFileInMemory));
|
||||
}
|
||||
|
||||
FileLoader::~FileLoader() {
|
||||
_downloader->remove(this);
|
||||
}
|
||||
|
||||
Main::Session &FileLoader::session() const {
|
||||
return _downloader->api().session();
|
||||
}
|
||||
@ -257,46 +337,7 @@ void FileLoader::permitLoadFromCloud() {
|
||||
}
|
||||
|
||||
void FileLoader::notifyAboutProgress() {
|
||||
const auto queue = _queue;
|
||||
emit progress(this);
|
||||
LoadNextFromQueue(queue);
|
||||
}
|
||||
|
||||
void FileLoader::LoadNextFromQueue(not_null<Queue*> queue) {
|
||||
if (queue->queriesCount >= queue->queriesLimit) {
|
||||
return;
|
||||
}
|
||||
for (auto i = queue->start; i;) {
|
||||
if (i->loadPart()) {
|
||||
if (queue->queriesCount >= queue->queriesLimit) {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
i = i->_next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void FileLoader::removeFromQueue() {
|
||||
if (!_inQueue) return;
|
||||
if (_next) {
|
||||
_next->_prev = _prev;
|
||||
}
|
||||
if (_prev) {
|
||||
_prev->_next = _next;
|
||||
}
|
||||
if (_queue->end == this) {
|
||||
_queue->end = _prev;
|
||||
}
|
||||
if (_queue->start == this) {
|
||||
_queue->start = _next;
|
||||
}
|
||||
_next = _prev = nullptr;
|
||||
_inQueue = false;
|
||||
}
|
||||
|
||||
FileLoader::~FileLoader() {
|
||||
removeFromQueue();
|
||||
}
|
||||
|
||||
void FileLoader::localLoaded(
|
||||
@ -331,71 +372,7 @@ void FileLoader::start() {
|
||||
return cancel(true);
|
||||
}
|
||||
}
|
||||
|
||||
auto currentPriority = _downloader->currentPriority();
|
||||
FileLoader *before = nullptr, *after = nullptr;
|
||||
if (_inQueue && _priority == currentPriority) {
|
||||
if (!_next || _next->_priority < currentPriority) return startLoading();
|
||||
after = _next;
|
||||
while (after->_next && after->_next->_priority == currentPriority) {
|
||||
after = after->_next;
|
||||
}
|
||||
} else {
|
||||
_priority = currentPriority;
|
||||
if (_inQueue) {
|
||||
if (_next && _next->_priority == currentPriority) {
|
||||
after = _next;
|
||||
} else if (_prev && _prev->_priority < currentPriority) {
|
||||
before = _prev;
|
||||
while (before->_prev && before->_prev->_priority < currentPriority) {
|
||||
before = before->_prev;
|
||||
}
|
||||
} else {
|
||||
return startLoading();
|
||||
}
|
||||
} else {
|
||||
if (_queue->start && _queue->start->_priority == currentPriority) {
|
||||
after = _queue->start;
|
||||
} else {
|
||||
before = _queue->start;
|
||||
}
|
||||
}
|
||||
if (after) {
|
||||
while (after->_next && after->_next->_priority == currentPriority) {
|
||||
after = after->_next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
removeFromQueue();
|
||||
|
||||
_inQueue = true;
|
||||
if (!_queue->start) {
|
||||
_queue->start = _queue->end = this;
|
||||
} else if (before) {
|
||||
if (before != _next) {
|
||||
_prev = before->_prev;
|
||||
_next = before;
|
||||
_next->_prev = this;
|
||||
if (_prev) {
|
||||
_prev->_next = this;
|
||||
}
|
||||
if (_queue->start->_prev) _queue->start = _queue->start->_prev;
|
||||
}
|
||||
} else if (after) {
|
||||
if (after != _prev) {
|
||||
_next = after->_next;
|
||||
_prev = after;
|
||||
after->_next = this;
|
||||
if (_next) {
|
||||
_next->_prev = this;
|
||||
}
|
||||
if (_queue->end->_next) _queue->end = _queue->end->_next;
|
||||
}
|
||||
} else {
|
||||
LOG(("Queue Error: _start && !before && !after"));
|
||||
}
|
||||
return startLoading();
|
||||
_downloader->enqueue(this);
|
||||
}
|
||||
|
||||
void FileLoader::loadLocal(const Storage::Cache::Key &key) {
|
||||
@ -480,9 +457,8 @@ void FileLoader::cancel(bool fail) {
|
||||
_file.remove();
|
||||
}
|
||||
_data = QByteArray();
|
||||
removeFromQueue();
|
||||
|
||||
const auto queue = _queue;
|
||||
const auto downloader = _downloader;
|
||||
const auto sessionGuard = &session();
|
||||
const auto weak = QPointer<FileLoader>(this);
|
||||
if (fail) {
|
||||
@ -494,16 +470,6 @@ void FileLoader::cancel(bool fail) {
|
||||
_filename = QString();
|
||||
_file.setFileName(_filename);
|
||||
}
|
||||
|
||||
// Current cancel() call could be made from ~Main::Session().
|
||||
crl::on_main(sessionGuard, [=] { LoadNextFromQueue(queue); });
|
||||
}
|
||||
|
||||
void FileLoader::startLoading() {
|
||||
if ((_queue->queriesCount >= _queue->queriesLimit) || _finished) {
|
||||
return;
|
||||
}
|
||||
loadPart();
|
||||
}
|
||||
|
||||
int FileLoader::currentOffset() const {
|
||||
@ -594,7 +560,7 @@ bool FileLoader::finalizeResult() {
|
||||
Platform::File::PostprocessDownloaded(
|
||||
QFileInfo(_file).absoluteFilePath());
|
||||
}
|
||||
removeFromQueue();
|
||||
_downloader->remove(this);
|
||||
|
||||
if (_localStatus == LocalStatus::NotFound) {
|
||||
if (const auto key = fileLocationKey()) {
|
||||
@ -627,6 +593,7 @@ mtpFileLoader::mtpFileLoader(
|
||||
uint8 cacheTag)
|
||||
: FileLoader(
|
||||
to,
|
||||
location.dcId(),
|
||||
size,
|
||||
type,
|
||||
toCache,
|
||||
@ -635,7 +602,6 @@ mtpFileLoader::mtpFileLoader(
|
||||
cacheTag)
|
||||
, _location(location)
|
||||
, _origin(origin) {
|
||||
_queue = _downloader->queueForDc(dcId());
|
||||
}
|
||||
|
||||
mtpFileLoader::mtpFileLoader(
|
||||
@ -646,6 +612,7 @@ mtpFileLoader::mtpFileLoader(
|
||||
uint8 cacheTag)
|
||||
: FileLoader(
|
||||
QString(),
|
||||
Global::WebFileDcId(),
|
||||
size,
|
||||
UnknownFileLocation,
|
||||
LoadToCacheAsWell,
|
||||
@ -653,7 +620,6 @@ mtpFileLoader::mtpFileLoader(
|
||||
autoLoading,
|
||||
cacheTag)
|
||||
, _location(location) {
|
||||
_queue = _downloader->queueForDc(dcId());
|
||||
}
|
||||
|
||||
mtpFileLoader::mtpFileLoader(
|
||||
@ -664,6 +630,7 @@ mtpFileLoader::mtpFileLoader(
|
||||
uint8 cacheTag)
|
||||
: FileLoader(
|
||||
QString(),
|
||||
Global::WebFileDcId(),
|
||||
size,
|
||||
UnknownFileLocation,
|
||||
LoadToCacheAsWell,
|
||||
@ -671,7 +638,10 @@ mtpFileLoader::mtpFileLoader(
|
||||
autoLoading,
|
||||
cacheTag)
|
||||
, _location(location) {
|
||||
_queue = _downloader->queueForDc(dcId());
|
||||
}
|
||||
|
||||
mtpFileLoader::~mtpFileLoader() {
|
||||
cancelRequests();
|
||||
}
|
||||
|
||||
Data::FileOrigin mtpFileLoader::fileOrigin() const {
|
||||
@ -703,31 +673,26 @@ void mtpFileLoader::refreshFileReferenceFrom(
|
||||
makeRequest(offset);
|
||||
}
|
||||
|
||||
bool mtpFileLoader::loadPart() {
|
||||
if (_finished || _lastComplete || (!_sentRequests.empty() && !_size)) {
|
||||
return false;
|
||||
} else if (_size && _nextRequestOffset >= _size) {
|
||||
return false;
|
||||
}
|
||||
bool mtpFileLoader::readyToRequest() const {
|
||||
return !_finished
|
||||
&& !_lastComplete
|
||||
&& (_sentRequests.empty() || _size != 0)
|
||||
&& (!_size || _nextRequestOffset < _size);
|
||||
}
|
||||
|
||||
makeRequest(_nextRequestOffset);
|
||||
void mtpFileLoader::loadPart(int dcIndex) {
|
||||
Expects(readyToRequest());
|
||||
|
||||
makeRequest(_nextRequestOffset, dcIndex);
|
||||
_nextRequestOffset += Storage::kPartSize;
|
||||
return true;
|
||||
}
|
||||
|
||||
MTP::DcId mtpFileLoader::dcId() const {
|
||||
if (const auto storage = base::get_if<StorageFileLocation>(&_location)) {
|
||||
return storage->dcId();
|
||||
}
|
||||
return Global::WebFileDcId();
|
||||
}
|
||||
|
||||
mtpFileLoader::RequestData mtpFileLoader::prepareRequest(int offset) const {
|
||||
mtpFileLoader::RequestData mtpFileLoader::prepareRequest(
|
||||
int offset,
|
||||
int dcIndex) const {
|
||||
auto result = RequestData();
|
||||
result.dcId = _cdnDcId ? _cdnDcId : dcId();
|
||||
result.dcIndex = _size
|
||||
? _downloader->chooseDcIndexForRequest(result.dcId)
|
||||
: 0;
|
||||
result.dcIndex = dcIndex;
|
||||
result.offset = offset;
|
||||
return result;
|
||||
}
|
||||
@ -796,13 +761,17 @@ mtpRequestId mtpFileLoader::sendRequest(const RequestData &requestData) {
|
||||
});
|
||||
}
|
||||
|
||||
void mtpFileLoader::makeRequest(int offset) {
|
||||
void mtpFileLoader::makeRequest(int offset, int dcIndex) {
|
||||
Expects(!_finished);
|
||||
|
||||
auto requestData = prepareRequest(offset);
|
||||
auto requestData = prepareRequest(offset, dcIndex);
|
||||
placeSentRequest(sendRequest(requestData), requestData);
|
||||
}
|
||||
|
||||
void mtpFileLoader::makeRequest(int offset) {
|
||||
makeRequest(offset, _downloader->chooseDcIndexForRequest(dcId()));
|
||||
}
|
||||
|
||||
void mtpFileLoader::requestMoreCdnFileHashes() {
|
||||
Expects(!_finished);
|
||||
|
||||
@ -1011,7 +980,6 @@ void mtpFileLoader::placeSentRequest(
|
||||
requestData.dcId,
|
||||
requestData.dcIndex,
|
||||
Storage::kPartSize);
|
||||
++_queue->queriesCount;
|
||||
_sentRequests.emplace(requestId, requestData);
|
||||
}
|
||||
|
||||
@ -1024,8 +992,6 @@ int mtpFileLoader::finishSentRequestGetOffset(mtpRequestId requestId) {
|
||||
requestData.dcId,
|
||||
requestData.dcIndex,
|
||||
-Storage::kPartSize);
|
||||
|
||||
--_queue->queriesCount;
|
||||
_sentRequests.erase(it);
|
||||
|
||||
return requestData.offset;
|
||||
@ -1196,10 +1162,6 @@ std::optional<MediaKey> mtpFileLoader::fileLocationKey() const {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
mtpFileLoader::~mtpFileLoader() {
|
||||
cancelRequests();
|
||||
}
|
||||
|
||||
webFileLoader::webFileLoader(
|
||||
const QString &url,
|
||||
const QString &to,
|
||||
@ -1209,21 +1171,28 @@ webFileLoader::webFileLoader(
|
||||
: FileLoader(
|
||||
QString(),
|
||||
0,
|
||||
0,
|
||||
UnknownFileLocation,
|
||||
LoadToCacheAsWell,
|
||||
fromCloud,
|
||||
autoLoading,
|
||||
cacheTag)
|
||||
, _url(url) {
|
||||
_queue = _downloader->queueForWeb();
|
||||
}
|
||||
|
||||
bool webFileLoader::loadPart() {
|
||||
if (_finished
|
||||
|| _requestSent
|
||||
|| _webLoadManager == FinishedWebLoadManager) {
|
||||
return false;
|
||||
}
|
||||
webFileLoader::~webFileLoader() {
|
||||
markAsNotSent();
|
||||
}
|
||||
|
||||
bool webFileLoader::readyToRequest() const {
|
||||
return !_finished
|
||||
&& !_requestSent
|
||||
&& (_webLoadManager != FinishedWebLoadManager);
|
||||
}
|
||||
|
||||
void webFileLoader::loadPart(int dcIndex) {
|
||||
Expects(readyToRequest());
|
||||
|
||||
if (!_webLoadManager) {
|
||||
_webLoadMainManager = new WebLoadMainManager();
|
||||
|
||||
@ -1233,9 +1202,8 @@ bool webFileLoader::loadPart() {
|
||||
_webLoadThread->start();
|
||||
}
|
||||
|
||||
_requestSent = true;
|
||||
markAsSent();
|
||||
_webLoadManager->append(this, _url);
|
||||
return false;
|
||||
}
|
||||
|
||||
int webFileLoader::currentOffset() const {
|
||||
@ -1249,6 +1217,7 @@ void webFileLoader::loadProgress(qint64 already, qint64 size) {
|
||||
}
|
||||
|
||||
void webFileLoader::loadFinished(const QByteArray &data) {
|
||||
markAsNotSent();
|
||||
if (writeResultPart(0, bytes::make_span(data))) {
|
||||
if (finalizeResult()) {
|
||||
notifyAboutProgress();
|
||||
@ -1257,6 +1226,7 @@ void webFileLoader::loadFinished(const QByteArray &data) {
|
||||
}
|
||||
|
||||
void webFileLoader::loadError() {
|
||||
markAsNotSent();
|
||||
cancel(true);
|
||||
}
|
||||
|
||||
@ -1269,11 +1239,27 @@ std::optional<MediaKey> webFileLoader::fileLocationKey() const {
|
||||
}
|
||||
|
||||
void webFileLoader::cancelRequests() {
|
||||
if (!webLoadManager()) return;
|
||||
if (!webLoadManager()) {
|
||||
return;
|
||||
}
|
||||
webLoadManager()->stop(this);
|
||||
markAsNotSent();
|
||||
}
|
||||
|
||||
webFileLoader::~webFileLoader() {
|
||||
void webFileLoader::markAsSent() {
|
||||
if (_requestSent) {
|
||||
return;
|
||||
}
|
||||
_requestSent = true;
|
||||
_downloader->requestedAmountIncrement(0, 0, 1);
|
||||
}
|
||||
|
||||
void webFileLoader::markAsNotSent() {
|
||||
if (!_requestSent) {
|
||||
return;
|
||||
}
|
||||
_requestSent = false;
|
||||
_downloader->requestedAmountIncrement(0, 0, -1);
|
||||
}
|
||||
|
||||
class webFileLoaderPrivate {
|
||||
@ -1372,6 +1358,10 @@ WebLoadManager::WebLoadManager(QThread *thread) {
|
||||
#endif // OS_MAC_OLD
|
||||
}
|
||||
|
||||
WebLoadManager::~WebLoadManager() {
|
||||
clear();
|
||||
}
|
||||
|
||||
void WebLoadManager::append(webFileLoader *loader, const QString &url) {
|
||||
loader->_private = new webFileLoaderPrivate(loader, url);
|
||||
|
||||
@ -1606,10 +1596,6 @@ void WebLoadManager::clear() {
|
||||
_replies.clear();
|
||||
}
|
||||
|
||||
WebLoadManager::~WebLoadManager() {
|
||||
clear();
|
||||
}
|
||||
|
||||
void WebLoadMainManager::progress(webFileLoader *loader, qint64 already, qint64 size) {
|
||||
if (webLoadManager() && webLoadManager()->carries(loader)) {
|
||||
loader->loadProgress(already, size);
|
||||
|
||||
@ -16,6 +16,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
#include <QtNetwork/QNetworkReply>
|
||||
|
||||
class ApiWrap;
|
||||
class FileLoader;
|
||||
|
||||
namespace Main {
|
||||
class Session;
|
||||
@ -35,48 +36,51 @@ constexpr auto kMaxWallPaperInMemory = kMaxFileInMemory;
|
||||
constexpr auto kMaxAnimationInMemory = kMaxFileInMemory; // 10 MB gif and mp4 animations held in memory while playing
|
||||
constexpr auto kMaxWallPaperDimension = 4096; // 4096x4096 is max area.
|
||||
|
||||
class Downloader final {
|
||||
class Downloader final : public base::has_weak_ptr {
|
||||
public:
|
||||
struct Queue {
|
||||
Queue(int queriesLimit) : queriesLimit(queriesLimit) {
|
||||
}
|
||||
int queriesCount = 0;
|
||||
int queriesLimit = 0;
|
||||
FileLoader *start = nullptr;
|
||||
FileLoader *end = nullptr;
|
||||
};
|
||||
|
||||
explicit Downloader(not_null<ApiWrap*> api);
|
||||
~Downloader();
|
||||
|
||||
ApiWrap &api() const {
|
||||
[[nodiscard]] ApiWrap &api() const {
|
||||
return *_api;
|
||||
}
|
||||
|
||||
int currentPriority() const {
|
||||
return _priority;
|
||||
}
|
||||
void clearPriorities();
|
||||
void enqueue(not_null<FileLoader*> loader);
|
||||
void remove(not_null<FileLoader*> loader);
|
||||
|
||||
base::Observable<void> &taskFinished() {
|
||||
[[nodiscard]] base::Observable<void> &taskFinished() {
|
||||
return _taskFinishedObservable;
|
||||
}
|
||||
|
||||
// dcId == 0 is for web requests.
|
||||
void requestedAmountIncrement(MTP::DcId dcId, int index, int amount);
|
||||
int chooseDcIndexForRequest(MTP::DcId dcId) const;
|
||||
|
||||
not_null<Queue*> queueForDc(MTP::DcId dcId);
|
||||
not_null<Queue*> queueForWeb();
|
||||
[[nodiscard]] int chooseDcIndexForRequest(MTP::DcId dcId);
|
||||
|
||||
private:
|
||||
class Queue final {
|
||||
public:
|
||||
void enqueue(not_null<FileLoader*> loader);
|
||||
void remove(not_null<FileLoader*> loader);
|
||||
void resetGeneration();
|
||||
[[nodiscard]] FileLoader *nextLoader() const;
|
||||
|
||||
private:
|
||||
std::vector<not_null<FileLoader*>> _loaders;
|
||||
std::vector<not_null<FileLoader*>> _previousGeneration;
|
||||
|
||||
};
|
||||
|
||||
void checkSendNext();
|
||||
|
||||
void killDownloadSessionsStart(MTP::DcId dcId);
|
||||
void killDownloadSessionsStop(MTP::DcId dcId);
|
||||
void killDownloadSessions();
|
||||
|
||||
not_null<ApiWrap*> _api;
|
||||
void resetGeneration();
|
||||
|
||||
const not_null<ApiWrap*> _api;
|
||||
|
||||
base::Observable<void> _taskFinishedObservable;
|
||||
int _priority = 1;
|
||||
|
||||
using RequestedInDc = std::array<int64, MTP::kDownloadSessionsCount>;
|
||||
base::flat_map<MTP::DcId, RequestedInDc> _requestedBytesAmount;
|
||||
@ -84,8 +88,9 @@ private:
|
||||
base::flat_map<MTP::DcId, crl::time> _killDownloadSessionTimes;
|
||||
base::Timer _killDownloadSessionsTimer;
|
||||
|
||||
std::map<MTP::DcId, Queue> _queuesForDc;
|
||||
Queue _queueForWeb;
|
||||
base::flat_map<MTP::DcId, Queue> _mtprotoLoaders;
|
||||
Queue _webLoaders;
|
||||
bool _resettingGeneration = false;
|
||||
|
||||
};
|
||||
|
||||
@ -100,15 +105,13 @@ struct StorageImageSaved {
|
||||
|
||||
};
|
||||
|
||||
class mtpFileLoader;
|
||||
class webFileLoader;
|
||||
|
||||
class FileLoader : public QObject {
|
||||
Q_OBJECT
|
||||
|
||||
public:
|
||||
FileLoader(
|
||||
const QString &toFile,
|
||||
MTP::DcId dcId,
|
||||
int32 size,
|
||||
LocationType locationType,
|
||||
LoadToCacheSetting toCache,
|
||||
@ -147,12 +150,6 @@ public:
|
||||
void start();
|
||||
void cancel();
|
||||
|
||||
bool loading() const {
|
||||
return _inQueue;
|
||||
}
|
||||
bool started() const {
|
||||
return _inQueue;
|
||||
}
|
||||
bool loadingLocal() const {
|
||||
return (_localStatus == LocalStatus::Loading);
|
||||
}
|
||||
@ -174,7 +171,7 @@ signals:
|
||||
void failed(FileLoader *loader, bool started);
|
||||
|
||||
protected:
|
||||
using Queue = Storage::Downloader::Queue;
|
||||
friend class Storage::Downloader;
|
||||
|
||||
enum class LocalStatus {
|
||||
NotTried,
|
||||
@ -183,6 +180,10 @@ protected:
|
||||
Loaded,
|
||||
};
|
||||
|
||||
[[nodiscard]] MTP::DcId dcId() const {
|
||||
return _dcId;
|
||||
}
|
||||
|
||||
void readImage(const QSize &shrinkBox) const;
|
||||
|
||||
bool tryLoadLocal();
|
||||
@ -191,27 +192,21 @@ protected:
|
||||
virtual std::optional<MediaKey> fileLocationKey() const = 0;
|
||||
virtual void cancelRequests() = 0;
|
||||
|
||||
void startLoading();
|
||||
void removeFromQueue();
|
||||
void cancel(bool failed);
|
||||
|
||||
void notifyAboutProgress();
|
||||
static void LoadNextFromQueue(not_null<Queue*> queue);
|
||||
virtual bool loadPart() = 0;
|
||||
[[nodiscard]] virtual bool readyToRequest() const = 0;
|
||||
virtual void loadPart(int dcIndex) = 0;
|
||||
|
||||
bool writeResultPart(int offset, bytes::const_span buffer);
|
||||
bool finalizeResult();
|
||||
[[nodiscard]] QByteArray readLoadedPartBack(int offset, int size);
|
||||
|
||||
not_null<Storage::Downloader*> _downloader;
|
||||
FileLoader *_prev = nullptr;
|
||||
FileLoader *_next = nullptr;
|
||||
int _priority = 0;
|
||||
Queue *_queue = nullptr;
|
||||
const MTP::DcId _dcId = 0;
|
||||
const not_null<Storage::Downloader*> _downloader;
|
||||
|
||||
bool _autoLoading = false;
|
||||
uint8 _cacheTag = 0;
|
||||
bool _inQueue = false;
|
||||
bool _finished = false;
|
||||
bool _cancelled = false;
|
||||
mutable LocalStatus _localStatus = LocalStatus::NotTried;
|
||||
@ -237,7 +232,7 @@ protected:
|
||||
|
||||
class StorageImageLocation;
|
||||
class WebFileLocation;
|
||||
class mtpFileLoader : public FileLoader, public RPCSender {
|
||||
class mtpFileLoader final : public FileLoader, public RPCSender {
|
||||
public:
|
||||
mtpFileLoader(
|
||||
const StorageFileLocation &location,
|
||||
@ -277,6 +272,8 @@ public:
|
||||
~mtpFileLoader();
|
||||
|
||||
private:
|
||||
friend class Downloader;
|
||||
|
||||
struct RequestData {
|
||||
MTP::DcId dcId = 0;
|
||||
int dcIndex = 0;
|
||||
@ -292,11 +289,12 @@ private:
|
||||
std::optional<MediaKey> fileLocationKey() const override;
|
||||
void cancelRequests() override;
|
||||
|
||||
MTP::DcId dcId() const;
|
||||
RequestData prepareRequest(int offset) const;
|
||||
[[nodiscard]] RequestData prepareRequest(int offset, int dcIndex) const;
|
||||
void makeRequest(int offset, int dcIndex);
|
||||
void makeRequest(int offset);
|
||||
|
||||
bool loadPart() override;
|
||||
bool readyToRequest() const override;
|
||||
void loadPart(int dcIndex) override;
|
||||
void normalPartLoaded(const MTPupload_File &result, mtpRequestId requestId);
|
||||
void webPartLoaded(const MTPupload_WebFile &result, mtpRequestId requestId);
|
||||
void cdnPartLoaded(const MTPupload_CdnFile &result, mtpRequestId requestId);
|
||||
@ -334,7 +332,6 @@ private:
|
||||
StorageFileLocation,
|
||||
WebFileLocation,
|
||||
GeoPointLocation> _location;
|
||||
|
||||
Data::FileOrigin _origin;
|
||||
|
||||
MTP::DcId _cdnDcId = 0;
|
||||
@ -349,7 +346,7 @@ private:
|
||||
|
||||
class webFileLoaderPrivate;
|
||||
|
||||
class webFileLoader : public FileLoader {
|
||||
class webFileLoader final : public FileLoader {
|
||||
public:
|
||||
webFileLoader(
|
||||
const QString &url,
|
||||
@ -370,11 +367,15 @@ public:
|
||||
|
||||
~webFileLoader();
|
||||
|
||||
protected:
|
||||
private:
|
||||
void cancelRequests() override;
|
||||
Storage::Cache::Key cacheKey() const override;
|
||||
std::optional<MediaKey> fileLocationKey() const override;
|
||||
bool loadPart() override;
|
||||
bool readyToRequest() const override;
|
||||
void loadPart(int dcIndex) override;
|
||||
|
||||
void markAsSent();
|
||||
void markAsNotSent();
|
||||
|
||||
QString _url;
|
||||
|
||||
|
||||
@ -37,6 +37,7 @@ StreamedFileDownloader::StreamedFileDownloader(
|
||||
uint8 cacheTag)
|
||||
: FileLoader(
|
||||
toFile,
|
||||
dcId,
|
||||
size,
|
||||
locationType,
|
||||
toCache,
|
||||
@ -59,8 +60,6 @@ StreamedFileDownloader::StreamedFileDownloader(
|
||||
savePart(std::move(part));
|
||||
}
|
||||
}, _lifetime);
|
||||
|
||||
_queue = _downloader->queueForDc(dcId);
|
||||
}
|
||||
|
||||
StreamedFileDownloader::~StreamedFileDownloader() {
|
||||
@ -98,21 +97,25 @@ std::optional<MediaKey> StreamedFileDownloader::fileLocationKey() const {
|
||||
}
|
||||
|
||||
void StreamedFileDownloader::cancelRequests() {
|
||||
//_partsRequested == std::count(
|
||||
// begin(_partIsSaved),
|
||||
// begin(_partIsSaved) + _nextPartIndex,
|
||||
// false);
|
||||
_queue->queriesCount -= _partsRequested;
|
||||
_partsRequested = 0;
|
||||
_nextPartIndex = 0;
|
||||
|
||||
_reader->cancelForDownloader(this);
|
||||
}
|
||||
|
||||
bool StreamedFileDownloader::loadPart() {
|
||||
bool StreamedFileDownloader::readyToRequest() const {
|
||||
if (_finished || _nextPartIndex >= _partsCount) {
|
||||
return false;
|
||||
}
|
||||
_nextPartIndex = std::find(
|
||||
begin(_partIsSaved) + _nextPartIndex,
|
||||
end(_partIsSaved),
|
||||
false
|
||||
) - begin(_partIsSaved);
|
||||
return (_nextPartIndex < _partsCount);
|
||||
}
|
||||
|
||||
void StreamedFileDownloader::loadPart(int dcIndex) {
|
||||
const auto index = std::find(
|
||||
begin(_partIsSaved) + _nextPartIndex,
|
||||
end(_partIsSaved),
|
||||
@ -120,15 +123,11 @@ bool StreamedFileDownloader::loadPart() {
|
||||
) - begin(_partIsSaved);
|
||||
if (index == _partsCount) {
|
||||
_nextPartIndex = _partsCount;
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
_nextPartIndex = index + 1;
|
||||
_reader->loadForDownloader(this, index * kPartSize);
|
||||
|
||||
++_partsRequested;
|
||||
++_queue->queriesCount;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void StreamedFileDownloader::savePart(const LoadedPart &part) {
|
||||
@ -150,7 +149,6 @@ void StreamedFileDownloader::savePart(const LoadedPart &part) {
|
||||
|
||||
if (index < _nextPartIndex) {
|
||||
--_partsRequested;
|
||||
--_queue->queriesCount;
|
||||
}
|
||||
if (!writeResultPart(offset, bytes::make_span(part.bytes))) {
|
||||
return;
|
||||
|
||||
@ -49,7 +49,8 @@ private:
|
||||
Cache::Key cacheKey() const override;
|
||||
std::optional<MediaKey> fileLocationKey() const override;
|
||||
void cancelRequests() override;
|
||||
bool loadPart() override;
|
||||
bool readyToRequest() const override;
|
||||
void loadPart(int dcIndex) override;
|
||||
|
||||
void savePart(const Media::Streaming::LoadedPart &part);
|
||||
|
||||
@ -60,7 +61,7 @@ private:
|
||||
std::shared_ptr<Media::Streaming::Reader> _reader;
|
||||
|
||||
std::vector<bool> _partIsSaved; // vector<bool> :D
|
||||
int _nextPartIndex = 0;
|
||||
mutable int _nextPartIndex = 0;
|
||||
int _partsCount = 0;
|
||||
int _partsRequested = 0;
|
||||
int _partsSaved = 0;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user