/* This file is part of Telegram Desktop, the official desktop application for the Telegram messaging service. For license and copyright information please follow this link: https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL */ #include "data/data_download_manager.h" #include "data/data_session.h" #include "data/data_photo.h" #include "data/data_document.h" #include "data/data_changes.h" #include "data/data_user.h" #include "data/data_channel.h" #include "base/unixtime.h" #include "base/random.h" #include "main/main_session.h" #include "main/main_account.h" #include "storage/storage_account.h" #include "history/history.h" #include "history/history_item.h" #include "history/history_message.h" #include "core/application.h" #include "core/mime_type.h" #include "ui/controls/download_bar.h" #include "ui/text/format_song_document_name.h" #include "storage/serialize_common.h" #include "apiwrap.h" namespace Data { namespace { constexpr auto kClearLoadingTimeout = 5 * crl::time(1000); constexpr auto kMaxFileSize = 2000 * 1024 * 1024; constexpr auto kMaxResolvePerAttempt = 100; constexpr auto ByItem = [](const auto &entry) { if constexpr (std::is_same_v) { return entry.object.item; } else { const auto resolved = entry.object.get(); return resolved ? resolved->item.get() : nullptr; } }; [[nodiscard]] uint64 PeerAccessHash(not_null peer) { if (const auto user = peer->asUser()) { return user->accessHash(); } else if (const auto channel = peer->asChannel()) { return channel->access; } return 0; } } // namespace DownloadManager::DownloadManager() : _clearLoadingTimer([=] { clearLoading(); }) { } DownloadManager::~DownloadManager() = default; void DownloadManager::trackSession(not_null session) { auto &data = _sessions.emplace(session, SessionData()).first->second; data.downloaded = deserialize(session); data.resolveNeeded = data.downloaded.size(); session->data().itemRepaintRequest( ) | rpl::filter([=](not_null item) { return _loading.contains(item); }) | rpl::start_with_next([=](not_null item) { check(item); }, data.lifetime); session->data().itemLayoutChanged( ) | rpl::filter([=](not_null item) { return _loading.contains(item); }) | rpl::start_with_next([=](not_null item) { check(item); }, data.lifetime); session->data().itemViewRefreshRequest( ) | rpl::start_with_next([=](not_null item) { changed(item); }, data.lifetime); session->changes().messageUpdates( MessageUpdate::Flag::Destroyed ) | rpl::start_with_next([=](const MessageUpdate &update) { removed(update.item); }, data.lifetime); session->account().sessionChanges( ) | rpl::filter( rpl::mappers::_1 != session ) | rpl::take(1) | rpl::start_with_next([=] { untrack(session); }, data.lifetime); } int64 DownloadManager::computeNextStartDate() { const auto now = base::unixtime::now(); if (_lastStartedBase != now) { _lastStartedBase = now; _lastStartedAdded = 0; } else { ++_lastStartedAdded; } return int64(_lastStartedBase) * 1000 + _lastStartedAdded; } void DownloadManager::addLoading(DownloadObject object) { Expects(object.item != nullptr); Expects(object.document != nullptr); const auto item = object.item; auto &data = sessionData(item); const auto already = ranges::find(data.downloading, item, ByItem); if (already != end(data.downloading)) { const auto document = already->object.document; const auto photo = already->object.photo; if (document == object.document && photo == object.photo) { check(item); return; } remove(data, already); } const auto size = object.document->size; const auto path = object.document->loadingFilePath(); if (path.isEmpty()) { return; } data.downloading.push_back({ .object = object, .started = computeNextStartDate(), .path = path, .total = size, }); _loading.emplace(item); _loadingProgress = DownloadProgress{ .ready = _loadingProgress.current().ready, .total = _loadingProgress.current().total + size, }; _loadingListChanges.fire({}); _clearLoadingTimer.cancel(); check(item); } void DownloadManager::check(not_null item) { auto &data = sessionData(item); const auto i = ranges::find(data.downloading, item, ByItem); Assert(i != end(data.downloading)); auto &entry = *i; const auto media = item->media(); const auto photo = media ? media->photo() : nullptr; const auto document = media ? media->document() : nullptr; if (entry.object.photo != photo || entry.object.document != document) { cancel(data, i); return; } // Load with progress only documents for now. Assert(document != nullptr); const auto path = document->filepath(true); if (!path.isEmpty()) { if (_loading.contains(item)) { addLoaded(entry.object, path, entry.started); } } else if (!document->loading()) { remove(data, i); } else { const auto totalChange = document->size - entry.total; const auto readyChange = document->loadOffset() - entry.ready; if (!readyChange && !totalChange) { return; } entry.ready += readyChange; entry.total += totalChange; _loadingProgress = DownloadProgress{ .ready = _loadingProgress.current().ready + readyChange, .total = _loadingProgress.current().total + totalChange, }; } } void DownloadManager::addLoaded( DownloadObject object, const QString &path, DownloadDate started) { Expects(object.item != nullptr); Expects(object.document || object.photo); const auto size = QFileInfo(path).size(); if (size <= 0 || size > kMaxFileSize) { return; } const auto item = object.item; auto &data = sessionData(item); const auto id = object.document ? DownloadId{ object.document->id, DownloadType::Document } : DownloadId{ object.photo->id, DownloadType::Photo }; data.downloaded.push_back({ .download = id, .started = started, .path = path, .size = int32(size), .itemId = item->fullId(), .peerAccessHash = PeerAccessHash(item->history()->peer), .object = std::make_unique(object), }); _loaded.emplace(item); _loadedAdded.fire(&data.downloaded.back()); writePostponed(&item->history()->session()); const auto i = ranges::find(data.downloading, item, ByItem); if (i != end(data.downloading)) { auto &entry = *i; const auto j = _loading.find(entry.object.item); if (j == end(_loading)) { return; } const auto document = entry.object.document; const auto totalChange = document->size - entry.total; const auto readyChange = document->size - entry.ready; entry.ready += readyChange; entry.total += totalChange; entry.done = true; _loading.erase(j); _loadingDone.emplace(entry.object.item); _loadingProgress = DownloadProgress{ .ready = _loadingProgress.current().ready + readyChange, .total = _loadingProgress.current().total + totalChange, }; if (_loading.empty()) { _clearLoadingTimer.callOnce(kClearLoadingTimeout); } } } void DownloadManager::clearIfFinished() { if (_clearLoadingTimer.isActive()) { _clearLoadingTimer.cancel(); clearLoading(); } } void DownloadManager::deleteFiles(const std::vector &ids) { struct DocumentDescriptor { uint64 sessionUniqueId = 0; DocumentId documentId = 0; FullMsgId itemId; }; auto sessions = base::flat_set>(); auto files = base::flat_map(); for (const auto &id : ids) { if (const auto item = MessageByGlobalId(id)) { const auto session = &item->history()->session(); const auto i = _sessions.find(session); if (i == end(_sessions)) { continue; } auto &data = i->second; const auto j = ranges::find( data.downloading, not_null{ item }, ByItem); if (j != end(data.downloading)) { cancel(data, j); } const auto k = ranges::find(data.downloaded, item, ByItem); if (k != end(data.downloaded)) { const auto document = k->object->document; files.emplace(k->path, DocumentDescriptor{ .sessionUniqueId = id.sessionUniqueId, .documentId = document ? document->id : DocumentId(), .itemId = id.itemId, }); _loaded.remove(item); _generated.remove(item); if (document) { _generatedDocuments.remove(document); } data.downloaded.erase(k); _loadedRemoved.fire_copy(item); sessions.emplace(session); } } } for (const auto &session : sessions) { writePostponed(session); } crl::async([files = std::move(files)] { for (const auto &file : files) { QFile(file.first).remove(); crl::on_main([descriptor = file.second] { if (const auto session = SessionByUniqueId( descriptor.sessionUniqueId)) { if (const auto id = descriptor.documentId) { [[maybe_unused]] const auto location = session->data().document(id)->location(true); } const auto itemId = descriptor.itemId; if (const auto item = session->data().message(itemId)) { session->data().requestItemRepaint(item); } } }); } }); } auto DownloadManager::loadingList() const -> ranges::any_view { return ranges::views::all( _sessions ) | ranges::views::transform([=](const auto &pair) { return ranges::views::all( pair.second.downloading ) | ranges::views::transform([](const DownloadingId &id) { return &id; }); }) | ranges::views::join; } DownloadProgress DownloadManager::loadingProgress() const { return _loadingProgress.current(); } rpl::producer<> DownloadManager::loadingListChanges() const { return _loadingListChanges.events(); } auto DownloadManager::loadingProgressValue() const -> rpl::producer { return _loadingProgress.value(); } void DownloadManager::clearLoading() { Expects(_loading.empty()); for (auto &[session, data] : _sessions) { while (!data.downloading.empty()) { remove(data, data.downloading.end() - 1); } } } auto DownloadManager::loadedList() -> ranges::any_view { for (auto &[session, data] : _sessions) { resolve(session, data); } return ranges::views::all( _sessions ) | ranges::views::transform([=](const auto &pair) { return ranges::views::all( pair.second.downloaded ) | ranges::views::filter([](const DownloadedId &id) { return (id.object != nullptr); }) | ranges::views::transform([](const DownloadedId &id) { return &id; }); }) | ranges::views::join; } void DownloadManager::resolve( not_null session, SessionData &data) { if (data.resolveSentTotal >= data.resolveNeeded || data.resolveSentTotal >= kMaxResolvePerAttempt) { return; } struct Prepared { uint64 peerAccessHash = 0; QVector ids; }; auto &owner = session->data(); auto prepared = base::flat_map(); auto last = begin(data.downloaded); auto from = last + (data.resolveNeeded - data.resolveSentTotal); for (auto i = from; i != last;) { auto &id = *--i; const auto msgId = id.itemId.msg; const auto info = QFileInfo(id.path); if (!info.exists() || info.size() != id.size) { // Mark as deleted. id.path = QString(); } else if (!owner.message(id.itemId) && IsServerMsgId(msgId)) { const auto groupByPeer = peerIsChannel(id.itemId.peer) ? id.itemId.peer : session->userPeerId(); auto &perPeer = prepared[groupByPeer]; if (peerIsChannel(id.itemId.peer) && !perPeer.peerAccessHash) { perPeer.peerAccessHash = id.peerAccessHash; } perPeer.ids.push_back(MTP_inputMessageID(MTP_int(msgId.bare))); } if (++data.resolveSentTotal >= kMaxResolvePerAttempt) { break; } } const auto check = [=] { auto &data = sessionData(session); if (!data.resolveSentRequests) { resolveRequestsFinished(session, data); } }; const auto requestFinished = [=] { --sessionData(session).resolveSentRequests; check(); }; for (auto &[peer, perPeer] : prepared) { if (const auto channelId = peerToChannel(peer)) { session->api().request(MTPchannels_GetMessages( MTP_inputChannel( MTP_long(channelId.bare), MTP_long(perPeer.peerAccessHash)), MTP_vector(perPeer.ids) )).done([=](const MTPmessages_Messages &result) { session->data().processExistingMessages( session->data().channelLoaded(channelId), result); requestFinished(); }).fail(requestFinished).send(); } else { session->api().request(MTPmessages_GetMessages( MTP_vector(perPeer.ids) )).done([=](const MTPmessages_Messages &result) { session->data().processExistingMessages(nullptr, result); requestFinished(); }).fail(requestFinished).send(); } } data.resolveSentRequests += prepared.size(); check(); } void DownloadManager::resolveRequestsFinished( not_null session, SessionData &data) { auto &owner = session->data(); for (; data.resolveSentTotal > 0; --data.resolveSentTotal) { const auto i = begin(data.downloaded) + (--data.resolveNeeded); if (i->path.isEmpty()) { data.downloaded.erase(i); continue; } const auto item = owner.message(i->itemId); const auto media = item ? item->media() : nullptr; const auto document = media ? media->document() : nullptr; const auto photo = media ? media->photo() : nullptr; if (i->download.type == DownloadType::Document && (!document || document->id != i->download.objectId)) { generateEntry(session, *i); } else if (i->download.type == DownloadType::Photo && (!photo || photo->id != i->download.objectId)) { generateEntry(session, *i); } else { i->object = std::make_unique(DownloadObject{ .item = item, .document = document, .photo = photo, }); _loaded.emplace(item); } _loadedAdded.fire(&*i); } crl::on_main(session, [=] { resolve(session, sessionData(session)); }); } void DownloadManager::generateEntry( not_null session, DownloadedId &id) { Expects(!id.object); const auto info = QFileInfo(id.path); const auto document = session->data().document( base::RandomValue(), 0, // accessHash QByteArray(), // fileReference TimeId(id.started / 1000), QVector( 1, MTP_documentAttributeFilename( MTP_string(info.fileName()))), Core::MimeTypeForFile(info).name(), InlineImageLocation(), // inlineThumbnail ImageWithLocation(), // thumbnail ImageWithLocation(), // videoThumbnail 0, // dc id.size); document->setLocation(Core::FileLocation(info)); _generatedDocuments.emplace(document); id.object = std::make_unique(DownloadObject{ .item = generateFakeItem(document), .document = document, }); _loaded.emplace(id.object->item); } auto DownloadManager::loadedAdded() const -> rpl::producer> { return _loadedAdded.events(); } auto DownloadManager::loadedRemoved() const -> rpl::producer> { return _loadedRemoved.events(); } void DownloadManager::remove( SessionData &data, std::vector::iterator i) { const auto now = DownloadProgress{ .ready = _loadingProgress.current().ready - i->ready, .total = _loadingProgress.current().total - i->total, }; _loading.remove(i->object.item); _loadingDone.remove(i->object.item); data.downloading.erase(i); _loadingListChanges.fire({}); _loadingProgress = now; if (_loading.empty() && !_loadingDone.empty()) { _clearLoadingTimer.callOnce(kClearLoadingTimeout); } } void DownloadManager::cancel( SessionData &data, std::vector::iterator i) { const auto object = i->object; remove(data, i); if (const auto document = object.document) { document->cancel(); } else if (const auto photo = object.photo) { photo->cancel(); } } void DownloadManager::changed(not_null item) { if (_loaded.contains(item)) { auto &data = sessionData(item); const auto i = ranges::find(data.downloaded, item.get(), ByItem); Assert(i != end(data.downloaded)); const auto media = item->media(); const auto photo = media ? media->photo() : nullptr; const auto document = media ? media->document() : nullptr; if (i->object->photo != photo || i->object->document != document) { detach(*i); } } if (_loading.contains(item) || _loadingDone.contains(item)) { check(item); } } void DownloadManager::removed(not_null item) { if (_loaded.contains(item)) { auto &data = sessionData(item); const auto i = ranges::find(data.downloaded, item.get(), ByItem); Assert(i != end(data.downloaded)); detach(*i); } if (_loading.contains(item) || _loadingDone.contains(item)) { auto &data = sessionData(item); const auto i = ranges::find(data.downloading, item, ByItem); Assert(i != end(data.downloading)); // We don't want to download files without messages. // For example, there is no way to refresh a file reference for them. //entry.object.item = nullptr; cancel(data, i); } } not_null DownloadManager::regenerateItem( const DownloadObject &previous) { return generateItem(previous.item, previous.document, previous.photo); } not_null DownloadManager::generateFakeItem( not_null document) { return generateItem(nullptr, document, nullptr); } not_null DownloadManager::generateItem( HistoryItem *previousItem, DocumentData *document, PhotoData *photo) { Expects(document || photo); const auto session = document ? &document->session() : &photo->session(); const auto fromId = previousItem ? previousItem->from()->id : session->userPeerId(); const auto history = previousItem ? previousItem->history() : session->data().history(session->user()); const auto flags = MessageFlag::FakeHistoryItem; const auto replyTo = MsgId(); const auto viaBotId = UserId(); const auto date = base::unixtime::now(); const auto postAuthor = QString(); const auto caption = TextWithEntities(); const auto make = [&](const auto media) { return history->makeMessage( history->nextNonHistoryEntryId(), flags, replyTo, viaBotId, date, fromId, QString(), media, caption, HistoryMessageMarkupData()); }; const auto result = document ? make(document) : make(photo); _generated.emplace(result); return result; } void DownloadManager::detach(DownloadedId &id) { Expects(id.object != nullptr); Expects(_loaded.contains(id.object->item)); Expects(!_generated.contains(id.object->item)); // Maybe generate new document? const auto was = id.object->item; const auto now = regenerateItem(*id.object); _loaded.remove(was); _loaded.emplace(now); id.object->item = now; _loadedRemoved.fire_copy(was); _loadedAdded.fire_copy(&id); } DownloadManager::SessionData &DownloadManager::sessionData( not_null session) { const auto i = _sessions.find(session); Assert(i != end(_sessions)); return i->second; } const DownloadManager::SessionData &DownloadManager::sessionData( not_null session) const { const auto i = _sessions.find(session); Assert(i != end(_sessions)); return i->second; } DownloadManager::SessionData &DownloadManager::sessionData( not_null item) { return sessionData(&item->history()->session()); } void DownloadManager::writePostponed(not_null session) { session->account().local().updateDownloads(serializator(session)); } Fn()> DownloadManager::serializator( not_null session) const { return [this, weak = base::make_weak(session.get())]() -> std::optional { const auto strong = weak.get(); if (!strong) { return std::nullopt; } else if (!_sessions.contains(strong)) { return QByteArray(); } auto result = QByteArray(); const auto &data = sessionData(strong); const auto count = data.downloaded.size(); const auto constant = sizeof(quint64) // download.objectId + sizeof(qint32) // download.type + sizeof(qint64) // started + sizeof(qint32) // size + sizeof(quint64) // itemId.peer + sizeof(qint64) // itemId.msg + sizeof(quint64); // peerAccessHash auto size = sizeof(qint32) // count + count * constant; for (const auto &id : data.downloaded) { size += Serialize::stringSize(id.path); } result.reserve(size); auto stream = QDataStream(&result, QIODevice::WriteOnly); stream.setVersion(QDataStream::Qt_5_1); stream << qint32(count); for (const auto &id : data.downloaded) { stream << quint64(id.download.objectId) << qint32(id.download.type) << qint64(id.started) << qint32(id.size) << quint64(id.itemId.peer.value) << qint64(id.itemId.msg.bare) << quint64(id.peerAccessHash) << id.path; } stream.device()->close(); return result; }; } std::vector DownloadManager::deserialize( not_null session) const { const auto serialized = session->account().local().downloadsSerialized(); if (serialized.isEmpty()) { return {}; } QDataStream stream(serialized); stream.setVersion(QDataStream::Qt_5_1); auto count = qint32(); stream >> count; if (stream.status() != QDataStream::Ok || count <= 0 || count > 99'999) { return {}; } auto result = std::vector(); result.reserve(count); for (auto i = 0; i != count; ++i) { auto downloadObjectId = quint64(); auto uncheckedDownloadType = qint32(); auto started = qint64(); auto size = qint32(); auto itemIdPeer = quint64(); auto itemIdMsg = qint64(); auto peerAccessHash = quint64(); auto path = QString(); stream >> downloadObjectId >> uncheckedDownloadType >> started >> size >> itemIdPeer >> itemIdMsg >> peerAccessHash >> path; const auto downloadType = DownloadType(uncheckedDownloadType); if (stream.status() != QDataStream::Ok || path.isEmpty() || size <= 0 || size > kMaxFileSize || (downloadType != DownloadType::Document && downloadType != DownloadType::Photo)) { return {}; } result.push_back({ .download = { .objectId = downloadObjectId, .type = downloadType, }, .started = started, .path = path, .size = size, .itemId = { PeerId(itemIdPeer), MsgId(itemIdMsg) }, .peerAccessHash = peerAccessHash, }); } return result; } void DownloadManager::untrack(not_null session) { const auto i = _sessions.find(session); Assert(i != end(_sessions)); for (const auto &entry : i->second.downloaded) { if (const auto resolved = entry.object.get()) { const auto item = resolved->item; _loaded.remove(item); _generated.remove(item); if (const auto document = resolved->document) { _generatedDocuments.remove(document); } } } while (!i->second.downloading.empty()) { remove(i->second, i->second.downloading.end() - 1); } _sessions.erase(i); } rpl::producer MakeDownloadBarProgress() { return Core::App().downloadManager().loadingProgressValue( ) | rpl::map([=](const DownloadProgress &progress) { return Ui::DownloadBarProgress{ .ready = progress.ready, .total = progress.total, }; }); } rpl::producer MakeDownloadBarContent() { auto &manager = Core::App().downloadManager(); return rpl::single( rpl::empty_value() ) | rpl::then( manager.loadingListChanges() | rpl::to_empty ) | rpl::map([=, &manager] { auto result = Ui::DownloadBarContent(); for (const auto id : manager.loadingList()) { if (result.singleName.text.isEmpty()) { const auto document = id->object.document; result.singleName = Ui::Text::FormatDownloadsName(document); result.singleThumbnail = QImage(); } ++result.count; if (id->done) { ++result.done; } } return result; }); } } // namespace Data