/* This file is part of Telegram Desktop, the official desktop application for the Telegram messaging service. For license and copyright information please follow this link: https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL */ #include "export/export_api_wrap.h" #include "export/export_settings.h" #include "export/data/export_data_types.h" #include "export/output/export_output_file.h" #include "mtproto/rpc_sender.h" #include namespace Export { namespace { constexpr auto kUserpicsSliceLimit = 100; constexpr auto kFileChunkSize = 128 * 1024; constexpr auto kFileRequestsCount = 2; constexpr auto kFileNextRequestDelay = TimeMs(20); constexpr auto kChatsSliceLimit = 100; constexpr auto kMessagesSliceLimit = 100; bool WillLoadFile(const Data::File &file) { return file.relativePath.isEmpty() && (!file.content.isEmpty() || file.location.dcId != 0); } } // namespace struct ApiWrap::UserpicsProcess { FnMut start; Fn handleSlice; FnMut finish; base::optional slice; bool lastSlice = false; int fileIndex = -1; }; struct ApiWrap::FileProcess { FileProcess(const QString &path); Output::File file; QString relativePath; FnMut done; Data::FileLocation location; int offset = 0; int size = 0; struct Request { int offset = 0; QByteArray bytes; }; std::deque requests; }; struct ApiWrap::DialogsProcess { Data::DialogsInfo info; FnMut start; Fn startOne; Fn sliceOne; Fn finishOne; FnMut finish; Data::TimeId offsetDate = 0; int32 offsetId = 0; MTPInputPeer offsetPeer = MTP_inputPeerEmpty(); struct Single; std::unique_ptr single; int singleIndex = -1; }; struct ApiWrap::DialogsProcess::Single { Single(const Data::DialogInfo &info); Data::DialogInfo info; int32 offsetId = 1; base::optional slice; bool lastSlice = false; int fileIndex = -1; }; ApiWrap::FileProcess::FileProcess(const QString &path) : file(path) { } ApiWrap::DialogsProcess::Single::Single(const Data::DialogInfo &info) : info(info) { } template auto ApiWrap::mainRequest(Request &&request) { return std::move(_mtp.request( std::move(request) ).fail([=](RPCError &&result) { error(std::move(result)); }).toDC(MTP::ShiftDcId(0, MTP::kExportDcShift))); } auto ApiWrap::fileRequest(const Data::FileLocation &location, int offset) { Expects(location.dcId != 0); return std::move(_mtp.request(MTPupload_GetFile( location.data, MTP_int(offset), MTP_int(kFileChunkSize) )).fail([=](RPCError &&result) { error(std::move(result)); }).toDC(MTP::ShiftDcId(location.dcId, MTP::kExportDcShift))); } ApiWrap::ApiWrap(Fn)> runner) : _mtp(std::move(runner)) { } rpl::producer ApiWrap::errors() const { return _errors.events(); } void ApiWrap::startExport(const Settings &settings) { Expects(_settings == nullptr); _settings = std::make_unique(settings); } void ApiWrap::requestPersonalInfo(FnMut done) { mainRequest(MTPusers_GetFullUser( _user )).done([=, done = std::move(done)](const MTPUserFull &result) mutable { Expects(result.type() == mtpc_userFull); const auto &full = result.c_userFull(); if (full.vuser.type() == mtpc_user) { done(Data::ParsePersonalInfo(result)); } else { error("Bad user type."); } }).send(); } void ApiWrap::requestUserpics( FnMut start, Fn slice, FnMut finish) { Expects(_userpicsProcess == nullptr); _userpicsProcess = std::make_unique(); _userpicsProcess->start = std::move(start); _userpicsProcess->handleSlice = std::move(slice); _userpicsProcess->finish = std::move(finish); mainRequest(MTPphotos_GetUserPhotos( _user, MTP_int(0), MTP_long(0), MTP_int(kUserpicsSliceLimit) )).done([=](const MTPphotos_Photos &result) mutable { Expects(_userpicsProcess != nullptr); _userpicsProcess->start([&] { auto info = Data::UserpicsInfo(); result.match([&](const MTPDphotos_photos &data) { info.count = data.vphotos.v.size(); }, [&](const MTPDphotos_photosSlice &data) { info.count = data.vcount.v; }); return info; }()); handleUserpicsSlice(result); }).send(); } void ApiWrap::handleUserpicsSlice(const MTPphotos_Photos &result) { Expects(_userpicsProcess != nullptr); result.match([&](const auto &data) { if constexpr (MTPDphotos_photos::Is()) { _userpicsProcess->lastSlice = true; } loadUserpicsFiles(Data::ParseUserpicsSlice(data.vphotos)); }); } void ApiWrap::loadUserpicsFiles(Data::UserpicsSlice &&slice) { Expects(_userpicsProcess != nullptr); Expects(!_userpicsProcess->slice.has_value()); if (slice.list.empty()) { _userpicsProcess->lastSlice = true; } _userpicsProcess->slice = std::move(slice); _userpicsProcess->fileIndex = -1; loadNextUserpic(); } void ApiWrap::loadNextUserpic() { Expects(_userpicsProcess != nullptr); Expects(_userpicsProcess->slice.has_value()); const auto &list = _userpicsProcess->slice->list; while (true) { const auto index = ++_userpicsProcess->fileIndex; if (index >= list.size()) { break; } const auto &file = list[index].image.file; if (WillLoadFile(file)) { loadFile( file, [=](const QString &path) { loadUserpicDone(path); }); return; } } const auto lastUserpicId = list.empty() ? base::none : base::make_optional(list.back().id); _userpicsProcess->handleSlice(*base::take(_userpicsProcess->slice)); if (_userpicsProcess->lastSlice) { finishUserpics(); return; } Assert(lastUserpicId.has_value()); mainRequest(MTPphotos_GetUserPhotos( _user, MTP_int(0), MTP_long(*lastUserpicId), MTP_int(kUserpicsSliceLimit) )).done([=](const MTPphotos_Photos &result) { handleUserpicsSlice(result); }).send(); } void ApiWrap::loadUserpicDone(const QString &relativePath) { Expects(_userpicsProcess != nullptr); Expects(_userpicsProcess->slice.has_value()); Expects((_userpicsProcess->fileIndex >= 0) && (_userpicsProcess->fileIndex < _userpicsProcess->slice->list.size())); const auto index = _userpicsProcess->fileIndex; auto &file = _userpicsProcess->slice->list[index].image.file; file.relativePath = relativePath; loadNextUserpic(); } void ApiWrap::finishUserpics() { Expects(_userpicsProcess != nullptr); base::take(_userpicsProcess)->finish(); } void ApiWrap::requestContacts(FnMut done) { const auto hash = 0; mainRequest(MTPcontacts_GetContacts( MTP_int(hash) )).done([=, done = std::move(done)]( const MTPcontacts_Contacts &result) mutable { if (result.type() == mtpc_contacts_contacts) { done(Data::ParseContactsList(result)); } else { error("Bad contacts type."); } }).send(); } void ApiWrap::requestSessions(FnMut done) { mainRequest(MTPaccount_GetAuthorizations( )).done([=, done = std::move(done)]( const MTPaccount_Authorizations &result) mutable { done(Data::ParseSessionsList(result)); }).send(); } void ApiWrap::requestDialogs( FnMut start, Fn startOne, Fn sliceOne, Fn finishOne, FnMut finish) { Expects(_dialogsProcess == nullptr); _dialogsProcess = std::make_unique(); _dialogsProcess->start = std::move(start); _dialogsProcess->startOne = std::move(startOne); _dialogsProcess->sliceOne = std::move(sliceOne); _dialogsProcess->finishOne = std::move(finishOne); _dialogsProcess->finish = std::move(finish); requestDialogsSlice(); } void ApiWrap::requestDialogsSlice() { Expects(_dialogsProcess != nullptr); mainRequest(MTPmessages_GetDialogs( MTP_flags(0), MTP_int(_dialogsProcess->offsetDate), MTP_int(_dialogsProcess->offsetId), _dialogsProcess->offsetPeer, MTP_int(kChatsSliceLimit) )).done([=](const MTPmessages_Dialogs &result) mutable { const auto finished = [&] { switch (result.type()) { case mtpc_messages_dialogs: return true; case mtpc_messages_dialogsSlice: { const auto &data = result.c_messages_dialogsSlice(); return data.vdialogs.v.isEmpty(); } break; default: Unexpected("Type in ApiWrap::requestChatsSlice."); } }(); auto info = Data::ParseDialogsInfo(result); if (finished || info.list.empty()) { finishDialogsList(); } else { const auto &last = info.list.back(); _dialogsProcess->offsetId = last.topMessageId; _dialogsProcess->offsetDate = last.topMessageDate; _dialogsProcess->offsetPeer = last.input; appendDialogsSlice(std::move(info)); requestDialogsSlice(); } }).send(); } void ApiWrap::appendDialogsSlice(Data::DialogsInfo &&info) { Expects(_dialogsProcess != nullptr); Expects(_settings != nullptr); const auto types = _settings->types | _settings->fullChats; auto filtered = ranges::view::all( info.list ) | ranges::view::filter([&](const Data::DialogInfo &info) { const auto bit = [&] { using DialogType = Data::DialogInfo::Type; switch (info.type) { case DialogType::Personal: return Settings::Type::PersonalChats; case DialogType::PrivateGroup: return Settings::Type::PrivateGroups; case DialogType::PublicGroup: return Settings::Type::PublicGroups; case DialogType::Channel: return Settings::Type::MyChannels; } return Settings::Type(0); }(); return (types & bit) != 0; }); auto &list = _dialogsProcess->info.list; list.reserve(list.size()); for (auto &info : filtered) { list.push_back(std::move(info)); } } void ApiWrap::finishDialogsList() { Expects(_dialogsProcess != nullptr); ranges::reverse(_dialogsProcess->info.list); fillDialogsPaths(); _dialogsProcess->start(_dialogsProcess->info); requestNextDialog(); } void ApiWrap::fillDialogsPaths() { Expects(_dialogsProcess != nullptr); auto &list = _dialogsProcess->info.list; const auto digits = Data::NumberToString(list.size() - 1).size(); auto index = 0; for (auto &dialog : list) { const auto number = Data::NumberToString(++index, digits, '0'); dialog.relativePath = "Chats/chat_" + number + '/'; } } void ApiWrap::requestNextDialog() { Expects(_dialogsProcess != nullptr); Expects(_dialogsProcess->single == nullptr); const auto index = ++_dialogsProcess->singleIndex; if (index < 11) {// _dialogsProcess->info.list.size()) { const auto &one = _dialogsProcess->info.list[index]; _dialogsProcess->single = std::make_unique(one); _dialogsProcess->startOne(one); requestMessagesSlice(); return; } finishDialogs(); } void ApiWrap::requestMessagesSlice() { Expects(_dialogsProcess != nullptr); Expects(_dialogsProcess->single != nullptr); const auto process = _dialogsProcess->single.get(); mainRequest(MTPmessages_GetHistory( process->info.input, MTP_int(process->offsetId), MTP_int(0), // offset_date MTP_int(-kMessagesSliceLimit), MTP_int(kMessagesSliceLimit), MTP_int(0), // max_id MTP_int(0), // min_id MTP_int(0) // hash )).done([=](const MTPmessages_Messages &result) mutable { Expects(_dialogsProcess != nullptr); Expects(_dialogsProcess->single != nullptr); const auto process = _dialogsProcess->single.get(); result.match([&](const MTPDmessages_messagesNotModified &data) { error("Unexpected messagesNotModified received."); }, [&](const auto &data) { if constexpr (MTPDmessages_messages::Is()) { process->lastSlice = true; } loadMessagesFiles(Data::ParseMessagesSlice( data.vmessages, data.vusers, data.vchats, process->info.relativePath)); }); }).send(); } void ApiWrap::loadMessagesFiles(Data::MessagesSlice &&slice) { Expects(_dialogsProcess != nullptr); Expects(_dialogsProcess->single != nullptr); Expects(!_dialogsProcess->single->slice.has_value()); const auto process = _dialogsProcess->single.get(); if (slice.list.empty()) { process->lastSlice = true; } process->slice = std::move(slice); process->fileIndex = -1; loadNextMessageFile(); } void ApiWrap::loadNextMessageFile() { Expects(_dialogsProcess != nullptr); Expects(_dialogsProcess->single != nullptr); Expects(_dialogsProcess->single->slice.has_value()); const auto process = _dialogsProcess->single.get(); const auto &list = process->slice->list; while (true) { const auto index = ++process->fileIndex; if (index >= list.size()) { break; } const auto &file = list[index].file(); if (WillLoadFile(file)) { loadFile( file, [=](const QString &path) { loadMessageFileDone(path); }); return; } } if (!list.empty()) { process->offsetId = list.back().id + 1; } _dialogsProcess->sliceOne(*base::take(process->slice)); if (process->lastSlice) { finishMessages(); } else { requestMessagesSlice(); } } void ApiWrap::loadMessageFileDone(const QString &relativePath) { Expects(_dialogsProcess != nullptr); Expects(_dialogsProcess->single != nullptr); Expects(_dialogsProcess->single->slice.has_value()); Expects((_dialogsProcess->single->fileIndex >= 0) && (_dialogsProcess->single->fileIndex < _dialogsProcess->single->slice->list.size())); const auto process = _dialogsProcess->single.get(); const auto index = process->fileIndex; process->slice->list[index].file().relativePath = relativePath; loadNextMessageFile(); } void ApiWrap::finishMessages() { Expects(_dialogsProcess != nullptr); Expects(_dialogsProcess->single != nullptr); Expects(!_dialogsProcess->single->slice.has_value()); _dialogsProcess->single = nullptr; _dialogsProcess->finishOne(); requestNextDialog(); } void ApiWrap::finishDialogs() { Expects(_dialogsProcess != nullptr); Expects(_dialogsProcess->single == nullptr); base::take(_dialogsProcess)->finish(); } void ApiWrap::loadFile(const Data::File &file, FnMut done) { Expects(_fileProcess == nullptr); Expects(_settings != nullptr); Expects(WillLoadFile(file)); using namespace Output; const auto relativePath = File::PrepareRelativePath( _settings->path, file.suggestedPath); _fileProcess = std::make_unique( _settings->path + relativePath); _fileProcess->relativePath = relativePath; _fileProcess->location = file.location; _fileProcess->done = std::move(done); if (!file.content.isEmpty()) { auto &output = _fileProcess->file; if (output.writeBlock(file.content) == File::Result::Success) { _fileProcess->done(relativePath); } else { error(QString("Could not open '%1'.").arg(relativePath)); } } else { loadFilePart(); } } void ApiWrap::loadFilePart() { if (!_fileProcess || _fileProcess->requests.size() >= kFileRequestsCount || (_fileProcess->size > 0 && _fileProcess->offset >= _fileProcess->size)) { return; } const auto offset = _fileProcess->offset; _fileProcess->requests.push_back({ offset }); fileRequest( _fileProcess->location, _fileProcess->offset ).done([=](const MTPupload_File &result) { filePartDone(offset, result); }).send(); _fileProcess->offset += kFileChunkSize; if (_fileProcess->size > 0 && _fileProcess->requests.size() < kFileRequestsCount) { //const auto runner = _runner; //crl::on_main([=] { // QTimer::singleShot(kFileNextRequestDelay, [=] { // runner([=] { // loadFilePart(); // }); // }); //}); } } void ApiWrap::filePartDone(int offset, const MTPupload_File &result) { Expects(_fileProcess != nullptr); Expects(!_fileProcess->requests.empty()); if (result.type() == mtpc_upload_fileCdnRedirect) { error("Cdn redirect is not supported."); return; } const auto &data = result.c_upload_file(); if (data.vbytes.v.isEmpty()) { if (_fileProcess->size > 0) { error("Empty bytes received in file part."); return; } } else { using Request = FileProcess::Request; auto &requests = _fileProcess->requests; const auto i = ranges::find( requests, offset, [](const Request &request) { return request.offset; }); Assert(i != end(requests)); i->bytes = data.vbytes.v; auto &file = _fileProcess->file; while (!requests.empty() && !requests.front().bytes.isEmpty()) { const auto &bytes = requests.front().bytes; if (file.writeBlock(bytes) != Output::File::Result::Success) { error(QString("Could not write bytes to '%1'." ).arg(_fileProcess->relativePath)); return; } requests.pop_front(); } if (!requests.empty() || !_fileProcess->size || _fileProcess->size > _fileProcess->offset) { loadFilePart(); return; } } auto process = base::take(_fileProcess); process->done(process->relativePath); } void ApiWrap::error(RPCError &&error) { _errors.fire(std::move(error)); } void ApiWrap::error(const QString &text) { error(MTP_rpc_error(MTP_int(0), MTP_string("API_ERROR: " + text))); } ApiWrap::~ApiWrap() = default; } // namespace Export