tdesktop/Telegram/SourceFiles/storage/file_upload.cpp

580 lines
17 KiB
C++
Raw Normal View History

/*
This file is part of Telegram Desktop,
the official desktop application for the Telegram messaging service.
For license and copyright information please follow this link:
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
2017-03-04 10:23:56 +00:00
#include "storage/file_upload.h"
2017-12-19 16:57:42 +00:00
#include "storage/localimageloader.h"
#include "storage/file_download.h"
#include "data/data_document.h"
#include "data/data_photo.h"
#include "data/data_session.h"
2019-07-24 11:45:24 +00:00
#include "main/main_session.h"
namespace Storage {
namespace {
2019-01-18 11:26:43 +00:00
// max 512kb uploaded at the same time in each session
constexpr auto kMaxUploadFileParallelSize = MTP::kUploadSessionsCount * 512 * 1024;
constexpr auto kDocumentMaxPartsCount = 3000;
// 32kb for tiny document ( < 1mb )
constexpr auto kDocumentUploadPartSize0 = 32 * 1024;
// 64kb for little document ( <= 32mb )
constexpr auto kDocumentUploadPartSize1 = 64 * 1024;
// 128kb for small document ( <= 375mb )
constexpr auto kDocumentUploadPartSize2 = 128 * 1024;
// 256kb for medium document ( <= 750mb )
constexpr auto kDocumentUploadPartSize3 = 256 * 1024;
// 512kb for large document ( <= 1500mb )
constexpr auto kDocumentUploadPartSize4 = 512 * 1024;
// One part each half second, if not uploaded faster.
constexpr auto kUploadRequestInterval = crl::time(500);
// How much time without upload causes additional session kill.
2019-12-02 13:10:19 +00:00
constexpr auto kKillSessionTimeout = 15 * crl::time(000);
} // namespace
2017-12-19 16:57:42 +00:00
struct Uploader::File {
File(const SendMediaReady &media);
File(const std::shared_ptr<FileLoadResult> &file);
void setDocSize(int32 size);
bool setPartSize(uint32 partSize);
std::shared_ptr<FileLoadResult> file;
SendMediaReady media;
int32 partsCount = 0;
mutable int32 fileSentSize = 0;
2017-12-19 16:57:42 +00:00
uint64 id() const;
SendMediaType type() const;
uint64 thumbId() const;
const QString &filename() const;
HashMd5 md5Hash;
std::unique_ptr<QFile> docFile;
int32 docSentParts = 0;
int32 docSize = 0;
int32 docPartSize = 0;
int32 docPartsCount = 0;
};
Uploader::File::File(const SendMediaReady &media) : media(media) {
partsCount = media.parts.size();
2019-02-08 16:20:08 +00:00
if (type() == SendMediaType::File
2019-09-03 08:25:19 +00:00
|| type() == SendMediaType::ThemeFile
2019-02-08 16:20:08 +00:00
|| type() == SendMediaType::Audio) {
2017-12-19 16:57:42 +00:00
setDocSize(media.file.isEmpty()
? media.data.size()
: media.filesize);
} else {
docSize = docPartSize = docPartsCount = 0;
}
}
Uploader::File::File(const std::shared_ptr<FileLoadResult> &file)
: file(file) {
partsCount = (type() == SendMediaType::Photo
|| type() == SendMediaType::Secure)
2017-12-19 16:57:42 +00:00
? file->fileparts.size()
: file->thumbparts.size();
2019-02-08 16:20:08 +00:00
if (type() == SendMediaType::File
2019-09-03 08:25:19 +00:00
|| type() == SendMediaType::ThemeFile
2019-02-08 16:20:08 +00:00
|| type() == SendMediaType::Audio) {
2017-12-19 16:57:42 +00:00
setDocSize(file->filesize);
} else {
docSize = docPartSize = docPartsCount = 0;
}
}
void Uploader::File::setDocSize(int32 size) {
docSize = size;
constexpr auto limit0 = 1024 * 1024;
constexpr auto limit1 = 32 * limit0;
2019-01-18 11:26:43 +00:00
if (docSize >= limit0 || !setPartSize(kDocumentUploadPartSize0)) {
if (docSize > limit1 || !setPartSize(kDocumentUploadPartSize1)) {
if (!setPartSize(kDocumentUploadPartSize2)) {
if (!setPartSize(kDocumentUploadPartSize3)) {
if (!setPartSize(kDocumentUploadPartSize4)) {
2017-12-19 16:57:42 +00:00
LOG(("Upload Error: bad doc size: %1").arg(docSize));
}
}
}
}
}
}
bool Uploader::File::setPartSize(uint32 partSize) {
docPartSize = partSize;
docPartsCount = (docSize / docPartSize)
+ ((docSize % docPartSize) ? 1 : 0);
2019-01-18 11:26:43 +00:00
return (docPartsCount <= kDocumentMaxPartsCount);
2017-12-19 16:57:42 +00:00
}
uint64 Uploader::File::id() const {
return file ? file->id : media.id;
}
SendMediaType Uploader::File::type() const {
return file ? file->type : media.type;
}
uint64 Uploader::File::thumbId() const {
return file ? file->thumbId : media.thumbId;
}
const QString &Uploader::File::filename() const {
return file ? file->filename : media.filename;
}
Uploader::Uploader(not_null<ApiWrap*> api)
: _api(api) {
nextTimer.setSingleShot(true);
connect(&nextTimer, SIGNAL(timeout()), this, SLOT(sendNext()));
stopSessionsTimer.setSingleShot(true);
connect(&stopSessionsTimer, SIGNAL(timeout()), this, SLOT(stopSessions()));
}
2019-01-18 12:27:37 +00:00
void Uploader::uploadMedia(
const FullMsgId &msgId,
const SendMediaReady &media) {
if (media.type == SendMediaType::Photo) {
2019-01-18 12:27:37 +00:00
Auth().data().processPhoto(media.photo, media.photoThumbs);
} else if (media.type == SendMediaType::File
2019-09-03 08:25:19 +00:00
|| media.type == SendMediaType::ThemeFile
2019-01-18 12:27:37 +00:00
|| media.type == SendMediaType::Audio) {
const auto document = media.photoThumbs.empty()
2019-01-18 12:27:37 +00:00
? Auth().data().processDocument(media.document)
: Auth().data().processDocument(
media.document,
base::duplicate(media.photoThumbs.front().second));
if (!media.data.isEmpty()) {
document->setDataAndCache(media.data);
2019-09-03 08:25:19 +00:00
if (media.type == SendMediaType::ThemeFile) {
2019-02-08 16:20:08 +00:00
document->checkWallPaperProperties();
}
}
if (!media.file.isEmpty()) {
document->setLocation(FileLocation(media.file));
}
}
queue.emplace(msgId, File(media));
sendNext();
}
2017-12-19 16:57:42 +00:00
void Uploader::upload(
const FullMsgId &msgId,
const std::shared_ptr<FileLoadResult> &file) {
if (file->type == SendMediaType::Photo) {
2019-01-18 12:27:37 +00:00
const auto photo = Auth().data().processPhoto(
file->photo,
file->photoThumbs);
photo->uploadingData = std::make_unique<Data::UploadState>(
file->partssize);
} else if (file->type == SendMediaType::File
2019-09-03 08:25:19 +00:00
|| file->type == SendMediaType::ThemeFile
2019-01-18 12:27:37 +00:00
|| file->type == SendMediaType::Audio) {
const auto document = file->thumb.isNull()
? Auth().data().processDocument(file->document)
: Auth().data().processDocument(
file->document,
std::move(file->thumb));
2019-01-18 12:27:37 +00:00
document->uploadingData = std::make_unique<Data::UploadState>(
document->size);
2019-03-01 11:16:55 +00:00
document->setGoodThumbnailOnUpload(
std::move(file->goodThumbnail),
std::move(file->goodThumbnailBytes));
if (!file->content.isEmpty()) {
document->setDataAndCache(file->content);
2019-09-03 08:25:19 +00:00
if (file->type == SendMediaType::ThemeFile) {
2019-02-08 16:20:08 +00:00
document->checkWallPaperProperties();
}
}
if (!file->filepath.isEmpty()) {
document->setLocation(FileLocation(file->filepath));
}
}
queue.emplace(msgId, File(file));
sendNext();
}
void Uploader::currentFailed() {
auto j = queue.find(uploadingId);
if (j != queue.end()) {
if (j->second.type() == SendMediaType::Photo) {
_photoFailed.fire_copy(j->first);
} else if (j->second.type() == SendMediaType::File
2019-09-03 08:25:19 +00:00
|| j->second.type() == SendMediaType::ThemeFile
|| j->second.type() == SendMediaType::Audio) {
const auto document = Auth().data().document(j->second.id());
if (document->uploading()) {
document->status = FileUploadFailed;
}
_documentFailed.fire_copy(j->first);
} else if (j->second.type() == SendMediaType::Secure) {
_secureFailed.fire_copy(j->first);
} else {
Unexpected("Type in Uploader::currentFailed.");
}
queue.erase(j);
}
requestsSent.clear();
docRequestsSent.clear();
dcMap.clear();
uploadingId = FullMsgId();
sentSize = 0;
for (int i = 0; i < MTP::kUploadSessionsCount; ++i) {
sentSizes[i] = 0;
}
sendNext();
}
void Uploader::stopSessions() {
for (int i = 0; i < MTP::kUploadSessionsCount; ++i) {
MTP::stopSession(MTP::uploadDcId(i));
}
}
void Uploader::sendNext() {
if (sentSize >= kMaxUploadFileParallelSize || _pausedId.msg) return;
bool stopping = stopSessionsTimer.isActive();
if (queue.empty()) {
if (!stopping) {
2019-12-02 13:10:19 +00:00
stopSessionsTimer.start(kKillSessionTimeout);
}
return;
}
if (stopping) {
stopSessionsTimer.stop();
}
auto i = uploadingId.msg ? queue.find(uploadingId) : queue.begin();
if (!uploadingId.msg) {
uploadingId = i->first;
} else if (i == queue.end()) {
i = queue.begin();
uploadingId = i->first;
}
auto &uploadingData = i->second;
auto todc = 0;
for (auto dc = 1; dc != MTP::kUploadSessionsCount; ++dc) {
if (sentSizes[dc] < sentSizes[todc]) {
todc = dc;
}
}
auto &parts = uploadingData.file
? ((uploadingData.type() == SendMediaType::Photo
|| uploadingData.type() == SendMediaType::Secure)
? uploadingData.file->fileparts
: uploadingData.file->thumbparts)
: uploadingData.media.parts;
const auto partsOfId = uploadingData.file
? ((uploadingData.type() == SendMediaType::Photo
|| uploadingData.type() == SendMediaType::Secure)
? uploadingData.file->id
: uploadingData.file->thumbId)
: uploadingData.media.thumbId;
if (parts.isEmpty()) {
if (uploadingData.docSentParts >= uploadingData.docPartsCount) {
if (requestsSent.empty() && docRequestsSent.empty()) {
const auto options = uploadingData.file
? uploadingData.file->to.options
: Api::SendOptions();
const auto edit = uploadingData.file &&
uploadingData.file->edit;
if (uploadingData.type() == SendMediaType::Photo) {
auto photoFilename = uploadingData.filename();
if (!photoFilename.endsWith(qstr(".jpg"), Qt::CaseInsensitive)) {
// Server has some extensions checking for inputMediaUploadedPhoto,
// so force the extension to be .jpg anyway. It doesn't matter,
// because the filename from inputFile is not used anywhere.
photoFilename += qstr(".jpg");
}
const auto md5 = uploadingData.file
? uploadingData.file->filemd5
: uploadingData.media.jpeg_md5;
const auto file = MTP_inputFile(
MTP_long(uploadingData.id()),
MTP_int(uploadingData.partsCount),
MTP_string(photoFilename),
MTP_bytes(md5));
_photoReady.fire({ uploadingId, options, file, edit });
} else if (uploadingData.type() == SendMediaType::File
2019-09-03 08:25:19 +00:00
|| uploadingData.type() == SendMediaType::ThemeFile
|| uploadingData.type() == SendMediaType::Audio) {
QByteArray docMd5(32, Qt::Uninitialized);
hashMd5Hex(uploadingData.md5Hash.result(), docMd5.data());
2019-01-18 11:26:43 +00:00
const auto file = (uploadingData.docSize > kUseBigFilesFrom)
? MTP_inputFileBig(
MTP_long(uploadingData.id()),
MTP_int(uploadingData.docPartsCount),
MTP_string(uploadingData.filename()))
: MTP_inputFile(
MTP_long(uploadingData.id()),
MTP_int(uploadingData.docPartsCount),
MTP_string(uploadingData.filename()),
MTP_bytes(docMd5));
if (uploadingData.partsCount) {
const auto thumbFilename = uploadingData.file
? uploadingData.file->thumbname
: (qsl("thumb.") + uploadingData.media.thumbExt);
const auto thumbMd5 = uploadingData.file
? uploadingData.file->thumbmd5
: uploadingData.media.jpeg_md5;
const auto thumb = MTP_inputFile(
MTP_long(uploadingData.thumbId()),
MTP_int(uploadingData.partsCount),
MTP_string(thumbFilename),
MTP_bytes(thumbMd5));
_thumbDocumentReady.fire({
uploadingId,
options,
file,
thumb,
edit });
} else {
_documentReady.fire({
uploadingId,
options,
file,
edit });
}
} else if (uploadingData.type() == SendMediaType::Secure) {
_secureReady.fire({
uploadingId,
uploadingData.id(),
uploadingData.partsCount });
}
queue.erase(uploadingId);
uploadingId = FullMsgId();
sendNext();
}
return;
}
auto &content = uploadingData.file
? uploadingData.file->content
: uploadingData.media.data;
QByteArray toSend;
if (content.isEmpty()) {
if (!uploadingData.docFile) {
const auto filepath = uploadingData.file
? uploadingData.file->filepath
: uploadingData.media.file;
uploadingData.docFile = std::make_unique<QFile>(filepath);
if (!uploadingData.docFile->open(QIODevice::ReadOnly)) {
currentFailed();
return;
}
}
toSend = uploadingData.docFile->read(uploadingData.docPartSize);
2019-01-18 11:26:43 +00:00
if (uploadingData.docSize <= kUseBigFilesFrom) {
uploadingData.md5Hash.feed(toSend.constData(), toSend.size());
}
} else {
const auto offset = uploadingData.docSentParts
* uploadingData.docPartSize;
toSend = content.mid(offset, uploadingData.docPartSize);
if ((uploadingData.type() == SendMediaType::File
2019-09-03 08:25:19 +00:00
|| uploadingData.type() == SendMediaType::ThemeFile
|| uploadingData.type() == SendMediaType::Audio)
2019-01-18 11:26:43 +00:00
&& uploadingData.docSentParts <= kUseBigFilesFrom) {
uploadingData.md5Hash.feed(toSend.constData(), toSend.size());
}
}
if ((toSend.size() > uploadingData.docPartSize)
|| ((toSend.size() < uploadingData.docPartSize
&& uploadingData.docSentParts + 1 != uploadingData.docPartsCount))) {
currentFailed();
return;
}
mtpRequestId requestId;
2019-01-18 11:26:43 +00:00
if (uploadingData.docSize > kUseBigFilesFrom) {
requestId = MTP::send(
MTPupload_SaveBigFilePart(
MTP_long(uploadingData.id()),
MTP_int(uploadingData.docSentParts),
MTP_int(uploadingData.docPartsCount),
MTP_bytes(toSend)),
rpcDone(&Uploader::partLoaded),
rpcFail(&Uploader::partFailed),
MTP::uploadDcId(todc));
} else {
requestId = MTP::send(
MTPupload_SaveFilePart(
MTP_long(uploadingData.id()),
MTP_int(uploadingData.docSentParts),
MTP_bytes(toSend)),
rpcDone(&Uploader::partLoaded),
rpcFail(&Uploader::partFailed),
MTP::uploadDcId(todc));
}
docRequestsSent.emplace(requestId, uploadingData.docSentParts);
dcMap.emplace(requestId, todc);
sentSize += uploadingData.docPartSize;
sentSizes[todc] += uploadingData.docPartSize;
uploadingData.docSentParts++;
} else {
auto part = parts.begin();
const auto requestId = MTP::send(
MTPupload_SaveFilePart(
MTP_long(partsOfId),
MTP_int(part.key()),
MTP_bytes(part.value())),
rpcDone(&Uploader::partLoaded),
rpcFail(&Uploader::partFailed),
MTP::uploadDcId(todc));
requestsSent.emplace(requestId, part.value());
dcMap.emplace(requestId, todc);
sentSize += part.value().size();
sentSizes[todc] += part.value().size();
parts.erase(part);
}
2019-01-18 11:26:43 +00:00
nextTimer.start(kUploadRequestInterval);
}
void Uploader::cancel(const FullMsgId &msgId) {
uploaded.erase(msgId);
if (uploadingId == msgId) {
currentFailed();
} else {
queue.erase(msgId);
}
}
void Uploader::pause(const FullMsgId &msgId) {
_pausedId = msgId;
}
void Uploader::unpause() {
_pausedId = FullMsgId();
sendNext();
}
void Uploader::confirm(const FullMsgId &msgId) {
}
void Uploader::clear() {
uploaded.clear();
queue.clear();
2017-12-18 15:52:58 +00:00
for (const auto &requestData : requestsSent) {
MTP::cancel(requestData.first);
}
requestsSent.clear();
2017-12-18 15:52:58 +00:00
for (const auto &requestData : docRequestsSent) {
MTP::cancel(requestData.first);
}
docRequestsSent.clear();
dcMap.clear();
sentSize = 0;
for (int i = 0; i < MTP::kUploadSessionsCount; ++i) {
MTP::stopSession(MTP::uploadDcId(i));
sentSizes[i] = 0;
}
stopSessionsTimer.stop();
}
void Uploader::partLoaded(const MTPBool &result, mtpRequestId requestId) {
auto j = docRequestsSent.end();
auto i = requestsSent.find(requestId);
if (i == requestsSent.cend()) {
j = docRequestsSent.find(requestId);
}
if (i != requestsSent.cend() || j != docRequestsSent.cend()) {
2015-10-29 00:16:52 +00:00
if (mtpIsFalse(result)) { // failed to upload current file
currentFailed();
return;
} else {
auto dcIt = dcMap.find(requestId);
if (dcIt == dcMap.cend()) { // must not happen
currentFailed();
return;
}
auto dc = dcIt->second;
dcMap.erase(dcIt);
int32 sentPartSize = 0;
auto k = queue.find(uploadingId);
Assert(k != queue.cend());
auto &[fullId, file] = *k;
if (i != requestsSent.cend()) {
sentPartSize = i->second.size();
requestsSent.erase(i);
} else {
sentPartSize = file.docPartSize;
docRequestsSent.erase(j);
}
sentSize -= sentPartSize;
sentSizes[dc] -= sentPartSize;
if (file.type() == SendMediaType::Photo) {
file.fileSentSize += sentPartSize;
const auto photo = Auth().data().photo(file.id());
if (photo->uploading() && file.file) {
photo->uploadingData->size = file.file->partssize;
photo->uploadingData->offset = file.fileSentSize;
}
_photoProgress.fire_copy(fullId);
} else if (file.type() == SendMediaType::File
2019-09-03 08:25:19 +00:00
|| file.type() == SendMediaType::ThemeFile
|| file.type() == SendMediaType::Audio) {
const auto document = Auth().data().document(file.id());
if (document->uploading()) {
const auto doneParts = file.docSentParts
- int(docRequestsSent.size());
document->uploadingData->offset = std::min(
document->uploadingData->size,
doneParts * file.docPartSize);
}
_documentProgress.fire_copy(fullId);
} else if (file.type() == SendMediaType::Secure) {
file.fileSentSize += sentPartSize;
_secureProgress.fire_copy({
fullId,
file.fileSentSize,
file.file->partssize });
}
}
}
sendNext();
}
bool Uploader::partFailed(const RPCError &error, mtpRequestId requestId) {
if (MTP::isDefaultHandledError(error)) return false;
2015-04-04 20:01:34 +00:00
// failed to upload current file
if ((requestsSent.find(requestId) != requestsSent.cend())
|| (docRequestsSent.find(requestId) != docRequestsSent.cend())) {
currentFailed();
}
sendNext();
return true;
}
Uploader::~Uploader() {
clear();
}
} // namespace Storage