mirror of
https://github.com/telegramdesktop/tdesktop
synced 2025-03-19 01:48:09 +00:00
Add downloaded files cache in export.
Also write downloaded photo paths.
This commit is contained in:
parent
07ff7c6cb0
commit
2dc3ec955a
@ -7,6 +7,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace base {
|
||||
namespace details {
|
||||
|
||||
|
@ -48,6 +48,10 @@ struct UserpicsInfo {
|
||||
struct FileLocation {
|
||||
int dcId = 0;
|
||||
MTPInputFileLocation data;
|
||||
|
||||
explicit operator bool() const {
|
||||
return dcId != 0;
|
||||
}
|
||||
};
|
||||
|
||||
struct File {
|
||||
|
@ -11,6 +11,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
#include "export/data/export_data_types.h"
|
||||
#include "export/output/export_output_file.h"
|
||||
#include "mtproto/rpc_sender.h"
|
||||
#include "base/value_ordering.h"
|
||||
#include "base/bytes.h"
|
||||
|
||||
#include <deque>
|
||||
@ -25,13 +26,61 @@ constexpr auto kFileNextRequestDelay = TimeMs(20);
|
||||
constexpr auto kChatsSliceLimit = 100;
|
||||
constexpr auto kMessagesSliceLimit = 100;
|
||||
constexpr auto kFileMaxSize = 1500 * 1024 * 1024;
|
||||
constexpr auto kLocationCacheSize = 100'000;
|
||||
|
||||
bool FileIsAvailable(const Data::File &file) {
|
||||
return file.relativePath.isEmpty() && (file.location.dcId != 0);
|
||||
struct LocationKey {
|
||||
uint64 type;
|
||||
uint64 id;
|
||||
|
||||
inline bool operator<(const LocationKey &other) const {
|
||||
return std::tie(type, id) < std::tie(other.type, other.id);
|
||||
}
|
||||
};
|
||||
|
||||
std::tuple<const uint64 &, const uint64 &> value_ordering_helper(const LocationKey &value) {
|
||||
return std::tie(
|
||||
value.type,
|
||||
value.id);
|
||||
}
|
||||
|
||||
LocationKey ComputeLocationKey(const Data::FileLocation &value) {
|
||||
auto result = LocationKey();
|
||||
result.type = value.dcId;
|
||||
value.data.match([&](const MTPDinputFileLocation &data) {
|
||||
result.type |= (1ULL << 24);
|
||||
result.type |= (uint64(uint32(data.vlocal_id.v)) << 32);
|
||||
result.id = data.vvolume_id.v;
|
||||
}, [&](const MTPDinputDocumentFileLocation &data) {
|
||||
result.type |= (2ULL << 24);
|
||||
result.id = data.vid.v;
|
||||
}, [&](const MTPDinputSecureFileLocation &data) {
|
||||
result.type |= (3ULL << 24);
|
||||
result.id = data.vid.v;
|
||||
}, [&](const MTPDinputEncryptedFileLocation &data) {
|
||||
result.type |= (4ULL << 24);
|
||||
result.id = data.vid.v;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
class ApiWrap::LoadedFileCache {
|
||||
public:
|
||||
using Location = Data::FileLocation;
|
||||
|
||||
LoadedFileCache(int limit);
|
||||
|
||||
void save(const Location &location, const QString &relativePath);
|
||||
base::optional<QString> find(const Location &location) const;
|
||||
|
||||
private:
|
||||
int _limit = 0;
|
||||
std::map<LocationKey, QString> _map;
|
||||
std::deque<LocationKey> _list;
|
||||
|
||||
};
|
||||
|
||||
struct ApiWrap::UserpicsProcess {
|
||||
FnMut<void(Data::UserpicsInfo&&)> start;
|
||||
Fn<void(Data::UserpicsSlice&&)> handleSlice;
|
||||
@ -94,6 +143,32 @@ struct ApiWrap::DialogsProcess::Single {
|
||||
|
||||
};
|
||||
|
||||
ApiWrap::LoadedFileCache::LoadedFileCache(int limit) : _limit(limit) {
|
||||
Expects(limit >= 0);
|
||||
}
|
||||
|
||||
void ApiWrap::LoadedFileCache::save(
|
||||
const Location &location,
|
||||
const QString &relativePath) {
|
||||
const auto key = ComputeLocationKey(location);
|
||||
_map[key] = relativePath;
|
||||
_list.push_back(key);
|
||||
if (_list.size() > _limit) {
|
||||
const auto key = _list.front();
|
||||
_list.pop_front();
|
||||
_map.erase(key);
|
||||
}
|
||||
}
|
||||
|
||||
base::optional<QString> ApiWrap::LoadedFileCache::find(
|
||||
const Location &location) const {
|
||||
const auto key = ComputeLocationKey(location);
|
||||
if (const auto i = _map.find(key); i != end(_map)) {
|
||||
return i->second;
|
||||
}
|
||||
return base::none;
|
||||
}
|
||||
|
||||
ApiWrap::FileProcess::FileProcess(const QString &path) : file(path) {
|
||||
}
|
||||
|
||||
@ -129,7 +204,8 @@ auto ApiWrap::fileRequest(const Data::FileLocation &location, int offset) {
|
||||
}
|
||||
|
||||
ApiWrap::ApiWrap(Fn<void(FnMut<void()>)> runner)
|
||||
: _mtp(std::move(runner)) {
|
||||
: _mtp(std::move(runner))
|
||||
, _fileCache(std::make_unique<LoadedFileCache>(kLocationCacheSize)) {
|
||||
}
|
||||
|
||||
rpl::producer<RPCError> ApiWrap::errors() const {
|
||||
@ -613,11 +689,11 @@ bool ApiWrap::processFileLoad(
|
||||
|
||||
if (!file.relativePath.isEmpty()) {
|
||||
return true;
|
||||
} else if (writePreloadedFile(file)) {
|
||||
return true;
|
||||
} else if (!FileIsAvailable(file)) {
|
||||
} else if (!file.location) {
|
||||
file.skipReason = SkipReason::Unavailable;
|
||||
return true;
|
||||
} else if (writePreloadedFile(file)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
using Type = MediaSettings::Type;
|
||||
@ -656,11 +732,15 @@ bool ApiWrap::writePreloadedFile(Data::File &file) {
|
||||
|
||||
using namespace Output;
|
||||
|
||||
if (!file.content.isEmpty()) {
|
||||
if (const auto path = _fileCache->find(file.location)) {
|
||||
file.relativePath = *path;
|
||||
return true;
|
||||
} else if (!file.content.isEmpty()) {
|
||||
const auto process = prepareFileProcess(file);
|
||||
auto &output = _fileProcess->file;
|
||||
if (output.writeBlock(file.content) == File::Result::Success) {
|
||||
file.relativePath = process->relativePath;
|
||||
_fileCache->save(file.location, file.relativePath);
|
||||
return true;
|
||||
}
|
||||
error(QString("Could not write '%1'.").arg(process->relativePath));
|
||||
@ -670,7 +750,7 @@ bool ApiWrap::writePreloadedFile(Data::File &file) {
|
||||
|
||||
void ApiWrap::loadFile(const Data::File &file, FnMut<void(QString)> done) {
|
||||
Expects(_fileProcess == nullptr);
|
||||
Expects(FileIsAvailable(file));
|
||||
Expects(file.location.dcId != 0);
|
||||
|
||||
_fileProcess = prepareFileProcess(file);
|
||||
_fileProcess->done = std::move(done);
|
||||
@ -767,7 +847,10 @@ void ApiWrap::filePartDone(int offset, const MTPupload_File &result) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
auto process = base::take(_fileProcess);
|
||||
const auto relativePath = process->relativePath;
|
||||
_fileCache->save(process->location, relativePath);
|
||||
process->done(process->relativePath);
|
||||
}
|
||||
|
||||
|
@ -60,6 +60,7 @@ private:
|
||||
struct UserpicsProcess;
|
||||
struct FileProcess;
|
||||
struct DialogsProcess;
|
||||
class LoadedFileCache;
|
||||
|
||||
void startMainSession(FnMut<void()> done);
|
||||
|
||||
@ -111,6 +112,7 @@ private:
|
||||
std::unique_ptr<Settings> _settings;
|
||||
MTPInputUser _user = MTP_inputUserSelf();
|
||||
|
||||
std::unique_ptr<LoadedFileCache> _fileCache;
|
||||
std::unique_ptr<UserpicsProcess> _userpicsProcess;
|
||||
std::unique_ptr<FileProcess> _fileProcess;
|
||||
std::unique_ptr<DialogsProcess> _dialogsProcess;
|
||||
|
@ -71,7 +71,8 @@ struct Settings {
|
||||
}
|
||||
|
||||
static inline Types DefaultFullChats() {
|
||||
return Type::PersonalChats;
|
||||
return Type::PersonalChats
|
||||
| Type::BotChats;
|
||||
}
|
||||
|
||||
};
|
||||
|
@ -178,6 +178,31 @@ QByteArray SerializeMessage(
|
||||
}
|
||||
};
|
||||
|
||||
using SkipReason = Data::File::SkipReason;
|
||||
const auto pushPath = [&](
|
||||
const Data::File &file,
|
||||
const QByteArray &label) {
|
||||
Expects(!file.relativePath.isEmpty()
|
||||
|| file.skipReason != SkipReason::None);
|
||||
|
||||
push(label, [&]() -> QByteArray {
|
||||
switch (file.skipReason) {
|
||||
case SkipReason::Unavailable: return "(file unavailable)";
|
||||
case SkipReason::FileSize: return "(file too large)";
|
||||
case SkipReason::FileType: return "(file skipped)";
|
||||
case SkipReason::None: return FormatFilePath(file);
|
||||
}
|
||||
Unexpected("Skip reason while writing file path.");
|
||||
}());
|
||||
};
|
||||
const auto pushPhoto = [&](const Image &image) {
|
||||
pushPath(image.file, "Photo");
|
||||
if (image.width && image.height) {
|
||||
push("Width", NumberToString(image.width));
|
||||
push("Height", NumberToString(image.height));
|
||||
}
|
||||
};
|
||||
|
||||
message.action.content.match([&](const ActionChatCreate &data) {
|
||||
pushActor();
|
||||
pushAction("Create group");
|
||||
@ -190,7 +215,7 @@ QByteArray SerializeMessage(
|
||||
}, [&](const ActionChatEditPhoto &data) {
|
||||
pushActor();
|
||||
pushAction("Edit group photo");
|
||||
push("Photo", FormatFilePath(data.photo.image.file));
|
||||
pushPhoto(data.photo.image);
|
||||
}, [&](const ActionChatDeletePhoto &data) {
|
||||
pushActor();
|
||||
pushAction("Delete group photo");
|
||||
@ -306,28 +331,29 @@ QByteArray SerializeMessage(
|
||||
}
|
||||
|
||||
message.media.content.match([&](const Photo &photo) {
|
||||
pushPhoto(photo.image);
|
||||
pushTTL();
|
||||
}, [&](const Document &data) {
|
||||
const auto pushPath = [&](const QByteArray &label) {
|
||||
push(label, FormatFilePath(data.file));
|
||||
const auto pushMyPath = [&](const QByteArray &label) {
|
||||
return pushPath(data.file, label);
|
||||
};
|
||||
if (data.isSticker) {
|
||||
pushPath("Sticker");
|
||||
pushMyPath("Sticker");
|
||||
push("Emoji", data.stickerEmoji);
|
||||
} else if (data.isVideoMessage) {
|
||||
pushPath("Video message");
|
||||
pushMyPath("Video message");
|
||||
} else if (data.isVoiceMessage) {
|
||||
pushPath("Voice message");
|
||||
pushMyPath("Voice message");
|
||||
} else if (data.isAnimated) {
|
||||
pushPath("Animation");
|
||||
pushMyPath("Animation");
|
||||
} else if (data.isVideoFile) {
|
||||
pushPath("Video file");
|
||||
pushMyPath("Video file");
|
||||
} else if (data.isAudioFile) {
|
||||
pushPath("Audio file");
|
||||
pushMyPath("Audio file");
|
||||
push("Performer", data.songPerformer);
|
||||
push("Title", data.songTitle);
|
||||
} else {
|
||||
pushPath("File");
|
||||
pushMyPath("File");
|
||||
}
|
||||
if (!data.isSticker) {
|
||||
push("Mime type", data.mime);
|
||||
|
Loading…
Reference in New Issue
Block a user