tdesktop/Telegram/SourceFiles/chat_helpers/emoji_keywords.cpp

719 lines
18 KiB
C++
Raw Normal View History

/*
This file is part of Telegram Desktop,
the official desktop application for the Telegram messaging service.
For license and copyright information please follow this link:
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#include "chat_helpers/emoji_keywords.h"
2019-09-17 16:13:12 +00:00
#include "emoji_suggestions_helper.h"
#include "lang/lang_instance.h"
#include "lang/lang_cloud_manager.h"
#include "core/application.h"
2019-09-17 16:13:12 +00:00
#include "base/platform/base_platform_info.h"
#include "ui/emoji_config.h"
#include "main/main_domain.h"
2019-07-24 11:45:24 +00:00
#include "main/main_session.h"
#include "apiwrap.h"
2019-09-04 07:19:15 +00:00
#include <QtGui/QGuiApplication>
namespace ChatHelpers {
namespace {
constexpr auto kRefreshEach = 60 * 60 * crl::time(1000); // 1 hour.
2019-03-29 14:23:15 +00:00
constexpr auto kKeepNotUsedLangPacksCount = 4;
constexpr auto kKeepNotUsedInputLanguagesCount = 4;
using namespace Ui::Emoji;
using Result = EmojiKeywords::Result;
struct LangPackEmoji {
EmojiPtr emoji = nullptr;
QString text;
};
struct LangPackData {
int version = 0;
int maxKeyLength = 0;
std::map<QString, std::vector<LangPackEmoji>> emoji;
};
[[nodiscard]] bool MustAddPostfix(const QString &text) {
if (text.size() != 1) {
return false;
}
const auto code = text[0].unicode();
return (code == 0x2122U) || (code == 0xA9U) || (code == 0xAEU);
}
2019-04-01 10:19:36 +00:00
[[nodiscard]] bool SkipExactKeyword(
const QString &language,
const QString &word) {
2019-05-27 10:22:13 +00:00
if ((word.size() == 1) && !word[0].isLetter()) {
return true;
} else if (word == qstr("10")) {
return true;
} else if (language != qstr("en")) {
2019-04-01 10:19:36 +00:00
return false;
} else if ((word.size() == 1)
&& (word[0] != '$')
&& (word[0].unicode() != 8364)) { // Euro.
return true;
} else if ((word.size() == 2)
&& (word != qstr("us"))
&& (word != qstr("uk"))
&& (word != qstr("hi"))
&& (word != qstr("ok"))) {
return true;
}
return false;
}
[[nodiscard]] EmojiPtr FindExact(const QString &text) {
auto length = 0;
const auto result = Find(text, &length);
return (length < text.size()) ? nullptr : result;
}
2019-03-29 14:23:21 +00:00
void CreateCacheFilePath() {
QDir().mkpath(internal::CacheFileFolder() + qstr("/keywords"));
}
[[nodiscard]] QString CacheFilePath(QString id) {
static const auto BadSymbols = QRegularExpression("[^a-zA-Z0-9_\\.\\-]");
id.replace(BadSymbols, QString());
if (id.isEmpty()) {
return QString();
}
return internal::CacheFileFolder() + qstr("/keywords/") + id;
}
[[nodiscard]] LangPackData ReadLocalCache(const QString &id) {
auto file = QFile(CacheFilePath(id));
if (!file.open(QIODevice::ReadOnly)) {
return {};
}
auto result = LangPackData();
2019-03-29 14:23:21 +00:00
auto stream = QDataStream(&file);
stream.setVersion(QDataStream::Qt_5_1);
auto version = qint32();
auto count = qint32();
stream
>> version
>> count;
if (version < 0 || count < 0 || stream.status() != QDataStream::Ok) {
return {};
}
for (auto i = 0; i != count; ++i) {
auto key = QString();
auto size = qint32();
stream
>> key
>> size;
if (size < 0 || stream.status() != QDataStream::Ok) {
return {};
}
auto &list = result.emoji[key];
for (auto j = 0; j != size; ++j) {
auto text = QString();
stream >> text;
if (stream.status() != QDataStream::Ok) {
return {};
}
const auto emoji = MustAddPostfix(text)
? (text + QChar(Ui::Emoji::kPostfix))
: text;
const auto entry = LangPackEmoji{ FindExact(emoji), text };
2019-03-29 14:23:21 +00:00
if (!entry.emoji) {
return {};
}
list.push_back(entry);
}
result.maxKeyLength = std::max(result.maxKeyLength, key.size());
}
result.version = version;
return result;
}
void WriteLocalCache(const QString &id, const LangPackData &data) {
2019-03-29 14:23:21 +00:00
if (!data.version && data.emoji.empty()) {
return;
}
CreateCacheFilePath();
auto file = QFile(CacheFilePath(id));
if (!file.open(QIODevice::WriteOnly)) {
return;
}
2019-03-29 14:23:21 +00:00
auto stream = QDataStream(&file);
stream.setVersion(QDataStream::Qt_5_1);
stream
<< qint32(data.version)
<< qint32(data.emoji.size());
for (const auto &[key, list] : data.emoji) {
stream
<< key
<< qint32(list.size());
for (const auto &emoji : list) {
stream << emoji.text;
}
}
}
[[nodiscard]] QString NormalizeQuery(const QString &query) {
2019-03-29 14:23:21 +00:00
return query.toLower();
}
[[nodiscard]] QString NormalizeKey(const QString &key) {
return key.toLower().trimmed();
}
void AppendFoundEmoji(
std::vector<Result> &result,
const QString &label,
const std::vector<LangPackEmoji> &list) {
// It is important that the 'result' won't relocate while inserting.
result.reserve(result.size() + list.size());
const auto alreadyBegin = begin(result);
const auto alreadyEnd = alreadyBegin + result.size();
auto &&add = ranges::view::all(
list
) | ranges::view::filter([&](const LangPackEmoji &entry) {
const auto i = ranges::find(
alreadyBegin,
alreadyEnd,
entry.emoji,
&Result::emoji);
return (i == alreadyEnd);
}) | ranges::view::transform([&](const LangPackEmoji &entry) {
return Result{ entry.emoji, label, entry.text };
});
result.insert(end(result), add.begin(), add.end());
}
void AppendLegacySuggestions(
std::vector<Result> &result,
const QString &query) {
2019-03-29 13:51:18 +00:00
const auto badSuggestionChar = [](QChar ch) {
return (ch < 'a' || ch > 'z')
&& (ch < 'A' || ch > 'Z')
&& (ch < '0' || ch > '9')
&& (ch != '_')
&& (ch != '-')
&& (ch != '+');
};
if (ranges::any_of(query, badSuggestionChar)) {
2019-03-29 13:51:18 +00:00
return;
}
const auto suggestions = GetSuggestions(QStringToUTF16(query));
// It is important that the 'result' won't relocate while inserting.
result.reserve(result.size() + suggestions.size());
const auto alreadyBegin = begin(result);
const auto alreadyEnd = alreadyBegin + result.size();
auto &&add = ranges::view::all(
suggestions
) | ranges::view::transform([](const Suggestion &suggestion) {
return Result{
Find(QStringFromUTF16(suggestion.emoji())),
QStringFromUTF16(suggestion.label()),
QStringFromUTF16(suggestion.replacement())
};
}) | ranges::view::filter([&](const Result &entry) {
const auto i = entry.emoji
? ranges::find(
alreadyBegin,
alreadyEnd,
entry.emoji,
&Result::emoji)
: alreadyEnd;
return (entry.emoji != nullptr)
&& (i == alreadyEnd);
});
result.insert(end(result), add.begin(), add.end());
}
void ApplyDifference(
LangPackData &data,
const QVector<MTPEmojiKeyword> &keywords,
int version) {
data.version = version;
for (const auto &keyword : keywords) {
keyword.match([&](const MTPDemojiKeyword &keyword) {
2019-07-05 13:38:38 +00:00
const auto word = NormalizeKey(qs(keyword.vkeyword()));
if (word.isEmpty()) {
return;
}
auto &list = data.emoji[word];
auto &&emoji = ranges::view::all(
2019-07-05 13:38:38 +00:00
keyword.vemoticons().v
) | ranges::view::transform([](const MTPstring &string) {
const auto text = qs(string);
const auto emoji = MustAddPostfix(text)
? (text + QChar(Ui::Emoji::kPostfix))
: text;
return LangPackEmoji{ FindExact(emoji), text };
}) | ranges::view::filter([&](const LangPackEmoji &entry) {
if (!entry.emoji) {
LOG(("API Warning: emoji %1 is not supported, word: %2."
).arg(entry.text
).arg(word));
}
return (entry.emoji != nullptr);
});
list.insert(end(list), emoji.begin(), emoji.end());
}, [&](const MTPDemojiKeywordDeleted &keyword) {
2019-07-05 13:38:38 +00:00
const auto word = NormalizeKey(qs(keyword.vkeyword()));
if (word.isEmpty()) {
return;
}
const auto i = data.emoji.find(word);
if (i == end(data.emoji)) {
return;
}
auto &list = i->second;
2019-07-05 13:38:38 +00:00
for (const auto &emoji : keyword.vemoticons().v) {
list.erase(
ranges::remove(list, qs(emoji), &LangPackEmoji::text),
end(list));
}
if (list.empty()) {
data.emoji.erase(i);
}
});
}
if (data.emoji.empty()) {
data.maxKeyLength = 0;
} else {
auto &&lengths = ranges::view::all(
data.emoji
) | ranges::view::transform([](auto &&pair) {
return pair.first.size();
});
data.maxKeyLength = *ranges::max_element(lengths);
}
}
} // namespace
class EmojiKeywords::LangPack final {
public:
using Delegate = details::EmojiKeywordsLangPackDelegate;
LangPack(not_null<Delegate*> delegate, const QString &id);
LangPack(const LangPack &other) = delete;
LangPack &operator=(const LangPack &other) = delete;
~LangPack();
2019-03-29 12:21:11 +00:00
[[nodiscard]] QString id() const;
void refresh();
void apiChanged();
[[nodiscard]] std::vector<Result> query(
const QString &normalized,
bool exact) const;
2019-03-29 13:51:18 +00:00
[[nodiscard]] int maxQueryLength() const;
private:
enum class State {
ReadingCache,
PendingRequest,
Requested,
Refreshed,
};
void readLocalCache();
void applyDifference(const MTPEmojiKeywordsDifference &result);
void applyData(LangPackData &&data);
not_null<Delegate*> _delegate;
QString _id;
State _state = State::ReadingCache;
LangPackData _data;
crl::time _lastRefreshTime = 0;
mtpRequestId _requestId = 0;
base::binary_guard _guard;
};
EmojiKeywords::LangPack::LangPack(
not_null<Delegate*> delegate,
const QString &id)
: _delegate(delegate)
, _id(id) {
readLocalCache();
}
EmojiKeywords::LangPack::~LangPack() {
if (_requestId) {
if (const auto api = _delegate->api()) {
api->request(_requestId).cancel();
}
}
}
void EmojiKeywords::LangPack::readLocalCache() {
const auto id = _id;
auto callback = crl::guard(_guard.make_guard(), [=](
LangPackData &&result) {
applyData(std::move(result));
refresh();
});
crl::async([id, callback = std::move(callback)]() mutable {
crl::on_main([
callback = std::move(callback),
result = ReadLocalCache(id)
]() mutable {
callback(std::move(result));
});
});
}
2019-03-29 12:21:11 +00:00
QString EmojiKeywords::LangPack::id() const {
return _id;
}
void EmojiKeywords::LangPack::refresh() {
if (_state != State::Refreshed) {
return;
} else if (_lastRefreshTime > 0
&& crl::now() - _lastRefreshTime < kRefreshEach) {
return;
}
const auto api = _delegate->api();
if (!api) {
_state = State::PendingRequest;
return;
}
_state = State::Requested;
const auto send = [&](auto &&request) {
return api->request(
std::move(request)
).done([=](const MTPEmojiKeywordsDifference &result) {
_requestId = 0;
_lastRefreshTime = crl::now();
applyDifference(result);
}).fail([=](const RPCError &error) {
_requestId = 0;
_lastRefreshTime = crl::now();
}).send();
};
_requestId = (_data.version > 0)
? send(MTPmessages_GetEmojiKeywordsDifference(
MTP_string(_id),
MTP_int(_data.version)))
: send(MTPmessages_GetEmojiKeywords(
MTP_string(_id)));
}
void EmojiKeywords::LangPack::applyDifference(
const MTPEmojiKeywordsDifference &result) {
result.match([&](const MTPDemojiKeywordsDifference &data) {
2019-07-05 13:38:38 +00:00
const auto code = qs(data.vlang_code());
const auto version = data.vversion().v;
const auto &keywords = data.vkeywords().v;
if (code != _id) {
LOG(("API Error: Bad lang_code for emoji keywords %1 -> %2"
).arg(_id
).arg(code));
_data.version = 0;
_state = State::Refreshed;
return;
2019-03-29 14:23:21 +00:00
} else if (keywords.isEmpty() && _data.version >= version) {
_state = State::Refreshed;
return;
}
2019-03-29 14:23:21 +00:00
const auto id = _id;
auto copy = _data;
auto callback = crl::guard(_guard.make_guard(), [=](
LangPackData &&result) {
applyData(std::move(result));
});
crl::async([=,
copy = std::move(copy),
callback = std::move(callback)]() mutable {
ApplyDifference(copy, keywords, version);
2019-03-29 14:23:21 +00:00
WriteLocalCache(id, copy);
crl::on_main([
result = std::move(copy),
callback = std::move(callback)
]() mutable {
callback(std::move(result));
});
});
});
}
void EmojiKeywords::LangPack::applyData(LangPackData &&data) {
_data = std::move(data);
_state = State::Refreshed;
_delegate->langPackRefreshed();
}
void EmojiKeywords::LangPack::apiChanged() {
if (_state == State::Requested && !_delegate->api()) {
_requestId = 0;
} else if (_state != State::PendingRequest) {
return;
}
_state = State::Refreshed;
refresh();
}
std::vector<Result> EmojiKeywords::LangPack::query(
const QString &normalized,
bool exact) const {
2019-04-01 10:19:36 +00:00
if (normalized.size() > _data.maxKeyLength
|| _data.emoji.empty()
|| (exact && SkipExactKeyword(_id, normalized))) {
return {};
}
const auto from = _data.emoji.lower_bound(normalized);
2019-10-02 09:54:29 +00:00
auto &&chosen = ranges::make_subrange(
from,
end(_data.emoji)
) | ranges::view::take_while([&](const auto &pair) {
const auto &key = pair.first;
return exact ? (key == normalized) : key.startsWith(normalized);
});
auto result = std::vector<Result>();
for (const auto &[key, list] : chosen) {
AppendFoundEmoji(result, key, list);
}
return result;
}
2019-03-29 13:51:18 +00:00
int EmojiKeywords::LangPack::maxQueryLength() const {
return _data.maxKeyLength;
}
EmojiKeywords::EmojiKeywords() {
crl::on_main(&_guard, [=] {
2019-07-24 11:45:24 +00:00
handleSessionChanges();
});
}
EmojiKeywords::~EmojiKeywords() = default;
not_null<details::EmojiKeywordsLangPackDelegate*> EmojiKeywords::delegate() {
return static_cast<details::EmojiKeywordsLangPackDelegate*>(this);
}
ApiWrap *EmojiKeywords::api() {
return _api;
}
void EmojiKeywords::langPackRefreshed() {
_refreshed.fire({});
}
2019-07-24 11:45:24 +00:00
void EmojiKeywords::handleSessionChanges() {
Core::App().domain().activeSessionValue( // #TODO multi someSessionValue
2019-07-24 11:45:24 +00:00
) | rpl::map([](Main::Session *session) {
return session ? &session->api() : nullptr;
}) | rpl::start_with_next([=](ApiWrap *api) {
apiChanged(api);
}, _lifetime);
}
void EmojiKeywords::apiChanged(ApiWrap *api) {
_api = api;
if (_api) {
2020-06-08 09:06:50 +00:00
crl::on_main(&_api->session(), crl::guard(&_guard, [=] {
2019-03-29 12:21:11 +00:00
base::ObservableViewer(
Lang::CurrentCloudManager().firstLanguageSuggestion()
) | rpl::filter([=] {
// Refresh with the suggested language if we already were asked.
return !_data.empty();
}) | rpl::start_with_next([=] {
refresh();
}, _suggestedChangeLifetime);
}));
} else {
2019-03-29 11:25:10 +00:00
_langsRequestId = 0;
_suggestedChangeLifetime.destroy();
}
for (const auto &[language, item] : _data) {
item->apiChanged();
}
}
void EmojiKeywords::refresh() {
2019-03-29 14:23:15 +00:00
auto list = languages();
if (_localList != list) {
_localList = std::move(list);
refreshRemoteList();
} else {
refreshFromRemoteList();
}
}
2019-03-29 14:23:15 +00:00
std::vector<QString> EmojiKeywords::languages() {
if (!_api) {
2019-03-29 14:23:15 +00:00
return {};
}
refreshInputLanguages();
auto result = std::vector<QString>();
const auto yield = [&](const QString &language) {
result.push_back(language);
};
const auto yieldList = [&](const QStringList &list) {
result.insert(end(result), list.begin(), list.end());
};
2020-09-30 09:11:44 +00:00
yield(Lang::Id());
2019-03-29 14:23:15 +00:00
yield(Lang::DefaultLanguageId());
yield(Lang::CurrentCloudManager().suggestedLanguage());
yield(Platform::SystemLanguage());
yieldList(QLocale::system().uiLanguages());
for (const auto &list : _inputLanguages) {
yieldList(list);
}
ranges::sort(result);
return result;
}
void EmojiKeywords::refreshInputLanguages() {
const auto method = QGuiApplication::inputMethod();
if (!method) {
return;
}
const auto list = method->locale().uiLanguages();
const auto i = ranges::find(_inputLanguages, list);
if (i != end(_inputLanguages)) {
std::rotate(i, i + 1, end(_inputLanguages));
} else {
if (_inputLanguages.size() >= kKeepNotUsedInputLanguagesCount) {
_inputLanguages.pop_front();
}
_inputLanguages.push_back(list);
}
}
rpl::producer<> EmojiKeywords::refreshed() const {
return _refreshed.events();
}
std::vector<Result> EmojiKeywords::query(
const QString &query,
bool exact) const {
const auto normalized = NormalizeQuery(query);
2019-03-29 12:21:11 +00:00
if (normalized.isEmpty()) {
return {};
}
auto result = std::vector<Result>();
for (const auto &[language, item] : _data) {
const auto list = item->query(normalized, exact);
// It is important that the 'result' won't relocate while inserting.
result.reserve(result.size() + list.size());
const auto alreadyBegin = begin(result);
const auto alreadyEnd = alreadyBegin + result.size();
auto &&add = ranges::view::all(
list
) | ranges::view::filter([&](Result entry) {
// In each item->query() result the list has no duplicates.
// So we need to check only for duplicates between queries.
const auto i = ranges::find(
alreadyBegin,
alreadyEnd,
entry.emoji,
&Result::emoji);
return (i == alreadyEnd);
});
result.insert(end(result), add.begin(), add.end());
}
if (!exact) {
AppendLegacySuggestions(result, query);
}
return result;
}
2019-03-29 13:51:18 +00:00
int EmojiKeywords::maxQueryLength() const {
if (_data.empty()) {
return 0;
}
auto &&lengths = _data | ranges::view::transform([](const auto &pair) {
return pair.second->maxQueryLength();
});
return *ranges::max_element(lengths);
}
void EmojiKeywords::refreshRemoteList() {
if (!_api) {
_localList.clear();
setRemoteList({});
return;
}
2019-03-29 11:25:10 +00:00
_api->request(base::take(_langsRequestId)).cancel();
auto languages = QVector<MTPstring>();
for (const auto &id : _localList) {
languages.push_back(MTP_string(id));
}
_langsRequestId = _api->request(MTPmessages_GetEmojiKeywordsLanguages(
MTP_vector<MTPstring>(languages)
)).done([=](const MTPVector<MTPEmojiLanguage> &result) {
setRemoteList(ranges::view::all(
result.v
) | ranges::view::transform([](const MTPEmojiLanguage &language) {
return language.match([&](const MTPDemojiLanguage &language) {
2019-07-05 13:38:38 +00:00
return qs(language.vlang_code());
2019-03-29 11:25:10 +00:00
});
}) | ranges::to_vector);
_langsRequestId = 0;
}).fail([=](const RPCError &error) {
_langsRequestId = 0;
}).send();
}
void EmojiKeywords::setRemoteList(std::vector<QString> &&list) {
if (_remoteList == list) {
return;
}
_remoteList = std::move(list);
for (auto i = begin(_data); i != end(_data);) {
if (ranges::find(_remoteList, i->first) != end(_remoteList)) {
++i;
} else {
2019-03-29 14:23:15 +00:00
if (_notUsedData.size() >= kKeepNotUsedLangPacksCount) {
2019-03-29 12:21:11 +00:00
_notUsedData.pop_front();
}
_notUsedData.push_back(std::move(i->second));
i = _data.erase(i);
}
}
refreshFromRemoteList();
}
void EmojiKeywords::refreshFromRemoteList() {
for (const auto &id : _remoteList) {
if (const auto i = _data.find(id); i != end(_data)) {
i->second->refresh();
2019-03-29 12:21:11 +00:00
continue;
}
const auto i = ranges::find(
_notUsedData,
id,
[](const std::unique_ptr<LangPack> &p) { return p->id(); });
if (i != end(_notUsedData)) {
_data.emplace(id, std::move(*i));
_notUsedData.erase(i);
} else {
_data.emplace(
id,
std::make_unique<LangPack>(delegate(), id));
}
}
}
} // namespace ChatHelpers