Use serverside keywords for emoji suggestions.

Telegram/SourceFiles/chat_helpers/emoji_keywords.cpp

@@ -0,0 +1,520 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "chat_helpers/emoji_keywords.h"
+
+#include "chat_helpers/emoji_suggestions_helper.h"
+#include "lang/lang_instance.h"
+#include "lang/lang_cloud_manager.h"
+#include "core/application.h"
+#include "platform/platform_specific.h"
+#include "ui/emoji_config.h"
+#include "auth_session.h"
+#include "apiwrap.h"
+
+namespace ChatHelpers {
+namespace {
+
+constexpr auto kRefreshEach = 60 * 60 * crl::time(1000); // 1 hour.
+
+using namespace Ui::Emoji;
+
+using Result = EmojiKeywords::Result;
+
+struct LangPackEmoji {
+	EmojiPtr emoji = nullptr;
+	QString text;
+};
+
+struct LangPackData {
+	int version = 0;
+	int maxKeyLength = 0;
+	std::map<QString, std::vector<LangPackEmoji>> emoji;
+};
+
+[[nodiscard]] bool MustAddPostfix(const QString &text) {
+	if (text.size() != 1) {
+		return false;
+	}
+	const auto code = text[0].unicode();
+	return (code == 0x2122U) || (code == 0xA9U) || (code == 0xAEU);
+}
+
+[[nodiscard]] std::vector<QString> KeywordLanguages() {
+	if (!AuthSession::Exists()) {
+		return {};
+	}
+	auto result = std::vector<QString>();
+	const auto yield = [&](const QString &language) {
+		result.push_back(language);
+	};
+	const auto yieldLocale = [&](const QLocale &locale) {
+		for (const auto &language : locale.uiLanguages()) {
+			yield(language);
+		}
+	};
+	yield(Lang::Current().id());
+	yield(Lang::DefaultLanguageId());
+	yield(Lang::CurrentCloudManager().suggestedLanguage());
+	yield(Platform::SystemLanguage());
+	if (const auto method = QGuiApplication::inputMethod()) {
+		yieldLocale(method->locale());
+	}
+	yieldLocale(QLocale::system());
+	return result;
+}
+
+[[nodiscard]] QString CacheFilePath(QString id) {
+	static const auto BadSymbols = QRegularExpression("[^a-zA-Z0-9_\\.\\-]");
+	id.replace(BadSymbols, QString());
+	if (id.isEmpty()) {
+		return QString();
+	}
+	return internal::CacheFileFolder() + qstr("/keywords/") + id;
+}
+
+[[nodiscard]] LangPackData ReadLocalCache(const QString &id) {
+	auto file = QFile(CacheFilePath(id));
+	if (!file.open(QIODevice::ReadOnly)) {
+		return {};
+	}
+	// #TODO emoji
+	auto result = LangPackData();
+	return result;
+}
+
+void WriteLocalCache(const QString &id, const LangPackData &data) {
+	auto file = QFile(CacheFilePath(id));
+	if (!file.open(QIODevice::WriteOnly)) {
+		return;
+	}
+
+}
+
+[[nodiscard]] QString NormalizeQuery(const QString &query) {
+	return query.toLower().trimmed();
+}
+
+void AppendFoundEmoji(
+		std::vector<Result> &result,
+		const QString &label,
+		const std::vector<LangPackEmoji> &list) {
+	auto &&add = ranges::view::all(
+		list
+	) | ranges::view::filter([&](const LangPackEmoji &entry) {
+		const auto i = ranges::find(result, entry.emoji, &Result::emoji);
+		return (i == end(result));
+	}) | ranges::view::transform([&](const LangPackEmoji &entry) {
+		return Result{ entry.emoji, label, entry.text };
+	});
+	result.insert(end(result), add.begin(), add.end());
+}
+
+void AppendLegacySuggestions(
+		std::vector<Result> &result,
+		const QString &query) {
+	const auto suggestions = GetSuggestions(QStringToUTF16(query));
+	auto &&add = ranges::view::all(
+		suggestions
+	) | ranges::view::transform([](const Suggestion &suggestion) {
+		return Result{
+			Find(QStringFromUTF16(suggestion.emoji())),
+			QStringFromUTF16(suggestion.label()),
+			QStringFromUTF16(suggestion.replacement())
+		};
+	}) | ranges::view::filter([&](const Result &entry) {
+		const auto i = entry.emoji
+			? ranges::find(result, entry.emoji, &Result::emoji)
+			: end(result);
+		return (entry.emoji != nullptr)
+			&& (i == end(result));
+	});
+	result.insert(end(result), add.begin(), add.end());
+}
+
+void ApplyDifference(
+		LangPackData &data,
+		const QVector<MTPEmojiKeyword> &keywords,
+		int version) {
+	data.version = version;
+	for (const auto &keyword : keywords) {
+		keyword.match([&](const MTPDemojiKeyword &keyword) {
+			const auto word = NormalizeQuery(qs(keyword.vkeyword));
+			if (word.isEmpty()) {
+				return;
+			}
+			auto &list = data.emoji[word];
+			auto &&emoji = ranges::view::all(
+				keyword.vemoticons.v
+			) | ranges::view::transform([](const MTPstring &string) {
+				const auto text = qs(string);
+				const auto emoji = MustAddPostfix(text)
+					? (text + QChar(Ui::Emoji::kPostfix))
+					: text;
+				return LangPackEmoji{ Find(emoji), text };
+			}) | ranges::view::filter([&](const LangPackEmoji &entry) {
+				if (!entry.emoji) {
+					LOG(("API Warning: emoji %1 is not supported, word: %2."
+						).arg(entry.text
+						).arg(word));
+				}
+				return (entry.emoji != nullptr);
+			});
+			list.insert(end(list), emoji.begin(), emoji.end());
+		}, [&](const MTPDemojiKeywordDeleted &keyword) {
+			const auto word = NormalizeQuery(qs(keyword.vkeyword));
+			if (word.isEmpty()) {
+				return;
+			}
+			const auto i = data.emoji.find(word);
+			if (i == end(data.emoji)) {
+				return;
+			}
+			auto &list = i->second;
+			for (const auto &emoji : keyword.vemoticons.v) {
+				list.erase(
+					ranges::remove(list, qs(emoji), &LangPackEmoji::text),
+					end(list));
+			}
+			if (list.empty()) {
+				data.emoji.erase(i);
+			}
+		});
+	}
+	if (data.emoji.empty()) {
+		data.maxKeyLength = 0;
+	} else {
+		auto &&lengths = ranges::view::all(
+			data.emoji
+		) | ranges::view::transform([](auto &&pair) {
+			return pair.first.size();
+		});
+		data.maxKeyLength = *ranges::max_element(lengths);
+	}
+}
+
+} // namespace
+
+class EmojiKeywords::LangPack final {
+public:
+	using Delegate = details::EmojiKeywordsLangPackDelegate;
+
+	LangPack(not_null<Delegate*> delegate, const QString &id);
+	LangPack(const LangPack &other) = delete;
+	LangPack &operator=(const LangPack &other) = delete;
+	~LangPack();
+
+	void refresh();
+	void apiChanged();
+
+	[[nodiscard]] std::vector<Result> query(
+		const QString &normalized,
+		bool exact) const;
+
+private:
+	enum class State {
+		ReadingCache,
+		PendingRequest,
+		Requested,
+		Refreshed,
+	};
+
+	void readLocalCache();
+	void applyDifference(const MTPEmojiKeywordsDifference &result);
+	void applyData(LangPackData &&data);
+
+	not_null<Delegate*> _delegate;
+	QString _id;
+	State _state = State::ReadingCache;
+	LangPackData _data;
+	crl::time _lastRefreshTime = 0;
+	mtpRequestId _requestId = 0;
+	base::binary_guard _guard;
+
+};
+
+EmojiKeywords::LangPack::LangPack(
+	not_null<Delegate*> delegate,
+	const QString &id)
+: _delegate(delegate)
+, _id(id) {
+	readLocalCache();
+}
+
+EmojiKeywords::LangPack::~LangPack() {
+	if (_requestId) {
+		if (const auto api = _delegate->api()) {
+			api->request(_requestId).cancel();
+		}
+	}
+}
+
+void EmojiKeywords::LangPack::readLocalCache() {
+	const auto id = _id;
+	auto callback = crl::guard(_guard.make_guard(), [=](
+			LangPackData &&result) {
+		applyData(std::move(result));
+		refresh();
+	});
+	crl::async([id, callback = std::move(callback)]() mutable {
+		crl::on_main([
+			callback = std::move(callback),
+			result = ReadLocalCache(id)
+		]() mutable {
+			callback(std::move(result));
+		});
+	});
+}
+
+void EmojiKeywords::LangPack::refresh() {
+	if (_state != State::Refreshed) {
+		return;
+	} else if (_lastRefreshTime > 0
+		&& crl::now() - _lastRefreshTime < kRefreshEach) {
+		return;
+	}
+	const auto api = _delegate->api();
+	if (!api) {
+		_state = State::PendingRequest;
+		return;
+	}
+	_state = State::Requested;
+	const auto send = [&](auto &&request) {
+		return api->request(
+			std::move(request)
+		).done([=](const MTPEmojiKeywordsDifference &result) {
+			_requestId = 0;
+			_lastRefreshTime = crl::now();
+			applyDifference(result);
+		}).fail([=](const RPCError &error) {
+			_requestId = 0;
+			_lastRefreshTime = crl::now();
+		}).send();
+	};
+	_requestId = (_data.version > 0)
+		? send(MTPmessages_GetEmojiKeywordsDifference(
+			MTP_string(_id),
+			MTP_int(_data.version)))
+		: send(MTPmessages_GetEmojiKeywords(
+			MTP_string(_id)));
+}
+
+void EmojiKeywords::LangPack::applyDifference(
+		const MTPEmojiKeywordsDifference &result) {
+	result.match([&](const MTPDemojiKeywordsDifference &data) {
+		const auto code = qs(data.vlang_code);
+		const auto version = data.vversion.v;
+		const auto &keywords = data.vkeywords.v;
+		if (code != _id) {
+			LOG(("API Error: Bad lang_code for emoji keywords %1 -> %2"
+				).arg(_id
+				).arg(code));
+			_data.version = 0;
+			_state = State::Refreshed;
+			return;
+		} else if (keywords.isEmpty()) {
+			if (_data.version < version) {
+				auto moved = std::move(_data);
+				moved.version = version;
+				applyData(std::move(moved));
+			} else {
+				_state = State::Refreshed;
+			}
+			return;
+		}
+		auto callback = crl::guard(_guard.make_guard(), [=](
+				LangPackData &&result) {
+			applyData(std::move(result));
+		});
+		auto copy = _data;
+		crl::async([=, callback = std::move(callback)]() mutable {
+			ApplyDifference(copy, keywords, version);
+			crl::on_main([
+				result = std::move(copy),
+				callback = std::move(callback)
+			]() mutable {
+				callback(std::move(result));
+			});
+		});
+	});
+}
+
+void EmojiKeywords::LangPack::applyData(LangPackData &&data) {
+	_data = std::move(data);
+	_state = State::Refreshed;
+	_delegate->langPackRefreshed();
+}
+
+void EmojiKeywords::LangPack::apiChanged() {
+	if (_state == State::Requested && !_delegate->api()) {
+		_requestId = 0;
+	} else if (_state != State::PendingRequest) {
+		return;
+	}
+	_state = State::Refreshed;
+	refresh();
+}
+
+std::vector<Result> EmojiKeywords::LangPack::query(
+		const QString &normalized,
+		bool exact) const {
+	if (normalized.size() > _data.maxKeyLength || _data.emoji.empty()) {
+		return {};
+	}
+
+	const auto from = _data.emoji.lower_bound(normalized);
+	auto &&chosen = ranges::make_iterator_range(
+		from,
+		end(_data.emoji)
+	) | ranges::view::take_while([&](const auto &pair) {
+		const auto &key = pair.first;
+		return exact ? (key == normalized) : key.startsWith(normalized);
+	});
+
+	auto result = std::vector<Result>();
+	for (const auto &[key, list] : chosen) {
+		AppendFoundEmoji(result, key, list);
+	}
+	return result;
+}
+
+EmojiKeywords::EmojiKeywords() {
+	crl::on_main(&_guard, [=] {
+		handleAuthSessionChanges();
+	});
+}
+
+EmojiKeywords::~EmojiKeywords() = default;
+
+not_null<details::EmojiKeywordsLangPackDelegate*> EmojiKeywords::delegate() {
+	return static_cast<details::EmojiKeywordsLangPackDelegate*>(this);
+}
+
+ApiWrap *EmojiKeywords::api() {
+	return _api;
+}
+
+void EmojiKeywords::langPackRefreshed() {
+	_refreshed.fire({});
+}
+
+void EmojiKeywords::handleAuthSessionChanges() {
+	rpl::single(
+		rpl::empty_value()
+	) | rpl::then(base::ObservableViewer(
+		Core::App().authSessionChanged()
+	)) | rpl::map([] {
+		return AuthSession::Exists() ? &Auth().api() : nullptr;
+	}) | rpl::start_with_next([=](ApiWrap *api) {
+		apiChanged(api);
+	}, _lifetime);
+}
+
+void EmojiKeywords::apiChanged(ApiWrap *api) {
+	_api = api;
+	if (_api) {
+		base::ObservableViewer(
+			Lang::CurrentCloudManager().firstLanguageSuggestion()
+		) | rpl::filter([=] {
+			// Refresh with the suggested language if we already were asked.
+			return !_data.empty();
+		}) | rpl::start_with_next([=] {
+			refresh();
+		}, _suggestedChangeLifetime);
+	} else {
+		_remoteListRequestId = 0;
+		_suggestedChangeLifetime.destroy();
+	}
+	for (const auto &[language, item] : _data) {
+		item->apiChanged();
+	}
+}
+
+void EmojiKeywords::refresh() {
+	auto list = KeywordLanguages();
+	if (_localList != list) {
+		_localList = std::move(list);
+		refreshRemoteList();
+	} else {
+		refreshFromRemoteList();
+	}
+}
+
+rpl::producer<> EmojiKeywords::refreshed() const {
+	return _refreshed.events();
+}
+
+std::vector<Result> EmojiKeywords::query(
+		const QString &query,
+		bool exact) const {
+	const auto normalized = NormalizeQuery(query);
+	auto result = std::vector<Result>();
+	for (const auto &[language, item] : _data) {
+		const auto oldcount = result.size();
+		const auto list = item->query(normalized, exact);
+		auto &&add = ranges::view::all(
+			list
+		) | ranges::view::filter([&](Result entry) {
+			// In each item->query() result the list has no duplicates.
+			// So we need to check only for duplicates between queries.
+			const auto oldbegin = begin(result);
+			const auto oldend = oldbegin + oldcount;
+			const auto i = ranges::find(
+				oldbegin,
+				oldend,
+				entry.emoji,
+				&Result::emoji);
+			return (i == oldend);
+		});
+		result.insert(end(result), add.begin(), add.end());
+	}
+	if (!exact) {
+		AppendLegacySuggestions(result, query);
+	}
+	return result;
+}
+
+void EmojiKeywords::refreshRemoteList() {
+	if (!_api) {
+		_localList.clear();
+		setRemoteList({});
+		return;
+	}
+	_api->request(base::take(_remoteListRequestId)).cancel();
+	//_remoteListRequestId = _api->request() // #TODO emoji
+	setRemoteList(base::duplicate(_localList));
+	auto list = _localList;
+}
+
+void EmojiKeywords::setRemoteList(std::vector<QString> &&list) {
+	if (_remoteList == list) {
+		return;
+	}
+	_remoteList = std::move(list);
+	for (auto i = begin(_data); i != end(_data);) {
+		if (ranges::find(_remoteList, i->first) != end(_remoteList)) {
+			++i;
+		} else {
+			i = _data.erase(i);
+		}
+	}
+	refreshFromRemoteList();
+}
+
+void EmojiKeywords::refreshFromRemoteList() {
+	for (const auto &id : _remoteList) {
+		if (const auto i = _data.find(id); i != end(_data)) {
+			i->second->refresh();
+		} else {
+			_data.emplace(
+				id,
+				std::make_unique<LangPack>(delegate(), id));
+		}
+	}
+}
+
+} // namespace ChatHelpers

Telegram/SourceFiles/chat_helpers/emoji_keywords.h

@@ -0,0 +1,74 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+class ApiWrap;
+
+namespace ChatHelpers {
+namespace details {
+
+class EmojiKeywordsLangPackDelegate {
+public:
+	virtual ApiWrap *api() = 0;
+	virtual void langPackRefreshed() = 0;
+
+protected:
+	~EmojiKeywordsLangPackDelegate() = default;
+
+};
+
+} // namespace details
+
+class EmojiKeywords final : private details::EmojiKeywordsLangPackDelegate {
+public:
+	EmojiKeywords();
+	EmojiKeywords(const EmojiKeywords &other) = delete;
+	EmojiKeywords &operator=(const EmojiKeywords &other) = delete;
+	~EmojiKeywords();
+
+	void refresh();
+
+	[[nodiscard]] rpl::producer<> refreshed() const;
+
+	struct Result {
+		EmojiPtr emoji = nullptr;
+		QString label;
+		QString replacement;
+	};
+	[[nodiscard]] std::vector<Result> query(
+		const QString &query,
+		bool exact = false) const;
+
+private:
+	class LangPack;
+
+	not_null<details::EmojiKeywordsLangPackDelegate*> delegate();
+	ApiWrap *api() override;
+	void langPackRefreshed() override;
+
+	void handleAuthSessionChanges();
+	void apiChanged(ApiWrap *api);
+	void refreshRemoteList();
+	void setRemoteList(std::vector<QString> &&list);
+	void refreshFromRemoteList();
+
+	ApiWrap *_api = nullptr;
+	std::vector<QString> _localList;
+	std::vector<QString> _remoteList;
+	mtpRequestId _remoteListRequestId = 0;
+	base::flat_map<QString, std::unique_ptr<LangPack>> _data;
+	rpl::event_stream<> _refreshed;
+
+	rpl::lifetime _suggestedChangeLifetime;
+
+	rpl::lifetime _lifetime;
+	base::has_weak_ptr _guard;
+
+};
+
+} // namespace ChatHelpers

Telegram/SourceFiles/chat_helpers/emoji_suggestions_widget.cpp

@@ -7,6 +7,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 */
 #include "chat_helpers/emoji_suggestions_widget.h"
 
+#include "chat_helpers/emoji_keywords.h"
 #include "chat_helpers/emoji_suggestions_helper.h"
 #include "ui/effects/ripple_animation.h"
 #include "ui/widgets/shadow.h"
@@ -14,6 +15,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #include "ui/widgets/input_fields.h"
 #include "ui/emoji_config.h"
 #include "platform/platform_specific.h"
+#include "core/application.h"
 #include "core/event_filter.h"
 #include "styles/style_chat_helpers.h"
 
@@ -109,53 +111,58 @@ std::vector<SuggestionsWidget::Row> SuggestionsWidget::getRowsByQuery() const {
 	if (_query.isEmpty()) {
 		return result;
 	}
-	auto suggestions = GetSuggestions(QStringToUTF16(_query));
+	auto suggestions = std::vector<Row>();
+	const auto results = Core::App().emojiKeywords().query(_query.mid(1));
+	for (const auto &result : results) {
+		suggestions.emplace_back(
+			result.emoji,
+			result.label,
+			result.replacement);
+	}
 	if (suggestions.empty()) {
 		return result;
 	}
-	auto count = suggestions.size();
-	auto suggestionsEmoji = std::vector<EmojiPtr>(count, nullptr);
-	for (auto i = 0; i != count; ++i) {
-		suggestionsEmoji[i] = Find(QStringFromUTF16(suggestions[i].emoji()));
-	}
 	auto recents = 0;
-	auto &recent = GetRecent();
-	for (auto &item : recent) {
-		auto emoji = item.first->original();
-		if (!emoji) emoji = item.first;
-		auto it = std::find(suggestionsEmoji.begin(), suggestionsEmoji.end(), emoji);
-		if (it != suggestionsEmoji.end()) {
-			auto index = (it - suggestionsEmoji.begin());
-			if (index >= recents) {
-				if (index > recents) {
-					auto recentEmoji = suggestionsEmoji[index];
-					auto recentSuggestion = suggestions[index];
-					for (auto i = index; i != recents; --i) {
-						suggestionsEmoji[i] = suggestionsEmoji[i - 1];
-						suggestions[i] = suggestions[i - 1];
-					}
-					suggestionsEmoji[recents] = recentEmoji;
-					suggestions[recents] = recentSuggestion;
-				}
-				++recents;
-			}
+	const auto &recent = GetRecent();
+	for (const auto &item : recent) {
+		const auto emoji = item.first->original()
+			? item.first->original()
+			: item.first;
+		const auto it = ranges::find(suggestions, emoji, [](const Row &row) {
+			return row.emoji().get();
+		});
+		if (it == end(suggestions)) {
+			continue;
 		}
+		const auto index = (it - begin(suggestions));
+		if (index < recents) {
+			continue;
+		} else if (index > recents) {
+			auto recentSuggestion = std::move(suggestions[index]);
+			for (auto i = index; i != recents; --i) {
+				suggestions[i] = std::move(suggestions[i - 1]);
+			}
+			suggestions[recents] = std::move(recentSuggestion);
+		}
+		++recents;
 	}
 
 	result.reserve(kRowLimit);
 	auto index = 0;
-	for (auto &item : suggestions) {
-		if (auto emoji = suggestionsEmoji[index++]) {
-			if (emoji->hasVariants()) {
-				auto it = cEmojiVariants().constFind(emoji->nonColoredId());
-				if (it != cEmojiVariants().cend()) {
-					emoji = emoji->variant(it.value());
-				}
-			}
-			result.emplace_back(emoji, QStringFromUTF16(item.label()), QStringFromUTF16(item.replacement()));
-			if (result.size() == kRowLimit) {
-				break;
-			}
+	for (const auto &item : suggestions) {
+		const auto emoji = [&] {
+			const auto result = item.emoji();
+			const auto &variants = cEmojiVariants();
+			const auto i = result->hasVariants()
+				? variants.constFind(result->nonColoredId())
+				: variants.cend();
+			return (i != variants.cend())
+				? result->variant(i.value())
+				: result.get();
+		}();
+		result.emplace_back(emoji, item.label(), item.replacement());
+		if (result.size() == kRowLimit) {
+			break;
 		}
 	}
 	return result;
@@ -450,6 +457,10 @@ void SuggestionsController::setReplaceCallback(
 }
 
 void SuggestionsController::handleTextChange() {
+	if (Global::SuggestEmoji() && _field->textCursor().position() > 0) {
+		Core::App().emojiKeywords().refresh();
+	}
+
 	_ignoreCursorPositionChange = true;
 	InvokeQueued(_container, [=] { _ignoreCursorPositionChange = false; });
 

Telegram/SourceFiles/codegen/emoji/data.cpp

@@ -1920,9 +1920,20 @@ std::map<InputId, InputId> FlagAliases = {
  { { 0xD83CDDE8U, 0xD83CDDF5U, }, { 0xD83CDDEBU, 0xD83CDDF7U, } },
  { { 0xD83CDDE7U, 0xD83CDDFBU, }, { 0xD83CDDF3U, 0xD83CDDF4U, } },
  { { 0xD83CDDE6U, 0xD83CDDE8U, }, { 0xD83CDDF8U, 0xD83CDDEDU, } },
+
+ // This is different flag, but macOS shows that glyph :(
+ { { 0xD83CDDE9U, 0xD83CDDECU, }, { 0xD83CDDEEU, 0xD83CDDF4U, } },
+
+ { { 0xD83CDDF9U, 0xD83CDDE6U, }, { 0xD83CDDF8U, 0xD83CDDEDU, } },
+ { { 0xD83CDDF2U, 0xD83CDDEBU, }, { 0xD83CDDEBU, 0xD83CDDF7U, } },
+ { { 0xD83CDDEAU, 0xD83CDDE6U, }, { 0xD83CDDEAU, 0xD83CDDF8U, } },
 };
 
-std::map<Id, Id> Aliases;
+std::map<Id, std::vector<Id>> Aliases; // original -> list of aliased
+
+void AddAlias(const Id &original, const Id &aliased) {
+	Aliases[original].push_back(aliased);
+}
 
 constexpr auto kErrorBadData = 401;
 
@@ -1983,7 +1994,7 @@ void appendCategory(
 		const InputCategory &category,
 		const set<Id> &variatedIds,
 		const set<Id> &postfixRequiredIds) {
-	result.categories.push_back(vector<int>());
+	result.categories.emplace_back();
 	for (auto &id : category) {
 		auto emoji = Emoji();
 		auto bareId = BareIdFromInput(id);
@@ -2013,7 +2024,14 @@ void appendCategory(
 			it = result.map.emplace(bareId, index).first;
 			result.list.push_back(move(emoji));
 			if (const auto a = Aliases.find(bareId); a != end(Aliases)) {
-				result.map.emplace(a->second, index);
+				for (const auto &alias : a->second) {
+					const auto ok = result.map.emplace(alias, index).second;
+					if (!ok) {
+						logDataError() << "some emoji alias already in the map.";
+						result = Data();
+						return;
+					}
+				}
 			}
 			if (postfixRequiredIds.find(bareId) != end(postfixRequiredIds)) {
 				result.postfixRequired.emplace(index);
@@ -2055,7 +2073,14 @@ void appendCategory(
 					it = result.map.emplace(bareColoredId, index).first;
 					result.list.push_back(move(colored));
 					if (const auto a = Aliases.find(bareColoredId); a != end(Aliases)) {
-						result.map.emplace(a->second, index);
+						for (const auto &alias : a->second) {
+							const auto ok = result.map.emplace(alias, index).second;
+							if (!ok) {
+								logDataError() << "some emoji alias already in the map.";
+								result = Data();
+								return;
+							}
+						}
 					}
 					if (postfixRequiredIds.find(bareColoredId) != end(postfixRequiredIds)) {
 						result.postfixRequired.emplace(index);
@@ -2172,7 +2197,7 @@ bool CheckOldInCurrent(std::set<Id> variatedIds) {
 		key[1] = color;
 		auto value = alias;
 		value[1] = color;
-		Aliases.emplace(BareIdFromInput(key), BareIdFromInput(value));
+		AddAlias(BareIdFromInput(key), BareIdFromInput(value));
 		return true;
 	};
 	auto result = true;
@@ -2239,12 +2264,8 @@ bool CheckOldInCurrent(std::set<Id> variatedIds) {
 					<< genderIndex
 					<< ".";
 				result = false;
-			} else if (Aliases.find(bare) != end(Aliases)) {
-				common::logError(kErrorBadData, "input")
-					<< "Bad data: two aliases for a gendered emoji.";
-				result = false;
 			} else {
-				Aliases.emplace(bare, BareIdFromInput(*i));
+				AddAlias(bare, BareIdFromInput(*i));
 			}
 		}
 	}
@@ -2286,10 +2307,6 @@ bool CheckOldInCurrent(std::set<Id> variatedIds) {
 				<< ".";
 			result = false;
 			continue;
-		} else if (Aliases.find(bare) != end(Aliases)) {
-			common::logError(kErrorBadData, "input")
-				<< "Bad data: two aliases for a gendered emoji.";
-			result = false;
 		} else {
 			for (const auto color : Colors) {
 				if (!emplaceColoredAlias(real, *i, color)) {
@@ -2321,12 +2338,8 @@ bool CheckOldInCurrent(std::set<Id> variatedIds) {
 			common::logError(kErrorBadData, "input")
 				<< "Bad data: without gender alias not found with gender.";
 			result = false;
-		} else if (Aliases.find(bare) != end(Aliases)) {
-			common::logError(kErrorBadData, "input")
-				<< "Bad data: two aliases for a gendered emoji.";
-			result = false;
 		} else {
-			Aliases.emplace(bare, BareIdFromInput(inputId));
+			AddAlias(bare, BareIdFromInput(inputId));
 		}
 		if (variatedIds.find(bare) != variatedIds.end()) {
 			auto colorReal = real;
@@ -2345,16 +2358,8 @@ bool CheckOldInCurrent(std::set<Id> variatedIds) {
 		}
 	}
 
-	for (const auto [inputId, real] : FlagAliases) {
-		const auto bare = BareIdFromInput(real);
-		if (Aliases.find(bare) != end(Aliases)) {
-			common::logError(kErrorBadData, "input")
-				<< "Bad data: two aliases for a flag emoji.";
-			result = false;
-		}
-		else {
-			Aliases.emplace(bare, BareIdFromInput(inputId));
-		}
+	for (const auto &[inputId, real] : FlagAliases) {
+		AddAlias(BareIdFromInput(real), BareIdFromInput(inputId));
 	}
 
 	return result;

Telegram/SourceFiles/core/application.cpp

@@ -18,6 +18,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #include "core/sandbox.h"
 #include "core/local_url_handlers.h"
 #include "core/launcher.h"
+#include "chat_helpers/emoji_keywords.h"
 #include "storage/localstorage.h"
 #include "platform/platform_specific.h"
 #include "mainwindow.h"
@@ -83,6 +84,7 @@ Application::Application(not_null<Launcher*> launcher)
 , _databases(std::make_unique<Storage::Databases>())
 , _animationsManager(std::make_unique<Ui::Animations::Manager>())
 , _langpack(std::make_unique<Lang::Instance>())
+, _emojiKeywords(std::make_unique<ChatHelpers::EmojiKeywords>())
 , _audio(std::make_unique<Media::Audio::Instance>())
 , _logo(Window::LoadLogo())
 , _logoNoMargin(Window::LoadLogoNoMargin()) {
@@ -93,6 +95,41 @@ Application::Application(not_null<Launcher*> launcher)
 	Instance = this;
 }
 
+Application::~Application() {
+	_window.reset();
+	_mediaView.reset();
+
+	// Some MTP requests can be cancelled from data clearing.
+	authSessionDestroy();
+
+	// The langpack manager should be destroyed before MTProto instance,
+	// because it is MTP::Sender and it may have pending requests.
+	_langCloudManager.reset();
+
+	_mtproto.reset();
+	_mtprotoForKeysDestroy.reset();
+
+	Shortcuts::Finish();
+
+	Ui::Emoji::Clear();
+
+	anim::stopManager();
+
+	stopWebLoadManager();
+	App::deinitMedia();
+
+	Window::Theme::Unload();
+
+	Media::Player::finish(_audio.get());
+	style::stopManager();
+
+	Local::finish();
+	Global::finish();
+	ThirdParty::finish();
+
+	Instance = nullptr;
+}
+
 void Application::run() {
 	Fonts::Start();
 
@@ -1127,41 +1164,6 @@ void Application::startShortcuts() {
 	}, _lifetime);
 }
 
-Application::~Application() {
-	_window.reset();
-	_mediaView.reset();
-
-	// Some MTP requests can be cancelled from data clearing.
-	authSessionDestroy();
-
-	// The langpack manager should be destroyed before MTProto instance,
-	// because it is MTP::Sender and it may have pending requests.
-	_langCloudManager.reset();
-
-	_mtproto.reset();
-	_mtprotoForKeysDestroy.reset();
-
-	Shortcuts::Finish();
-
-	Ui::Emoji::Clear();
-
-	anim::stopManager();
-
-	stopWebLoadManager();
-	App::deinitMedia();
-
-	Window::Theme::Unload();
-
-	Media::Player::finish(_audio.get());
-	style::stopManager();
-
-	Local::finish();
-	Global::finish();
-	ThirdParty::finish();
-
-	Instance = nullptr;
-}
-
 bool IsAppLaunched() {
 	return (Application::Instance != nullptr);
 }

Telegram/SourceFiles/core/application.h

@@ -26,6 +26,10 @@ namespace Window {
 struct TermsLock;
 } // namespace Window
 
+namespace ChatHelpers {
+class EmojiKeywords;
+} // namespace ChatHelpers
+
 namespace App {
 void quit();
 } // namespace App
@@ -67,9 +71,9 @@ struct LocalUrlHandler;
 class Application final : public QObject, private base::Subscriber {
 public:
 	Application(not_null<Launcher*> launcher);
-
 	Application(const Application &other) = delete;
 	Application &operator=(const Application &other) = delete;
+	~Application();
 
 	not_null<Launcher*> launcher() const {
 		return _launcher;
@@ -146,12 +150,6 @@ public:
 	AuthSession *authSession() {
 		return _authSession.get();
 	}
-	Lang::Instance &langpack() {
-		return *_langpack;
-	}
-	Lang::CloudManager *langCloudManager() {
-		return _langCloudManager.get();
-	}
 	void authSessionCreate(const MTPUser &user);
 	base::Observable<void> &authSessionChanged() {
 		return _authSessionChanged;
@@ -165,6 +163,17 @@ public:
 		return *_audio;
 	}
 
+	// Langpack and emoji keywords.
+	Lang::Instance &langpack() {
+		return *_langpack;
+	}
+	Lang::CloudManager *langCloudManager() {
+		return _langCloudManager.get();
+	}
+	ChatHelpers::EmojiKeywords &emojiKeywords() {
+		return *_emojiKeywords;
+	}
+
 	// Internal links.
 	void setInternalLinkDomain(const QString &domain) const;
 	QString createInternalLink(const QString &query) const;
@@ -222,8 +231,6 @@ public:
 		_callDelayedTimer.call(duration, std::move(lambda));
 	}
 
-	~Application();
-
 protected:
 	bool eventFilter(QObject *object, QEvent *event) override;
 
@@ -264,6 +271,7 @@ private:
 	std::unique_ptr<Media::View::OverlayWidget> _mediaView;
 	const std::unique_ptr<Lang::Instance> _langpack;
 	std::unique_ptr<Lang::CloudManager> _langCloudManager;
+	const std::unique_ptr<ChatHelpers::EmojiKeywords> _emojiKeywords;
 	std::unique_ptr<Lang::Translator> _translator;
 	std::unique_ptr<MTP::DcOptions> _dcOptions;
 	std::unique_ptr<MTP::Instance> _mtproto;

Telegram/SourceFiles/mtproto/core_types.h

@@ -174,6 +174,9 @@ private:
 
 };
 
+struct ZeroFlagsHelper {
+};
+
 } // namespace internal
 } // namespace MTP
 
@@ -400,13 +403,6 @@ inline MTPint MTP_int(int32 v) {
 }
 using MTPInt = MTPBoxed<MTPint>;
 
-namespace internal {
-
-struct ZeroFlagsHelper {
-};
-
-} // namespace internal
-
 template <typename Flags>
 class MTPflags {
 public:
@@ -416,7 +412,7 @@ public:
 		"MTPflags are allowed only wrapping int32 flag types!");
 
 	MTPflags() = default;
-	MTPflags(internal::ZeroFlagsHelper helper) {
+	MTPflags(MTP::internal::ZeroFlagsHelper helper) {
 	}
 
 	uint32 innerLength() const {
@@ -456,8 +452,8 @@ inline MTPflags<base::flags<T>> MTP_flags(T v) {
 	return MTPflags<base::flags<T>>(v);
 }
 
-inline internal::ZeroFlagsHelper MTP_flags(void(internal::ZeroFlagsHelper::*)()) {
-	return internal::ZeroFlagsHelper();
+inline MTP::internal::ZeroFlagsHelper MTP_flags(void(MTP::internal::ZeroFlagsHelper::*)()) {
+	return MTP::internal::ZeroFlagsHelper();
 }
 
 template <typename Flags>

Telegram/SourceFiles/ui/emoji_config.cpp

@@ -101,23 +101,19 @@ int RowsCount(int index) {
 		+ ((count % kImagesPerRow) ? 1 : 0);
 }
 
-QString CacheFileFolder() {
-	return cWorkingDir() + "tdata/emoji";
-}
-
 QString CacheFileNameMask(int size) {
 	return "cache_" + QString::number(size) + '_';
 }
 
 QString CacheFilePath(int size, int index) {
-	return CacheFileFolder()
+	return internal::CacheFileFolder()
 		+ '/'
 		+ CacheFileNameMask(size)
 		+ QString::number(index);
 }
 
 QString CurrentSettingPath() {
-	return CacheFileFolder() + "/current";
+	return internal::CacheFileFolder() + "/current";
 }
 
 bool IsValidSetId(int id) {
@@ -188,7 +184,7 @@ void SaveToFile(int id, const QImage &image, int size, int index) {
 
 	QFile f(CacheFilePath(size, index));
 	if (!f.open(QIODevice::WriteOnly)) {
-		if (!QDir::current().mkpath(CacheFileFolder())
+		if (!QDir::current().mkpath(internal::CacheFileFolder())
 			|| !f.open(QIODevice::WriteOnly)) {
 			LOG(("App Error: Could not open emoji cache '%1' for size %2_%3"
 				).arg(f.fileName()
@@ -500,6 +496,10 @@ void ClearUniversalChecked() {
 
 namespace internal {
 
+QString CacheFileFolder() {
+	return cWorkingDir() + "tdata/emoji";
+}
+
 QString SetDataPath(int id) {
 	Expects(IsValidSetId(id) && id != 0);
 
@@ -536,7 +536,7 @@ void ClearIrrelevantCache() {
 	Expects(SizeLarge > 0);
 
 	crl::async([] {
-		const auto folder = CacheFileFolder();
+		const auto folder = internal::CacheFileFolder();
 		const auto list = QDir(folder).entryList(QDir::Files);
 		const auto good1 = CacheFileNameMask(SizeNormal);
 		const auto good2 = CacheFileNameMask(SizeLarge);

Telegram/SourceFiles/ui/emoji_config.h

@@ -14,6 +14,7 @@ namespace Ui {
 namespace Emoji {
 namespace internal {
 
+[[nodiscard]] QString CacheFileFolder();
 [[nodiscard]] QString SetDataPath(int id);
 
 } // namespace internal

Telegram/ThirdParty/emoji_suggestions/emoji_suggestions.cpp

@@ -372,15 +372,15 @@ int Completer::findEqualCharsCount(int position, const utf16string *word) {
 }
 
 std::vector<Suggestion> Completer::prepareResult() {
-	auto firstCharOfQuery = _query[0];
-	auto reorder = [&](auto &&predicate) {
+	const auto firstCharOfQuery = _query[0];
+	const auto reorder = [&](auto &&predicate) {
 		std::stable_partition(
 			std::begin(_result),
 			std::end(_result),
 			std::forward<decltype(predicate)>(predicate));
 	};
-	reorder([firstCharOfQuery](Result &result) {
-		auto firstCharAfterColon = result.replacement->replacement[1];
+	reorder([&](Result &result) {
+		const auto firstCharAfterColon = result.replacement->replacement[1];
 		return (firstCharAfterColon == firstCharOfQuery);
 	});
 	reorder([](Result &result) {

Telegram/gyp/telegram_sources.txt

@@ -94,6 +94,8 @@
 <(src_loc)/calls/calls_top_bar.h
 <(src_loc)/chat_helpers/bot_keyboard.cpp
 <(src_loc)/chat_helpers/bot_keyboard.h
+<(src_loc)/chat_helpers/emoji_keywords.cpp
+<(src_loc)/chat_helpers/emoji_keywords.h
 <(src_loc)/chat_helpers/emoji_list_widget.cpp
 <(src_loc)/chat_helpers/emoji_list_widget.h
 <(src_loc)/chat_helpers/emoji_sets_manager.cpp