tdesktop/Telegram/SourceFiles/chat_helpers/emoji_keywords.cpp

/*
This file is part of Telegram Desktop,
the official desktop application for the Telegram messaging service.

For license and copyright information please follow this link:
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#include "chat_helpers/emoji_keywords.h"

#include "chat_helpers/emoji_suggestions_helper.h"
#include "lang/lang_instance.h"
#include "lang/lang_cloud_manager.h"
#include "core/application.h"
#include "platform/platform_specific.h"
#include "ui/emoji_config.h"
#include "auth_session.h"
#include "apiwrap.h"

namespace ChatHelpers {
namespace {

constexpr auto kRefreshEach = 60 * 60 * crl::time(1000); // 1 hour.
constexpr auto kKeepNotUsedLangPacksCount = 10;

using namespace Ui::Emoji;

using Result = EmojiKeywords::Result;

struct LangPackEmoji {
	EmojiPtr emoji = nullptr;
	QString text;
};

struct LangPackData {
	int version = 0;
	int maxKeyLength = 0;
	std::map<QString, std::vector<LangPackEmoji>> emoji;
};

[[nodiscard]] bool MustAddPostfix(const QString &text) {
	if (text.size() != 1) {
		return false;
	}
	const auto code = text[0].unicode();
	return (code == 0x2122U) || (code == 0xA9U) || (code == 0xAEU);
}

[[nodiscard]] std::vector<QString> KeywordLanguages() {
	if (!AuthSession::Exists()) {
		return {};
	}
	auto result = std::vector<QString>();
	const auto yield = [&](const QString &language) {
		result.push_back(language);
	};
	const auto yieldLocale = [&](const QLocale &locale) {
		for (const auto &language : locale.uiLanguages()) {
			yield(language);
		}
	};
	yield(Lang::Current().id());
	yield(Lang::DefaultLanguageId());
	yield(Lang::CurrentCloudManager().suggestedLanguage());
	yield(Platform::SystemLanguage());
	yieldLocale(QLocale::system());
	if (const auto method = QGuiApplication::inputMethod()) {
		yieldLocale(method->locale());
	}
	return result;
}

[[nodiscard]] QString CacheFilePath(QString id) {
	static const auto BadSymbols = QRegularExpression("[^a-zA-Z0-9_\\.\\-]");
	id.replace(BadSymbols, QString());
	if (id.isEmpty()) {
		return QString();
	}
	return internal::CacheFileFolder() + qstr("/keywords/") + id;
}

[[nodiscard]] LangPackData ReadLocalCache(const QString &id) {
	auto file = QFile(CacheFilePath(id));
	if (!file.open(QIODevice::ReadOnly)) {
		return {};
	}
	// #TODO emoji
	auto result = LangPackData();
	return result;
}

void WriteLocalCache(const QString &id, const LangPackData &data) {
	auto file = QFile(CacheFilePath(id));
	if (!file.open(QIODevice::WriteOnly)) {
		return;
	}

}

[[nodiscard]] QString NormalizeQuery(const QString &query) {
	return query.toLower().trimmed();
}

void AppendFoundEmoji(
		std::vector<Result> &result,
		const QString &label,
		const std::vector<LangPackEmoji> &list) {
	auto &&add = ranges::view::all(
		list
	) | ranges::view::filter([&](const LangPackEmoji &entry) {
		const auto i = ranges::find(result, entry.emoji, &Result::emoji);
		return (i == end(result));
	}) | ranges::view::transform([&](const LangPackEmoji &entry) {
		return Result{ entry.emoji, label, entry.text };
	});
	result.insert(end(result), add.begin(), add.end());
}

void AppendLegacySuggestions(
		std::vector<Result> &result,
		const QString &query) {
	const auto badSuggestionChar = [](QChar ch) {
		return (ch < 'a' || ch > 'z')
			&& (ch < 'A' || ch > 'Z')
			&& (ch < '0' || ch > '9')
			&& (ch != '_')
			&& (ch != '-')
			&& (ch != '+');
	};
	if (ranges::find_if(query, badSuggestionChar) != query.end()) {
		return;
	}

	const auto suggestions = GetSuggestions(QStringToUTF16(query));
	auto &&add = ranges::view::all(
		suggestions
	) | ranges::view::transform([](const Suggestion &suggestion) {
		return Result{
			Find(QStringFromUTF16(suggestion.emoji())),
			QStringFromUTF16(suggestion.label()),
			QStringFromUTF16(suggestion.replacement())
		};
	}) | ranges::view::filter([&](const Result &entry) {
		const auto i = entry.emoji
			? ranges::find(result, entry.emoji, &Result::emoji)
			: end(result);
		return (entry.emoji != nullptr)
			&& (i == end(result));
	});
	result.insert(end(result), add.begin(), add.end());
}

void ApplyDifference(
		LangPackData &data,
		const QVector<MTPEmojiKeyword> &keywords,
		int version) {
	data.version = version;
	for (const auto &keyword : keywords) {
		keyword.match([&](const MTPDemojiKeyword &keyword) {
			const auto word = NormalizeQuery(qs(keyword.vkeyword));
			if (word.isEmpty()) {
				return;
			}
			auto &list = data.emoji[word];
			auto &&emoji = ranges::view::all(
				keyword.vemoticons.v
			) | ranges::view::transform([](const MTPstring &string) {
				const auto text = qs(string);
				const auto emoji = MustAddPostfix(text)
					? (text + QChar(Ui::Emoji::kPostfix))
					: text;
				return LangPackEmoji{ Find(emoji), text };
			}) | ranges::view::filter([&](const LangPackEmoji &entry) {
				if (!entry.emoji) {
					LOG(("API Warning: emoji %1 is not supported, word: %2."
						).arg(entry.text
						).arg(word));
				}
				return (entry.emoji != nullptr);
			});
			list.insert(end(list), emoji.begin(), emoji.end());
		}, [&](const MTPDemojiKeywordDeleted &keyword) {
			const auto word = NormalizeQuery(qs(keyword.vkeyword));
			if (word.isEmpty()) {
				return;
			}
			const auto i = data.emoji.find(word);
			if (i == end(data.emoji)) {
				return;
			}
			auto &list = i->second;
			for (const auto &emoji : keyword.vemoticons.v) {
				list.erase(
					ranges::remove(list, qs(emoji), &LangPackEmoji::text),
					end(list));
			}
			if (list.empty()) {
				data.emoji.erase(i);
			}
		});
	}
	if (data.emoji.empty()) {
		data.maxKeyLength = 0;
	} else {
		auto &&lengths = ranges::view::all(
			data.emoji
		) | ranges::view::transform([](auto &&pair) {
			return pair.first.size();
		});
		data.maxKeyLength = *ranges::max_element(lengths);
	}
}

} // namespace

class EmojiKeywords::LangPack final {
public:
	using Delegate = details::EmojiKeywordsLangPackDelegate;

	LangPack(not_null<Delegate*> delegate, const QString &id);
	LangPack(const LangPack &other) = delete;
	LangPack &operator=(const LangPack &other) = delete;
	~LangPack();

	[[nodiscard]] QString id() const;

	void refresh();
	void apiChanged();

	[[nodiscard]] std::vector<Result> query(
		const QString &normalized,
		bool exact) const;
	[[nodiscard]] int maxQueryLength() const;

private:
	enum class State {
		ReadingCache,
		PendingRequest,
		Requested,
		Refreshed,
	};

	void readLocalCache();
	void applyDifference(const MTPEmojiKeywordsDifference &result);
	void applyData(LangPackData &&data);

	not_null<Delegate*> _delegate;
	QString _id;
	State _state = State::ReadingCache;
	LangPackData _data;
	crl::time _lastRefreshTime = 0;
	mtpRequestId _requestId = 0;
	base::binary_guard _guard;

};

EmojiKeywords::LangPack::LangPack(
	not_null<Delegate*> delegate,
	const QString &id)
: _delegate(delegate)
, _id(id) {
	readLocalCache();
}

EmojiKeywords::LangPack::~LangPack() {
	if (_requestId) {
		if (const auto api = _delegate->api()) {
			api->request(_requestId).cancel();
		}
	}
}

void EmojiKeywords::LangPack::readLocalCache() {
	const auto id = _id;
	auto callback = crl::guard(_guard.make_guard(), [=](
			LangPackData &&result) {
		applyData(std::move(result));
		refresh();
	});
	crl::async([id, callback = std::move(callback)]() mutable {
		crl::on_main([
			callback = std::move(callback),
			result = ReadLocalCache(id)
		]() mutable {
			callback(std::move(result));
		});
	});
}

QString EmojiKeywords::LangPack::id() const {
	return _id;
}

void EmojiKeywords::LangPack::refresh() {
	if (_state != State::Refreshed) {
		return;
	} else if (_lastRefreshTime > 0
		&& crl::now() - _lastRefreshTime < kRefreshEach) {
		return;
	}
	const auto api = _delegate->api();
	if (!api) {
		_state = State::PendingRequest;
		return;
	}
	_state = State::Requested;
	const auto send = [&](auto &&request) {
		return api->request(
			std::move(request)
		).done([=](const MTPEmojiKeywordsDifference &result) {
			_requestId = 0;
			_lastRefreshTime = crl::now();
			applyDifference(result);
		}).fail([=](const RPCError &error) {
			_requestId = 0;
			_lastRefreshTime = crl::now();
		}).send();
	};
	_requestId = (_data.version > 0)
		? send(MTPmessages_GetEmojiKeywordsDifference(
			MTP_string(_id),
			MTP_int(_data.version)))
		: send(MTPmessages_GetEmojiKeywords(
			MTP_string(_id)));
}

void EmojiKeywords::LangPack::applyDifference(
		const MTPEmojiKeywordsDifference &result) {
	result.match([&](const MTPDemojiKeywordsDifference &data) {
		const auto code = qs(data.vlang_code);
		const auto version = data.vversion.v;
		const auto &keywords = data.vkeywords.v;
		if (code != _id) {
			LOG(("API Error: Bad lang_code for emoji keywords %1 -> %2"
				).arg(_id
				).arg(code));
			_data.version = 0;
			_state = State::Refreshed;
			return;
		} else if (keywords.isEmpty()) {
			if (_data.version < version) {
				auto moved = std::move(_data);
				moved.version = version;
				applyData(std::move(moved));
			} else {
				_state = State::Refreshed;
			}
			return;
		}
		auto callback = crl::guard(_guard.make_guard(), [=](
				LangPackData &&result) {
			applyData(std::move(result));
		});
		auto copy = _data;
		crl::async([=, callback = std::move(callback)]() mutable {
			ApplyDifference(copy, keywords, version);
			crl::on_main([
				result = std::move(copy),
				callback = std::move(callback)
			]() mutable {
				callback(std::move(result));
			});
		});
	});
}

void EmojiKeywords::LangPack::applyData(LangPackData &&data) {
	_data = std::move(data);
	_state = State::Refreshed;
	_delegate->langPackRefreshed();
}

void EmojiKeywords::LangPack::apiChanged() {
	if (_state == State::Requested && !_delegate->api()) {
		_requestId = 0;
	} else if (_state != State::PendingRequest) {
		return;
	}
	_state = State::Refreshed;
	refresh();
}

std::vector<Result> EmojiKeywords::LangPack::query(
		const QString &normalized,
		bool exact) const {
	if (normalized.size() > _data.maxKeyLength || _data.emoji.empty()) {
		return {};
	}

	const auto from = _data.emoji.lower_bound(normalized);
	auto &&chosen = ranges::make_iterator_range(
		from,
		end(_data.emoji)
	) | ranges::view::take_while([&](const auto &pair) {
		const auto &key = pair.first;
		return exact ? (key == normalized) : key.startsWith(normalized);
	});

	auto result = std::vector<Result>();
	for (const auto &[key, list] : chosen) {
		AppendFoundEmoji(result, key, list);
	}
	return result;
}

int EmojiKeywords::LangPack::maxQueryLength() const {
	return _data.maxKeyLength;
}

EmojiKeywords::EmojiKeywords() {
	crl::on_main(&_guard, [=] {
		handleAuthSessionChanges();
	});
}

EmojiKeywords::~EmojiKeywords() = default;

not_null<details::EmojiKeywordsLangPackDelegate*> EmojiKeywords::delegate() {
	return static_cast<details::EmojiKeywordsLangPackDelegate*>(this);
}

ApiWrap *EmojiKeywords::api() {
	return _api;
}

void EmojiKeywords::langPackRefreshed() {
	_refreshed.fire({});
}

void EmojiKeywords::handleAuthSessionChanges() {
	rpl::single(
		rpl::empty_value()
	) | rpl::then(base::ObservableViewer(
		Core::App().authSessionChanged()
	)) | rpl::map([] {
		return AuthSession::Exists() ? &Auth().api() : nullptr;
	}) | rpl::start_with_next([=](ApiWrap *api) {
		apiChanged(api);
	}, _lifetime);
}

void EmojiKeywords::apiChanged(ApiWrap *api) {
	_api = api;
	if (_api) {
		crl::on_main(&Auth(), crl::guard(&_guard, [=] {
			base::ObservableViewer(
				Lang::CurrentCloudManager().firstLanguageSuggestion()
			) | rpl::filter([=] {
				// Refresh with the suggested language if we already were asked.
				return !_data.empty();
			}) | rpl::start_with_next([=] {
				refresh();
			}, _suggestedChangeLifetime);
		}));
	} else {
		_langsRequestId = 0;
		_suggestedChangeLifetime.destroy();
	}
	for (const auto &[language, item] : _data) {
		item->apiChanged();
	}
}

void EmojiKeywords::refresh() {
	auto list = KeywordLanguages();
	if (_localList != list) {
		_localList = std::move(list);
		refreshRemoteList();
	} else {
		refreshFromRemoteList();
	}
}

rpl::producer<> EmojiKeywords::refreshed() const {
	return _refreshed.events();
}

std::vector<Result> EmojiKeywords::query(
		const QString &query,
		bool exact) const {
	const auto normalized = NormalizeQuery(query);
	if (normalized.isEmpty()) {
		return {};
	}
	auto result = std::vector<Result>();
	for (const auto &[language, item] : _data) {
		const auto oldcount = result.size();
		const auto list = item->query(normalized, exact);
		auto &&add = ranges::view::all(
			list
		) | ranges::view::filter([&](Result entry) {
			// In each item->query() result the list has no duplicates.
			// So we need to check only for duplicates between queries.
			const auto oldbegin = begin(result);
			const auto oldend = oldbegin + oldcount;
			const auto i = ranges::find(
				oldbegin,
				oldend,
				entry.emoji,
				&Result::emoji);
			return (i == oldend);
		});
		result.insert(end(result), add.begin(), add.end());
	}
	if (!exact) {
		AppendLegacySuggestions(result, query);
	}
	return result;
}

int EmojiKeywords::maxQueryLength() const {
	if (_data.empty()) {
		return 0;
	}
	auto &&lengths = _data | ranges::view::transform([](const auto &pair) {
		return pair.second->maxQueryLength();
	});
	return *ranges::max_element(lengths);
}

void EmojiKeywords::refreshRemoteList() {
	if (!_api) {
		_localList.clear();
		setRemoteList({});
		return;
	}
	_api->request(base::take(_langsRequestId)).cancel();
	auto languages = QVector<MTPstring>();
	for (const auto &id : _localList) {
		languages.push_back(MTP_string(id));
	}
	_langsRequestId = _api->request(MTPmessages_GetEmojiKeywordsLanguages(
		MTP_vector<MTPstring>(languages)
	)).done([=](const MTPVector<MTPEmojiLanguage> &result) {
		setRemoteList(ranges::view::all(
			result.v
		) | ranges::view::transform([](const MTPEmojiLanguage &language) {
			return language.match([&](const MTPDemojiLanguage &language) {
				return qs(language.vlang_code);
			});
		}) | ranges::to_vector);
		_langsRequestId = 0;
	}).fail([=](const RPCError &error) {
		_langsRequestId = 0;
	}).send();
}

void EmojiKeywords::setRemoteList(std::vector<QString> &&list) {
	if (_remoteList == list) {
		return;
	}
	_remoteList = std::move(list);
	for (auto i = begin(_data); i != end(_data);) {
		if (ranges::find(_remoteList, i->first) != end(_remoteList)) {
			++i;
		} else {
			if (_notUsedData.size() > kKeepNotUsedLangPacksCount) {
				_notUsedData.pop_front();
			}
			_notUsedData.push_back(std::move(i->second));
			i = _data.erase(i);
		}
	}
	refreshFromRemoteList();
}

void EmojiKeywords::refreshFromRemoteList() {
	for (const auto &id : _remoteList) {
		if (const auto i = _data.find(id); i != end(_data)) {
			i->second->refresh();
			continue;
		}
		const auto i = ranges::find(
			_notUsedData,
			id,
			[](const std::unique_ptr<LangPack> &p) { return p->id(); });
		if (i != end(_notUsedData)) {
			_data.emplace(id, std::move(*i));
			_notUsedData.erase(i);
		} else {
			_data.emplace(
				id,
				std::make_unique<LangPack>(delegate(), id));
		}
	}
}

} // namespace ChatHelpers