From d80cf5d149dc36091907b6292b7309e02d3a61df Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 28 Jul 2022 17:48:28 +0300 Subject: [PATCH] Download stickers for custom emoji in export. --- Telegram/Resources/export_html/js/script.js | 10 ++ .../export/data/export_data_types.h | 11 ++ .../SourceFiles/export/export_api_wrap.cpp | 127 +++++++++++++++++- Telegram/SourceFiles/export/export_api_wrap.h | 8 ++ .../export/output/export_output_html.cpp | 17 ++- Telegram/lib_ui | 2 +- 6 files changed, 167 insertions(+), 8 deletions(-) diff --git a/Telegram/Resources/export_html/js/script.js b/Telegram/Resources/export_html/js/script.js index 23b4009e5..8d25f5302 100644 --- a/Telegram/Resources/export_html/js/script.js +++ b/Telegram/Resources/export_html/js/script.js @@ -52,6 +52,16 @@ function ShowMentionName() { return false; } +function ShowNotLoadedEmoji() { + ShowToast("This custom emoji is not included, change data exporting settings to download."); + return false; +} + +function ShowNotAvailableEmoji() { + ShowToast("This custom emoji is not available."); + return false; +} + function ShowSpoiler(target) { if (target.classList.contains("hidden")) { target.classList.toggle("hidden"); diff --git a/Telegram/SourceFiles/export/data/export_data_types.h b/Telegram/SourceFiles/export/data/export_data_types.h index 07b03aece..32a4a502b 100644 --- a/Telegram/SourceFiles/export/data/export_data_types.h +++ b/Telegram/SourceFiles/export/data/export_data_types.h @@ -340,6 +340,12 @@ struct ParseMediaContext { UserId botId = 0; }; +Document ParseDocument( + ParseMediaContext &context, + const MTPDocument &data, + const QString &suggestedFolder, + TimeId date); + Media ParseMedia( ParseMediaContext &context, const MTPMessageMedia &data, @@ -560,6 +566,10 @@ struct TextPart { Type type = Type::Text; Utf8String text; Utf8String additional; + + [[nodiscard]] static Utf8String UnavailableEmoji() { + return "(unavailable)"; + } }; struct MessageId { @@ -619,6 +629,7 @@ struct FileOrigin { int split = 0; MTPInputPeer peer; int32 messageId = 0; + uint64 customEmojiId = 0; }; Message ParseMessage( diff --git a/Telegram/SourceFiles/export/export_api_wrap.cpp b/Telegram/SourceFiles/export/export_api_wrap.cpp index 21312886b..45e9e0665 100644 --- a/Telegram/SourceFiles/export/export_api_wrap.cpp +++ b/Telegram/SourceFiles/export/export_api_wrap.cpp @@ -30,6 +30,7 @@ constexpr auto kMessagesSliceLimit = 100; constexpr auto kTopPeerSliceLimit = 100; constexpr auto kFileMaxSize = 4000 * int64(1024 * 1024); constexpr auto kLocationCacheSize = 100'000; +constexpr auto kMaxEmojiPerRequest = 100; struct LocationKey { uint64 type; @@ -1468,13 +1469,79 @@ void ApiWrap::loadMessagesFiles(Data::MessagesSlice &&slice) { Expects(_chatProcess != nullptr); Expects(!_chatProcess->slice.has_value()); + collectMessagesCustomEmoji(slice); + if (slice.list.empty()) { _chatProcess->lastSlice = true; } _chatProcess->slice = std::move(slice); _chatProcess->fileIndex = 0; - loadNextMessageFile(); + resolveCustomEmoji(); +} + +void ApiWrap::collectMessagesCustomEmoji(const Data::MessagesSlice &slice) { + for (const auto &message : slice.list) { + for (const auto &part : message.text) { + if (part.type == Data::TextPart::Type::CustomEmoji) { + if (const auto id = part.additional.toULongLong()) { + if (!_resolvedCustomEmoji.contains(id)) { + _unresolvedCustomEmoji.emplace(id); + } + } + } + } + } +} + +void ApiWrap::resolveCustomEmoji() { + if (_unresolvedCustomEmoji.empty()) { + loadNextMessageFile(); + return; + } + const auto count = std::min( + int(_unresolvedCustomEmoji.size()), + kMaxEmojiPerRequest); + auto v = QVector(); + v.reserve(count); + const auto till = end(_unresolvedCustomEmoji); + const auto from = end(_unresolvedCustomEmoji) - count; + for (auto i = from; i != till; ++i) { + v.push_back(MTP_long(*i)); + } + _unresolvedCustomEmoji.erase(from, till); + const auto finalize = [=] { + for (const auto &id : v) { + if (_resolvedCustomEmoji.contains(id.v)) { + continue; + } + _resolvedCustomEmoji.emplace( + id.v, + Data::Document{ + .file = { + .skipReason = Data::File::SkipReason::Unavailable, + }, + }); + } + resolveCustomEmoji(); + }; + mainRequest(MTPmessages_GetCustomEmojiDocuments( + MTP_vector(v) + )).fail([=](const MTP::Error &error) { + LOG(("Export Error: Failed to get documents for emoji.")); + finalize(); + return true; + }).done([=](const MTPVector &result) { + for (const auto &entry : result.v) { + auto document = Data::ParseDocument( + _chatProcess->context, + entry, + _chatProcess->info.relativePath, + TimeId()); + _resolvedCustomEmoji.emplace(document.id, std::move(document)); + } + finalize(); + }).send(); } Data::Message *ApiWrap::currentFileMessage() const { @@ -1501,6 +1568,44 @@ Data::FileOrigin ApiWrap::currentFileMessageOrigin() const { return result; } +bool ApiWrap::messageCustomEmojiReady(Data::Message &message) { + for (auto &part : message.text) { + if (part.type == Data::TextPart::Type::CustomEmoji) { + if (const auto id = part.additional.toULongLong()) { + const auto i = _resolvedCustomEmoji.find(id); + if (i == end(_resolvedCustomEmoji)) { + part.additional = Data::TextPart::UnavailableEmoji(); + } else { + auto &file = i->second.file; + const auto fileProgress = [=](FileProgress value) { + return loadMessageEmojiProgress(value); + }; + const auto ready = processFileLoad( + file, + { .customEmojiId = id }, + fileProgress, + [=](const QString &path) { + loadMessageEmojiDone(id, path); + }); + if (!ready) { + return false; + } + using SkipReason = Data::File::SkipReason; + if (file.skipReason == SkipReason::Unavailable) { + part.additional = Data::TextPart::UnavailableEmoji(); + } else if (file.skipReason == SkipReason::FileType + || file.skipReason == SkipReason::FileSize) { + part.additional = QByteArray(); + } else { + part.additional = file.relativePath.toUtf8(); + } + } + } + } + } + return true; +} + void ApiWrap::loadNextMessageFile() { Expects(_chatProcess != nullptr); Expects(_chatProcess->slice.has_value()); @@ -1508,10 +1613,13 @@ void ApiWrap::loadNextMessageFile() { for (auto &list = _chatProcess->slice->list ; _chatProcess->fileIndex < list.size() ; ++_chatProcess->fileIndex) { - const auto &message = list[_chatProcess->fileIndex]; + auto &message = list[_chatProcess->fileIndex]; if (Data::SkipMessageByDate(message, *_settings)) { continue; } + if (!messageCustomEmojiReady(message)) { + return; + } const auto fileProgress = [=](FileProgress value) { return loadMessageFileProgress(value); }; @@ -1618,6 +1726,21 @@ void ApiWrap::loadMessageThumbDone(const QString &relativePath) { loadNextMessageFile(); } +bool ApiWrap::loadMessageEmojiProgress(FileProgress progress) { + return loadMessageFileProgress(progress); +} + +void ApiWrap::loadMessageEmojiDone(uint64 id, const QString &relativePath) { + const auto i = _resolvedCustomEmoji.find(id); + if (i != end(_resolvedCustomEmoji)) { + i->second.file.relativePath = relativePath; + if (relativePath.isEmpty()) { + i->second.file.skipReason = Data::File::SkipReason::Unavailable; + } + } + loadNextMessageFile(); +} + void ApiWrap::finishMessages() { Expects(_chatProcess != nullptr); Expects(!_chatProcess->slice.has_value()); diff --git a/Telegram/SourceFiles/export/export_api_wrap.h b/Telegram/SourceFiles/export/export_api_wrap.h index a297dd92c..6384457d7 100644 --- a/Telegram/SourceFiles/export/export_api_wrap.h +++ b/Telegram/SourceFiles/export/export_api_wrap.h @@ -14,6 +14,7 @@ namespace Export { namespace Data { struct File; struct Chat; +struct Document; struct FileLocation; struct PersonalInfo; struct UserpicsInfo; @@ -156,12 +157,17 @@ private: int addOffset, int limit, FnMut done); + void collectMessagesCustomEmoji(const Data::MessagesSlice &slice); + void resolveCustomEmoji(); void loadMessagesFiles(Data::MessagesSlice &&slice); void loadNextMessageFile(); + bool messageCustomEmojiReady(Data::Message &message); bool loadMessageFileProgress(FileProgress value); void loadMessageFileDone(const QString &relativePath); bool loadMessageThumbProgress(FileProgress value); void loadMessageThumbDone(const QString &relativePath); + bool loadMessageEmojiProgress(FileProgress progress); + void loadMessageEmojiDone(uint64 id, const QString &relativePath); void finishMessagesSlice(); void finishMessages(); @@ -227,6 +233,8 @@ private: std::unique_ptr _leftChannelsProcess; std::unique_ptr _dialogsProcess; std::unique_ptr _chatProcess; + base::flat_set _unresolvedCustomEmoji; + base::flat_map _resolvedCustomEmoji; QVector _splits; rpl::event_stream _errors; diff --git a/Telegram/SourceFiles/export/output/export_output_html.cpp b/Telegram/SourceFiles/export/output/export_output_html.cpp index 0d79af922..f2bb88d7e 100644 --- a/Telegram/SourceFiles/export/output/export_output_html.cpp +++ b/Telegram/SourceFiles/export/output/export_output_html.cpp @@ -220,7 +220,8 @@ QByteArray JoinList( QByteArray FormatText( const std::vector &data, - const QString &internalLinksDomain) { + const QString &internalLinksDomain, + const QString &relativeLinkBase) { return JoinList(QByteArray(), ranges::views::all( data ) | ranges::views::transform([&](const Data::TextPart &part) { @@ -274,9 +275,15 @@ QByteArray FormatText( "onclick=\"ShowSpoiler(this)\">" "" + text + ""; - case Type::CustomEmoji: return SerializeString("{custom_emoji}") - + text // TODO custom-emoji - + SerializeString("{/custom_emoji}"); + case Type::CustomEmoji: return (part.additional.isEmpty() + ? "" + : (part.additional == Data::TextPart::UnavailableEmoji()) + ? "" + : ("")) + + text + + ""; } Unexpected("Type in text entities serialization."); }) | ranges::to_vector); @@ -1257,7 +1264,7 @@ auto HtmlWriter::Wrap::pushMessage( block.append(pushMedia(message, basePath, peers, internalLinksDomain)); - const auto text = FormatText(message.text, internalLinksDomain); + const auto text = FormatText(message.text, internalLinksDomain, _base); if (!text.isEmpty()) { block.append(pushDiv("text")); block.append(text); diff --git a/Telegram/lib_ui b/Telegram/lib_ui index 9b0f4df00..4768e7ee0 160000 --- a/Telegram/lib_ui +++ b/Telegram/lib_ui @@ -1 +1 @@ -Subproject commit 9b0f4df00715f4dfaac81e17148ca37df26fb301 +Subproject commit 4768e7ee03aa22f64f73dc13016d5bd94a047496