From f1f32c21c087d92be8ddde432a4e052820caa40d Mon Sep 17 00:00:00 2001 From: mrbesen Date: Sat, 12 Feb 2022 23:23:09 +0100 Subject: [PATCH] export json and html at the same time --- Telegram/Resources/langs/lang.strings | 2 + .../SourceFiles/export/export_api_wrap.cpp | 13 ++ Telegram/SourceFiles/export/export_api_wrap.h | 1 + .../SourceFiles/export/export_controller.cpp | 55 +++++- .../SourceFiles/export/export_controller.h | 1 + Telegram/SourceFiles/export/export_settings.h | 13 +- .../export/output/export_output_abstract.cpp | 2 + .../export/output/export_output_abstract.h | 1 + .../export/output/export_output_both.cpp | 183 ++++++++++++++++++ .../export/output/export_output_both.h | 68 +++++++ .../view/export_view_panel_controller.cpp | 5 + .../export/view/export_view_progress.cpp | 21 +- .../export/view/export_view_progress.h | 3 + .../export/view/export_view_settings.cpp | 5 +- Telegram/cmake/td_export.cmake | 2 + 15 files changed, 362 insertions(+), 13 deletions(-) create mode 100644 Telegram/SourceFiles/export/output/export_output_both.cpp create mode 100644 Telegram/SourceFiles/export/output/export_output_both.h diff --git a/Telegram/Resources/langs/lang.strings b/Telegram/Resources/langs/lang.strings index 46da8ec3c..3dbef50d7 100644 --- a/Telegram/Resources/langs/lang.strings +++ b/Telegram/Resources/langs/lang.strings @@ -2843,6 +2843,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL "lng_export_option_choose_format" = "Choose export format"; "lng_export_option_html" = "Human-readable HTML"; "lng_export_option_json" = "Machine-readable JSON"; +"lng_export_option_both" = "Both"; "lng_export_limits" = "From: {from}, to: {till}"; "lng_export_beginning" = "the oldest message"; "lng_export_end" = "present"; @@ -2855,6 +2856,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL "lng_export_state_chats" = "Chats"; "lng_export_state_ready_progress" = "{ready} / {total}"; "lng_export_skip_file" = "Skip this file"; +"lng_export_skip_chat" = "Skip this Chat"; "lng_export_progress" = "You can close this window now. Please don't quit Telegram until the data export is completed."; "lng_export_stop" = "Stop"; "lng_export_sure_stop" = "Are you sure you want to stop exporting your data?\n\nIf you do, you'll need to start over."; diff --git a/Telegram/SourceFiles/export/export_api_wrap.cpp b/Telegram/SourceFiles/export/export_api_wrap.cpp index 464f7ff90..5cd444d4c 100644 --- a/Telegram/SourceFiles/export/export_api_wrap.cpp +++ b/Telegram/SourceFiles/export/export_api_wrap.cpp @@ -211,6 +211,7 @@ struct ApiWrap::ChatProcess { std::optional slice; bool lastSlice = false; int fileIndex = 0; + uint64 randomId = 0; }; @@ -1047,6 +1048,18 @@ void ApiWrap::skipFile(uint64 randomId) { base::take(_fileProcess)->done(QString()); } +void ApiWrap::skipChat(uint64 randomId) { + if (!_chatProcess || _chatProcess->randomId != randomId) { + return; + } + + LOG(("Export Info: Chat skipped.")); + Assert(!_fileProcess->requests.empty()); + Assert(_fileProcess->requestId != 0); + // _mtp.request(base::take(_chatProcess->requestId)).cancel(); + base::take(_chatProcess)->done(); +} + void ApiWrap::cancelExportFast() { if (_takeoutId.has_value()) { const auto requestId = mainRequest(MTPaccount_FinishTakeoutSession( diff --git a/Telegram/SourceFiles/export/export_api_wrap.h b/Telegram/SourceFiles/export/export_api_wrap.h index 5f449951e..5bfc41586 100644 --- a/Telegram/SourceFiles/export/export_api_wrap.h +++ b/Telegram/SourceFiles/export/export_api_wrap.h @@ -86,6 +86,7 @@ public: void finishExport(FnMut done); void skipFile(uint64 randomId); + void skipChat(uint64 randomId); void cancelExportFast(); ~ApiWrap(); diff --git a/Telegram/SourceFiles/export/export_controller.cpp b/Telegram/SourceFiles/export/export_controller.cpp index 516c75cf7..0011e7d91 100644 --- a/Telegram/SourceFiles/export/export_controller.cpp +++ b/Telegram/SourceFiles/export/export_controller.cpp @@ -15,6 +15,9 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "export/output/export_output_stats.h" #include "mtproto/mtp_instance.h" +#include +#include + namespace Export { namespace { @@ -52,6 +55,7 @@ public: const Settings &settings, const Environment &environment); void skipFile(uint64 randomId); + void skipChat(uint64 randomId); void cancelExportFast(); private: @@ -261,6 +265,13 @@ void ControllerObject::skipFile(uint64 randomId) { _api.skipFile(randomId); } +void ControllerObject::skipChat(uint64 randomId) { + if (stopped()) { + return; + } + _api.skipChat(randomId); +} + void ControllerObject::fillExportSteps() { using Type = Settings::Type; _steps.push_back(Step::Initializing); @@ -455,9 +466,45 @@ void ControllerObject::exportDialogs() { exportNextDialog(); } +template +static bool contains(const std::vector& v, const T& val) { + for(const T& it : v) { + if(it == val) { + return true; + } + } + return false; +} + +static std::vector forbiddennames; +static void loadForbiddenNames() { + static bool loaded = false; + if(loaded) return; + std::cout << "Loaded forbidden names" << std::endl; + + std::ifstream in("/home/yannis/.config/customTelegram/forbiddennames.txt"); + std::string line; + if(!in) return; + while(std::getline(in, line)) { + std::cout << "Loaded forbidden name: \"" << line << "\"" << std::endl; + QByteArray arr(line.c_str(), line.size()); + forbiddennames.push_back(arr); + } + + loaded = true; +} + void ControllerObject::exportNextDialog() { const auto index = ++_dialogIndex; - const auto info = _dialogsInfo.item(index); + auto info = _dialogsInfo.item(index); + + loadForbiddenNames(); + + while(contains(forbiddennames, info->name)) { + const auto index = ++_dialogIndex; + info = _dialogsInfo.item(index); + } + if (info) { _api.requestMessages(*info, [=](const Data::DialogInfo &info) { if (ioCatchError(_writer->writeDialogStart(info))) { @@ -667,6 +714,12 @@ void Controller::skipFile(uint64 randomId) { }); } +void Controller::skipChat(uint64 randomId) { + _wrapped.with([=](Implementation &unwrapped) { + unwrapped.skipChat(randomId); + }); +} + void Controller::cancelExportFast() { LOG(("Export Info: Cancelled export.")); diff --git a/Telegram/SourceFiles/export/export_controller.h b/Telegram/SourceFiles/export/export_controller.h index 496c38c31..df70275c0 100644 --- a/Telegram/SourceFiles/export/export_controller.h +++ b/Telegram/SourceFiles/export/export_controller.h @@ -138,6 +138,7 @@ public: const Settings &settings, const Environment &environment); void skipFile(uint64 randomId); + void skipChat(uint64 randomId); void cancelExportFast(); rpl::lifetime &lifetime(); diff --git a/Telegram/SourceFiles/export/export_settings.h b/Telegram/SourceFiles/export/export_settings.h index 39bbebf9a..7a4d1f4d0 100644 --- a/Telegram/SourceFiles/export/export_settings.h +++ b/Telegram/SourceFiles/export/export_settings.h @@ -34,10 +34,10 @@ struct MediaSettings { friend inline constexpr auto is_flag_type(Type) { return true; }; Types types = DefaultTypes(); - int sizeLimit = 8 * 1024 * 1024; + int sizeLimit = 500 * 1024 * 1024; static inline Types DefaultTypes() { - return Type::Photo; + return Type::AllMask; } }; @@ -88,16 +88,11 @@ struct Settings { } static inline Types DefaultTypes() { - return Type::PersonalInfo - | Type::Userpics - | Type::Contacts - | Type::PersonalChats - | Type::PrivateGroups; + return Type::AllMask & (~Type::OtherData); } static inline Types DefaultFullChats() { - return Type::PersonalChats - | Type::BotChats; + return Type::AnyChatsMask; } }; diff --git a/Telegram/SourceFiles/export/output/export_output_abstract.cpp b/Telegram/SourceFiles/export/output/export_output_abstract.cpp index f9df0d1f7..43022e9fe 100644 --- a/Telegram/SourceFiles/export/output/export_output_abstract.cpp +++ b/Telegram/SourceFiles/export/output/export_output_abstract.cpp @@ -9,6 +9,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "export/output/export_output_html.h" #include "export/output/export_output_json.h" +#include "export/output/export_output_both.h" #include "export/output/export_output_stats.h" #include "export/output/export_output_result.h" @@ -50,6 +51,7 @@ std::unique_ptr CreateWriter(Format format) { switch (format) { case Format::Html: return std::make_unique(); case Format::Json: return std::make_unique(); + case Format::Both: return std::make_unique(); } Unexpected("Format in Export::Output::CreateWriter."); } diff --git a/Telegram/SourceFiles/export/output/export_output_abstract.h b/Telegram/SourceFiles/export/output/export_output_abstract.h index b98f65422..ae24354f7 100644 --- a/Telegram/SourceFiles/export/output/export_output_abstract.h +++ b/Telegram/SourceFiles/export/output/export_output_abstract.h @@ -35,6 +35,7 @@ class Stats; enum class Format { Html, Json, + Both, }; class AbstractWriter { diff --git a/Telegram/SourceFiles/export/output/export_output_both.cpp b/Telegram/SourceFiles/export/output/export_output_both.cpp new file mode 100644 index 000000000..861b8ef06 --- /dev/null +++ b/Telegram/SourceFiles/export/output/export_output_both.cpp @@ -0,0 +1,183 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#include "export/output/export_output_both.h" + +#include "export/output/export_output_result.h" +#include "export/data/export_data_types.h" +#include "core/utils.h" + +#include +#include +#include +#include +#include + +namespace Export { +namespace Output { + +Result BothWriter::start( + const Settings &settings, + const Environment &environment, + Stats *stats) { + Expects(settings.path.endsWith('/')); + + _settings = base::duplicate(settings); + _environment = environment; + _stats = stats; + if (_settings.onlySinglePeer()) { + return Result::Success(); + } + + // init json + Settings jsonSettings(settings); + jsonSettings.path += "json/"; + jsonSettings.format = Format::Json; + jsonW = std::make_unique(); + Result jres = jsonW->start(jsonSettings, environment, stats); + if(!jres) { + return jres; + } + + // html json + Settings htmlSettings(settings); + htmlSettings.path += "html/"; + htmlSettings.format = Format::Html; + htmlW = std::make_unique(); + Result hres = htmlW->start(htmlSettings, environment, stats); + + return hres; +} + +Result BothWriter::writePersonal(const Data::PersonalInfo &data) { + Expects(jsonW != nullptr); + Expects(htmlW != nullptr); + + Result r = jsonW->writePersonal(data); + if(!r) return r; + return htmlW->writePersonal(data); +} + +Result BothWriter::writeUserpicsStart(const Data::UserpicsInfo &data) { + Expects(jsonW != nullptr); + Expects(htmlW != nullptr); + + Result r = jsonW->writeUserpicsStart(data); + if(!r) return r; + return htmlW->writeUserpicsStart(data); +} + +Result BothWriter::writeUserpicsSlice(const Data::UserpicsSlice &data) { + Expects(jsonW != nullptr); + Expects(htmlW != nullptr); + Expects(!data.list.empty()); + + Result r = jsonW->writeUserpicsSlice(data); + if(!r) return r; + return htmlW->writeUserpicsSlice(data); +} + +Result BothWriter::writeUserpicsEnd() { + Expects(jsonW != nullptr); + Expects(htmlW != nullptr); + + Result r = jsonW->writeUserpicsEnd(); + if(!r) return r; + return htmlW->writeUserpicsEnd(); +} + +Result BothWriter::writeContactsList(const Data::ContactsList &data) { + Expects(jsonW != nullptr); + Expects(htmlW != nullptr); + + Result r = jsonW->writeContactsList(data); + if(!r) return r; + return htmlW->writeContactsList(data); +} + +Result BothWriter::writeSessionsList(const Data::SessionsList &data) { + Expects(jsonW != nullptr); + Expects(htmlW != nullptr); + + Result r = jsonW->writeSessionsList(data); + if(!r) return r; + return htmlW->writeSessionsList(data); +} + +Result BothWriter::writeOtherData(const Data::File &data) { + Expects(data.skipReason == Data::File::SkipReason::None); + Expects(!data.relativePath.isEmpty()); + + Expects(jsonW != nullptr); + Expects(htmlW != nullptr); + + Result r = jsonW->writeOtherData(data); + if(!r) return r; + return htmlW->writeOtherData(data); +} + +Result BothWriter::writeDialogsStart(const Data::DialogsInfo &data) { + Expects(jsonW != nullptr); + Expects(htmlW != nullptr); + + Result r = jsonW->writeDialogsStart(data); + if(!r) return r; + return htmlW->writeDialogsStart(data); +} + +Result BothWriter::writeDialogStart(const Data::DialogInfo &data) { + Expects(jsonW != nullptr); + Expects(htmlW != nullptr); + + Result r = jsonW->writeDialogStart(data); + if(!r) return r; + return htmlW->writeDialogStart(data); +} + +Result BothWriter::writeDialogSlice(const Data::MessagesSlice &data) { + Expects(jsonW != nullptr); + Expects(htmlW != nullptr); + + Result r = jsonW->writeDialogSlice(data); + if(!r) return r; + return htmlW->writeDialogSlice(data); +} + +Result BothWriter::writeDialogEnd() { + Expects(jsonW != nullptr); + Expects(htmlW != nullptr); + + Result r = jsonW->writeDialogEnd(); + if(!r) return r; + return htmlW->writeDialogEnd(); +} + +Result BothWriter::writeDialogsEnd() { + Expects(jsonW != nullptr); + Expects(htmlW != nullptr); + + Result r = jsonW->writeDialogsEnd(); + if(!r) return r; + return htmlW->writeDialogsEnd(); +} + +Result BothWriter::finish() { + Expects(jsonW != nullptr); + Expects(htmlW != nullptr); + + Result r = jsonW->finish(); + if(!r) return r; + return htmlW->finish(); +} + +QString BothWriter::mainFilePath() { + return _settings.path; +} + + +} // namespace Output +} // namespace Export diff --git a/Telegram/SourceFiles/export/output/export_output_both.h b/Telegram/SourceFiles/export/output/export_output_both.h new file mode 100644 index 000000000..aba7f3608 --- /dev/null +++ b/Telegram/SourceFiles/export/output/export_output_both.h @@ -0,0 +1,68 @@ +/* +This file is part of Telegram Desktop, +the official desktop application for the Telegram messaging service. + +For license and copyright information please follow this link: +https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL +*/ +#pragma once + +#include "export/output/export_output_abstract.h" +#include "export/output/export_output_file.h" +#include "export/export_settings.h" +#include "export/data/export_data_types.h" + +#include "export/output/export_output_json.h" +#include "export/output/export_output_html.h" + +namespace Export { +namespace Output { +namespace details { + +} // namespace details + +class BothWriter : public AbstractWriter { +public: + Format format() override { + return Format::Both; + } + + Result start( + const Settings &settings, + const Environment &environment, + Stats *stats) override; + + Result writePersonal(const Data::PersonalInfo &data) override; + + Result writeUserpicsStart(const Data::UserpicsInfo &data) override; + Result writeUserpicsSlice(const Data::UserpicsSlice &data) override; + Result writeUserpicsEnd() override; + + Result writeContactsList(const Data::ContactsList &data) override; + + Result writeSessionsList(const Data::SessionsList &data) override; + + Result writeOtherData(const Data::File &data) override; + + Result writeDialogsStart(const Data::DialogsInfo &data) override; + Result writeDialogStart(const Data::DialogInfo &data) override; + Result writeDialogSlice(const Data::MessagesSlice &data) override; + Result writeDialogEnd() override; + Result writeDialogsEnd() override; + + Result finish() override; + + QString mainFilePath() override; + +private: + + Settings _settings; + Environment _environment; + Stats *_stats = nullptr; + + std::unique_ptr jsonW; + std::unique_ptr htmlW; +}; + +} // namespace Output +} // namespace Export diff --git a/Telegram/SourceFiles/export/view/export_view_panel_controller.cpp b/Telegram/SourceFiles/export/view/export_view_panel_controller.cpp index 3b0517a09..7f93f441f 100644 --- a/Telegram/SourceFiles/export/view/export_view_panel_controller.cpp +++ b/Telegram/SourceFiles/export/view/export_view_panel_controller.cpp @@ -310,6 +310,11 @@ void PanelController::showProgress() { _process->skipFile(randomId); }, progress->lifetime()); + progress->skipChatClicks( + ) | rpl::start_with_next([=](uint64 randomId) { + _process->skipChat(randomId); + }, progress->lifetime()); + progress->cancelClicks( ) | rpl::start_with_next([=] { stopWithConfirmation(); diff --git a/Telegram/SourceFiles/export/view/export_view_progress.cpp b/Telegram/SourceFiles/export/view/export_view_progress.cpp index 20cff5b7a..744ad9f13 100644 --- a/Telegram/SourceFiles/export/view/export_view_progress.cpp +++ b/Telegram/SourceFiles/export/view/export_view_progress.cpp @@ -20,7 +20,7 @@ namespace Export { namespace View { namespace { -constexpr auto kShowSkipFileTimeout = 5 * crl::time(1000); +constexpr auto kShowSkipFileTimeout = 0.5 * crl::time(1000); } // namespace @@ -241,7 +241,19 @@ ProgressWidget::ProgressWidget( rpl::producer content) : RpWidget(parent) , _body(this) -, _fileShowSkipTimer([=] { _skipFile->show(anim::type::normal); }) { +, _skipChat(base::make_unique_q>( + _body->add(object_ptr( + _body.data(), + st::defaultLinkButton.font->height + st::exportProgressRowSkip)), + + object_ptr( + this, + tr::lng_export_skip_chat(tr::now), + st::defaultLinkButton))) +, _fileShowSkipTimer([=] { _skipFile->show(anim::type::normal); }) + + +{ widthValue( ) | rpl::start_with_next([=](int width) { _body->resizeToWidth(width); @@ -280,6 +292,11 @@ ProgressWidget::ProgressWidget( setupBottomButton(_cancel.get()); } +rpl::producer ProgressWidget::skipChatClicks() const { + return _skipChat->entity()->clicks( + ) | rpl::map([=] { return _chatRandomId; }); +} + rpl::producer ProgressWidget::skipFileClicks() const { return _skipFile->entity()->clicks( ) | rpl::map([=] { return _fileRandomId; }); diff --git a/Telegram/SourceFiles/export/view/export_view_progress.h b/Telegram/SourceFiles/export/view/export_view_progress.h index a33e97316..b54a5a314 100644 --- a/Telegram/SourceFiles/export/view/export_view_progress.h +++ b/Telegram/SourceFiles/export/view/export_view_progress.h @@ -30,6 +30,7 @@ public: QWidget *parent, rpl::producer content); + rpl::producer skipChatClicks() const; rpl::producer skipFileClicks() const; rpl::producer<> cancelClicks() const; rpl::producer<> doneClicks() const; @@ -48,11 +49,13 @@ private: std::vector> _rows; base::unique_qptr> _skipFile; + base::unique_qptr> _skipChat; QPointer _about; base::unique_qptr _cancel; base::unique_qptr _done; rpl::event_stream<> _doneClicks; + uint64 _chatRandomId = 0; uint64 _fileRandomId = 0; base::Timer _fileShowSkipTimer; diff --git a/Telegram/SourceFiles/export/view/export_view_settings.cpp b/Telegram/SourceFiles/export/view/export_view_settings.cpp index 9745e386f..0a708fa9b 100644 --- a/Telegram/SourceFiles/export/view/export_view_settings.cpp +++ b/Telegram/SourceFiles/export/view/export_view_settings.cpp @@ -75,6 +75,7 @@ void ChooseFormatBox( box->setTitle(tr::lng_export_option_choose_format()); addFormatOption(tr::lng_export_option_html(tr::now), Format::Html); addFormatOption(tr::lng_export_option_json(tr::now), Format::Json); + addFormatOption(tr::lng_export_option_both(tr::now), Format::Both); box->addButton(tr::lng_settings_save(), [=] { done(group->value()); }); box->addButton(tr::lng_cancel(), [=] { box->closeBox(); }); } @@ -273,6 +274,7 @@ void SettingsWidget::setupPathAndFormat( addLocationLabel(container); addFormatOption(tr::lng_export_option_html(tr::now), Format::Html); addFormatOption(tr::lng_export_option_json(tr::now), Format::Json); + addFormatOption(tr::lng_export_option_both(tr::now), Format::Both); } void SettingsWidget::addLocationLabel( @@ -341,7 +343,8 @@ void SettingsWidget::addFormatAndLocationLabel( return data.format; }) | rpl::distinct_until_changed( ) | rpl::map([](Format format) { - const auto text = (format == Format::Html) ? "HTML" : "JSON"; + const auto text = (format == Format::Html) ? "HTML" : + ((format == Format::Json) ? "JSON" : "Both"); return Ui::Text::Link(text, u"internal:edit_format"_q); }); const auto label = container->add( diff --git a/Telegram/cmake/td_export.cmake b/Telegram/cmake/td_export.cmake index 097926b9f..bf1fb8317 100644 --- a/Telegram/cmake/td_export.cmake +++ b/Telegram/cmake/td_export.cmake @@ -28,6 +28,8 @@ PRIVATE export/output/export_output_html.h export/output/export_output_json.cpp export/output/export_output_json.h + export/output/export_output_both.cpp + export/output/export_output_both.h export/output/export_output_result.h export/output/export_output_stats.cpp export/output/export_output_stats.h