export json and html at the same time

Telegram/Resources/langs/lang.strings

@@ -2843,6 +2843,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 "lng_export_option_choose_format" = "Choose export format";
 "lng_export_option_html" = "Human-readable HTML";
 "lng_export_option_json" = "Machine-readable JSON";
+"lng_export_option_both" = "Both";
 "lng_export_limits" = "From: {from}, to: {till}";
 "lng_export_beginning" = "the oldest message";
 "lng_export_end" = "present";
@@ -2855,6 +2856,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 "lng_export_state_chats" = "Chats";
 "lng_export_state_ready_progress" = "{ready} / {total}";
 "lng_export_skip_file" = "Skip this file";
+"lng_export_skip_chat" = "Skip this Chat";
 "lng_export_progress" = "You can close this window now. Please don't quit Telegram until the data export is completed.";
 "lng_export_stop" = "Stop";
 "lng_export_sure_stop" = "Are you sure you want to stop exporting your data?\n\nIf you do, you'll need to start over.";

Telegram/SourceFiles/export/export_api_wrap.cpp

@@ -211,6 +211,7 @@ struct ApiWrap::ChatProcess {
 	std::optional<Data::MessagesSlice> slice;
 	bool lastSlice = false;
 	int fileIndex = 0;
+	uint64 randomId = 0;
 };
 
 
@@ -1047,6 +1048,18 @@ void ApiWrap::skipFile(uint64 randomId) {
 	base::take(_fileProcess)->done(QString());
 }
 
+void ApiWrap::skipChat(uint64 randomId) {
+	if (!_chatProcess || _chatProcess->randomId != randomId) {
+		return;
+	}
+
+	LOG(("Export Info: Chat skipped."));
+	Assert(!_fileProcess->requests.empty());
+	Assert(_fileProcess->requestId != 0);
+	// _mtp.request(base::take(_chatProcess->requestId)).cancel();
+	base::take(_chatProcess)->done();
+}
+
 void ApiWrap::cancelExportFast() {
 	if (_takeoutId.has_value()) {
 		const auto requestId = mainRequest(MTPaccount_FinishTakeoutSession(

Telegram/SourceFiles/export/export_api_wrap.h

@@ -86,6 +86,7 @@ public:
 
 	void finishExport(FnMut<void()> done);
 	void skipFile(uint64 randomId);
+	void skipChat(uint64 randomId);
 	void cancelExportFast();
 
 	~ApiWrap();

Telegram/SourceFiles/export/export_controller.cpp

@@ -15,6 +15,9 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #include "export/output/export_output_stats.h"
 #include "mtproto/mtp_instance.h"
 
+#include <iostream>
+#include <fstream>
+
 namespace Export {
 namespace {
 
@@ -52,6 +55,7 @@ public:
 		const Settings &settings,
 		const Environment &environment);
 	void skipFile(uint64 randomId);
+	void skipChat(uint64 randomId);
 	void cancelExportFast();
 
 private:
@@ -261,6 +265,13 @@ void ControllerObject::skipFile(uint64 randomId) {
 	_api.skipFile(randomId);
 }
 
+void ControllerObject::skipChat(uint64 randomId) {
+	if (stopped()) {
+		return;
+	}
+	_api.skipChat(randomId);
+}
+
 void ControllerObject::fillExportSteps() {
 	using Type = Settings::Type;
 	_steps.push_back(Step::Initializing);
@@ -455,9 +466,45 @@ void ControllerObject::exportDialogs() {
 	exportNextDialog();
 }
 
+template<typename T>
+static bool contains(const std::vector<T>& v, const T& val) {
+	for(const T& it : v) {
+		if(it == val) {
+			return true;
+		}
+	}
+	return false;
+}
+
+static std::vector<Export::Data::Utf8String> forbiddennames;
+static void loadForbiddenNames() {
+	static bool loaded = false;
+	if(loaded) return;
+	std::cout << "Loaded forbidden names" << std::endl;
+
+	std::ifstream in("/home/yannis/.config/customTelegram/forbiddennames.txt");
+	std::string line;
+	if(!in) return;
+	while(std::getline(in, line)) {
+		std::cout << "Loaded forbidden name: \"" << line << "\"" << std::endl;
+		QByteArray arr(line.c_str(), line.size());
+		forbiddennames.push_back(arr);
+	}
+
+	loaded = true;
+}
+
 void ControllerObject::exportNextDialog() {
 	const auto index = ++_dialogIndex;
-	const auto info = _dialogsInfo.item(index);
+	auto info = _dialogsInfo.item(index);
+
+	loadForbiddenNames();
+
+	while(contains<Export::Data::Utf8String>(forbiddennames, info->name)) {
+		const auto index = ++_dialogIndex;
+		info = _dialogsInfo.item(index);
+	}
+
 	if (info) {
 		_api.requestMessages(*info, [=](const Data::DialogInfo &info) {
 			if (ioCatchError(_writer->writeDialogStart(info))) {
@@ -667,6 +714,12 @@ void Controller::skipFile(uint64 randomId) {
 	});
 }
 
+void Controller::skipChat(uint64 randomId) {
+	_wrapped.with([=](Implementation &unwrapped) {
+		unwrapped.skipChat(randomId);
+	});
+}
+
 void Controller::cancelExportFast() {
 	LOG(("Export Info: Cancelled export."));
 

Telegram/SourceFiles/export/export_controller.h

@@ -138,6 +138,7 @@ public:
 		const Settings &settings,
 		const Environment &environment);
 	void skipFile(uint64 randomId);
+	void skipChat(uint64 randomId);
 	void cancelExportFast();
 
 	rpl::lifetime &lifetime();

Telegram/SourceFiles/export/export_settings.h

@@ -34,10 +34,10 @@ struct MediaSettings {
 	friend inline constexpr auto is_flag_type(Type) { return true; };
 
 	Types types = DefaultTypes();
-	int sizeLimit = 8 * 1024 * 1024;
+	int sizeLimit = 500 * 1024 * 1024;
 
 	static inline Types DefaultTypes() {
-		return Type::Photo;
+		return Type::AllMask;
 	}
 
 };
@@ -88,16 +88,11 @@ struct Settings {
 	}
 
 	static inline Types DefaultTypes() {
-		return Type::PersonalInfo
-			| Type::Userpics
-			| Type::Contacts
-			| Type::PersonalChats
-			| Type::PrivateGroups;
+		return Type::AllMask & (~Type::OtherData);
 	}
 
 	static inline Types DefaultFullChats() {
-		return Type::PersonalChats
-			| Type::BotChats;
+		return Type::AnyChatsMask;
 	}
 
 };

Telegram/SourceFiles/export/output/export_output_abstract.cpp

@@ -9,6 +9,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 
 #include "export/output/export_output_html.h"
 #include "export/output/export_output_json.h"
+#include "export/output/export_output_both.h"
 #include "export/output/export_output_stats.h"
 #include "export/output/export_output_result.h"
 
@@ -50,6 +51,7 @@ std::unique_ptr<AbstractWriter> CreateWriter(Format format) {
 	switch (format) {
 	case Format::Html: return std::make_unique<HtmlWriter>();
 	case Format::Json: return std::make_unique<JsonWriter>();
+	case Format::Both: return std::make_unique<BothWriter>();
 	}
 	Unexpected("Format in Export::Output::CreateWriter.");
 }

Telegram/SourceFiles/export/output/export_output_abstract.h

@@ -35,6 +35,7 @@ class Stats;
 enum class Format {
 	Html,
 	Json,
+	Both,
 };
 
 class AbstractWriter {

Telegram/SourceFiles/export/output/export_output_both.cpp

@@ -0,0 +1,183 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "export/output/export_output_both.h"
+
+#include "export/output/export_output_result.h"
+#include "export/data/export_data_types.h"
+#include "core/utils.h"
+
+#include <QtCore/QDateTime>
+#include <QtCore/QJsonDocument>
+#include <QtCore/QJsonObject>
+#include <QtCore/QJsonArray>
+#include <QtCore/QJsonValue>
+
+namespace Export {
+namespace Output {
+
+Result BothWriter::start(
+		const Settings &settings,
+		const Environment &environment,
+		Stats *stats) {
+	Expects(settings.path.endsWith('/'));
+
+	_settings = base::duplicate(settings);
+	_environment = environment;
+	_stats = stats;
+	if (_settings.onlySinglePeer()) {
+		return Result::Success();
+	}
+
+	// init json
+	Settings jsonSettings(settings);
+	jsonSettings.path += "json/";
+	jsonSettings.format = Format::Json;
+	jsonW = std::make_unique<JsonWriter>();
+	Result jres = jsonW->start(jsonSettings, environment, stats);
+	if(!jres) {
+		return jres;
+	}
+
+	// html json
+	Settings htmlSettings(settings);
+	htmlSettings.path += "html/";
+	htmlSettings.format = Format::Html;
+	htmlW = std::make_unique<HtmlWriter>();
+	Result hres = htmlW->start(htmlSettings, environment, stats);
+
+	return hres;
+}
+
+Result BothWriter::writePersonal(const Data::PersonalInfo &data) {
+	Expects(jsonW != nullptr);
+	Expects(htmlW != nullptr);
+
+	Result r = jsonW->writePersonal(data);
+	if(!r) return r;
+	return htmlW->writePersonal(data);
+}
+
+Result BothWriter::writeUserpicsStart(const Data::UserpicsInfo &data) {
+	Expects(jsonW != nullptr);
+	Expects(htmlW != nullptr);
+
+	Result r = jsonW->writeUserpicsStart(data);
+	if(!r) return r;
+	return htmlW->writeUserpicsStart(data);
+}
+
+Result BothWriter::writeUserpicsSlice(const Data::UserpicsSlice &data) {
+	Expects(jsonW != nullptr);
+	Expects(htmlW != nullptr);
+	Expects(!data.list.empty());
+
+	Result r = jsonW->writeUserpicsSlice(data);
+	if(!r) return r;
+	return htmlW->writeUserpicsSlice(data);
+}
+
+Result BothWriter::writeUserpicsEnd() {
+	Expects(jsonW != nullptr);
+	Expects(htmlW != nullptr);
+
+	Result r = jsonW->writeUserpicsEnd();
+	if(!r) return r;
+	return htmlW->writeUserpicsEnd();
+}
+
+Result BothWriter::writeContactsList(const Data::ContactsList &data) {
+	Expects(jsonW != nullptr);
+	Expects(htmlW != nullptr);
+
+	Result r = jsonW->writeContactsList(data);
+	if(!r) return r;
+	return htmlW->writeContactsList(data);
+}
+
+Result BothWriter::writeSessionsList(const Data::SessionsList &data) {
+	Expects(jsonW != nullptr);
+	Expects(htmlW != nullptr);
+
+	Result r = jsonW->writeSessionsList(data);
+	if(!r) return r;
+	return htmlW->writeSessionsList(data);
+}
+
+Result BothWriter::writeOtherData(const Data::File &data) {
+	Expects(data.skipReason == Data::File::SkipReason::None);
+	Expects(!data.relativePath.isEmpty());
+
+	Expects(jsonW != nullptr);
+	Expects(htmlW != nullptr);
+
+	Result r = jsonW->writeOtherData(data);
+	if(!r) return r;
+	return htmlW->writeOtherData(data);
+}
+
+Result BothWriter::writeDialogsStart(const Data::DialogsInfo &data) {
+	Expects(jsonW != nullptr);
+	Expects(htmlW != nullptr);
+
+	Result r = jsonW->writeDialogsStart(data);
+	if(!r) return r;
+	return htmlW->writeDialogsStart(data);
+}
+
+Result BothWriter::writeDialogStart(const Data::DialogInfo &data) {
+	Expects(jsonW != nullptr);
+	Expects(htmlW != nullptr);
+
+	Result r = jsonW->writeDialogStart(data);
+	if(!r) return r;
+	return htmlW->writeDialogStart(data);
+}
+
+Result BothWriter::writeDialogSlice(const Data::MessagesSlice &data) {
+	Expects(jsonW != nullptr);
+	Expects(htmlW != nullptr);
+
+	Result r = jsonW->writeDialogSlice(data);
+	if(!r) return r;
+	return htmlW->writeDialogSlice(data);
+}
+
+Result BothWriter::writeDialogEnd() {
+	Expects(jsonW != nullptr);
+	Expects(htmlW != nullptr);
+
+	Result r = jsonW->writeDialogEnd();
+	if(!r) return r;
+	return htmlW->writeDialogEnd();
+}
+
+Result BothWriter::writeDialogsEnd() {
+	Expects(jsonW != nullptr);
+	Expects(htmlW != nullptr);
+
+	Result r = jsonW->writeDialogsEnd();
+	if(!r) return r;
+	return htmlW->writeDialogsEnd();
+}
+
+Result BothWriter::finish() {
+	Expects(jsonW != nullptr);
+	Expects(htmlW != nullptr);
+
+	Result r = jsonW->finish();
+	if(!r) return r;
+	return htmlW->finish();
+}
+
+QString BothWriter::mainFilePath() {
+	return _settings.path;
+}
+
+
+} // namespace Output
+} // namespace Export

Telegram/SourceFiles/export/output/export_output_both.h

@@ -0,0 +1,68 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "export/output/export_output_abstract.h"
+#include "export/output/export_output_file.h"
+#include "export/export_settings.h"
+#include "export/data/export_data_types.h"
+
+#include "export/output/export_output_json.h"
+#include "export/output/export_output_html.h"
+
+namespace Export {
+namespace Output {
+namespace details {
+
+} // namespace details
+
+class BothWriter : public AbstractWriter {
+public:
+	Format format() override {
+		return Format::Both;
+	}
+
+	Result start(
+		const Settings &settings,
+		const Environment &environment,
+		Stats *stats) override;
+
+	Result writePersonal(const Data::PersonalInfo &data) override;
+
+	Result writeUserpicsStart(const Data::UserpicsInfo &data) override;
+	Result writeUserpicsSlice(const Data::UserpicsSlice &data) override;
+	Result writeUserpicsEnd() override;
+
+	Result writeContactsList(const Data::ContactsList &data) override;
+
+	Result writeSessionsList(const Data::SessionsList &data) override;
+
+	Result writeOtherData(const Data::File &data) override;
+
+	Result writeDialogsStart(const Data::DialogsInfo &data) override;
+	Result writeDialogStart(const Data::DialogInfo &data) override;
+	Result writeDialogSlice(const Data::MessagesSlice &data) override;
+	Result writeDialogEnd() override;
+	Result writeDialogsEnd() override;
+
+	Result finish() override;
+
+	QString mainFilePath() override;
+
+private:
+
+	Settings _settings;
+	Environment _environment;
+	Stats *_stats = nullptr;
+
+	std::unique_ptr<JsonWriter> jsonW;
+	std::unique_ptr<HtmlWriter> htmlW;
+};
+
+} // namespace Output
+} // namespace Export

Telegram/SourceFiles/export/view/export_view_panel_controller.cpp

@@ -310,6 +310,11 @@ void PanelController::showProgress() {
 		_process->skipFile(randomId);
 	}, progress->lifetime());
 
+	progress->skipChatClicks(
+	) | rpl::start_with_next([=](uint64 randomId) {
+		_process->skipChat(randomId);
+	}, progress->lifetime());
+
 	progress->cancelClicks(
 	) | rpl::start_with_next([=] {
 		stopWithConfirmation();

Telegram/SourceFiles/export/view/export_view_progress.cpp

@@ -20,7 +20,7 @@ namespace Export {
 namespace View {
 namespace {
 
-constexpr auto kShowSkipFileTimeout = 5 * crl::time(1000);
+constexpr auto kShowSkipFileTimeout = 0.5 * crl::time(1000);
 
 } // namespace
 
@@ -241,7 +241,19 @@ ProgressWidget::ProgressWidget(
 	rpl::producer<Content> content)
 : RpWidget(parent)
 , _body(this)
-, _fileShowSkipTimer([=] { _skipFile->show(anim::type::normal); }) {
+, _skipChat(base::make_unique_q<Ui::FadeWrap<Ui::LinkButton>>(
+		_body->add(object_ptr<Ui::FixedHeightWidget>(
+			_body.data(),
+			st::defaultLinkButton.font->height + st::exportProgressRowSkip)),
+
+		object_ptr<Ui::LinkButton>(
+			this,
+			tr::lng_export_skip_chat(tr::now),
+			st::defaultLinkButton)))
+, _fileShowSkipTimer([=] { _skipFile->show(anim::type::normal); })
+
+
+{
 	widthValue(
 	) | rpl::start_with_next([=](int width) {
 		_body->resizeToWidth(width);
@@ -280,6 +292,11 @@ ProgressWidget::ProgressWidget(
 	setupBottomButton(_cancel.get());
 }
 
+rpl::producer<uint64> ProgressWidget::skipChatClicks() const {
+	return _skipChat->entity()->clicks(
+	) | rpl::map([=] { return _chatRandomId; });
+}
+
 rpl::producer<uint64> ProgressWidget::skipFileClicks() const {
 	return _skipFile->entity()->clicks(
 	) | rpl::map([=] { return _fileRandomId; });

Telegram/SourceFiles/export/view/export_view_progress.h

@@ -30,6 +30,7 @@ public:
 		QWidget *parent,
 		rpl::producer<Content> content);
 
+	rpl::producer<uint64> skipChatClicks() const;
 	rpl::producer<uint64> skipFileClicks() const;
 	rpl::producer<> cancelClicks() const;
 	rpl::producer<> doneClicks() const;
@@ -48,11 +49,13 @@ private:
 	std::vector<not_null<Row*>> _rows;
 
 	base::unique_qptr<Ui::FadeWrap<Ui::LinkButton>> _skipFile;
+	base::unique_qptr<Ui::FadeWrap<Ui::LinkButton>> _skipChat;
 	QPointer<Ui::FlatLabel> _about;
 	base::unique_qptr<Ui::RoundButton> _cancel;
 	base::unique_qptr<Ui::RoundButton> _done;
 	rpl::event_stream<> _doneClicks;
 
+	uint64 _chatRandomId = 0;
 	uint64 _fileRandomId = 0;
 	base::Timer _fileShowSkipTimer;
 

Telegram/SourceFiles/export/view/export_view_settings.cpp

@@ -75,6 +75,7 @@ void ChooseFormatBox(
 	box->setTitle(tr::lng_export_option_choose_format());
 	addFormatOption(tr::lng_export_option_html(tr::now), Format::Html);
 	addFormatOption(tr::lng_export_option_json(tr::now), Format::Json);
+	addFormatOption(tr::lng_export_option_both(tr::now), Format::Both);
 	box->addButton(tr::lng_settings_save(), [=] { done(group->value()); });
 	box->addButton(tr::lng_cancel(), [=] { box->closeBox(); });
 }
@@ -273,6 +274,7 @@ void SettingsWidget::setupPathAndFormat(
 	addLocationLabel(container);
 	addFormatOption(tr::lng_export_option_html(tr::now), Format::Html);
 	addFormatOption(tr::lng_export_option_json(tr::now), Format::Json);
+	addFormatOption(tr::lng_export_option_both(tr::now), Format::Both);
 }
 
 void SettingsWidget::addLocationLabel(
@@ -341,7 +343,8 @@ void SettingsWidget::addFormatAndLocationLabel(
 		return data.format;
 	}) | rpl::distinct_until_changed(
 	) | rpl::map([](Format format) {
-		const auto text = (format == Format::Html) ? "HTML" : "JSON";
+		const auto text = (format == Format::Html) ? "HTML" :
+			((format == Format::Json) ?  "JSON" : "Both");
 		return Ui::Text::Link(text, u"internal:edit_format"_q);
 	});
 	const auto label = container->add(

Telegram/cmake/td_export.cmake

@@ -28,6 +28,8 @@ PRIVATE
     export/output/export_output_html.h
     export/output/export_output_json.cpp
     export/output/export_output_json.h
+    export/output/export_output_both.cpp
+    export/output/export_output_both.h
     export/output/export_output_result.h
     export/output/export_output_stats.cpp
     export/output/export_output_stats.h