Add downloaded files cache in export.

Also write downloaded photo paths.

Telegram/SourceFiles/base/value_ordering.h

@@ -7,6 +7,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 */
 #pragma once
 
+#include <utility>
+
 namespace base {
 namespace details {
 

Telegram/SourceFiles/export/data/export_data_types.h

@@ -48,6 +48,10 @@ struct UserpicsInfo {
 struct FileLocation {
 	int dcId = 0;
 	MTPInputFileLocation data;
+
+	explicit operator bool() const {
+		return dcId != 0;
+	}
 };
 
 struct File {

Telegram/SourceFiles/export/export_api_wrap.cpp

@@ -11,6 +11,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #include "export/data/export_data_types.h"
 #include "export/output/export_output_file.h"
 #include "mtproto/rpc_sender.h"
+#include "base/value_ordering.h"
 #include "base/bytes.h"
 
 #include <deque>
@@ -25,13 +26,61 @@ constexpr auto kFileNextRequestDelay = TimeMs(20);
 constexpr auto kChatsSliceLimit = 100;
 constexpr auto kMessagesSliceLimit = 100;
 constexpr auto kFileMaxSize = 1500 * 1024 * 1024;
+constexpr auto kLocationCacheSize = 100'000;
 
-bool FileIsAvailable(const Data::File &file) {
-	return file.relativePath.isEmpty() && (file.location.dcId != 0);
+struct LocationKey {
+	uint64 type;
+	uint64 id;
+
+	inline bool operator<(const LocationKey &other) const {
+		return std::tie(type, id) < std::tie(other.type, other.id);
+	}
+};
+
+std::tuple<const uint64 &, const uint64 &> value_ordering_helper(const LocationKey &value) {
+	return std::tie(
+		value.type,
+		value.id);
+}
+
+LocationKey ComputeLocationKey(const Data::FileLocation &value) {
+	auto result = LocationKey();
+	result.type = value.dcId;
+	value.data.match([&](const MTPDinputFileLocation &data) {
+		result.type |= (1ULL << 24);
+		result.type |= (uint64(uint32(data.vlocal_id.v)) << 32);
+		result.id = data.vvolume_id.v;
+	}, [&](const MTPDinputDocumentFileLocation &data) {
+		result.type |= (2ULL << 24);
+		result.id = data.vid.v;
+	}, [&](const MTPDinputSecureFileLocation &data) {
+		result.type |= (3ULL << 24);
+		result.id = data.vid.v;
+	}, [&](const MTPDinputEncryptedFileLocation &data) {
+		result.type |= (4ULL << 24);
+		result.id = data.vid.v;
+	});
+	return result;
 }
 
 } // namespace
 
+class ApiWrap::LoadedFileCache {
+public:
+	using Location = Data::FileLocation;
+
+	LoadedFileCache(int limit);
+
+	void save(const Location &location, const QString &relativePath);
+	base::optional<QString> find(const Location &location) const;
+
+private:
+	int _limit = 0;
+	std::map<LocationKey, QString> _map;
+	std::deque<LocationKey> _list;
+
+};
+
 struct ApiWrap::UserpicsProcess {
 	FnMut<void(Data::UserpicsInfo&&)> start;
 	Fn<void(Data::UserpicsSlice&&)> handleSlice;
@@ -94,6 +143,32 @@ struct ApiWrap::DialogsProcess::Single {
 
 };
 
+ApiWrap::LoadedFileCache::LoadedFileCache(int limit) : _limit(limit) {
+	Expects(limit >= 0);
+}
+
+void ApiWrap::LoadedFileCache::save(
+		const Location &location,
+		const QString &relativePath) {
+	const auto key = ComputeLocationKey(location);
+	_map[key] = relativePath;
+	_list.push_back(key);
+	if (_list.size() > _limit) {
+		const auto key = _list.front();
+		_list.pop_front();
+		_map.erase(key);
+	}
+}
+
+base::optional<QString> ApiWrap::LoadedFileCache::find(
+		const Location &location) const {
+	const auto key = ComputeLocationKey(location);
+	if (const auto i = _map.find(key); i != end(_map)) {
+		return i->second;
+	}
+	return base::none;
+}
+
 ApiWrap::FileProcess::FileProcess(const QString &path) : file(path) {
 }
 
@@ -129,7 +204,8 @@ auto ApiWrap::fileRequest(const Data::FileLocation &location, int offset) {
 }
 
 ApiWrap::ApiWrap(Fn<void(FnMut<void()>)> runner)
-: _mtp(std::move(runner)) {
+: _mtp(std::move(runner))
+, _fileCache(std::make_unique<LoadedFileCache>(kLocationCacheSize)) {
 }
 
 rpl::producer<RPCError> ApiWrap::errors() const {
@@ -613,11 +689,11 @@ bool ApiWrap::processFileLoad(
 
 	if (!file.relativePath.isEmpty()) {
 		return true;
-	} else if (writePreloadedFile(file)) {
-		return true;
-	} else if (!FileIsAvailable(file)) {
+	} else if (!file.location) {
 		file.skipReason = SkipReason::Unavailable;
 		return true;
+	} else if (writePreloadedFile(file)) {
+		return true;
 	}
 
 	using Type = MediaSettings::Type;
@@ -656,11 +732,15 @@ bool ApiWrap::writePreloadedFile(Data::File &file) {
 
 	using namespace Output;
 
-	if (!file.content.isEmpty()) {
+	if (const auto path = _fileCache->find(file.location)) {
+		file.relativePath = *path;
+		return true;
+	} else if (!file.content.isEmpty()) {
 		const auto process = prepareFileProcess(file);
 		auto &output = _fileProcess->file;
 		if (output.writeBlock(file.content) == File::Result::Success) {
 			file.relativePath = process->relativePath;
+			_fileCache->save(file.location, file.relativePath);
 			return true;
 		}
 		error(QString("Could not write '%1'.").arg(process->relativePath));
@@ -670,7 +750,7 @@ bool ApiWrap::writePreloadedFile(Data::File &file) {
 
 void ApiWrap::loadFile(const Data::File &file, FnMut<void(QString)> done) {
 	Expects(_fileProcess == nullptr);
-	Expects(FileIsAvailable(file));
+	Expects(file.location.dcId != 0);
 
 	_fileProcess = prepareFileProcess(file);
 	_fileProcess->done = std::move(done);
@@ -767,7 +847,10 @@ void ApiWrap::filePartDone(int offset, const MTPupload_File &result) {
 			return;
 		}
 	}
+
 	auto process = base::take(_fileProcess);
+	const auto relativePath = process->relativePath;
+	_fileCache->save(process->location, relativePath);
 	process->done(process->relativePath);
 }
 

Telegram/SourceFiles/export/export_api_wrap.h

@@ -60,6 +60,7 @@ private:
 	struct UserpicsProcess;
 	struct FileProcess;
 	struct DialogsProcess;
+	class LoadedFileCache;
 
 	void startMainSession(FnMut<void()> done);
 
@@ -111,6 +112,7 @@ private:
 	std::unique_ptr<Settings> _settings;
 	MTPInputUser _user = MTP_inputUserSelf();
 
+	std::unique_ptr<LoadedFileCache> _fileCache;
 	std::unique_ptr<UserpicsProcess> _userpicsProcess;
 	std::unique_ptr<FileProcess> _fileProcess;
 	std::unique_ptr<DialogsProcess> _dialogsProcess;

Telegram/SourceFiles/export/export_settings.h

@@ -71,7 +71,8 @@ struct Settings {
 	}
 
 	static inline Types DefaultFullChats() {
-		return Type::PersonalChats;
+		return Type::PersonalChats
+			| Type::BotChats;
 	}
 
 };

Telegram/SourceFiles/export/output/export_output_text.cpp

@@ -178,6 +178,31 @@ QByteArray SerializeMessage(
 		}
 	};
 
+	using SkipReason = Data::File::SkipReason;
+	const auto pushPath = [&](
+			const Data::File &file,
+			const QByteArray &label) {
+		Expects(!file.relativePath.isEmpty()
+			|| file.skipReason != SkipReason::None);
+
+		push(label, [&]() -> QByteArray {
+			switch (file.skipReason) {
+			case SkipReason::Unavailable: return "(file unavailable)";
+			case SkipReason::FileSize: return "(file too large)";
+			case SkipReason::FileType: return "(file skipped)";
+			case SkipReason::None: return FormatFilePath(file);
+			}
+			Unexpected("Skip reason while writing file path.");
+		}());
+	};
+	const auto pushPhoto = [&](const Image &image) {
+		pushPath(image.file, "Photo");
+		if (image.width && image.height) {
+			push("Width", NumberToString(image.width));
+			push("Height", NumberToString(image.height));
+		}
+	};
+
 	message.action.content.match([&](const ActionChatCreate &data) {
 		pushActor();
 		pushAction("Create group");
@@ -190,7 +215,7 @@ QByteArray SerializeMessage(
 	}, [&](const ActionChatEditPhoto &data) {
 		pushActor();
 		pushAction("Edit group photo");
-		push("Photo", FormatFilePath(data.photo.image.file));
+		pushPhoto(data.photo.image);
 	}, [&](const ActionChatDeletePhoto &data) {
 		pushActor();
 		pushAction("Delete group photo");
@@ -306,28 +331,29 @@ QByteArray SerializeMessage(
 	}
 
 	message.media.content.match([&](const Photo &photo) {
+		pushPhoto(photo.image);
 		pushTTL();
 	}, [&](const Document &data) {
-		const auto pushPath = [&](const QByteArray &label) {
-			push(label, FormatFilePath(data.file));
+		const auto pushMyPath = [&](const QByteArray &label) {
+			return pushPath(data.file, label);
 		};
 		if (data.isSticker) {
-			pushPath("Sticker");
+			pushMyPath("Sticker");
 			push("Emoji", data.stickerEmoji);
 		} else if (data.isVideoMessage) {
-			pushPath("Video message");
+			pushMyPath("Video message");
 		} else if (data.isVoiceMessage) {
-			pushPath("Voice message");
+			pushMyPath("Voice message");
 		} else if (data.isAnimated) {
-			pushPath("Animation");
+			pushMyPath("Animation");
 		} else if (data.isVideoFile) {
-			pushPath("Video file");
+			pushMyPath("Video file");
 		} else if (data.isAudioFile) {
-			pushPath("Audio file");
+			pushMyPath("Audio file");
 			push("Performer", data.songPerformer);
 			push("Title", data.songTitle);
 		} else {
-			pushPath("File");
+			pushMyPath("File");
 		}
 		if (!data.isSticker) {
 			push("Mime type", data.mime);