Clear old versions of Storage::Cache::Database.

This commit is contained in:
John Preston 2018-07-30 23:34:23 +03:00
parent 9147c12687
commit cb371f09ac
26 changed files with 404 additions and 85 deletions

View File

@ -7,7 +7,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#pragma once
#include "core/basic_types.h"
#include "data/data_types.h"
#include "data/data_peer.h"

View File

@ -7,6 +7,13 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#pragma once
#include "base/build_config.h"
#include "base/ordered_set.h"
#include "base/unique_function.h"
#include "base/functors.h"
#include <QtGlobal>
#include <string>
#include <exception>
#include <memory>
@ -14,11 +21,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include <functional>
#include <gsl/gsl>
#include "base/build_config.h"
#include "base/ordered_set.h"
#include "base/unique_function.h"
#include "base/functors.h"
namespace func = base::functors;
using gsl::not_null;
@ -45,6 +47,3 @@ using float64 = double;
using TimeMs = int64;
using TimeId = int32;
#define qsl(s) QStringLiteral(s)
#define qstr(s) QLatin1String((s), sizeof(s) - 1)

View File

@ -0,0 +1,71 @@
/*
This file is part of Telegram Desktop,
the official desktop application for the Telegram messaging service.
For license and copyright information please follow this link:
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#pragma once
#include "base/algorithm.h"
#include <atomic>
namespace base {
class binary_guard {
public:
binary_guard() = default;
binary_guard(binary_guard &&other);
binary_guard &operator=(binary_guard &&other);
~binary_guard();
bool alive() const;
private:
void destroy();
std::atomic<bool> *_bothAlive = nullptr;
friend std::pair<binary_guard, binary_guard> make_binary_guard();
};
inline binary_guard::binary_guard(binary_guard &&other)
: _bothAlive(base::take(other._bothAlive)) {
}
inline binary_guard &binary_guard::operator=(binary_guard &&other) {
if (this != &other) {
destroy();
_bothAlive = base::take(other._bothAlive);
}
return *this;
}
inline binary_guard::~binary_guard() {
destroy();
}
inline bool binary_guard::alive() const {
return _bothAlive && _bothAlive->load();
}
inline void binary_guard::destroy() {
if (_bothAlive) {
auto old = true;
if (!_bothAlive->compare_exchange_strong(old, false)) {
delete _bothAlive;
}
}
}
inline std::pair<binary_guard, binary_guard> make_binary_guard() {
auto result = std::pair<binary_guard, binary_guard>();
result.first._bothAlive
= result.second._bothAlive
= new std::atomic<bool>(true);
return result;
}
} // namespace base

View File

@ -9,7 +9,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include "base/bytes.h"
#include "base/algorithm.h"
#include "core/basic_types.h"
#include "base/basic_types.h"
extern "C" {
#include <openssl/bn.h>

View File

@ -7,8 +7,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#pragma once
#include "core/basic_types.h"
class SingleTimer : public QTimer { // single shot timer with check
Q_OBJECT

View File

@ -8,7 +8,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#pragma once
#include "logs.h"
#include "core/basic_types.h"
#include "base/basic_types.h"
#include "base/flags.h"
#include "base/algorithm.h"
#include "base/assertion.h"
@ -21,6 +21,9 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include <cmath>
#include <set>
#define qsl(s) QStringLiteral(s)
#define qstr(s) QLatin1String((s), sizeof(s) - 1)
// Define specializations for QByteArray for Qt 5.3.2, because
// QByteArray in Qt 5.3.2 doesn't declare "pointer" subtype.
#ifdef OS_MAC_OLD

View File

@ -10,7 +10,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include "export/export_settings.h"
#include "export/output/export_output_file.h"
#include "core/mime_type.h"
#include "core/utils.h"
#include <QtCore/QDateTime>
#include <QtCore/QRegularExpression>
#include <QtGui/QImageReader>

View File

@ -14,7 +14,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include "mtproto/rpc_sender.h"
#include "base/value_ordering.h"
#include "base/bytes.h"
#include <set>
#include <deque>

View File

@ -7,8 +7,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#pragma once
#include "core/basic_types.h"
class FileLoader;
namespace InlineBots {

View File

@ -7,7 +7,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#pragma once
#include "core/basic_types.h"
#include "history/history_location_manager.h"
namespace InlineBots {

View File

@ -7,7 +7,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#pragma once
#include "core/basic_types.h"
#include "base/basic_types.h"
namespace Core {
class Launcher;

View File

@ -12,7 +12,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include <QtCore/QString>
#include <QtCore/QByteArray>
#include <rpl/details/callable.h>
#include "core/basic_types.h"
#include "base/basic_types.h"
#include "base/match_method.h"
#include "base/flags.h"
#include "base/bytes.h"

View File

@ -7,8 +7,9 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#pragma once
#include <rpl/details/callable.h>
#include "base/flat_set.h"
#include "core/utils.h"
#include <rpl/details/callable.h>
class RPCError {
public:

View File

@ -78,7 +78,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include "base/flat_map.h"
#include "base/weak_ptr.h"
#include "core/basic_types.h"
#include "base/basic_types.h"
#include "logs.h"
#include "core/utils.h"
#include "config.h"

View File

@ -0,0 +1,113 @@
/*
This file is part of Telegram Desktop,
the official desktop application for the Telegram messaging service.
For license and copyright information please follow this link:
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#include "storage/cache/storage_cache_cleaner.h"
#include <crl/crl.h>
#include <QtCore/QDir>
#include <QtCore/QFile>
#include <unordered_map>
#include <set>
namespace Storage {
namespace Cache {
namespace details {
class CleanerObject {
public:
using Wrapper = Cache::Cleaner;
CleanerObject(
crl::weak_on_queue<CleanerObject> weak,
const QString &base,
base::binary_guard &&guard,
FnMut<void(Error)> done);
private:
void start();
void scheduleNext();
void cleanNext();
void done();
crl::weak_on_queue<CleanerObject> _weak;
QString _base, _errorPath;
std::vector<QString> _queue;
base::binary_guard _guard;
FnMut<void(Error)> _done;
};
CleanerObject::CleanerObject(
crl::weak_on_queue<CleanerObject> weak,
const QString &base,
base::binary_guard &&guard,
FnMut<void(Error)> done)
: _weak(std::move(weak))
, _base(base)
, _guard(std::move(guard))
, _done(std::move(done)) {
start();
}
void CleanerObject::start() {
const auto entries = QDir(_base).entryList(
QDir::Dirs | QDir::NoDotAndDotDot);
for (const auto entry : entries) {
_queue.push_back(entry);
}
if (const auto version = ReadVersionValue(_base)) {
_queue.erase(
ranges::remove(_queue, QString::number(*version)),
end(_queue));
scheduleNext();
} else {
_errorPath = VersionFilePath(_base);
done();
}
}
void CleanerObject::scheduleNext() {
if (_queue.empty()) {
done();
return;
}
_weak.with([](CleanerObject &that) {
if (that._guard.alive()) {
that.cleanNext();
}
});
}
void CleanerObject::cleanNext() {
const auto path = _base + _queue.back();
_queue.pop_back();
if (!QDir(path).removeRecursively()) {
_errorPath = path;
}
scheduleNext();
}
void CleanerObject::done() {
if (_done) {
_done(_errorPath.isEmpty()
? Error::NoError()
: Error{ Error::Type::IO, _errorPath });
}
}
} // namespace details
Cleaner::Cleaner(
const QString &base,
base::binary_guard &&guard,
FnMut<void(Error)> done)
: _wrapped(base, std::move(guard), std::move(done)) {
}
Cleaner::~Cleaner() = default;
} // namespace Cache
} // namespace Storage

View File

@ -0,0 +1,35 @@
/*
This file is part of Telegram Desktop,
the official desktop application for the Telegram messaging service.
For license and copyright information please follow this link:
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#pragma once
#include "storage/cache/storage_cache_types.h"
#include "base/binary_guard.h"
namespace Storage {
namespace Cache {
namespace details {
class CleanerObject;
} // namespace details
class Cleaner {
public:
Cleaner(
const QString &base,
base::binary_guard &&guard,
FnMut<void(Error)> done);
~Cleaner();
private:
using Implementation = details::CleanerObject;
crl::object_on_queue<Implementation> _wrapped;
};
} // namespace Cache
} // namespace Storage

View File

@ -7,6 +7,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#include "storage/cache/storage_cache_database.h"
#include "storage/cache/storage_cache_cleaner.h"
#include "storage/storage_encryption.h"
#include "storage/storage_encrypted_file.h"
#include "base/flat_set.h"
@ -160,9 +161,11 @@ class Database {
public:
using Wrapper = Cache::Database;
using Settings = Wrapper::Settings;
Database(const QString &path, const Settings &settings);
Database(
crl::weak_on_queue<Database> weak,
const QString &path,
const Settings &settings);
using Error = Wrapper::Error;
void open(EncryptionKey key, FnMut<void(Error)> done);
void close(FnMut<void()> done);
@ -173,7 +176,6 @@ public:
void clear(FnMut<void(Error)> done);
private:
using Version = int32;
struct Entry {
Entry() = default;
Entry(PlaceId place, uint8 tag, uint32 checksum, size_type size);
@ -183,6 +185,10 @@ private:
size_type size = 0;
PlaceId place = { { 0 } };
};
struct CleanerWrap {
std::unique_ptr<Cleaner> object;
base::binary_guard guard;
};
template <typename Callback, typename ...Args>
void invokeCallback(Callback &&callback, Args &&...args);
@ -215,6 +221,10 @@ private:
QString writeKeyPlace(const Key &key, size_type size, uint32 checksum);
void writeMultiRemove();
void createCleaner();
void cleanerDone(Error error);
crl::weak_on_queue<Database> _weak;
QString _base, _path;
Settings _settings;
EncryptionKey _key;
@ -222,6 +232,8 @@ private:
std::unordered_map<Key, Entry> _map;
std::set<Key> _removing;
CleanerWrap _cleaner;
};
Database::Entry::Entry(
@ -235,8 +247,12 @@ Database::Entry::Entry(
, size(size) {
}
Database::Database(const QString &path, const Settings &settings)
: _base(QDir(path).absolutePath() + '/')
Database::Database(
crl::weak_on_queue<Database> weak,
const QString &path,
const Settings &settings)
: _weak(std::move(weak))
, _base(ComputeBasePath(path))
, _settings(settings) {
}
@ -253,7 +269,7 @@ void Database::invokeCallback(Callback &&callback, Args &&...args) {
}
}
auto Database::ioError(const QString &path) const -> Error {
Error Database::ioError(const QString &path) const {
return { Error::Type::IO, path };
}
@ -276,15 +292,15 @@ void Database::open(EncryptionKey key, FnMut<void(Error)> done) {
break;
case File::Result::Failed: {
const auto available = findAvailableVersion();
const auto retry = openBinlog(available, File::Mode::Write, key);
if (retry == File::Result::Success) {
if (writeVersion(available)) {
if (writeVersion(available)) {
const auto open = openBinlog(available, File::Mode::Write, key);
if (open == File::Result::Success) {
invokeCallback(done, Error::NoError());
} else {
invokeCallback(done, ioError(versionPath()));
invokeCallback(done, ioError(binlogPath(available)));
}
} else {
invokeCallback(done, ioError(binlogPath(available)));
invokeCallback(done, ioError(versionPath()));
}
} break;
default: Unexpected("Result from Database::openBinlog.");
@ -296,7 +312,7 @@ QString Database::computePath(Version version) const {
}
QString Database::binlogFilename() const {
return qsl("binlog");
return QStringLiteral("binlog");
}
QString Database::binlogPath(Version version) const {
@ -316,6 +332,7 @@ File::Result Database::openBinlog(
if (result == File::Result::Success) {
_path = computePath(version);
_key = std::move(key);
createCleaner();
readBinlog();
}
return result;
@ -466,6 +483,7 @@ bool Database::readRecordMultiRemove(bytes::const_span data) {
}
void Database::close(FnMut<void()> done) {
_cleaner = CleanerWrap();
_binlog.close();
invokeCallback(done);
}
@ -604,6 +622,24 @@ void Database::writeMultiRemove() {
}
}
void Database::createCleaner() {
auto [left, right] = base::make_binary_guard();
_cleaner.guard = std::move(left);
auto done = [weak = _weak](Error error) {
weak.with([=](Database &that) {
that.cleanerDone(error);
});
};
_cleaner.object = std::make_unique<Cleaner>(
_base,
std::move(right),
std::move(done));
}
void Database::cleanerDone(Error error) {
_cleaner = CleanerWrap();
}
void Database::clear(FnMut<void(Error)> done) {
Expects(_key.empty());
@ -631,36 +667,18 @@ auto Database::findAvailableVersion() const -> Version {
}
QString Database::versionPath() const {
return _base + "version";
return VersionFilePath(_base);
}
bool Database::writeVersion(Version version) {
const auto bytes = QByteArray::fromRawData(
reinterpret_cast<const char*>(&version),
sizeof(version));
if (!QDir().mkpath(_base)) {
return false;
}
QFile file(versionPath());
if (!file.open(QIODevice::WriteOnly)) {
return false;
} else if (file.write(bytes) != bytes.size()) {
return false;
}
return file.flush();
return WriteVersionValue(_base, version);
}
auto Database::readVersion() const -> Version {
QFile file(versionPath());
if (!file.open(QIODevice::ReadOnly)) {
return Version();
if (const auto result = ReadVersionValue(_base)) {
return *result;
}
const auto bytes = file.read(sizeof(Version));
if (bytes.size() != sizeof(Version)) {
return Version();
}
return *reinterpret_cast<const Version*>(bytes.data());
return Version();
}
QString Database::placePath(PlaceId place) const {

View File

@ -7,7 +7,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#pragma once
#include "core/basic_types.h"
#include "storage/cache/storage_cache_types.h"
#include "base/basic_types.h"
#include <crl/crl_object_on_queue.h>
#include <QtCore/QString>
@ -42,20 +43,6 @@ public:
};
Database(const QString &path, const Settings &settings);
struct Error {
enum class Type {
None,
IO,
WrongKey,
LockFailed,
};
Type type = Type::None;
QString path;
static Error NoError() {
return Error();
}
};
void open(EncryptionKey key, FnMut<void(Error)> done);
void close(FnMut<void()> done);

View File

@ -9,8 +9,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include "storage/cache/storage_cache_database.h"
#include "storage/storage_encryption.h"
#include <crl/crl.h>
#include <thread>
using namespace Storage::Cache;
@ -29,8 +29,8 @@ const auto TestValue2 = QByteArray("bytetestbytetestb");
crl::semaphore Semaphore;
auto Result = Database::Error();
const auto GetResult = [](Database::Error error) {
auto Result = Error();
const auto GetResult = [](Error error) {
Result = error;
Semaphore.release();
};
@ -49,15 +49,15 @@ TEST_CASE("encrypted cache db", "[storage_cache_database]") {
db.clear(GetResult);
Semaphore.acquire();
REQUIRE(Result.type == Database::Error::Type::None);
REQUIRE(Result.type == Error::Type::None);
db.open(key, GetResult);
Semaphore.acquire();
REQUIRE(Result.type == Database::Error::Type::None);
REQUIRE(Result.type == Error::Type::None);
db.put(Key{ 0, 1 }, TestValue1, GetResult);
Semaphore.acquire();
REQUIRE(Result.type == Database::Error::Type::None);
REQUIRE(Result.type == Error::Type::None);
db.close([&] { Semaphore.release(); });
Semaphore.acquire();
@ -67,7 +67,7 @@ TEST_CASE("encrypted cache db", "[storage_cache_database]") {
db.open(key, GetResult);
Semaphore.acquire();
REQUIRE(Result.type == Database::Error::Type::None);
REQUIRE(Result.type == Error::Type::None);
db.get(Key{ 0, 1 }, GetValue);
Semaphore.acquire();
@ -75,7 +75,7 @@ TEST_CASE("encrypted cache db", "[storage_cache_database]") {
db.put(Key{ 1, 0 }, TestValue2, GetResult);
Semaphore.acquire();
REQUIRE(Result.type == Database::Error::Type::None);
REQUIRE(Result.type == Error::Type::None);
db.get(Key{ 1, 0 }, GetValue);
Semaphore.acquire();
@ -93,7 +93,7 @@ TEST_CASE("encrypted cache db", "[storage_cache_database]") {
db.open(key, GetResult);
Semaphore.acquire();
REQUIRE(Result.type == Database::Error::Type::None);
REQUIRE(Result.type == Error::Type::None);
db.get(Key{ 0, 1 }, GetValue);
Semaphore.acquire();

View File

@ -0,0 +1,55 @@
/*
This file is part of Telegram Desktop,
the official desktop application for the Telegram messaging service.
For license and copyright information please follow this link:
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#include "storage/cache/storage_cache_types.h"
#include <QtCore/QDir>
namespace Storage {
namespace Cache {
QString ComputeBasePath(const QString &original) {
const auto result = QDir(original).absolutePath();
return result.endsWith('/') ? result : (result + '/');
}
QString VersionFilePath(const QString &base) {
Expects(base.endsWith('/'));
return base + QStringLiteral("version");
}
base::optional<Version> ReadVersionValue(const QString &base) {
QFile file(VersionFilePath(base));
if (!file.open(QIODevice::ReadOnly)) {
return base::none;
}
const auto bytes = file.read(sizeof(Version));
if (bytes.size() != sizeof(Version)) {
return base::none;
}
return *reinterpret_cast<const Version*>(bytes.data());
}
bool WriteVersionValue(const QString &base, Version value) {
if (!QDir().mkpath(base)) {
return false;
}
const auto bytes = QByteArray::fromRawData(
reinterpret_cast<const char*>(&value),
sizeof(value));
QFile file(VersionFilePath(base));
if (!file.open(QIODevice::WriteOnly)) {
return false;
} else if (file.write(bytes) != bytes.size()) {
return false;
}
return file.flush();
}
} // namespace Cache
} // namespace Storage

View File

@ -0,0 +1,41 @@
/*
This file is part of Telegram Desktop,
the official desktop application for the Telegram messaging service.
For license and copyright information please follow this link:
https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#pragma once
#include "base/basic_types.h"
#include "base/optional.h"
namespace Storage {
namespace Cache {
using Version = int32;
QString ComputeBasePath(const QString &original);
QString VersionFilePath(const QString &base);
base::optional<Version> ReadVersionValue(const QString &base);
bool WriteVersionValue(const QString &base, Version value);
struct Error {
enum class Type {
None,
IO,
WrongKey,
LockFailed,
};
Type type = Type::None;
QString path;
static Error NoError();
};
inline Error Error::NoError() {
return Error();
}
} // namespace Cache
} // namespace Storage

View File

@ -7,7 +7,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#pragma once
#include "core/basic_types.h"
#include "storage/file_download.h"
#include "auth_session.h"

View File

@ -7,7 +7,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#pragma once
#include "core/basic_types.h"
#include "base/basic_types.h"
#include <QtCore/QFile>
namespace Storage {

View File

@ -7,7 +7,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#pragma once
#include "core/basic_types.h"
#include <QtCore/QTimer>
#include <QtGui/QColor>

View File

@ -52,8 +52,12 @@
'<(src_loc)/storage/storage_file_lock_posix.cpp',
'<(src_loc)/storage/storage_file_lock_win.cpp',
'<(src_loc)/storage/storage_file_lock.h',
'<(src_loc)/storage/cache/storage_cache_cleaner.cpp',
'<(src_loc)/storage/cache/storage_cache_cleaner.h',
'<(src_loc)/storage/cache/storage_cache_database.cpp',
'<(src_loc)/storage/cache/storage_cache_database.h',
'<(src_loc)/storage/cache/storage_cache_types.cpp',
'<(src_loc)/storage/cache/storage_cache_types.h',
],
'conditions': [[ 'build_macold', {
'xcode_settings': {

View File

@ -1,5 +1,7 @@
<(src_loc)/base/algorithm.h
<(src_loc)/base/assertion.h
<(src_loc)/base/basic_types.h
<(src_loc)/base/binary_guard.h
<(src_loc)/base/build_config.h
<(src_loc)/base/bytes.h
<(src_loc)/base/flags.h
@ -136,7 +138,6 @@
<(src_loc)/chat_helpers/tabbed_section.h
<(src_loc)/chat_helpers/tabbed_selector.cpp
<(src_loc)/chat_helpers/tabbed_selector.h
<(src_loc)/core/basic_types.h
<(src_loc)/core/changelogs.cpp
<(src_loc)/core/changelogs.h
<(src_loc)/core/click_handler.cpp