MrBesen
/
dedup
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more options

This commit is contained in:
mrbesen 2022-11-08 21:51:29 +01:00
parent aae71672f5
commit 09d285d94d
Signed by: MrBesen
GPG Key ID: 596B2350DCD67504
4 changed files with 34 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
inc/dedup.h
View File

@ -35,6 +35,8 @@ private:
bool hardlink = true;
bool ignoredotfiles = false;
bool dryrun = true;
bool deleteDuplicates = true;
bool ignoreInodeID = true;
uint64_t deduplicatedBytes = 0; // amount of bytes deduplicated
uint64_t deduplicatedFiles = 0;
};

4
inc/fileindexer.h
View File

@ -16,10 +16,14 @@ public:
using map_t = std::multimap<uint64_t, std::shared_ptr<File>>;
map_t index(const SearchFolder& sf);
void setIgnoreDotFiles(bool b = true);
void setIgnoreInodeIDs(bool b = true);
private:
void handleFolderContent(dirent* dircont, const SearchFolder& sf, std::list<SearchFolder>& newfolders, map_t& newfiles);
void handleNewFile(ino_t inode, const std::string& path, map_t& newfiles);
std::set<uint64_t> knownInodes; // file inodes, that are known and should not be indexed again
bool ignoredotfiles = false;
bool ignoreInodeID = false;
};

18
src/dedup.cpp
View File

@ -38,6 +38,8 @@ Files& getFiles(uint64_t fs, std::map<uint64_t, Files>& m) {
void Dedup::start() {
FileIndexer::map_t foundfiles;
indexer.setIgnoreDotFiles(ignoredotfiles);
indexer.setIgnoreInodeIDs(ignoreInodeID);
for(const SearchFolder& sf : folders) {
foundfiles.merge(indexer.index(sf));
}
@ -192,7 +194,9 @@ bool Dedup::relinkFile(const std::string& linkbase, const std::string& replacedf
if(dryrun || !hardlink) {
Log::note << "delete " << replacedfile;
Log::note << "link " << linkbase << " -> " << replacedfile;
if(!deleteDuplicates) {
Log::note << "link " << linkbase << " -> " << replacedfile;
}
} else {
int res = ::unlink(replacedfile.c_str());
if(res != 0) {
@ -200,11 +204,13 @@ bool Dedup::relinkFile(const std::string& linkbase, const std::string& replacedf
return false;
}
res = ::link(linkbase.c_str(), replacedfile.c_str());
if(res != 0) {
Log::error << "link(" << linkbase << ", " << replacedfile << ") failed: " << strerror(errno) << " (" << errno << ')';
// TODO try to symlink?
return false;
if(!deleteDuplicates) {
res = ::link(linkbase.c_str(), replacedfile.c_str());
if(res != 0) {
Log::error << "link(" << linkbase << ", " << replacedfile << ") failed: " << strerror(errno) << " (" << errno << ')';
// TODO try to symlink?
return false;
}
}
}
return true;

22
src/fileindexer.cpp
View File

@ -35,6 +35,13 @@ std::multimap<uint64_t, std::shared_ptr<File>> FileIndexer::index(const SearchFo
return out;
}
void FileIndexer::setIgnoreDotFiles(bool b) {
ignoredotfiles = b;
}
void FileIndexer::setIgnoreInodeIDs(bool b) {
ignoreInodeID = b;
}
void FileIndexer::handleFolderContent(dirent* dircont, const SearchFolder& sf, std::list<SearchFolder>& newfolders, map_t& newfiles) {
std::string name(dircont->d_name);
@ -62,10 +69,12 @@ void FileIndexer::handleFolderContent(dirent* dircont, const SearchFolder& sf, s
void FileIndexer::handleNewFile(ino_t inode, const std::string& path, map_t& newfiles) {
// check for already scanned inodes
auto it = knownInodes.find(inode);
if(it != knownInodes.end()) {
Log::note << "found already detected file: " << inode << ' ' << path;
return;
if(!ignoreInodeID) {
auto it = knownInodes.find(inode);
if(it != knownInodes.end()) {
Log::note << "found already detected file: " << inode << ' ' << path;
return;
}
}
struct stat statbuf;
@ -78,8 +87,9 @@ void FileIndexer::handleNewFile(ino_t inode, const std::string& path, map_t& new
loff_t fileSize = statbuf.st_size;
nlink_t linkCount = statbuf.st_nlink;
knownInodes.insert(inode);
if(!ignoreInodeID) {
knownInodes.insert(inode);
}
newfiles.insert({fileSize, std::make_shared<File>(fileSize, inode, linkCount, path)});
}