thiesyy
/
YoutubeCrawler
forked from MrBesen/YoutubeCrawler
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

merged db stop after crawl, lock status

This commit is contained in:
mrbesen 2018-10-08 16:35:02 +02:00
parent c47ab96996
commit 33bdc8cf5c
2 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/de/mrbesen/youtubecrawler/Crawler.java
View File

@ -59,7 +59,6 @@ public class Crawler implements Runnable {
public void stop() {
crawl = false;
db.stop();
}
public synchronized void addtoCrawl(String videoid) {
@ -220,6 +219,7 @@ public class Crawler implements Runnable {
log.info("get report");
int count = 0;
for (CrawlerThread crawlerThread : threads) {
currentstate = "get report: " + crawlerThread.thread.getName();
LinkedList<String>[] report = crawlerThread.report();
crawlcount+= report[0].size();
toSave.addAll(report[0]);
@ -283,7 +283,10 @@ public class Crawler implements Runnable {
Main.getMain().stop();
}
}
currentstate = "delete Double";
db.deleteDouble();
currentstate = "stop DB";
db.stop();
//end
long runtimes = (System.currentTimeMillis() - start) / 1000;
@ -292,6 +295,7 @@ public class Crawler implements Runnable {
int runtimem = (int) (runtimes / 60);
float vidps = (crawlcount / (float) runtimes);//videos per second
log.info("Crawling Stopped. Runtime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )");
currentstate = "ended.";
}
public DB getDB() {
@ -317,10 +321,10 @@ public class Crawler implements Runnable {
out += "\nRuntime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )";
out += "\nState: " + currentstate;
out += "\nDBSize: " + db.getDBSize();
out += "\nThread Nr, todo size, requested, crawledsize, foundsize";
out += "\nThread Nr, todo size, locked, requested, crawledsize, foundsize";
for (int i = 0; i < threads.size(); i++) {
CrawlerThread thre = threads.get(i);
out += "\n " + i + " " + thre.todo.size() + " " + thre.requested + " " + thre.crawled.size() + " " + thre.found.size();
out += "\n " + i + " " + thre.todo.size() + " " + thre.lockforreport + " " + thre.requested + " " + thre.crawled.size() + " " + thre.found.size();
}
return out;
}

4
src/de/mrbesen/youtubecrawler/CrawlerThread.java
View File

@ -20,7 +20,7 @@ public class CrawlerThread implements Runnable {
LinkedList<String> found = new LinkedList<>();//videos this thread had found
boolean requested = true;//is a request pending?
private boolean lockforreport = false;
boolean lockforreport = false;
public CrawlerThread( Crawler root) {
parent = root;
@ -69,7 +69,7 @@ public class CrawlerThread implements Runnable {
*/
LinkedList<String>[] report() {
lockforreport = true;
return new LinkedList[] {crawled, found};
return new LinkedList[] {(LinkedList) crawled.clone(), (LinkedList) found.clone()};
}
private void crawl(String videoid) {