thiesyy
/
YoutubeCrawler
forked from MrBesen/YoutubeCrawler
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

merged db stop after crawl, lock status

This commit is contained in:
mrbesen 2018-10-08 16:35:02 +02:00
parent c47ab96996
commit 33bdc8cf5c
2 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/de/mrbesen/youtubecrawler/Crawler.java
View File

@ -59,7 +59,6 @@ public class Crawler implements Runnable {
public void stop() { public void stop() {
crawl = false; crawl = false;
db.stop();
} }
public synchronized void addtoCrawl(String videoid) { public synchronized void addtoCrawl(String videoid) {
@ -220,6 +219,7 @@ public class Crawler implements Runnable {
log.info("get report"); log.info("get report");
int count = 0; int count = 0;
for (CrawlerThread crawlerThread : threads) { for (CrawlerThread crawlerThread : threads) {
currentstate = "get report: " + crawlerThread.thread.getName();
LinkedList<String>[] report = crawlerThread.report(); LinkedList<String>[] report = crawlerThread.report();
crawlcount+= report[0].size(); crawlcount+= report[0].size();
toSave.addAll(report[0]); toSave.addAll(report[0]);
@ -283,7 +283,10 @@ public class Crawler implements Runnable {
Main.getMain().stop(); Main.getMain().stop();
} }
} }
currentstate = "delete Double";
db.deleteDouble(); db.deleteDouble();
currentstate = "stop DB";
db.stop();
//end //end
long runtimes = (System.currentTimeMillis() - start) / 1000; long runtimes = (System.currentTimeMillis() - start) / 1000;
@ -292,6 +295,7 @@ public class Crawler implements Runnable {
int runtimem = (int) (runtimes / 60); int runtimem = (int) (runtimes / 60);
float vidps = (crawlcount / (float) runtimes);//videos per second float vidps = (crawlcount / (float) runtimes);//videos per second
log.info("Crawling Stopped. Runtime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )"); log.info("Crawling Stopped. Runtime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )");
currentstate = "ended.";
} }
public DB getDB() { public DB getDB() {
@ -317,10 +321,10 @@ public class Crawler implements Runnable {
out += "\nRuntime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )"; out += "\nRuntime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )";
out += "\nState: " + currentstate; out += "\nState: " + currentstate;
out += "\nDBSize: " + db.getDBSize(); out += "\nDBSize: " + db.getDBSize();
out += "\nThread Nr, todo size, requested, crawledsize, foundsize"; out += "\nThread Nr, todo size, locked, requested, crawledsize, foundsize";
for (int i = 0; i < threads.size(); i++) { for (int i = 0; i < threads.size(); i++) {
CrawlerThread thre = threads.get(i); CrawlerThread thre = threads.get(i);
out += "\n " + i + " " + thre.todo.size() + " " + thre.requested + " " + thre.crawled.size() + " " + thre.found.size(); out += "\n " + i + " " + thre.todo.size() + " " + thre.lockforreport + " " + thre.requested + " " + thre.crawled.size() + " " + thre.found.size();
} }
return out; return out;
} }

4
src/de/mrbesen/youtubecrawler/CrawlerThread.java
View File

@ -20,7 +20,7 @@ public class CrawlerThread implements Runnable {
LinkedList<String> found = new LinkedList<>();//videos this thread had found LinkedList<String> found = new LinkedList<>();//videos this thread had found
boolean requested = true;//is a request pending? boolean requested = true;//is a request pending?
private boolean lockforreport = false; boolean lockforreport = false;
public CrawlerThread( Crawler root) { public CrawlerThread( Crawler root) {
parent = root; parent = root;
@ -69,7 +69,7 @@ public class CrawlerThread implements Runnable {
*/ */
LinkedList<String>[] report() { LinkedList<String>[] report() {
lockforreport = true; lockforreport = true;
return new LinkedList[] {crawled, found}; return new LinkedList[] {(LinkedList) crawled.clone(), (LinkedList) found.clone()};
} }
private void crawl(String videoid) { private void crawl(String videoid) {