stats added
This commit is contained in:
parent
10fdcda095
commit
0aa196339b
|
@ -121,7 +121,8 @@ public class Crawler implements Runnable {
|
|||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
currentstate = "populate threads";
|
||||
//populate threads
|
||||
int threadcount = 4;
|
||||
try {
|
||||
|
@ -137,8 +138,10 @@ public class Crawler implements Runnable {
|
|||
threads.add(thr);
|
||||
thr.thread.start();
|
||||
}
|
||||
currentstate = "delete Double";
|
||||
long lastdoubledelete = System.currentTimeMillis();
|
||||
db.deleteDouble();
|
||||
currentstate = "crawl";
|
||||
while(crawl) {
|
||||
log.info("to Crawl: " + toCrawl.size() + " known: " + toknown.size() + " Time: " + dateform.format(new Date()));
|
||||
try {
|
||||
|
@ -310,6 +313,7 @@ public class Crawler implements Runnable {
|
|||
out += "\nToknown: " + toknown.size();
|
||||
out += "\nToSave: " + toSave.size();
|
||||
out += "\nrequested: " + requested.size();
|
||||
out += "\nRandomBuffer: " + db.getRandomCount();
|
||||
out += "\nRuntime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )";
|
||||
out += "\nState: " + currentstate;
|
||||
out += "\nThread Nr, todo size, requested, crawledsize, foundsize";
|
||||
|
|
|
@ -207,6 +207,10 @@ public class DB implements Runnable {
|
|||
return randombuffer.remove(0);
|
||||
}
|
||||
|
||||
public int getRandomCount() {
|
||||
return randombuffer.size();
|
||||
}
|
||||
|
||||
public LinkedList<String> restoreTemp() {
|
||||
ResultSet res = query("SELECT * FROM `ytcrawler`.`temp` LIMIT 500;");
|
||||
LinkedList<String> out = new LinkedList<>();
|
||||
|
|
Loading…
Reference in New Issue