stats added
This commit is contained in:
parent
10fdcda095
commit
0aa196339b
|
@ -121,7 +121,8 @@ public class Crawler implements Runnable {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
currentstate = "populate threads";
|
||||||
//populate threads
|
//populate threads
|
||||||
int threadcount = 4;
|
int threadcount = 4;
|
||||||
try {
|
try {
|
||||||
|
@ -137,8 +138,10 @@ public class Crawler implements Runnable {
|
||||||
threads.add(thr);
|
threads.add(thr);
|
||||||
thr.thread.start();
|
thr.thread.start();
|
||||||
}
|
}
|
||||||
|
currentstate = "delete Double";
|
||||||
long lastdoubledelete = System.currentTimeMillis();
|
long lastdoubledelete = System.currentTimeMillis();
|
||||||
db.deleteDouble();
|
db.deleteDouble();
|
||||||
|
currentstate = "crawl";
|
||||||
while(crawl) {
|
while(crawl) {
|
||||||
log.info("to Crawl: " + toCrawl.size() + " known: " + toknown.size() + " Time: " + dateform.format(new Date()));
|
log.info("to Crawl: " + toCrawl.size() + " known: " + toknown.size() + " Time: " + dateform.format(new Date()));
|
||||||
try {
|
try {
|
||||||
|
@ -310,6 +313,7 @@ public class Crawler implements Runnable {
|
||||||
out += "\nToknown: " + toknown.size();
|
out += "\nToknown: " + toknown.size();
|
||||||
out += "\nToSave: " + toSave.size();
|
out += "\nToSave: " + toSave.size();
|
||||||
out += "\nrequested: " + requested.size();
|
out += "\nrequested: " + requested.size();
|
||||||
|
out += "\nRandomBuffer: " + db.getRandomCount();
|
||||||
out += "\nRuntime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )";
|
out += "\nRuntime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )";
|
||||||
out += "\nState: " + currentstate;
|
out += "\nState: " + currentstate;
|
||||||
out += "\nThread Nr, todo size, requested, crawledsize, foundsize";
|
out += "\nThread Nr, todo size, requested, crawledsize, foundsize";
|
||||||
|
|
|
@ -207,6 +207,10 @@ public class DB implements Runnable {
|
||||||
return randombuffer.remove(0);
|
return randombuffer.remove(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public int getRandomCount() {
|
||||||
|
return randombuffer.size();
|
||||||
|
}
|
||||||
|
|
||||||
public LinkedList<String> restoreTemp() {
|
public LinkedList<String> restoreTemp() {
|
||||||
ResultSet res = query("SELECT * FROM `ytcrawler`.`temp` LIMIT 500;");
|
ResultSet res = query("SELECT * FROM `ytcrawler`.`temp` LIMIT 500;");
|
||||||
LinkedList<String> out = new LinkedList<>();
|
LinkedList<String> out = new LinkedList<>();
|
||||||
|
|
Loading…
Reference in New Issue