stats
This commit is contained in:
parent
deb6f83152
commit
442c4c0630
|
@ -27,6 +27,7 @@ public class Crawler implements Runnable {
|
|||
private List<CrawlerThread> threads;//list of all threads
|
||||
private List<CrawlerThread> requested = new LinkedList<>();
|
||||
private static DateFormat dateform = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss");
|
||||
private long start;
|
||||
|
||||
private boolean crawl = true;
|
||||
private int crawlcount = 0;
|
||||
|
@ -82,7 +83,7 @@ public class Crawler implements Runnable {
|
|||
|
||||
@Override
|
||||
public void run() {
|
||||
long start = System.currentTimeMillis();
|
||||
start = System.currentTimeMillis();
|
||||
log.info("Try to load crawlfile");
|
||||
if(crawlfile.exists()) {
|
||||
listlock.writeLock().lock();
|
||||
|
@ -145,7 +146,9 @@ public class Crawler implements Runnable {
|
|||
Thread.yield();
|
||||
try {
|
||||
Thread.sleep(100);
|
||||
} catch(InterruptedException ignored) { }
|
||||
} catch(InterruptedException ignored) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
//nothing left?
|
||||
if(toknown.isEmpty() && toCrawl.isEmpty() && requested.size() == threads.size()) {//very uncommon
|
||||
|
@ -219,6 +222,8 @@ public class Crawler implements Runnable {
|
|||
|
||||
//end
|
||||
long runtimes = (System.currentTimeMillis() - start) / 1000;
|
||||
if(runtimes < 0)
|
||||
runtimes = 1;
|
||||
int runtimem = (int) (runtimes / 60);
|
||||
float vidps = (crawlcount / (float) runtimes);//videos per second
|
||||
log.info("Crawling Stopped. Runtime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )");
|
||||
|
@ -232,6 +237,18 @@ public class Crawler implements Runnable {
|
|||
return new Video();
|
||||
}
|
||||
|
||||
public void printStats() {
|
||||
long runtimes = (System.currentTimeMillis() - start) / 1000;
|
||||
if(runtimes < 0)
|
||||
runtimes = 1;
|
||||
int runtimem = (int) (runtimes / 60);
|
||||
float vidps = (crawlcount / (float) runtimes);//videos per second
|
||||
log.info("ToCrawl:" + toCrawl.size());
|
||||
log.info("Toknown:" + toknown.size());
|
||||
log.info("ToSave:" + toSave.size());
|
||||
log.info("Runtime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )");
|
||||
}
|
||||
|
||||
public static class Video {
|
||||
String id;
|
||||
int length;//the length of the video in seconds
|
||||
|
@ -239,4 +256,5 @@ public class Crawler implements Runnable {
|
|||
byte categorie;
|
||||
long created;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -15,6 +15,10 @@ public class HTTPS {
|
|||
con = (HttpsURLConnection) (new URL(url)).openConnection();
|
||||
con.setDoInput(true);
|
||||
con.setDefaultUseCaches(true);
|
||||
int code = con.getResponseCode();
|
||||
if(code != 200) {
|
||||
Log.l.warn("recived Response code: " + code);
|
||||
}
|
||||
}
|
||||
|
||||
public String getContent() throws IOException {
|
||||
|
|
|
@ -51,6 +51,9 @@ public class Main implements CommandHandler{
|
|||
cra.addtoCrawl(id);
|
||||
log.info("added.");
|
||||
}
|
||||
} else if(in.equalsIgnoreCase("stats")) {
|
||||
log.info("Getting Stats");
|
||||
cra.printStats();
|
||||
}
|
||||
}
|
||||
s.close();
|
||||
|
|
Loading…
Reference in New Issue