forked from MrBesen/YoutubeCrawler
stats
This commit is contained in:
parent
deb6f83152
commit
442c4c0630
|
@ -27,6 +27,7 @@ public class Crawler implements Runnable {
|
||||||
private List<CrawlerThread> threads;//list of all threads
|
private List<CrawlerThread> threads;//list of all threads
|
||||||
private List<CrawlerThread> requested = new LinkedList<>();
|
private List<CrawlerThread> requested = new LinkedList<>();
|
||||||
private static DateFormat dateform = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss");
|
private static DateFormat dateform = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss");
|
||||||
|
private long start;
|
||||||
|
|
||||||
private boolean crawl = true;
|
private boolean crawl = true;
|
||||||
private int crawlcount = 0;
|
private int crawlcount = 0;
|
||||||
|
@ -82,7 +83,7 @@ public class Crawler implements Runnable {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
long start = System.currentTimeMillis();
|
start = System.currentTimeMillis();
|
||||||
log.info("Try to load crawlfile");
|
log.info("Try to load crawlfile");
|
||||||
if(crawlfile.exists()) {
|
if(crawlfile.exists()) {
|
||||||
listlock.writeLock().lock();
|
listlock.writeLock().lock();
|
||||||
|
@ -145,7 +146,9 @@ public class Crawler implements Runnable {
|
||||||
Thread.yield();
|
Thread.yield();
|
||||||
try {
|
try {
|
||||||
Thread.sleep(100);
|
Thread.sleep(100);
|
||||||
} catch(InterruptedException ignored) { }
|
} catch(InterruptedException ignored) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
//nothing left?
|
//nothing left?
|
||||||
if(toknown.isEmpty() && toCrawl.isEmpty() && requested.size() == threads.size()) {//very uncommon
|
if(toknown.isEmpty() && toCrawl.isEmpty() && requested.size() == threads.size()) {//very uncommon
|
||||||
|
@ -219,6 +222,8 @@ public class Crawler implements Runnable {
|
||||||
|
|
||||||
//end
|
//end
|
||||||
long runtimes = (System.currentTimeMillis() - start) / 1000;
|
long runtimes = (System.currentTimeMillis() - start) / 1000;
|
||||||
|
if(runtimes < 0)
|
||||||
|
runtimes = 1;
|
||||||
int runtimem = (int) (runtimes / 60);
|
int runtimem = (int) (runtimes / 60);
|
||||||
float vidps = (crawlcount / (float) runtimes);//videos per second
|
float vidps = (crawlcount / (float) runtimes);//videos per second
|
||||||
log.info("Crawling Stopped. Runtime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )");
|
log.info("Crawling Stopped. Runtime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )");
|
||||||
|
@ -232,6 +237,18 @@ public class Crawler implements Runnable {
|
||||||
return new Video();
|
return new Video();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void printStats() {
|
||||||
|
long runtimes = (System.currentTimeMillis() - start) / 1000;
|
||||||
|
if(runtimes < 0)
|
||||||
|
runtimes = 1;
|
||||||
|
int runtimem = (int) (runtimes / 60);
|
||||||
|
float vidps = (crawlcount / (float) runtimes);//videos per second
|
||||||
|
log.info("ToCrawl:" + toCrawl.size());
|
||||||
|
log.info("Toknown:" + toknown.size());
|
||||||
|
log.info("ToSave:" + toSave.size());
|
||||||
|
log.info("Runtime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )");
|
||||||
|
}
|
||||||
|
|
||||||
public static class Video {
|
public static class Video {
|
||||||
String id;
|
String id;
|
||||||
int length;//the length of the video in seconds
|
int length;//the length of the video in seconds
|
||||||
|
@ -239,4 +256,5 @@ public class Crawler implements Runnable {
|
||||||
byte categorie;
|
byte categorie;
|
||||||
long created;
|
long created;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,6 +15,10 @@ public class HTTPS {
|
||||||
con = (HttpsURLConnection) (new URL(url)).openConnection();
|
con = (HttpsURLConnection) (new URL(url)).openConnection();
|
||||||
con.setDoInput(true);
|
con.setDoInput(true);
|
||||||
con.setDefaultUseCaches(true);
|
con.setDefaultUseCaches(true);
|
||||||
|
int code = con.getResponseCode();
|
||||||
|
if(code != 200) {
|
||||||
|
Log.l.warn("recived Response code: " + code);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getContent() throws IOException {
|
public String getContent() throws IOException {
|
||||||
|
|
|
@ -51,6 +51,9 @@ public class Main implements CommandHandler{
|
||||||
cra.addtoCrawl(id);
|
cra.addtoCrawl(id);
|
||||||
log.info("added.");
|
log.info("added.");
|
||||||
}
|
}
|
||||||
|
} else if(in.equalsIgnoreCase("stats")) {
|
||||||
|
log.info("Getting Stats");
|
||||||
|
cra.printStats();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
s.close();
|
s.close();
|
||||||
|
|
Loading…
Reference in New Issue