diff --git a/src/de/mrbesen/youtubecrawler/Crawler.java b/src/de/mrbesen/youtubecrawler/Crawler.java index 1f3167f..585017f 100644 --- a/src/de/mrbesen/youtubecrawler/Crawler.java +++ b/src/de/mrbesen/youtubecrawler/Crawler.java @@ -27,6 +27,7 @@ public class Crawler implements Runnable { private List threads;//list of all threads private List requested = new LinkedList<>(); private static DateFormat dateform = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss"); + private long start; private boolean crawl = true; private int crawlcount = 0; @@ -82,7 +83,7 @@ public class Crawler implements Runnable { @Override public void run() { - long start = System.currentTimeMillis(); + start = System.currentTimeMillis(); log.info("Try to load crawlfile"); if(crawlfile.exists()) { listlock.writeLock().lock(); @@ -145,7 +146,9 @@ public class Crawler implements Runnable { Thread.yield(); try { Thread.sleep(100); - } catch(InterruptedException ignored) { } + } catch(InterruptedException ignored) { + break; + } } //nothing left? if(toknown.isEmpty() && toCrawl.isEmpty() && requested.size() == threads.size()) {//very uncommon @@ -219,6 +222,8 @@ public class Crawler implements Runnable { //end long runtimes = (System.currentTimeMillis() - start) / 1000; + if(runtimes < 0) + runtimes = 1; int runtimem = (int) (runtimes / 60); float vidps = (crawlcount / (float) runtimes);//videos per second log.info("Crawling Stopped. Runtime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )"); @@ -232,6 +237,18 @@ public class Crawler implements Runnable { return new Video(); } + public void printStats() { + long runtimes = (System.currentTimeMillis() - start) / 1000; + if(runtimes < 0) + runtimes = 1; + int runtimem = (int) (runtimes / 60); + float vidps = (crawlcount / (float) runtimes);//videos per second + log.info("ToCrawl:" + toCrawl.size()); + log.info("Toknown:" + toknown.size()); + log.info("ToSave:" + toSave.size()); + log.info("Runtime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )"); + } + public static class Video { String id; int length;//the length of the video in seconds @@ -239,4 +256,5 @@ public class Crawler implements Runnable { byte categorie; long created; } + } diff --git a/src/de/mrbesen/youtubecrawler/HTTPS.java b/src/de/mrbesen/youtubecrawler/HTTPS.java index faa17e9..956460e 100644 --- a/src/de/mrbesen/youtubecrawler/HTTPS.java +++ b/src/de/mrbesen/youtubecrawler/HTTPS.java @@ -15,6 +15,10 @@ public class HTTPS { con = (HttpsURLConnection) (new URL(url)).openConnection(); con.setDoInput(true); con.setDefaultUseCaches(true); + int code = con.getResponseCode(); + if(code != 200) { + Log.l.warn("recived Response code: " + code); + } } public String getContent() throws IOException { diff --git a/src/de/mrbesen/youtubecrawler/Main.java b/src/de/mrbesen/youtubecrawler/Main.java index 080a5d6..1c07d0a 100644 --- a/src/de/mrbesen/youtubecrawler/Main.java +++ b/src/de/mrbesen/youtubecrawler/Main.java @@ -51,6 +51,9 @@ public class Main implements CommandHandler{ cra.addtoCrawl(id); log.info("added."); } + } else if(in.equalsIgnoreCase("stats")) { + log.info("Getting Stats"); + cra.printStats(); } } s.close();