diff --git a/src/de/mrbesen/youtubecrawler/Crawler.java b/src/de/mrbesen/youtubecrawler/Crawler.java index 585017f..b28698b 100644 --- a/src/de/mrbesen/youtubecrawler/Crawler.java +++ b/src/de/mrbesen/youtubecrawler/Crawler.java @@ -28,7 +28,7 @@ public class Crawler implements Runnable { private List requested = new LinkedList<>(); private static DateFormat dateform = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss"); private long start; - + private boolean crawl = true; private int crawlcount = 0; @@ -71,7 +71,7 @@ public class Crawler implements Runnable { requested.add(t); } } - + private void send(CrawlerThread t) { listlock.writeLock().lock(); for(int i = 0; i < jobspeerthread && !toCrawl.isEmpty(); i++) { @@ -125,13 +125,13 @@ public class Crawler implements Runnable { log.warn("Could not read the Number \"" + Config.prop.getProperty("crawler.threadcount") + "\" from the Config."); } threads = new ArrayList<>(threadcount); - + for(int i = 0; i < threadcount; i++) { CrawlerThread thr = new CrawlerThread( this); new Thread(thr, "Crawler #" + i).start(); threads.add(thr); } - + int updateOffset = 0; while(crawl) { log.info("to Crawl: " + toCrawl.size() + " known: " + toknown.size() + " Time: " + dateform.format(new Date())); @@ -139,23 +139,39 @@ public class Crawler implements Runnable { while(!requested.isEmpty() && !toCrawl.isEmpty() && crawl) { send(requested.remove(0)); } - + //kindof idle while(toCrawl.size() > (jobspeerthread * threads.size()) && crawl && requested.isEmpty()) { startup = 0;//stop startup count Thread.yield(); try { - Thread.sleep(100); + Thread.sleep(5000); } catch(InterruptedException ignored) { break; } + log.info("updating DB Offset= " + updateOffset); + LinkedList vids = db.getUncompleted(50, updateOffset); + LinkedList