diff --git a/src/de/mrbesen/youtubecrawler/Crawler.java b/src/de/mrbesen/youtubecrawler/Crawler.java index bc0707c..c74ef5a 100644 --- a/src/de/mrbesen/youtubecrawler/Crawler.java +++ b/src/de/mrbesen/youtubecrawler/Crawler.java @@ -59,6 +59,7 @@ public class Crawler implements Runnable { public void stop() { crawl = false; + db.stop(); } public synchronized void addtoCrawl(String videoid) { @@ -132,9 +133,12 @@ public class Crawler implements Runnable { for(int i = 0; i < threadcount; i++) { CrawlerThread thr = new CrawlerThread( this); - new Thread(thr, "Crawler #" + i).start(); + thr.setThread(new Thread(thr, "Crawler #" + i)); threads.add(thr); + thr.thread.start(); } + long lastdoubledelete = System.currentTimeMillis(); + db.deleteDouble(); while(crawl) { log.info("to Crawl: " + toCrawl.size() + " known: " + toknown.size() + " Time: " + dateform.format(new Date())); try { @@ -149,11 +153,16 @@ public class Crawler implements Runnable { while(toCrawl.size() > (jobspeerthread * threads.size()) && crawl && requested.isEmpty()) { startup = 0;//stop startup count currentstate = "idle"; - Thread.yield(); - try { - Thread.sleep(100); - } catch(InterruptedException ignored) { - break; + if((System.currentTimeMillis() - lastdoubledelete) / 1000 > 1800) { + db.deleteDouble(); + lastdoubledelete = System.currentTimeMillis(); + } else { + Thread.yield(); + try { + Thread.sleep(100); + } catch(InterruptedException ignored) { + break; + } } // updateDB(); } @@ -212,7 +221,8 @@ public class Crawler implements Runnable { crawlcount+= report[0].size(); toSave.addAll(report[0]); crawlerThread.crawled.clear(); - while(report[1].size() > 15) { + + while(report[1].size() > 1) {//2 videos werden ggf. gelöscht ohne gesehen zu werden. LinkedList store = new LinkedList<>(); try { while(!report[1].isEmpty() && store.size() < 50) { @@ -227,8 +237,8 @@ public class Crawler implements Runnable { } log.info(count + " videos added."); crawlerThread.found.clear(); + crawlerThread.thread.interrupt();//free from lock } - db.deleteDouble(); long runtimes = (System.currentTimeMillis() - start) / 1000; if(runtimes < 0) @@ -270,6 +280,7 @@ public class Crawler implements Runnable { Main.getMain().stop(); } } + db.deleteDouble(); //end long runtimes = (System.currentTimeMillis() - start) / 1000; diff --git a/src/de/mrbesen/youtubecrawler/CrawlerThread.java b/src/de/mrbesen/youtubecrawler/CrawlerThread.java index afa090b..1317932 100644 --- a/src/de/mrbesen/youtubecrawler/CrawlerThread.java +++ b/src/de/mrbesen/youtubecrawler/CrawlerThread.java @@ -13,22 +13,35 @@ public class CrawlerThread implements Runnable { private Logger log = Logger.getLogger(this.getClass().getName()); private Crawler parent; - + Thread thread; + LinkedList todo = new LinkedList<>();//videos, this thread should crawl LinkedList crawled = new LinkedList<>();//videos this thread had crawled LinkedList found = new LinkedList<>();//videos this thread had found boolean requested = true;//is a request pending? - + private boolean lockforreport = false; + public CrawlerThread( Crawler root) { parent = root; root.request(this); } + + void setThread(Thread t) { + thread = t; + } @Override public void run() { while(parent.isCrawling()) { while(!todo.isEmpty() && parent.isCrawling()) { + if(lockforreport) { + try { + Thread.sleep(10); + } catch(InterruptedException e) { + lockforreport = false; + } + } crawl(todo.removeFirst()); if(todo.size() < 5 && !requested) { requested = true; @@ -55,6 +68,7 @@ public class CrawlerThread implements Runnable { * @return */ LinkedList[] report() { + lockforreport = true; return new LinkedList[] {crawled, found}; } diff --git a/src/de/mrbesen/youtubecrawler/DB.java b/src/de/mrbesen/youtubecrawler/DB.java index 9f27b53..ef9a564 100644 --- a/src/de/mrbesen/youtubecrawler/DB.java +++ b/src/de/mrbesen/youtubecrawler/DB.java @@ -4,8 +4,10 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import java.util.Random; import org.apache.log4j.Logger; @@ -13,13 +15,16 @@ import com.mysql.cj.jdbc.exceptions.MysqlDataTruncation; import de.mrbesen.youtubecrawler.Crawler.Video; -public class DB { +public class DB implements Runnable { private Connection con; private String server = Config.prop.getProperty("db.host", "localhost"), user = Config.prop.getProperty("db.user", "ytcrawler"), pw = Config.prop.getProperty("db.pw", ""), db = Config.prop.getProperty("db.dbname", "ytcrawler"); private int port = Integer.parseInt(Config.prop.getProperty("db.port", "3306")); private Logger log = Logger.getLogger(DB.class.getName()); - + private ArrayList randombuffer = new ArrayList<>(100); + private Random rand = new Random(); + private Server serv = new Server(this); + private Thread randomrefill = null; public DB() { try { @@ -46,6 +51,9 @@ public class DB { log.info("Database is set up!"); } + + serv.start(); + refillbuffer(); } catch (SQLException e) { log.error("Error while connecting to the database! ", e); } @@ -180,19 +188,23 @@ public class DB { } } - - public String getRandom() { - ResultSet set = query("SELECT `id`, rand() as 'r' FROM `videos` ORDER BY r LIMIT 1;"); - if(set == null) - return null; - try { - if(set.next()) { - return set.getString(1); - } - } catch (SQLException e) { - log.warn("error getting a random video", e); + private void refillbuffer() { + if(randomrefill == null) { + randomrefill = new Thread(this, "Randomrefill"); + randomrefill.start(); } - return null; + } + + public String getRandom() { + log.info("Get random Video"); + if(randombuffer.size() < 10 ) { + refillbuffer(); + } + if(randombuffer.isEmpty()) { + log.warn("randombuffer is empty!"); + return null; + } + return randombuffer.remove(0); } public LinkedList restoreTemp() { @@ -209,7 +221,10 @@ public class DB { } public void deleteDouble() { + log.info("Started Delete Double"); + long start = System.currentTimeMillis(); update("call ytcrawler.deletedouble();"); + log.info("Delete Double done in " + ((System.currentTimeMillis() - start)/60000) + " min"); } public void storeTemp(LinkedList strings) { @@ -221,4 +236,49 @@ public class DB { update("INSERT IGNORE INTO `ytcrawler`.`temp` (`ytid`) VALUES ('" + sb.substring(6).toString() + "');"); } } + + + /** + * Stops the randomnes-Server and disconnect + */ + public void stop() { + serv.stop(); + try { + if(con != null) { + if(!con.isClosed()) { + con.close(); + } + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + /** + * runable, of Thread for randomrefill + */ + @Override + public void run() { + ResultSet count = query("SELECT `" + db + "`.`getLimit`();"); + if(count != null) { + try { + if(count.next()) { + int max = count.getInt(1); + ResultSet set = query("SELECT `id` FROM `videos` LIMIT " + rand.nextInt(max) + ",100;"); + if(set != null) { + while(set.next()) { + randombuffer.add(set.getString(1)); + } + log.info("refilled randombuffer to " + randombuffer.size() + " videos."); + } + } + } catch (SQLException e) { + log.warn("error getting a random video", e); + } + } + if(randombuffer.isEmpty()) { + log.error("Unable to retrieve RandomVideos"); + } + randomrefill = null; + } } \ No newline at end of file diff --git a/src/de/mrbesen/youtubecrawler/Main.java b/src/de/mrbesen/youtubecrawler/Main.java index ee76eed..fef93d1 100644 --- a/src/de/mrbesen/youtubecrawler/Main.java +++ b/src/de/mrbesen/youtubecrawler/Main.java @@ -60,7 +60,7 @@ public class Main implements CommandHandler, EventListener{ tapi.getCommandManager().registerCommand("stop", this); tapi.getEventManager().registerEvent(this); tapi.setHelpText("Send the command /random to get a random video."); - tapi.setUpdateInterval(25000); + tapi.setUpdateInterval(2000); tapi.start(); //load admins @@ -104,8 +104,15 @@ public class Main implements CommandHandler, EventListener{ public void stop() { log.info("Stop."); - tapi.stop(); cra.stop(); + log.info("cra stopped"); + try { + Thread.sleep(100); + } catch(InterruptedException e) { + e.printStackTrace(); + } + tapi.stop(); + log.info("tapi stopped"); mainthread.interrupt(); } @@ -142,7 +149,9 @@ public class Main implements CommandHandler, EventListener{ public void broadcastAdmin(String msg) { for(String admin : admins) { - tapi.getUser(admin).sendMessage(msg); + TUser adm = tapi.getUser(admin); + if(adm != null) + adm.sendMessage(msg); } } diff --git a/src/de/mrbesen/youtubecrawler/Server.java b/src/de/mrbesen/youtubecrawler/Server.java new file mode 100644 index 0000000..16dc1af --- /dev/null +++ b/src/de/mrbesen/youtubecrawler/Server.java @@ -0,0 +1,62 @@ +package de.mrbesen.youtubecrawler; + +import java.io.IOException; +import java.io.PrintWriter; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; + +import org.apache.log4j.Logger; + +public class Server implements Runnable { + + private ServerSocket ssoc; + private Thread t; + private boolean run = false; + private Logger log = Logger.getLogger(this.getClass().getName()); + private DB db; + + public Server(DB d) { + db = d; + } + + void start() { + run = true; + t = new Thread(this, "Server"); + t.start(); + } + + void stop() { + run = false; + t.interrupt(); + } + + public void run() { + try { + final int port = 2419; + ssoc = new ServerSocket(); + ssoc.bind(new InetSocketAddress(InetAddress.getByName("::1"), port)); + ssoc.setSoTimeout(5); + log.info("opened Server at port " + port); + while(run) { + try { + Socket client = ssoc.accept(); + if(client.getInetAddress().isLoopbackAddress()) { + PrintWriter out = new PrintWriter(client.getOutputStream()); + out.println(db.getRandom()); + out.flush(); + out.close(); + } else { + log.info("client connected: " + client.getInetAddress().toString()); + } + client.close(); + } catch(SocketTimeoutException ignored) {} + } + ssoc.close(); + } catch(IOException e) { + e.printStackTrace(); + } + } +} diff --git a/src/de/mrbesen/youtubecrawler/YoutubeAPI.java b/src/de/mrbesen/youtubecrawler/YoutubeAPI.java index 594d28b..574f760 100644 --- a/src/de/mrbesen/youtubecrawler/YoutubeAPI.java +++ b/src/de/mrbesen/youtubecrawler/YoutubeAPI.java @@ -157,3 +157,4 @@ public class YoutubeAPI { return null; } } +// no suchelement bla \ No newline at end of file