randombuffer in second thread, server local only

src/de/mrbesen/youtubecrawler/Crawler.java

@@ -59,6 +59,7 @@ public class Crawler implements Runnable {
 
 	public void stop() {
 		crawl = false;
+		db.stop();
 	}
 
 	public synchronized void addtoCrawl(String videoid) {
@@ -132,9 +133,12 @@ public class Crawler implements Runnable {
 
 		for(int i = 0; i < threadcount; i++) {
 			CrawlerThread thr = new CrawlerThread( this);
-			new Thread(thr, "Crawler #" + i).start();
+			thr.setThread(new Thread(thr, "Crawler #" + i));
 			threads.add(thr);
+			thr.thread.start();
 		}
+		long lastdoubledelete = System.currentTimeMillis();
+		db.deleteDouble();
 		while(crawl) {
 			log.info("to Crawl: " + toCrawl.size() + " known: " + toknown.size() + " Time: " + dateform.format(new Date()));
 			try {
@@ -149,11 +153,16 @@ public class Crawler implements Runnable {
 				while(toCrawl.size() > (jobspeerthread * threads.size()) && crawl && requested.isEmpty()) {
 					startup = 0;//stop startup count
 					currentstate = "idle";
-					Thread.yield();
-					try {
-						Thread.sleep(100);
-					} catch(InterruptedException ignored) {
-						break;
+					if((System.currentTimeMillis() - lastdoubledelete) / 1000 > 1800) {
+						db.deleteDouble();
+						lastdoubledelete = System.currentTimeMillis();
+					} else {
+						Thread.yield();
+						try {
+							Thread.sleep(100);
+						} catch(InterruptedException ignored) {
+							break;
+						}
 					}
 					//				updateDB();
 				}
@@ -212,7 +221,8 @@ public class Crawler implements Runnable {
 					crawlcount+= report[0].size();
 					toSave.addAll(report[0]);
 					crawlerThread.crawled.clear();
-					while(report[1].size() > 15) {
+					
+					while(report[1].size() > 1) {//2 videos werden ggf. gelöscht ohne gesehen zu werden.
 						LinkedList<String> store = new LinkedList<>();
 						try {
 							while(!report[1].isEmpty() && store.size() < 50) {
@@ -227,8 +237,8 @@ public class Crawler implements Runnable {
 					}
 					log.info(count + " videos added.");
 					crawlerThread.found.clear();
+					crawlerThread.thread.interrupt();//free from lock
 				}
-				db.deleteDouble();
 
 				long runtimes = (System.currentTimeMillis() - start) / 1000;
 				if(runtimes < 0)
@@ -270,6 +280,7 @@ public class Crawler implements Runnable {
 				Main.getMain().stop();
 			}
 		}
+		db.deleteDouble();
 
 		//end
 		long runtimes = (System.currentTimeMillis() - start) / 1000;

src/de/mrbesen/youtubecrawler/CrawlerThread.java

@@ -13,22 +13,35 @@ public class CrawlerThread implements Runnable {
 
 	private Logger log  = Logger.getLogger(this.getClass().getName());
 	private Crawler parent;
-
+	Thread thread;
+	
 	LinkedList<String> todo = new LinkedList<>();//videos, this thread should crawl
 	LinkedList<String> crawled = new LinkedList<>();//videos this thread had crawled
 	LinkedList<String> found = new LinkedList<>();//videos this thread had found
 
 	boolean requested = true;//is a request pending?
-
+	private boolean lockforreport = false;
+	
 	public CrawlerThread( Crawler root) {
 		parent = root;
 		root.request(this);
 	}
+	
+	void setThread(Thread t) {
+		thread = t;
+	}
 
 	@Override
 	public void run() {
 		while(parent.isCrawling()) {
 			while(!todo.isEmpty() && parent.isCrawling()) {
+				if(lockforreport) {
+					try {
+						Thread.sleep(10);
+					} catch(InterruptedException e) {
+						lockforreport = false;
+					}
+				}
 				crawl(todo.removeFirst());
 				if(todo.size() < 5 && !requested) {
 					requested = true;
@@ -55,6 +68,7 @@ public class CrawlerThread implements Runnable {
 	 * @return
 	 */
 	LinkedList<String>[] report() {
+		lockforreport = true;
 		return new LinkedList[] {crawled, found};
 	}
 

src/de/mrbesen/youtubecrawler/DB.java

@@ -4,8 +4,10 @@ import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Random;
 
 import org.apache.log4j.Logger;
 
@@ -13,13 +15,16 @@ import com.mysql.cj.jdbc.exceptions.MysqlDataTruncation;
 
 import de.mrbesen.youtubecrawler.Crawler.Video;
 
-public class DB {
+public class DB implements Runnable {
 
 	private Connection con;
 	private String server = Config.prop.getProperty("db.host", "localhost"), user = Config.prop.getProperty("db.user", "ytcrawler"), pw = Config.prop.getProperty("db.pw", ""), db = Config.prop.getProperty("db.dbname", "ytcrawler");
 	private int port = Integer.parseInt(Config.prop.getProperty("db.port", "3306")); 
 	private Logger log = Logger.getLogger(DB.class.getName());
-
+	private ArrayList<String> randombuffer = new ArrayList<>(100);
+	private Random rand = new Random();
+	private Server serv = new Server(this);
+	private Thread randomrefill = null;
 
 	public DB() {
 		try {
@@ -46,6 +51,9 @@ public class DB {
 
 				log.info("Database is set up!");
 			}
+			
+			serv.start();
+			refillbuffer();
 		} catch (SQLException e) {
 			log.error("Error while connecting to the database! ", e);
 		}
@@ -180,19 +188,23 @@ public class DB {
 		}
 	}
 
-
-	public String getRandom() {
-		ResultSet set = query("SELECT `id`, rand() as 'r' FROM `videos` ORDER BY r LIMIT 1;");
-		if(set == null)
-			return null;
-		try {
-			if(set.next()) {
-				return set.getString(1);
-			}
-		} catch (SQLException e) {
-			log.warn("error getting a random video", e);
+	private void refillbuffer() {
+		if(randomrefill == null) {
+			randomrefill = new Thread(this, "Randomrefill");
+			randomrefill.start();
 		}
-		return null;
+	}
+	
+	public String getRandom() {
+		log.info("Get random Video");
+		if(randombuffer.size() < 10 ) {
+			refillbuffer();
+		}
+		if(randombuffer.isEmpty()) {
+			log.warn("randombuffer is empty!");
+			return null;
+		}
+		return randombuffer.remove(0);
 	}
 
 	public LinkedList<String> restoreTemp() {
@@ -209,7 +221,10 @@ public class DB {
 	}
 
 	public void deleteDouble() {
+		log.info("Started Delete Double");
+		long start = System.currentTimeMillis();
 		update("call ytcrawler.deletedouble();");
+		log.info("Delete Double done in " + ((System.currentTimeMillis() - start)/60000) + " min");
 	}
 
 	public void storeTemp(LinkedList<String> strings) {
@@ -221,4 +236,49 @@ public class DB {
 			update("INSERT IGNORE INTO `ytcrawler`.`temp` (`ytid`) VALUES ('" + sb.substring(6).toString() + "');");
 		}
 	}
+
+	
+	/**
+	 * Stops the randomnes-Server and disconnect
+	 */
+	public void stop() {
+		serv.stop();
+		try {
+			if(con != null) {
+				if(!con.isClosed()) {
+					con.close();
+				}
+			}
+		} catch (SQLException e) {
+			e.printStackTrace();
+		}
+	}
+
+	/**
+	 * runable, of Thread for randomrefill
+	 */
+	@Override
+	public void run() {
+		ResultSet count = query("SELECT `" + db + "`.`getLimit`();");
+		if(count != null) {
+			try {
+				if(count.next()) {
+					int max = count.getInt(1);
+					ResultSet set = query("SELECT `id` FROM `videos` LIMIT " + rand.nextInt(max) + ",100;");
+					if(set != null) {
+						while(set.next()) {
+							randombuffer.add(set.getString(1));
+						}
+						log.info("refilled randombuffer to " + randombuffer.size() + " videos.");
+					}
+				}
+			} catch (SQLException e) {
+				log.warn("error getting a random video", e);
+			}
+		}
+		if(randombuffer.isEmpty()) {
+			log.error("Unable to retrieve RandomVideos");
+		}
+		randomrefill = null;
+	}
 }

src/de/mrbesen/youtubecrawler/Main.java

@@ -60,7 +60,7 @@ public class Main implements CommandHandler, EventListener{
 		tapi.getCommandManager().registerCommand("stop", this);
 		tapi.getEventManager().registerEvent(this);
 		tapi.setHelpText("Send the command /random to get a random video.");
-		tapi.setUpdateInterval(25000);
+		tapi.setUpdateInterval(2000);
 		tapi.start();
 
 		//load admins
@@ -104,8 +104,15 @@ public class Main implements CommandHandler, EventListener{
 
 	public void stop() {
 		log.info("Stop.");
-		tapi.stop();
 		cra.stop();
+		log.info("cra stopped");
+		try {
+			Thread.sleep(100);
+		} catch(InterruptedException e) {
+			e.printStackTrace();
+		}
+		tapi.stop();
+		log.info("tapi stopped");
 		mainthread.interrupt();
 	}
 
@@ -142,7 +149,9 @@ public class Main implements CommandHandler, EventListener{
 
 	public void broadcastAdmin(String msg) {
 		for(String admin : admins) {
-			tapi.getUser(admin).sendMessage(msg);
+			TUser adm = tapi.getUser(admin);
+			if(adm != null)
+				adm.sendMessage(msg);
 		}
 	}
 

src/de/mrbesen/youtubecrawler/Server.java

@@ -0,0 +1,62 @@
+package de.mrbesen.youtubecrawler;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
+
+import org.apache.log4j.Logger;
+
+public class Server implements Runnable {
+
+	private ServerSocket ssoc;
+	private Thread t;
+	private boolean run = false;
+	private Logger log = Logger.getLogger(this.getClass().getName());
+	private DB db;
+
+	public Server(DB d) {
+		db = d;
+	}
+	
+	void start() {
+		run = true;
+		t = new Thread(this, "Server");
+		t.start();
+	}
+
+	void stop() {
+		run = false;
+		t.interrupt();
+	}
+
+	public void run() {
+		try {
+			final int port = 2419;
+			ssoc = new ServerSocket();
+			ssoc.bind(new InetSocketAddress(InetAddress.getByName("::1"), port));
+			ssoc.setSoTimeout(5);
+			log.info("opened Server at port " + port);
+			while(run) {
+				try {
+					Socket client = ssoc.accept();
+					if(client.getInetAddress().isLoopbackAddress()) {
+						PrintWriter out = new PrintWriter(client.getOutputStream());
+						out.println(db.getRandom());
+						out.flush();
+						out.close();
+					} else {
+						log.info("client connected: " + client.getInetAddress().toString());
+					}
+					client.close();
+				} catch(SocketTimeoutException ignored) {}
+			}
+			ssoc.close();
+		} catch(IOException e) {
+			e.printStackTrace();
+		}
+	}
+}

src/de/mrbesen/youtubecrawler/YoutubeAPI.java

@@ -157,3 +157,4 @@ public class YoutubeAPI {
 		return null;
 	}
 }
+// no suchelement bla