randombuffer in second thread, server local only
This commit is contained in:
parent
b67952ce88
commit
a5abe7f08e
|
@ -59,6 +59,7 @@ public class Crawler implements Runnable {
|
|||
|
||||
public void stop() {
|
||||
crawl = false;
|
||||
db.stop();
|
||||
}
|
||||
|
||||
public synchronized void addtoCrawl(String videoid) {
|
||||
|
@ -132,9 +133,12 @@ public class Crawler implements Runnable {
|
|||
|
||||
for(int i = 0; i < threadcount; i++) {
|
||||
CrawlerThread thr = new CrawlerThread( this);
|
||||
new Thread(thr, "Crawler #" + i).start();
|
||||
thr.setThread(new Thread(thr, "Crawler #" + i));
|
||||
threads.add(thr);
|
||||
thr.thread.start();
|
||||
}
|
||||
long lastdoubledelete = System.currentTimeMillis();
|
||||
db.deleteDouble();
|
||||
while(crawl) {
|
||||
log.info("to Crawl: " + toCrawl.size() + " known: " + toknown.size() + " Time: " + dateform.format(new Date()));
|
||||
try {
|
||||
|
@ -149,11 +153,16 @@ public class Crawler implements Runnable {
|
|||
while(toCrawl.size() > (jobspeerthread * threads.size()) && crawl && requested.isEmpty()) {
|
||||
startup = 0;//stop startup count
|
||||
currentstate = "idle";
|
||||
Thread.yield();
|
||||
try {
|
||||
Thread.sleep(100);
|
||||
} catch(InterruptedException ignored) {
|
||||
break;
|
||||
if((System.currentTimeMillis() - lastdoubledelete) / 1000 > 1800) {
|
||||
db.deleteDouble();
|
||||
lastdoubledelete = System.currentTimeMillis();
|
||||
} else {
|
||||
Thread.yield();
|
||||
try {
|
||||
Thread.sleep(100);
|
||||
} catch(InterruptedException ignored) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// updateDB();
|
||||
}
|
||||
|
@ -212,7 +221,8 @@ public class Crawler implements Runnable {
|
|||
crawlcount+= report[0].size();
|
||||
toSave.addAll(report[0]);
|
||||
crawlerThread.crawled.clear();
|
||||
while(report[1].size() > 15) {
|
||||
|
||||
while(report[1].size() > 1) {//2 videos werden ggf. gelöscht ohne gesehen zu werden.
|
||||
LinkedList<String> store = new LinkedList<>();
|
||||
try {
|
||||
while(!report[1].isEmpty() && store.size() < 50) {
|
||||
|
@ -227,8 +237,8 @@ public class Crawler implements Runnable {
|
|||
}
|
||||
log.info(count + " videos added.");
|
||||
crawlerThread.found.clear();
|
||||
crawlerThread.thread.interrupt();//free from lock
|
||||
}
|
||||
db.deleteDouble();
|
||||
|
||||
long runtimes = (System.currentTimeMillis() - start) / 1000;
|
||||
if(runtimes < 0)
|
||||
|
@ -270,6 +280,7 @@ public class Crawler implements Runnable {
|
|||
Main.getMain().stop();
|
||||
}
|
||||
}
|
||||
db.deleteDouble();
|
||||
|
||||
//end
|
||||
long runtimes = (System.currentTimeMillis() - start) / 1000;
|
||||
|
|
|
@ -13,22 +13,35 @@ public class CrawlerThread implements Runnable {
|
|||
|
||||
private Logger log = Logger.getLogger(this.getClass().getName());
|
||||
private Crawler parent;
|
||||
|
||||
Thread thread;
|
||||
|
||||
LinkedList<String> todo = new LinkedList<>();//videos, this thread should crawl
|
||||
LinkedList<String> crawled = new LinkedList<>();//videos this thread had crawled
|
||||
LinkedList<String> found = new LinkedList<>();//videos this thread had found
|
||||
|
||||
boolean requested = true;//is a request pending?
|
||||
|
||||
private boolean lockforreport = false;
|
||||
|
||||
public CrawlerThread( Crawler root) {
|
||||
parent = root;
|
||||
root.request(this);
|
||||
}
|
||||
|
||||
void setThread(Thread t) {
|
||||
thread = t;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
while(parent.isCrawling()) {
|
||||
while(!todo.isEmpty() && parent.isCrawling()) {
|
||||
if(lockforreport) {
|
||||
try {
|
||||
Thread.sleep(10);
|
||||
} catch(InterruptedException e) {
|
||||
lockforreport = false;
|
||||
}
|
||||
}
|
||||
crawl(todo.removeFirst());
|
||||
if(todo.size() < 5 && !requested) {
|
||||
requested = true;
|
||||
|
@ -55,6 +68,7 @@ public class CrawlerThread implements Runnable {
|
|||
* @return
|
||||
*/
|
||||
LinkedList<String>[] report() {
|
||||
lockforreport = true;
|
||||
return new LinkedList[] {crawled, found};
|
||||
}
|
||||
|
||||
|
|
|
@ -4,8 +4,10 @@ import java.sql.Connection;
|
|||
import java.sql.DriverManager;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
|
@ -13,13 +15,16 @@ import com.mysql.cj.jdbc.exceptions.MysqlDataTruncation;
|
|||
|
||||
import de.mrbesen.youtubecrawler.Crawler.Video;
|
||||
|
||||
public class DB {
|
||||
public class DB implements Runnable {
|
||||
|
||||
private Connection con;
|
||||
private String server = Config.prop.getProperty("db.host", "localhost"), user = Config.prop.getProperty("db.user", "ytcrawler"), pw = Config.prop.getProperty("db.pw", ""), db = Config.prop.getProperty("db.dbname", "ytcrawler");
|
||||
private int port = Integer.parseInt(Config.prop.getProperty("db.port", "3306"));
|
||||
private Logger log = Logger.getLogger(DB.class.getName());
|
||||
|
||||
private ArrayList<String> randombuffer = new ArrayList<>(100);
|
||||
private Random rand = new Random();
|
||||
private Server serv = new Server(this);
|
||||
private Thread randomrefill = null;
|
||||
|
||||
public DB() {
|
||||
try {
|
||||
|
@ -46,6 +51,9 @@ public class DB {
|
|||
|
||||
log.info("Database is set up!");
|
||||
}
|
||||
|
||||
serv.start();
|
||||
refillbuffer();
|
||||
} catch (SQLException e) {
|
||||
log.error("Error while connecting to the database! ", e);
|
||||
}
|
||||
|
@ -180,19 +188,23 @@ public class DB {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
public String getRandom() {
|
||||
ResultSet set = query("SELECT `id`, rand() as 'r' FROM `videos` ORDER BY r LIMIT 1;");
|
||||
if(set == null)
|
||||
return null;
|
||||
try {
|
||||
if(set.next()) {
|
||||
return set.getString(1);
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
log.warn("error getting a random video", e);
|
||||
private void refillbuffer() {
|
||||
if(randomrefill == null) {
|
||||
randomrefill = new Thread(this, "Randomrefill");
|
||||
randomrefill.start();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public String getRandom() {
|
||||
log.info("Get random Video");
|
||||
if(randombuffer.size() < 10 ) {
|
||||
refillbuffer();
|
||||
}
|
||||
if(randombuffer.isEmpty()) {
|
||||
log.warn("randombuffer is empty!");
|
||||
return null;
|
||||
}
|
||||
return randombuffer.remove(0);
|
||||
}
|
||||
|
||||
public LinkedList<String> restoreTemp() {
|
||||
|
@ -209,7 +221,10 @@ public class DB {
|
|||
}
|
||||
|
||||
public void deleteDouble() {
|
||||
log.info("Started Delete Double");
|
||||
long start = System.currentTimeMillis();
|
||||
update("call ytcrawler.deletedouble();");
|
||||
log.info("Delete Double done in " + ((System.currentTimeMillis() - start)/60000) + " min");
|
||||
}
|
||||
|
||||
public void storeTemp(LinkedList<String> strings) {
|
||||
|
@ -221,4 +236,49 @@ public class DB {
|
|||
update("INSERT IGNORE INTO `ytcrawler`.`temp` (`ytid`) VALUES ('" + sb.substring(6).toString() + "');");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Stops the randomnes-Server and disconnect
|
||||
*/
|
||||
public void stop() {
|
||||
serv.stop();
|
||||
try {
|
||||
if(con != null) {
|
||||
if(!con.isClosed()) {
|
||||
con.close();
|
||||
}
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* runable, of Thread for randomrefill
|
||||
*/
|
||||
@Override
|
||||
public void run() {
|
||||
ResultSet count = query("SELECT `" + db + "`.`getLimit`();");
|
||||
if(count != null) {
|
||||
try {
|
||||
if(count.next()) {
|
||||
int max = count.getInt(1);
|
||||
ResultSet set = query("SELECT `id` FROM `videos` LIMIT " + rand.nextInt(max) + ",100;");
|
||||
if(set != null) {
|
||||
while(set.next()) {
|
||||
randombuffer.add(set.getString(1));
|
||||
}
|
||||
log.info("refilled randombuffer to " + randombuffer.size() + " videos.");
|
||||
}
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
log.warn("error getting a random video", e);
|
||||
}
|
||||
}
|
||||
if(randombuffer.isEmpty()) {
|
||||
log.error("Unable to retrieve RandomVideos");
|
||||
}
|
||||
randomrefill = null;
|
||||
}
|
||||
}
|
|
@ -60,7 +60,7 @@ public class Main implements CommandHandler, EventListener{
|
|||
tapi.getCommandManager().registerCommand("stop", this);
|
||||
tapi.getEventManager().registerEvent(this);
|
||||
tapi.setHelpText("Send the command /random to get a random video.");
|
||||
tapi.setUpdateInterval(25000);
|
||||
tapi.setUpdateInterval(2000);
|
||||
tapi.start();
|
||||
|
||||
//load admins
|
||||
|
@ -104,8 +104,15 @@ public class Main implements CommandHandler, EventListener{
|
|||
|
||||
public void stop() {
|
||||
log.info("Stop.");
|
||||
tapi.stop();
|
||||
cra.stop();
|
||||
log.info("cra stopped");
|
||||
try {
|
||||
Thread.sleep(100);
|
||||
} catch(InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
tapi.stop();
|
||||
log.info("tapi stopped");
|
||||
mainthread.interrupt();
|
||||
}
|
||||
|
||||
|
@ -142,7 +149,9 @@ public class Main implements CommandHandler, EventListener{
|
|||
|
||||
public void broadcastAdmin(String msg) {
|
||||
for(String admin : admins) {
|
||||
tapi.getUser(admin).sendMessage(msg);
|
||||
TUser adm = tapi.getUser(admin);
|
||||
if(adm != null)
|
||||
adm.sendMessage(msg);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,62 @@
|
|||
package de.mrbesen.youtubecrawler;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.net.InetAddress;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.ServerSocket;
|
||||
import java.net.Socket;
|
||||
import java.net.SocketTimeoutException;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
public class Server implements Runnable {
|
||||
|
||||
private ServerSocket ssoc;
|
||||
private Thread t;
|
||||
private boolean run = false;
|
||||
private Logger log = Logger.getLogger(this.getClass().getName());
|
||||
private DB db;
|
||||
|
||||
public Server(DB d) {
|
||||
db = d;
|
||||
}
|
||||
|
||||
void start() {
|
||||
run = true;
|
||||
t = new Thread(this, "Server");
|
||||
t.start();
|
||||
}
|
||||
|
||||
void stop() {
|
||||
run = false;
|
||||
t.interrupt();
|
||||
}
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
final int port = 2419;
|
||||
ssoc = new ServerSocket();
|
||||
ssoc.bind(new InetSocketAddress(InetAddress.getByName("::1"), port));
|
||||
ssoc.setSoTimeout(5);
|
||||
log.info("opened Server at port " + port);
|
||||
while(run) {
|
||||
try {
|
||||
Socket client = ssoc.accept();
|
||||
if(client.getInetAddress().isLoopbackAddress()) {
|
||||
PrintWriter out = new PrintWriter(client.getOutputStream());
|
||||
out.println(db.getRandom());
|
||||
out.flush();
|
||||
out.close();
|
||||
} else {
|
||||
log.info("client connected: " + client.getInetAddress().toString());
|
||||
}
|
||||
client.close();
|
||||
} catch(SocketTimeoutException ignored) {}
|
||||
}
|
||||
ssoc.close();
|
||||
} catch(IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -157,3 +157,4 @@ public class YoutubeAPI {
|
|||
return null;
|
||||
}
|
||||
}
|
||||
// no suchelement bla
|
Loading…
Reference in New Issue