358 lines
11 KiB
Java
358 lines
11 KiB
Java
package de.mrbesen.youtubecrawler;
|
|
|
|
import java.io.BufferedWriter;
|
|
import java.io.File;
|
|
import java.io.FileWriter;
|
|
import java.io.IOException;
|
|
import java.io.PrintWriter;
|
|
import java.text.DateFormat;
|
|
import java.text.SimpleDateFormat;
|
|
import java.util.ArrayList;
|
|
import java.util.Date;
|
|
import java.util.LinkedList;
|
|
import java.util.List;
|
|
import java.util.NoSuchElementException;
|
|
import java.util.Scanner;
|
|
|
|
import org.apache.log4j.Logger;
|
|
|
|
public class Crawler implements Runnable {
|
|
|
|
private int jobspeerthread = 100; //the amount of jobs a thread get peer request
|
|
|
|
private LinkedList<String> toSave = new LinkedList<>();//all found ytids, witch need to be analysed
|
|
private LinkedList<String> toCrawl = new LinkedList<>();//all videos tu crawl
|
|
private LinkedList<String> toknown = new LinkedList<>();//list with all videos, to test if they are allready known, if not they are moved to tocrawle
|
|
private List<CrawlerThread> threads;//list of all threads
|
|
private List<CrawlerThread> requested = new LinkedList<>();
|
|
|
|
private static DateFormat dateform = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss");
|
|
private String currentstate = "undefined";
|
|
private long start;
|
|
|
|
private boolean crawl = true;
|
|
private int crawlcount = 0;
|
|
private int updateOffset = 0;
|
|
|
|
private DB db = new DB();
|
|
private YoutubeAPI api = new YoutubeAPI();
|
|
private File crawlfile = new File("crawl.txt");
|
|
private Logger log = Logger.getLogger(this.getClass().getName());
|
|
|
|
private int maxvideostotest = 100;
|
|
private int startup = 2;//to keep the beginning cool
|
|
|
|
public Crawler() {
|
|
try {
|
|
maxvideostotest = Integer.parseInt(Config.prop.getProperty("crawler.maxvideos"));
|
|
} catch(NumberFormatException e) {
|
|
log.warn("could not read the number \"" + Config.prop.getProperty("crawler.maxvideos") + "\" from the config file. maxvideo");
|
|
maxvideostotest = 100;
|
|
}
|
|
try {
|
|
jobspeerthread = Integer.parseInt(Config.prop.getProperty("crawler.jobspeerthread"));
|
|
} catch(NumberFormatException e) {
|
|
log.warn("could not read the number \"" + Config.prop.getProperty("crawler.jobspeerthread") + "\" from the config file. maxvideo");
|
|
jobspeerthread = 100;
|
|
}
|
|
}
|
|
|
|
public void stop() {
|
|
crawl = false;
|
|
db.stop();
|
|
}
|
|
|
|
public synchronized void addtoCrawl(String videoid) {
|
|
if(! (toCrawl.contains(videoid) || toknown.contains(videoid)))
|
|
toknown.add(videoid);
|
|
}
|
|
|
|
public boolean isCrawling() {
|
|
return crawl;
|
|
}
|
|
|
|
public void request(CrawlerThread t) {
|
|
if(!toCrawl.isEmpty()) {
|
|
send(t);
|
|
} else {
|
|
requested.add(t);
|
|
}
|
|
}
|
|
|
|
private void send(CrawlerThread t) {
|
|
// listlock.writeLock().lock();
|
|
for(int i = 0; i < jobspeerthread && !toCrawl.isEmpty(); i++) {
|
|
t.todo.add(toCrawl.removeFirst());
|
|
}
|
|
// listlock.writeLock().unlock();
|
|
t.requested = false;
|
|
}
|
|
|
|
@Override
|
|
public void run() {
|
|
currentstate = "loading crawlfile";
|
|
start = System.currentTimeMillis();
|
|
log.info("Try to load crawlfile");
|
|
if(crawlfile.exists()) {
|
|
try {
|
|
Scanner in = new Scanner(crawlfile);
|
|
boolean crawl = true;//section of file
|
|
while(in.hasNextLine()) {
|
|
String line = in.nextLine();
|
|
if(line == null) {
|
|
break;
|
|
} else {
|
|
if(!line.isEmpty()) {
|
|
if(line.equals("-")) {//section delimiter
|
|
crawl = false;
|
|
} else {
|
|
if(crawl) {
|
|
toCrawl.add(line);
|
|
} else {
|
|
toknown.add(line);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
in.close();
|
|
} catch(IOException e) {
|
|
log.warn("Error while loading crawl file.");
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
|
|
//populate threads
|
|
int threadcount = 4;
|
|
try {
|
|
threadcount = Integer.parseInt(Config.prop.getProperty("crawler.threadcount"));
|
|
} catch(NumberFormatException e) {
|
|
log.warn("Could not read the Number \"" + Config.prop.getProperty("crawler.threadcount") + "\" from the Config.");
|
|
}
|
|
threads = new ArrayList<>(threadcount);
|
|
|
|
for(int i = 0; i < threadcount; i++) {
|
|
CrawlerThread thr = new CrawlerThread( this);
|
|
thr.setThread(new Thread(thr, "Crawler #" + i));
|
|
threads.add(thr);
|
|
thr.thread.start();
|
|
}
|
|
long lastdoubledelete = System.currentTimeMillis();
|
|
db.deleteDouble();
|
|
while(crawl) {
|
|
log.info("to Crawl: " + toCrawl.size() + " known: " + toknown.size() + " Time: " + dateform.format(new Date()));
|
|
try {
|
|
//fullfill request
|
|
while(!requested.isEmpty() && !toCrawl.isEmpty() && crawl) {
|
|
log.info("fullfill request");
|
|
currentstate = "fullfill requests";
|
|
send(requested.remove(0));
|
|
}
|
|
|
|
//kindof idle
|
|
while(toCrawl.size() > (jobspeerthread * threads.size()) && crawl && requested.isEmpty()) {
|
|
startup = 0;//stop startup count
|
|
currentstate = "idle";
|
|
if((System.currentTimeMillis() - lastdoubledelete) / 1000 > 1800) {
|
|
db.deleteDouble();
|
|
lastdoubledelete = System.currentTimeMillis();
|
|
} else {
|
|
Thread.yield();
|
|
try {
|
|
Thread.sleep(100);
|
|
} catch(InterruptedException ignored) {
|
|
break;
|
|
}
|
|
}
|
|
// updateDB();
|
|
}
|
|
//nothing left?
|
|
if(toknown.isEmpty() && toCrawl.isEmpty() && requested.size() == threads.size()) {//very uncommon
|
|
log.warn("nothing left to crawl");
|
|
crawl = false;
|
|
}
|
|
|
|
//refil the tocrawl list.
|
|
if(!toknown.isEmpty()) {
|
|
//check in db for known videos
|
|
log.info("Checking the DB");
|
|
currentstate = "get new tocrawl";
|
|
// listlock.writeLock().lock();
|
|
while(toCrawl.size() < jobspeerthread * threads.size() * 2 && crawl && !toknown.isEmpty()) {
|
|
LinkedList<String> tocheck = new LinkedList<>();
|
|
for(int i = 0; i < toknown.size() && i < maxvideostotest; i++) {
|
|
tocheck.add(toknown.removeFirst());
|
|
}
|
|
toCrawl.addAll(db.checkvideos(tocheck));
|
|
}
|
|
// listlock.writeLock().unlock();
|
|
}
|
|
if(toknown.size() < threadcount * jobspeerthread * 20 && crawl) {
|
|
currentstate = "restoretemp";
|
|
log.info("restoreTemp");
|
|
LinkedList<String> rest = db.restoreTemp();
|
|
toknown.addAll(rest);
|
|
}
|
|
|
|
//writing crawlfile
|
|
log.info("Writing Crawlfile");
|
|
currentstate = "writing crawlfile";
|
|
// listlock.writeLock().lock();
|
|
try {
|
|
PrintWriter p = new PrintWriter(new BufferedWriter(new FileWriter(crawlfile)));
|
|
for(String t : toCrawl) {
|
|
p.println(t);
|
|
}
|
|
p.println("-");
|
|
for(String t : toknown) {
|
|
p.println(t);
|
|
}
|
|
p.close();
|
|
} catch (IOException e) {
|
|
log.error("Error writing crawlfile.", e);
|
|
}
|
|
|
|
//get reports
|
|
currentstate = "get report";
|
|
log.info("get report");
|
|
int count = 0;
|
|
for (CrawlerThread crawlerThread : threads) {
|
|
LinkedList<String>[] report = crawlerThread.report();
|
|
crawlcount+= report[0].size();
|
|
toSave.addAll(report[0]);
|
|
crawlerThread.crawled.clear();
|
|
|
|
while(report[1].size() > 1) {//2 videos werden ggf. gelöscht ohne gesehen zu werden.
|
|
LinkedList<String> store = new LinkedList<>();
|
|
try {
|
|
while(!report[1].isEmpty() && store.size() < 50) {
|
|
store.add(report[1].removeFirst());
|
|
|
|
count++;
|
|
}
|
|
} catch(NoSuchElementException ignored) {//concurrentmodification fuckery
|
|
log.info("no suchelement bla");
|
|
}
|
|
db.storeTemp(store);
|
|
}
|
|
log.info(count + " videos added.");
|
|
crawlerThread.found.clear();
|
|
crawlerThread.thread.interrupt();//free from lock
|
|
}
|
|
|
|
long runtimes = (System.currentTimeMillis() - start) / 1000;
|
|
if(runtimes < 0)
|
|
runtimes = 1;
|
|
float vidps = (crawlcount / (float) runtimes);//videos per second
|
|
Main.getMain().broadcastAdmin(vidps + "v/s " + crawlcount + " total V");
|
|
|
|
//save to db
|
|
currentstate = "save to DB";
|
|
log.info("save " + toSave.size() + " videos to DB.");
|
|
while(!toSave.isEmpty()) {
|
|
LinkedList<String> videoids = new LinkedList<>();
|
|
for(int i = 0; i < 50 && !toSave.isEmpty(); i++) {
|
|
videoids.add(toSave.remove(0));
|
|
}
|
|
if(videoids.size() > 0) {
|
|
List<Video> videos = api.getInfos(videoids)[0];
|
|
db.addVideos(videos);
|
|
}
|
|
}
|
|
|
|
//at the beginning there is maybe just one video to crawl, so keep it calm.
|
|
if(startup > 0) {
|
|
startup --;
|
|
currentstate = "startup sleep";
|
|
log.info("startup sleep");
|
|
try {
|
|
Thread.sleep(2000);
|
|
} catch(InterruptedException e) {}
|
|
}
|
|
} catch(Throwable t) {
|
|
log.warn("exception in Crawler!", t);
|
|
StringBuilder sb = new StringBuilder();
|
|
for(StackTraceElement elem : t.getStackTrace()) {
|
|
sb.append(elem.getFileName() + "(").append(elem.getMethodName() + ":").append(elem.getLineNumber() + ")\n");
|
|
}
|
|
Main.getMain().broadcastAdmin("Excpetion in crawler: " + t.toString() + "\n" + sb.toString() );
|
|
crawl = false;
|
|
Main.getMain().stop();
|
|
}
|
|
}
|
|
db.deleteDouble();
|
|
|
|
//end
|
|
long runtimes = (System.currentTimeMillis() - start) / 1000;
|
|
if(runtimes < 0)
|
|
runtimes = 1;
|
|
int runtimem = (int) (runtimes / 60);
|
|
float vidps = (crawlcount / (float) runtimes);//videos per second
|
|
log.info("Crawling Stopped. Runtime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )");
|
|
}
|
|
|
|
public DB getDB() {
|
|
return db;
|
|
}
|
|
|
|
public static Video getVideo() {
|
|
return new Video();
|
|
}
|
|
|
|
public String getStats() {
|
|
long runtimes = (System.currentTimeMillis() - start) / 1000;
|
|
if(runtimes < 0)
|
|
runtimes = 1;
|
|
float vidps = (crawlcount / (float) runtimes);//videos per second
|
|
int runtimem = (int) (runtimes / 60);
|
|
String out = "";
|
|
out += "ToCrawl: " + toCrawl.size();
|
|
out += "\nToknown: " + toknown.size();
|
|
out += "\nToSave: " + toSave.size();
|
|
out += "\nrequested: " + requested.size();
|
|
out += "\nRuntime: " + runtimem + "min and " + crawlcount + " videos crawled. ( " + vidps + " v/s )";
|
|
out += "\nState: " + currentstate;
|
|
out += "\nThread Nr, todo size, requested, crawledsize, foundsize";
|
|
for (int i = 0; i < threads.size(); i++) {
|
|
CrawlerThread thre = threads.get(i);
|
|
out += "\n " + i + " " + thre.todo.size() + " " + thre.requested + " " + thre.crawled.size() + " " + thre.found.size();
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/**
|
|
* Updates old entrys of the DB. currently unused.
|
|
*/
|
|
private void updateDB() {
|
|
log.info("updating DB Offset= " + updateOffset);
|
|
LinkedList<String> vids = db.getUncompleted(50, updateOffset);
|
|
LinkedList<Video>[] infos = api.getInfos(vids);
|
|
if(infos != null) {
|
|
int size = infos[0].size() + infos[1].size();
|
|
if(size < 50) {
|
|
updateOffset += ((50-size)/2)+1;
|
|
}
|
|
if(infos[1].size() > 0) {
|
|
log.info("delete " + infos[1].size() + " livestreams");
|
|
db.removeVideos(infos[1]);
|
|
}
|
|
|
|
db.updateVideos(infos[0]);
|
|
log.info("Updated " + infos[0].size() + " Videos.");
|
|
}
|
|
}
|
|
|
|
public static class Video {
|
|
String id = "";
|
|
String title = "";
|
|
String channel = "";
|
|
String tags = "";
|
|
int length = 0;//the length of the video in seconds
|
|
String languageCode = "";
|
|
byte categorie = 0;
|
|
long created = 0;
|
|
boolean live = false;
|
|
}
|
|
|
|
}
|