forked from MrBesen/YoutubeCrawler
kleine bugfixes, admins file
This commit is contained in:
parent
4917369b34
commit
b67952ce88
|
@ -137,7 +137,7 @@ public class Crawler implements Runnable {
|
|||
}
|
||||
while(crawl) {
|
||||
log.info("to Crawl: " + toCrawl.size() + " known: " + toknown.size() + " Time: " + dateform.format(new Date()));
|
||||
|
||||
try {
|
||||
//fullfill request
|
||||
while(!requested.isEmpty() && !toCrawl.isEmpty() && crawl) {
|
||||
log.info("fullfill request");
|
||||
|
@ -212,7 +212,7 @@ public class Crawler implements Runnable {
|
|||
crawlcount+= report[0].size();
|
||||
toSave.addAll(report[0]);
|
||||
crawlerThread.crawled.clear();
|
||||
while(report[1].size() > 0) {
|
||||
while(report[1].size() > 15) {
|
||||
LinkedList<String> store = new LinkedList<>();
|
||||
try {
|
||||
while(!report[1].isEmpty() && store.size() < 50) {
|
||||
|
@ -221,12 +221,20 @@ public class Crawler implements Runnable {
|
|||
count++;
|
||||
}
|
||||
} catch(NoSuchElementException ignored) {//concurrentmodification fuckery
|
||||
log.info("no suchelement bla");
|
||||
}
|
||||
db.storeTemp(store);
|
||||
}
|
||||
log.info(count + " videos added.");
|
||||
crawlerThread.found.clear();
|
||||
}
|
||||
db.deleteDouble();
|
||||
|
||||
long runtimes = (System.currentTimeMillis() - start) / 1000;
|
||||
if(runtimes < 0)
|
||||
runtimes = 1;
|
||||
float vidps = (crawlcount / (float) runtimes);//videos per second
|
||||
Main.getMain().broadcastAdmin(vidps + "v/s " + crawlcount + " total V");
|
||||
|
||||
//save to db
|
||||
currentstate = "save to DB";
|
||||
|
@ -251,6 +259,16 @@ public class Crawler implements Runnable {
|
|||
Thread.sleep(2000);
|
||||
} catch(InterruptedException e) {}
|
||||
}
|
||||
} catch(Throwable t) {
|
||||
log.warn("exception in Crawler!", t);
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for(StackTraceElement elem : t.getStackTrace()) {
|
||||
sb.append(elem.getFileName() + "(").append(elem.getMethodName() + ":").append(elem.getLineNumber() + ")\n");
|
||||
}
|
||||
Main.getMain().broadcastAdmin("Excpetion in crawler: " + t.toString() + "\n" + sb.toString() );
|
||||
crawl = false;
|
||||
Main.getMain().stop();
|
||||
}
|
||||
}
|
||||
|
||||
//end
|
||||
|
@ -270,12 +288,12 @@ public class Crawler implements Runnable {
|
|||
return new Video();
|
||||
}
|
||||
|
||||
public String printStats() {
|
||||
public String getStats() {
|
||||
long runtimes = (System.currentTimeMillis() - start) / 1000;
|
||||
if(runtimes < 0)
|
||||
runtimes = 1;
|
||||
int runtimem = (int) (runtimes / 60);
|
||||
float vidps = (crawlcount / (float) runtimes);//videos per second
|
||||
int runtimem = (int) (runtimes / 60);
|
||||
String out = "";
|
||||
out += "ToCrawl: " + toCrawl.size();
|
||||
out += "\nToknown: " + toknown.size();
|
||||
|
|
|
@ -97,6 +97,7 @@ public class DB {
|
|||
if(i > 0)
|
||||
sb.append(',');
|
||||
Video v = input.get(i);
|
||||
if(v != null)
|
||||
sb.append("('").append(v.id).append("','").append(v.length).append("','").append(v.created).append("','").append(v.languageCode).append("','").append(v.categorie).append("','").append(v.title).append("','").append(v.channel).append("','").append(v.tags).append("') ");
|
||||
}
|
||||
if(sb.length() > 2) {
|
||||
|
@ -195,18 +196,22 @@ public class DB {
|
|||
}
|
||||
|
||||
public LinkedList<String> restoreTemp() {
|
||||
ResultSet res = query("SELECT * FROM `ytcrawler`.`temp` LIMIT 0,500;");
|
||||
ResultSet res = query("SELECT * FROM `ytcrawler`.`temp` LIMIT 500;");
|
||||
LinkedList<String> out = new LinkedList<>();
|
||||
log.info("RestoreTemp");
|
||||
try {
|
||||
while(res.next()) {
|
||||
out.add(res.getString(1));
|
||||
}
|
||||
update("DELETE FROM `ytcrawler`.`temp` LIMIT 0,500;");
|
||||
update("DELETE FROM `ytcrawler`.`temp` LIMIT 500;");
|
||||
} catch (Exception e) {}
|
||||
return out;
|
||||
}
|
||||
|
||||
public void deleteDouble() {
|
||||
update("call ytcrawler.deletedouble();");
|
||||
}
|
||||
|
||||
public void storeTemp(LinkedList<String> strings) {
|
||||
if(!strings.isEmpty()) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package de.mrbesen.youtubecrawler;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Random;
|
||||
import java.util.Scanner;
|
||||
|
@ -16,16 +17,22 @@ import de.mrbesen.telegram.objects.TUser;
|
|||
|
||||
public class Main implements CommandHandler, EventListener{
|
||||
|
||||
private ArrayList<TUser> admins = new ArrayList<>();
|
||||
private ArrayList<String> admins = new ArrayList<>();//usernames of admins
|
||||
private String adminstr = null;
|
||||
private long setadminstr = -1;
|
||||
private static String abc = "abcdefghijklmnopqrstuvwxyz";
|
||||
private Logger log = Logger.getLogger(this.getClass().getName());
|
||||
private TelegramAPI tapi;
|
||||
private Thread mainthread;
|
||||
private static Main main;
|
||||
|
||||
public static Main getMain() {
|
||||
return main;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
new Main().run();
|
||||
main = new Main();
|
||||
main.run();
|
||||
}
|
||||
|
||||
private Crawler cra;
|
||||
|
@ -53,8 +60,23 @@ public class Main implements CommandHandler, EventListener{
|
|||
tapi.getCommandManager().registerCommand("stop", this);
|
||||
tapi.getEventManager().registerEvent(this);
|
||||
tapi.setHelpText("Send the command /random to get a random video.");
|
||||
tapi.setUpdateInterval(25000);
|
||||
tapi.start();
|
||||
|
||||
//load admins
|
||||
Log.l.info("Loading admins.");
|
||||
try {
|
||||
Scanner scan = new Scanner(new File("admins"));
|
||||
while(scan.hasNext()) {
|
||||
String line = scan.nextLine().trim();
|
||||
if(!line.isEmpty()) {
|
||||
admins.add(line);
|
||||
}
|
||||
}
|
||||
scan.close();
|
||||
} catch (IOException | NumberFormatException | ArrayIndexOutOfBoundsException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
//CLI
|
||||
Scanner s = new Scanner(System.in);
|
||||
String in;
|
||||
|
@ -71,7 +93,7 @@ public class Main implements CommandHandler, EventListener{
|
|||
}
|
||||
} else if(in.equalsIgnoreCase("stats")) {
|
||||
log.info("Getting Stats");
|
||||
for(String line : cra.printStats().split("\n")) {
|
||||
for(String line : cra.getStats().split("\n")) {
|
||||
log.info(line);
|
||||
}
|
||||
}
|
||||
|
@ -80,7 +102,8 @@ public class Main implements CommandHandler, EventListener{
|
|||
log.info("Terminated.");
|
||||
}
|
||||
|
||||
private void stop() {
|
||||
public void stop() {
|
||||
log.info("Stop.");
|
||||
tapi.stop();
|
||||
cra.stop();
|
||||
mainthread.interrupt();
|
||||
|
@ -93,7 +116,7 @@ public class Main implements CommandHandler, EventListener{
|
|||
sender.sendMessage("https://youtube.com/watch?v=" + ytid);
|
||||
return true;
|
||||
} else if(cmd.equals("admin")) {
|
||||
if(admins.contains(sender)) {
|
||||
if(admins.contains(sender.getName())) {
|
||||
sender.sendMessage("You are admin.");
|
||||
return true;
|
||||
} else {
|
||||
|
@ -102,12 +125,12 @@ public class Main implements CommandHandler, EventListener{
|
|||
log.info("Adminstr: " + adminstr);
|
||||
}
|
||||
} else if(cmd.equals("stats")) {
|
||||
if(admins.contains(sender)) {
|
||||
sender.sendMessage(cra.printStats());
|
||||
if(admins.contains(sender.getName())) {
|
||||
sender.sendMessage(cra.getStats());
|
||||
return true;
|
||||
}
|
||||
} else if(cmd.equals("stop")) {
|
||||
if(admins.contains(sender)) {
|
||||
if(admins.contains(sender.getName())) {
|
||||
stop();
|
||||
sender.sendMessage("Stop.");
|
||||
log.info("Stopped via Telegram by " + sender.getFirstName());
|
||||
|
@ -117,6 +140,12 @@ public class Main implements CommandHandler, EventListener{
|
|||
return false;
|
||||
}
|
||||
|
||||
public void broadcastAdmin(String msg) {
|
||||
for(String admin : admins) {
|
||||
tapi.getUser(admin).sendMessage(msg);
|
||||
}
|
||||
}
|
||||
|
||||
private String getRandomStr(int length) {
|
||||
Random rand = new Random();
|
||||
String out = "";
|
||||
|
@ -132,11 +161,11 @@ public class Main implements CommandHandler, EventListener{
|
|||
if(e.getMessage() != null && (System.currentTimeMillis() - setadminstr) / 1000 < 60) {
|
||||
if(e.getMessage().getText() != null) {
|
||||
if(e.getMessage().getText().equals(adminstr)) {
|
||||
admins.add(e.getUser());
|
||||
admins.add(e.getUser().getName());
|
||||
e.getMessage().reply("You are now Admin!");
|
||||
adminstr = null;
|
||||
setadminstr = -1;
|
||||
log.info(e.getUser().getFirstName() + " is now Admin!");
|
||||
log.info(e.getUser().getID() + " " + e.getUser().getName() + " " + e.getUser().getFirstName() + " is now Admin!");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue