This commit is contained in:
mrbesen 2018-10-21 21:57:36 +02:00
parent c4eaa4fa6e
commit 5519ab072d
3 changed files with 13 additions and 34 deletions

View File

@ -326,12 +326,6 @@ public class Crawler implements Runnable {
} }
profiler.endSection();//main profiler.endSection();//main
} }
profiler.startSection("cleanup");
profiler.startSection("deleteDouble");
//db.deleteDouble();
profiler.endStartSection("stopDB");
db.stop();
profiler.endSection();
profiler.endSection();//root profiler.endSection();//root
log.info("Profiler:"); log.info("Profiler:");
for(String s : profiler.getTreeView()) { for(String s : profiler.getTreeView()) {

View File

@ -133,6 +133,7 @@ public class DB implements Runnable {
//log.info("add " + input.size() + " videos"); //log.info("add " + input.size() + " videos");
if(input != null) { if(input != null) {
if(input.size() > 0) { if(input.size() > 0) {
writebuffercurrentsize += input.size();
for(int i = 0; i < input.size(); i++) { for(int i = 0; i < input.size(); i++) {
Video v = input.get(i); Video v = input.get(i);
if(v != null) if(v != null)
@ -142,6 +143,7 @@ public class DB implements Runnable {
} }
if(writebuffercurrentsize > writebuffersize || force) { if(writebuffercurrentsize > writebuffersize || force) {
if(tostorebuffer.length() > 10) { if(tostorebuffer.length() > 10) {
log.info("Write data to DB video count: " + writebuffercurrentsize);
dbsize += writebuffercurrentsize; dbsize += writebuffercurrentsize;
tostorebuffer.deleteCharAt(0);//delete leading ',' tostorebuffer.deleteCharAt(0);//delete leading ','
String qu = "INSERT IGNORE INTO `ytcrawler`.`videos`(`id`, `length`, `created`, `langcode`, `category`, `videotitle`, `channel`, `tags`) VALUES " + tostorebuffer.toString(); String qu = "INSERT IGNORE INTO `ytcrawler`.`videos`(`id`, `length`, `created`, `langcode`, `category`, `videotitle`, `channel`, `tags`) VALUES " + tostorebuffer.toString();

View File

@ -2,7 +2,6 @@ package de.mrbesen.youtubecrawler;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.lang.Thread.State;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Random; import java.util.Random;
import java.util.Scanner; import java.util.Scanner;
@ -118,6 +117,7 @@ public class Main implements JSONCommandHandler, EventListener{
} }
} else if(in.equalsIgnoreCase("crastop")) { } else if(in.equalsIgnoreCase("crastop")) {
log.info("Stop crawler"); log.info("Stop crawler");
startcrawler = false;
cra.stop(); cra.stop();
} else if(in.equalsIgnoreCase("profiler")) { } else if(in.equalsIgnoreCase("profiler")) {
for(String profline : cra.getProfiling()) { for(String profline : cra.getProfiling()) {
@ -143,35 +143,18 @@ public class Main implements JSONCommandHandler, EventListener{
public void stop() { public void stop() {
log.info("Stop."); log.info("Stop.");
cra.stop(); cra.stop();
if(startcrawler) { if(!startcrawler) {//crawler not running
new Thread(new Runnable() { cra.getDB().stop();
@Override tapi.stop();
public void run() {
int count = 0;
while(crawlerthread.isAlive()) {
try {
Thread.sleep(20000);
} catch(InterruptedException ignored) {}
count = 0;
for(Thread t : Thread.getAllStackTraces().keySet()) {
String name = format(t.getName(), 15);
log.info(name + ":\t" + t.getState().name());
if(t.getState() == State.RUNNABLE) {
count ++;
}
}
log.info("count: " + count);
}
log.info("Stoped Shutdown Watchdog");
}
}, "Shutdown Watchdog").start();
} }
} }
void stopcallback() { void stopcallback() {
tapi.stop(); if(startcrawler) {//crawler should run
log.info("tapi stopped"); tapi.stop();
mainthread.interrupt(); cra.getDB().stop();
mainthread.interrupt();
}
} }
@Override @Override
@ -179,7 +162,7 @@ public class Main implements JSONCommandHandler, EventListener{
//unused //unused
return false; return false;
} }
@Override @Override
public boolean onCommand(TUser sender, String cmd, String[] args, JSONObject j) { public boolean onCommand(TUser sender, String cmd, String[] args, JSONObject j) {
if(cmd.startsWith("random")) { if(cmd.startsWith("random")) {
@ -222,7 +205,7 @@ public class Main implements JSONCommandHandler, EventListener{
return true; return true;
} }
} }
/*else if(cmd.equals("stop")) { /*else if(cmd.equals("stop")) {
if(admins.contains(sender.getName())) { if(admins.contains(sender.getName())) {
stop(); stop();