more performance improvements
This commit is contained in:
parent
4471c0f01d
commit
c4eaa4fa6e
|
@ -237,11 +237,6 @@ public class Crawler implements Runnable {
|
||||||
for(String t : toCrawl) {
|
for(String t : toCrawl) {
|
||||||
p.println(t);
|
p.println(t);
|
||||||
}
|
}
|
||||||
/*
|
|
||||||
p.println("-");
|
|
||||||
for(String t : toknown) {
|
|
||||||
p.println(t);
|
|
||||||
}*/
|
|
||||||
p.close();
|
p.close();
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
log.error("Error writing crawlfile.", e);
|
log.error("Error writing crawlfile.", e);
|
||||||
|
@ -278,7 +273,7 @@ public class Crawler implements Runnable {
|
||||||
}
|
}
|
||||||
db.storeTemp(store);
|
db.storeTemp(store);
|
||||||
}
|
}
|
||||||
log.info(count + " videos added.");
|
log.info(count + " videos added from " + threadname);
|
||||||
profiler.endSection();
|
profiler.endSection();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -302,7 +297,7 @@ public class Crawler implements Runnable {
|
||||||
ArrayList<Video> videos = (ArrayList<Video>) api.getInfos(videoids)[0];
|
ArrayList<Video> videos = (ArrayList<Video>) api.getInfos(videoids)[0];
|
||||||
profiler.endStartSection("sendtoDB");
|
profiler.endStartSection("sendtoDB");
|
||||||
db.addVideos(videos, false);
|
db.addVideos(videos, false);
|
||||||
profiler.endSection();
|
profiler.endSection();//sendtoDB
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
profiler.endSection();//save2DB
|
profiler.endSection();//save2DB
|
||||||
|
|
|
@ -26,8 +26,9 @@ public class DB implements Runnable {
|
||||||
private Server serv = new Server(this);
|
private Server serv = new Server(this);
|
||||||
private Thread randomrefill = null;
|
private Thread randomrefill = null;
|
||||||
private int dbsize = 0;
|
private int dbsize = 0;
|
||||||
private ArrayList<Video> tostorebuffer;
|
private StringBuilder tostorebuffer ;
|
||||||
private int writebuffersize = 500;
|
private int writebuffersize = 500;
|
||||||
|
private int writebuffercurrentsize = 0;
|
||||||
|
|
||||||
public DB() {
|
public DB() {
|
||||||
try {
|
try {
|
||||||
|
@ -67,7 +68,7 @@ public class DB implements Runnable {
|
||||||
} catch(NumberFormatException e) {
|
} catch(NumberFormatException e) {
|
||||||
log.warn("could not read the number \"" + Config.prop.getProperty("db.writebuffersize") + "\" from the config file. db.writebuffersize");
|
log.warn("could not read the number \"" + Config.prop.getProperty("db.writebuffersize") + "\" from the config file. db.writebuffersize");
|
||||||
}
|
}
|
||||||
tostorebuffer = new ArrayList<>(writebuffersize);
|
tostorebuffer = new StringBuilder(writebuffersize);
|
||||||
} catch (SQLException e) {
|
} catch (SQLException e) {
|
||||||
log.error("Error while connecting to the database! ", e);
|
log.error("Error while connecting to the database! ", e);
|
||||||
}
|
}
|
||||||
|
@ -128,28 +129,27 @@ public class DB implements Runnable {
|
||||||
* save the list of videos to the DB
|
* save the list of videos to the DB
|
||||||
* @param input
|
* @param input
|
||||||
*/
|
*/
|
||||||
public void addVideos(List<Video> input, boolean force) {
|
public void addVideos(ArrayList<Video> input, boolean force) {
|
||||||
//log.info("add " + input.size() + " videos");
|
//log.info("add " + input.size() + " videos");
|
||||||
if(input != null) {
|
if(input != null) {
|
||||||
if(input.size() > 0) {
|
if(input.size() > 0) {
|
||||||
tostorebuffer.addAll(input);
|
for(int i = 0; i < input.size(); i++) {
|
||||||
|
Video v = input.get(i);
|
||||||
|
if(v != null)
|
||||||
|
tostorebuffer.append(",('").append(v.id).append("','").append(v.length).append("','").append(v.created).append("','").append(v.languageCode).append("','").append(v.categorie).append("','").append(v.title).append("','").append(v.channel).append("','").append(v.tags).append("') ");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(tostorebuffer.size() > writebuffersize || force) {
|
if(writebuffercurrentsize > writebuffersize || force) {
|
||||||
dbsize += tostorebuffer.size();
|
if(tostorebuffer.length() > 10) {
|
||||||
StringBuilder sb = new StringBuilder();
|
dbsize += writebuffercurrentsize;
|
||||||
for(int i = 0; i < tostorebuffer.size(); i++) {
|
tostorebuffer.deleteCharAt(0);//delete leading ','
|
||||||
if(i > 0)
|
String qu = "INSERT IGNORE INTO `ytcrawler`.`videos`(`id`, `length`, `created`, `langcode`, `category`, `videotitle`, `channel`, `tags`) VALUES " + tostorebuffer.toString();
|
||||||
sb.append(',');
|
|
||||||
Video v = tostorebuffer.get(i);
|
|
||||||
if(v != null)
|
|
||||||
sb.append("('").append(v.id).append("','").append(v.length).append("','").append(v.created).append("','").append(v.languageCode).append("','").append(v.categorie).append("','").append(v.title).append("','").append(v.channel).append("','").append(v.tags).append("') ");
|
|
||||||
}
|
|
||||||
tostorebuffer.clear();
|
|
||||||
if(sb.length() > 2) {
|
|
||||||
String qu = "INSERT IGNORE INTO `ytcrawler`.`videos`(`id`, `length`, `created`, `langcode`, `category`, `videotitle`, `channel`, `tags`) VALUES " + sb.toString();
|
|
||||||
update(qu);
|
update(qu);
|
||||||
}
|
}
|
||||||
|
//reset buffer
|
||||||
|
writebuffercurrentsize = 0;
|
||||||
|
tostorebuffer = new StringBuilder(writebuffersize);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -37,7 +37,7 @@ public class YoutubeAPI {
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<Video>[] getInfos(List<String> ids) {
|
public List<Video>[] getInfos(List<String> ids) {
|
||||||
log.info("get " + ids.size() + " infos");
|
//log.info("get " + ids.size() + " infos");
|
||||||
if(ids.isEmpty())
|
if(ids.isEmpty())
|
||||||
return null;
|
return null;
|
||||||
|
|
||||||
|
@ -138,7 +138,7 @@ public class YoutubeAPI {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} while(!nextpage.equals(""));
|
} while(!nextpage.equals(""));
|
||||||
log.info("got " + (out.size() + livestr.size()) + " infos");
|
//log.info("got " + (out.size() + livestr.size()) + " infos");
|
||||||
return new List[] {out, livestr};
|
return new List[] {out, livestr};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue