more performance improvements
This commit is contained in:
parent
4471c0f01d
commit
c4eaa4fa6e
|
@ -237,11 +237,6 @@ public class Crawler implements Runnable {
|
|||
for(String t : toCrawl) {
|
||||
p.println(t);
|
||||
}
|
||||
/*
|
||||
p.println("-");
|
||||
for(String t : toknown) {
|
||||
p.println(t);
|
||||
}*/
|
||||
p.close();
|
||||
} catch (IOException e) {
|
||||
log.error("Error writing crawlfile.", e);
|
||||
|
@ -278,7 +273,7 @@ public class Crawler implements Runnable {
|
|||
}
|
||||
db.storeTemp(store);
|
||||
}
|
||||
log.info(count + " videos added.");
|
||||
log.info(count + " videos added from " + threadname);
|
||||
profiler.endSection();
|
||||
}
|
||||
|
||||
|
@ -302,7 +297,7 @@ public class Crawler implements Runnable {
|
|||
ArrayList<Video> videos = (ArrayList<Video>) api.getInfos(videoids)[0];
|
||||
profiler.endStartSection("sendtoDB");
|
||||
db.addVideos(videos, false);
|
||||
profiler.endSection();
|
||||
profiler.endSection();//sendtoDB
|
||||
}
|
||||
}
|
||||
profiler.endSection();//save2DB
|
||||
|
|
|
@ -26,8 +26,9 @@ public class DB implements Runnable {
|
|||
private Server serv = new Server(this);
|
||||
private Thread randomrefill = null;
|
||||
private int dbsize = 0;
|
||||
private ArrayList<Video> tostorebuffer;
|
||||
private StringBuilder tostorebuffer ;
|
||||
private int writebuffersize = 500;
|
||||
private int writebuffercurrentsize = 0;
|
||||
|
||||
public DB() {
|
||||
try {
|
||||
|
@ -67,7 +68,7 @@ public class DB implements Runnable {
|
|||
} catch(NumberFormatException e) {
|
||||
log.warn("could not read the number \"" + Config.prop.getProperty("db.writebuffersize") + "\" from the config file. db.writebuffersize");
|
||||
}
|
||||
tostorebuffer = new ArrayList<>(writebuffersize);
|
||||
tostorebuffer = new StringBuilder(writebuffersize);
|
||||
} catch (SQLException e) {
|
||||
log.error("Error while connecting to the database! ", e);
|
||||
}
|
||||
|
@ -128,28 +129,27 @@ public class DB implements Runnable {
|
|||
* save the list of videos to the DB
|
||||
* @param input
|
||||
*/
|
||||
public void addVideos(List<Video> input, boolean force) {
|
||||
public void addVideos(ArrayList<Video> input, boolean force) {
|
||||
//log.info("add " + input.size() + " videos");
|
||||
if(input != null) {
|
||||
if(input.size() > 0) {
|
||||
tostorebuffer.addAll(input);
|
||||
for(int i = 0; i < input.size(); i++) {
|
||||
Video v = input.get(i);
|
||||
if(v != null)
|
||||
tostorebuffer.append(",('").append(v.id).append("','").append(v.length).append("','").append(v.created).append("','").append(v.languageCode).append("','").append(v.categorie).append("','").append(v.title).append("','").append(v.channel).append("','").append(v.tags).append("') ");
|
||||
}
|
||||
}
|
||||
}
|
||||
if(tostorebuffer.size() > writebuffersize || force) {
|
||||
dbsize += tostorebuffer.size();
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for(int i = 0; i < tostorebuffer.size(); i++) {
|
||||
if(i > 0)
|
||||
sb.append(',');
|
||||
Video v = tostorebuffer.get(i);
|
||||
if(v != null)
|
||||
sb.append("('").append(v.id).append("','").append(v.length).append("','").append(v.created).append("','").append(v.languageCode).append("','").append(v.categorie).append("','").append(v.title).append("','").append(v.channel).append("','").append(v.tags).append("') ");
|
||||
}
|
||||
tostorebuffer.clear();
|
||||
if(sb.length() > 2) {
|
||||
String qu = "INSERT IGNORE INTO `ytcrawler`.`videos`(`id`, `length`, `created`, `langcode`, `category`, `videotitle`, `channel`, `tags`) VALUES " + sb.toString();
|
||||
if(writebuffercurrentsize > writebuffersize || force) {
|
||||
if(tostorebuffer.length() > 10) {
|
||||
dbsize += writebuffercurrentsize;
|
||||
tostorebuffer.deleteCharAt(0);//delete leading ','
|
||||
String qu = "INSERT IGNORE INTO `ytcrawler`.`videos`(`id`, `length`, `created`, `langcode`, `category`, `videotitle`, `channel`, `tags`) VALUES " + tostorebuffer.toString();
|
||||
update(qu);
|
||||
}
|
||||
//reset buffer
|
||||
writebuffercurrentsize = 0;
|
||||
tostorebuffer = new StringBuilder(writebuffersize);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -37,7 +37,7 @@ public class YoutubeAPI {
|
|||
}
|
||||
|
||||
public List<Video>[] getInfos(List<String> ids) {
|
||||
log.info("get " + ids.size() + " infos");
|
||||
//log.info("get " + ids.size() + " infos");
|
||||
if(ids.isEmpty())
|
||||
return null;
|
||||
|
||||
|
@ -138,7 +138,7 @@ public class YoutubeAPI {
|
|||
}
|
||||
}
|
||||
} while(!nextpage.equals(""));
|
||||
log.info("got " + (out.size() + livestr.size()) + " infos");
|
||||
//log.info("got " + (out.size() + livestr.size()) + " infos");
|
||||
return new List[] {out, livestr};
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue