improved db handling
This commit is contained in:
parent
e02d51b72c
commit
928942e3ef
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Linux",
|
||||
"includePath": [
|
||||
"${workspaceFolder}/**",
|
||||
"${workspaceFolder}/includes/",
|
||||
"/usr/lib/jvm/java-8-openjdk-amd64/include/",
|
||||
"/usr/lib/jvm/java-8-openjdk-amd64/include/linux/"
|
||||
],
|
||||
"defines": [],
|
||||
"compilerPath": "/usr/bin/clang",
|
||||
"cStandard": "c11",
|
||||
"cppStandard": "c++17"
|
||||
}
|
||||
],
|
||||
"version": 4
|
||||
}
|
|
@ -33,6 +33,9 @@ public class DB implements Runnable {
|
|||
private StringBuilder totempbuffer;
|
||||
private int writetempbuffercurrentsize = 0;
|
||||
|
||||
private final int TEMPBUFFERRATIO = 15;
|
||||
private final int STOREBUFFERRATIO = 100;
|
||||
|
||||
public DB() {
|
||||
try {
|
||||
connect(false);
|
||||
|
@ -70,8 +73,8 @@ public class DB implements Runnable {
|
|||
} catch(NumberFormatException e) {
|
||||
log.warn("could not read the number \"" + Config.prop.getProperty("db.writebuffersize") + "\" from the config file. db.writebuffersize");
|
||||
}
|
||||
tostorebuffer = new StringBuilder(writebuffersize);
|
||||
totempbuffer = new StringBuilder(writebuffersize);
|
||||
tostorebuffer = new StringBuilder(writebuffersize * STOREBUFFERRATIO);
|
||||
totempbuffer = new StringBuilder(writebuffersize * TEMPBUFFERRATIO);
|
||||
} catch (SQLException e) {
|
||||
log.error("Error while connecting to the database! ", e);
|
||||
}
|
||||
|
@ -138,7 +141,7 @@ public class DB implements Runnable {
|
|||
for(int i = 0; i < input.size(); i++) {
|
||||
Video v = input.get(i);
|
||||
if(v != null)
|
||||
tostorebuffer.append(",('").append(escape(v.id)).append("',").append(v.length).append(",").append(v.created).append(",'").append(escape(v.languageCode)).append("',").append(v.categorie).append(",'").append(escape(v.title)).append("','").append(escape(v.channel)).append("','").append(escape(v.tags)).append("') ");
|
||||
tostorebuffer.append("('").append(escape(v.id)).append("',").append(v.length).append(",").append(v.created).append(",'").append(escape(v.languageCode)).append("',").append(v.categorie).append(",'").append(escape(v.title)).append("','").append(escape(v.channel)).append("','").append(escape(v.tags)).append("'),");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -146,13 +149,13 @@ public class DB implements Runnable {
|
|||
if(tostorebuffer.length() > 10) {
|
||||
log.info("Write databuffer to DB video count: " + writebuffercurrentsize);
|
||||
dbsize += writebuffercurrentsize;
|
||||
tostorebuffer.deleteCharAt(0);//delete leading ','
|
||||
tostorebuffer.deleteCharAt(tostorebuffer.length()-1);//delete trailing ','
|
||||
String qu = "INSERT IGNORE INTO `videos`(`id`, `length`, `created`, `langcode`, `category`, `videotitle`, `channel`, `tags`) VALUES " + tostorebuffer.toString();
|
||||
update(qu);
|
||||
|
||||
//reset buffer
|
||||
writebuffercurrentsize = 0;
|
||||
tostorebuffer = new StringBuilder(writebuffersize);
|
||||
tostorebuffer = new StringBuilder(writebuffersize * STOREBUFFERRATIO);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -284,18 +287,18 @@ public class DB implements Runnable {
|
|||
log.info("store Temp to buffer: " + strings.size());
|
||||
writetempbuffercurrentsize += strings.size();
|
||||
for(String s : strings) {
|
||||
totempbuffer.append(", ('").append(escape(s)).append("')");
|
||||
totempbuffer.append("('").append(escape(s)).append("'),");
|
||||
}
|
||||
}
|
||||
if(writetempbuffercurrentsize > writebuffersize || force) {
|
||||
log.info("Write Buffer: " + writetempbuffercurrentsize);
|
||||
totempbuffer.deleteCharAt(0);//delete leading ','
|
||||
totempbuffer.deleteCharAt(totempbuffer.length()-1);//delete trailing ','
|
||||
String qu = "INSERT IGNORE INTO `temp` (`ytid`) VALUES " + totempbuffer.toString() + ";";
|
||||
update(qu);
|
||||
|
||||
//reset
|
||||
writetempbuffercurrentsize = 0;
|
||||
totempbuffer = new StringBuilder(writebuffersize);
|
||||
totempbuffer = new StringBuilder(writebuffersize * TEMPBUFFERRATIO);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -308,7 +311,6 @@ public class DB implements Runnable {
|
|||
if(con != null) {
|
||||
if(!con.isClosed()) {
|
||||
addVideos(null, true);
|
||||
con.commit();
|
||||
con.close();
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue