MrBesen
/
YoutubeCrawler
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

improved db handling

This commit is contained in:
mrbesen 2021-10-26 01:06:47 +02:00
parent e02d51b72c
commit 928942e3ef
Signed by: MrBesen
GPG Key ID: 596B2350DCD67504
2 changed files with 30 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
.vscode/c_cpp_properties.json vendored Normal file
View File

@ -0,0 +1,18 @@
{
"configurations": [
{
"name": "Linux",
"includePath": [
"${workspaceFolder}/**",
"${workspaceFolder}/includes/",
"/usr/lib/jvm/java-8-openjdk-amd64/include/",
"/usr/lib/jvm/java-8-openjdk-amd64/include/linux/"
],
"defines": [],
"compilerPath": "/usr/bin/clang",
"cStandard": "c11",
"cppStandard": "c++17"
}
],
"version": 4
}

22
src/main/de/mrbesen/youtubecrawler/DB.java
View File

@ -26,13 +26,16 @@ public class DB implements Runnable {
private Thread randomrefill = null; private Thread randomrefill = null;
private int dbsize = 0; private int dbsize = 0;
private StringBuilder tostorebuffer ; private StringBuilder tostorebuffer;
private int writebuffersize = 500; private int writebuffersize = 500;
private int writebuffercurrentsize = 0; private int writebuffercurrentsize = 0;
private StringBuilder totempbuffer; private StringBuilder totempbuffer;
private int writetempbuffercurrentsize = 0; private int writetempbuffercurrentsize = 0;
private final int TEMPBUFFERRATIO = 15;
private final int STOREBUFFERRATIO = 100;
public DB() { public DB() {
try { try {
connect(false); connect(false);
@ -70,8 +73,8 @@ public class DB implements Runnable {
} catch(NumberFormatException e) { } catch(NumberFormatException e) {
log.warn("could not read the number \"" + Config.prop.getProperty("db.writebuffersize") + "\" from the config file. db.writebuffersize"); log.warn("could not read the number \"" + Config.prop.getProperty("db.writebuffersize") + "\" from the config file. db.writebuffersize");
} }
tostorebuffer = new StringBuilder(writebuffersize); tostorebuffer = new StringBuilder(writebuffersize * STOREBUFFERRATIO);
totempbuffer = new StringBuilder(writebuffersize); totempbuffer = new StringBuilder(writebuffersize * TEMPBUFFERRATIO);
} catch (SQLException e) { } catch (SQLException e) {
log.error("Error while connecting to the database! ", e); log.error("Error while connecting to the database! ", e);
} }
@ -138,7 +141,7 @@ public class DB implements Runnable {
for(int i = 0; i < input.size(); i++) { for(int i = 0; i < input.size(); i++) {
Video v = input.get(i); Video v = input.get(i);
if(v != null) if(v != null)
tostorebuffer.append(",('").append(escape(v.id)).append("',").append(v.length).append(",").append(v.created).append(",'").append(escape(v.languageCode)).append("',").append(v.categorie).append(",'").append(escape(v.title)).append("','").append(escape(v.channel)).append("','").append(escape(v.tags)).append("') "); tostorebuffer.append("('").append(escape(v.id)).append("',").append(v.length).append(",").append(v.created).append(",'").append(escape(v.languageCode)).append("',").append(v.categorie).append(",'").append(escape(v.title)).append("','").append(escape(v.channel)).append("','").append(escape(v.tags)).append("'),");
} }
} }
} }
@ -146,13 +149,13 @@ public class DB implements Runnable {
if(tostorebuffer.length() > 10) { if(tostorebuffer.length() > 10) {
log.info("Write databuffer to DB video count: " + writebuffercurrentsize); log.info("Write databuffer to DB video count: " + writebuffercurrentsize);
dbsize += writebuffercurrentsize; dbsize += writebuffercurrentsize;
tostorebuffer.deleteCharAt(0);//delete leading ',' tostorebuffer.deleteCharAt(tostorebuffer.length()-1);//delete trailing ','
String qu = "INSERT IGNORE INTO `videos`(`id`, `length`, `created`, `langcode`, `category`, `videotitle`, `channel`, `tags`) VALUES " + tostorebuffer.toString(); String qu = "INSERT IGNORE INTO `videos`(`id`, `length`, `created`, `langcode`, `category`, `videotitle`, `channel`, `tags`) VALUES " + tostorebuffer.toString();
update(qu); update(qu);
//reset buffer //reset buffer
writebuffercurrentsize = 0; writebuffercurrentsize = 0;
tostorebuffer = new StringBuilder(writebuffersize); tostorebuffer = new StringBuilder(writebuffersize * STOREBUFFERRATIO);
} }
} }
} }
@ -284,18 +287,18 @@ public class DB implements Runnable {
log.info("store Temp to buffer: " + strings.size()); log.info("store Temp to buffer: " + strings.size());
writetempbuffercurrentsize += strings.size(); writetempbuffercurrentsize += strings.size();
for(String s : strings) { for(String s : strings) {
totempbuffer.append(", ('").append(escape(s)).append("')"); totempbuffer.append("('").append(escape(s)).append("'),");
} }
} }
if(writetempbuffercurrentsize > writebuffersize || force) { if(writetempbuffercurrentsize > writebuffersize || force) {
log.info("Write Buffer: " + writetempbuffercurrentsize); log.info("Write Buffer: " + writetempbuffercurrentsize);
totempbuffer.deleteCharAt(0);//delete leading ',' totempbuffer.deleteCharAt(totempbuffer.length()-1);//delete trailing ','
String qu = "INSERT IGNORE INTO `temp` (`ytid`) VALUES " + totempbuffer.toString() + ";"; String qu = "INSERT IGNORE INTO `temp` (`ytid`) VALUES " + totempbuffer.toString() + ";";
update(qu); update(qu);
//reset //reset
writetempbuffercurrentsize = 0; writetempbuffercurrentsize = 0;
totempbuffer = new StringBuilder(writebuffersize); totempbuffer = new StringBuilder(writebuffersize * TEMPBUFFERRATIO);
} }
} }
@ -308,7 +311,6 @@ public class DB implements Runnable {
if(con != null) { if(con != null) {
if(!con.isClosed()) { if(!con.isClosed()) {
addVideos(null, true); addVideos(null, true);
con.commit();
con.close(); con.close();
} }
} }