diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json new file mode 100644 index 0000000..4662699 --- /dev/null +++ b/.vscode/c_cpp_properties.json @@ -0,0 +1,18 @@ +{ + "configurations": [ + { + "name": "Linux", + "includePath": [ + "${workspaceFolder}/**", + "${workspaceFolder}/includes/", + "/usr/lib/jvm/java-8-openjdk-amd64/include/", + "/usr/lib/jvm/java-8-openjdk-amd64/include/linux/" + ], + "defines": [], + "compilerPath": "/usr/bin/clang", + "cStandard": "c11", + "cppStandard": "c++17" + } + ], + "version": 4 +} \ No newline at end of file diff --git a/src/main/de/mrbesen/youtubecrawler/DB.java b/src/main/de/mrbesen/youtubecrawler/DB.java index 7c11ae7..8c4856b 100644 --- a/src/main/de/mrbesen/youtubecrawler/DB.java +++ b/src/main/de/mrbesen/youtubecrawler/DB.java @@ -26,13 +26,16 @@ public class DB implements Runnable { private Thread randomrefill = null; private int dbsize = 0; - private StringBuilder tostorebuffer ; + private StringBuilder tostorebuffer; private int writebuffersize = 500; private int writebuffercurrentsize = 0; private StringBuilder totempbuffer; private int writetempbuffercurrentsize = 0; + private final int TEMPBUFFERRATIO = 15; + private final int STOREBUFFERRATIO = 100; + public DB() { try { connect(false); @@ -70,8 +73,8 @@ public class DB implements Runnable { } catch(NumberFormatException e) { log.warn("could not read the number \"" + Config.prop.getProperty("db.writebuffersize") + "\" from the config file. db.writebuffersize"); } - tostorebuffer = new StringBuilder(writebuffersize); - totempbuffer = new StringBuilder(writebuffersize); + tostorebuffer = new StringBuilder(writebuffersize * STOREBUFFERRATIO); + totempbuffer = new StringBuilder(writebuffersize * TEMPBUFFERRATIO); } catch (SQLException e) { log.error("Error while connecting to the database! ", e); } @@ -138,7 +141,7 @@ public class DB implements Runnable { for(int i = 0; i < input.size(); i++) { Video v = input.get(i); if(v != null) - tostorebuffer.append(",('").append(escape(v.id)).append("',").append(v.length).append(",").append(v.created).append(",'").append(escape(v.languageCode)).append("',").append(v.categorie).append(",'").append(escape(v.title)).append("','").append(escape(v.channel)).append("','").append(escape(v.tags)).append("') "); + tostorebuffer.append("('").append(escape(v.id)).append("',").append(v.length).append(",").append(v.created).append(",'").append(escape(v.languageCode)).append("',").append(v.categorie).append(",'").append(escape(v.title)).append("','").append(escape(v.channel)).append("','").append(escape(v.tags)).append("'),"); } } } @@ -146,13 +149,13 @@ public class DB implements Runnable { if(tostorebuffer.length() > 10) { log.info("Write databuffer to DB video count: " + writebuffercurrentsize); dbsize += writebuffercurrentsize; - tostorebuffer.deleteCharAt(0);//delete leading ',' + tostorebuffer.deleteCharAt(tostorebuffer.length()-1);//delete trailing ',' String qu = "INSERT IGNORE INTO `videos`(`id`, `length`, `created`, `langcode`, `category`, `videotitle`, `channel`, `tags`) VALUES " + tostorebuffer.toString(); update(qu); //reset buffer writebuffercurrentsize = 0; - tostorebuffer = new StringBuilder(writebuffersize); + tostorebuffer = new StringBuilder(writebuffersize * STOREBUFFERRATIO); } } } @@ -284,18 +287,18 @@ public class DB implements Runnable { log.info("store Temp to buffer: " + strings.size()); writetempbuffercurrentsize += strings.size(); for(String s : strings) { - totempbuffer.append(", ('").append(escape(s)).append("')"); + totempbuffer.append("('").append(escape(s)).append("'),"); } } if(writetempbuffercurrentsize > writebuffersize || force) { log.info("Write Buffer: " + writetempbuffercurrentsize); - totempbuffer.deleteCharAt(0);//delete leading ',' + totempbuffer.deleteCharAt(totempbuffer.length()-1);//delete trailing ',' String qu = "INSERT IGNORE INTO `temp` (`ytid`) VALUES " + totempbuffer.toString() + ";"; update(qu); //reset writetempbuffercurrentsize = 0; - totempbuffer = new StringBuilder(writebuffersize); + totempbuffer = new StringBuilder(writebuffersize * TEMPBUFFERRATIO); } } @@ -308,7 +311,6 @@ public class DB implements Runnable { if(con != null) { if(!con.isClosed()) { addVideos(null, true); - con.commit(); con.close(); } }