From c4eaa4fa6ee9556b47956f6db5e7d15b685b43c2 Mon Sep 17 00:00:00 2001
From: mrbesen <y.g.2@gmx.de>
Date: Fri, 12 Oct 2018 20:03:53 +0200
Subject: [PATCH] more performance improvements

---
 src/de/mrbesen/youtubecrawler/Crawler.java    |  9 ++---
 src/de/mrbesen/youtubecrawler/DB.java         | 34 +++++++++----------
 src/de/mrbesen/youtubecrawler/YoutubeAPI.java |  4 +--
 3 files changed, 21 insertions(+), 26 deletions(-)

diff --git a/src/de/mrbesen/youtubecrawler/Crawler.java b/src/de/mrbesen/youtubecrawler/Crawler.java
index 5346ac8..4336371 100644
--- a/src/de/mrbesen/youtubecrawler/Crawler.java
+++ b/src/de/mrbesen/youtubecrawler/Crawler.java
@@ -237,11 +237,6 @@ public class Crawler implements Runnable {
 					for(String t : toCrawl) {
 						p.println(t);
 					}
-					/*
-					p.println("-");
-					for(String t : toknown) {
-						p.println(t);
-					}*/
 					p.close();
 				} catch (IOException e) {
 					log.error("Error writing crawlfile.", e);
@@ -278,7 +273,7 @@ public class Crawler implements Runnable {
 						}
 						db.storeTemp(store);
 					}
-					log.info(count + " videos added.");
+					log.info(count + " videos added from " + threadname);
 					profiler.endSection();
 				}
 
@@ -302,7 +297,7 @@ public class Crawler implements Runnable {
 						ArrayList<Video> videos = (ArrayList<Video>) api.getInfos(videoids)[0];
 						profiler.endStartSection("sendtoDB");
 						db.addVideos(videos, false);
-						profiler.endSection();
+						profiler.endSection();//sendtoDB
 					}
 				}
 				profiler.endSection();//save2DB
diff --git a/src/de/mrbesen/youtubecrawler/DB.java b/src/de/mrbesen/youtubecrawler/DB.java
index c8d2d4f..1a71606 100644
--- a/src/de/mrbesen/youtubecrawler/DB.java
+++ b/src/de/mrbesen/youtubecrawler/DB.java
@@ -26,8 +26,9 @@ public class DB implements Runnable {
 	private Server serv = new Server(this);
 	private Thread randomrefill = null;
 	private int dbsize = 0;
-	private ArrayList<Video> tostorebuffer;
+	private StringBuilder tostorebuffer ;
 	private int writebuffersize = 500;
+	private int writebuffercurrentsize = 0;
 
 	public DB() {
 		try {
@@ -67,7 +68,7 @@ public class DB implements Runnable {
 			} catch(NumberFormatException e) {
 				log.warn("could not read the number \"" + Config.prop.getProperty("db.writebuffersize") + "\" from the config file. db.writebuffersize");
 			}
-			tostorebuffer = new ArrayList<>(writebuffersize);
+			tostorebuffer = new StringBuilder(writebuffersize);
 		} catch (SQLException e) {
 			log.error("Error while connecting to the database! ", e);
 		}
@@ -128,28 +129,27 @@ public class DB implements Runnable {
 	 * save the list of videos to the DB
 	 * @param input
 	 */
-	public void addVideos(List<Video> input, boolean force) {
+	public void addVideos(ArrayList<Video> input, boolean force) {
 		//log.info("add " + input.size() + " videos");
 		if(input != null) {
 			if(input.size() > 0) {
-				tostorebuffer.addAll(input);
+				for(int i = 0; i < input.size(); i++) {
+					Video v = input.get(i);
+					if(v != null)
+						tostorebuffer.append(",('").append(v.id).append("','").append(v.length).append("','").append(v.created).append("','").append(v.languageCode).append("','").append(v.categorie).append("','").append(v.title).append("','").append(v.channel).append("','").append(v.tags).append("') ");
+				}
 			}
 		}
-		if(tostorebuffer.size() > writebuffersize || force) {
-			dbsize += tostorebuffer.size();
-			StringBuilder sb = new StringBuilder();
-			for(int i = 0; i < tostorebuffer.size(); i++) {
-				if(i > 0)
-					sb.append(',');
-				Video v = tostorebuffer.get(i);
-				if(v != null)
-					sb.append("('").append(v.id).append("','").append(v.length).append("','").append(v.created).append("','").append(v.languageCode).append("','").append(v.categorie).append("','").append(v.title).append("','").append(v.channel).append("','").append(v.tags).append("') ");
-			}
-			tostorebuffer.clear();
-			if(sb.length() > 2) {
-				String qu = "INSERT IGNORE INTO `ytcrawler`.`videos`(`id`, `length`, `created`, `langcode`, `category`, `videotitle`, `channel`, `tags`) VALUES " + sb.toString();
+		if(writebuffercurrentsize > writebuffersize || force) {
+			if(tostorebuffer.length() > 10) {
+				dbsize += writebuffercurrentsize;
+				tostorebuffer.deleteCharAt(0);//delete leading ','
+				String qu = "INSERT IGNORE INTO `ytcrawler`.`videos`(`id`, `length`, `created`, `langcode`, `category`, `videotitle`, `channel`, `tags`) VALUES " + tostorebuffer.toString();
 				update(qu);
 			}
+			//reset buffer
+			writebuffercurrentsize = 0;
+			tostorebuffer = new StringBuilder(writebuffersize);
 		}
 	}
 
diff --git a/src/de/mrbesen/youtubecrawler/YoutubeAPI.java b/src/de/mrbesen/youtubecrawler/YoutubeAPI.java
index c782ca0..2ac0bff 100644
--- a/src/de/mrbesen/youtubecrawler/YoutubeAPI.java
+++ b/src/de/mrbesen/youtubecrawler/YoutubeAPI.java
@@ -37,7 +37,7 @@ public class YoutubeAPI {
 	}
 
 	public List<Video>[] getInfos(List<String> ids) {
-		log.info("get " + ids.size() + " infos");
+		//log.info("get " + ids.size() + " infos");
 		if(ids.isEmpty())
 			return null;
 
@@ -138,7 +138,7 @@ public class YoutubeAPI {
 				}
 			}
 		} while(!nextpage.equals(""));
-		log.info("got " + (out.size() + livestr.size()) + " infos");
+		//log.info("got " + (out.size() + livestr.size()) + " infos");
 		return new List[] {out, livestr};
 	}