From a0b0ce6bfb3120589d0d34d2f3e503bb8ab55bc9 Mon Sep 17 00:00:00 2001
From: mrbesen <y.g.2@gmx.de>
Date: Mon, 18 Oct 2021 22:04:52 +0200
Subject: [PATCH] utf8 in DB and youtubeapi parsing problem fixed

---
 src/main/de/mrbesen/youtubecrawler/DB.java             | 3 +--
 src/main/de/mrbesen/youtubecrawler/YoutubeAPI.java     | 6 +++---
 src/test/de/mrbesen/youtubecrawler/YoutubeAPITest.java | 3 +--
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/main/de/mrbesen/youtubecrawler/DB.java b/src/main/de/mrbesen/youtubecrawler/DB.java
index 1dd7965..741838d 100644
--- a/src/main/de/mrbesen/youtubecrawler/DB.java
+++ b/src/main/de/mrbesen/youtubecrawler/DB.java
@@ -97,7 +97,7 @@ public class DB implements Runnable {
 	private void connect(boolean selectdb) {
 		try {
 			//verbinden
-			con = DriverManager.getConnection("jdbc:mysql://" + server + ":" + port + "/" + (selectdb ? db : "") + "?serverTimezone=UTC&verifyServerCertificate=false&useSSL=true", user, pw);
+			con = DriverManager.getConnection("jdbc:mysql://" + server + ":" + port + "/" + (selectdb ? db : "") + "?serverTimezone=UTC&verifyServerCertificate=false&useSSL=true&useUnicode=true&characterEncoding=utf-8", user, pw);
 		}catch (SQLException e) {
 			log.error("Error while connecting to the database! ", e);
 		}
@@ -132,7 +132,6 @@ public class DB implements Runnable {
 	 * @param input
 	 */
 	public void addVideos(ArrayList<Video> input, boolean force) {
-		//log.info("add " + input.size() + " videos");
 		if(input != null) {
 			if(input.size() > 0) {
 				writebuffercurrentsize += input.size();
diff --git a/src/main/de/mrbesen/youtubecrawler/YoutubeAPI.java b/src/main/de/mrbesen/youtubecrawler/YoutubeAPI.java
index dd23076..7cce399 100644
--- a/src/main/de/mrbesen/youtubecrawler/YoutubeAPI.java
+++ b/src/main/de/mrbesen/youtubecrawler/YoutubeAPI.java
@@ -66,7 +66,7 @@ public class YoutubeAPI {
 				try {
 					int intid = Integer.parseInt(id);
 					out.put(intid, name);
-					System.out.println(intid + ";" + name);
+					// System.out.println(intid + ";" + name);
 				} catch (NumberFormatException e) {
 					e.printStackTrace();
 				}
@@ -104,7 +104,7 @@ public class YoutubeAPI {
 		String title = snippet.getString("title"); //maxlen: 100
 		long published = getDate(snippet.getString("publishedAt"));
 		String channel = snippet.getString("channelTitle");
-		String tags = snippet.getJSONArray("tags").toList().stream().map(o -> (String) o).collect(Collectors.joining(", ")); // max len: ~500
+		String tags = snippet.getJSONArray("tags").toList().stream().map(o -> (String) o).collect(Collectors.joining(",")); // max len: ~500
 
 		byte category = 0;
 		try {
@@ -115,7 +115,7 @@ public class YoutubeAPI {
 		int duration = (int) getDuration(contentDetails.getString("duration"));
 		boolean live = !snippet.getString("liveBroadcastContent").equalsIgnoreCase("none");
 
-		String langCode = snippet.getString("defaultLanguage");
+		String langCode = snippet.optString("defaultLanguage", snippet.optString("defaultAudioLanguage", ""));
 		if(langCode.length() > 3) {
 			langCode = langCode.substring(0, 3);
 		}
diff --git a/src/test/de/mrbesen/youtubecrawler/YoutubeAPITest.java b/src/test/de/mrbesen/youtubecrawler/YoutubeAPITest.java
index dcb6939..47bb00b 100644
--- a/src/test/de/mrbesen/youtubecrawler/YoutubeAPITest.java
+++ b/src/test/de/mrbesen/youtubecrawler/YoutubeAPITest.java
@@ -18,14 +18,13 @@ public class YoutubeAPITest {
 		Assertions.assertEquals(vid.id, "gsvKF2ojUzs");
 		Assertions.assertEquals(vid.title, "Mikebøi - Missed");
 		Assertions.assertEquals(vid.channel, "Trap Nation");
-		Assertions.assertEquals(vid.tags, "mikeboi missed, trap nation, trapnation, трап натион, electronic music 2020, Trap, Electronic Dance Music, missed trap nation, trap music, Electronic Music, Trap Music, Dance Music, missed mike boi, gaming music, Trap Music 2017, mike boy missed, mikebøi - missed, нас не догонят ремикс, Trap Nation, TrapNation, Mikebøi - Missed, trap nation 2020, trap music 2020 remix, EDM, missed mikeboi, music");
+		Assertions.assertEquals(vid.tags, "mikeboi missed,trap nation,trapnation,трап натион,electronic music 2020,Trap,Electronic Dance Music,missed trap nation,trap music,Electronic Music,Trap Music,Dance Music,missed mike boi,gaming music,Trap Music 2017,mike boy missed,mikebøi - missed,нас не догонят ремикс,Trap Nation,TrapNation,Mikebøi - Missed,trap nation 2020,trap music 2020 remix,EDM,missed mikeboi,music");
 		Assertions.assertEquals(vid.length, 213);
 		Assertions.assertEquals(vid.languageCode, "en");
 		Assertions.assertEquals(vid.categorie, 10);
 		Assertions.assertEquals(vid.created, 1491571496);
 		Assertions.assertFalse(vid.live);
 
-
 		Map<Integer, String> obj = api.getCategories();
 
 		Assertions.assertNotNull(obj);