MrBesen
/
YoutubeCrawler
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

utf8 in DB and youtubeapi parsing problem fixed

This commit is contained in:
mrbesen 2021-10-18 22:04:52 +02:00
parent bdcc276d58
commit a0b0ce6bfb
Signed by: MrBesen
GPG Key ID: 596B2350DCD67504
3 changed files with 5 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/de/mrbesen/youtubecrawler/DB.java
View File

@ -97,7 +97,7 @@ public class DB implements Runnable {
private void connect(boolean selectdb) {
try {
//verbinden
con = DriverManager.getConnection("jdbc:mysql://" + server + ":" + port + "/" + (selectdb ? db : "") + "?serverTimezone=UTC&verifyServerCertificate=false&useSSL=true", user, pw);
con = DriverManager.getConnection("jdbc:mysql://" + server + ":" + port + "/" + (selectdb ? db : "") + "?serverTimezone=UTC&verifyServerCertificate=false&useSSL=true&useUnicode=true&characterEncoding=utf-8", user, pw);
}catch (SQLException e) {
log.error("Error while connecting to the database! ", e);
}
@ -132,7 +132,6 @@ public class DB implements Runnable {
* @param input
*/
public void addVideos(ArrayList<Video> input, boolean force) {
//log.info("add " + input.size() + " videos");
if(input != null) {
if(input.size() > 0) {
writebuffercurrentsize += input.size();

6
src/main/de/mrbesen/youtubecrawler/YoutubeAPI.java
View File

@ -66,7 +66,7 @@ public class YoutubeAPI {
try {
int intid = Integer.parseInt(id);
out.put(intid, name);
System.out.println(intid + ";" + name);
// System.out.println(intid + ";" + name);
} catch (NumberFormatException e) {
e.printStackTrace();
}
@ -104,7 +104,7 @@ public class YoutubeAPI {
String title = snippet.getString("title"); //maxlen: 100
long published = getDate(snippet.getString("publishedAt"));
String channel = snippet.getString("channelTitle");
String tags = snippet.getJSONArray("tags").toList().stream().map(o -> (String) o).collect(Collectors.joining(", ")); // max len: ~500
String tags = snippet.getJSONArray("tags").toList().stream().map(o -> (String) o).collect(Collectors.joining(",")); // max len: ~500
byte category = 0;
try {
@ -115,7 +115,7 @@ public class YoutubeAPI {
int duration = (int) getDuration(contentDetails.getString("duration"));
boolean live = !snippet.getString("liveBroadcastContent").equalsIgnoreCase("none");
String langCode = snippet.getString("defaultLanguage");
String langCode = snippet.optString("defaultLanguage", snippet.optString("defaultAudioLanguage", ""));
if(langCode.length() > 3) {
langCode = langCode.substring(0, 3);
}

3
src/test/de/mrbesen/youtubecrawler/YoutubeAPITest.java
View File

@ -18,14 +18,13 @@ public class YoutubeAPITest {
Assertions.assertEquals(vid.id, "gsvKF2ojUzs");
Assertions.assertEquals(vid.title, "Mikebøi - Missed");
Assertions.assertEquals(vid.channel, "Trap Nation");
Assertions.assertEquals(vid.tags, "mikeboi missed, trap nation, trapnation, трап натион, electronic music 2020, Trap, Electronic Dance Music, missed trap nation, trap music, Electronic Music, Trap Music, Dance Music, missed mike boi, gaming music, Trap Music 2017, mike boy missed, mikebøi - missed, нас не догонят ремикс, Trap Nation, TrapNation, Mikebøi - Missed, trap nation 2020, trap music 2020 remix, EDM, missed mikeboi, music");
Assertions.assertEquals(vid.tags, "mikeboi missed,trap nation,trapnation,трап натион,electronic music 2020,Trap,Electronic Dance Music,missed trap nation,trap music,Electronic Music,Trap Music,Dance Music,missed mike boi,gaming music,Trap Music 2017,mike boy missed,mikebøi - missed,нас не догонят ремикс,Trap Nation,TrapNation,Mikebøi - Missed,trap nation 2020,trap music 2020 remix,EDM,missed mikeboi,music");
Assertions.assertEquals(vid.length, 213);
Assertions.assertEquals(vid.languageCode, "en");
Assertions.assertEquals(vid.categorie, 10);
Assertions.assertEquals(vid.created, 1491571496);
Assertions.assertFalse(vid.live);
Map<Integer, String> obj = api.getCategories();
Assertions.assertNotNull(obj);