162 lines
5.5 KiB
Java
162 lines
5.5 KiB
Java
package de.mrbesen.youtubecrawler;
|
|
|
|
import java.io.BufferedReader;
|
|
import java.io.IOException;
|
|
import java.io.InputStreamReader;
|
|
import java.net.URL;
|
|
import java.text.DateFormat;
|
|
import java.text.ParseException;
|
|
import java.text.SimpleDateFormat;
|
|
import java.util.ArrayList;
|
|
import java.util.Date;
|
|
import java.util.LinkedList;
|
|
import java.util.List;
|
|
|
|
import javax.net.ssl.HttpsURLConnection;
|
|
|
|
import org.apache.log4j.Logger;
|
|
|
|
import de.mrbesen.youtubecrawler.Crawler.Video;
|
|
|
|
public class YoutubeAPI {
|
|
|
|
private final String api_key = Config.prop.getProperty("youtube.apikey");
|
|
private static String basequery = "https://www.googleapis.com/youtube/v3/videos?part=snippet,contentDetails&id=";
|
|
private static DateFormat dateformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
|
private Logger log = Logger.getLogger(YoutubeAPI.class.getName());
|
|
|
|
public YoutubeAPI() {
|
|
if(api_key.isEmpty()) {
|
|
log.error("apikey is not defined!");
|
|
System.exit(1);
|
|
}
|
|
}
|
|
|
|
public Video getInfo(String id) {
|
|
return (Video) getInfos(id)[0].get(0);
|
|
}
|
|
|
|
public List<Video>[] getInfos(List<String> ids) {
|
|
//log.info("get " + ids.size() + " infos");
|
|
if(ids.isEmpty())
|
|
return null;
|
|
|
|
StringBuilder sb = new StringBuilder(ids.remove(0));
|
|
while(!ids.isEmpty()) {
|
|
sb.append(',').append(ids.remove(0));
|
|
}
|
|
return getInfos(sb.toString());
|
|
}
|
|
|
|
public List<Video>[] getInfos(String idlist) {
|
|
ArrayList<Video> out = new ArrayList<Video>(idlist.length() / 12);//approximierte vorraussichtliche länge
|
|
LinkedList<Video> livestr = new LinkedList<Video>();
|
|
String nextpage = "";
|
|
do {
|
|
String query = basequery + idlist + "&key=" + api_key;
|
|
BufferedReader br = connect(query);
|
|
nextpage = "";
|
|
if(br != null) {
|
|
try {
|
|
String line;
|
|
Video v = null;
|
|
boolean tags = false;
|
|
while((line = br.readLine()) != null) {
|
|
String split[] = line.split(":",2);
|
|
if(split.length == 2) {
|
|
split[0] = removeunwanted(split[0]);
|
|
|
|
//System.out.println(split[0] + " " + split[1]);
|
|
if(split[0].equals("defaultAudioLanguage")) {
|
|
v.languageCode = removeunwanted(split[1]);
|
|
} else if(split[0].equals("defaultLanguage")) {
|
|
v.languageCode = removeunwanted(split[1]);
|
|
} else if(split[0].equals("title")) {
|
|
if(v.title.isEmpty())
|
|
v.title = removeunwanted(split[1]);
|
|
} else if(split[0].equals("channelTitle")) {
|
|
v.channel = removeunwanted(split[1]);
|
|
} else if(split[0].equals("defaultLanguage")) {
|
|
v.languageCode = removeunwanted(split[1]);
|
|
} else if(split[0].equals("tags")) {
|
|
tags = true;
|
|
} else if(split[0].equals("liveBroadcastContent")) {
|
|
v.live = !removeunwanted(split[1]).equalsIgnoreCase("none");
|
|
} else if(split[0].equals("id")) {
|
|
if(v != null) {
|
|
if(!v.live)
|
|
out.add(v);
|
|
else {
|
|
livestr.add(v);
|
|
log.info("livestream found! " + v.id + " " + v.channel);
|
|
}
|
|
}
|
|
v = new Video();
|
|
v.id = removeunwanted(split[1]);
|
|
//System.out.println("new video: " + v.id + " " + v.length + " " + v.languageCode);
|
|
} else if(split[0].equals("categoryId")) {
|
|
v.categorie = Byte.parseByte(removeunwanted(split[1]));
|
|
} else if(split[0].equals("duration")) {
|
|
String timeparts[] = removeunwanted(split[1]).substring(2).split("[H,M,S]");
|
|
try {
|
|
if(timeparts.length > 2) {//hours
|
|
v.length = 3600 * Integer.parseInt(timeparts[0]);
|
|
}
|
|
if(timeparts.length > 1) {//minutes
|
|
v.length += 60 * Integer.parseInt(timeparts[timeparts.length -2]);
|
|
}
|
|
//Seconds
|
|
v.length += Integer.parseInt(timeparts[timeparts.length-1]);
|
|
} catch(NumberFormatException e) {//failed: P6DT17H59M53S and P15W3DT4H1M11S and P1W2DT20H47M55S video id: 1NPyC0psMaI and P2W2DT23H58M58S video id: Jd9KjbRxhN4 For input string: "W2DT23"
|
|
Main.getMain().broadcastAdmin(removeunwanted(split[1]) + " video id: " + v.id);
|
|
log.warn("Error saving the time string: " + removeunwanted(split[1]) + " video id: " + v.id, e);
|
|
}
|
|
} else if(split[0].equals("publishedAt")) {
|
|
String tmp = removeunwanted(split[1]);
|
|
tmp = tmp.replace('T', ' ');
|
|
tmp = tmp.substring(0, 19);
|
|
Date d = dateformat.parse(tmp);
|
|
v.created = d.getTime() / 1000;
|
|
}else if(split[0].equals("nextPageToken")) {
|
|
nextpage = "&pageToken=" + removeunwanted(split[1]);
|
|
// System.out.println("nextpage set to " + nextpage);
|
|
}
|
|
} else {
|
|
if(line.contains("]")) {
|
|
if(v.tags.length() > 1)
|
|
v.tags = v.tags.substring(1);
|
|
tags = false;
|
|
} else if(tags) {
|
|
v.tags += ", " + removeunwanted(line);
|
|
}
|
|
}
|
|
}
|
|
out.add(v);//add the last video
|
|
br.close();
|
|
} catch(IOException | ParseException e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
} while(!nextpage.equals(""));
|
|
//log.info("got " + (out.size() + livestr.size()) + " infos");
|
|
return new List[] {out, livestr};
|
|
}
|
|
|
|
private String removeunwanted(String in) {
|
|
return in.replaceAll("[\"}{\\,\\\\]", "").replaceAll("'", "").trim();
|
|
}
|
|
|
|
public BufferedReader connect(String url) {
|
|
try {
|
|
URL urll = new URL(url);
|
|
HttpsURLConnection con = (HttpsURLConnection) urll.openConnection();
|
|
con.connect();
|
|
//System.out.println(con.getResponseCode());
|
|
return new BufferedReader(new InputStreamReader(con.getInputStream()));
|
|
} catch(IOException e) {
|
|
e.printStackTrace();
|
|
}
|
|
return null;
|
|
}
|
|
}
|
|
// no suchelement bla |