2018-07-15 21:30:12 +02:00
package de.mrbesen.youtubecrawler ;
import java.io.BufferedReader ;
import java.io.IOException ;
import java.io.InputStreamReader ;
import java.net.URL ;
import java.text.DateFormat ;
import java.text.ParseException ;
import java.text.SimpleDateFormat ;
2018-10-12 17:22:36 +02:00
import java.util.ArrayList ;
2018-07-15 21:30:12 +02:00
import java.util.Date ;
import java.util.LinkedList ;
import java.util.List ;
import javax.net.ssl.HttpsURLConnection ;
import org.apache.log4j.Logger ;
import de.mrbesen.youtubecrawler.Crawler.Video ;
public class YoutubeAPI {
private final String api_key = Config . prop . getProperty ( " youtube.apikey " ) ;
private static String basequery = " https://www.googleapis.com/youtube/v3/videos?part=snippet,contentDetails&id= " ;
private static DateFormat dateformat = new SimpleDateFormat ( " yyyy-MM-dd HH:mm:ss " ) ;
2018-07-15 22:09:37 +02:00
private Logger log = Logger . getLogger ( YoutubeAPI . class . getName ( ) ) ;
2018-07-15 21:30:12 +02:00
public YoutubeAPI ( ) {
if ( api_key . isEmpty ( ) ) {
log . error ( " apikey is not defined! " ) ;
System . exit ( 1 ) ;
}
}
public Video getInfo ( String id ) {
2018-07-20 19:57:43 +02:00
return ( Video ) getInfos ( id ) [ 0 ] . get ( 0 ) ;
2018-07-15 21:30:12 +02:00
}
2018-10-12 17:22:36 +02:00
public List < Video > [ ] getInfos ( List < String > ids ) {
2018-10-12 20:03:53 +02:00
//log.info("get " + ids.size() + " infos");
2018-07-15 21:30:12 +02:00
if ( ids . isEmpty ( ) )
2018-07-20 19:57:43 +02:00
return null ;
2018-07-15 21:30:12 +02:00
StringBuilder sb = new StringBuilder ( ids . remove ( 0 ) ) ;
while ( ! ids . isEmpty ( ) ) {
sb . append ( ',' ) . append ( ids . remove ( 0 ) ) ;
}
return getInfos ( sb . toString ( ) ) ;
}
2018-10-12 17:22:36 +02:00
public List < Video > [ ] getInfos ( String idlist ) {
ArrayList < Video > out = new ArrayList < Video > ( idlist . length ( ) / 12 ) ; //approximierte vorraussichtliche länge
2018-07-20 19:57:43 +02:00
LinkedList < Video > livestr = new LinkedList < Video > ( ) ;
2018-07-15 21:30:12 +02:00
String nextpage = " " ;
do {
String query = basequery + idlist + " &key= " + api_key ;
BufferedReader br = connect ( query ) ;
nextpage = " " ;
if ( br ! = null ) {
try {
String line ;
Video v = null ;
2018-07-20 19:57:43 +02:00
boolean tags = false ;
2018-07-15 21:30:12 +02:00
while ( ( line = br . readLine ( ) ) ! = null ) {
String split [ ] = line . split ( " : " , 2 ) ;
if ( split . length = = 2 ) {
split [ 0 ] = removeunwanted ( split [ 0 ] ) ;
//System.out.println(split[0] + " " + split[1]);
if ( split [ 0 ] . equals ( " defaultAudioLanguage " ) ) {
v . languageCode = removeunwanted ( split [ 1 ] ) ;
} else if ( split [ 0 ] . equals ( " defaultLanguage " ) ) {
v . languageCode = removeunwanted ( split [ 1 ] ) ;
2018-07-20 19:57:43 +02:00
} else if ( split [ 0 ] . equals ( " title " ) ) {
if ( v . title . isEmpty ( ) )
v . title = removeunwanted ( split [ 1 ] ) ;
} else if ( split [ 0 ] . equals ( " channelTitle " ) ) {
v . channel = removeunwanted ( split [ 1 ] ) ;
} else if ( split [ 0 ] . equals ( " defaultLanguage " ) ) {
v . languageCode = removeunwanted ( split [ 1 ] ) ;
} else if ( split [ 0 ] . equals ( " tags " ) ) {
tags = true ;
} else if ( split [ 0 ] . equals ( " liveBroadcastContent " ) ) {
v . live = ! removeunwanted ( split [ 1 ] ) . equalsIgnoreCase ( " none " ) ;
2018-07-15 21:30:12 +02:00
} else if ( split [ 0 ] . equals ( " id " ) ) {
2018-07-20 19:57:43 +02:00
if ( v ! = null ) {
if ( ! v . live )
out . add ( v ) ;
else {
livestr . add ( v ) ;
log . info ( " livestream found! " + v . id + " " + v . channel ) ;
}
}
2018-07-15 21:30:12 +02:00
v = new Video ( ) ;
v . id = removeunwanted ( split [ 1 ] ) ;
//System.out.println("new video: " + v.id + " " + v.length + " " + v.languageCode);
} else if ( split [ 0 ] . equals ( " categoryId " ) ) {
v . categorie = Byte . parseByte ( removeunwanted ( split [ 1 ] ) ) ;
} else if ( split [ 0 ] . equals ( " duration " ) ) {
String timeparts [ ] = removeunwanted ( split [ 1 ] ) . substring ( 2 ) . split ( " [H,M,S] " ) ;
try {
if ( timeparts . length > 2 ) { //hours
v . length = 3600 * Integer . parseInt ( timeparts [ 0 ] ) ;
}
if ( timeparts . length > 1 ) { //minutes
v . length + = 60 * Integer . parseInt ( timeparts [ timeparts . length - 2 ] ) ;
}
//Seconds
v . length + = Integer . parseInt ( timeparts [ timeparts . length - 1 ] ) ;
2018-07-23 12:27:51 +02:00
} catch ( NumberFormatException e ) { //failed: P6DT17H59M53S and P15W3DT4H1M11S and P1W2DT20H47M55S video id: 1NPyC0psMaI and P2W2DT23H58M58S video id: Jd9KjbRxhN4 For input string: "W2DT23"
2018-10-11 15:32:19 +02:00
Main . getMain ( ) . broadcastAdmin ( removeunwanted ( split [ 1 ] ) + " video id: " + v . id ) ;
2018-07-20 19:57:43 +02:00
log . warn ( " Error saving the time string: " + removeunwanted ( split [ 1 ] ) + " video id: " + v . id , e ) ;
2018-07-15 21:30:12 +02:00
}
} else if ( split [ 0 ] . equals ( " publishedAt " ) ) {
String tmp = removeunwanted ( split [ 1 ] ) ;
tmp = tmp . replace ( 'T' , ' ' ) ;
tmp = tmp . substring ( 0 , tmp . length ( ) - 5 ) ;
Date d = dateformat . parse ( tmp ) ;
2018-07-20 19:57:43 +02:00
v . created = d . getTime ( ) / 1000 ;
2018-07-15 21:30:12 +02:00
} else if ( split [ 0 ] . equals ( " nextPageToken " ) ) {
nextpage = " &pageToken= " + removeunwanted ( split [ 1 ] ) ;
// System.out.println("nextpage set to " + nextpage);
}
2018-07-20 19:57:43 +02:00
} else {
if ( line . contains ( " ] " ) ) {
if ( v . tags . length ( ) > 1 )
v . tags = v . tags . substring ( 1 ) ;
tags = false ;
} else if ( tags ) {
v . tags + = " , " + removeunwanted ( line ) ;
}
2018-07-15 21:30:12 +02:00
}
}
out . add ( v ) ; //add the last video
br . close ( ) ;
} catch ( IOException | ParseException e ) {
e . printStackTrace ( ) ;
}
}
} while ( ! nextpage . equals ( " " ) ) ;
2018-10-12 20:03:53 +02:00
//log.info("got " + (out.size() + livestr.size()) + " infos");
2018-10-12 17:22:36 +02:00
return new List [ ] { out , livestr } ;
2018-07-15 21:30:12 +02:00
}
private String removeunwanted ( String in ) {
2018-07-20 19:57:43 +02:00
return in . replaceAll ( " [ \" }{ \\ , \\ \\ ] " , " " ) . replaceAll ( " ' " , " " ) . trim ( ) ;
2018-07-15 21:30:12 +02:00
}
public BufferedReader connect ( String url ) {
try {
URL urll = new URL ( url ) ;
HttpsURLConnection con = ( HttpsURLConnection ) urll . openConnection ( ) ;
con . connect ( ) ;
//System.out.println(con.getResponseCode());
return new BufferedReader ( new InputStreamReader ( con . getInputStream ( ) ) ) ;
} catch ( IOException e ) {
e . printStackTrace ( ) ;
}
return null ;
}
2018-07-20 19:57:43 +02:00
}
2018-09-11 12:12:51 +02:00
// no suchelement bla