forked from MrBesen/YoutubeCrawler
gitignore and cap checkvideos request
This commit is contained in:
parent
9221b03120
commit
73643625f3
|
@ -0,0 +1,5 @@
|
|||
/bin/
|
||||
/target/
|
||||
.project
|
||||
.settings/*
|
||||
.classpath
|
|
@ -25,6 +25,7 @@ public class Crawler implements Runnable {
|
|||
private YoutubeAPI api = new YoutubeAPI();
|
||||
private File crawlfile = new File("crawl.txt");
|
||||
private Logger log = Logger.getLogger(Crawler.class.getName());
|
||||
private static int maxvideostotest = 100;
|
||||
|
||||
public void stop() {
|
||||
crawl = false;
|
||||
|
@ -96,14 +97,18 @@ public class Crawler implements Runnable {
|
|||
crawl = false;
|
||||
//delete / clear crawl file
|
||||
} else {
|
||||
toCrawl.addAll(db.checkvideos(toknown));//TODO cap it at something like 1000 videos
|
||||
toknown.clear();
|
||||
try {
|
||||
PrintWriter p = new PrintWriter(new BufferedWriter(new FileWriter(crawlfile)));
|
||||
for(String t : toCrawl) {
|
||||
p.println(t);
|
||||
LinkedList<String> tocheck = new LinkedList<>();
|
||||
for(int i = 0; i < toknown.size() && i < maxvideostotest; i++) {
|
||||
tocheck.add(toknown.removeFirst());
|
||||
}
|
||||
p.close();
|
||||
toCrawl.addAll(db.checkvideos(tocheck));
|
||||
|
||||
try {
|
||||
PrintWriter p = new PrintWriter(new BufferedWriter(new FileWriter(crawlfile)));
|
||||
for(String t : toCrawl) {
|
||||
p.println(t);
|
||||
}
|
||||
p.close();
|
||||
} catch (IOException e) {
|
||||
log.error("Error writing crawlfile.", e);
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue