gitignore and cap checkvideos request

This commit is contained in:
mrbesen 2018-07-16 20:12:20 +02:00
parent 9221b03120
commit 73643625f3
2 changed files with 17 additions and 7 deletions

5
.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
/bin/
/target/
.project
.settings/*
.classpath

View File

@ -25,6 +25,7 @@ public class Crawler implements Runnable {
private YoutubeAPI api = new YoutubeAPI();
private File crawlfile = new File("crawl.txt");
private Logger log = Logger.getLogger(Crawler.class.getName());
private static int maxvideostotest = 100;
public void stop() {
crawl = false;
@ -96,14 +97,18 @@ public class Crawler implements Runnable {
crawl = false;
//delete / clear crawl file
} else {
toCrawl.addAll(db.checkvideos(toknown));//TODO cap it at something like 1000 videos
toknown.clear();
try {
PrintWriter p = new PrintWriter(new BufferedWriter(new FileWriter(crawlfile)));
for(String t : toCrawl) {
p.println(t);
LinkedList<String> tocheck = new LinkedList<>();
for(int i = 0; i < toknown.size() && i < maxvideostotest; i++) {
tocheck.add(toknown.removeFirst());
}
p.close();
toCrawl.addAll(db.checkvideos(tocheck));
try {
PrintWriter p = new PrintWriter(new BufferedWriter(new FileWriter(crawlfile)));
for(String t : toCrawl) {
p.println(t);
}
p.close();
} catch (IOException e) {
log.error("Error writing crawlfile.", e);
}