forked from MrBesen/YoutubeCrawler
gitignore and cap checkvideos request
This commit is contained in:
parent
9221b03120
commit
73643625f3
|
@ -0,0 +1,5 @@
|
||||||
|
/bin/
|
||||||
|
/target/
|
||||||
|
.project
|
||||||
|
.settings/*
|
||||||
|
.classpath
|
|
@ -25,6 +25,7 @@ public class Crawler implements Runnable {
|
||||||
private YoutubeAPI api = new YoutubeAPI();
|
private YoutubeAPI api = new YoutubeAPI();
|
||||||
private File crawlfile = new File("crawl.txt");
|
private File crawlfile = new File("crawl.txt");
|
||||||
private Logger log = Logger.getLogger(Crawler.class.getName());
|
private Logger log = Logger.getLogger(Crawler.class.getName());
|
||||||
|
private static int maxvideostotest = 100;
|
||||||
|
|
||||||
public void stop() {
|
public void stop() {
|
||||||
crawl = false;
|
crawl = false;
|
||||||
|
@ -96,8 +97,12 @@ public class Crawler implements Runnable {
|
||||||
crawl = false;
|
crawl = false;
|
||||||
//delete / clear crawl file
|
//delete / clear crawl file
|
||||||
} else {
|
} else {
|
||||||
toCrawl.addAll(db.checkvideos(toknown));//TODO cap it at something like 1000 videos
|
LinkedList<String> tocheck = new LinkedList<>();
|
||||||
toknown.clear();
|
for(int i = 0; i < toknown.size() && i < maxvideostotest; i++) {
|
||||||
|
tocheck.add(toknown.removeFirst());
|
||||||
|
}
|
||||||
|
toCrawl.addAll(db.checkvideos(tocheck));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
PrintWriter p = new PrintWriter(new BufferedWriter(new FileWriter(crawlfile)));
|
PrintWriter p = new PrintWriter(new BufferedWriter(new FileWriter(crawlfile)));
|
||||||
for(String t : toCrawl) {
|
for(String t : toCrawl) {
|
||||||
|
|
Loading…
Reference in New Issue