forked from MrBesen/YoutubeCrawler
speedup at inserting to DB
This commit is contained in:
parent
5519ab072d
commit
bb46e9a5e6
|
@ -70,7 +70,7 @@ public class Crawler implements Runnable {
|
||||||
//if(! (toCrawl.contains(videoid) || toknown.contains(videoid)))
|
//if(! (toCrawl.contains(videoid) || toknown.contains(videoid)))
|
||||||
//toknown.add(videoid);
|
//toknown.add(videoid);
|
||||||
if(toCrawl.contains(videoid)) {
|
if(toCrawl.contains(videoid)) {
|
||||||
LinkedList<String> str = new LinkedList<String>();
|
ArrayList<String> str = new ArrayList<String>(1);
|
||||||
str.add(videoid);
|
str.add(videoid);
|
||||||
db.storeTemp(str);
|
db.storeTemp(str);
|
||||||
}
|
}
|
||||||
|
@ -255,18 +255,16 @@ public class Crawler implements Runnable {
|
||||||
|
|
||||||
int count = 0;
|
int count = 0;
|
||||||
while(report[1].size() > 1) {//2 videos werden ggf. gelöscht ohne gesehen zu werden.
|
while(report[1].size() > 1) {//2 videos werden ggf. gelöscht ohne gesehen zu werden.
|
||||||
LinkedList<String> store = null;
|
ArrayList<String> store = null;
|
||||||
try {
|
try {
|
||||||
if(report[1].size() <= 50) {
|
if(report[1].size() <= 50) {
|
||||||
store = report[1];
|
store = new ArrayList<>(report[1]);
|
||||||
count += report[1].size();
|
count += report[1].size();
|
||||||
report[1].clear();
|
report[1].clear();
|
||||||
} else {
|
} else {
|
||||||
store = new LinkedList<>();
|
store = new ArrayList<>(report[1].subList(0, 50));
|
||||||
while(!report[1].isEmpty() && store.size() < 50) {
|
report[1].removeAll(store);
|
||||||
store.add(report[1].removeFirst());
|
count+=50;
|
||||||
count++;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} catch(NoSuchElementException ignored) {//concurrentmodification fuckery
|
} catch(NoSuchElementException ignored) {//concurrentmodification fuckery
|
||||||
log.info("no suchelement bla");
|
log.info("no suchelement bla");
|
||||||
|
|
|
@ -271,7 +271,7 @@ public class DB implements Runnable {
|
||||||
log.info("Delete Double done in " + ((System.currentTimeMillis() - start)/60000) + " min");
|
log.info("Delete Double done in " + ((System.currentTimeMillis() - start)/60000) + " min");
|
||||||
}
|
}
|
||||||
|
|
||||||
public void storeTemp(LinkedList<String> strings) {
|
public void storeTemp(ArrayList<String> strings) {
|
||||||
if(strings == null)
|
if(strings == null)
|
||||||
return;
|
return;
|
||||||
if(!strings.isEmpty()) {
|
if(!strings.isEmpty()) {
|
||||||
|
|
Loading…
Reference in New Issue