From bb46e9a5e65c872277f91dda92ffc211e4a0eca6 Mon Sep 17 00:00:00 2001 From: MrBesen Date: Tue, 13 Nov 2018 14:49:04 +0100 Subject: [PATCH] speedup at inserting to DB --- src/de/mrbesen/youtubecrawler/Crawler.java | 14 ++++++-------- src/de/mrbesen/youtubecrawler/DB.java | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/de/mrbesen/youtubecrawler/Crawler.java b/src/de/mrbesen/youtubecrawler/Crawler.java index 2639726..174483d 100644 --- a/src/de/mrbesen/youtubecrawler/Crawler.java +++ b/src/de/mrbesen/youtubecrawler/Crawler.java @@ -70,7 +70,7 @@ public class Crawler implements Runnable { //if(! (toCrawl.contains(videoid) || toknown.contains(videoid))) //toknown.add(videoid); if(toCrawl.contains(videoid)) { - LinkedList str = new LinkedList(); + ArrayList str = new ArrayList(1); str.add(videoid); db.storeTemp(str); } @@ -255,18 +255,16 @@ public class Crawler implements Runnable { int count = 0; while(report[1].size() > 1) {//2 videos werden ggf. gelöscht ohne gesehen zu werden. - LinkedList store = null; + ArrayList store = null; try { if(report[1].size() <= 50) { - store = report[1]; + store = new ArrayList<>(report[1]); count += report[1].size(); report[1].clear(); } else { - store = new LinkedList<>(); - while(!report[1].isEmpty() && store.size() < 50) { - store.add(report[1].removeFirst()); - count++; - } + store = new ArrayList<>(report[1].subList(0, 50)); + report[1].removeAll(store); + count+=50; } } catch(NoSuchElementException ignored) {//concurrentmodification fuckery log.info("no suchelement bla"); diff --git a/src/de/mrbesen/youtubecrawler/DB.java b/src/de/mrbesen/youtubecrawler/DB.java index 471671c..0898181 100644 --- a/src/de/mrbesen/youtubecrawler/DB.java +++ b/src/de/mrbesen/youtubecrawler/DB.java @@ -271,7 +271,7 @@ public class DB implements Runnable { log.info("Delete Double done in " + ((System.currentTimeMillis() - start)/60000) + " min"); } - public void storeTemp(LinkedList strings) { + public void storeTemp(ArrayList strings) { if(strings == null) return; if(!strings.isEmpty()) {