Merge branch 'ait-qa' of https://github.com/internetarchive/heritrix3 …

…into 2021-07-18-ait-qa-merge
internetarchive · Aug 2, 2021 · 9ff04fc · 9ff04fc
2 parents 8fe92c3 + d2d8c49
commit 9ff04fc
Show file tree

Hide file tree

Showing 22 changed files with 375 additions and 2,072 deletions.
diff --git a/contrib/pom.xml b/contrib/pom.xml
@@ -84,9 +84,15 @@
 			<version>2.8.6</version>
 		</dependency>
 		<dependency>
+<<<<<<< HEAD
 			<groupId>org.java-websocket</groupId>
 			<artifactId>Java-WebSocket</artifactId>
 			<version>1.5.2</version>
+=======
+			<groupId>org.xerial</groupId>
+			<artifactId>sqlite-jdbc</artifactId>
+			<version>3.34.0</version>
+>>>>>>> d2d8c490bcca93ca96da952d904969dd424b2920
 		</dependency>
 	</dependencies>
 	<repositories>

diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java
@@ -33,6 +33,7 @@
 import java.net.URI;
 import java.nio.channels.Channels;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
@@ -113,6 +114,9 @@ public class ExtractorYoutubeDL extends Extractor
 
     protected static final int MAX_VIDEOS_PER_PAGE = 1000;
 
+    // for shouldExtract
+    protected HashMap<String, Boolean> seedsYDLd = new HashMap<String, Boolean>();
+
     protected transient Logger ydlLogger = null;
 
     // unnamed toethread-local temporary file

diff --git a/contrib/src/main/java/org/archive/modules/postprocessor/KafkaCrawlLogFeed.java b/contrib/src/main/java/org/archive/modules/postprocessor/KafkaCrawlLogFeed.java