bugfix to also read WET files with WarcReader

huggingface · Dec 12, 2023 · 6014a6f · 6014a6f
1 parent 476de37
commit 6014a6f
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/datatrove/pipeline/readers/warc.py b/src/datatrove/pipeline/readers/warc.py
@@ -41,7 +41,7 @@ def read_file(self, datafile: BaseInputDataFile):
 
 def process_record(record: ArcWarcRecord) -> dict | None:
     # record type
-    if record.rec_type != "response":
+    if record.rec_type != "response" and record.rec_type != "conversion":  # wet files have "conversion" type
         return
 
     # content type filtering