changed trafilatura default args

huggingface · Dec 28, 2024 · f816913 · f816913
1 parent aae7e33
commit f816913
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/src/datatrove/pipeline/extractors/trafilatura.py b/src/datatrove/pipeline/extractors/trafilatura.py
@@ -22,13 +22,15 @@ class Trafilatura(BaseExtractor):
     def __init__(
         self,
         favour_precision: bool = True,
+        include_comments: bool = False,
         include_images: bool = False,
         timeout: float = 0.1,
         deduplicate: bool = True,
         **kwargs,
     ):
         super().__init__(timeout)
         self.favour_precision = favour_precision
+        self.include_comments = include_comments
         self.include_images = include_images
         self.deduplicate = deduplicate
         self.kwargs = kwargs
@@ -65,7 +67,7 @@ def extract(self, text: str) -> str:
         return extract(
             text,
             favor_precision=self.favour_precision,
-            include_comments=False,
+            include_comments=self.include_comments,
             deduplicate=self.deduplicate,
             **self.kwargs,
         )