improved test case robustness

huggingface · Dec 22, 2024 · cd18c59 · cd18c59
1 parent 56a71ed
commit cd18c59
Show file tree

Hide file tree

Showing 3 changed files with 171 additions and 23 deletions.
diff --git a/src/datatrove/pipeline/extractors/__init__.py b/src/datatrove/pipeline/extractors/__init__.py
@@ -1,6 +1,5 @@
 from .inscriptis import Inscriptis
 from .justext import Justext
-from .modular import ReadabilityInscriptis
 from .readabilipy import ReadabiliPy
 from .readability import Readability
 from .resiliparse import Resiliparse

diff --git a/src/datatrove/pipeline/extractors/inscriptis.py b/src/datatrove/pipeline/extractors/inscriptis.py
@@ -39,6 +39,9 @@ def __init__(
         self.kwargs = kwargs
         self.regex_excessive_lines = re.compile(r"(" + self.new_line_chars + "\n+)")
 
+    def clean_html(self, html: str) -> str:
+        return self.preprocessor.clean_html(html)
+
     def extract(self, text: str) -> str:
         """
         Args:
@@ -50,7 +53,7 @@ def extract(self, text: str) -> str:
         from inscriptis.css_profiles import CSS_PROFILES
         from inscriptis.model.config import ParserConfig
 
-        cleaned_html = self.preprocessor.clean_html(text)
+        cleaned_html = self.clean_html(text)
 
         text = get_text(
             html_content=cleaned_html,