add a min extracted size when merging multiple nodes

adbar · Jan 25, 2024 · 3f50d4e · 3f50d4e
1 parent 65eccab
commit 3f50d4e
Showing 1 changed file with 5 additions and 3 deletions.
diff --git a/trafilatura/core.py b/trafilatura/core.py
@@ -550,11 +550,13 @@ def extract_content(tree, options):
                 new_subtree = HtmlElement(subtrees[0].tag)
                 for _subtree in subtrees:
                     for child in _subtree:
-                        # if len(' '.join(child.itertext()).strip()) > MIN_EXTRACTED_SIZE ?
-                        new_subtree.append(child)
+                        if len(''.join(child.itertext()).strip()) > options.config.getint('DEFAULT', 'MIN_EXTRACTED_SIZE'):
+                            new_subtree.append(child)
                 subtree = new_subtree
-            else:
+            elif len(subtrees) == 1:
                 subtree = subtrees[0]
+            else:
+                continue
         except IndexError:
             continue
         # prune the subtree