From de57ac1d878ef1e8944741e891288c144b8eff0c Mon Sep 17 00:00:00 2001
From: Vincent Barbaresi <vincent.barbaresi@gmail.com>
Date: Tue, 2 Jan 2024 14:01:51 +0100
Subject: [PATCH] Drop invalid XML element attributes (#462)

* drop invalid XML element attributes

Fixes issue #375

The bug happened when we had a `:` in an element attribute that didn't match any XML namespace (invalid XML). In the example it was `padding:1px=""; margin:15px=""`
We can workaround it by manually dropping those bad elements.
I hope it doesn't impact performance too much

To reproduce:
`trafilatura -u  https://web.archive.org/web/20230619162141/https://www.tristatetelecom.com/productdetailI2.aspx?dataid=IPGSM-4G --xml`

Minimal reproduction example:
```
echo 'Testing<ul style="" padding:1px; margin:15px""><b>Features:</b> <li>Saves the cost of two dedicated phone lines.</li> al station using Internet or cellular technology.</li> <li>Requires no change to the existing Fire Alarm Control Panel configuration. The IPGSM-4G connects directly to the primary and secondary telephone ports.</li>
' | trafilatura --xml
```

* pin lxml to < 5

* syntax

---------

Co-authored-by: Adrien Barbaresi <adbar@users.noreply.github.com>
Co-authored-by: Adrien Barbaresi <barbaresi@bbaw.de>
---
 tests/unit_tests.py  | 5 ++++-
 trafilatura/utils.py | 6 ++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/tests/unit_tests.py b/tests/unit_tests.py
index ed4bb046..48fef5f0 100644
--- a/tests/unit_tests.py
+++ b/tests/unit_tests.py
@@ -387,7 +387,10 @@ def test_external():
         teststring = f.read()
     assert extract(teststring, no_fallback=True, include_tables=False) == ''
     assert extract(teststring, no_fallback=False, include_tables=False) == ''
-
+    # invalid XML attributes: namespace colon in attribute key (issue #375). Those attributes should be stripped
+    bad_xml = 'Testing<ul style="" padding:1px; margin:15px""><b>Features:</b> <li>Saves the cost of two dedicated phone lines.</li> al station using Internet or cellular technology.</li> <li>Requires no change to the existing Fire Alarm Control Panel configuration. The IPGSM-4G connects directly to the primary and secondary telephone ports.</li>'
+    res = extract(bad_xml, output_format='xml')
+    assert "Features" in res
 
 def test_images():
     '''Test image extraction function'''
diff --git a/trafilatura/utils.py b/trafilatura/utils.py
index 3b8bbc04..7cc49ff5 100644
--- a/trafilatura/utils.py
+++ b/trafilatura/utils.py
@@ -303,6 +303,12 @@ def sanitize_tree(tree):
         preserve_space = elem.tag in SPACING_PROTECTED or parent_tag in SPACING_PROTECTED
         trailing_space = elem.tag in FORMATTING_PROTECTED or parent_tag in FORMATTING_PROTECTED or preserve_space
 
+        # remove invalid attributes
+        for attribute in elem.attrib:
+            if ':' in attribute:  # colon is reserved for namespaces in XML
+                if not elem.attrib[attribute] or attribute.split(':', 1)[0] not in tree.nsmap:
+                    elem.attrib.pop(attribute)
+
         if elem.text:
             elem.text = sanitize(elem.text, preserve_space, trailing_space)
         if elem.tail: