add code formatting in TXT/Markdown output (#498)

* add code formatting in TXT/Markdown output * distinguish between inline code and code snippets * use f-strings
adbar · Feb 7, 2024 · 6cc2c69 · 6cc2c69
1 parent 5024da1
commit 6cc2c69
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 0 deletions.
diff --git a/tests/unit_tests.py b/tests/unit_tests.py
@@ -293,6 +293,21 @@ def test_formatting():
     my_document = html.fromstring('<html><body><article><h3>Title</h3><p><b>This here is in bold font.</b></p></article></body></html>')
     my_result = extract(my_document, output_format='txt', include_formatting=True, config=ZERO_CONFIG)
     assert my_result == '### Title\n**This here is in bold font.**'
+    # code sections
+    my_document = html.fromstring('<html><body><article><h3>Title</h3><p>Here is a code sample:</p><code>import trafilatura</code></p></article></body></html>')
+    my_result = extract(my_document, output_format='txt', include_formatting=True, config=ZERO_CONFIG)
+    assert my_result == """### Title
+Here is a code sample:
+`import trafilatura`"""
+    my_document = html.fromstring('<html><body><article><h3>Title</h3><p>Here is a code sample:</p><code>import trafilatura\ntrafilatura.extract("")</code></p></article></body></html>')
+    my_result = extract(my_document, output_format='txt', include_formatting=True, config=ZERO_CONFIG)
+    assert my_result == """### Title
+Here is a code sample:
+```
+import trafilatura
+trafilatura.extract("")
+```"""
+
     # nested
     my_document = html.fromstring('<html><body><p><b>This here is in bold and <i>italic</i> font.</b></p></body></html>')
     my_result = extract(my_document, output_format='xml', include_formatting=True, config=ZERO_CONFIG)

diff --git a/trafilatura/xml.py b/trafilatura/xml.py
@@ -230,6 +230,11 @@ def replace_element_text(element, include_formatting):
             rend = element.get('rend')
             if rend in HI_FORMATTING:
                 element.text = f'{HI_FORMATTING[rend]}{element.text}{HI_FORMATTING[rend]}'
+        elif element.tag == 'code':
+            if '\n' in element.text:
+                element.text = f'```\n{element.text}\n```'
+            else:
+                element.text = f'`{element.text}`'
     # handle links
     if element.tag == 'ref':
         if element.text is not None: