Merge branch 'main' into fix-opt

.github/workflows/tests.yml

@@ -122,6 +122,30 @@ jobs:
 
       - name: Run tests
         run: |
+          # Attempt to work around https://github.com/pypa/pip/issues/12781
+          PIP_CONSTRAINT= hatch env run -e test -- pip install 'pip>=24.2'
+          xvfb-run --auto-servernum hatch run test:nowarn || xvfb-run --auto-servernum hatch run test:nowarn --lf
+
+  test_mistune_30:
+    name: Test Mistune 3.0
+    timeout-minutes: 20
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install texlive-plain-generic inkscape texlive-xetex latexmk
+          sudo apt-get install xvfb x11-utils libxkbcommon-x11-0  libxcb-xinerama0 python3-pyqt5
+
+          # pandoc is not up to date in the ubuntu repos, so we install directly
+          wget https://github.com/jgm/pandoc/releases/download/2.9.2.1/pandoc-2.9.2.1-1-amd64.deb && sudo dpkg -i pandoc-2.9.2.1-1-amd64.deb
+
+      - name: Run tests
+        run: |
+          hatch env run -e test -- pip install 'mistune~=3.0.0'
           xvfb-run --auto-servernum hatch run test:nowarn || xvfb-run --auto-servernum hatch run test:nowarn --lf
 
   test_prereleases:

CHANGELOG.md

@@ -2,6 +2,35 @@
 
 <!-- <START NEW CHANGELOG ENTRY> -->
 
+## 7.16.5
+
+([Full Changelog](https://github.com/jupyter/nbconvert/compare/v7.16.4...18e10f6484afdfee39105248c28abe5ef0835b22))
+
+### Enhancements made
+
+- Add support for mistune 3.1.0 [#2199](https://github.com/jupyter/nbconvert/pull/2199) ([@fcollonval](https://github.com/fcollonval))
+- Allow including `text/x-rst` outputs in rst conversion, transition away from `text/restructuredtext` [#2167](https://github.com/jupyter/nbconvert/pull/2167) ([@takluyver](https://github.com/takluyver))
+
+### Bugs fixed
+
+- Do not display mathjax overlay [#2181](https://github.com/jupyter/nbconvert/pull/2181) ([@timkpaine](https://github.com/timkpaine))
+- Work around pip 24.1 bug which prevents installing pandocfilters 1.4.1 [#2168](https://github.com/jupyter/nbconvert/pull/2168) ([@takluyver](https://github.com/takluyver))
+- Don't die if template path cannot be read [#2162](https://github.com/jupyter/nbconvert/pull/2162) ([@stuaxo](https://github.com/stuaxo))
+- Fix markdown2asciidoc function for pandoc >= 3.0 (closes #2017) [#2152](https://github.com/jupyter/nbconvert/pull/2152) ([@thomasjm](https://github.com/thomasjm))
+
+### Maintenance and upkeep improvements
+
+- enhancement dep-chain: directly depend on bleach\[css\], instead of pulling in tinycss2. [#2166](https://github.com/jupyter/nbconvert/pull/2166) ([@xiacunshun](https://github.com/xiacunshun))
+- chore: update pre-commit hooks [#2146](https://github.com/jupyter/nbconvert/pull/2146) ([@pre-commit-ci](https://github.com/pre-commit-ci))
+
+### Contributors to this release
+
+([GitHub contributors page for this release](https://github.com/jupyter/nbconvert/graphs/contributors?from=2024-04-29&to=2025-01-02&type=c))
+
+[@bollwyvl](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Abollwyvl+updated%3A2024-04-29..2025-01-02&type=Issues) | [@fcollonval](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Afcollonval+updated%3A2024-04-29..2025-01-02&type=Issues) | [@krassowski](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Akrassowski+updated%3A2024-04-29..2025-01-02&type=Issues) | [@pre-commit-ci](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Apre-commit-ci+updated%3A2024-04-29..2025-01-02&type=Issues) | [@stuaxo](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Astuaxo+updated%3A2024-04-29..2025-01-02&type=Issues) | [@t-makaro](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3At-makaro+updated%3A2024-04-29..2025-01-02&type=Issues) | [@takluyver](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Atakluyver+updated%3A2024-04-29..2025-01-02&type=Issues) | [@thomasjm](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Athomasjm+updated%3A2024-04-29..2025-01-02&type=Issues) | [@timkpaine](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Atimkpaine+updated%3A2024-04-29..2025-01-02&type=Issues) | [@xiacunshun](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Axiacunshun+updated%3A2024-04-29..2025-01-02&type=Issues)
+
+<!-- <END NEW CHANGELOG ENTRY> -->
+
 ## 7.16.4
 
 ([Full Changelog](https://github.com/jupyter/nbconvert/compare/v7.16.3...71fde294f623083f364bf68e14f07459dee952e6))
@@ -22,8 +51,6 @@
 
 [@blink1073](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Ablink1073+updated%3A2024-03-21..2024-04-29&type=Issues) | [@mgeier](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Amgeier+updated%3A2024-03-21..2024-04-29&type=Issues) | [@pre-commit-ci](https://github.com/search?q=repo%3Ajupyter%2Fnbconvert+involves%3Apre-commit-ci+updated%3A2024-03-21..2024-04-29&type=Issues)
 
-<!-- <END NEW CHANGELOG ENTRY> -->
-
 ## 7.16.3
 
 ([Full Changelog](https://github.com/jupyter/nbconvert/compare/v7.16.2...9019037e300dfdbac5a8cb330d0b09eb82e12f62))

docs/source/nbconvert_library.ipynb

@@ -486,7 +486,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "@damianavila wrote the Nikola Plugin to [write blog post as Notebooks](http://damianavila.github.io/blog/posts/one-line-deployment-of-your-site-to-gh-pages.html) and is developing a js-extension to publish notebooks via one click from the web app."
+    "@damianavila wrote the Nikola Plugin to [write blog post as Notebooks](https://damianavila.github.io/blog/posts/one-line-deployment-of-your-site-to-gh-pages) and is developing a js-extension to publish notebooks via one click from the web app."
    ]
   }
  ],

nbconvert/_version.py

@@ -4,7 +4,7 @@
 from typing import List
 
 # Version string must appear intact for versioning
-__version__ = "7.16.4"
+__version__ = "7.16.5"
 
 # Build up version_info tuple for backwards compatibility
 pattern = r"(?P<major>\d+).(?P<minor>\d+).(?P<patch>\d+)(?P<rest>.*)"

nbconvert/exporters/html.py

@@ -15,6 +15,7 @@
 from bs4 import BeautifulSoup
 from jupyter_core.paths import jupyter_path
 from traitlets import Bool, Unicode, default, validate
+from traitlets import Dict as TraitletsDict
 from traitlets.config import Config
 
 if tuple(int(x) for x in jinja2.__version__.split(".")[:3]) < (3, 0, 0):
@@ -183,6 +184,14 @@ def _template_name_default(self):
 
     output_mimetype = "text/html"
 
+    lexer_options = TraitletsDict(
+        {},
+        help=(
+            "Options to be passed to the pygments lexer for highlighting markdown code blocks. "
+            "See https://pygments.org/docs/lexers/#available-lexers for available options."
+        ),
+    ).tag(config=True)
+
     @property
     def default_config(self):
         c = Config(
@@ -239,6 +248,7 @@ def markdown2html(self, context, source):
             path=path,
             anchor_link_text=self.anchor_link_text,
             exclude_anchor_links=self.exclude_anchor_links,
+            **self.lexer_options,
         )
         return MarkdownWithMath(renderer=renderer).render(source)
 

nbconvert/exporters/rst.py

@@ -6,6 +6,7 @@
 from traitlets import default
 from traitlets.config import Config
 
+from ..filters import DataTypeFilter
 from .templateexporter import TemplateExporter
 
 
@@ -22,9 +23,24 @@ def _file_extension_default(self):
     def _template_name_default(self):
         return "rst"
 
-    output_mimetype = "text/restructuredtext"
+    @default("raw_mimetypes")
+    def _raw_mimetypes_default(self):
+        # Up to summer 2024, nbconvert had a mistaken output_mimetype.
+        # Listing that as an extra option here maintains compatibility for
+        # notebooks with raw cells marked as that mimetype.
+        return [self.output_mimetype, "text/restructuredtext", ""]
+
+    output_mimetype = "text/x-rst"
     export_from_notebook = "reST"
 
+    def default_filters(self):
+        """Override filter_data_type to use native rst outputs"""
+        dtf = DataTypeFilter()
+        dtf.display_data_priority = [self.output_mimetype, *dtf.display_data_priority]
+        filters = dict(super().default_filters())
+        filters["filter_data_type"] = dtf
+        return filters.items()
+
     @property
     def default_config(self):
         c = Config(

nbconvert/exporters/templateexporter.py

@@ -411,6 +411,7 @@ def from_notebook_node(  # type:ignore[explicit-override, override]
         """
         nb_copy, resources = super().from_notebook_node(nb, resources, **kw)
         resources.setdefault("raw_mimetypes", self.raw_mimetypes)
+        resources.setdefault("output_mimetype", self.output_mimetype)
         resources["global_content_filter"] = {
             "include_code": not self.exclude_code_cell,
             "include_markdown": not self.exclude_markdown,
@@ -569,13 +570,19 @@ def _template_paths(self, prune=True, root_dirs=None):
         for template_name in template_names:
             for base_dir in self.extra_template_basedirs:
                 path = os.path.join(base_dir, template_name)
-                if not prune or os.path.exists(path):
-                    paths.append(path)
+                try:
+                    if not prune or os.path.exists(path):
+                        paths.append(path)
+                except PermissionError:
+                    pass
             for root_dir in root_dirs:
                 base_dir = os.path.join(root_dir, "nbconvert", "templates")
                 path = os.path.join(base_dir, template_name)
-                if not prune or os.path.exists(path):
-                    paths.append(path)
+                try:
+                    if not prune or os.path.exists(path):
+                        paths.append(path)
+                except PermissionError:
+                    pass
 
         for root_dir in root_dirs:
             # we include root_dir for when we want to be very explicit, e.g.

nbconvert/filters/highlight.py

@@ -136,7 +136,9 @@ def __call__(self, source, language=None, metadata=None, strip_verbatim=False):
         return latex
 
 
-def _pygments_highlight(source, output_formatter, language="ipython", metadata=None):
+def _pygments_highlight(
+    source, output_formatter, language="ipython", metadata=None, **lexer_options
+):
     """
     Return a syntax-highlighted version of the input source
 
@@ -149,6 +151,10 @@ def _pygments_highlight(source, output_formatter, language="ipython", metadata=N
         language to highlight the syntax of
     metadata : NotebookNode cell metadata
         metadata of the cell to highlight
+    lexer_options : dict
+        Options to pass to the pygments lexer. See
+        https://pygments.org/docs/lexers/#available-lexers for more information about
+        valid lexer options
     """
     from pygments import highlight
     from pygments.lexers import get_lexer_by_name
@@ -179,7 +185,7 @@ def _pygments_highlight(source, output_formatter, language="ipython", metadata=N
 
     if lexer is None:
         try:
-            lexer = get_lexer_by_name(language, stripall=True)
+            lexer = get_lexer_by_name(language, **lexer_options)
         except ClassNotFound:
             warn("No lexer found for language %r. Treating as plain text." % language, stacklevel=2)
             from pygments.lexers.special import TextLexer

nbconvert/filters/markdown.py

@@ -8,6 +8,10 @@
 
 import re
 
+from packaging.version import Version
+
+from nbconvert.utils.pandoc import get_pandoc_version
+
 try:
     from .markdown_mistune import markdown2html_mistune
 
@@ -66,7 +70,16 @@ def markdown2html_pandoc(source, extra_args=None):
 
 def markdown2asciidoc(source, extra_args=None):
     """Convert a markdown string to asciidoc via pandoc"""
-    extra_args = extra_args or ["--atx-headers"]
+
+    # Prior to version 3.0, pandoc supported the --atx-headers flag.
+    # For later versions, we must instead pass --markdown-headings=atx.
+    # See https://pandoc.org/releases.html#pandoc-3.0-2023-01-18
+    atx_args = ["--atx-headers"]
+    pandoc_version = get_pandoc_version()
+    if pandoc_version and Version(pandoc_version) >= Version("3.0"):
+        atx_args = ["--markdown-headings=atx"]
+
+    extra_args = extra_args or atx_args
     asciidoc = convert_pandoc(source, "markdown", "asciidoc", extra_args=extra_args)
     # workaround for https://github.com/jgm/pandoc/issues/3068
     if "__" in asciidoc: