diff --git a/.coveragerc b/.coveragerc index f46fbb8..ad6dc16 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,3 @@ [run] omit = - ipypublish/ipysphinx/docutils_transforms.py - ipypublish/ipysphinx/extension.py - ipypublish/ipysphinx/directives.py ipypublish/scripts/nb_setup.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..e9008a6 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +exclude = + setup.py, + ipypublish/scripts/ipynb_latex_setup.py, + ipypublish/tests/test_files/basic_nb/expected/python_with_meta.py \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 8a966f1..8ee3454 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,22 +3,37 @@ matrix: include: - os: linux sudo: required - python: 2.7 + python: 3.6 dist: trusty + env: TEST_TYPE="flake8" - os: linux sudo: required - python: 3.4 + python: 3.6 dist: trusty + env: TEST_TYPE="rtd" - os: linux sudo: required - python: 3.5 + python: 3.6 dist: trusty + env: TEST_TYPE="pytest" PYPI_DEPLOY=true - os: linux sudo: required - python: 3.6 + python: 2.7 + dist: trusty + env: TEST_TYPE="pytest" + - os: linux + sudo: required + python: 3.4 dist: trusty + env: TEST_TYPE="pytest" + - os: linux + sudo: required + python: 3.5 + dist: trusty + env: TEST_TYPE="pytest" - os: osx language: generic + env: TEST_TYPE="pytest" before_install: - travis_wait brew update # TODO currently by default python 2.7 is already installed (see https://github.com/travis-ci/travis-ci/issues/9929) @@ -51,59 +66,72 @@ matrix: - travis_retry sudo tlmgr install biblatex - travis_retry sudo tlmgr install needspace - travis_retry sudo tlmgr install collection-fontsrecommended + # glossaries dependencies + - travis_retry sudo tlmgr install glossaries # NB: for different languages glossaries- + - travis_retry sudo tlmgr install mfirstuc # see https://tex.stackexchange.com/questions/268216/usepackageglossaries-wont-work-after-miktex-update-reinstallation + - travis_retry sudo tlmgr install xfor + - travis_retry sudo tlmgr install datatool + - travis_retry sudo tlmgr install substr allow_failures: - os: linux sudo: required python: 3.4 dist: trusty + env: TEST_TYPE="pytest" - os: linux sudo: required python: 3.5 dist: trusty + env: TEST_TYPE="pytest" - os: osx language: generic - -# TODO add read the docs test `pip install .[docs] cd docs; make` + env: TEST_TYPE="pytest" before_install: # Pandoc -- url="https://github.com/jgm/pandoc/releases/tag/2.6" -- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then path=$(curl -L $url | grep -o '/jgm/pandoc/releases/download/.*-amd64\.deb') ; fi -- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then downloadUrl="https://github.com$path" ; fi -- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then file=${path##*/} ; fi -- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then wget $downloadUrl && sudo dpkg -i $file ; fi +- | + if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + url="https://github.com/jgm/pandoc/releases/tag/2.6" + path=$(curl -L $url | grep -o '/jgm/pandoc/releases/download/.*-amd64\.deb') + downloadUrl="https://github.com$path" + file=${path##*/} + wget $downloadUrl && sudo dpkg -i $file + fi # LaTeX -- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get -qq update ; fi -# - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y pandoc ; fi -- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y texlive ; fi -- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y texlive-xetex ; fi -- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y texlive-latex-extra ; fi -- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y texlive-lang-portuguese ; fi -# need up-to-date koma-script, which isn't supplied with this version of debian -- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo wget http://mirrors.ctan.org/install/macros/latex/contrib/koma-script.tds.zip ; fi -- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo unzip koma-script.tds.zip -d ~/texmf/ ; fi -- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y latexmk ; fi -- pip install -U pip setuptools wheel +- | + if [[ "$TRAVIS_OS_NAME" == "linux" && "$TEST_TYPE" == "pytest" ]]; then + sudo apt-get -qq update + # sudo apt-get install -y pandoc is too old + sudo apt-get install -y texlive + sudo apt-get install -y texlive-xetex + sudo apt-get install -y texlive-latex-extra + sudo apt-get install -y texlive-lang-portuguese + # texlive-glossaries doesn't appear to be available + sudo wget http://mirrors.ctan.org/install/macros/latex/contrib/glossaries.tds.zip + sudo unzip glossaries.tds.zip -d ~/texmf/ + # need up-to-date koma-script, which isn't supplied with this version of debian + sudo wget http://mirrors.ctan.org/install/macros/latex/contrib/koma-script.tds.zip + sudo unzip koma-script.tds.zip -d ~/texmf/ + sudo apt-get install -y latexmk + fi + install: -- travis_wait pip install .[tests,sphinx] -- pip install --quiet coveralls +- pip install -U pip setuptools wheel +- if [[ "$TEST_TYPE" == "pytest" ]]; then travis_wait pip install .[tests] ; fi +- if [[ "$TEST_TYPE" == "pytest" ]]; then pip install --quiet coveralls ; fi +- if [[ "$TEST_TYPE" == "flake8" ]]; then travis_wait pip install "flake8(>=3.7,<3.8)" ; fi +- if [[ "$TEST_TYPE" == "rtd" ]]; then travis_wait pip install .[rtd] ; fi + script: -# - nosetests -v --nocapture --with-doctest --with-coverage --exe --cover-package=ipypublish -- pytest -v --cov=ipypublish --cov-config .coveragerc --cov-report= ipypublish -- nbpublish -pdf --pdf-debug -log debug example/notebooks/Example.ipynb +- if [[ "$TEST_TYPE" == "pytest" ]]; then pytest -v --cov=ipypublish --cov-config .coveragerc --cov-report= ipypublish ; fi +- if [[ "$TEST_TYPE" == "pytest" ]]; then nbpublish -pdf --pdf-debug -log debug example/notebooks/Example.ipynb ; fi +- if [[ "$TEST_TYPE" == "flake8" ]]; then flake8 . ; fi +- if [[ "$TEST_TYPE" == "rtd" ]]; then cd docs; make ; fi + after_success: -- coveralls +- if [[ "$TEST_TYPE" == "pytest" ]]; then coveralls ; fi + deploy: -# - provider: pypi -# distributions: bdist_wheel -# server: https://test.pypi.org/legacy/ -# user: cjsewell -# password: -# secure: en0mNL9+rzlhUpnsLsdd3kskGYWVHL+bUyKnPJCZq0d0AH7tmAC7/J+KkR9iv+pHwVHJphqd+8X3p7k+eXOjjSGlY4mnTmz84iQrRilRzEsK53IvlU+0tayKv23ZlUVG8ecKT/33BOdqO/QZNOz09DViTABj65iIoTzd3pM1b94ycCOjWJV2EzSn8UQXe0p90mO7iE4aoyOOimweezbaZ1FrmZatUuRWeFGuzhopHF1FHtd/xk8Q2rWItUqwIgHiESknAAm5fFsgfJtEqenvmoGbeuBJ7ImCQzEOtF30Im7HJRwd8tXRJHVrTS0PpQpZLjhS/nGors8RRC0DW7r1FdYTUKWUUypDZJSphpyAkfvdVtlOX0NBBuCU71X8yx3yZWLsSv84PulRN6YASPSV3FvptMtam1J2/eoAm2aNy+/RyCvZpe8VtlE+7CSQX41T/1FdIyMzThlJHG0n3Kx92YBTUVuNLTlawTfK4WfS8NJhW1H74XoIRzrKn41EVoup4G0HAPj3vFhezwMdJ7CFIPyav5RdYf3h60smE0qpJl+qIMo5QD7AEWpvg+AIR8C4e0fRE6Iqovqzk+vM4Jq1QlI/mghDkRTFKEES5v0dObaIG8Gi+F3TfBjlyssHA0YREBaZRfVaO2Ov8wLBNaqr3QyJniJ5+eK2yL406DCqKOc= -# on: -# branch: master -# tags: false -# condition: $TRAVIS_PYTHON_VERSION = "3.6" - provider: pypi distributions: "bdist_wheel sdist" user: cjsewell @@ -112,4 +140,4 @@ deploy: on: branch: master tags: true - condition: $TRAVIS_PYTHON_VERSION = "3.6" + condition: $PYPI_DEPLOY = true diff --git a/.vscode/jinja2-latex.code-snippets b/.vscode/jinja2-latex.code-snippets new file mode 100644 index 0000000..3d5e90d --- /dev/null +++ b/.vscode/jinja2-latex.code-snippets @@ -0,0 +1,76 @@ +{ + "super": { + "prefix": "super", + "scope": "jinja-latex", + "body": "((( super() )))", + "description": "call inherited block" + }, + "set": { + "prefix": "set", + "scope": "jinja-latex", + "body": "((* set ${1:name} = ${2:value} *))", + "description": "set variable" + }, + "print": { + "prefix": "print", + "scope": "inja-latex", + "body": "((( ${1:variable} )))", + "description": "print variable" + }, + "block": { + "prefix": "block", + "scope": "jinja-latex", + "body": [ + "((* block ${1:name} *))", + "$2", + "((* endblock ${1:name} *))" + ], + "description": "jinja block" + }, + "macro": { + "prefix": "macro", + "scope": "jinja-latex", + "body": [ + "((* macro ${1:name} *))", + "$2", + "((* endmacro *))" + ], + "description": "macro function" + }, + "if": { + "prefix": "if", + "scope": "jinja-latex", + "body": [ + "((* if ${1:condition} *))", + "$2", + "((* endif *))" + ], + "description": "if condition" + }, + "if-else": { + "prefix": "if-else", + "scope": "jinja-latex", + "body": [ + "((* if ${1:condition} *))", + "$2", + "((* else *))", + "$3", + "((* endif *))" + ], + "description": "if-else condition" + }, + "if-elif-else": { + "prefix": "if-elif-else", + "scope": "jinja-latex", + "body": [ + "((* if ${1:condition} *))", + "$2", + "((* else *))", + "$3", + "((* elif ${1:condition2} *))", + "", + "((* endif *))" + ], + "description": "if-else condition" + } +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 30cd7cd..98f8039 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -29,11 +29,13 @@ "latex-workshop.latex.autoBuild.onSave.enabled": false, "cSpell.words": [ "Jupyter", + "docutils", "ipynb", "ipypublish", "jupytext", "nbconvert", "nbpresent", + "noqa", "placeholders", "plugins" ], diff --git a/MANIFEST.in b/MANIFEST.in index 94dd60a..00b80a8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,5 +4,5 @@ include requirements.txt recursive-include ipypublish *.json recursive-include ipypublish *.j2 recursive-include ipypublish *.yaml -recursive-include ipypublish/ipysphinx/css *.css -recursive-include ipypublish/tests/test_files * \ No newline at end of file +recursive-include ipypublish/sphinx/notebook/css *.css +include ipypublish/tests/test_files/example.jpg diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..1ece6b4 --- /dev/null +++ b/conftest.py @@ -0,0 +1 @@ +pytest_plugins = 'sphinx.testing.fixtures' diff --git a/docs/environment.yaml b/docs/environment.yaml index dd72385..66a7291 100644 --- a/docs/environment.yaml +++ b/docs/environment.yaml @@ -2,4 +2,8 @@ channels: - conda-forge dependencies: - python =3.6 + # TODO pip>=10 fails du to this issue https://github.com/pypa/pip/issues/5247 + # raises error `Cannot uninstall 'docutils'. It is a distutils installed project` + # even though docutils appear to already be at the correct version (0.14)!? + - pip <10.0 - pandoc diff --git a/docs/get_intersphinx_inv.py b/docs/get_intersphinx_inv.py index 1222bef..214f1f6 100644 --- a/docs/get_intersphinx_inv.py +++ b/docs/get_intersphinx_inv.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python def fetch_inventory(uri): @@ -26,7 +26,9 @@ def warn(self, msg): # uri = "https://docutils.readthedocs.io/en/sphinx-docs/objects.inv" # uri = "https://traitlets.readthedocs.io/en/latest/objects.inv" # uri = "https://networkx.github.io/documentation/stable/objects.inv" - uri = "http://docs.scipy.org/doc/scipy/reference/objects.inv" + # uri = "http://docs.scipy.org/doc/scipy/reference/objects.inv" + # uri = "http://pillow.readthedocs.org/en/latest/objects.inv" + uri = 'http://www.sphinx-doc.org/en/latest/objects.inv' # Read inventory into a dictionary inv = fetch_inventory(uri) diff --git a/docs/source/_static/example_glossary.bib b/docs/source/_static/example_glossary.bib new file mode 100644 index 0000000..81c2a8f --- /dev/null +++ b/docs/source/_static/example_glossary.bib @@ -0,0 +1,24 @@ +@glsterm{gtkey1, + name = {name}, + description = {full description \textbf{with latex}}, + plural = {some names}, + text = {alternative text}, + sort = {a}, + symbol = {\ensuremath{n}} +} + +@glsacronym{akey1, + abbreviation = {MA}, + longname = {My Abbreviation}, + description = {full description}, + plural = {MAs}, + longplural = {Some Abbreviations} +} + +@glssymbol{symbol1, + name = {\ensuremath{\pi}}, + description = {full description}, + plural = {\ensuremath{\pi}s}, + text = {alternative text}, + sort = {b} +} \ No newline at end of file diff --git a/docs/source/_static/other_glossary.bib b/docs/source/_static/other_glossary.bib new file mode 100644 index 0000000..2fe4999 --- /dev/null +++ b/docs/source/_static/other_glossary.bib @@ -0,0 +1,24 @@ +@glsterm{gtkey2, + name = {name}, + description = {full description \textbf{with latex}}, + plural = {some names}, + text = {alternative text}, + sort = {a}, + symbol = {\ensuremath{n}} +} + +@glsacronym{akey2, + abbreviation = {MA}, + longname = {My Abbreviation}, + description = {full description}, + plural = {MAs}, + longplural = {Some Abbreviations} +} + +@glssymbol{symbol2, + name = {\ensuremath{\pi}}, + description = {full description}, + plural = {\ensuremath{\pi}s}, + text = {alternative text}, + sort = {b} +} \ No newline at end of file diff --git a/docs/source/api/ipypublish.schema.rst b/docs/source/api/ipypublish.schema.rst index 9de9894..24e197c 100644 --- a/docs/source/api/ipypublish.schema.rst +++ b/docs/source/api/ipypublish.schema.rst @@ -1,5 +1,5 @@ -ipypublish\.schema package -========================== +ipypublish.schema package +========================= Module contents --------------- diff --git a/docs/source/code_cells.ipynb b/docs/source/code_cells.ipynb index 418f5d1..547b149 100644 --- a/docs/source/code_cells.ipynb +++ b/docs/source/code_cells.ipynb @@ -155,8 +155,7 @@ "outputs": [], "source": [ "import os\n", - "from ipypublish.tests import TEST_FILES_DIR\n", - "example_pic = os.path.join(TEST_FILES_DIR, 'example.jpg')" + "from ipypublish.tests import TEST_PIC_PATH\n" ] }, { @@ -173,7 +172,7 @@ }, "outputs": [], "source": [ - "nb_setup.images_hconcat([example_pic, example_pic],\n", + "nb_setup.images_hconcat([TEST_PIC_PATH, TEST_PIC_PATH],\n", " width=600, gap=10)" ] }, @@ -192,7 +191,7 @@ }, "outputs": [], "source": [ - "nb_setup.images_vconcat([example_pic, example_pic],\n", + "nb_setup.images_vconcat([TEST_PIC_PATH, TEST_PIC_PATH],\n", " height=400, gap=10)" ] }, @@ -211,7 +210,7 @@ }, "outputs": [], "source": [ - "nb_setup.images_gridconcat([[_,_] for _ in [example_pic, example_pic]],\n", + "nb_setup.images_gridconcat([[_,_] for _ in [TEST_PIC_PATH, TEST_PIC_PATH]],\n", " height=300, vgap=10,hgap=20)" ] }, diff --git a/docs/source/conf.py b/docs/source/conf.py index 52d0e98..f935902 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -18,6 +18,8 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +import docutils +import sphinxcontrib.bibtex import os import io import urllib @@ -59,37 +61,62 @@ 'sphinx.ext.autosummary', # 'sphinx.ext.imgconverter' # converts svg to pdf in latex output # TODO imgconverter failing (I guess for process.svg), - 'ipypublish.ipysphinx', - 'sphinxcontrib.bibtex' + 'ipypublish.sphinx.notebook', + 'ipypublish.sphinx.gls', + 'sphinxcontrib.bibtex', + 'recommonmark' ] +logger = sphinx.util.logging.getLogger(__name__) + + +# TODO this is a workaround until +# https://github.com/mcmtroffaes/sphinxcontrib-bibtex/pull/162 is merged +def process_citations(app, doctree, docname): + """Replace labels of citation nodes by actual labels. + + :param app: The sphinx application. + :type app: :class:`sphinx.application.Sphinx` + :param doctree: The document tree. + :type doctree: :class:`docutils.nodes.document` + :param docname: The document name. + :type docname: :class:`str` + """ + for node in doctree.traverse(docutils.nodes.citation): + key = node[0].astext() + try: + label = app.env.bibtex_cache.get_label_from_key(key) + except KeyError: + logger.warning("could not relabel citation [%s]" % key, + type="bibtex", subtype="relabel") + else: + node[0] = docutils.nodes.label('', label) + + +sphinxcontrib.bibtex.process_citations = process_citations + +suppress_warnings = ['bibtex.relabel'] + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = { -# '.rst': 'restructuredtext', -# '.md': 'markdown', -# '.ipynb': 'jupyter_notebook' -# } -# source_suffix = ['.rst', '.md', '.ipynb'] - if sphinx.version_info[0:2] < (1, 8): source_parsers = { - '.md': 'recommonmark.parser.CommonMarkParser', - '.Rmd': 'ipypublish.ipysphinx.parser.NBParser' + # '.md': 'recommonmark.parser.CommonMarkParser', + '.Rmd': 'ipypublish.sphinx.notebook.parser.NBParser' } else: - source_parsers = { - '.md': 'recommonmark.parser.CommonMarkParser' - } - import jupytext - ipysphinx_preconverters = { - ".Rmd": jupytext.readf + source_suffix = { + '.rst': 'restructuredtext', + '.md': 'markdown', + '.ipynb': 'jupyter_notebook', + '.Rmd': 'jupyter_notebook' } + # import jupytext + # ipysphinx_preconverters = { + # ".Rmd": jupytext.readf + # } ipysphinx_show_prompts = True # List of patterns, relative to source directory, that match files and @@ -204,10 +231,10 @@ numfig = True math_numfig = True numfig_secnum_depth = 2 -numfig_format: {'section': 'Section %s', - 'figure': 'Fig. %s', - 'table': 'Table %s', - 'code-block': 'Code Block %s'} +numfig_format = {'section': 'Section %s', + 'figure': 'Fig. %s', + 'table': 'Table %s', + 'code-block': 'Code Block %s'} math_number_all = True math_eqref_format = "Eq. {number}" # TODO this isn't working @@ -243,16 +270,23 @@ 'tornado': ("https://www.tornadoweb.org/en/stable/", None), 'traitlets': ("https://traitlets.readthedocs.io/en/stable/", None), 'jinja': ('http://jinja.pocoo.org/docs/dev', None), + 'bibtexparser': ('https://bibtexparser.readthedocs.io/en/master/', None), # 'docutils': ("https://docutils.readthedocs.io/en/sphinx-docs", None), # # TODO docutils intersphinx - # 'sphinx': ('http://www.sphinx-doc.org/en/latest/', None) + 'sphinx': ('http://www.sphinx-doc.org/en/latest/', None) } intersphinx_aliases = { + ('py:class', 'dictionary'): + ('py:class', 'dict'), + ('py:class', 'PIL.Image'): + ('py:class', 'PIL.Image.Image'), ('py:class', 'nbconvert.preprocessors.base.Preprocessor'): ('py:class', 'nbconvert.preprocessors.Preprocessor'), ('py:class', 'nbformat.notebooknode.NotebookNode'): ('py:class', 'nbformat.NotebookNode'), + ('py:class', 'NotebookNode'): + ('py:class', 'nbformat.NotebookNode'), ('py:class', 'traitlets.config.configurable.Configurable'): ('py:module', 'traitlets.config') } @@ -298,11 +332,13 @@ ('py:class', 'traitlets.config.configurable.LoggingConfigurable'), ('py:class', 'docutils.nodes.Element'), + ('py:class', 'docutils.nodes.General'), + ('py:class', 'docutils.nodes.document'), ('py:class', 'docutils.parsers.rst.Directive'), ('py:class', 'docutils.transforms.Transform'), ('py:class', 'docutils.parsers.rst.Parser'), ('py:class', 'sphinx.parsers.RSTParser'), - ('py:obj', 'sphinx.application.Sphinx'), + ('py:class', 'sphinx.roles.XRefRole'), ('py:exc', 'nbconvert.pandoc.PandocMissing') ] @@ -317,6 +353,9 @@ if gitbranch is not None and "develop" in gitbranch: gitpath = "blob/develop" binderpath = "develop" +elif gitbranch is not None and "glossary" in gitbranch: + gitpath = "blob/glossary" + binderpath = "glossary" else: gitpath = "blob/v{}".format(ipypublish.__version__) binderpath = "v{}".format(ipypublish.__version__) @@ -339,7 +378,7 @@ {{%- endif %}} __ https://github.com/chrisjsewell/ipypublish/{gitpath}/{{{{ docname }}}} -""".format(gitpath=gitpath, binderpath=binderpath) +""".format(gitpath=gitpath, binderpath=binderpath) # noqa: E501 def create_git_releases(app): @@ -399,10 +438,11 @@ def add_intersphinx_aliases_to_inv(app): def run_apidoc(app): - """ generate apidoc + """ generate apidoc See: https://github.com/rtfd/readthedocs.org/issues/1139 """ + logger.info("running apidoc") # get correct paths this_folder = os.path.abspath( os.path.dirname(os.path.realpath(__file__))) @@ -410,16 +450,22 @@ def run_apidoc(app): # module_path = ipypublish.utils.get_module_path(ipypublish) module_path = os.path.normpath( os.path.join(this_folder, "../../")) - ignore_setup = os.path.normpath( - os.path.join(this_folder, "../../setup.py")) - ignore_tests = os.path.normpath( - os.path.join(this_folder, "../../ipypublish/tests")) + + ignore_paths = [ + "../../setup.py", + "../../conftest.py", + "../../ipypublish/tests", + "../../ipypublish/sphinx/tests" + ] + ignore_paths = [ + os.path.normpath( + os.path.join(this_folder, p)) for p in ignore_paths] + if os.path.exists(api_folder): shutil.rmtree(api_folder) os.mkdir(api_folder) - argv = ["--separate", "-o", api_folder, - module_path, ignore_setup, ignore_tests] + argv = ["--separate", "-o", api_folder, module_path] + ignore_paths try: # Sphinx 1.7+ diff --git a/docs/source/index.rst b/docs/source/index.rst index 2de024d..b3f44e2 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -64,7 +64,7 @@ Badges nb_conversion metadata_tags custom_export_config - sphinx_extension + sphinx_extensions examples applications additional_tools diff --git a/docs/source/ipynb_with_external.txt b/docs/source/ipynb_with_external.txt deleted file mode 100644 index a67937e..0000000 --- a/docs/source/ipynb_with_external.txt +++ /dev/null @@ -1,162 +0,0 @@ ---- -jupyter: - ipub: - at_notation: true - bibliography: _static/example.bib - biboptions: - - super - - sort - bibstyle: unsrtnat - language: portuges - listcode: true - listfigures: true - listtables: true - titlepage: - author: Authors Name - email: authors@email.com - institution: - - Institution1 - - Institution2 - logo: _static/logo_name.png - subtitle: Sub-Title - supervisors: - - First Supervisor - - Second Supervisor - tagline: A tagline for the report. - title: Main-Title - toc: - depth: 2 - use_numref: true - jupytext: - metadata_filter: - notebook: ipub - text_representation: - extension: .Rmd - format_name: rmarkdown - format_version: '1.0' - jupytext_version: 0.8.6 - kernelspec: - display_name: Python 3 - language: python - name: python3 ---- - -```{python init_cell=TRUE, slideshow={'slide_type': 'skip'}} -from ipypublish import nb_setup -plt = nb_setup.setup_matplotlib( - print_errors=True, - output=('pdf',)) -pd = nb_setup.setup_pandas() -sym = nb_setup.setup_sympy() -import numpy as np -from IPython.display import Image, Latex -print("ok") -``` - -# Markdown - -Hallo .aljfnfa/dkfnadflkn - - -## General - - -Some markdown text. - -A list: - -- something -- something else - -A numbered list - -1. something -2. something else - -non-ascii characters TODO - - -This is a long section of text, which we only want in a document (not a presentation) -some text -some more text -some more text -some more text -some more text -some more text -some more text -some more text -some more text - - - -This is an abbreviated section of the document text, which we only want in a presentation - -- summary of document text - - -## References and Citations - - -References to \cref{fig:example}, \cref{tbl:example}, =@eqn:example_sympy and \cref{code:example_mpl}. - -A latex citation.\cite{zelenyak_molecular_2016} - -A html citation.(Kirkeminde, 2012) - - -## Todo notes - - -\todo[inline]{an inline todo} - -Some text.\todo{a todo in the margins} - - -# Text Output - -```{python ipub={'text': {'format': {'backgroundcolor': '\\color{blue!10}'}}}} -print(""" -This is some printed text, -with a nicely formatted output. -""") -``` - -# Images and Figures - -```{python ipub={'figure': {'caption': 'A nice picture.', 'label': 'fig:example', 'placement': '!bh'}}} -Image(filename='_static/example.jpg',height=400) -``` - -## Displaying a plot with its code - - -```{python ipub={'code': {'asfloat': True, 'caption': 'The plotting code for a matplotlib figure (\cref{fig:example_mpl}).', 'label': 'code:example_mpl', 'widefigure': False}, 'figure': {'caption': 'A matplotlib figure, with the caption set in the markdowncell above the figure.', 'label': 'fig:example_mpl', 'widefigure': False}}} -plt.scatter(np.random.rand(10), np.random.rand(10), - label='data label') -plt.ylabel(r'a y label with latex $\alpha$') -plt.legend(); -``` - -# Tables (with pandas) - - -```{python ipub={'code': {'asfloat': True, 'caption': 'The plotting code for a pandas Dataframe table (\cref{tbl:example}).', 'label': 'code:example_pd', 'placement': 'H', 'widefigure': False}, 'table': {'alternate': 'gray!20', 'caption': 'An example of a table created with pandas dataframe.', 'label': 'tbl:example', 'placement': 'H'}}} -df = pd.DataFrame(np.random.rand(3,4),columns=['a','b','c','d']) -df.a = ['$\delta$','x','y'] -df.b = ['l','m','n'] -df.set_index(['a','b']) -df.round(3) -``` - -# Equations (with ipython or sympy) - -```{python ipub={'equation': {'label': 'eqn:example_ipy'}}} -Latex('$$ a = b+c $$') -``` - -```{python ipub={'code': {'asfloat': True, 'caption': 'The plotting code for a sympy equation (=@eqn:example_sympy).', 'label': 'code:example_sym', 'placement': 'H', 'widefigure': False}, 'equation': {'environment': 'equation', 'label': 'eqn:example_sympy'}}} -y = sym.Function('y') -n = sym.symbols(r'\alpha') -f = y(n)-2*y(n-1/sym.pi)-5*y(n-2) -sym.rsolve(f,y(n),[1,4]) -``` diff --git a/docs/source/markdown_cells.Rmd b/docs/source/markdown_cells.Rmd index 786c56e..3e7ccb0 100644 --- a/docs/source/markdown_cells.Rmd +++ b/docs/source/markdown_cells.Rmd @@ -25,6 +25,13 @@ jupyter: at_notation: true use_numref: true bibliography: _static/example.bib + bibglossary: + filepath: _static/other_glossary.bib + sphinx: + no_bib: false + no_glossary: false + bib_title: Bibliography + glossary_title: Glossary jupytext: metadata_filter: notebook: ipub @@ -284,23 +291,30 @@ $$2x &= 8 \\ 3x + 9y &= -12$$ {#eqn:example2 env=align} Labelled math must be 'display math', rather than inline, i.e. wrapped in double dollars. +.. _markdown_references: + ### References -Pandoc references are start with @ and multiple references can be wrapped in square brackets: +Pandoc references are denoted by ``@``, with an optional prefix, and multiple references can be wrapped in square brackets: ```md Citation [@zelenyak_molecular_2016; @kirkeminde_thermodynamic_2012] + +Glossary Terms: %@gtkey2, &@akey2, &@symbol2 ``` Citation [@zelenyak_molecular_2016; @kirkeminde_thermodynamic_2012] +Glossary Terms: %@gtkey2, &@akey2, &@symbol2 + .. note:: - Citation in sphinx are provided by the excellent - [sphinxcontrib-bibtex extension](https://sphinxcontrib-bibtex.readthedocs.io) + Citations in sphinx are provided by the excellent + [sphinxcontrib-bibtex extension](https://sphinxcontrib-bibtex.readthedocs.io), + and glossary referencing is provided by the equally good :ref:`sphinx_ext_gls` References can have attributes; `latex` (defining the LaTeX tag to use), -`rst` (defining the RST role to use) and class `.capital` (defining if HTML naming if capitalized) +`rst` (defining the RST role to use) and class `.capital` (defining if HTML naming is capitalized) ```md @fig:example {.capital latex=cref rst=numref}, [@eqnlabel;@eqn:example2] {latex=eqref rst=eq} @@ -317,6 +331,8 @@ prefix attribute "!" {latex=ref rst=ref} "=" {latex=eqref rst=eq} "?" {.capital latex=Cref rst=numref} +"&" {latex=gls rst=gls} +"%" {.capital latex=Gls rst=glsc} Table: Prefixes for reference attributes {#tbl:prefixes} @@ -379,4 +395,5 @@ xyz

``` -Formatter filters, then look for these parent spans, to provide identifiers, classes and attributes. \ No newline at end of file +Formatter filters, then look for these parent spans, +to provide identifiers, classes and attributes. \ No newline at end of file diff --git a/docs/source/nb_conversion.rst b/docs/source/nb_conversion.rst index d80352d..26bf0bd 100644 --- a/docs/source/nb_conversion.rst +++ b/docs/source/nb_conversion.rst @@ -64,7 +64,7 @@ Some of note are: sphinx_ipypublish_main.run The same as sphinx_ipypublish_main, but also creates a conf.py file and runs `sphinx-build `_, - to create HTML documentation. + to create HTML documentation (see :ref:`sphinx_extensions`). html_ipypublish_main converts the entire notebook(s) to HTML and adds a table of contents @@ -92,6 +92,8 @@ Variants ending **.exec** will additionally execute the entire notebook ``pip install ipypublish[sphinx]`` + These are already included in the conda install. + A Note on PDF Conversion ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/releases.rst b/docs/source/releases.rst index d43ecfc..b92bfbb 100644 --- a/docs/source/releases.rst +++ b/docs/source/releases.rst @@ -10,6 +10,50 @@ Releases but anyone using custom converter plugins will be required to update them (see :ref:`convert_from_old_api`) +Version 0.10 +------------ + +v0.10.0 +~~~~~~~ + +- Added Sphinx extension for glossary referencing: ``ipypublish.sphinx.gls``. + See :ref:`sphinx_ext_gls` + +- Added ``ConvertBibGloss`` post-processor, + to convert a bibglossary to the required format + +- Added notebook-level metadata options for ``bibglossary`` and ``sphinx`` + (see :ref:`meta_doclevel_schema`) + +- Large refactoring and improvements for test suite, particularly for testing + of Sphinx extensions (using the Sphinx pytest fixtures) and creation of the + ``IpyTestApp`` fixture + +- fixed `tornado version restriction `_ + +Back-compatibility breaking changes: + +- renamed Sphinx notebook extension from + ``ipypublish.ipysphinx`` to ``ipypublish.sphinx.notebook`` + (see :ref:`sphinx_ext_notebook`) + +- :py:meth:`ipypublish.postprocessors.base.IPyPostProcessor.run_postprocess` + input signature changed (and consequently it has changes for all post-processors) + +v0.9 + +.. code-block:: python + + def run_postprocess(self, stream, filepath, resources): + output_folder = filepath.parent + +v0.10 + +.. code-block:: python + + def run_postprocess(self, stream, mimetype, filepath, resources): + output_folder = filepath.parent + Version 0.9 ----------- @@ -43,7 +87,7 @@ v0.9.0 - Refactored conversion process to :py:class:`ipypublish.convert.main.IpyPubMain` configurable class - Added postprocessors (see :ref:`post-processors`) -- Added Sphinx extension (see :ref:`sphinx_extension`) +- Added Sphinx notebook extension (see :ref:`sphinx_extensions`) - Added Binder examples to documentation (see :ref:`code_cells`) Version 0.8 diff --git a/docs/source/sphinx_ext_bibgloss.rst b/docs/source/sphinx_ext_bibgloss.rst new file mode 100644 index 0000000..bddd2e2 --- /dev/null +++ b/docs/source/sphinx_ext_bibgloss.rst @@ -0,0 +1,307 @@ +.. _sphinx_ext_gls: + +ipypublish.sphinx.gls +===================== + +:py:mod:`ipypublish.sphinx.gls` is adapted from +`sphinxcontrib-bibtex `_, +to provide a +`sphinx extension `_ +that closely replicates the functionality of the +`LaTeX glossaries `_ package, +for referencing glossaries terms, acronyms and symbols +provided in a separate file. + +This extension loads: + +- The ``bibglossary`` directive, for setting a file containing terms + (see :py:mod:`ipypublish.sphinx.gls.directives`) +- ``gls``, ``glsc``, ``glspl``, ``glscpl`` roles, for referencing the terms + (see :py:mod:`ipypublish.sphinx.gls.roles`) + +.. seealso:: + + :ref:`RMarkdown References `, + for an example of using this extension + within the conversion of a Jupyter Notebook. + +Installation +------------ + +Install ipypublish: + +.. code-block:: console + + conda install "ipypublish>=0.10" + +or + +.. code-block:: console + + pip install "ipypublish[sphinx]>=0.10" + +The key addition to the sphinx configuration file (conf.py) is: + +.. code-block:: python + + extensions = [ + 'ipypublish.sphinx.gls' + ] + +Also ``sphinx.ext.mathjax`` is recommended for rendering math. + +The Glossary File Formats +------------------------- + +The glossary file can be in one of two formats; +a .bib file (recommended) or a .tex file. + +The .bib file is a standard `BibTeX `_ file +(which is initially parsed by +`bibtexparser `_), and should contain +the following custom entry types: + +.. example taken from tests/sourcedirs/bibgloss_sortkeys + +**glossary term** (``name`` and ``description`` are required fields): + +.. code-block:: bibtex + + @glsterm{gtkey1, + name = {name}, + description = {full description \textbf{with latex}}, + plural = {some names}, + text = {alternative text}, + sort = {a}, + symbol = {\ensuremath{n}} + } + +**acronym** (``abbreviation`` and ``longname`` are required fields): + +.. code-block:: bibtex + + @glsacronym{akey1, + abbreviation = {MA}, + longname = {My Abbreviation}, + description = {full description}, + plural = {MAs}, + longplural = {Some Abbreviations} + } + +**symbol** (``name`` and ``description`` are required fields): + +.. code-block:: bibtex + + @glssymbol{symbol1, + name = {\ensuremath{\pi}}, + description = {full description}, + plural = {\ensuremath{\pi}s}, + text = {alternative text}, + sort = {sortkey} + } + +Alternatively, the glossary can be supplied as a TeX file: + +.. code-block:: tex + + \newglossaryentry{gtkey1}{ + name={name}, + description={full description \textbf{with latex}} + plural={names}, + text={alternative text}, + sort={a}, + symbol = {\ensuremath{n}} + } + \newacronym[plural={AAs}]{akey1}{AA}{An Abbreviation} + \newglossaryentry{symbol1}{ + name={\ensuremath{\pi}}, + description={full description}, + plural={\ensuremath{\pi}s}, + text={alternative text}, + sort={b}, + type={symbols} + } + +.. note:: + + To parse a glossary in TeX format, the + `TexSoup `_ package is required. + + ``newglossaryentry`` with ``type={symbols}`` are considered to be symbols, + no other types are recognised. + +.. attention:: + + All labels and description text are converted from latex to rst + by `Pandoc `_, then rst to docutils, + before being output to the final document. + + To skip this conversion (and only output as plain text) set + the sphinx configuration variable ``bibgloss_convert_latex = False`` + +.. seealso:: + + The `LaTeX/Glossary guide `_, + for further description of each field. + + +Usage +----- + +.. rst:role:: gls + + The ``gls`` role will output the 'name' or 'abbreviation' field of the entry: + + .. code-block:: rst + + :gls:`gtkey1`, :gls:`akey1`, :gls:`symbol1` + + :gls:`gtkey1`, :gls:`akey1`, :gls:`symbol1` + +.. rst:role:: glspl + + The ``glspl`` role will output the 'plural' field of the entry, + or (if not present) will append an 's' to the 'name' or 'abbreviation' field. + + .. code-block:: rst + + :glspl:`gtkey1`, :glspl:`akey1`, :glspl:`symbol1` + + :glspl:`gtkey1`, :glspl:`akey1`, :glspl:`symbol1` + +.. rst:role:: glsc + + The ``glsc`` and ``glscpl`` capitalise the respective labels. + + .. code-block:: rst + + :glsc:`gtkey1`, :glscpl:`gtkey1` + + :glsc:`gtkey1`, :glscpl:`gtkey1` + +.. rst:directive:: .. bibglossary:: path/to/glossary + + When creating the glossary, it is of note that, if the file extension is not + given, then ``bibglossary`` will attempt to find the best match + in the parent folder. The glossary will be sorted by lower case, + 'name'/'abbreviation' field, or 'sort' field (if it exists). + + .. code-block:: rst + + .. rubric:: Glossary + + .. bibglossary:: _static/example_glossary + + .. rubric:: Glossary + + .. bibglossary:: _static/example_glossary + + In order to use multiple glossaries, across one or more files, and avoid + hyperlink clashes, it is possible to set a ``keyprefix`` to distinguish + which glossary is being referenced. + + .. code-block:: rst + + :gls:`a-gtkey1` + + .. bibglossary:: _static/example_glossary + :keyprefix: a- + + :gls:`a-gtkey1` + + .. bibglossary:: _static/example_glossary + :keyprefix: a- + + Additional options include: + + - ``encoding`` to specify the encoding of the glossary file + - ``unsorted`` to sort the glossary by order of first use (rather than alphanumerically) + - ``all`` to output all glossary terms (including unused) + + .. code-block:: rst + + :gls:`b-akey1` + + .. bibglossary:: _static/example_glossary + :encoding: utf8 + :unsorted: + :keyprefix: b- + + :gls:`b-akey1`, :glsc:`b-gtkey1` + + .. bibglossary:: _static/example_glossary + :encoding: utf8 + :unsorted: + :keyprefix: b- + +.. seealso:: + + Additional options, known issues, and workarounds can be found in the + `sphinxcontrib-bibtex `_, + documentation. + +Python API +---------- + +The loading, conversion and storage of each glossary file is handled by a +:py:class:`~ipypublish.bib2glossary.classes.BibGlossDB` instance. + +.. nbinput:: python + :execution-count: 1 + + from ipypublish.bib2glossary import BibGlossDB + bibdb = BibGlossDB() + bibdb.load("_static/example_glossary") + len(bibdb) + +.. nboutput:: + :execution-count: 1 + + 3 + +This class is a subclass of :py:class:`collections.abc.MutableMapping`, +and so can be used as a dictionary. + +.. nbinput:: python + :execution-count: 2 + + print("gtkey1" in bibdb) + entry = bibdb["gtkey1"] + entry + +.. nboutput:: + :execution-count: 2 + + True + BibGlossEntry(key=gtkey1,label=name) + +Entries have attributes for the main fields, and can output to latex. + +.. nbinput:: python + :execution-count: 3 + + print(entry.key) + print(entry.label) + print(entry.plural) + print(entry.to_latex()) + +.. nboutput:: + :execution-count: 3 + + gtkey1 + name + some names + \newglossaryentry{gtkey1}{ + description={full description \textbf{with latex}}, + name={name}, + plural={some names}, + sort={a}, + symbol={\ensuremath{n}}, + text={alternative text} + } + + + + + + diff --git a/docs/source/sphinx_ext_notebook.rst b/docs/source/sphinx_ext_notebook.rst new file mode 100644 index 0000000..8a90253 --- /dev/null +++ b/docs/source/sphinx_ext_notebook.rst @@ -0,0 +1,158 @@ +.. _sphinx_ext_notebook: + +ipypublish.sphinx.notebook +========================== + +:py:mod:`ipypublish.sphinx.notebook` is adapted from +`nbshinx `_, to provide a +`sphinx extension `_ +for converting notebooks with :py:class:`ipypublish.convert.main.IpyPubMain`. + +This website is built using it, +and a good example its use would be to look at the +`ipypublish/docs/source/conf.py `_. + +This extension loads: + +- ``nbinput``, ``nboutput``, ``nbinfo`` and ``nbwarning`` directives, + that *wrap* input and output notebook cells + (see :py:mod:`ipypublish.sphinx.notebook.directives`) +- a source parser for files with the ``.ipynb`` extension + (see :py:class:`ipypublish.sphinx.notebook.parser.NBParser`) + +Usage +----- + +The key addition to the sphinx configuration file (conf.py) is: + +.. code-block:: python + + extensions = [ + 'ipypublish.sphinx.notebook' + ] + +The extension is also pre-configured to convert .Rmd files, +using `jupytext `_ (see :ref:`markdown_cells`). +To turn this feature on (for sphinx>=1.8): + +.. code-block:: python + + source_suffix = { + '.rst': 'restructuredtext', + '.ipynb': 'jupyter_notebook', + '.Rmd': 'jupyter_notebook' + } + +or for sphinx<1.8: + +.. code-block:: python + + source_parsers = { + '.Rmd': 'ipypublish.sphinx.notebook.parser.NBParser' + } + +.. important:: + + To use the sphinx extension, + IPyPublish must be installed with the sphinx extras: + + ``pip install ipypublish[sphinx]`` + + or the conda install already contains these extras. + +.. tip:: + + To convert a notebook directly to HTML *via* sphinx, + you can run: + + ``nbpublish -f sphinx_ipypublish_main.run notebook.ipynb`` + + This will convert the notebook to .rst, create a basic conf.py file + (including the ipypublish extensions), and + call `sphinx-build `_. + +Configuration +------------- + +Additional configuration can be added, +as described in :numref:`tbl:sphinx_config`, and numbered figures etc can be +setup by adding to the conf.py: + +.. code-block:: python + + numfig = True + math_numfig = True + numfig_secnum_depth = 2 + numfig_format: {'section': 'Section %s', + 'figure': 'Fig. %s', + 'table': 'Table %s', + 'code-block': 'Code Block %s'} + math_number_all = True + + mathjax_config = { + 'TeX': {'equationNumbers': {'autoNumber': 'AMS', 'useLabelIds': True}}, + } + +.. important:: + + To number items, the initial toctree must include the ``:numbered:`` option + +.. table:: Configuration values to use in conf.py + :name: tbl:sphinx_config + + ============================= =========================== ================================================================== + Name Default Description + ============================= =========================== ================================================================== + ipysphinx_export_config "sphinx_ipypublish_all.ext" ipypublish configuration file to use for conversion to .rst + ipysphinx_folder_suffix "_nbfiles" for dumping internal images, etc + ipysphinx_overwrite_existing False raise error if nb_name.rst already exists + ipysphinx_config_folders () additional folders containing ipypublish configuration files + ipysphinx_show_prompts False show cell prompts + ipysphinx_input_prompt "[%s]" format of input prompts + ipysphinx_output_prompt "[%s]" format of output prompts + ipysphinx_preconverters {} a mapping of additional file extensions to preconversion functions + ============================= =========================== ================================================================== + +Examples +-------- + +.. code-block:: rst + + .. nbinput:: python + :execution-count: 2 + :caption: A caption for the code cell + :name: ref_label + + print("hallo") + +.. nbinput:: python + :execution-count: 2 + :caption: A caption for the code cell + :name: ref_label + + print("hallo") + +.. code-block:: rst + + .. nboutput:: + :execution-count: 2 + + hallo + +.. nboutput:: + :execution-count: 2 + + hallo + +.. code-block:: rst + + .. nbinfo:: Some information + +.. nbinfo:: Some information + +.. code-block:: rst + + .. nbwarning:: This is a warning + +.. nbwarning:: This is a warning + diff --git a/docs/source/sphinx_extension.rst b/docs/source/sphinx_extension.rst deleted file mode 100644 index c5e8529..0000000 --- a/docs/source/sphinx_extension.rst +++ /dev/null @@ -1,67 +0,0 @@ -.. _sphinx_extension: - -Sphinx Extension -================ - -:py:mod:`ipypublish.ipysphinx` is adapted from -`nbshinx `_, to provide a -`sphinx extension `_ -for converting notebooks with :py:class:`ipypublish.convert.main.IpyPubMain`. - -This entire website is built using it, -and a good example of using it would be to look at its conf.py. - -The key addition to the configuration file (conf.py) is: - -.. code-block:: python - - extensions = [ - 'sphinx.ext.mathjax', - 'ipypublish.ipysphinx', - 'sphinxcontrib.bibtex' - ] - -.. important:: - - To use the sphinx extension, - IPyPublish must be installed with the sphinx extras: - - ``pip install ipypublish[sphinx]`` - -Additional configuration can be added, -as described in :numref:`tbl:sphinx_config`, and numbered figures etc can be -setup by adding: - -.. code-block:: python - - numfig = True - math_numfig = True - numfig_secnum_depth = 2 - numfig_format: {'section': 'Section %s', - 'figure': 'Fig. %s', - 'table': 'Table %s', - 'code-block': 'Code Block %s'} - math_number_all = True - - mathjax_config = { - 'TeX': {'equationNumbers': {'autoNumber': 'AMS', 'useLabelIds': True}}, - } - -.. important:: - - To number items, the initial toctree must include the ``:numbered:`` option - -.. table:: Configuration values to use in conf.py - :name: tbl:sphinx_config - - ============================= =========================== ================================================ - Name Default Description - ============================= =========================== ================================================ - ipysphinx_export_config "sphinx_ipypublish_all.ext" ipypublish configuration file - ipysphinx_folder_suffix "_nbfiles" for dumping internal images, etc - ipysphinx_overwrite_existing False raise error if nb_name.rst already exists - ipysphinx_config_folders () additional folders containing conversion files - ipysphinx_show_prompts False show cell prompts - ipysphinx_input_prompt "[%s]" format of input prompts - ipysphinx_output_prompt "[%s]" format of output prompts - ============================= =========================== ================================================ diff --git a/docs/source/sphinx_extensions.rst b/docs/source/sphinx_extensions.rst new file mode 100644 index 0000000..202d40b --- /dev/null +++ b/docs/source/sphinx_extensions.rst @@ -0,0 +1,24 @@ +.. _sphinx_extensions: + +Sphinx Extensions +================= + +IPyPublish packages a number of `sphinx `_ +extensions which are used to convert notebooks to (primarily) HTML. + +.. tip:: + + To convert a notebook directly to HTML *via* sphinx, + you can run: + + ``nbpublish -f sphinx_ipypublish_main.run notebook.ipynb`` + + This will convert the notebook to .rst, create a basic conf.py file + (including the ipypublish extensions), and + call `sphinx-build `_. + +.. toctree:: + :maxdepth: 2 + + sphinx_ext_notebook + sphinx_ext_bibgloss \ No newline at end of file diff --git a/ipypublish/__init__.py b/ipypublish/__init__.py index 72fd147..d87d33d 100644 --- a/ipypublish/__init__.py +++ b/ipypublish/__init__.py @@ -1,3 +1,3 @@ from ipypublish.scripts import nb_setup # noqa: F401 -__version__ = '0.9.4' +__version__ = '0.10.0' diff --git a/ipypublish/bib2glossary/__init__.py b/ipypublish/bib2glossary/__init__.py new file mode 100644 index 0000000..33475f6 --- /dev/null +++ b/ipypublish/bib2glossary/__init__.py @@ -0,0 +1,2 @@ +from ipypublish.bib2glossary.classes import ( # noqa: F401 + BibGlossEntry, BibGlossDB) diff --git a/ipypublish/bib2glossary/classes.py b/ipypublish/bib2glossary/classes.py new file mode 100644 index 0000000..920f19b --- /dev/null +++ b/ipypublish/bib2glossary/classes.py @@ -0,0 +1,381 @@ +import copy +import io +import logging +import os + +import bibtexparser + +from ipypublish.bib2glossary.definitions import ( + ETYPE_GLOSS, ETYPE_ACRONYM, ETYPE_SYMBOL, + NEWGLOSS_FIELDS, NEWACRONYM_FIELDS +) + +try: + from collections.abc import MutableMapping +except ImportError: + from collections import MutableMapping + +logger = logging.getLogger(__name__) + + +class BibGlossEntry(object): + _allowed_types = ( + ETYPE_GLOSS, ETYPE_ACRONYM, ETYPE_SYMBOL + ) + + def __init__(self, entry_dict): + + self._validate_dict(entry_dict) + self._entry_dict = entry_dict + + def _validate_dict(self, dct): + if 'ID' not in dct: + raise KeyError + if 'ENTRYTYPE' not in dct: + raise KeyError + + if dct['ENTRYTYPE'] not in self._allowed_types: + raise TypeError( + 'ENTRYTYPE must be one of: {}'.format(self._allowed_types)) + + if dct['ENTRYTYPE'] == ETYPE_ACRONYM: + if 'abbreviation' not in dct or 'longname' not in dct: + raise KeyError + elif (dct['ENTRYTYPE'] == ETYPE_GLOSS + or dct['ENTRYTYPE'] == ETYPE_SYMBOL): + if 'name' not in dct or 'description' not in dct: + raise KeyError + + def _get_key(self): + return self._entry_dict['ID'] + + def _set_key(self, key): + self._entry_dict['ID'] = key + + key = property(_get_key, _set_key) + + @property + def type(self): + return self._entry_dict['ENTRYTYPE'] + + def __contains__(self, key): + return key in self._entry_dict + + def get(self, key): + return self._entry_dict[key] + + @property + def label(self): + if self.type == ETYPE_ACRONYM: + return self.get('abbreviation') + elif self.type == ETYPE_GLOSS: + return self.get('name') + elif self.type == ETYPE_SYMBOL: + return self.get('name') + else: + raise NotImplementedError + + @property + def sortkey(self): + if "sort" in self: + return self.get("sort") + else: + return self.label.lower() + + @property + def plural(self): + if 'plural' in self: + return self.get('plural') + else: + return "{}s".format(self.label) + + @property + def text(self): + if self.type == ETYPE_ACRONYM: + return self.get('longname') + elif self.type == ETYPE_GLOSS: + return self.get('description') + elif self.type == ETYPE_SYMBOL: + return self.get('description') + else: + raise NotImplementedError + + def __repr__(self): + return "BibGlossEntry(key={0},label={1})".format(self.key, self.label) + + def to_dict(self): + return copy.deepcopy(self._entry_dict) + + def to_latex(self): + + if self.type in [ETYPE_GLOSS, ETYPE_SYMBOL]: + options = [] + for field in sorted(NEWGLOSS_FIELDS): + if field in self: + options.append("{0}={{{1}}}".format( + field, self.get(field))) + if self.type == ETYPE_SYMBOL: + options.append("type={symbols}") + body = "{{{key}}}{{\n {options}\n}}".format( + key=self.key, options=",\n ".join(options)) + return "\\newglossaryentry" + body + + elif self.type == ETYPE_ACRONYM: + body = "{{{key}}}{{{abbrev}}}{{{long}}}".format( + key=self.key, abbrev=self.label, long=self.text) + options = [] + for field in sorted(NEWACRONYM_FIELDS): + if field in self: + options.append("{0}={{{1}}}".format( + field, self.get(field))) + if options: + body = "[" + ",".join(options) + "]" + body + + return "\\newacronym" + body + + +class BibGlossDB(MutableMapping): + + def __init__(self): + self._entries = {} + + def __getitem__(self, key): + return self._entries[key] + + def __setitem__(self, key, entry): + if not isinstance(entry, BibGlossEntry): + raise ValueError('value must be a BibGlossEntry') + if key != entry.key: + raise ValueError('key must equal entry.key') + self._entries[key] = entry + + def __delitem__(self, key): + del self._entries[key] + + def __iter__(self): + return iter(self._entries) + + def __len__(self): + return len(self._entries) + + @staticmethod + def get_fake_entry_obj(key): + return BibGlossEntry({ + 'ENTRYTYPE': ETYPE_GLOSS, + 'ID': key, + 'name': key, + 'description': '' + }) + + def load_bib(self, text_str=None, path=None, bibdb=None, encoding="utf8", + ignore_nongloss_types=False, + ignore_duplicates=False): + """load a bib file + + Parameters + ---------- + text_str=None: str or None + string representing the bib file contents + path=None: str or None + path to bibfile + bibdb=None: bibtexparser.bibdatabase.BibDatabase or None + encoding="utf8": str + bib file encoding + ignore_nongloss_types: bool + if False, a KeyError will be raised for non-gloss types + ignore_duplicates: bool + if False, a KeyError will be raised if multiple entries are found + with the same key, otherwise only the first entry will be used + + """ + bib = None + + if sum([e is not None for e in [text_str, path, bibdb]]) != 1: + raise ValueError( + "only one of text_str, path or bib must be supplied") + if bibdb is not None: + if not isinstance(bibdb, bibtexparser.bibdatabase.BibDatabase): + raise ValueError("bib is not a BibDatabase instance") + bib = bibdb + elif path is not None: + if text_str is not None: + raise ValueError( + 'text_str and path cannot be set at the same time') + with io.open(path, encoding=encoding) as fobj: + text_str = fobj.read() + + if bib is None: + parser = bibtexparser.bparser.BibTexParser() + parser.ignore_nonstandard_types = False + parser.encoding = encoding + bib = parser.parse(text_str) + # TODO doesn't appear to check for key duplication + # see https://github.com/sciunto-org/python-bibtexparser/issues/237 + + entries = {} + for entry_dict in bib.entries: + + try: + entry = BibGlossEntry(entry_dict) + except TypeError: + if ignore_nongloss_types: + logger.warning('Skipping non-glossary entry') + continue + else: + raise + + if entry.key in entries: + if ignore_duplicates: + logger.warning('Skipping duplicate key entry') + continue + else: + raise KeyError( + "the bib file contains " + "multiple entries with the key: {}".format(entry.key)) + + entries[entry.key] = entry + + # self._bib = bib + self._entries = entries + + return True + + def load_tex(self, text_str=None, path=None, encoding='utf8', + skip_ioerrors=False, ignore_unknown_types=True): + """load a tex file + + Parameters + ---------- + text_str=None: str or None + string representing the bib file contents + path=None: str or None + path to bibfile + bibdb=None: bibtexparser.bibdatabase.BibDatabase or None + encoding="utf8": str + bib file encoding + skip_ioerrors: bool + if False, an IOError will be raised if + newglossaryterm or newacronym is badly formatted + ignore_unknown_types: bool + if True, strip unknown types, otherwise raise a ValueError + + Notes + ----- + the texsoup package is required. + + if a newglossaryterm has field 'type={symbols}', then + it will be loaded as a symbol + + """ + from ipypublish.bib2glossary.parse_tex import parse_tex + gterms, acronyms = parse_tex( + text_str=text_str, path=path, encoding=encoding, + skip_ioerrors=skip_ioerrors + ) + entries = {} + for key, fields in gterms.items(): + + fields["ENTRYTYPE"] = ETYPE_GLOSS + if fields.get("type", None) == "symbols": + fields["ENTRYTYPE"] = ETYPE_SYMBOL + fields.pop("type") + elif "type" in fields: + if not ignore_unknown_types: + raise ValueError("the 'type' is not recognised: " + "{}".format(fields['type'])) + fields.pop("type") + + fields["ID"] = key + entry = BibGlossEntry(fields) + entries[entry.key] = entry + + for key, fields in acronyms.items(): + fields["ENTRYTYPE"] = ETYPE_ACRONYM + fields["ID"] = key + entry = BibGlossEntry(fields) + entries[entry.key] = entry + + self._entries = entries + + return True + + @staticmethod + def guess_path(path): + """ guess the path of a bib file, with or without a file extension, + from the available files in the path folder + """ + basepath, extension = os.path.splitext(str(path)) + if extension in [".bib", ".biblatex", ".bibtex"]: + return path + elif extension in [".tex", ".latex"]: + return path + elif os.path.exists(basepath + ".bib"): + return basepath + ".bib" + elif os.path.exists(basepath + ".bibtex"): + return basepath + ".bibtex" + elif os.path.exists(basepath + ".biblatex"): + return basepath + ".biblatex" + elif os.path.exists(basepath + ".tex"): + return basepath + ".tex" + elif os.path.exists(basepath + ".latex"): + return basepath + ".latex" + else: + return None + + def load(self, path, encoding='utf8'): + """load a file, the type will be guessed from the extension, + or (if no extension is given), the available files in the path folder + + Parameters + ---------- + path: str + encoding='utf8': str + encoding of the file + + """ + path = self.guess_path(path) + if path is None: + raise IOError( + "no acceptable loader found for path: {}".format(path)) + basepath, extension = os.path.splitext(str(path)) + if extension in [".bib", ".biblatex", ".bibtex"]: + self.load_bib(path=path, encoding=encoding) + elif extension in [".tex", ".latex"]: + self.load_tex(path=path, encoding=encoding) + + def to_dict(self): + return {k: e.to_dict() for k, e in self.items()} + + def to_bib_string(self): + bibdb = bibtexparser.bibdatabase.BibDatabase() + bibdb.entries = [e.to_dict() for e in self.values()] + writer = bibtexparser.bwriter.BibTexWriter() + writer.contents = ['comments', 'entries'] + writer.indent = ' ' + # writer.order_entries_by = ('ENTRYTYPE', 'ID') + return writer.write(bibdb) + + def to_latex_dict(self, splitlines=True): + """convert to dict of latex strings + + Returns + ------- + dict: + {(, ): } + + """ + latex_stings = {} + for entry in self.values(): + string = entry.to_latex() + if splitlines: + string = string.splitlines() + latex_stings[ + (entry.type, entry.key)] = string + return latex_stings + + def to_latex_string(self): + lines = [] + latex_dict = self.to_latex_dict(splitlines=False) + for key in sorted(list(latex_dict.keys())): + lines.append(latex_dict[key]) + return "\n".join(lines) diff --git a/ipypublish/bib2glossary/definitions.py b/ipypublish/bib2glossary/definitions.py new file mode 100644 index 0000000..924ee8b --- /dev/null +++ b/ipypublish/bib2glossary/definitions.py @@ -0,0 +1,28 @@ +ETYPE_GLOSS = 'glsterm' +ETYPE_ACRONYM = 'glsacronym' +ETYPE_SYMBOL = 'glssymbol' + +NEWGLOSS_FIELDS = ( + "name", "description", "plural", "symbol", "text", "sort" +) + +NEWACRONYM_FIELDS = ( + "description", "plural", "longplural", "firstplural" +) + + +# TODO allow mapping +# DEFAULT_GLOSS_P2F = (("name", "name"), +# ("description", "description"), +# ("plural", "plural"), +# ("symbol", "symbol"), +# ("text", "text"), +# ("sort", "sort")) + + +# DEFAULT_ACRONYM_P2F = (("abbreviation", "abbreviation"), +# ("longname", "longname"), +# ("description", "description"), +# ("plural", "plural"), +# ("longplural", "longplural"), +# ("firstplural", "firstplural")) diff --git a/ipypublish/bib2glossary/parse_tex.py b/ipypublish/bib2glossary/parse_tex.py new file mode 100644 index 0000000..e042004 --- /dev/null +++ b/ipypublish/bib2glossary/parse_tex.py @@ -0,0 +1,252 @@ +from collections import deque +import io +import logging + +logger = logging.getLogger(__name__) + + +def import_texsoup(): + try: + from TexSoup import TexSoup + from TexSoup.utils import TokenWithPosition + from TexSoup.data import RArg, OArg + except ImportError: + raise ImportError( + "to parse tex files, TexSoup must be installed: \n" + "pip install texsoup\n" + "conda install -c conda-forge texsoup") + except SyntaxError: + raise ImportError('TexSoup package is broken on python 2.7, ' + 'so can not be imported for tex parsing') + return { + "TexSoup": TexSoup, + "RArg": RArg, + "OArg": OArg, + "TokenWithPosition": TokenWithPosition + } + + +def _create_msg_error(msg, node=None, row=None): + """create error message, optionally including TexNode and row""" + text = msg.strip() + if row is not None: + text = "(row {}) ".format(row) + text + if hasattr(node, "name"): + text = text + ": {}".format(node.name) + return text + + +def extract_required_val(rarg): + """extract the value of a TexSoup RArg""" + RArg = import_texsoup()["RArg"] + if not isinstance(rarg, RArg): + raise ValueError( + "expected {} to be a required argument".format(type(rarg))) + return rarg.value + + +def _extract_parameters(texsoup_exprs): + """extract the parameters from a TexSoup expression list""" + RArg = import_texsoup()["RArg"] + TokenWithPosition = import_texsoup()["TokenWithPosition"] + expressions = deque(texsoup_exprs) + param_name = None + params = {} + errors = [] + while expressions: + expr = expressions.popleft() + if isinstance(expr, TokenWithPosition): + # TODO is this the best way to extract parameter name? + param_name = expr.text.replace(",", "").replace("=", "").strip() + elif isinstance(expr, RArg): + if param_name is None: + errors.append( + "expected expression " + "'{}' to precede a parameter name".format(expr)) + break + if param_name in params: + errors.append( + "parameter '{}' already defined".format(param_name)) + else: + params[param_name] = expr.value + param_name = None + else: + errors.append( + "expected expression '{}' ".format(expr) + + "to be a parameter name or required argument") + break + + if param_name is not None: + pass # allowed since last expr may be new line + # errors.append( + # "parameter '{}' is not assigned a value".format(param_name)) + + return params, errors + + +def extract_parameters(argument): + """extract parameters from a TexSoup OArg or Arg""" + RArg = import_texsoup()["RArg"] + OArg = import_texsoup()["OArg"] + if not isinstance(argument, (OArg, RArg)): + raise ValueError( + "expected {} to be of type OArg or RArg".format(type(argument))) + + opt_params, errors = _extract_parameters(argument.exprs) + + return opt_params, errors + + +def create_newgloss_dict(gterm, row=None): + """ + """ + arguments = list(gterm.args) + fields = {} + + if len(arguments) != 2: + msg = _create_msg_error( + "could not parse newglossaryterm (arguments != 2)", gterm, row) + raise IOError(msg) + + key = extract_required_val(arguments[0]) + + params, errors = extract_parameters(arguments[1]) + + for error in errors: + msg = _create_msg_error( + "error reading 'parameter' block: {}".format(error), + gterm, row) + raise IOError(msg) + + for param_name, param_value in params.items(): + + if param_name in fields: + raise IOError( + "duplicate parameter '{0}' in key '{1}'".format( + param_name, key)) + + fields[param_name] = param_value + + return key, fields + + +def create_newacronym_dict(acronym, row=None): + """ + """ + OArg = import_texsoup()["OArg"] + + arguments = list(acronym.args) + fields = {} + + if len(arguments) < 3: + msg = _create_msg_error( + "could not parse newacronym (too few arguments)", acronym, row) + raise IOError(msg) + if len(arguments) > 4: + msg = _create_msg_error( + "could not parse newacronym (too many arguments)", acronym, row) + raise IOError(msg) + + key = extract_required_val(arguments[-3]) + abbreviation = extract_required_val(arguments[-2]) + name = extract_required_val(arguments[-1]) + + if len(arguments) == 4: + options = arguments[0] + + if not isinstance(options, OArg): + msg = _create_msg_error( + "expected first argument of newacronym to be 'optional", + acronym, row) + raise IOError(msg) + + opt_params, errors = extract_parameters(options) + + for error in errors: + msg = _create_msg_error( + "error reading newacronym 'optional' block: {}".format(error), + acronym, row) + raise IOError(msg) + + for opt_name, opt_value in opt_params.items(): + if opt_name in fields: + raise IOError( + "duplicate parameter '{0}' in key '{1}'".format( + opt_name, key)) + fields[opt_name] = opt_value + + return key, abbreviation, name, fields + + +def parse_tex(text_str=None, path=None, encoding='utf8', + abbrev_field="abbreviation", fname_field="longname", + skip_ioerrors=False): + """parse a tex file containing newglossaryentry and/or newacronym to dict + + Parameters + ---------- + text_str=None: str + string representing the tex file + path=None: str + path to the tex file + encoding='utf8': str + tex file encoding + abbrev_field="abbreviation": str + field key for acronym abbreviation + fname_field="longname": str + field key for acronym full name + skip_ioerrors=False: bool + skip errors on reading a single entry + + Returns + ------- + dict: glossaryterms + {key: fields} + dict: acronyms + {key: fields} + + """ + TexSoup = import_texsoup()["TexSoup"] + + if sum([e is not None for e in [text_str, path]]) != 1: + raise ValueError("only one of text_str or path must be supplied") + elif path is not None: + if text_str is not None: + raise ValueError( + 'text_str and path cannot be set at the same time') + with io.open(path, encoding=encoding) as fobj: + text_str = fobj.read() + + latex_tree = TexSoup(text_str) + + keys = [] + gterms = {} + acronyms = {} + + for gterm in latex_tree.find_all("newglossaryentry"): + try: + key, fields = create_newgloss_dict(gterm) + except IOError: + if skip_ioerrors: + continue + raise + if key in keys: + raise KeyError("duplicate key: {}".format(key)) + keys.append(key) + gterms[key] = fields + + for acronym in latex_tree.find_all("newacronym"): + try: + key, abbreviation, name, fields = create_newacronym_dict(acronym) + except IOError: + if skip_ioerrors: + continue + raise + if key in keys: + raise KeyError("duplicate key: {}".format(key)) + keys.append(key) + fields[abbrev_field] = abbreviation + fields[fname_field] = name + acronyms[key] = fields + + return gterms, acronyms diff --git a/ipypublish/bib2glossary/test_bib2gloss.py b/ipypublish/bib2glossary/test_bib2gloss.py new file mode 100644 index 0000000..6ad05c8 --- /dev/null +++ b/ipypublish/bib2glossary/test_bib2gloss.py @@ -0,0 +1,133 @@ +import re +import sys +from textwrap import dedent +import pytest +from ipypublish.bib2glossary import BibGlossDB + +bib_str = """\ + @glsterm{gtkey1, + description = {the description}, + name = {name} + } + @glsterm{gtkey2, + description = {the description of other}, + name = {other name} + } + @glsacronym{akey1, + abbreviation = {ABRV}, + longname = {Abbreviation}, + description = {a description} + } + @glsacronym{akey2, + abbreviation = {OTHER}, + longname = {Abbrev of other}, + plural = {OTHERs} + } + @glssymbol{skey1, + description = {the description of symbol}, + name = {\\pi} + } + @badtype{bkey1, + field = {text} + } + """ + +# TODO check for key duplication +# see https://github.com/sciunto-org/python-bibtexparser/issues/237 + +tex_str = """\ + \\newacronym[description={a description}]{akey1}{OTHER}{Abbreviation of other} + \\newglossaryentry{gtkey1}{ + name={other name}, + description={the description of other} + } + \\newglossaryentry{skey1}{ + name={name}, + description={the description}, + type={symbols} + } + """ # noqa: E501 + + +def test_load_bib_type_error(): + + bibgloss = BibGlossDB() + with pytest.raises(TypeError): + bibgloss.load_bib( + text_str=dedent(bib_str), ignore_nongloss_types=False) + + +def test_load_bib_type_ignore(): + + bibgloss = BibGlossDB() + bibgloss.load_bib(text_str=dedent(bib_str), ignore_nongloss_types=True) + assert set(bibgloss.keys()) == { + 'gtkey1', 'gtkey2', 'akey1', 'akey2', 'skey1'} + + +@pytest.mark.skipif( + sys.version_info < (3, 0), + reason="SyntaxError on import of texsoup/data.py line 135") +def test_load_tex(): + + bibgloss = BibGlossDB() + bibgloss.load_tex(text_str=dedent(tex_str)) + assert {k: e.type for k, e in bibgloss.items()} == { + 'gtkey1': 'glsterm', + 'akey1': 'glsacronym', + 'skey1': 'glssymbol'} + + +def test_to_dict(): + bibgloss = BibGlossDB() + bibgloss.load_bib(text_str=dedent(bib_str), ignore_nongloss_types=True) + dct = bibgloss.to_dict() + assert set(dct.keys()) == { + 'gtkey1', 'gtkey2', 'akey1', 'akey2', 'skey1'} + + +def test_to_bib_string(): + bibgloss = BibGlossDB() + bibgloss.load_bib(text_str=dedent(bib_str), ignore_nongloss_types=True) + string = bibgloss.to_bib_string() + assert re.search( + "@glsacronym\\{akey1,.*@glsterm\\{gtkey1,.*@glssymbol\\{skey1.*", + string, + re.DOTALL + ) + + +def test_to_latex_dict(): + bibgloss = BibGlossDB() + bibgloss.load_bib(text_str=dedent(bib_str), ignore_nongloss_types=True) + latex_dict = bibgloss.to_latex_dict() + print(latex_dict) + assert latex_dict == { + ('glsacronym', + 'akey1'): [( + '\\newacronym[description={a description}]{' + 'akey1}{ABRV}{Abbreviation}')], + ('glsacronym', + 'akey2'): [( + '\\newacronym[plural={OTHERs}]{' + 'akey2}{OTHER}{Abbrev of other}')], + ('glsterm', + 'gtkey1'): [ + '\\newglossaryentry{gtkey1}{', + ' description={the description},', + ' name={name}', + '}'], + ('glsterm', + 'gtkey2'): [ + '\\newglossaryentry{gtkey2}{', + ' description={the description of other},', + ' name={other name}', + '}'], + ('glssymbol', + 'skey1'): [ + '\\newglossaryentry{skey1}{', + ' description={the description of symbol},', + ' name={\\pi},', + ' type={symbols}', + '}'] + } diff --git a/ipypublish/bib2glossary/test_parse_tex.py b/ipypublish/bib2glossary/test_parse_tex.py new file mode 100644 index 0000000..34514ce --- /dev/null +++ b/ipypublish/bib2glossary/test_parse_tex.py @@ -0,0 +1,161 @@ +import sys +import pytest + +if sys.version_info >= (3, 0): + from ipypublish.bib2glossary.parse_tex import parse_tex + + +@pytest.mark.skipif( + sys.version_info < (3, 0), + reason="SyntaxError on import of texsoup/data.py line 135") +def test_parse_acronyms(): + + text_str = """ + \\newacronym{otherkey}{OTHER}{Abbreviation of other} + \\newacronym{thekey}{ABRV}{Abbreviation} + """ + gterms, acronyms = parse_tex(text_str=text_str) + assert gterms == {} + assert acronyms == { + 'otherkey': { + 'abbreviation': 'OTHER', + 'longname': 'Abbreviation of other'}, + 'thekey': { + 'abbreviation': 'ABRV', + 'longname': 'Abbreviation'} + } + + +@pytest.mark.skipif( + sys.version_info < (3, 0), + reason="SyntaxError on import of texsoup/data.py line 135") +def test_parse_acronyms_with_options(): + + text_str = """ + \\newacronym[description={a description}]{otherkey}{OTHER}{Abbreviation of other} + \\newacronym[plural={ABRVs},longplural={Abbreviations}]{thekey}{ABRV}{Abbreviation} + """ # noqa: E501 + gterms, acronyms = parse_tex(text_str=text_str) + assert gterms == {} + assert acronyms == { + 'otherkey': { + 'abbreviation': 'OTHER', + 'longname': 'Abbreviation of other', + 'description': 'a description'}, + 'thekey': { + 'abbreviation': 'ABRV', + 'longname': 'Abbreviation', + 'longplural': 'Abbreviations', + 'plural': 'ABRVs'} + } + + +@pytest.mark.skipif( + sys.version_info < (3, 0), + reason="SyntaxError on import of texsoup/data.py line 135") +def test_parse_gterms(): + + text_str = """ + \\newglossaryentry{otherkey}{ + name={other name}, + description={the description of other} + } + \\newglossaryentry{thekey}{ + name={name}, + description={the description}, + type={symbols} + } + """ + gterms, acronyms = parse_tex(text_str=text_str) + assert acronyms == {} + assert gterms == { + 'otherkey': { + 'description': 'the description of other', + 'name': 'other name'}, + 'thekey': { + 'description': 'the description', + 'name': 'name', + 'type': 'symbols'} + } + + +@pytest.mark.skipif( + sys.version_info < (3, 0), + reason="SyntaxError on import of texsoup/data.py line 135") +def test_parse_mixed(): + text_str = """ + \\newacronym{otherkey}{OTHER}{Abbreviation of other} + \\newglossaryentry{thekey}{ + name={name}, + description={the description}, + type={symbols} + } + """ + gterms, acronyms = parse_tex(text_str=text_str) + assert acronyms == { + 'otherkey': { + 'abbreviation': 'OTHER', + 'longname': 'Abbreviation of other'} + } + assert gterms == { + 'thekey': { + 'description': 'the description', + 'name': 'name', + 'type': 'symbols'} + } + + +@pytest.mark.skipif( + sys.version_info < (3, 0), + reason="SyntaxError on import of texsoup/data.py line 135") +def test_duplicate_key(): + text_str = """ + \\newacronym{thekey}{OTHER}{Abbreviation of other} + \\newglossaryentry{thekey}{ + name={name}, + description={the description}, + type={symbols} + } + """ + with pytest.raises(KeyError): + parse_tex(text_str=text_str) + + +@pytest.mark.skipif( + sys.version_info < (3, 0), + reason="SyntaxError on import of texsoup/data.py line 135") +def test_acronym_ioerror(): + text_str = """ + \\newacronym{thekey}{Abbreviation of other} + """ + with pytest.raises(IOError): + parse_tex(text_str=text_str) + + +@pytest.mark.skipif( + sys.version_info < (3, 0), + reason="SyntaxError on import of texsoup/data.py line 135") +def test_gterm_ioerror(): + text_str = """ + \\newglossaryentry{} + """ + with pytest.raises(IOError): + parse_tex(text_str=text_str) + + +@pytest.mark.skipif( + sys.version_info < (3, 0), + reason="SyntaxError on import of texsoup/data.py line 135") +def test_ioerror_skip(): + text_str = """ + \\newacronym{thekey}{Abbreviation of other} + \\newacronym{thekey2}{ABBR}{Abbreviation of other} + """ + gterms, acronyms = parse_tex(text_str=text_str, skip_ioerrors=True) + assert gterms == {} + assert acronyms == { + "thekey2": { + 'abbreviation': 'ABBR', + 'longname': 'Abbreviation of other' + } + } diff --git a/ipypublish/convert/config_manager.py b/ipypublish/convert/config_manager.py index 197859c..13309bb 100644 --- a/ipypublish/convert/config_manager.py +++ b/ipypublish/convert/config_manager.py @@ -6,6 +6,7 @@ from six import string_types from jinja2 import DictLoader import jsonschema +import nbconvert # noqa: F401 from ipypublish.utils import (pathlib, handle_error, get_module_path, read_file_from_directory, read_file_from_module) @@ -100,10 +101,11 @@ def create_exporter_cls(class_str): class_name = export_class_path[-1] try: export_module = importlib.import_module(module_path) - except ModuleNotFoundError: + except ModuleNotFoundError: # noqa: F821 handle_error( "module {} containing exporter class {} not found".format( - module_path, class_name), ModuleNotFoundError, logger=logger) + module_path, class_name), + ModuleNotFoundError, logger=logger) # noqa: F821 if hasattr(export_module, class_name): export_class = getattr(export_module, class_name) else: diff --git a/ipypublish/convert/main.py b/ipypublish/convert/main.py index 53dc3f2..62f9358 100755 --- a/ipypublish/convert/main.py +++ b/ipypublish/convert/main.py @@ -173,7 +173,7 @@ def _validate_pre_conversion_funcs(self, proposal): ).tag(config=True) default_ppconfig_kwargs = T.Dict( - trait=T.Bool, + trait=T.Bool(), default_value=( ('pdf_in_temp', False), ('pdf_debug', False), @@ -183,7 +183,7 @@ def _validate_pre_conversion_funcs(self, proposal): ).tag(config=True) default_pporder_kwargs = T.Dict( - trait=T.Bool, + trait=T.Bool(), default_value=( ('dry_run', False), ('clear_existing', False), @@ -226,6 +226,9 @@ def _create_default_ppconfig(self, pdf_in_temp=False, pdf_debug=False, }, "CopyResourcePaths": { "files_folder": "${files_path}" + }, + "ConvertBibGloss": { + "files_folder": "${files_path}" } }) @@ -245,7 +248,10 @@ def _create_default_pporder(self, dry_run=False, clear_existing=False, default_pprocs.append('write-text-file') if dump_files or create_pdf or serve_html: default_pprocs.extend( - ['write-resource-files', 'copy-resource-paths']) + [ + 'write-resource-files', + 'copy-resource-paths', + 'convert-bibgloss']) if create_pdf: default_pprocs.append('pdf-export') elif serve_html: @@ -343,8 +349,8 @@ def publish(self, ipynb_path, nb_node=None): if isinstance(ipynb_path, string_types): ipynb_path = pathlib.Path(ipynb_path) ipynb_name, ipynb_ext = os.path.splitext(ipynb_path.name) - outdir = os.path.join( - os.getcwd(), 'converted') if self.outpath is None else self.outpath + outdir = (os.path.join(os.getcwd(), 'converted') + if self.outpath is None else str(self.outpath)) self._setup_logger(ipynb_name, outdir) diff --git a/ipypublish/export_plugins/latex_ipypublish_all.exec.json b/ipypublish/export_plugins/latex_ipypublish_all.exec.json index 180676d..0321762 100644 --- a/ipypublish/export_plugins/latex_ipypublish_all.exec.json +++ b/ipypublish/export_plugins/latex_ipypublish_all.exec.json @@ -110,6 +110,10 @@ { "module": "ipypublish.templates.segments", "file": "ipy-contents_framed_code-new.yaml.tex.j2" + }, + { + "module": "ipypublish.templates.segments", + "file": "ipy-glossary.yaml.tex.j2" } ] } diff --git a/ipypublish/export_plugins/latex_ipypublish_all.json b/ipypublish/export_plugins/latex_ipypublish_all.json index ac731d2..b1ed41a 100644 --- a/ipypublish/export_plugins/latex_ipypublish_all.json +++ b/ipypublish/export_plugins/latex_ipypublish_all.json @@ -102,6 +102,10 @@ { "module": "ipypublish.templates.segments", "file": "ipy-contents_framed_code-new.yaml.tex.j2" + }, + { + "module": "ipypublish.templates.segments", + "file": "ipy-glossary.yaml.tex.j2" } ] } diff --git a/ipypublish/export_plugins/latex_ipypublish_main.json b/ipypublish/export_plugins/latex_ipypublish_main.json index 41043d0..ba1e698 100644 --- a/ipypublish/export_plugins/latex_ipypublish_main.json +++ b/ipypublish/export_plugins/latex_ipypublish_main.json @@ -71,6 +71,10 @@ { "module": "ipypublish.templates.segments", "file": "ipy-contents_framed_code-new.yaml.tex.j2" + }, + { + "module": "ipypublish.templates.segments", + "file": "ipy-glossary.yaml.tex.j2" } ] } diff --git a/ipypublish/export_plugins/latex_ipypublish_nocode.json b/ipypublish/export_plugins/latex_ipypublish_nocode.json index 9738f86..e9d2a0b 100644 --- a/ipypublish/export_plugins/latex_ipypublish_nocode.json +++ b/ipypublish/export_plugins/latex_ipypublish_nocode.json @@ -104,6 +104,10 @@ { "module": "ipypublish.templates.segments", "file": "ipy-contents_framed_code-new.yaml.tex.j2" + }, + { + "module": "ipypublish.templates.segments", + "file": "ipy-glossary.yaml.tex.j2" } ] } diff --git a/ipypublish/export_plugins/sphinx_ipypublish_all.exec.json b/ipypublish/export_plugins/sphinx_ipypublish_all.exec.json index 414426c..1d8fcc9 100644 --- a/ipypublish/export_plugins/sphinx_ipypublish_all.exec.json +++ b/ipypublish/export_plugins/sphinx_ipypublish_all.exec.json @@ -14,7 +14,8 @@ "choose_output_type": "ipypublish.filters.rst_choose_output.choose_output_type", "ipypandoc": "ipypublish.filters_pandoc.main.jinja_filter", "is_equation": "ipypublish.filters.filters.is_equation", - "basename": "ipypublish.filters.filters.basename" + "basename": "ipypublish.filters.filters.basename", + "strip_ext": "ipypublish.filters.filters.strip_ext" }, "preprocessors": [ { diff --git a/ipypublish/export_plugins/sphinx_ipypublish_all.ext.json b/ipypublish/export_plugins/sphinx_ipypublish_all.ext.json index 112cdb5..99f67f9 100644 --- a/ipypublish/export_plugins/sphinx_ipypublish_all.ext.json +++ b/ipypublish/export_plugins/sphinx_ipypublish_all.ext.json @@ -17,7 +17,8 @@ "choose_output_type": "ipypublish.filters.rst_choose_output.choose_output_type", "ipypandoc": "ipypublish.filters_pandoc.main.jinja_filter", "is_equation": "ipypublish.filters.filters.is_equation", - "basename": "ipypublish.filters.filters.basename" + "basename": "ipypublish.filters.filters.basename", + "strip_ext": "ipypublish.filters.filters.strip_ext" }, "preprocessors": [ { diff --git a/ipypublish/export_plugins/sphinx_ipypublish_all.ext.noexec.json b/ipypublish/export_plugins/sphinx_ipypublish_all.ext.noexec.json new file mode 100644 index 0000000..9909518 --- /dev/null +++ b/ipypublish/export_plugins/sphinx_ipypublish_all.ext.noexec.json @@ -0,0 +1,101 @@ +{ + "$schema": "../schema/export_config.schema.json", + "description": [ + "For use with the ipypublish.ipyshinx Sphinx extension (no initial notebook execution)", + "- unlike the, conventional version, this does not output a file, ", + " and does not redirect/copy external files to the files_folder", + "- execute the notebook", + "- only output cells with metadata tags are used", + "- code, figures, tables and code are formatted accordingly" + ], + "exporter": { + "class": "nbconvert.exporters.RSTExporter", + "filters": { + "get_empty_lines": "ipypublish.filters.filters.get_empty_lines", + "get_caption": "ipypublish.filters.filters.get_caption", + "wrap_eqn": "ipypublish.filters.filters.wrap_eqn", + "choose_output_type": "ipypublish.filters.rst_choose_output.choose_output_type", + "ipypandoc": "ipypublish.filters_pandoc.main.jinja_filter", + "is_equation": "ipypublish.filters.filters.is_equation", + "basename": "ipypublish.filters.filters.basename", + "strip_ext": "ipypublish.filters.filters.strip_ext" + }, + "preprocessors": [ + { + "class": "ipypublish.preprocessors.latex_doc_defaults.MetaDefaults", + "args": { + "cell_defaults": { + "ipub": { + "figure": { + "placement": "H" + }, + "table": { + "placement": "H" + }, + "equation": { + "environment": "equation" + }, + "text": true, + "mkdown": true, + "code": true, + "error": true + } + }, + "nb_defaults": { + "ipub": { + "titlepage": {}, + "toc": true, + "listfigures": true, + "listtables": true, + "listcode": true + } + } + } + }, + { + "class": "ipypublish.preprocessors.split_outputs.SplitOutputs", + "args": { + "split": true + } + }, + { + "class": "ipypublish.preprocessors.latex_doc_links.LatexDocLinks", + "args": { + "metapath": "${meta_path}", + "filesfolder": "${files_path}", + "redirect_external": false + } + }, + { + "class": "ipypublish.preprocessors.latex_doc_captions.LatexCaptions", + "args": {} + } + ], + "other_args": { + "ExecutePreprocessor": { + "enabled": false + } + } + }, + "template": { + "outline": { + "module": "ipypublish.templates.outline_schemas", + "file": "rst_outline.rst.j2" + }, + "segments": [ + { + "module": "ipypublish.templates.segments", + "file": "ipy-sphinx.yaml.j2" + } + ] + }, + "postprocessors": { + "order": [ + "remove-blank-lines", + "remove-trailing-space", + "filter-output-files", + "remove-folder", + "write-resource-files" + ] + } +} \ No newline at end of file diff --git a/ipypublish/export_plugins/sphinx_ipypublish_all.json b/ipypublish/export_plugins/sphinx_ipypublish_all.json index fed64d9..4469efc 100644 --- a/ipypublish/export_plugins/sphinx_ipypublish_all.json +++ b/ipypublish/export_plugins/sphinx_ipypublish_all.json @@ -13,7 +13,8 @@ "choose_output_type": "ipypublish.filters.rst_choose_output.choose_output_type", "ipypandoc": "ipypublish.filters_pandoc.main.jinja_filter", "is_equation": "ipypublish.filters.filters.is_equation", - "basename": "ipypublish.filters.filters.basename" + "basename": "ipypublish.filters.filters.basename", + "strip_ext": "ipypublish.filters.filters.strip_ext" }, "preprocessors": [ { diff --git a/ipypublish/export_plugins/sphinx_ipypublish_all.run.json b/ipypublish/export_plugins/sphinx_ipypublish_all.run.json index 70b1641..dc59150 100644 --- a/ipypublish/export_plugins/sphinx_ipypublish_all.run.json +++ b/ipypublish/export_plugins/sphinx_ipypublish_all.run.json @@ -14,7 +14,8 @@ "choose_output_type": "ipypublish.filters.rst_choose_output.choose_output_type", "ipypandoc": "ipypublish.filters_pandoc.main.jinja_filter", "is_equation": "ipypublish.filters.filters.is_equation", - "basename": "ipypublish.filters.filters.basename" + "basename": "ipypublish.filters.filters.basename", + "strip_ext": "ipypublish.filters.filters.strip_ext" }, "preprocessors": [ { diff --git a/ipypublish/export_plugins/sphinx_ipypublish_main.json b/ipypublish/export_plugins/sphinx_ipypublish_main.json index ed523a1..8b4b402 100644 --- a/ipypublish/export_plugins/sphinx_ipypublish_main.json +++ b/ipypublish/export_plugins/sphinx_ipypublish_main.json @@ -13,7 +13,8 @@ "choose_output_type": "ipypublish.filters.rst_choose_output.choose_output_type", "ipypandoc": "ipypublish.filters_pandoc.main.jinja_filter", "is_equation": "ipypublish.filters.filters.is_equation", - "basename": "ipypublish.filters.filters.basename" + "basename": "ipypublish.filters.filters.basename", + "strip_ext": "ipypublish.filters.filters.strip_ext" }, "preprocessors": [ { diff --git a/ipypublish/export_plugins/sphinx_ipypublish_main.run.json b/ipypublish/export_plugins/sphinx_ipypublish_main.run.json index b48517a..b6f8bbf 100644 --- a/ipypublish/export_plugins/sphinx_ipypublish_main.run.json +++ b/ipypublish/export_plugins/sphinx_ipypublish_main.run.json @@ -13,7 +13,8 @@ "choose_output_type": "ipypublish.filters.rst_choose_output.choose_output_type", "ipypandoc": "ipypublish.filters_pandoc.main.jinja_filter", "is_equation": "ipypublish.filters.filters.is_equation", - "basename": "ipypublish.filters.filters.basename" + "basename": "ipypublish.filters.filters.basename", + "strip_ext": "ipypublish.filters.filters.strip_ext" }, "preprocessors": [ { diff --git a/ipypublish/export_plugins/sphinx_ipypublish_nocode.json b/ipypublish/export_plugins/sphinx_ipypublish_nocode.json index 8b1aa0e..874ed57 100644 --- a/ipypublish/export_plugins/sphinx_ipypublish_nocode.json +++ b/ipypublish/export_plugins/sphinx_ipypublish_nocode.json @@ -13,7 +13,8 @@ "choose_output_type": "ipypublish.filters.rst_choose_output.choose_output_type", "ipypandoc": "ipypublish.filters_pandoc.main.jinja_filter", "is_equation": "ipypublish.filters.filters.is_equation", - "basename": "ipypublish.filters.filters.basename" + "basename": "ipypublish.filters.filters.basename", + "strip_ext": "ipypublish.filters.filters.strip_ext" }, "preprocessors": [ { diff --git a/ipypublish/filters/rst_choose_output.py b/ipypublish/filters/rst_choose_output.py index 8123a5d..a7a8479 100644 --- a/ipypublish/filters/rst_choose_output.py +++ b/ipypublish/filters/rst_choose_output.py @@ -45,4 +45,4 @@ def choose_output_type(output): break else: latex_datatype = ', '.join(output.data.keys()) - return html_datatype, latex_datatype \ No newline at end of file + return html_datatype, latex_datatype diff --git a/ipypublish/filters_pandoc/definitions.py b/ipypublish/filters_pandoc/definitions.py index 921e24e..a60661b 100644 --- a/ipypublish/filters_pandoc/definitions.py +++ b/ipypublish/filters_pandoc/definitions.py @@ -30,16 +30,33 @@ ("classes", ("capital",)), ("attributes", (("latex", "Cref"), ("rst", "numref"))) )), + ("&", ( + ("classes", ()), + ("attributes", (("latex", "gls"), ("rst", "gls"))), + )), + ("%", ( + ("classes", ("capital",)), + ("attributes", (("latex", "Gls"), ("rst", "glsc"))) + )), ) -# PREFIX_ALLOWED = ('+', '!', '=', '?') -# PREFIX_MAP_LATEX = (('+', 'cref'), ('?', 'Cref'), -# ('!', 'ref'), ('=', 'eqref'), ("", "cite")) -PREFIX_MAP_LATEX_R = (('cref', '+'), ('Cref', '?'), - ('ref', '!'), ('eqref', '='), ("cite", "")) -# PREFIX_MAP_RST = (('+', 'numref'), ('?', 'numref'), ('!', 'ref'), -# ('=', 'eq'), ("", "cite")) -PREFIX_MAP_RST_R = (('numref', '+'), ('ref', '!'), ('eq', '='), ("cite", "")) +PREFIX_MAP_LATEX_R = ( + ('cref', '+'), + ('Cref', '?'), + ('ref', '!'), + ('eqref', '='), + ("cite", ""), + ("gls", "&"), + ("Gls", "%") + ) +PREFIX_MAP_RST_R = ( + ('numref', '+'), + ('ref', '!'), + ('eq', '='), + ("cite", ""), + ("gls", "&"), + ("glsc", "%") + ) CITE_HTML_NAMES = ( ("Math", "eqn."), diff --git a/ipypublish/filters_pandoc/format_cite_elements.py b/ipypublish/filters_pandoc/format_cite_elements.py index b8e8be6..b6b2f59 100644 --- a/ipypublish/filters_pandoc/format_cite_elements.py +++ b/ipypublish/filters_pandoc/format_cite_elements.py @@ -112,7 +112,7 @@ def format_cites(cite, doc): names.setdefault(prefix, set()).add( '{1}'.format(citation.id, ref["number"]) ) - + elif citation.id in doc.bibdatabase: cites.add(process_bib_entry( citation.id, doc.bibdatabase, doc.bibnums)) @@ -153,7 +153,7 @@ def format_cites(cite, doc): # 'No reference found for: {}'.format( '{}'.format( ", ".join([l for l in unknown])))) - + return elements diff --git a/ipypublish/filters_pandoc/format_label_elements.py b/ipypublish/filters_pandoc/format_label_elements.py index 7e5f82c..70576a9 100644 --- a/ipypublish/filters_pandoc/format_label_elements.py +++ b/ipypublish/filters_pandoc/format_label_elements.py @@ -5,7 +5,7 @@ first to access the functionality below: If the parent of the element is a Span (or Div for Table), with a class -labelled-Math/labelled-Image/labelled-Table, +labelled-Math/labelled-Image/labelled-Table, then the label of the element will be span.identifier, and the attributes and classes from this Span will be used to inform the format. @@ -19,7 +19,7 @@ """ # TODO format headers with section labels -# (see ipysphinx.docutils_transforms.CreateNotebookSectionAnchors) +# (see ipysphinx.transforms.CreateNotebookSectionAnchors) import json from panflute import Element, Doc, Span, Div, Math, Image, Table # noqa: F401 import panflute as pf diff --git a/ipypublish/filters_pandoc/format_raw_spans.py b/ipypublish/filters_pandoc/format_raw_spans.py index fc7b6fb..9719ed8 100644 --- a/ipypublish/filters_pandoc/format_raw_spans.py +++ b/ipypublish/filters_pandoc/format_raw_spans.py @@ -30,6 +30,9 @@ def process_raw_spans(container, doc): return pf.Str("\n\n.. {}:: {}\n\n".format( container.attributes["tag"], container.attributes["content"])) + if container.attributes["tag"] == "ensuremath": + return pf.RawInline(":math:`{}`".format( + container.attributes["content"]), format='rst') return pf.RawInline(container.attributes.get("original"), format=container.attributes["format"]) diff --git a/ipypublish/filters_pandoc/main.py b/ipypublish/filters_pandoc/main.py index 7194286..dc508fc 100644 --- a/ipypublish/filters_pandoc/main.py +++ b/ipypublish/filters_pandoc/main.py @@ -8,7 +8,7 @@ - [reStructuredText Directives](http://docutils.sourceforge.net/docs/ref/rst/directives.html#figure) - [sphinxcontrib-bibtex](https://sphinxcontrib-bibtex.readthedocs.io/en/latest/usage.html) -""" +""" # noqa: E501 import panflute as pf from ipypublish.filters_pandoc.definitions import IPUB_META_ROUTE @@ -64,7 +64,7 @@ def pandoc_filters(): out_doc = doc for func in filters: - out_doc = func(out_doc) # type: Doc + out_doc = func(out_doc) # type: pf.Doc # TODO strip meta? pf.dump(doc) diff --git a/ipypublish/filters_pandoc/prepare_labels.py b/ipypublish/filters_pandoc/prepare_labels.py index 3f452ec..b4d206f 100644 --- a/ipypublish/filters_pandoc/prepare_labels.py +++ b/ipypublish/filters_pandoc/prepare_labels.py @@ -4,7 +4,7 @@ Then, for each Image, Math and Table found; -2) Extract labels and attributes to the right of Math or Table captions, +2) Extract labels and attributes to the right of Math or Table captions, in the form; ``{#id .class-name a="an attribute"}`` 3) If attributes found, remove them from the document and wrap the associated @@ -122,7 +122,7 @@ def resolve_equations_images(element, doc): # "attributes": subel.attributes, "elements": [] } - + else: attributes = find_attributes(subel) if attributes: diff --git a/ipypublish/filters_pandoc/prepare_raw.py b/ipypublish/filters_pandoc/prepare_raw.py index 98377f6..742191e 100644 --- a/ipypublish/filters_pandoc/prepare_raw.py +++ b/ipypublish/filters_pandoc/prepare_raw.py @@ -119,7 +119,7 @@ def process_html_cites(container, doc): def process_latex_raw(element, doc): - # type: (Union[RawInline, RawBlock], Doc) -> Element + # type: (Union[pf.RawInline, pf.RawBlock], pf.Doc) -> pf.Element """extract all latex adhering to \\tag{content} or \\tag[options]{content} to a Span element with class RAWSPAN_CLASS attributes: @@ -141,14 +141,13 @@ def process_latex_raw(element, doc): def process_latex_str(block, doc): # type: (pf.Block, Doc) -> Union[pf.Block,None] - """see process_latex_raw + """see process_latex_raw same but sometimes pandoc doesn't convert to a raw element """ - # TODO in the tests '\cite{a}' -> RawInline, - # yet running a file with pandoc \cite{a}' -> Str?! - # if not (isinstance(block, get_panflute_containers(pf.Str))): - # return None + # TODO why is pandoc sometimes converting latex tags to Str? + # >> echo "\cite{a}" | pandoc -f markdown -t json + # {"blocks":[{"t":"Para","c":[{"t":"RawInline","c":["tex","\\cite{a}"]}]}],"pandoc-api-version":[1,17,5,4],"meta":{}} content_attr = get_pf_content_attr(block, pf.Str) if not content_attr: @@ -180,6 +179,21 @@ def process_latex_str(block, doc): def assess_latex(text, is_block): + """ test if text is a latex command + ``\\tag{content}`` or ``\\tag[options]{content}`` + + if so return a panflute.Span, with attributes: + + - format: "latex" + - tag: + - options: + - content: + - original: + + """ + # TODO these regexes do not match labels containing nested {} braces + # use recursive regexes (https://stackoverflow.com/a/26386070/5033292) + # with https://pypi.org/project/regex/ # find tags with no option, i.e \tag{label} match_latex_noopts = re.match( @@ -214,8 +228,9 @@ def assess_latex(text, is_block): span = pf.Span( classes=[RAWSPAN_CLASS, CONVERTED_OTHER_CLASS], attributes={"format": "latex", - "tag": tag, "content": content, - options: "options", + "tag": tag, + "content": content, + "options": options, "original": text} ) if is_block: diff --git a/ipypublish/filters_pandoc/tests/test_convert_raw.py b/ipypublish/filters_pandoc/tests/test_convert_raw.py index 4eea349..bc826fb 100644 --- a/ipypublish/filters_pandoc/tests/test_convert_raw.py +++ b/ipypublish/filters_pandoc/tests/test_convert_raw.py @@ -102,7 +102,7 @@ def test_cite_in_table_caption(): '- -', '1 2', '', - 'Table: Caption \cite{a}' + 'Table: Caption \\cite{a}' ] out_string = apply_filter( @@ -181,8 +181,8 @@ def test_citations_latex(): "", "\\begin{figure}", "\\centering", - "\includegraphics{path/to/image.png}", - "\\caption{a citation \cite{label}}", + "\\includegraphics{path/to/image.png}", + "\\caption{a citation \\cite{label}}", "\\end{figure}" ]) @@ -292,7 +292,7 @@ def test_rst_directive_to_latex(): '', ' xyz' ] - + out_string = apply_filter( in_string, [prepare_raw.main, format_raw_spans.main], "latex") @@ -353,4 +353,4 @@ def test_rst_label_to_rst(): assert out_string.strip() == "\n".join([ '.. _alabel:' - ]) \ No newline at end of file + ]) diff --git a/ipypublish/filters_pandoc/tests/test_format_cite_elements.py b/ipypublish/filters_pandoc/tests/test_format_cite_elements.py index 1fb60b7..cb12b2f 100644 --- a/ipypublish/filters_pandoc/tests/test_format_cite_elements.py +++ b/ipypublish/filters_pandoc/tests/test_format_cite_elements.py @@ -33,5 +33,39 @@ def test_multiple_references_latex(): print(out_string) assert out_string == "\n".join([ - r"multiple references \cref{fig:id,tbl:id,eq:id1}" - ]) \ No newline at end of file + "multiple references \\cref{fig:id,tbl:id,eq:id1}" + ]) + + +def test_reference_prefixes_latex(): + """ + """ + in_string = [ + '(?@key1 &@key2 =@key3)' + ] + + out_string = apply_filter(in_string, + [prepare_cites.main, + format_cite_elements.main], "latex") + + print(out_string) + assert out_string == "\n".join([ + "(\\Cref{key1} \\gls{key2} \\eqref{key3})" + ]) + + +def test_reference_prefixes_rst(): + """ + """ + in_string = [ + '(?@key1 &@key2 %@key3 =@key4)' + ] + + out_string = apply_filter(in_string, + [prepare_cites.main, + format_cite_elements.main], "rst") + + print(out_string) + assert out_string == "\n".join([ + "(:ref:`key1` :gls:`key2` :glsc:`key3` :eq:`key4`)" + ]) diff --git a/ipypublish/filters_pandoc/tests/test_jinja_filter.py b/ipypublish/filters_pandoc/tests/test_jinja_filter.py index 01e626d..4766f6b 100644 --- a/ipypublish/filters_pandoc/tests/test_jinja_filter.py +++ b/ipypublish/filters_pandoc/tests/test_jinja_filter.py @@ -76,4 +76,4 @@ def test_remove_filter(): out_str = jinja_filter( "+@label", "rst", create_ipub_meta({"apply_filters": False}), {} ) - assert out_str == "+@label" \ No newline at end of file + assert out_str == "+@label" diff --git a/ipypublish/filters_pandoc/tests/test_prepare_labels.py b/ipypublish/filters_pandoc/tests/test_prepare_labels.py index d3878a2..a64222d 100644 --- a/ipypublish/filters_pandoc/tests/test_prepare_labels.py +++ b/ipypublish/filters_pandoc/tests/test_prepare_labels.py @@ -7,7 +7,6 @@ from panflute import Element, Doc # noqa: F401 from types import FunctionType # noqa: F401 -from ipypublish.filters_pandoc.utils import apply_filter from ipypublish.filters_pandoc.prepare_labels import main diff --git a/ipypublish/filters_pandoc/utils.py b/ipypublish/filters_pandoc/utils.py index c927925..3a0cbc9 100644 --- a/ipypublish/filters_pandoc/utils.py +++ b/ipypublish/filters_pandoc/utils.py @@ -21,7 +21,7 @@ def apply_filter(in_object, filter_func=None, replace_api_version=True, dry_run=False, **kwargs): # type: (list[str], FunctionType) -> str - """convenience function to apply a panflute filter(s) + """convenience function to apply a panflute filter(s) to a string, list of string lines, pandoc AST or panflute.Doc Parameters @@ -178,7 +178,7 @@ def strip_quotes(string): def find_attributes(element, allow_space=True, search_left=False, include_element=False): - """find an attribute 'container' for an element, + """find an attribute 'container' for an element, of the form {#id .class1 .class2 a=1 b="a string"} and extract its content @@ -265,7 +265,7 @@ def _search_attribute_right(element, include_element, allow_space): pf.Para(*attr_elements)).replace("\n", " ").strip() # split into the label and the rest - match = re.match("^\\{(#[^\s]+|)([^\\}]*)\\}", attribute_str) + match = re.match(r"^\{(#[^\s]+|)([^\}]*)\}", attribute_str) if not match: raise ValueError(attribute_str) classes, attributes = process_attributes(match.group(2)) @@ -339,7 +339,7 @@ def _search_attribute_left(element, include_element, allow_space): pf.Para(*attr_elements)).replace("\n", " ").strip() # split into the label and the rest - match = re.match("^\\{(#[^\s]+|)([^\\}]*)\\}$", attribute_str) + match = re.match("^\\{(#[^\\s]+|)([^\\}]*)\\}$", attribute_str) if not match: raise ValueError(attribute_str) classes, attributes = process_attributes(match.group(2)) @@ -354,7 +354,7 @@ def _search_attribute_left(element, include_element, allow_space): def process_attributes(attr_string): - """process a string of classes and attributes, + """process a string of classes and attributes, e.g. '.class-name .other a=1 b="some text"' will be returned as: ["class-name", "other"], {"a": 1, "b": "some text"} @@ -363,11 +363,11 @@ def process_attributes(attr_string): dict: attributes """ # find classes, denoted by .class-name - classes = [c[1][1:] for c in re.findall('(^|\s)(\\.[\\-\\_a-zA-Z]+)', + classes = [c[1][1:] for c in re.findall('(^|\\s)(\\.[\\-\\_a-zA-Z]+)', attr_string)] # find attributes, denoted by a=b, respecting quotes attr = {c[1]: strip_quotes(c[2]) for c in re.findall( - '(^|\s)([\\-\\_a-zA-Z]+)\s*=\s*(\\".+\\"|\\\'.+\\\'|[^\s\\"\\\']+)', + '(^|\\s)([\\-\\_a-zA-Z]+)\\s*=\\s*(\\".+\\"|\\\'.+\\\'|[^\\s\\"\\\']+)', # noqa: E501 attr_string)} # TODO this generally works, but should be stricter against any weird # fringe cases diff --git a/ipypublish/frontend/nbpresent.py b/ipypublish/frontend/nbpresent.py index 5097951..14c5e28 100644 --- a/ipypublish/frontend/nbpresent.py +++ b/ipypublish/frontend/nbpresent.py @@ -41,11 +41,10 @@ def nbpresent(inpath, """ inpath_name, inpath_ext = os.path.splitext(os.path.basename(inpath)) - outpath = None output_mimetype = guess_type(inpath, strict=False)[0] output_mimetype = 'unknown' if output_mimetype is None else output_mimetype - if inpath_ext == '.ipynb': + if output_mimetype != "text/html": config = {"IpyPubMain": { "conversion": outformat, @@ -74,12 +73,13 @@ def nbpresent(inpath, except Exception as err: logger.error("Run Failed: {}".format(err)) if print_traceback: - raise err + raise return 1 else: + logging.basicConfig(stream=sys.stdout, level=logging.INFO) server = RevealServer() if not dry_run: - server.postprocess("", output_mimetype, outpath) + server.postprocess("", output_mimetype, os.path.abspath(inpath)) return 0 diff --git a/ipypublish/frontend/nbpublish.py b/ipypublish/frontend/nbpublish.py index 82e1ae7..837b38c 100644 --- a/ipypublish/frontend/nbpublish.py +++ b/ipypublish/frontend/nbpublish.py @@ -76,7 +76,7 @@ def nbpublish(ipynb_path, except Exception as err: logger.error("Run Failed: {}".format(err)) if print_traceback: - raise err + raise return 1 return 0 diff --git a/ipypublish/ipysphinx/__init__.py b/ipypublish/ipysphinx/__init__.py deleted file mode 100644 index 51e7c1a..0000000 --- a/ipypublish/ipysphinx/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# expose setup function for sphinx -from ipypublish.ipysphinx.extension import setup # noqa: F401 diff --git a/ipypublish/port_api/convert_format_str.py b/ipypublish/port_api/convert_format_str.py index 6927ea6..020e604 100644 --- a/ipypublish/port_api/convert_format_str.py +++ b/ipypublish/port_api/convert_format_str.py @@ -37,7 +37,7 @@ def convert_format_str(template): "{{{{ super() }}}}", "{{% endblock codecell %}}", "{{% block in_prompt %}}{{% endblock in_prompt %}}", - "{{% block input %}}{{{{ cell.metadata | meta2yaml('#~~ ') }}}}", + "{{% block input %}}{{{{ cell.metadata | meta2yaml('#~~ ') }}}}", # noqa: E501 "{{{{ cell.source | ipython2python }}}}", "{{% endblock input %}}", "{{% block markdowncell scoped %}}#%% [markdown]", diff --git a/ipypublish/port_api/plugin_to_json.py b/ipypublish/port_api/plugin_to_json.py index 009de38..8e67852 100644 --- a/ipypublish/port_api/plugin_to_json.py +++ b/ipypublish/port_api/plugin_to_json.py @@ -25,8 +25,8 @@ def assess_syntax(path): elif isinstance(child, ast.ImportFrom): module = child.module for n in child.names: - importPath = module + "." + n.name - imported[n.name if n.asname is None else n.asname] = importPath + import_pth = module + "." + n.name + imported[n.name if n.asname is None else n.asname] = import_pth elif isinstance(child, ast.Assign): targets = child.targets if len(targets) > 1: @@ -79,14 +79,14 @@ def ast_to_json(item, imported, assignments): def convert_dict(dct, imported, assignments): # type: (ast.Dict, Dict[str, str], dict) -> dict """recurse through and replace keys""" - outDict = {} + out_dict = {} for key, val in zip(dct.keys, dct.values): if not isinstance(key, ast.Str): raise ValueError( "expected key to be a Str; {}".format(key)) - outDict[key.s] = ast_to_json(val, imported, assignments) + out_dict[key.s] = ast_to_json(val, imported, assignments) - return outDict + return out_dict def convert_oformat(oformat): @@ -142,8 +142,6 @@ def convert_config(config, exporter_class, allow_other): preprocs[pname]["args"]["metapath"] = "${meta_path}" preprocs[pname]["args"]["filesfolder"] = "${files_path}" - - # second parse for key, val in config.items(): if key in ["Exporter.filters", "TemplateExporter.filters", @@ -245,27 +243,27 @@ def create_json(docstring, imported, assignments, allow_other=True): keywords = expr.keywords if len(args) != 1 or len(keywords) > 0: raise ValueError("expected create_tpl(x) to have one argument") - segList = args[0] - if isinstance(segList, ast.ListComp): - segList = segList.generators[0].iter - if not isinstance(segList, ast.List): + seg_list = args[0] + if isinstance(seg_list, ast.ListComp): + seg_list = seg_list.generators[0].iter + if not isinstance(seg_list, ast.List): raise ValueError( "expected create_tpl(x) arg to be a List; {}".format( - segList)) + seg_list)) segments = [] - for seg in segList.elts: + for seg in seg_list.elts: if isinstance(seg, ast.Attribute): - segName = seg.value.id + seg_name = seg.value.id elif isinstance(seg, ast.Name): - segName = seg.id + seg_name = seg.id else: raise ValueError( "expected seg in template to be an Attribute; " + "{1}".format(seg)) - if segName not in imported: - raise ValueError("segment '{}' not found".format(segName)) - segments.append(imported[segName]) + if seg_name not in imported: + raise ValueError("segment '{}' not found".format(seg_name)) + segments.append(imported[seg_name]) template = segments if oformat is None: diff --git a/ipypublish/port_api/tpl_dct_to_json.py b/ipypublish/port_api/tpl_dct_to_json.py index bb90b5c..a257c91 100644 --- a/ipypublish/port_api/tpl_dct_to_json.py +++ b/ipypublish/port_api/tpl_dct_to_json.py @@ -36,7 +36,7 @@ def assess_syntax(path): dtype, child.value)) dct = child.value break - + if dct is None: raise IOError("could not find tpl(x)_dict") @@ -51,7 +51,7 @@ def assess_syntax(path): "expected {} value be of type Str: {}".format( dtype, value)) output[key.s] = value.s - + return { "identifier": os.path.splitext(os.path.basename(path))[0], "description": docstring, diff --git a/ipypublish/postprocessors/base.py b/ipypublish/postprocessors/base.py index cf8e021..8ccdb51 100644 --- a/ipypublish/postprocessors/base.py +++ b/ipypublish/postprocessors/base.py @@ -115,8 +115,7 @@ def postprocess(self, stream, mimetype, filepath, if not filepath.is_absolute(): self.handle_error( - "the post-processor requires a folder, " - "but the filepath is not absolute", + "the post-processor requires an absolute folder path", IOError) if filepath.parent.exists() and not filepath.parent.is_dir(): @@ -131,9 +130,9 @@ def postprocess(self, stream, mimetype, filepath, if resources is None: resources = {} - return self.run_postprocess(stream, filepath, resources) + return self.run_postprocess(stream, mimetype, filepath, resources) - def run_postprocess(self, stream, filepath, resources): + def run_postprocess(self, stream, mimetype, filepath, resources): """ should not be called directly override in sub-class diff --git a/ipypublish/postprocessors/convert_bibgloss.py b/ipypublish/postprocessors/convert_bibgloss.py new file mode 100644 index 0000000..cd1f627 --- /dev/null +++ b/ipypublish/postprocessors/convert_bibgloss.py @@ -0,0 +1,98 @@ +import os +import sys +from traitlets import Unicode +from ipypublish.postprocessors.base import IPyPostProcessor +from ipypublish.bib2glossary import BibGlossDB + + +class ConvertBibGloss(IPyPostProcessor): + """ convert a bibglossary to the required format + """ + @property + def allowed_mimetypes(self): + return None + + @property + def requires_path(self): + return True + + @property + def logger_name(self): + return "convert-bibgloss" + + encoding = Unicode( + "utf8", + help="the encoding of the input file" + ).tag(config=True) + + resource_key = Unicode( + "bibglosspath", + help="the key in the resources dict containing the path to the file" + ).tag(config=True) + + files_folder = Unicode( + "_static", + help="the path (relative to the main file path) to dump to" + ).tag(config=True) + + def run_postprocess(self, stream, mimetype, filepath, resources): + + if "bibglosspath" not in resources: + return stream, filepath, resources + + bibpath = resources["bibglosspath"] + + if not os.path.exists(str(bibpath)): + self.logger.warning( + "the bibglossary could not be converted, " + "since its path does not exist: {}".format(bibpath)) + return stream, filepath, resources + + bibname, extension = os.path.splitext(os.path.basename(bibpath)) + outstr = None + outext = None + + if extension in [".bib"]: + if mimetype == "text/restructuredtext": + pass + elif mimetype == "text/latex": + self.logger.info("converting bibglossary to tex") + bibdb = BibGlossDB() + bibdb.load_bib(path=str(bibpath), encoding=self.encoding) + outstr = bibdb.to_latex_string() + outext = ".tex" + + elif extension in [".tex"]: + if mimetype == "text/latex": + pass + elif mimetype == "text/restructuredtext": + self.logger.info("converting bibglossary to bibtex") + bibdb = BibGlossDB() + bibdb.load_tex(path=str(bibpath), encoding=self.encoding) + outstr = bibdb.to_bib_string() + outext = ".bib" + + else: + self.logger.warning( + "the bibglossary could not be converted, " + "since its file extension was not one of: " + "bib, tex") + + if outstr is None: + return stream, filepath, resources + + if sys.version_info < (3, 0): + outstr = unicode(outstr, encoding=self.encoding) # noqa: F821 + + output_folder = filepath.parent.joinpath(self.files_folder) + if not output_folder.exists(): + output_folder.mkdir(parents=True) + + outfile = output_folder.joinpath(bibname + outext) + self.logger.info("writing bibglossary: {}".format(outfile)) + with outfile.open("w", encoding=self.encoding) as fh: + fh.write(outstr) + + self.logger.debug("finished") + + return stream, filepath, resources diff --git a/ipypublish/postprocessors/file_actions.py b/ipypublish/postprocessors/file_actions.py index 98a1ca7..12615f1 100644 --- a/ipypublish/postprocessors/file_actions.py +++ b/ipypublish/postprocessors/file_actions.py @@ -26,7 +26,7 @@ def logger_name(self): help="the encoding of the output file" ).tag(config=True) - def run_postprocess(self, stream, filepath, resources): + def run_postprocess(self, stream, mimetype, filepath, resources): self.logger.info('writing stream to file: {}'.format(filepath)) with filepath.open("w", encoding=self.encoding) as fh: @@ -55,13 +55,13 @@ def logger_name(self): help="the path (relative to the main file path) to remove" ).tag(config=True) - def run_postprocess(self, stream, filepath, resources): + def run_postprocess(self, stream, mimetype, filepath, resources): remove_folder = filepath.parent.joinpath(self.files_folder) if remove_folder.exists() and remove_folder.is_dir(): self.logger.info( 'removing folder: {0}'.format(remove_folder)) - shutil.rmtree(remove_folder) + shutil.rmtree(str(remove_folder)) return stream, filepath, resources @@ -82,7 +82,7 @@ def logger_name(self): return "write-resource-files" resource_keys = List( - Unicode, + Unicode(), ["outputs"], help="the key names in the resources dict that contain files" ).tag(config=True) @@ -93,7 +93,7 @@ def logger_name(self): # help="the path (relative to the main file path) to write to" # ).tag(config=True) - def run_postprocess(self, stream, filepath, resources): + def run_postprocess(self, stream, mimetype, filepath, resources): output_folder = filepath.parent if not output_folder.exists(): @@ -139,7 +139,7 @@ def logger_name(self): return "copy-resource-paths" resource_keys = List( - Unicode, + Unicode(), ["external_file_paths"], help="the key names in the resources dict that contain filepaths" ).tag(config=True) @@ -149,7 +149,7 @@ def logger_name(self): help="the path (relative to the main file path) to copy to" ).tag(config=True) - def run_postprocess(self, stream, filepath, resources): + def run_postprocess(self, stream, mimetype, filepath, resources): output_folder = filepath.parent.joinpath(self.files_folder) if not output_folder.exists(): diff --git a/ipypublish/postprocessors/pdfexport.py b/ipypublish/postprocessors/pdfexport.py index 341b2a3..175db16 100755 --- a/ipypublish/postprocessors/pdfexport.py +++ b/ipypublish/postprocessors/pdfexport.py @@ -49,7 +49,7 @@ def logger_name(self): False, help="launch a html page containing a pdf browser").tag(config=True) - def run_postprocess(self, stream, filepath, resources): + def run_postprocess(self, stream, mimetype, filepath, resources): """ should not be called directly Parameters @@ -170,7 +170,7 @@ def log_latexmk_output(pipe): return exitcode -class change_dir: +class change_dir: # noqa: N801 """Context manager for changing the current working directory""" def __init__(self, new_path): @@ -240,4 +240,4 @@ def __exit__(self, etype, value, traceback): -""" +""" # noqa: E501 diff --git a/ipypublish/postprocessors/reveal_serve.py b/ipypublish/postprocessors/reveal_serve.py index caddacf..7525e68 100644 --- a/ipypublish/postprocessors/reveal_serve.py +++ b/ipypublish/postprocessors/reveal_serve.py @@ -11,7 +11,7 @@ import signal import webbrowser -from tornado import web, ioloop, httpserver, log +from tornado import web, ioloop, httpserver, log, gen from tornado.httpclient import AsyncHTTPClient from traitlets import Bool, Unicode, Int from ipypublish.postprocessors.base import IPyPostProcessor @@ -20,17 +20,13 @@ class ProxyHandler(web.RequestHandler): """handler the proxies requests from a local prefix to a CDN""" - @web.asynchronous + @gen.coroutine def get(self, prefix, url): """proxy a request to a CDN""" proxy_url = "/".join([self.settings['cdn'], url]) client = self.settings['client'] client.fetch(proxy_url, callback=self.finish_get) - - def finish_get(self, response): - """finish the request""" - # rethrow errors - response.rethrow() + response = yield client.fetch(proxy_url) for header in ["Content-Type", "Cache-Control", "Date", "Last-Modified", "Expires"]: @@ -75,7 +71,7 @@ def logger_name(self): port = Int( 8000, help="port for the server to listen on.").tag(config=True) - def run_postprocess(self, stream, filepath, resources): + def run_postprocess(self, stream, mimetype, filepath, resources): """Serve the build directory with a webserver.""" if not filepath.exists(): @@ -96,11 +92,11 @@ def run_postprocess(self, stream, filepath, resources): pass elif os.path.isdir(os.path.join(dirname, self.reveal_prefix)): # reveal prefix exists - self.log.info("Serving local %s", self.reveal_prefix) + self.logger.info("Serving local %s", self.reveal_prefix) self.logger.info("Serving local %s", self.reveal_prefix) else: - self.log.info("Redirecting %s requests to %s", - self.reveal_prefix, self.reveal_cdn) + self.logger.info("Redirecting %s requests to %s", + self.reveal_prefix, self.reveal_cdn) self.logger.info("Redirecting %s requests to %s", self.reveal_prefix, self.reveal_cdn) handlers.insert(0, (r"/(%s)/(.*)" % @@ -112,7 +108,7 @@ def run_postprocess(self, stream, filepath, resources): ) # hook up tornado logging to our self.logger - log.app_log = self.log + log.app_log = self.logger http_server = httpserver.HTTPServer(app) @@ -149,7 +145,7 @@ def handler(signum, frame): ioloop.IOLoop.instance().start() except KeyboardInterrupt: # dosen't look like line below is necessary - # ioloop.IOLoop.instance().stop() + ioloop.IOLoop.instance().stop() self.logger.info("\nInterrupted") return stream, filepath, resources diff --git a/ipypublish/postprocessors/sphinx.py b/ipypublish/postprocessors/sphinx.py index b1af3c5..44d8f55 100644 --- a/ipypublish/postprocessors/sphinx.py +++ b/ipypublish/postprocessors/sphinx.py @@ -8,10 +8,9 @@ from traitlets import TraitError, validate, Bool, Dict, Unicode from ipypublish import __version__ -from ipypublish.utils import find_entry_point from ipypublish.postprocessors.base import IPyPostProcessor -from ipypublish.ipysphinx.utils import import_sphinx -from ipypublish.ipysphinx.create_setup import make_conf, make_index +from ipypublish.sphinx.utils import import_sphinx +from ipypublish.sphinx.create_setup import make_conf, make_index # NOTE Interesting note about adding a directive to actually run python code @@ -70,7 +69,7 @@ def _valid_prompt_style(self, proposal): help="nit-picky mode, warn about all missing references" ) - def run_postprocess(self, stream, filepath, resources): + def run_postprocess(self, stream, mimetype, filepath, resources): # check sphinx is available and the correct version try: @@ -142,7 +141,7 @@ def run_postprocess(self, stream, filepath, resources): build_dir = filepath.parent.joinpath('build/html') if build_dir.exists(): # >> rm -r build/html - shutil.rmtree(build_dir) + shutil.rmtree(str(build_dir)) build_dir.mkdir(parents=True) # run sphinx diff --git a/ipypublish/postprocessors/stream_modify.py b/ipypublish/postprocessors/stream_modify.py index 0a11b0f..2922769 100644 --- a/ipypublish/postprocessors/stream_modify.py +++ b/ipypublish/postprocessors/stream_modify.py @@ -21,7 +21,7 @@ def requires_path(self): def logger_name(self): return "remove-blank-lines" - def run_postprocess(self, stream, filepath, resources): + def run_postprocess(self, stream, mimetype, filepath, resources): stream = re.sub(r'\n\s*\n', '\n\n', stream) return stream, filepath, resources @@ -41,7 +41,7 @@ def requires_path(self): def logger_name(self): return "remove-trailing-space" - def run_postprocess(self, stream, filepath, resources): + def run_postprocess(self, stream, mimetype, filepath, resources): stream = "\n".join([l.rstrip() for l in stream.splitlines()]) return stream, filepath, resources @@ -61,7 +61,7 @@ def requires_path(self): def logger_name(self): return "filter-output-files" - def run_postprocess(self, stream, filepath, resources): + def run_postprocess(self, stream, mimetype, filepath, resources): if 'outputs' in resources: for path in list(resources['outputs'].keys()): @@ -85,7 +85,7 @@ def requires_path(self): def logger_name(self): return "fix-slide-refs" - def run_postprocess(self, stream, filepath, resources): + def run_postprocess(self, stream, mimetype, filepath, resources): if resources and 'refslide' in resources: for k, (col, row) in resources['refslide'].items(): stream = stream.replace('{{id_home_prefix}}{0}'.format( diff --git a/ipypublish/postprocessors/to_stream.py b/ipypublish/postprocessors/to_stream.py index 2107cd1..5838520 100644 --- a/ipypublish/postprocessors/to_stream.py +++ b/ipypublish/postprocessors/to_stream.py @@ -31,7 +31,7 @@ def logger_name(self): help="where to write the output to" ).tag(config=True) - def run_postprocess(self, stream, filepath, resources): + def run_postprocess(self, stream, mimetype, filepath, resources): self.logger.info('writing stream to {}'.format(self.pipe)) io_type = {"stdout": sys.stdout, diff --git a/ipypublish/preprocessors/latex_doc_captions.py b/ipypublish/preprocessors/latex_doc_captions.py index fb25831..8e04355 100644 --- a/ipypublish/preprocessors/latex_doc_captions.py +++ b/ipypublish/preprocessors/latex_doc_captions.py @@ -16,8 +16,10 @@ class LatexCaptions(Preprocessor): """ - add_prefix = traits.Bool(False, help="add float type/number prefix to caption (from caption_prefix tag)").tag( - config=True) + add_prefix = traits.Bool( + False, + help=("add float type/number prefix to caption " + "(from caption_prefix tag)")).tag(config=True) def preprocess(self, nb, resources): @@ -30,10 +32,12 @@ def preprocess(self, nb, resources): if hasattr(cell.metadata, 'ipub'): if hasattr(cell.metadata.ipub.get('equation', False), 'get'): - if hasattr(cell.metadata.ipub.equation.get('environment', False), 'startswith'): - if cell.metadata.ipub.equation.environment.startswith('breqn'): + if hasattr(cell.metadata.ipub.equation.get( + 'environment', False), 'startswith'): + if cell.metadata.ipub.equation.environment.startswith('breqn'): # noqa: E501 if "ipub" not in nb.metadata: - nb.metadata["ipub"] = NotebookNode({'enable_breqn': True}) + nb.metadata["ipub"] = NotebookNode( + {'enable_breqn': True}) else: nb.metadata.ipub['enable_breqn'] = True @@ -47,11 +51,13 @@ def preprocess(self, nb, resources): if not cell.outputs: pass elif "text/latex" in cell.outputs[0].get('data', {}): - capt = cell.outputs[0].data["text/latex"].split(r'\n')[0] + capt = cell.outputs[0].data["text/latex"].split( + r'\n')[0] captions[cell.metadata.ipub.caption] = capt continue elif "text/plain" in cell.outputs[0].get('data', {}): - capt = cell.outputs[0].data["text/plain"].split(r'\n')[0] + capt = cell.outputs[0].data["text/plain"].split( + r'\n')[0] captions[cell.metadata.ipub.caption] = capt continue @@ -64,14 +70,18 @@ def preprocess(self, nb, resources): for key in cell.metadata.ipub: if hasattr(cell.metadata.ipub[key], 'label'): if cell.metadata.ipub[key]['label'] in captions: - logger.debug('replacing caption for: {}'.format(cell.metadata.ipub[key]['label'])) - cell.metadata.ipub[key]['caption'] = captions[cell.metadata.ipub[key]['label']] + logger.debug('replacing caption for: {}'.format( + cell.metadata.ipub[key]['label'])) + cell.metadata.ipub[key]['caption'] = captions[cell.metadata.ipub[key]['label']] # noqa: E501 # add float type/number prefix to caption, if required if self.add_prefix: if hasattr(cell.metadata.ipub[key], 'caption'): - if hasattr(cell.metadata.ipub[key], 'caption_prefix'): - newcaption = cell.metadata.ipub[key].caption_prefix + cell.metadata.ipub[key].caption + if hasattr(cell.metadata.ipub[key], + 'caption_prefix'): + newcaption = ( + cell.metadata.ipub[key].caption_prefix + + cell.metadata.ipub[key].caption) cell.metadata.ipub[key].caption = newcaption return nb, resources diff --git a/ipypublish/preprocessors/latex_doc_defaults.py b/ipypublish/preprocessors/latex_doc_defaults.py index 9c2884d..f0171db 100644 --- a/ipypublish/preprocessors/latex_doc_defaults.py +++ b/ipypublish/preprocessors/latex_doc_defaults.py @@ -6,7 +6,8 @@ def flatten(d, key_as_tuple=True, sep='.'): - """ get nested dict as {key:val,...}, where key is tuple/string of all nested keys + """ get nested dict as {key:val,...}, + where key is tuple/string of all nested keys Parameters ---------- @@ -34,9 +35,11 @@ def flatten(d, key_as_tuple=True, sep='.'): def expand(key, value): if isinstance(value, dict): if key_as_tuple: - return [(key + k, v) for k, v in flatten(value, key_as_tuple).items()] + return [(key + k, v) + for k, v in flatten(value, key_as_tuple).items()] else: - return [(str(key) + sep + k, v) for k, v in flatten(value, key_as_tuple).items()] + return [(str(key) + sep + k, v) + for k, v in flatten(value, key_as_tuple).items()] else: return [(key, value)] diff --git a/ipypublish/preprocessors/latex_doc_html.py b/ipypublish/preprocessors/latex_doc_html.py index 9b84b2e..4e543b3 100644 --- a/ipypublish/preprocessors/latex_doc_html.py +++ b/ipypublish/preprocessors/latex_doc_html.py @@ -50,9 +50,12 @@ def embed_html(self, cell, path): height = int(cell.metadata.ipub.embed_html.get('height', 0.5) * 100) width = int(cell.metadata.ipub.embed_html.get('width', 0.5) * 100) - embed_code = """ - - """.format(src=self.src_name, path=path, height=height, width=width) + embed_code = ( + '').format( + src=self.src_name, path=path, height=height, width=width) # add to the exising output or create a new one if cell.outputs: @@ -83,7 +86,8 @@ def preprocess(self, nb, resources): if hasattr(cell.metadata.ipub, 'embed_html'): if hasattr(cell.metadata.ipub.embed_html, 'filepath'): paths = [cell.metadata.ipub.embed_html.filepath] - if hasattr(cell.metadata.ipub.embed_html, 'other_files'): + if hasattr(cell.metadata.ipub.embed_html, + 'other_files'): if not isinstance( cell.metadata.ipub.embed_html.other_files, list): @@ -94,21 +98,24 @@ def preprocess(self, nb, resources): fpath = self.resolve_path(path, self.metapath) if not os.path.exists(fpath): logging.warning( - 'file in embed html metadata does not exist' - ': {}'.format(fpath)) + "file in embed html metadata doesn't exist" + ": {}".format(fpath)) else: resources.setdefault("external_file_paths", []) resources['external_file_paths'].append(fpath) if j == 0: - self.embed_html(cell, os.path.join( - self.filesfolder, os.path.basename(fpath))) + self.embed_html( + cell, os.path.join( + self.filesfolder, + os.path.basename(fpath))) elif hasattr(cell.metadata.ipub.embed_html, 'url'): self.embed_html( cell, cell.metadata.ipub.embed_html.url) else: logging.warning( - 'cell {} has no filepath or url key in its metadata.embed_html'.format(i)) + 'cell {} has no filepath or url key in its ' + 'metadata.embed_html'.format(i)) for floattype, floatabbr in [ ('figure', 'fig.'), ('table', 'tbl.'), ('code', 'code'), @@ -119,12 +126,13 @@ def preprocess(self, nb, resources): float_count[floattype] += 1 if not isinstance(cell.metadata.ipub[floattype], dict): continue - cell.metadata.ipub[floattype]['caption_prefix'] = '{0} {1}: '.format( + cell.metadata.ipub[floattype]['caption_prefix'] = '{0} {1}: '.format( # noqa: E501 floattype.capitalize(), float_count[floattype]) if 'label' in cell.metadata.ipub[floattype]: + label = '{0} {1}'.format( + floatabbr, float_count[floattype]) resources.setdefault('refmap', {})[ - cell.metadata.ipub[floattype]['label']] = '{0} {1}'.format(floatabbr, - float_count[floattype]) + cell.metadata.ipub[floattype]['label']] = label final_cells.append(cell) nb.cells = final_cells diff --git a/ipypublish/preprocessors/latex_doc_links.py b/ipypublish/preprocessors/latex_doc_links.py index 7639f3f..f9d7d35 100644 --- a/ipypublish/preprocessors/latex_doc_links.py +++ b/ipypublish/preprocessors/latex_doc_links.py @@ -242,7 +242,7 @@ def preprocess(self, nb, resources): if 'ipub' in nb.metadata: - if hasattr(nb.metadata.ipub, 'bibliography'): + if 'bibliography' in nb.metadata.ipub: bib = nb.metadata.ipub.bibliography bib = resolve_path(bib, self.metapath) if not os.path.exists(bib): @@ -255,18 +255,30 @@ def preprocess(self, nb, resources): nb.metadata.ipub.bibliography = os.path.join( self.filesfolder, os.path.basename(bib)) - if hasattr(nb.metadata.ipub, 'titlepage'): - if hasattr(nb.metadata.ipub.titlepage, 'logo'): - logo = nb.metadata.ipub.titlepage.logo - logo = resolve_path(logo, self.metapath) - if not os.path.exists(logo): - resources['unfound_file_paths'].append(logo) - else: - resources['external_file_paths'].append(logo) - - if self.redirect_external: - nb.metadata.ipub.titlepage.logo = os.path.join( - self.filesfolder, os.path.basename(logo)) + if "filepath" in nb.metadata.ipub.get('bibglossary', {}): + gloss = nb.metadata.ipub.bibglossary.filepath + gloss = resolve_path(gloss, self.metapath) + if not os.path.exists(gloss): + resources['unfound_file_paths'].append(gloss) + else: + resources['external_file_paths'].append(gloss) + resources['bibglosspath'] = gloss + + if self.redirect_external: + nb.metadata.ipub.bibglossary.filepath = os.path.join( + self.filesfolder, os.path.basename(gloss)) + + if 'logo' in nb.metadata.ipub.get('titlepage', {}): + logo = nb.metadata.ipub.titlepage.logo + logo = resolve_path(logo, self.metapath) + if not os.path.exists(logo): + resources['unfound_file_paths'].append(logo) + else: + resources['external_file_paths'].append(logo) + + if self.redirect_external: + nb.metadata.ipub.titlepage.logo = os.path.join( + self.filesfolder, os.path.basename(logo)) for index, cell in enumerate(nb.cells): nb.cells[index], resources = self.preprocess_cell( diff --git a/ipypublish/preprocessors/latextags_to_html.py b/ipypublish/preprocessors/latextags_to_html.py index 71dbbc0..98cd216 100644 --- a/ipypublish/preprocessors/latextags_to_html.py +++ b/ipypublish/preprocessors/latextags_to_html.py @@ -38,19 +38,24 @@ def safe_str(obj): class LatexTagsToHTML(Preprocessor): - r""" a preprocessor to find latex tags - (like \cite{abc} or \todo[color]{stuff}) and: + r""" a preprocessor to find latex tags + (like ``\cite{abc}`` or ``\todo[color]{stuff}``) and: 1. attempt to process them into a html friendly format 2. remove them entirely if this is not possible - for \ref or \cref, attempts to use resources.refmap to map labels to reference names + for ``\ref`` or ``\cref``, + attempts to use resources.refmap to map labels to reference names for labels not found in resources.refmap - the reference name is ' ', where; - - is either ref of, if labelbycolon is True and the label has a colon, all text before the colon + the reference name is ' ', where: + + - is either ref of, if labelbycolon is True and + the label has a colon, all text before the colon - iterate by order of first appearance of a particular label - NB: should be applied after LatexDocHTML, if you want resources.refmap to be available + + NB: should be applied after LatexDocHTML, + if you want resources.refmap to be available Examples -------- @@ -67,7 +72,8 @@ class LatexTagsToHTML(Preprocessor): ... date = {2016-09-01}, ... } ... ''') - >>> resources = NotebookNode({'bibliopath':bibfile, 'refmap':{"label":"label_name"}}) + >>> resources = NotebookNode( + ... {'bibliopath':bibfile, 'refmap':{"label":"label_name"}}) >>> cell = NotebookNode({ ... "cell_type":"markdown", @@ -104,18 +110,20 @@ class LatexTagsToHTML(Preprocessor): >>> print(nb.cells[0].source) \begin{equation}x=a+b\end{equation} - """ + """ # noqa: E501 - regex = traits.Unicode(r"\\(?:[^a-zA-Z]|[a-zA-Z]+[*=']?)(?:\[.*?\])?{.*?}", - help="the regex to identify latex tags").tag(config=True) + regex = traits.Unicode( + r"\\(?:[^a-zA-Z]|[a-zA-Z]+[*=']?)(?:\[.*?\])?{.*?}", + help="the regex to identify latex tags").tag(config=True) bibformat = traits.Unicode( "{author}, {year}.", help="the format to output \\cite{} tags found in the bibliography" - ).tag(config=True) + ).tag(config=True) labelbycolon = traits.Bool( True, - help='create reference label based on text before colon, e.g. \\ref{fig:example} -> fig 1' - ).tag(config=True) + help=('create reference label based on text before colon, ' + 'e.g. \\ref{fig:example} -> fig 1') + ).tag(config=True) def __init__(self, *args, **kwargs): # a dictionary to keep track of references, @@ -192,7 +200,8 @@ def replace_reflabel(self, name, resources): the links are left with a format hook in them: {id_home_prefix}, so that an nbconvert filter can later replace it - this is particularly useful for slides, which require a prefix #/