Handle Cell Attachments (#64)

Also, added ToC depth control

.gitignore

@@ -105,4 +105,4 @@ docs/source/_build
 
 # added by CJS
 releases.md
-.pytest_cache/v/cache/nodeids
+.pytest_cache/

docs/source/api/ipypublish.rst

@@ -12,6 +12,7 @@ Subpackages
     ipypublish.frontend
     ipypublish.port_api
     ipypublish.preprocessors
+    ipypublish.schema
     ipypublish.scripts
     ipypublish.templates
 

docs/source/api/ipypublish.schema.rst

@@ -0,0 +1,10 @@
+ipypublish\.schema package
+==========================
+
+Module contents
+---------------
+
+.. automodule:: ipypublish.schema
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/custom_export_config.rst

@@ -39,6 +39,13 @@ This process extends :py:mod:`nbconvert` in a number of ways:
 - The use of ``latexmk`` with XeLaTeX to convert TeX to PDF,
   and correct resolution of file references and citations.
 
+.. versionadded:: v0.8.3
+
+    Drag and drop cell attachments are now extracted and correctly referenced
+
+
+.. todo:: document this feature and other non-metadata features in a separate section 
+
 
 The Configuration File Format
 -----------------------------

docs/source/metadata_tags.rst

@@ -238,6 +238,22 @@ To control the output of **contents tables**:
      }
    }
 
+.. versionadded:: v0.8.3
+
+    You can now control the depth of the contents table:
+
+.. code:: json
+
+    {
+    "ipub": {
+      "toc": {"depth": 2}
+      }
+    }
+
+
+Figures and Tables
+~~~~~~~~~~~~~~~~~~
+
 To override the default **placement of figures and tables**:
 
 .. code:: json

ipypublish/__init__.py

@@ -1 +1 @@
-__version__ = '0.8.2'
+__version__ = '0.8.3'

ipypublish/convert/main.py

@@ -17,7 +17,7 @@
 
 
 import ipypublish
-from ipypublish.utils import pathlib, handle_error
+from ipypublish.utils import pathlib, handle_error, get_valid_filename
 from ipypublish.scripts.nbmerge import merge_notebooks
 from ipypublish.convert.config_manager import (get_export_config_path,
                                                load_export_config,
@@ -81,7 +81,7 @@ def publish(ipynb_path,
     if isinstance(ipynb_path, string_types):
         ipynb_path = pathlib.Path(ipynb_path)
     ipynb_name = os.path.splitext(ipynb_path.name)[0]
-    files_folder = ipynb_name + '_files'
+    files_folder = get_valid_filename(ipynb_name) + '_files'
     outdir = os.path.join(
         os.getcwd(), 'converted') if outpath is None else outpath
 
@@ -197,6 +197,9 @@ def create_config(exporter_data, template_name, replacements):
     files_path = "${files_path}"
     for instr, outstr in replacements.items():
         files_path = files_path.replace(instr, outstr)
+    # this ensured that the ExtractOutputPreprocessor sets extracted files to
+    # the required folder path
+    # (alternatively could set resources['output_files_dir'] = files_path)
     config[
         'ExtractOutputPreprocessor.output_filename_template'
     ] = files_path + '/{unique_key}_{cell_index}_{index}{extension}'

ipypublish/preprocessors/latex_doc_links.py

@@ -1,24 +1,51 @@
 import logging
 import os
+from binascii import a2b_base64
+import sys
+import json
+from mimetypes import guess_extension
 
+from six import string_types
 import traitlets as traits
 from nbconvert.preprocessors import Preprocessor
 
 
+def guess_extension_without_jpe(mimetype):
+    """
+    This function fixes a problem with '.jpe' extensions
+    of jpeg images which are then not recognised by latex.
+    For any other case, the function works in the same way
+    as mimetypes.guess_extension
+    """
+    ext = guess_extension(mimetype)
+    if ext == ".jpe":
+        ext = ".jpeg"
+    return ext
+
+
 class LatexDocLinks(Preprocessor):
-    """ a preprocessor to resolve file paths in the ipub metadata section
+    """ a preprocessor to resolve file paths in the notebook:
 
-    retrieve external file paths from metadata,
-    resolve where they are, if the path is relative
-    make sure that the link points to a single folder
-    add 'external_file_paths' and 'bibliopath' (if present) to resources
+    - retrieve external file paths from ipub metadata section,
+    - resolve where they are, if the path is relative
+    - make sure that the link points to a single folder
+    - add 'external_file_paths' and 'bibliopath' (if present) to resources
+    - extract attachments from input cells and rename their file links
 
     """
 
     metapath = traits.Unicode(
         '', help="the path to the meta data").tag(config=True)
     filesfolder = traits.Unicode(
         '', help="the folder to point towards").tag(config=True)
+    extract_attachments = traits.Bool(
+        True,
+        help=("extract attachments "
+              "(created by dragging and dropping files into markdown cells)")
+    ).tag(config=True)
+    output_attachment_template = traits.Unicode(
+        "{unique_key}_{cell_index}_{key}{extension}"
+    ).tag(config=True)
 
     def resolve_path(self, fpath, filepath):
         """resolve a relative path, w.r.t. another filepath """
@@ -28,12 +55,95 @@ def resolve_path(self, fpath, filepath):
 
         return fpath
 
+    def preprocess_cell(self, cell, resources, cell_index):
+        """
+        Extract attachment
+
+        Parameters
+        ----------
+        cell : nbformat.notebooknode.NotebookNode
+            Notebook cell being processed
+        resources : dict
+            Additional resources used in the conversion process.  Allows
+            preprocessors to pass variables into the Jinja engine.
+        cell_index : int
+            Index of the cell being processed (see base.py)
+        """
+        unique_key = resources.get('unique_key', 'attach')
+
+        if 'attachments' in cell and self.extract_attachments:
+            attachments = cell.pop('attachments')
+
+            for key, attachment in attachments.items():
+                # TODO this only works if there is a single MIME bundle
+                (mime_type, data), = attachment.items()
+
+                ext = guess_extension_without_jpe(mime_type)
+                if ext is None:
+                    ext = '.' + mime_type.rsplit('/')[-1]
+
+                # replace the pointer to the attachment
+                filepath = os.path.normpath(
+                    os.path.join(self.filesfolder,
+                                 self.output_attachment_template.format(
+                                     unique_key=unique_key,
+                                     cell_index=cell_index,
+                                     key=os.path.splitext(key)[0],
+                                     extension=ext))
+                )
+                if "source" in cell:
+                    cell["source"] = cell["source"].replace(
+                        'attachment:{}'.format(key), filepath)
+
+                # code taken from nbconvert.ExtractOutputPreprocessor
+                if (
+                    not isinstance(data, string_types)
+                    or mime_type == 'application/json'
+                ):
+                    # Data is either JSON-like and was parsed into a Python
+                    # object according to the spec, or data is for sure
+                    # JSON.
+                    # In the latter case we want to go extra sure that
+                    # we enclose a scalar string value into extra quotes by
+                    # serializing it properly.
+                    data = json.dumps(data)
+
+                # Binary files are base64-encoded, SVG is already XML
+                if mime_type in {
+                        'image/png', 'image/jpeg', 'application/pdf'}:
+                    # data is b64-encoded as text (str, unicode),
+                    # we want the original bytes
+                    data = a2b_base64(data)
+                elif sys.platform == 'win32':
+                    data = data.replace('\n', '\r\n').encode("UTF-8")
+                else:
+                    data = data.encode("UTF-8")
+
+                if filepath in resources['outputs']:
+                    raise ValueError(
+                        "Your outputs have filename metadata associated "
+                        "with them. Nbconvert saves these outputs to "
+                        "external files using this filename metadata. "
+                        "Filenames need to be unique across the notebook, "
+                        "or images will be overwritten. The filename {} is"
+                        " associated with more than one output. The second"
+                        " output associated with this filename is in cell "
+                        "{}.".format(filepath, cell_index)
+                    )
+                # In the resources, make the figure available
+                resources['outputs'][filepath] = data
+
+        return cell, resources
+
     def preprocess(self, nb, resources):
+        """
+        Preprocessing to apply on each notebook.
+        """
 
         logging.info('resolving external file paths' +
                      ' in ipub metadata to: {}'.format(self.metapath))
         external_files = []
-        if hasattr(nb.metadata, 'ipub'):
+        if 'ipub' in nb.metadata:
 
             # if hasattr(nb.metadata.ipub, 'files'):
             #     mfiles = []
@@ -78,4 +188,8 @@ def preprocess(self, nb, resources):
         resources.setdefault("external_file_paths", [])
         resources['external_file_paths'] += external_files
 
+        for index, cell in enumerate(nb.cells):
+            nb.cells[index], resources = self.preprocess_cell(
+                cell, resources, index)
+
         return nb, resources

ipypublish/schema/doc_metadata.schema.json

@@ -59,9 +59,16 @@
                     }
                 },
                 "toc": {
-                    "type": "boolean",
+                    "type": ["boolean", "object"],
                     "description": "output a table of contents",
-                    "default": "false"
+                    "default": "false",
+                    "properties": {
+                        "depth": {
+                            "description": "the depth of subheaders to show",
+                            "type": "number",
+                            "minimum": 1
+                        }
+                    }
                 },
                 "listfigures": {
                     "type": "boolean",

ipypublish/templates/segments/ipy-front_pages.latex-tpl.json

@@ -127,6 +127,14 @@
       "((*- endif *))",
       ""
     ],
+    "document_commands": [
+      "",
+      "((* if nb.metadata.ipub and nb.metadata.ipub.toc is mapping and 'depth' in nb.metadata.ipub.toc: -*))",
+      "% set depth of table of contents",
+      "\\setcounter{tocdepth}{((( nb.metadata.ipub.toc.depth )))}",
+      "((*- endif *))",
+      ""
+    ],
     "document_predoc": [
       "",
       "((*- if nb.metadata[\"ipub\"]: -*))",

ipypublish/tests/conftest.py

@@ -17,6 +17,18 @@ def ipynb2():
     return pathlib.Path(os.path.join(TEST_FILES_DIR, 'ipynb2.ipynb'))
 
 
+@pytest.fixture
+def ipynb_with_attach():
+    return {
+        "input_file": pathlib.Path(os.path.join(
+            TEST_FILES_DIR, 'nb_with_attachment',
+            'nb_with_attachment.ipynb')),
+        "latex_ipypublish_main": pathlib.Path(os.path.join(
+            TEST_FILES_DIR, 'nb_with_attachment',
+            'latex_ipypublish_main.tex'))
+    }
+
+
 @pytest.fixture
 def ipynb_with_bib():
     return pathlib.Path(os.path.join(TEST_FILES_DIR, 'ipynb_with_bib.ipynb'))
@@ -66,22 +78,16 @@ def ipynb_folder_with_external(temp_folder):
                                  'logo_example.png'),
                     os.path.join(folder, 'logo_example.png'))
 
-    yield folder
-
-
-@pytest.fixture
-def tex_with_external():
-    return pathlib.Path(os.path.join(TEST_FILES_DIR, 'ipynb_with_external',
-                                     'ipynb_with_external.tex'))
-
-
-@pytest.fixture
-def html_with_external():
-    return pathlib.Path(os.path.join(TEST_FILES_DIR, 'ipynb_with_external',
+    tex = pathlib.Path(os.path.join(TEST_FILES_DIR, 'ipynb_with_external',
+                                    'ipynb_with_external.tex'))
+    html = pathlib.Path(os.path.join(TEST_FILES_DIR, 'ipynb_with_external',
                                      'ipynb_with_external.html'))
-
-
-@pytest.fixture
-def slides_with_external():
-    return pathlib.Path(os.path.join(TEST_FILES_DIR, 'ipynb_with_external',
-                                     'ipynb_with_external.slides.html'))
+    slides = pathlib.Path(os.path.join(TEST_FILES_DIR, 'ipynb_with_external',
+                                       'ipynb_with_external.slides.html'))
+
+    yield {
+        "input_folder": folder,
+        "latex_ipypublish_main": tex,
+        "html_ipypublish_main": html,
+        "slides_ipypublish_main": slides
+    }

ipypublish/tests/test_files/ipynb_with_external/ipynb_with_external.ipynb

@@ -1297,7 +1297,7 @@
     "tagline": "A tagline for the report.",
     "title": "Main-Title"
    },
-   "toc": true
+   "toc": {"depth": 2}
   },
   "jupytext": {
    "metadata_filter": {

ipypublish/tests/test_files/ipynb_with_external/ipynb_with_external.tex

@@ -405,6 +405,9 @@
 \ofoot{\pagemark}
 \cfoot{}
 
+% set depth of table of contents
+\setcounter{tocdepth}{2}
+
 %%%%%%%%%%%%
 
 %%%%%%%%%%%% FINAL HEADER MATERIAL
@@ -615,7 +618,7 @@ \section{Equations (with ipython or
 \left(\sqrt{5} i\right)^{\alpha} \left(\frac{1}{2} - \frac{2 i}{5} \sqrt{5}\right) + \left(- \sqrt{5} i\right)^{\alpha} \left(\frac{1}{2} + \frac{2 i}{5} \sqrt{5}\right)
 \end{equation}
 
-\bibliographystyle{unsrtnat}
+      \bibliographystyle{unsrtnat}
 \bibliography{ipynb_with_external_files/example}
 
 \end{document}