Jupytext metadata moved to a jupytext section #91

HISTORY.rst

@@ -3,6 +3,21 @@
 Release History
 ---------------
 
+0.7.3 (2018-10-??)
+++++++++++++++++++++++
+
+**Improvements**
+
+- All `jupytext` related metadata goes to a `jupytext` section (#91)
+- Notebooks extensions can be prefixed with any prefix of at most three chars (#87)
+- Export of the same notebook to multiple formats is now supported. To export to all python formats,
+plus `.ipynb` and `.md`, use `"jupytext_formats": "ipynb,pct.py:percent,lgt.py:light,spx.py:sphinx,md",`.
+
+**BugFixes**
+
+- Trusting notebooks made functional again.
+- Command line `jupytext` returns a meaningful error when no argument is given.
+
 0.7.2 (2018-10-01)
 ++++++++++++++++++++++
 

demo/World population.ipynb

@@ -1452,7 +1452,9 @@
   }
  ],
  "metadata": {
-  "jupytext_formats": "ipynb,pct.py:percent,lgt.py:light,spx.py:sphinx,md",
+  "jupytext": {
+   "formats": "ipynb,pct.py:percent,lgt.py:light,spx.py:sphinx,md"
+  },
   "kernelspec": {
    "display_name": "Python 3",
    "language": "python",

demo/World population.lgt.py

@@ -1,7 +1,10 @@
 # ---
 # jupyter:
-#   jupytext_format_version: '1.3'
-#   jupytext_formats: ipynb,pct.py:percent,lgt.py:light,spx.py:sphinx,md
+#   jupytext:
+#     formats: ipynb,pct.py:percent,lgt.py:light,spx.py:sphinx,md
+#     this_document:
+#       format_name: light
+#       format_version: '1.3'
 #   kernelspec:
 #     display_name: Python 3
 #     language: python

demo/World population.md

@@ -1,7 +1,10 @@
 ---
 jupyter:
-  jupytext_format_version: '1.0'
-  jupytext_formats: ipynb,pct.py:percent,lgt.py:light,spx.py:sphinx,md:markdown
+  jupytext:
+    formats: ipynb,pct.py:percent,lgt.py:light,spx.py:sphinx,md:markdown
+    this_document:
+      format_name: markdown
+      format_version: '1.0'
   kernelspec:
     display_name: Python 3
     language: python

demo/World population.pct.py

@@ -1,7 +1,10 @@
 # ---
 # jupyter:
-#   jupytext_format_version: '1.1'
-#   jupytext_formats: ipynb,pct.py:percent,lgt.py:light,spx.py:sphinx,md
+#   jupytext:
+#     formats: ipynb,pct.py:percent,lgt.py:light,spx.py:sphinx,md
+#     this_document:
+#       format_name: percent
+#       format_version: '1.1'
 #   kernelspec:
 #     display_name: Python 3
 #     language: python

demo/World population.spx.py

@@ -1,7 +1,10 @@
 # ---
 # jupyter:
-#   jupytext_format_version: '1.1'
-#   jupytext_formats: ipynb,pct.py:percent,lgt.py:light,spx.py:sphinx,md
+#   jupytext:
+#     formats: ipynb,pct.py:percent,lgt.py:light,spx.py:sphinx,md
+#     this_document:
+#       format_name: sphinx
+#       format_version: '1.1'
 #   kernelspec:
 #     display_name: Python 3
 #     language: python

jupytext/compare.py

@@ -21,8 +21,7 @@ def filtered_cell(cell, preserve_outputs):
 
 def filtered_notebook_metadata(notebook):
     """Notebook metadata, filtered for metadata added by Jupytext itself"""
-    return {key: notebook.metadata[key] for key
-            in notebook.metadata if not key.startswith('jupytext_')}
+    return {key: notebook.metadata[key] for key in notebook.metadata if key != 'jupytext'}
 
 
 def compare_notebooks(notebook_expected, notebook_actual,

jupytext/contentsmanager.py

@@ -19,7 +19,7 @@
 import jupytext
 from .combine import combine_inputs_with_outputs
 from .formats import check_file_version, NOTEBOOK_EXTENSIONS, \
-    format_name_for_ext, parse_one_format, parse_formats
+    format_name_for_ext, parse_one_format, parse_formats, transition_to_jupytext_section_in_metadata
 
 
 def _jupytext_writes(ext, format_name):
@@ -85,7 +85,7 @@ def check_formats(formats):
                                  .format(str(group), fmt,
                                          str(NOTEBOOK_EXTENSIONS),
                                          expected_format))
-            if fmt == '.ipynb':
+            if fmt.endswith('.ipynb'):
                 has_ipynb = True
             else:
                 validated_group.append(fmt)
@@ -166,14 +166,10 @@ def all_nb_extensions(self):
 
     def format_group(self, fmt, nbk=None):
         """Return the group of extensions that contains 'fmt'"""
-        # Backward compatibility with nbrmd
-        for key in ['nbrmd_formats', 'nbrmd_format_version']:
-            if nbk and key in nbk.metadata:
-                nbk.metadata[key.replace('nbrmd', 'jupytext')] = \
-                    nbk.metadata.pop(key)
-
-        jupytext_formats = ((nbk.metadata.get('jupytext_formats')
-                             if nbk else None) or
+        if nbk:
+            transition_to_jupytext_section_in_metadata(nbk.metadata, fmt.endswith('.ipynb'))
+
+        jupytext_formats = ((nbk.metadata.get('jupytext', {}).get('formats') if nbk else None) or
                             self.default_jupytext_formats)
 
         try:

jupytext/formats.py

@@ -118,7 +118,7 @@ def get_format(ext, format_name=None):
     # remove pre-extension if any
     ext = '.' + ext.split('.')[-1]
 
-    if ext == '.ipynb':
+    if ext.endswith('.ipynb'):
         return None
 
     formats_for_extension = []
@@ -143,7 +143,14 @@ def read_format_from_metadata(text, ext):
     metadata, _, _ = header_to_metadata_and_cell(
         lines, "#'" if ext == '.R' else '#')
 
-    if set(metadata).difference(['encoding', 'main_language']):
+    transition_to_jupytext_section_in_metadata(metadata, ext.endswith('.ipynb'))
+
+    format_name = metadata.get('jupytext', {}).get('this_document', {}).get('format_name')
+    if format_name:
+        return format_name
+
+    if ('jupytext' in metadata and set(metadata['jupytext']).difference(['encoding', 'main_language'])) or \
+            set(metadata).difference(['jupytext']):
         return format_name_for_ext(metadata, ext)
 
     return None
@@ -156,7 +163,8 @@ def guess_format(text, ext):
     metadata, _, _ = header_to_metadata_and_cell(
         lines, "#'" if ext == '.R' else '#')
 
-    if set(metadata).difference(['encoding', 'main_language']):
+    if ('jupytext' in metadata and set(metadata['jupytext']).difference(['encoding', 'main_language'])) or \
+            set(metadata).difference(['jupytext']):
         return format_name_for_ext(metadata, ext)
 
     # Is this a Hydrogen-like script?
@@ -198,12 +206,12 @@ def check_file_version(notebook, source_path, outputs_path):
         return
 
     _, ext = os.path.splitext(source_path)
-    if ext == '.ipynb':
+    if ext.endswith('.ipynb'):
         return
-    version = notebook.metadata.get('jupytext_format_version')
+    version = notebook.metadata.get('jupytext', {}).get('this_document', {}).get('format_version')
     format_name = format_name_for_ext(notebook.metadata, ext)
-    if version:
-        del notebook.metadata['jupytext_format_version']
+    if 'this_document' in notebook.metadata['jupytext']:
+        del notebook.metadata['jupytext']['this_document']
 
     fmt = get_format(ext, format_name)
     current = fmt.current_version_number
@@ -289,7 +297,7 @@ def formats_as_string(formats):
 
 def format_name_for_ext(metadata, ext, explicit_default=True):
     """Return the format name for that extension"""
-    formats = metadata.get('jupytext_formats', '')
+    formats = metadata.get('jupytext', {}).get('formats', '')
     formats = parse_formats(formats)
     for fmt_ext, ext_format_name in formats:
         if fmt_ext.endswith(ext):
@@ -304,6 +312,23 @@ def format_name_for_ext(metadata, ext, explicit_default=True):
 
 def update_jupytext_formats_metadata(notebook, ext, format_name):
     """Update the jupytext_format metadata in the Jupyter notebook"""
-    formats = parse_formats(notebook.metadata.get('jupytext_formats', ''))
+    formats = parse_formats(notebook.metadata.get('jupytext', {}).get('formats', ''))
     formats = update_formats(formats, ext, format_name)
-    notebook.metadata['jupytext_formats'] = formats_as_string(formats)
+    notebook.metadata.setdefault('jupytext', {})['formats'] = formats_as_string(formats)
+
+
+def transition_to_jupytext_section_in_metadata(metadata, is_ipynb):
+    """Convert the jupytext_formats metadata entry to jupytext/formats, etc. See #91"""
+
+    # Backward compatibility with nbrmd
+    for key in ['nbrmd_formats', 'nbrmd_format_version']:
+        if key in metadata:
+            metadata[key.replace('nbrmd', 'jupytext')] = metadata.pop(key)
+
+    if 'jupytext_formats' in metadata:
+        metadata.setdefault('jupytext', {})['formats'] = metadata.pop('jupytext_formats')
+    if 'jupytext_format_version' in metadata:
+        metadata.setdefault('jupytext', {})['this_document'] = {'format_version': metadata.pop('jupytext_format_version')}
+    for entry in ['encoding', 'executable']:
+        if is_ipynb and entry in metadata:
+            metadata.setdefault('jupytext', {})[entry] = metadata.pop(entry)

jupytext/header.py

@@ -49,7 +49,7 @@ def encoding_and_executable(notebook, ext):
     :return:
     """
     lines = []
-    metadata = notebook.get('metadata', {})
+    metadata = notebook.get('metadata', {}).get('jupytext', {})
     comment = _SCRIPT_EXTENSIONS.get(ext, {}).get('comment')
 
     if ext not in ['.Rmd', '.md'] and 'executable' in metadata:
@@ -93,12 +93,14 @@ def metadata_and_cell_to_header(notebook, text_format):
     metadata = _as_dict(notebook.get('metadata', {}))
 
     if insert_or_test_version_number():
-        metadata['jupytext_format_version'] = \
-            text_format.current_version_number
+        metadata.setdefault('jupytext', {})['this_document'] = {'format_name': text_format.format_name,
+                                                           'format_version': text_format.current_version_number}
+
+    if 'jupytext' in metadata and not metadata['jupytext']:
+        del metadata['jupytext']
 
     if metadata:
-        header.extend(yaml.safe_dump({'jupyter': metadata},
-                                     default_flow_style=False).splitlines())
+        header.extend(yaml.safe_dump({'jupyter': metadata}, default_flow_style=False).splitlines())
 
     if header:
         header = ['---'] + header + ['---']
@@ -130,16 +132,15 @@ def header_to_metadata_and_cell(lines, header_prefix):
 
     for i, line in enumerate(lines):
         if i == 0 and line.startswith(comment + '!'):
-            metadata['executable'] = line[2:]
+            metadata.setdefault('jupytext', {})['executable'] = line[2:]
             start = i + 1
             continue
         if i == 0 or (i == 1 and not encoding_re.match(lines[0])):
             encoding = encoding_re.match(line)
             if encoding:
                 if encoding.group(1) != 'utf-8':
-                    raise ValueError('Encodings other than utf-8 '
-                                     'are not supported')
-                metadata['encoding'] = line
+                    raise ValueError('Encodings other than utf-8 are not supported')
+                metadata.setdefault('jupytext', {})['encoding'] = line
                 start = i + 1
                 continue
 

jupytext/jupytext.py

@@ -16,7 +16,7 @@
 from nbformat.v4.nbbase import new_notebook, new_code_cell
 import nbformat
 from .formats import get_format, read_format_from_metadata, guess_format, \
-    update_jupytext_formats_metadata, format_name_for_ext
+    update_jupytext_formats_metadata, format_name_for_ext, transition_to_jupytext_section_in_metadata
 from .header import header_to_metadata_and_cell, metadata_and_cell_to_header, \
     encoding_and_executable, insert_or_test_version_number
 from .languages import default_language_from_metadata_and_ext, \
@@ -113,7 +113,7 @@ def writes(self, nb, **kwargs):
 def reads(text, ext, format_name=None,
           rst2md=False, as_version=4, **kwargs):
     """Read a notebook from a string"""
-    if ext == '.ipynb':
+    if ext.endswith('.ipynb'):
         return nbformat.reads(text, as_version, **kwargs)
 
     format_name = read_format_from_metadata(text, ext) or format_name
@@ -125,6 +125,8 @@ def reads(text, ext, format_name=None,
 
     reader = TextNotebookReader(ext, format_name)
     notebook = reader.reads(text, **kwargs)
+    transition_to_jupytext_section_in_metadata(notebook.metadata, False)
+
     if format_name and insert_or_test_version_number():
         if format_name == 'sphinx-rst2md' and rst2md:
             format_name = 'sphinx'
@@ -135,11 +137,12 @@ def reads(text, ext, format_name=None,
 
 def read(file_or_stream, ext, format_name=None, as_version=4, **kwargs):
     """Read a notebook from a file"""
-    if ext == '.ipynb':
-        return nbformat.read(file_or_stream, as_version, **kwargs)
+    if ext.endswith('.ipynb'):
+        notebook = nbformat.read(file_or_stream, as_version, **kwargs)
+        transition_to_jupytext_section_in_metadata(notebook.metadata, True)
+        return notebook
 
-    return reads(file_or_stream.read(), ext=ext, format_name=format_name,
-                 **kwargs)
+    return reads(file_or_stream.read(), ext=ext, format_name=format_name, **kwargs)
 
 
 def readf(nb_file, format_name=None):
@@ -152,7 +155,9 @@ def readf(nb_file, format_name=None):
 def writes(notebook, ext, format_name=None,
            version=nbformat.NO_CONVERT, **kwargs):
     """Write a notebook to a string"""
-    if ext == '.ipynb':
+    transition_to_jupytext_section_in_metadata(notebook.metadata, ext.endswith('.ipynb'))
+
+    if ext.endswith('.ipynb'):
         return nbformat.writes(notebook, version, **kwargs)
 
     if not format_name:
@@ -168,7 +173,9 @@ def writes(notebook, ext, format_name=None,
 def write(notebook, file_or_stream, ext, format_name=None,
           version=nbformat.NO_CONVERT, **kwargs):
     """Write a notebook to a file"""
-    if ext == '.ipynb':
+    transition_to_jupytext_section_in_metadata(notebook.metadata, ext.endswith('.ipynb'))
+
+    if ext.endswith('.ipynb'):
         return nbformat.write(notebook, file_or_stream, version, **kwargs)
 
     if not format_name:
@@ -177,7 +184,8 @@ def write(notebook, file_or_stream, ext, format_name=None,
     if format_name and insert_or_test_version_number():
         update_jupytext_formats_metadata(notebook, ext, format_name)
 
-    return TextNotebookWriter(ext, format_name).write(notebook, file_or_stream)
+    writer = TextNotebookWriter(ext, format_name)
+    return writer.write(notebook, file_or_stream)
 
 
 def writef(notebook, nb_file, format_name=None):

setup.py

@@ -8,7 +8,7 @@
 
 setup(
     name='jupytext',
-    version='0.7.2',
+    version='0.7.3',
     author='Marc Wouts',
     author_email='marc.wouts@gmail.com',
     description='Jupyter notebooks as Markdown documents, '

tests/notebooks/mirror/Rmd_to_ipynb/R_sample.ipynb

@@ -45,4 +45,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

tests/notebooks/mirror/Rmd_to_ipynb/chunk_options.ipynb

@@ -16,6 +16,7 @@
    "execution_count": null,
    "metadata": {
     "hide_output": true,
+    "lines_to_next_cell": 2,
     "name": "knitr_setup"
    },
    "outputs": [],
@@ -24,13 +25,6 @@
     "knitr::opts_chunk$set(echo = FALSE, fig.width = 10, fig.height = 5)"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "noskipline": true
-   },
-   "source": []
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -50,8 +44,8 @@
     "fig.height": 5,
     "fig.width": 8,
     "hide_input": true,
-    "name": "bar_plot",
-    "noskipline": true
+    "lines_to_next_cell": 0,
+    "name": "bar_plot"
    },
    "outputs": [],
    "source": [