From 9a2213b430a999f60e698af3781eee70cc510d4b Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Fri, 7 Sep 2018 01:51:35 +0200 Subject: [PATCH 1/6] Start cell marker simplified to '# +' when possible #57 --- README.md | 2 +- ...ired Jupyter notebook and python script.py | 2 +- demo/World population.py | 4 +-- jupytext/cell_reader.py | 18 ++++++++++--- jupytext/file_format_version.py | 20 +++++++++++--- jupytext/jupytext.py | 4 +++ tests/python_notebook_sample.py | 2 +- tests/test_cli.py | 1 + tests/test_contentsmanager.py | 1 + tests/test_load_multiple.py | 5 +++- tests/test_read_simple_python.py | 26 +++++++++---------- 11 files changed, 59 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index ae012f903..d25dfc23d 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ Implement these [specifications](https://rmarkdown.rstudio.com/articles_report_f We wanted to represent Jupyter notebooks with the least explicit markers possible. The rationale for that is to allow **arbitrary** python files to open as Jupyter notebooks, even files which were never prepared to become a notebook. Precisely: - Jupyter metadata go to an escaped YAML header - Markdown cells are commented with `# `, and separated with a blank line -- Code cells are exported verbatim (except for Jupyter magics, which are escaped), and separated with blank lines. Code cells are reconstructed from consistent python paragraphs (no function, class or multiline comment will be broken). A start-of-cell delimiter `# + {}` is used for cells that have explicit metadata (inside the curly bracket, in JSON format), and for cells that include blank lines (outside of functions, classes, etc). The end of cell delimiter is `# -`, and is omitted when followed by another explicit start of cell marker. +- Code cells are exported verbatim (except for Jupyter magics, which are escaped), and separated with blank lines. Code cells are reconstructed from consistent python paragraphs (no function, class or multiline comment will be broken). A start-of-cell delimiter `# + {}` is used for cells that have explicit metadata (inside the curly bracket, in JSON format), and `# +` is used for cells that include blank lines (outside of functions, classes, etc). The end of cell delimiter is `# -`, and is omitted when followed by another explicit start of cell marker. ## Will my notebook really run in an IDE? diff --git a/demo/Paired Jupyter notebook and python script.py b/demo/Paired Jupyter notebook and python script.py index 7f33475de..e36ed98dd 100644 --- a/demo/Paired Jupyter notebook and python script.py +++ b/demo/Paired Jupyter notebook and python script.py @@ -36,7 +36,7 @@ # %matplotlib inline -# + {} +# + import matplotlib.pyplot as plt import numpy as np diff --git a/demo/World population.py b/demo/World population.py index 1c0875712..b9ed8b132 100644 --- a/demo/World population.py +++ b/demo/World population.py @@ -26,7 +26,7 @@ # [World Bank](http://www.worldbank.org/) # using the [wbdata](https://github.com/OliverSherouse/wbdata) python package -# + {} +# + import pandas as pd import wbdata as wb @@ -90,7 +90,7 @@ # [on their way](https://github.com/plotly/plotly.js/pull/2960) at Plotly. For # now we just do a stacked bar plot. -# + {} +# + import plotly.offline as offline import plotly.graph_objs as go diff --git a/jupytext/cell_reader.py b/jupytext/cell_reader.py index 7a1126074..4539fd33b 100644 --- a/jupytext/cell_reader.py +++ b/jupytext/cell_reader.py @@ -12,6 +12,7 @@ _END_CODE_MD = re.compile(r"^```\s*$") _CODE_OPTION_R = re.compile(r"^#\+(.*)\s*$") _CODE_OPTION_PY = re.compile(r"^(#|# )\+(\s*){(.*)}\s*$") +_SIMPLE_START_CODE_PY = re.compile(r"^(#|# )\+(\s*)$") _BLANK_LINE = re.compile(r"^\s*$") _PY_COMMENT = re.compile(r"^\s*#") _PY_INDENTED = re.compile(r"^\s") @@ -123,10 +124,14 @@ def metadata_and_language_from_option_line(self, line): self.language, self.metadata = \ rmd_options_to_metadata('r ' + _CODE_OPTION_R.findall(line)[0]) - if self.ext in ['.py', '.jl'] and _CODE_OPTION_PY.match(line): - self.language = 'python' if self.ext == '.py' else 'julia' - self.metadata = json_options_to_metadata( - _CODE_OPTION_PY.match(line).group(3)) + if self.ext in ['.py', '.jl']: + if _CODE_OPTION_PY.match(line): + self.language = 'python' if self.ext == '.py' else 'julia' + self.metadata = json_options_to_metadata( + _CODE_OPTION_PY.match(line).group(3)) + if _SIMPLE_START_CODE_PY.match(line): + self.language = 'python' if self.ext == '.py' else 'julia' + self.metadata = {} if self.metadata and 'language' in self.metadata: self.language = self.metadata['language'] @@ -225,6 +230,11 @@ def find_cell_end_code(self, lines, cell_end_re, return i - 1, i, False return i, i, False + if i > 0 and cell_start_re == _CODE_OPTION_PY and \ + _BLANK_LINE.match(lines[i - 1]) and \ + _SIMPLE_START_CODE_PY.match(line): + return i - 1, i, False + if cell_end_re: if cell_end_re.match(line): return i, i + 1, True diff --git a/jupytext/file_format_version.py b/jupytext/file_format_version.py index f421c50f6..b70e4b50d 100644 --- a/jupytext/file_format_version.py +++ b/jupytext/file_format_version.py @@ -12,8 +12,10 @@ # Version 1.0 on 2018-08-31 - jupytext v0.6.0 : Initial version # Python and Julia scripts - '.py': '1.1', - '.jl': '1.1', + '.py': '1.2', + '.jl': '1.2', + # Version 1.2 on 2018-09-05 - jupytext v0.6.3 : Metadata bracket can be + # omitted when empty, if previous line is empty #57 # Version 1.1 on 2018-08-25 - jupytext v0.6.0 : Cells separated with one # blank line #38 # Version 1.0 on 2018-08-22 - jupytext v0.5.2 : Initial version @@ -23,6 +25,9 @@ # Version 1.0 on 2018-08-22 - jupytext v0.5.2 : Initial version } +MIN_FILE_FORMAT_VERSION = {'.Rmd': '1.0', '.md': '1.0', '.py': '1.1', + '.jl': '1.1', '.R': '1.0'} + FILE_FORMAT_VERSION_ORG = FILE_FORMAT_VERSION @@ -31,6 +36,11 @@ def file_format_version(ext): return FILE_FORMAT_VERSION.get(ext) +def min_file_format_version(ext): + """Return minimum file format version for given ext""" + return MIN_FILE_FORMAT_VERSION.get(ext) + + def check_file_version(notebook, source_path, outputs_path): """Raise if file version in source file would override outputs""" _, ext = os.path.splitext(source_path) @@ -47,7 +57,11 @@ def check_file_version(notebook, source_path, outputs_path): if version == current: return - # Not merging? OK + # Version larger than minimum readable version + if version <= current and version >= min_file_format_version(ext): + return + + # Not merging? OK if source_path == outputs_path: return diff --git a/jupytext/jupytext.py b/jupytext/jupytext.py index 605ee0448..5a770a42f 100644 --- a/jupytext/jupytext.py +++ b/jupytext/jupytext.py @@ -125,6 +125,10 @@ def writes(self, nb, **kwargs): for i, cell in enumerate(cells): text = texts[i] + # Simplify cell marker when previous line is blank + if text[0] == '# + {}' and (not lines or not lines[-1]): + text[0] = '# +' + # remove end of cell marker when redundant # with next explicit marker if self.ext in ['.py', '.jl'] and cell.is_code() \ diff --git a/tests/python_notebook_sample.py b/tests/python_notebook_sample.py index 7777410aa..f1c3ba379 100755 --- a/tests/python_notebook_sample.py +++ b/tests/python_notebook_sample.py @@ -31,7 +31,7 @@ def f(x): # metadata and an end-of-cell marker. Metadata information in json format, # escaped with '#+' or '# +' -# + {} +# + def g(x): return x + 2 diff --git a/tests/test_cli.py b/tests/test_cli.py index a98821132..9223ebd7d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -10,6 +10,7 @@ from .utils import list_all_notebooks, list_py_notebooks file_format_version.FILE_FORMAT_VERSION = {} +file_format_version.MIN_FILE_FORMAT_VERSION = {} @pytest.mark.parametrize('nb_file', diff --git a/tests/test_contentsmanager.py b/tests/test_contentsmanager.py index 871a30104..a27d83d8e 100644 --- a/tests/test_contentsmanager.py +++ b/tests/test_contentsmanager.py @@ -7,6 +7,7 @@ from .utils import list_all_notebooks, list_py_notebooks jupytext.file_format_version.FILE_FORMAT_VERSION = {} +jupytext.file_format_version.MIN_FILE_FORMAT_VERSION = {} @pytest.mark.skipif(isinstance(TextFileContentsManager, str), diff --git a/tests/test_load_multiple.py b/tests/test_load_multiple.py index c35fd092d..380842b69 100644 --- a/tests/test_load_multiple.py +++ b/tests/test_load_multiple.py @@ -60,4 +60,7 @@ def test_combine_lower_version_raises(tmpdir): with pytest.raises(HTTPError): with mock.patch('jupytext.file_format_version.FILE_FORMAT_VERSION', {'.py': '1.0'}): - cm.get(tmp_ipynb) + with mock.patch( + 'jupytext.file_format_version.MIN_FILE_FORMAT_VERSION', + {'.py': '1.0'}): + cm.get(tmp_ipynb) diff --git a/tests/test_read_simple_python.py b/tests/test_read_simple_python.py index a8c8e1078..8d37224d0 100644 --- a/tests/test_read_simple_python.py +++ b/tests/test_read_simple_python.py @@ -116,7 +116,7 @@ def test_read_cell_two_blank_lines(pynb="""# --- # title: cell with two consecutive blank lines # --- -# + {} +# + a = 1 @@ -137,7 +137,7 @@ def test_read_cell_two_blank_lines(pynb="""# --- def test_read_cell_explicit_start(pynb=''' import pandas as pd -# + {} +# + def data(): return pd.DataFrame({'A': [0, 1]}) @@ -151,21 +151,21 @@ def data(): def test_read_complex_cells(pynb='''import pandas as pd -# + {} +# + def data(): return pd.DataFrame({'A': [0, 1]}) data() -# + {} +# + def data2(): return pd.DataFrame({'B': [0, 1]}) data2() -# + {} +# + # Finally we have a cell with only comments # This cell should remain a code cell and not get converted # to markdown @@ -203,7 +203,7 @@ def data2(): def test_read_prev_function( pynb="""def test_read_cell_explicit_start_end(pynb=''' import pandas as pd -# + {} +# + def data(): return pd.DataFrame({'A': [0, 1]}) @@ -228,7 +228,7 @@ def test_read_cell_with_one_blank_line_end(pynb="""import pandas compare(pynb, pynb2) -def test_read_code_cell_fully_commented(pynb="""# + {} +def test_read_code_cell_fully_commented(pynb="""# + # This is a code cell that # only contains comments """): @@ -250,7 +250,7 @@ def test_file_with_two_blank_line_end(pynb="""import pandas compare(pynb, pynb2) -def test_one_blank_lines_after_endofcell(pynb="""# + {} +def test_one_blank_lines_after_endofcell(pynb="""# + # This is a code cell with explicit end of cell 1 + 1 @@ -275,13 +275,13 @@ def test_one_blank_lines_after_endofcell(pynb="""# + {} compare(pynb, pynb2) -def test_two_cells_with_explicit_start(pynb="""# + {} +def test_two_cells_with_explicit_start(pynb="""# + # Cell one 1 + 1 1 + 1 -# + {} +# + # Cell two 2 + 2 @@ -303,7 +303,7 @@ def test_two_cells_with_explicit_start(pynb="""# + {} compare(pynb, pynb2) -def test_escape_start_pattern(pynb="""# The code start pattern '# + {}' can +def test_escape_start_pattern(pynb="""# The code start pattern '# +' can # appear in code and markdown cells. # In markdown cells it is escaped like here: @@ -454,7 +454,7 @@ def test_read_write_script(pynb="""#!/usr/bin/env python compare(pynb, pynb2) -def test_notebook_blank_lines(script="""# + {} +def test_notebook_blank_lines(script="""# + # This is a comment # followed by two variables a = 3 @@ -466,7 +466,7 @@ def test_notebook_blank_lines(script="""# + {} c = 5 -# + {} +# + # Now we have two functions def f(x): return x + x From 1eb64914823f2ec088b6ad70493a38d2b55c9ff3 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Fri, 7 Sep 2018 01:55:46 +0200 Subject: [PATCH 2/6] Testing Python 3.7 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 877b54bca..4af8fc718 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - "3.4" - "3.5" - "3.6" + - "3.7" # PyPy versions - "pypy3.5" # command to install dependencies From e015361baec178cd712eabd6ad6df19fcb40ce0c Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Fri, 7 Sep 2018 02:09:06 +0200 Subject: [PATCH 3/6] min_file_format_version required for that test --- tests/test_cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9223ebd7d..22b042e1e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -173,4 +173,7 @@ def test_combine_lower_version_raises(tmpdir): with pytest.raises(SystemExit): with mock.patch('jupytext.file_format_version.FILE_FORMAT_VERSION', {'.py': '1.0'}): - jupytext(args=[tmp_nbpy, '--to', 'ipynb', '--update']) + with mock.patch( + 'jupytext.file_format_version.MIN_FILE_FORMAT_VERSION', + {'.py': '1.0'}): + jupytext(args=[tmp_nbpy, '--to', 'ipynb', '--update']) From e640e7a2ee61f224f342c5fe17b92a34324a7b08 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Fri, 7 Sep 2018 02:10:02 +0200 Subject: [PATCH 4/6] Python 3.7 not yet in Travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4af8fc718..877b54bca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ python: - "3.4" - "3.5" - "3.6" - - "3.7" # PyPy versions - "pypy3.5" # command to install dependencies From 2087123dfcfa6b7c25676ce332e0db1b37affad7 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Fri, 7 Sep 2018 02:11:20 +0200 Subject: [PATCH 5/6] Update demo notebooks --- binder/requirements.txt | 2 +- demo/Paired Jupyter notebook and python script.py | 2 +- demo/World population.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/binder/requirements.txt b/binder/requirements.txt index c811e5871..e67c4f726 100644 --- a/binder/requirements.txt +++ b/binder/requirements.txt @@ -1,4 +1,4 @@ -jupytext>=0.6.2 +jupytext>=0.6.3 plotly matplotlib pandas diff --git a/demo/Paired Jupyter notebook and python script.py b/demo/Paired Jupyter notebook and python script.py index e36ed98dd..e4b94e688 100644 --- a/demo/Paired Jupyter notebook and python script.py +++ b/demo/Paired Jupyter notebook and python script.py @@ -14,7 +14,7 @@ # nbconvert_exporter: python # pygments_lexer: ipython3 # version: 3.6.5 -# jupytext_format_version: '1.1' +# jupytext_format_version: '1.2' # jupytext_formats: ipynb,py # --- diff --git a/demo/World population.py b/demo/World population.py index b9ed8b132..d08c0bff7 100644 --- a/demo/World population.py +++ b/demo/World population.py @@ -1,6 +1,6 @@ # --- # jupyter: -# jupytext_format_version: '1.1' +# jupytext_format_version: '1.2' # jupytext_formats: ipynb,py,md # kernelspec: # display_name: Python 3 From 130889f50dcf791053bca06819087bc16864ef84 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Fri, 7 Sep 2018 02:18:40 +0200 Subject: [PATCH 6/6] Version 0.6.3 --- HISTORY.rst | 13 +++++++++++++ setup.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6b110c0a5..883324e06 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,19 @@ Release History dev +++ +0.6.3 (2018-09-07) ++++++++++++++++++++ + +**Improvements** + +- Lighter cell markers for Python and Julia scripts (#57). Corresponding file +format version at 1.2. Scripts in previous version 1.1 can still be opened. +- New screenshots for the README. + +**BugFixes** + +- Command line conversion tool `jupytext` fixed on Python 2.7 (#46) + 0.6.2 (2018-09-05) +++++++++++++++++++ diff --git a/setup.py b/setup.py index 68d3db5b8..de2bb17ec 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='jupytext', - version='0.6.2', + version='0.6.3', author='Marc Wouts', author_email='marc.wouts@gmail.com', description='Jupyter notebooks as Markdown documents, '