Any instruction starting with % is magic #94

HISTORY.rst

@@ -9,10 +9,11 @@ Release History
 **Improvements**
 
 - All ``jupytext`` related metadata goes to a ``jupytext`` section (#91). Please make sure your collaborators use the same version of Jupytext, as the new version can read previous metadata, but not the opposite.
-- Notebooks extensions can be prefixed with any prefix of at most three chars (#87)
+- Notebooks extensions can be prefixed with any prefix of at most three chars (#87).
 - Export of the same notebook to multiple formats is now supported. To export to all python formats, plus ``.ipynb`` and ``.md``, use ``"jupytext": {"formats": "ipynb,pct.py:percent,lgt.py:light,spx.py:sphinx,md"},``.
-- README includes a short section on how to extend ``light`` and ``percent`` formats to more languages (#61)
-- Jupytext's contents manager accepts the ``auto`` extension in ``default_jupytext_formats`` (#93)
+- README includes a short section on how to extend ``light`` and ``percent`` formats to more languages (#61).
+- Jupytext's contents manager accepts the ``auto`` extension in ``default_jupytext_formats`` (#93).
+- All Jupyter magics are escaped in ``light`` scripts and R markdown documents. Escape magics in other formats with a ``comment_magics`` metadata (true or false), or with the contents manager ``comment_magics`` global flag (#94).
 
 **BugFixes**
 

README.md

@@ -252,7 +252,7 @@ That being said, using Jupytext from Jupyter Lab is also an option. Please note
 
 ## Will my notebook really run in an IDE?
 
-Well, that's what we expect. There's however a big difference in the python environments between Python IDEs and Jupyter: in most IDEs the code is executed with  `python` and not in a Jupyter kernel. For this reason, `jupytext` comments Jupyter magics found in your notebook when exporting to R Markdown, and to scripts in all format but the `percent` one. Magics are not commented in the plain Markdown representation, nor in the `percent` format, as some editors use that format in combination with Jupyter kernels. Change this by adding a `"comment_magics": true` or `false` entry in the notebook metadata, in the `"jupytext"` section. Or set your preference globally on the contents manager by adding this line to `.jupyter/jupyter_notebook_config.py`:
+Well, that's what we expect. There's however a big difference in the python environments between Python IDEs and Jupyter: in most IDEs the code is executed with  `python` and not in a Jupyter kernel. For this reason, `jupytext` comments Jupyter magics found in your notebook when exporting to R Markdown, and to scripts in all format but the `percent` one. Magics are not commented in the plain Markdown representation, nor in the `percent` format, as some editors use that format in combination with Jupyter kernels. Change this by adding a `#escape` or `#noescape` flag on the same line as the magic, or a "comment_magics": true` or `false` entry in the notebook metadata, in the `"jupytext"` section. Or set your preference globally on the contents manager by adding this line to `.jupyter/jupyter_notebook_config.py`:
 ```python
 c.ContentsManager.comment_magics = True # or False
 ```

jupytext/cell_to_text.py

@@ -100,9 +100,7 @@ def code_to_text(self):
         """Return the text representation of a code cell"""
         source = copy(self.source)
         escape_code_start(source, self.ext, self.language)
-
-        if self.comment_magics:
-            comment_magic(source, self.language)
+        comment_magic(source, self.language, self.comment_magics)
 
         options = []
         if self.cell_type == 'code' and self.language:
@@ -127,8 +125,8 @@ def code_to_text(self):
         source = copy(self.source)
         escape_code_start(source, self.ext, self.language)
 
-        if active and self.comment_magics:
-            comment_magic(source, self.language)
+        if active:
+            comment_magic(source, self.language, self.comment_magics)
 
         lines = []
         if not is_active('Rmd', self.metadata):
@@ -176,8 +174,7 @@ def code_to_text(self):
         escape_code_start(source, self.ext, self.language)
 
         if active:
-            if self.comment_magics:
-                comment_magic(source, self.language)
+            comment_magic(source, self.language, self.comment_magics)
         else:
             source = [self.comment + ' ' + line if line else self.comment for line in source]
 
@@ -246,8 +243,8 @@ def code_to_text(self):
         source = copy(self.source)
         escape_code_start(source, self.ext, self.language)
 
-        if active and self.comment_magics:
-            comment_magic(source, self.language)
+        if active:
+            comment_magic(source, self.language, self.comment_magics)
 
         if not active:
             source = ['# ' + line if line else '#' for line in source]
@@ -287,8 +284,7 @@ def cell_to_text(self):
 
         if self.cell_type == 'code':
             source = copy(self.source)
-            if self.comment_magics:
-                comment_magic(source, self.language)
+            comment_magic(source, self.language, self.comment_magics)
             return lines + source
 
         return lines + comment_lines(self.source, self.comment)
@@ -313,8 +309,7 @@ def cell_to_text(self):
         """Return the text representation for the cell"""
         if self.cell_type == 'code':
             source = copy(self.source)
-            if self.comment_magics:
-                comment_magic(source, self.language)
+            comment_magic(source, self.language, self.comment_magics)
             return source
 
         if 'cell_marker' in self.metadata:

jupytext/magics.py

@@ -4,63 +4,41 @@
 from .stringparser import StringParser
 from .languages import _SCRIPT_EXTENSIONS
 
-# Line magics retrieved manually (Aug 18) with %lsmagic
-_LINE_MAGICS = '%alias  %alias_magic  %autocall  %automagic  %autosave  ' \
-               '%bookmark  %cd  %clear  %cls  %colors  %config  ' \
-               '%connect_info  %copy  %ddir  %debug  %dhist  %dirs  ' \
-               '%doctest_mode  %echo  %ed  %edit  %env  %gui  %hist  ' \
-               '%history  %killbgscripts  %ldir  %less  %load  %load_ext  ' \
-               '%loadpy  %logoff  %logon  %logstart  %logstate  %logstop  ' \
-               '%ls  %lsmagic  %macro  %magic  %matplotlib  %mkdir  %more  ' \
-               '%notebook  %page  %pastebin  %pdb  %pdef  %pdoc  %pfile  ' \
-               '%pinfo  %pinfo2  %popd  %pprint  %precision  %profile  ' \
-               '%prun  %psearch  %psource  %pushd  %pwd  %pycat  %pylab  ' \
-               '%qtconsole  %quickref  %recall  %rehashx  %reload_ext  ' \
-               '%ren  %rep  %rerun  %reset  %reset_selective  %rmdir  %run  ' \
-               '%save  %sc  %set_env  %store  %sx  %system  %tb  %time  ' \
-               '%timeit  %unalias  %unload_ext  %who  %who_ls  %whos  ' \
-               '%xdel  %xmode'.split('  ')
-
-# Add classical line magics
-_LINE_MAGICS += ('%autoreload %aimport '  # autoreload
-                 '%R %Rpush %Rpull %Rget '  # rmagic
-                 '%store '  # storemagic
-                 ).split(' ')
-
-# Remove any blank line
-_LINE_MAGICS = [magic for magic in _LINE_MAGICS if magic.startswith('%')]
-
-# A magic expression is a line or cell magic escaped zero, or multiple times
-_FORCE_ESC_RE = {_SCRIPT_EXTENSIONS[ext]['language']: re.compile(
-    r"^({0} |{0})*%(.*)({0}| )escape".format(_SCRIPT_EXTENSIONS[ext]['comment'])) for ext in _SCRIPT_EXTENSIONS}
-_FORCE_NOT_ESC_RE = {_SCRIPT_EXTENSIONS[ext]['language']: re.compile(
-    r"^({0} |{0})*%(.*)({0}| )noescape".format(_SCRIPT_EXTENSIONS[ext]['comment'])) for ext in _SCRIPT_EXTENSIONS}
+# A magic expression is a line or cell or metakernel magic (#94, #61) escaped zero, or multiple times
 _MAGIC_RE = {_SCRIPT_EXTENSIONS[ext]['language']: re.compile(
-    r"^({0} |{0})*(%%[a-zA-Z]|{1})".format(
-        _SCRIPT_EXTENSIONS[ext]['comment'], '|'.join(_LINE_MAGICS))) for ext in _SCRIPT_EXTENSIONS}
+    r"^({0} |{0})*(%|%%|%%%)[a-zA-Z]".format(_SCRIPT_EXTENSIONS[ext]['comment'])) for ext in _SCRIPT_EXTENSIONS}
+_MAGIC_FORCE_ESC_RE = {_SCRIPT_EXTENSIONS[ext]['language']: re.compile(
+    r"^({0} |{0})*(%|%%|%%%)[a-zA-Z](.*){0}\s*escape".format(
+        _SCRIPT_EXTENSIONS[ext]['comment'])) for ext in _SCRIPT_EXTENSIONS}
+_MAGIC_NOT_ESC_RE = {_SCRIPT_EXTENSIONS[ext]['language']: re.compile(
+    r"^({0} |{0})*(%|%%|%%%)[a-zA-Z](.*){0}\s*noescape".format(
+        _SCRIPT_EXTENSIONS[ext]['comment'])) for ext in _SCRIPT_EXTENSIONS}
 _COMMENT = {_SCRIPT_EXTENSIONS[ext]['language']: _SCRIPT_EXTENSIONS[ext]['comment'] for ext in _SCRIPT_EXTENSIONS}
 
 # Commands starting with a question marks have to be escaped
 _HELP_RE = re.compile(r"^(# |#)*\?")
 
 
-def is_magic(line, language):
-    """Is the current line a (possibly escaped) Jupyter magic?"""
-    if _FORCE_ESC_RE.get(language, _FORCE_ESC_RE['python']).match(line):
+def is_magic(line, language, global_escape_flag=True):
+    """Is the current line a (possibly escaped) Jupyter magic, and should it be commented?"""
+    if _MAGIC_FORCE_ESC_RE.get(language, _MAGIC_FORCE_ESC_RE['python']).match(line):
         return True
-    if not _FORCE_NOT_ESC_RE.get(language, _FORCE_ESC_RE['python']).match(line) and \
-            _MAGIC_RE.get(language, _FORCE_ESC_RE['python']).match(line):
+    if _MAGIC_NOT_ESC_RE.get(language, _MAGIC_NOT_ESC_RE['python']).match(line):
+        return False
+    if not global_escape_flag:
+        return False
+    if _MAGIC_RE.get(language, _MAGIC_RE['python']).match(line):
         return True
     if language == 'python':
         return _HELP_RE.match(line)
     return False
 
 
-def comment_magic(source, language='python'):
+def comment_magic(source, language='python', global_escape_flag=True):
     """Escape Jupyter magics with '# '"""
     parser = StringParser(language)
     for pos, line in enumerate(source):
-        if not parser.is_quoted() and is_magic(line, language):
+        if not parser.is_quoted() and is_magic(line, language, global_escape_flag):
             source[pos] = _COMMENT[language] + ' ' + line
         parser.read_line(line)
     return source
@@ -76,11 +54,11 @@ def unesc(line, language):
     return line
 
 
-def uncomment_magic(source, language='python'):
+def uncomment_magic(source, language='python', global_escape_flag=True):
     """Unescape Jupyter magics"""
     parser = StringParser(language)
     for pos, line in enumerate(source):
-        if not parser.is_quoted() and is_magic(line, language):
+        if not parser.is_quoted() and is_magic(line, language, global_escape_flag):
             source[pos] = unesc(line, language)
         parser.read_line(line)
     return source

tests/test_escape_magics.py

@@ -19,7 +19,7 @@ def test_escape(line):
     assert uncomment_magic(comment_magic([line])) == [line]
 
 
-@pytest.mark.parametrize('line', ['%pytest.fixture'])
+@pytest.mark.parametrize('line', ['@pytest.fixture'])
 def test_escape_magic_only(line):
     assert comment_magic([line]) == [line]
 
@@ -29,9 +29,9 @@ def test_force_noescape(line):
     assert comment_magic([line]) == [line]
 
 
-@pytest.mark.parametrize('line', ['%pytest.fixture #escape'])
-def test_force_escape(line):
-    assert comment_magic([line]) == ['# ' + line]
+@pytest.mark.parametrize('line', ['%matplotlib inline #noescape'])
+def test_force_noescape_with_gbl_esc_flag(line):
+    assert comment_magic([line], global_escape_flag=True) == [line]
 
 
 @pytest.mark.parametrize('ext_and_format_name,commented',