From 65c5ed397593cfbdc465b397e8934ae83b4df974 Mon Sep 17 00:00:00 2001 From: James Knight Date: Fri, 11 May 2018 23:58:26 -0400 Subject: [PATCH] handle interpreter directives with long lengths (#794) * venv: prepend interpreter directive to argument list When preparing virtual environments in a file container which has large length, the system might not be able to invoke shebang scripts which define interpreters beyond system limits (e.x. Linux as a limit of 128; BINPRM_BUF_SIZE [1]). This method can be used to check if the executable is a script containing a shebang line [2]. If so, extract the interpreter (and possible optional argument) and prepend the values to the provided argument list. tox will only attempt to read an interpreter directive of a maximum size of 2048 bytes to limit excessive reading and support UNIX systems which may support a longer interpret length. [1]: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git/tree/include/uapi/linux/binfmts.h?h=linux-4.16.y#n19 [2]: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git/tree/fs/binfmt_script.c?h=linux-4.16.y#n24 Signed-off-by: James Knight * tests: validate shebang interpreter prepending Providing a series of test assertions to validate the recently added method `prepend_shebang_interpreter`. Ensures shell scripts with various interpreter directives (whitespaces, single argument, invalid entries) are appended or ignored respectfully. Signed-off-by: James Knight * changelog: track pr 794 Track change which supports handling interpreter directives in environments with long path lengths [1]. [1]: e913dd3f81692c8e8432354a097a51551a9aa160 Signed-off-by: James Knight * doc: add workaround for limited shebang environments Adding documentation to tox's configuration document describing how to take advantage of the `TOX_LIMITED_SHEBANG` environment variable to bypass system-defined shebang interpreter directive limits. Signed-off-by: James Knight --- changelog/794.feature.rst | 5 ++ doc/config.rst | 20 +++++++ tests/test_venv.py | 108 +++++++++++++++++++++++++++++++++++++- tox/venv.py | 26 +++++++++ 4 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 changelog/794.feature.rst diff --git a/changelog/794.feature.rst b/changelog/794.feature.rst new file mode 100644 index 000000000..085334b92 --- /dev/null +++ b/changelog/794.feature.rst @@ -0,0 +1,5 @@ +Add support to explicitly invoke interpreter directives for environments with +long path lengths. In the event that ``tox`` cannot invoke scripts with a +system-limited shebang (e.x. a Linux host running a Jenkins Pipeline), a user +can set the environment variable ``TOX_LIMITED_SHEBANG`` to workaround the +system's limitation (e.x. ``export TOX_LIMITED_SHEBANG=1``) - by @jdknight diff --git a/doc/config.rst b/doc/config.rst index cf1d9ec47..5a40420d1 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -699,6 +699,26 @@ With the previous configuration, it will install: - ``flake8`` package for ``py36`` environment. - ``flake8`` and ``coverage`` packages for ``coverage`` environment. +Advanced settings +----------------- + +Handle interpreter directives with long lengths ++++++++++++++++++++++++++++++++++++++++++++++++ + +For systems supporting executable text files (scripts with a shebang), the +system will attempt to parse the interpreter directive to determine the program +to execute on the target text file. When ``tox`` prepares a virtual environment +in a file container which has a large length (e.x. using Jenkins Pipelines), the +system might not be able to invoke shebang scripts which define interpreters +beyond system limits (e.x. Linux as a limit of 128; ``BINPRM_BUF_SIZE``). To +workaround an environment which suffers from an interpreter directive limit, a +user can bypass the system's interpreter parser by defining the +``TOX_LIMITED_SHEBANG`` environment variable before invoking ``tox``:: + + export TOX_LIMITED_SHEBANG=1 + +When the workaround is enabled, all tox-invoked text file executables will have +their interpreter directive parsed by and explicitly executed by ``tox``. Other Rules and notes ===================== diff --git a/tests/test_venv.py b/tests/test_venv.py index 5ff601b0c..912f03f68 100644 --- a/tests/test_venv.py +++ b/tests/test_venv.py @@ -7,7 +7,8 @@ import tox from tox.interpreters import NoInterpreterInfo from tox.venv import ( - CreationConfig, VirtualEnv, getdigest, tox_testenv_create, tox_testenv_install_deps) + CreationConfig, VirtualEnv, getdigest, prepend_shebang_interpreter, + tox_testenv_create, tox_testenv_install_deps) def test_getdigest(tmpdir): @@ -761,3 +762,108 @@ def tox_runtest_post(self): assert log == [] mocksession.runtestenv(venv) assert log == ['started', 'finished'] + + +@pytest.mark.skipif("sys.platform == 'win32'") +def test_tox_testenv_interpret_shebang_empty_instance(tmpdir): + testfile = tmpdir.join('check_shebang_empty_instance.py') + base_args = [str(testfile), 'arg1', 'arg2', 'arg3'] + + # empty instance + testfile.write('') + args = prepend_shebang_interpreter(base_args) + assert args == base_args + + +@pytest.mark.skipif("sys.platform == 'win32'") +def test_tox_testenv_interpret_shebang_empty_interpreter(tmpdir): + testfile = tmpdir.join('check_shebang_empty_interpreter.py') + base_args = [str(testfile), 'arg1', 'arg2', 'arg3'] + + # empty interpreter + testfile.write('#!') + args = prepend_shebang_interpreter(base_args) + assert args == base_args + + +@pytest.mark.skipif("sys.platform == 'win32'") +def test_tox_testenv_interpret_shebang_empty_interpreter_ws(tmpdir): + testfile = tmpdir.join('check_shebang_empty_interpreter_ws.py') + base_args = [str(testfile), 'arg1', 'arg2', 'arg3'] + + # empty interpreter (whitespaces) + testfile.write('#! \n') + args = prepend_shebang_interpreter(base_args) + assert args == base_args + + +@pytest.mark.skipif("sys.platform == 'win32'") +def test_tox_testenv_interpret_shebang_interpreter_simple(tmpdir): + testfile = tmpdir.join('check_shebang_interpreter_simple.py') + base_args = [str(testfile), 'arg1', 'arg2', 'arg3'] + + # interpreter (simple) + testfile.write('#!interpreter') + args = prepend_shebang_interpreter(base_args) + assert args == [b'interpreter'] + base_args + + +@pytest.mark.skipif("sys.platform == 'win32'") +def test_tox_testenv_interpret_shebang_interpreter_ws(tmpdir): + testfile = tmpdir.join('check_shebang_interpreter_ws.py') + base_args = [str(testfile), 'arg1', 'arg2', 'arg3'] + + # interpreter (whitespaces) + testfile.write('#! interpreter \n\n') + args = prepend_shebang_interpreter(base_args) + assert args == [b'interpreter'] + base_args + + +@pytest.mark.skipif("sys.platform == 'win32'") +def test_tox_testenv_interpret_shebang_interpreter_arg(tmpdir): + testfile = tmpdir.join('check_shebang_interpreter_arg.py') + base_args = [str(testfile), 'arg1', 'arg2', 'arg3'] + + # interpreter with argument + testfile.write('#!interpreter argx\n') + args = prepend_shebang_interpreter(base_args) + assert args == [b'interpreter', b'argx'] + base_args + + +@pytest.mark.skipif("sys.platform == 'win32'") +def test_tox_testenv_interpret_shebang_interpreter_args(tmpdir): + testfile = tmpdir.join('check_shebang_interpreter_args.py') + base_args = [str(testfile), 'arg1', 'arg2', 'arg3'] + + # interpreter with argument (ensure single argument) + testfile.write('#!interpreter argx argx-part2\n') + args = prepend_shebang_interpreter(base_args) + assert args == [b'interpreter', b'argx argx-part2'] + base_args + + +@pytest.mark.skipif("sys.platform == 'win32'") +def test_tox_testenv_interpret_shebang_real(tmpdir): + testfile = tmpdir.join('check_shebang_real.py') + base_args = [str(testfile), 'arg1', 'arg2', 'arg3'] + + # interpreter (real example) + testfile.write('#!/usr/bin/env python\n') + args = prepend_shebang_interpreter(base_args) + assert args == [b'/usr/bin/env', b'python'] + base_args + + +@pytest.mark.skipif("sys.platform == 'win32'") +def test_tox_testenv_interpret_shebang_long_example(tmpdir): + testfile = tmpdir.join('check_shebang_long_example.py') + base_args = [str(testfile), 'arg1', 'arg2', 'arg3'] + + # interpreter (long example) + testfile.write( + '#!this-is-an-example-of-a-very-long-interpret-directive-what-should-' + 'be-directly-invoked-when-tox-needs-to-invoked-the-provided-script-' + 'name-in-the-argument-list') + args = prepend_shebang_interpreter(base_args) + assert args == [ + b'this-is-an-example-of-a-very-long-interpret-directive-what-should-be-' + b'directly-invoked-when-tox-needs-to-invoked-the-provided-script-name-' + b'in-the-argument-list'] + base_args diff --git a/tox/venv.py b/tox/venv.py index 1d52e5216..1f05c3a1a 100755 --- a/tox/venv.py +++ b/tox/venv.py @@ -400,6 +400,8 @@ def _pcall(self, args, cwd, venv=True, testcommand=False, cwd.ensure(dir=1) args[0] = self.getcommandpath(args[0], venv, cwd) + if sys.platform != 'win32' and 'TOX_LIMITED_SHEBANG' in os.environ: + args = prepend_shebang_interpreter(args) env = self._getenv(testcommand=testcommand) bindir = str(self.envconfig.envbindir) env['PATH'] = p = os.pathsep.join([bindir, os.environ["PATH"]]) @@ -415,6 +417,30 @@ def getdigest(path): return path.computehash() +def prepend_shebang_interpreter(args): + # prepend interpreter directive (if any) to argument list + # + # When preparing virtual environments in a file container which has large + # length, the system might not be able to invoke shebang scripts which + # define interpreters beyond system limits (e.x. Linux as a limit of 128; + # BINPRM_BUF_SIZE). This method can be used to check if the executable is + # a script containing a shebang line. If so, extract the interpreter (and + # possible optional argument) and prepend the values to the provided + # argument list. tox will only attempt to read an interpreter directive of + # a maximum size of 2048 bytes to limit excessive reading and support UNIX + # systems which may support a longer interpret length. + try: + with open(args[0], 'rb') as f: + if f.read(1) == b'#' and f.read(1) == b'!': + MAXINTERP = 2048 + interp = f.readline(MAXINTERP).rstrip() + interp_args = interp.split(None, 1)[:2] + return interp_args + args + except IOError: + pass + return args + + @tox.hookimpl def tox_testenv_create(venv, action): config_interpreter = venv.getsupportedinterpreter()