From 7daa1fd199c788fcd91076aff237f1985f451f96 Mon Sep 17 00:00:00 2001 From: brandjon Date: Wed, 29 May 2019 14:04:39 -0700 Subject: [PATCH] Enable Python integration tests on Windows Now that we have both a Python 2 and 3 interpreter on our CI machines (bazelbuild/continuous-integration#578), we can turn on these version tests for Windows. Since there's no autodetecting toolchain for Windows yet (#7844) we define an explicit toolchain. Fixes #8411. RELNOTES: None PiperOrigin-RevId: 250562174 --- src/test/shell/bazel/BUILD | 4 - src/test/shell/bazel/python_version_test.sh | 62 ++++++++++------ .../shell/integration/python_stub_test.sh | 4 + src/test/shell/testenv.sh | 73 +++++++++++++++++++ 4 files changed, 118 insertions(+), 25 deletions(-) diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index 471c6f93869815..93dc52df63d2a2 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -720,10 +720,6 @@ sh_test( ":test-deps", "@bazel_tools//tools/bash/runfiles", ], - tags = [ - # Disabled on windows and mac; see TODOs in the test suite. - "no_windows", - ], ) sh_test( diff --git a/src/test/shell/bazel/python_version_test.sh b/src/test/shell/bazel/python_version_test.sh index a01977a8798ea8..73b173530da5c6 100755 --- a/src/test/shell/bazel/python_version_test.sh +++ b/src/test/shell/bazel/python_version_test.sh @@ -52,11 +52,6 @@ case "$(uname -s | tr [:upper:] [:lower:])" in msys*) # As of 2018-08-14, Bazel on Windows only supports MSYS Bash. declare -r is_windows=true - # As of 2019-05-18, this test is disabled on Windows (via "no_windows" tag), - # so this code shouldn't even have run. - # TODO(#7844): Enable this test for Windows once our autodetecting toolchain - # works transparently for this platform. - fail "This test does not run on Windows." ;; *) declare -r is_windows=false @@ -72,7 +67,7 @@ fi #### TESTS ############################################################# -# Sanity test that our environment setup above works. +# Sanity test that our environment setup works. function test_can_run_py_binaries() { mkdir -p test @@ -94,7 +89,6 @@ import platform print("I am Python " + platform.python_version_tuple()[0]) EOF cp test/main2.py test/main3.py - chmod u+x test/main2.py test/main3.py bazel run //test:main2 \ &> $TEST_log || fail "bazel run failed" @@ -142,18 +136,23 @@ EOF } # Regression test for #5104. This test ensures that it's possible to use -# --build_python_zip in combination with a py_runtime (as opposed to not using -# a py_runtime, i.e., the legacy --python_path mechanism). -# -# Note that with --incompatible_use_python_toolchains flipped, we're always -# using a py_runtime, so in that case this amounts to a test that -# --build_python_zip works at all. +# --build_python_zip in combination with an in-workspace runtime, as opposed to +# with a system runtime or not using a py_runtime at all (the legacy +# --python_path mechanism). # # The specific issue #5104 was caused by file permissions being lost when # unzipping runfiles, which led to an unexecutable runtime. -function test_build_python_zip_works_with_py_runtime() { +function test_build_python_zip_works_with_workspace_runtime() { mkdir -p test + # The runfiles interpreter is either a sh script or bat script depending on + # the current platform. + if "$is_windows"; then + INTERPRETER_FILE="mockpy.bat" + else + INTERPRETER_FILE="mockpy.sh" + fi + cat > test/BUILD << EOF load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") @@ -164,7 +163,7 @@ py_binary( py_runtime( name = "mock_runtime", - interpreter = ":mockpy.sh", + interpreter = ":$INTERPRETER_FILE", python_version = "PY3", ) @@ -184,11 +183,17 @@ EOF # executes the Python code. print("I am pybin!") EOF - cat > test/mockpy.sh < "test/$INTERPRETER_FILE" << EOF +@ECHO I am mockpy! +EOF + else + cat > "test/$INTERPRETER_FILE" << EOF +#!/bin/sh echo "I am mockpy!" EOF - chmod u+x test/mockpy.sh + chmod u+x test/mockpy.sh + fi bazel run //test:pybin \ --extra_toolchains=//test:mock_toolchain --build_python_zip \ @@ -197,6 +202,11 @@ EOF } function test_pybin_can_have_different_version_pybin_as_data_dep() { + # TODO(#8503): Fix this test for windows. + if "$is_windows"; then + return + fi + mkdir -p test cat > test/BUILD << EOF @@ -231,7 +241,6 @@ import platform print("Inner bin uses Python " + platform.python_version_tuple()[0]) EOF - chmod u+x test/py2bin.py cp test/py2bin.py test/py3bin.py cat > test/py2bin_calling_py3bin.py << EOF @@ -239,14 +248,15 @@ import platform import subprocess from bazel_tools.tools.python.runfiles import runfiles +print("Outer bin uses Python " + platform.python_version_tuple()[0]) + r = runfiles.Create() bin_path = r.Rlocation("$WORKSPACE_NAME/test/py3bin") +assert bin_path is not None -print("Outer bin uses Python " + platform.python_version_tuple()[0]) subprocess.call([bin_path]) EOF sed s/py3bin/py2bin/ test/py2bin_calling_py3bin.py > test/py3bin_calling_py2bin.py - chmod u+x test/py2bin_calling_py3bin.py test/py3bin_calling_py2bin.py EXPFLAG="--incompatible_allow_python_version_transitions=true \ --incompatible_py3_is_default=false \ @@ -267,6 +277,11 @@ EOF } function test_shbin_can_have_different_version_pybins_as_data_deps() { + # Uses bash, disable on windows. + if "$is_windows"; then + return + fi + mkdir -p test cat > test/BUILD << EOF @@ -394,6 +409,11 @@ EOF } function test_can_build_same_target_for_both_versions_in_one_build() { + # Uses bash, disable on windows. + if "$is_windows"; then + return + fi + mkdir -p test cat > test/BUILD << EOF diff --git a/src/test/shell/integration/python_stub_test.sh b/src/test/shell/integration/python_stub_test.sh index 1808cb4961b0e6..ff957df071b565 100755 --- a/src/test/shell/integration/python_stub_test.sh +++ b/src/test/shell/integration/python_stub_test.sh @@ -59,6 +59,10 @@ fi # Tests in this file do not actually start a Python interpreter, but plug in a # fake stub executable to serve as the "interpreter". +# +# Note that this means this suite cannot be used for tests of the actual stub +# script under Windows, since the stub script never runs (the launcher uses the +# mock interpreter rather than a system interpreter, see discussion in #7947). use_fake_python_runtimes_for_testsuite diff --git a/src/test/shell/testenv.sh b/src/test/shell/testenv.sh index 65f0ce0abe8f0d..73316d4426a0ca 100755 --- a/src/test/shell/testenv.sh +++ b/src/test/shell/testenv.sh @@ -19,6 +19,8 @@ # TODO(bazel-team): This file is currently an append of the old testenv.sh and # test-setup.sh files. This must be cleaned up eventually. +# TODO(bazel-team): Factor each test suite's is-this-windows setup check to use +# this var instead, or better yet a common $IS_WINDOWS var. PLATFORM="$(uname -s | tr [:upper:] [:lower:])" function is_darwin() { @@ -363,6 +365,54 @@ java_import( EOF } +# If the current platform is Windows, defines a Python toolchain for our +# Windows CI machines. Otherwise does nothing. +# +# Our Windows CI machines have Python 2 and 3 installed at C:\Python2 and +# C:\Python3 respectively. +# +# Since the tools directory is not cleared between test cases, this only needs +# to run once per suite. However, the toolchain must still be registered +# somehow. +# +# TODO(#7844): Delete this custom (and machine-specific) test setup once we have +# an autodetecting Python toolchain for Windows. +function maybe_setup_python_windows_tools() { + if [[ ! $PLATFORM =~ msys ]]; then + return + fi + + mkdir -p tools/python/windows + cat > tools/python/windows/BUILD << EOF +load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") + +py_runtime( + name = "py2_runtime", + interpreter_path = r"C:\Python2\python.exe", + python_version = "PY2", +) + +py_runtime( + name = "py3_runtime", + interpreter_path = r"C:\Python3\python.exe", + python_version = "PY3", +) + +py_runtime_pair( + name = "py_runtime_pair", + py2_runtime = ":py2_runtime", + py3_runtime = ":py3_runtime", +) + +toolchain( + name = "py_toolchain", + toolchain = ":py_runtime_pair", + toolchain_type = "@bazel_tools//tools/python:toolchain_type", + target_compatible_with = ["@bazel_tools//platforms:windows"], +) +EOF +} + function setup_skylark_javatest_support() { setup_javatest_common grep -q "name = \"junit4-jars\"" third_party/BUILD \ @@ -402,6 +452,27 @@ EOF function write_workspace_file() { cat > WORKSPACE << EOF workspace(name = '$WORKSPACE_NAME') +EOF + + maybe_setup_python_windows_workspace +} + +# If the current platform is Windows, registers our custom Windows Python +# toolchain. Otherwise does nothing. +# +# Since this modifies the WORKSPACE file, it must be called between test cases. +function maybe_setup_python_windows_workspace() { + if [[ ! $PLATFORM =~ msys ]]; then + return + fi + + # --extra_toolchains has left-to-right precedence semantics, but the bazelrc + # is processed before the command line. This means that any matching + # toolchains added to the bazelrc will always take precedence over toolchains + # set up by test cases. Instead, we add the toolchain to WORKSPACE so that it + # has lower priority than whatever is passed on the command line. + cat >> WORKSPACE << EOF +register_toolchains("//tools/python/windows:py_toolchain") EOF } @@ -422,6 +493,8 @@ function create_new_workspace() { || ln -s "${langtools_path}" third_party/java/jdk/langtools/javac-9+181-r4173-1.jar write_workspace_file + + maybe_setup_python_windows_tools }