Upgrading linux kokoro builds

kokoro/linux/bazel/build.sh

@@ -30,15 +30,3 @@ bazel_args=(
 )
 
 ${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh "${bazel_args[@]}"
-
-# Verify that we can build successfully from generated tar files.
-(
-  pyenv versions
-  pyenv shell 2.7.9  # python2 required for old googletest autotools support
-  git submodule update --init --recursive
-  ./autogen.sh && ./configure && make -j$(nproc) dist
-)
-DIST=`ls *.tar.gz`
-tar -xf $DIST
-cd ${DIST//.tar.gz}
-${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh build //:protobuf //:protobuf_java

kokoro/linux/benchmark/run.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
 #
+# Install Bazel 4.0.0.
+use_bazel.sh 4.0.0
+bazel version
+
 # Change to repo root
 cd $(dirname $0)/../../..
 
@@ -18,82 +22,42 @@ datasets=$(for file in $(find . -type f -name "dataset.*.pb" -not -path "./tmp/*
 echo $datasets
 popd
 
-# build Python protobuf
-./autogen.sh
-./configure CXXFLAGS="-fPIC -O2"
-make -j8
-pushd python
-python3 -m venv env
-source env/bin/activate
-python3 setup.py build --cpp_implementation
-pip3 install --install-option="--cpp_implementation" .
-popd
-
 # build and run Python benchmark
-# We do this before building protobuf C++ since C++ build
-# will rewrite some libraries used by protobuf python.
-pushd benchmarks
-make python-pure-python-benchmark
-make python-cpp-reflection-benchmark
-make -j8 python-cpp-generated-code-benchmark
-echo "[" > tmp/python_result.json
+echo "[" > python_result.json
 echo "benchmarking pure python..."
-./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets  >> tmp/python_result.json
-echo "," >> "tmp/python_result.json"
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python \
+${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh run //benchmarks/python:python_benchmark -- \
+	--json --behavior_prefix="pure-python-benchmark" $datasets >> python_result.json
+echo "," >> "python_result.json"
 echo "benchmarking python cpp reflection..."
-env LD_LIBRARY_PATH="${repo_root}/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets  >> tmp/python_result.json
-echo "," >> "tmp/python_result.json"
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp \
+${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh run //benchmarks/python:python_benchmark --define=use_fast_cpp_protos=true -- \
+	--json --behavior_prefix="cpp-reflection-benchmark" $datasets >> python_result.json
+echo "," >> "python_result.json"
 echo "benchmarking python cpp generated code..."
-env LD_LIBRARY_PATH="${repo_root}/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json
-echo "]" >> "tmp/python_result.json"
-popd
-
-# build CPP protobuf
-./configure
-make clean && make -j8
-
-pushd java
-mvn package -B -Dmaven.test.skip=true
-popd
-
-pushd benchmarks
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp \
+${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh run //benchmarks/python:python_benchmark --define=use_fast_cpp_protos=true -- \
+	--json --cpp_generated --behavior_prefix="cpp-generated-code-benchmark" $datasets >> python_result.json
+echo "]" >> "python_result.json"
 
 # build and run C++ benchmark
-# "make clean" deletes the contents of the tmp/ directory, so we move it elsewhere and then restore it once build is done.
-# TODO(jtattermusch): find a less clumsy way of protecting python_result.json contents
-mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp
 echo "benchmarking cpp..."
-env ./cpp-benchmark --benchmark_min_time=5.0 --benchmark_out_format=json --benchmark_out="tmp/cpp_result.json" $datasets
-
-# TODO(jtattermusch): add benchmarks for https://github.com/protocolbuffers/protobuf-go.
-# The original benchmarks for https://github.com/golang/protobuf were removed
-# because:
-# * they were broken and haven't been producing results for a long time
-# * the https://github.com/golang/protobuf implementation has been superseded by
-#   https://github.com/protocolbuffers/protobuf-go
+${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh run //benchmarks/cpp:cpp_benchmark -- \
+	--benchmark_min_time=5.0 --benchmark_out_format=json --benchmark_out="${repo_root}/cpp_result.json" $datasets
 
 # build and run java benchmark (java 11 is required)
-make java-benchmark
 echo "benchmarking java..."
-./java-benchmark -Cresults.file.options.file="tmp/java_result.json" $datasets
-
-# TODO(jtattermusch): re-enable JS benchmarks once https://github.com/protocolbuffers/protobuf/issues/8747 is fixed.
-# build and run js benchmark
-# make js-benchmark
-# echo "benchmarking js..."
-# ./js-benchmark $datasets  --json_output=$(pwd)/tmp/node_result.json
-
-# TODO(jtattermusch): add php-c-benchmark. Currently its build is broken.
+${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh run //benchmarks/java:java_benchmark -- \
+	-Cresults.file.options.file="${repo_root}/java_result.json" $datasets
 
 # persist raw the results in the build job log (for better debuggability)
-cat tmp/cpp_result.json
-cat tmp/java_result.json
-cat tmp/python_result.json
+cat cpp_result.json
+cat java_result.json
+cat python_result.json
 
 # print the postprocessed results to the build job log
 # TODO(jtattermusch): re-enable uploading results to bigquery (it is currently broken)
-make python_add_init
-env LD_LIBRARY_PATH="${repo_root}/src/.libs" python3 -m util.result_parser \
-	-cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" -python="../tmp/python_result.json"
-popd
-
+bazel run //benchmarks/util:result_parser -- \
+	-cpp="${repo_root}/cpp_result.json" \
+	-java="${repo_root}/java_result.json" \
+	-python="${repo_root}/python_result.json"

kokoro/linux/build_and_run_gcr_docker.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+#
+# Builds docker image and runs a command under it.
+# This is a generic script that is configured with the following variables:
+#
+# DOCKERHUB_ORGANIZATION - The organization on docker hub storing the
+# Dockerfile.
+# DOCKERFILE_DIR - Directory in which Dockerfile file is located.
+# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root)
+# OUTPUT_DIR - Directory that will be copied from inside docker after finishing.
+# $@ - Extra args to pass to docker run
+
+
+#### Work in progress ###
+
+set -ex
+
+cd $(dirname $0)/../..
+GIT_REPO_ROOT=$(pwd)
+cd -
+
+# Use image name based on Dockerfile sha1
+if [ -z "$DOCKERHUB_ORGANIZATION" ]
+then
+  DOCKERHUB_ORGANIZATION=grpctesting/protobuf
+  DOCKER_IMAGE_NAME=${DOCKERHUB_ORGANIZATION}_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ )
+else
+  # TODO(teboring): Remove this when all tests have been migrated to separate
+  # docker images.
+  DOCKERFILE_PREFIX=$(basename $DOCKERFILE_DIR)
+  DOCKER_IMAGE_NAME=${DOCKERHUB_ORGANIZATION}/${DOCKERFILE_PREFIX}_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ )
+fi
+CONTAINER_IMAGE=gcr.io/protobuf-build/bazel/linux@sha256:05a41edee21f620445ca01c46b9d33be2e4d06c3d8adced4341ac5abea754baa
+
+# Pull dockerimage from Dockerhub. This sometimes fails intermittently, so we
+# keep trying until we succeed.
+until docker pull $DOCKER_IMAGE_NAME; do sleep 10; done
+
+# Ensure existence of ccache directory
+CCACHE_DIR=/tmp/protobuf-ccache
+mkdir -p $CCACHE_DIR
+
+# Choose random name for docker container
+CONTAINER_NAME="build_and_run_docker_$(uuidgen)"
+
+echo $git_root
+
+# Run command inside docker
+docker run \
+  "$@" \
+  -e CCACHE_DIR=$CCACHE_DIR \
+  -e KOKORO_BUILD_NUMBER=$KOKORO_BUILD_NUMBER \
+  -e KOKORO_BUILD_ID=$KOKORO_BUILD_ID \
+  -e EXTERNAL_GIT_ROOT="/var/local/kokoro/protobuf" \
+  -e TEST_SET="$TEST_SET" \
+  -v "$git_root:/var/local/kokoro/protobuf:ro" \
+  -v $CCACHE_DIR:$CCACHE_DIR \
+  -w /var/local/git/protobuf \
+  --name=$CONTAINER_NAME \
+  $DOCKER_IMAGE_NAME \
+  bash -l "/var/local/kokoro/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true"
+
+# remove the container, possibly killing it first
+docker rm -f $CONTAINER_NAME || true
+
+[ -z "$FAILED" ] || {
+  exit 1
+}
+
+CONTAINER_IMAGE=gcr.io/protobuf-build/bazel/linux@sha256:05a41edee21f620445ca01c46b9d33be2e4d06c3d8adced4341ac5abea754baa
+
+if [[ -z "${PLATFORM}" ]]; then
+  PLATFORM_CONFIG=""
+else
+  PLATFORM_CONFIG="--config=${PLATFORM}"
+fi
+
+if $DEBUG_MODE; then
+  DEBUG_FLAG="-s"
+else
+  DEBUG_FLAG=""
+fi
+
+# Fetch external deps w/ network access enabled.
+docker run \
+  -v $GIT_REPO_ROOT:/workspace \
+  $CONTAINER_IMAGE \
+  build --nobuild --config=cross_config $BAZEL_TARGETS
+
+# Execute the requested command against the requested targets in a
+# fully offline context.
+docker run \
+  --network none \
+  -v $GIT_REPO_ROOT:/workspace \
+  $CONTAINER_IMAGE \
+  $BAZEL_COMMAND \
+  $DEBUG_FLAG \
+  $PLATFORM_CONFIG \
+  $BAZEL_TARGETS

kokoro/linux/csharp/build.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
 #
-# This is the top-level script we give to Kokoro as the entry point for
-# running the "pull request" project:
-#
-# This script selects a specific Dockerfile (for building a Docker image) and
-# a script to run inside that image.  Then we delegate to the general
-# build_and_run_docker.sh script.
+# Build file to set up and run tests
+
+set -eu
+
+# Install Bazel 4.0.0.
+use_bazel.sh 4.0.0
+bazel version
 
 # Change to repo root
 cd $(dirname $0)/../../..
 
-export DOCKERHUB_ORGANIZATION=protobuftesting
-export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/csharp
-export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
-export OUTPUT_DIR=testoutput
-export TEST_SET="csharp"
-./kokoro/linux/build_and_run_docker.sh
+# Get kokoro scripts from repo root by default.
+: ${SCRIPT_ROOT:=$(pwd)}
+source ${SCRIPT_ROOT}/kokoro/common/pyenv.sh
+
+./tests.sh csharp

kokoro/linux/csharp/common.cfg

@@ -0,0 +1,9 @@
+# Common config shared by presubmit and continuous.
+
+bazel_setting: {
+  project_id: "protobuf-build"
+  bes_backend_address: "buildeventservice.googleapis.com"
+  foundry_backend_address: "remotebuildexecution.googleapis.com"
+  upsalite_frontend_address: "https://source.cloud.google.com"
+  local_execution: true
+}

kokoro/linux/python310/build.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
 #
-# This is the top-level script we give to Kokoro as the entry point for
-# running the "pull request" project:
-#
-# This script selects a specific Dockerfile (for building a Docker image) and
-# a script to run inside that image.  Then we delegate to the general
-# build_and_run_docker.sh script.
+# Build file to set up and run tests
+
+set -eu
+
+# Install Bazel 4.0.0.
+use_bazel.sh 4.0.0
+bazel version
 
 # Change to repo root
 cd $(dirname $0)/../../..
 
-export DOCKERHUB_ORGANIZATION=protobuftesting
-export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python310
-export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
-export OUTPUT_DIR=testoutput
-export TEST_SET="python310"
-./kokoro/linux/build_and_run_docker.sh
+# Get kokoro scripts from repo root by default.
+: ${SCRIPT_ROOT:=$(pwd)}
+source ${SCRIPT_ROOT}/kokoro/common/pyenv.sh
+
+./tests.sh python310

kokoro/linux/python310/common.cfg

@@ -0,0 +1,9 @@
+# Common config shared by presubmit and continuous.
+
+bazel_setting: {
+  project_id: "protobuf-build"
+  bes_backend_address: "buildeventservice.googleapis.com"
+  foundry_backend_address: "remotebuildexecution.googleapis.com"
+  upsalite_frontend_address: "https://source.cloud.google.com"
+  local_execution: true
+}

kokoro/linux/python310_cpp/build.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
 #
-# This is the top-level script we give to Kokoro as the entry point for
-# running the "pull request" project:
-#
-# This script selects a specific Dockerfile (for building a Docker image) and
-# a script to run inside that image.  Then we delegate to the general
-# build_and_run_docker.sh script.
+# Build file to set up and run tests
+
+set -eu
+
+# Install Bazel 4.0.0.
+use_bazel.sh 4.0.0
+bazel version
 
 # Change to repo root
 cd $(dirname $0)/../../..
 
-export DOCKERHUB_ORGANIZATION=protobuftesting
-export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python310
-export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
-export OUTPUT_DIR=testoutput
-export TEST_SET="python310_cpp"
-./kokoro/linux/build_and_run_docker.sh
+# Get kokoro scripts from repo root by default.
+: ${SCRIPT_ROOT:=$(pwd)}
+source ${SCRIPT_ROOT}/kokoro/common/pyenv.sh
+
+./tests.sh python310_cpp

kokoro/linux/python310_cpp/common.cfg

@@ -0,0 +1,9 @@
+# Common config shared by presubmit and continuous.
+
+bazel_setting: {
+  project_id: "protobuf-build"
+  bes_backend_address: "buildeventservice.googleapis.com"
+  foundry_backend_address: "remotebuildexecution.googleapis.com"
+  upsalite_frontend_address: "https://source.cloud.google.com"
+  local_execution: true
+}