From a4c614f4a314b45bc23101c58cb8a49f05fbbd47 Mon Sep 17 00:00:00 2001 From: Mikhail Bautin Date: Wed, 3 Apr 2019 00:48:38 -0700 Subject: [PATCH] JSON-based test reports Summary: Adding a script for post-processing test results and generating JSON-based result files. Report format example: { "total_errors": 0, "total_skipped": 296, "total_failures": 1, "total_test_instances": 4086, "total_tests_run": 4062, "build_root": "build/asan-clang-dynamic-enterprise-ninja", "aggregation_errors": [], "compiler_type": "clang", "build_type": "asan", "tests": [ { "language": "java", "class_name": "org.yb.cql.TestSelect", "junit_xml_path": "java/yb-cql/target/...", "time": 12.804, "log_path": "java/yb-cql/target/...", "test_name": "testQualifiedColumnReference" }, ... ] } In these reports we can capture more information, such as paths to multiple types of per-test log files, than we could in JUnit-compatible XML test result files. Also we now correctly count skipped C++ tests, which are reported in a different way by the gtest framework (using `"status": "notrun"`) compared to the way skipped tests are reported in Java. Other test framework improvements: - Re-run Java tests with "Address already in use" and a couple more test patterns in the log (at most once). We already do this for C++ tests. - Adding an `YB_DEBUG_VIRTUALENV` environment variable for debugging virtualenv-related behavior. - Simplifying the logic for picking TEST_TMPDIR. Now it is always in `/tmp`. - Removing the old code for symbolizing C++ stack traces in the log which we don't use anyway, and which does not work for external mini-cluster tests. - Add a timeout in `update_test_result_xml.py`, because that script used to get stuck for hours in some cases on macOS. - Pick bind IPs on macOS in Java tests based on the actual list of available loopback interfaces, not assuming a specific range of pre-created interfaces. YSQL-specific test improvements: - Create a new flag `pg_yb_session_timeout_ms` and set it to 120 seconds by default on macOS (60 seconds otherwise). Refactoring: - Renaming test framework functions specific to C++ so that their name clearly states that. Test Plan: Jenkins Reviewers: bogdan, sergei, timur, bharat Reviewed By: bharat Subscribers: ybase Differential Revision: https://phabricator.dev.yugabyte.com/D6343 --- .gitignore | 4 + bin/repeat_unit_test.sh | 5 +- build-support/common-build-env.sh | 16 ++ build-support/common-test-env.sh | 226 +++++++--------- build-support/jenkins/build-and-test.sh | 21 +- .../python-wrappers/python-wrapper.sh | 13 +- build-support/run-test.sh | 7 +- build-support/run_tests_on_spark.py | 8 + build-support/stacktrace_addr2line.pl | 182 ------------- build-support/update_test_result_xml.py | 54 +++- .../org/yb/minicluster/MiniYBCluster.java | 54 ++-- .../src/test/java/org/yb/util/BindIpUtil.java | 39 +++ .../test/java/org/yb/util/SideBySideDiff.java | 6 +- python/yb/aggregate_test_reports.py | 216 +++++++++++++++ python/yb/postprocess_test_result.py | 256 ++++++++++++++++++ python/yb/process_tree_supervisor.py | 17 +- src/yb/gutil/strings/string_util-test.cc | 16 +- src/yb/util/debug-util.h | 5 +- src/yb/yql/pggate/pg_session.cc | 12 +- thirdparty/homebrew_version_for_jenkins.txt | 2 +- thirdparty/version_for_jenkins_mac.txt | 2 +- yb_build.sh | 7 +- 22 files changed, 800 insertions(+), 368 deletions(-) delete mode 100755 build-support/stacktrace_addr2line.pl create mode 100644 java/yb-client/src/test/java/org/yb/util/BindIpUtil.java create mode 100755 python/yb/aggregate_test_reports.py create mode 100755 python/yb/postprocess_test_result.py diff --git a/.gitignore b/.gitignore index d06dbf4d4509..537d576b1f45 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,7 @@ compile_commands.json # This might be added by some editors' Java plugins ent/java/yb-cqlent/bin/ + +test_results.json +test_failures.json +*_test_report.json diff --git a/bin/repeat_unit_test.sh b/bin/repeat_unit_test.sh index 80d18bfd7b0f..489275eeb831 100755 --- a/bin/repeat_unit_test.sh +++ b/bin/repeat_unit_test.sh @@ -259,8 +259,7 @@ if [[ $iteration -gt 0 ]]; then set +e current_timestamp=$( get_timestamp_for_filenames ) - export TEST_TMPDIR=/tmp/yb__${script_name_no_ext}_${build_type}_${current_timestamp}_\ -$RANDOM.$RANDOM.$RANDOM.$$ + export TEST_TMPDIR=/tmp/yb_tests__${current_timestamp}__$RANDOM.$RANDOM.$RANDOM mkdir -p "$TEST_TMPDIR" set_expected_core_dir "$TEST_TMPDIR" if ! "$is_java_test"; then @@ -298,7 +297,7 @@ $RANDOM.$RANDOM.$RANDOM.$$ pass_or_fail="PASSED" if ! did_test_succeed "$exit_code" "$raw_test_log_path"; then if ! "$is_java_test"; then - postprocess_test_log + postprocess_cxx_test_log process_core_file fi if "$skip_address_already_in_use" && \ diff --git a/build-support/common-build-env.sh b/build-support/common-build-env.sh index 03ff176d1466..c9584e9b8245 100644 --- a/build-support/common-build-env.sh +++ b/build-support/common-build-env.sh @@ -2047,6 +2047,22 @@ activate_virtualenv() { run_with_retries 10 0.5 pip2 install -r "$YB_SRC_ROOT/python_requirements_frozen.txt" \ $pip_no_cache fi + + if [[ ${YB_DEBUG_VIRTUALENV:-0} == "1" ]]; then + echo >&2 " +VIRTUALENV DEBUGGING +-------------------- + + Activated virtualenv in: $virtualenv_dir + Executable: $0 + PATH: $PATH + PYTHONPATH: ${PYTHONPATH:-undefined} + VIRTUAL_ENV: ${VIRTUAL_ENV:-undefined} + +" + fi + + export VIRTUAL_ENV } check_python_interpreter_version() { diff --git a/build-support/common-test-env.sh b/build-support/common-test-env.sh index f0134587acd4..b278ed7e241e 100644 --- a/build-support/common-test-env.sh +++ b/build-support/common-test-env.sh @@ -60,7 +60,8 @@ readonly RELEVANT_LOG_LINES_RE append_output_to=/dev/null readonly TEST_RESTART_PATTERN="Address already in use|\ -pthread .*: Device or resource busy" +pthread .*: Device or resource busy|\ +FATAL: could not create shared memory segment: No space left on device" # We use this to submit test jobs for execution on Spark. readonly SPARK_SUBMIT_CMD_PATH_NON_ASAN_TSAN=/n/tools/spark/current/bin/spark-submit @@ -365,6 +366,11 @@ using_nfs() { return 1 } +create_test_tmpdir() { + export TEST_TMPDIR="/tmp/yb_test.tmp.$RANDOM.$RANDOM.$RANDOM.pid$$" + mkdir_safe "$TEST_TMPDIR" +} + # Set a number of variables in preparation to running a test. # Inputs: # test_descriptor @@ -399,8 +405,6 @@ using_nfs() { # test_cmd_line # An array representing the test command line to run, including the JUnit-compatible test result # XML file location and --gtest_filter=..., if applicable. -# raw_test_log_path -# The path to output the raw test log to (before any filtering by stacktrace_addr2line.pl). # test_log_path # The final log path (after any applicable filtering and appending additional debug info). # xml_output_file @@ -412,7 +416,7 @@ using_nfs() { # # This function also removes the old XML output file at the path "$xml_output_file". -prepare_for_running_test() { +prepare_for_running_cxx_test() { expect_num_args 0 "$@" expect_vars_to_be_set \ YB_TEST_LOG_ROOT_DIR \ @@ -445,18 +449,8 @@ prepare_for_running_test() { local test_binary_sanitized=$( sanitize_for_path "$rel_test_binary" ) local test_name_sanitized=$( sanitize_for_path "$test_name" ) - # If there are ephermeral drives, pick a random one for this test and create a symlink from - # - # $BUILD_ROOT/yb-test-logs/$test_binary_sanitized - # ^^^^^^^^^^^^^^^^^^^^^^^^ - # ($YB_TEST_LOG_ROOT_DIR) - # - # to /mnt///test-workspace/. - # Otherwise, simply create the directory under $BUILD_ROOT/yb-test-logs. - test_dir="$YB_TEST_LOG_ROOT_DIR/$test_binary_sanitized" - if [[ ! -d $test_dir ]]; then - create_dir_on_ephemeral_drive "$test_dir" "test-workspace/$test_binary_sanitized" - fi + test_dir=$YB_TEST_LOG_ROOT_DIR/$test_binary_sanitized + mkdir_safe "$test_dir" rel_test_log_path_prefix="$test_binary_sanitized" if "$run_at_once"; then @@ -490,16 +484,8 @@ prepare_for_running_test() { test_cmd_line+=( "--gtest_filter=$test_name" ) fi - export TEST_TMPDIR="$test_log_path_prefix.tmp" - if is_src_root_on_nfs; then - export TEST_TMPDIR="/tmp/$test_log_path_prefix.tmp.$RANDOM.$RANDOM.$RANDOM.$$" - else - export TEST_TMPDIR="$test_log_path_prefix.tmp" - fi - - mkdir_safe "$TEST_TMPDIR" + create_test_tmpdir test_log_path="$test_log_path_prefix.log" - raw_test_log_path="${test_log_path_prefix}__raw.log" # gtest won't overwrite old junit test files, resulting in a build failure # even when retries are successful. @@ -677,6 +663,7 @@ process_core_file() { fi } +# Used for both C++ and Java tests. stop_process_tree_supervisor() { process_supervisor_success=true if [[ ${process_tree_supervisor_pid:-0} -eq 0 ]]; then @@ -736,7 +723,7 @@ stop_process_tree_supervisor() { # suite fails before it has a chance to generate an XML output file. In that case, this function # generates a substitute XML output file using parse_test_failure.py, but only if the log file # exists at its expected location. -handle_xml_output() { +handle_cxx_test_xml_output() { expect_vars_to_be_set \ rel_test_binary \ test_log_path \ @@ -792,58 +779,11 @@ handle_xml_output() { "$YB_SRC_ROOT"/build-support/update_test_result_xml.py \ --result-xml "$xml_output_file" \ --log-url "$test_log_url" \ - --mark-as-failed "$test_failed" # Useful for distributed builds in an NFS environment. chmod g+w "$xml_output_file" } -postprocess_test_log() { - expect_num_args 0 "$@" - expect_vars_to_be_set \ - STACK_TRACE_FILTER \ - abs_test_binary_path \ - raw_test_log_path \ - test_log_path \ - test_log_path_prefix - - if [[ ! -f $raw_test_log_path ]]; then - # We must have failed even before we could run the test. - return - fi - - local stack_trace_filter_err_path="${test_log_path_prefix}__stack_trace_filter_err.txt" - - if [[ $STACK_TRACE_FILTER == "cat" ]]; then - # Don't pass the binary name as an argument to the cat command. - set +e - "$STACK_TRACE_FILTER" <"$raw_test_log_path" 2>"$stack_trace_filter_err_path" >"$test_log_path" - else - set +ex - "$STACK_TRACE_FILTER" "$abs_test_binary_path" <"$raw_test_log_path" \ - >"$test_log_path" 2>"$stack_trace_filter_err_path" - fi - local stack_trace_filter_exit_code=$? - set -e - - if [[ $stack_trace_filter_exit_code -ne 0 ]]; then - # Stack trace filtering failed, create an output file with the error message. - ( - echo "Failed to run the stack trace filtering program '$STACK_TRACE_FILTER'" - echo - echo "Standard error from '$STACK_TRACE_FILTER'": - cat "$stack_trace_filter_err_path" - echo - - echo "Raw output:" - echo - cat "$raw_test_log_path" - ) >"$test_log_path" - test_failed=true - fi - rm -f "$raw_test_log_path" "$stack_trace_filter_err_path" -} - # Sets test log URL prefix if we're running in Jenkins. set_test_log_url_prefix() { test_log_url_prefix="" @@ -924,14 +864,7 @@ run_one_cxx_test() { # run tests on Jenkins or when running all tests using "ctest -j8" from the build root, one # should leave YB_CTEST_VERBOSE unset. if [[ -n ${YB_CTEST_VERBOSE:-} ]]; then - # In the verbose mode we have to perform stack trace filtering / symbolization right away. - local stack_trace_filter_cmd=( "$STACK_TRACE_FILTER" ) - if [[ $STACK_TRACE_FILTER != "cat" ]]; then - stack_trace_filter_cmd+=( "$abs_test_binary_path" ) - fi - ( set -x; "${test_wrapper_cmd_line[@]}" 2>&1 ) | \ - "${stack_trace_filter_cmd[@]}" | \ - tee "$test_log_path" + ( set -x; "${test_wrapper_cmd_line[@]}" 2>&1 ) | tee "$test_log_path" # Propagate the exit code of the test process, not any of the filters. This will only exit # this subshell, not the entire script calling this function. exit ${PIPESTATUS[0]} @@ -956,7 +889,7 @@ run_one_cxx_test() { egrep -q "$TEST_RESTART_PATTERN" "$test_log_path"; then log "Found one of the intermittent error patterns in the log, restarting the test (once):" egrep "$TEST_RESTART_PATTERN" "$test_log_path" >&2 - elif [[ $test_exit_code -ne 0 ]]; then + else # Avoid retrying any other kinds of failures. break fi @@ -969,7 +902,7 @@ run_one_cxx_test() { fi } -handle_test_failure() { +handle_cxx_test_failure() { expect_num_args 0 "$@" expect_vars_to_be_set \ TEST_TMPDIR \ @@ -1028,11 +961,33 @@ delete_successful_output_if_needed() { fi } -run_test_and_process_results() { +run_postproces_test_result_script() { + local args=( + --yb-src-root "$YB_SRC_ROOT" + --build-root "$BUILD_ROOT" + --extra-error-log-path "${YB_TEST_EXTRA_ERROR_LOG_PATH:-}" + ) + if [[ -n ${YB_FATAL_DETAILS_PATH_PREFIX:-} ]]; then + args+=( + --fatal-details-path-prefix "$YB_FATAL_DETAILS_PATH_PREFIX" + ) + fi + "$VIRTUAL_ENV/bin/python" "$YB_SRC_ROOT/python/yb/postprocess_test_result.py" \ + "${args[@]}" "$@" +} + +run_cxx_test_and_process_results() { run_one_cxx_test - postprocess_test_log - handle_test_failure - handle_xml_output + handle_cxx_test_failure + handle_cxx_test_xml_output + + run_postproces_test_result_script \ + --test-log-path "$test_log_path" \ + --junit-xml-path "$xml_output_file" \ + --language cxx \ + --cxx-rel-test-binary "$rel_test_binary" \ + --test-failed "$test_failed" + delete_successful_output_if_needed } @@ -1462,7 +1417,9 @@ run_java_test() { local timestamp=$( get_timestamp_for_filenames ) local surefire_rel_tmp_dir=surefire${timestamp}_${RANDOM}_${RANDOM}_${RANDOM}_$$ + # This should change the directory to either "java" or "ent/java". cd "$module_dir"/.. + # We specify tempDir to use a separate temporary directory for each test. # http://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html mvn_opts=( @@ -1485,22 +1442,9 @@ run_java_test() { local surefire_reports_dir=$module_dir/target/surefire-reports_${report_suffix} unset report_suffix - if [[ -n ${YB_SUREFIRE_REPORTS_DIR:-} ]]; then - surefire_reports_dir=$YB_SUREFIRE_REPORTS_DIR - elif should_run_java_test_methods_separately && - [[ -d $surefire_reports_dir && -n $test_method_name ]]; then - # If we are only running one test method, and it has its own surefire reports directory, it is - # OK to delete all other stuff in it. - if ! is_jenkins; then - log "Cleaning the existing contents of: $surefire_reports_dir" - fi - ( set -x; rm -f "$surefire_reports_dir"/* ) - fi - if ! is_jenkins; then - log "Using surefire reports directory: $surefire_reports_dir" - fi mvn_opts+=( "-Dyb.surefire.reports.directory=$surefire_reports_dir" + surefire:test ) if is_jenkins; then @@ -1508,34 +1452,59 @@ run_java_test() { mvn_opts+=( -X ) fi - local log_files_path_prefix=$surefire_reports_dir/$test_class - local test_log_path=$log_files_path_prefix-output.txt - local junit_xml_path=$surefire_reports_dir/TEST-$test_class.xml - - if ! is_jenkins; then - log "Test log path: $test_log_path" - fi - - mvn_opts+=( surefire:test ) - if ! which mvn >/dev/null; then fatal "Maven not found on PATH. PATH: $PATH" fi + + local should_clean_test_dir=false + if [[ -n ${YB_SUREFIRE_REPORTS_DIR:-} ]]; then + surefire_reports_dir=$YB_SUREFIRE_REPORTS_DIR + elif should_run_java_test_methods_separately && + [[ -d $surefire_reports_dir && -n $test_method_name ]]; then + should_clean_test_dir=true + fi + local junit_xml_path=$surefire_reports_dir/TEST-$test_class.xml + local log_files_path_prefix=$surefire_reports_dir/$test_class + local test_log_path=$log_files_path_prefix-output.txt + log "Using surefire reports directory: $surefire_reports_dir" + log "Test log path: $test_log_path" + local mvn_output_path="" about_to_start_running_test if [[ ${YB_REDIRECT_MVN_OUTPUT_TO_FILE:-0} == 1 ]]; then - mkdir -p "$surefire_reports_dir" mvn_output_path=$surefire_reports_dir/${test_class}__mvn_output.log - set +e - time ( try_set_ulimited_ulimit; set -x; mvn "${mvn_opts[@]}" ) &>"$mvn_output_path" - else - set +e - time ( try_set_ulimited_ulimit; set -x; mvn "${mvn_opts[@]}" ) fi - local mvn_exit_code=$? - set -e - log "Maven exited with code $mvn_exit_code" + local attempts_left + for attempts_left in {1..0}; do + if "$should_clean_test_dir"; then + log "Cleaning the existing contents of: $surefire_reports_dir" + ( set -x; rm -f "$surefire_reports_dir"/* ) + fi + if [[ ${YB_REDIRECT_MVN_OUTPUT_TO_FILE:-0} == 1 ]]; then + mkdir -p "$surefire_reports_dir" + set +e + time ( try_set_ulimited_ulimit; set -x; mvn "${mvn_opts[@]}" ) &>"$mvn_output_path" + else + set +e + time ( try_set_ulimited_ulimit; set -x; mvn "${mvn_opts[@]}" ) + fi + local mvn_exit_code=$? + set -e + log "Maven exited with code $mvn_exit_code" + if [[ $mvn_exit_code -eq 0 || $attempts_left -eq 0 ]]; then + break + fi + if [[ ! -f $test_log_path ]]; then + log "Warning: test log path not found: $test_log_path, not re-trying the test." + break + elif egrep -q "$TEST_RESTART_PATTERN" "$test_log_path"; then + log "Found an intermittent error in $test_log_path, retrying the test" + egrep "$TEST_RESTART_PATTERN" "$test_log_path" >&2 + else + break + fi + done local xml_valid=true if [[ -f $junit_xml_path ]]; then @@ -1617,6 +1586,15 @@ run_java_test() { if [[ $java_test_exit_code -eq 0 ]] && ! "$process_supervisor_success"; then java_test_exit_code=1 fi + + run_postproces_test_result_script \ + --test-log-path "$test_log_path" \ + --junit-xml-path "$junit_xml_path" \ + --language java \ + --class-name "$test_class" \ + --test-name "$test_method_name" \ + --java-module-dir "$module_dir" + return "$java_test_exit_code" } @@ -1839,12 +1817,4 @@ resolve_and_run_java_test() { # Initialization # ------------------------------------------------------------------------------------------------- -if is_mac; then - # Stack trace address to line number conversion is disabled on Mac OS X as of Apr 2016. - # See https://yugabyte.atlassian.net/browse/ENG-37 - STACK_TRACE_FILTER=cat -else - STACK_TRACE_FILTER=$YB_SRC_ROOT/build-support/stacktrace_addr2line.pl -fi - readonly jenkins_job_and_build=${JOB_NAME:-unknown_job}__${BUILD_NUMBER:-unknown_build} diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh index fccaa8e1c000..015de8d24072 100755 --- a/build-support/jenkins/build-and-test.sh +++ b/build-support/jenkins/build-and-test.sh @@ -145,6 +145,13 @@ cleanup() { cd "$YB_SRC_ROOT" +log "Removing old JSON-based test report files" +( + set -x + find . -name "*_test_report.json" -exec rm -f '{}' \; + rm -f test_results.json test_failures.json +) + export YB_RUN_JAVA_TEST_METHODS_SEPARATELY=1 log "Running with Bash version $BASH_VERSION" if ! "$YB_BUILD_SUPPORT_DIR/common-build-env-test.sh"; then @@ -279,9 +286,7 @@ if is_jenkins; then fi fi -if [[ ! -d $BUILD_ROOT ]]; then - create_dir_on_ephemeral_drive "$BUILD_ROOT" "build/${BUILD_ROOT##*/}" -fi +mkdir_safe "$BUILD_ROOT" if [[ -h $BUILD_ROOT ]]; then # If we ended up creating BUILD_ROOT as a symlink to an ephemeral drive, now make BUILD_ROOT @@ -711,6 +716,16 @@ fi # Finished running tests. remove_latest_symlink +log "Aggregating test reports" +cd "$YB_SRC_ROOT" # even though we should already be in this directory +find . -type f -name "*_test_report.json" | \ + "$YB_SRC_ROOT/python/yb/aggregate_test_reports.py" \ + --yb-src-root "$YB_SRC_ROOT" \ + --output-dir "$YB_SRC_ROOT" \ + --build-type "$build_type" \ + --compiler-type "$YB_COMPILER_TYPE" \ + --build-root "$BUILD_ROOT" + if [[ -n $FAILURES ]]; then heading "Failure summary" echo >&2 "$FAILURES" diff --git a/build-support/python-wrappers/python-wrapper.sh b/build-support/python-wrappers/python-wrapper.sh index 5961c20bf07d..e660aa12fa74 100755 --- a/build-support/python-wrappers/python-wrapper.sh +++ b/build-support/python-wrappers/python-wrapper.sh @@ -31,20 +31,23 @@ if [[ -n ${VIRTUAL_ENV:-} ]]; then python_interpreter_dirs+=( "$VIRTUAL_ENV/bin" ) fi -# Prefer system Python to Python coming from Homebrew/Linuxbrew. +if using_custom_homebrew; then + python_interpreter_dirs+=( "$YB_CUSTOM_HOMEBREW_DIR/bin" ) +fi + +# System Python installation. Also Homebrew Python on macOS. python_interpreter_dirs+=( /usr/local/bin /usr/bin ) +# On Linux, don't use Linuxbrew Python unless absolutely necessary (put it last). +# We currently run into this error when we use it: +# https://gist.githubusercontent.com/mbautin/d75fbc212a029c65577c4880aefcbb07/raw if using_linuxbrew; then python_interpreter_dirs+=( "$YB_LINUXBREW_DIR/bin" ) fi -if using_custom_homebrew; then - python_interpreter_dirs+=( "$YB_CUSTOM_HOMEBREW_DIR/bin" ) -fi - interpreter_name=${0##*/} interpreter_names=( "$interpreter_name" ) diff --git a/build-support/run-test.sh b/build-support/run-test.sh index 96bfb788a54a..f45d8a8942f0 100755 --- a/build-support/run-test.sh +++ b/build-support/run-test.sh @@ -63,6 +63,7 @@ cleanup() { if [[ $exit_code -eq 0 ]] && "$killed_stuck_processes"; then exit_code=1 fi + rm -rf "$TEST_TMPDIR" exit "$exit_code" } @@ -96,6 +97,8 @@ echo "Test is running on host $HOSTNAME, arguments: $*" set_java_home set_test_invocation_id +create_test_tmpdir + trap cleanup EXIT readonly process_supervisor_log_path=\ @@ -270,8 +273,8 @@ for test_descriptor in "${tests[@]}"; do else test_attempt_index="" fi - prepare_for_running_test - run_test_and_process_results + prepare_for_running_cxx_test + run_cxx_test_and_process_results done done diff --git a/build-support/run_tests_on_spark.py b/build-support/run_tests_on_spark.py index ffeb5a74a723..97a943b4bbd3 100755 --- a/build-support/run_tests_on_spark.py +++ b/build-support/run_tests_on_spark.py @@ -295,6 +295,10 @@ def parallel_run_test(test_descriptor_str): wait_for_path_to_exist(global_conf.build_root) yb_dist_tests.global_conf = global_conf test_descriptor = yb_dist_tests.TestDescriptor(test_descriptor_str) + + # This is saved in the test result file by process_test_result.py. + os.environ['YB_TEST_DESCRIPTOR_STR'] = test_descriptor_str + os.environ['YB_TEST_ATTEMPT_INDEX'] = str(test_descriptor.attempt_index) os.environ['build_type'] = global_conf.build_type os.environ['YB_RUNNING_TEST_ON_SPARK'] = '1' @@ -307,6 +311,7 @@ def parallel_run_test(test_descriptor_str): ''.join('%09d' % random.randrange(0, 1000000000) for i in xrange(4)))) os.environ['YB_TEST_STARTED_RUNNING_FLAG_FILE'] = test_started_running_flag_file + os.environ['YB_TEST_EXTRA_ERROR_LOG_PATH'] = test_descriptor.error_output_path yb_dist_tests.wait_for_clock_sync() @@ -994,6 +999,9 @@ def main(): "total_num_tests={}, len(test_descriptors)={}".format( total_num_tests, len(test_descriptors)) + # Randomize test order to avoid any kind of skew. + random.shuffle(test_descriptors) + test_names_rdd = spark_context.parallelize( [test_descriptor.descriptor_str for test_descriptor in test_descriptors], numSlices=total_num_tests) diff --git a/build-support/stacktrace_addr2line.pl b/build-support/stacktrace_addr2line.pl deleted file mode 100755 index 606effde4d7d..000000000000 --- a/build-support/stacktrace_addr2line.pl +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/perl -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -####################################################################### -# This script will convert a stack trace with addresses: -# 0x5fb015 yb::master::Master::Init() -# 0x5c2d38 yb::master::MiniMaster::Ports() -# 0x5c31fa yb::master::MiniMaster::Start() -# 0x58270a yb::MiniCluster::Start() -# 0x57dc71 yb::CreateTableStressTest::SetUp() -# To one with line numbers: -# 0x5fb015 yb::master::Master::Init() at /home/user/code/src/yb/src/master/master.cc:54 -# 0x5c2d38 yb::master::MiniMaster::Ports() at /home/user/code/src/yb/src/master/mini_master.cc:52 -# 0x5c31fa yb::master::MiniMaster::Start() at /home/user/code/src/yb/src/master/mini_master.cc:33 -# 0x58270a yb::MiniCluster::Start() at /home/user/code/src/yb/src/integration-tests/cluster.cc:48 -# 0x57dc71 yb::CreateTableStressTest::SetUp() at /home/user/code/src/yb/src/tests/stress-test.cc:61 -# -# If the script detects that the output is not symbolized, it will also attempt -# to determine the function names, i.e. it will convert: -# 0x5fb015 -# 0x5c2d38 -# 0x5c31fa -# To: -# 0x5fb015 yb::master::Master::Init() at /home/user/code/src/yb/src/master/master.cc:54 -# 0x5c2d38 yb::master::MiniMaster::Ports() at /home/user/code/src/yb/src/master/mini_master.cc:52 -# 0x5c31fa yb::master::MiniMaster::Start() at /home/user/code/src/yb/src/master/mini_master.cc:33 -####################################################################### -# -# The following only applies to changes made to this file as part of YugaByte development. -# -# Portions Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. -# -use strict; -use warnings; -no warnings 'portable'; # Support for 64-bit ints required - -use File::Basename; - -use constant DEBUG => 0; - -if (!@ARGV) { - die < is magical in Perl. -while (defined(my $input = )) { - # ExternalMiniCluster tests can produce stack traces prefixed by [m-1], [m-2], [m-3], [ts-1], - # [ts-2], [ts-3], e.g. - if ($input =~ /Shared library '(.*)' loaded at address 0x([0-9a-f]+)\s*$/) { - push(@library_offsets, [$1, hex($2)]); - $library_offsets_sorted = 0; - } - - if ($input =~ /^(?:\[([a-z]+)-\d+\])?\s+\@\s+(0x[[:xdigit:]]{6,})(?:\s+(\S+))?/ && - # Don't try to symbolize lines that already have a line number. - $input !~ /[.](c|cc|cxx|h|hpp):\d+\)$/) { - my $minicluster_daemon_prefix = $1; - my $addr = $2; - my $lookup_func_name = (!defined $3); - - my $current_binary = $binary; - if (defined($minicluster_daemon_prefix)) { - if ($minicluster_daemon_prefix eq "m") { - $current_binary = $master_binary; - } elsif ($minicluster_daemon_prefix eq "ts") { - $current_binary = $tserver_binary; - } - } - - if (!exists($addr2line_map{$addr})) { - if (!$library_offsets_sorted) { - @library_offsets = sort { $a->[1] <=> $b->[1] } @library_offsets; - if (DEBUG) { - for my $library_entry (@library_offsets) { - print "Library " . $library_entry->[0] . " is loaded at offset " . - to_hex_str($library_entry->[1]) . "\n";; - } - } - $library_offsets_sorted = 1; - } - - # The following could be optimized with binary search. - my $parsed_addr = hex($addr); - my $addr_to_use = $parsed_addr; - my $binary_to_use = $current_binary; - for my $library_entry (@library_offsets) { - my ($library_path, $library_offset) = @$library_entry; - if ($parsed_addr >= $library_offset) { - $binary_to_use = $library_path; - $addr_to_use = $parsed_addr - $library_offset; - } else { - # This library and all following libraries are loaded at higher addresses than the address - # we are looking for. - last; - } - } - if (DEBUG) { - print "For input line '$input' address '$addr' appears to be in the binary " . - "'$binary_to_use'\n"; - } - - my $addr2line_cmd = "addr2line -ifC -e \"" . $binary_to_use . '" ' . - to_hex_str($addr_to_use); - - $addr2line_map{$addr} = `$addr2line_cmd`; - } - chomp $input; - $input .= parse_addr2line_output($addr2line_map{$addr}, $lookup_func_name) . "\n"; - } - print $input; -} - -exit 0; diff --git a/build-support/update_test_result_xml.py b/build-support/update_test_result_xml.py index 409c07f2410b..4beaa405d79f 100755 --- a/build-support/update_test_result_xml.py +++ b/build-support/update_test_result_xml.py @@ -17,18 +17,46 @@ import argparse import os import sys +import signal +import time from xml.dom import minidom MAX_RESULT_XML_SIZE_BYTES = 16 * 1024 * 1024 +# This script sometimes gets stuck in our macOS NFS environment for unknown reasons, so we put a +# timeout around it. +TIMEOUT_SEC = 10 -def main(): + +# https://stackoverflow.com/questions/2281850/timeout-function-if-it-takes-too-long-to-finish +class Timeout: + def __init__(self, seconds=1, error_message='Timeout'): + self.seconds = seconds + self.start_time = None + + def handle_timeout(self, signum, frame): + if self.start_time is None: + raise RuntimeError("Timed out") + raise RuntimeError("Timed out after %.2f seconds" % (time.time() - self.start_time)) + + def __enter__(self): + self.start_time = time.time() + signal.signal(signal.SIGALRM, self.handle_timeout) + signal.alarm(self.seconds) + + def __exit__(self, type, value, traceback): + signal.alarm(0) + + +def initialize(): logging.basicConfig( level=logging.INFO, format="[" + os.path.basename(__file__) + "] %(asctime)s %(levelname)s: %(message)s") + +def parse_args(): parser = argparse.ArgumentParser( usage="usage: %(prog)s ", description="Updates a JUnit-style test result XML file, e.g. to add a log URL or " @@ -66,7 +94,10 @@ def main(): default='false', dest="mark_as_failed") - args = parser.parse_args() + return parser.parse_args() + + +def update_test_result_xml(args): result_xml_size = os.stat(args.result_xml).st_size if result_xml_size > MAX_RESULT_XML_SIZE_BYTES: logging.error( @@ -143,14 +174,27 @@ def main(): else: failure_node.appendChild(new_node) - output_xml_str = xml_dom.toxml() - with open(args.result_xml, 'w') as output_file: - output_file.write(output_xml_str) + output_xml_bytes = xml_dom.toxml().encode('utf-8') + with open(args.result_xml, 'wb') as output_file: + output_file.write(output_xml_bytes) return True +def main(): + args = None + try: + with Timeout(seconds=TIMEOUT_SEC): + args = parse_args() + return update_test_result_xml(args) + except: # noqa + if args: + logging.error("Error while trying to update test result XML: %s", args.result_xml) + raise + + if __name__ == "__main__": + initialize() if main(): exit_code = 0 else: diff --git a/java/yb-client/src/test/java/org/yb/minicluster/MiniYBCluster.java b/java/yb-client/src/test/java/org/yb/minicluster/MiniYBCluster.java index 12259bd705cb..775fa43eafb3 100644 --- a/java/yb-client/src/test/java/org/yb/minicluster/MiniYBCluster.java +++ b/java/yb-client/src/test/java/org/yb/minicluster/MiniYBCluster.java @@ -297,7 +297,7 @@ private String getDaemonBindAddress(MiniYBDaemonType daemonType) throws IOExcept return pickFreeRandomBindIpOnLinux(daemonType); } - return pickFreeBindIpOnlyVaryingLastTwoBytes(daemonType); + return pickFreeBindIpOnMacOrWithCertificate(daemonType); } private String getMasterBindAddress() throws IOException { @@ -320,15 +320,14 @@ private String pickFreeRandomBindIpOnLinux(MiniYBDaemonType daemonType) throws I daemonType.humanReadableName() + " in " + MAX_NUM_ATTEMPTS + " attempts"); } - private String getLoopbackIpWithLastByte(int lastByte) { - return "127.0.0." + lastByte; - } - - private String getLoopbackIpWithLastTwoBytesByte(int nextToLastByte, int lastByte) { + private String getLoopbackIpWithLastTwoBytes(int nextToLastByte, int lastByte) { return "127.0." + nextToLastByte + "." + lastByte; } - private String pickFreeBindIpOnlyVaryingLastTwoBytes( + private final int MIN_LAST_IP_BYTE = 2; + private final int MAX_LAST_IP_BYTE = 254; + + private String pickFreeBindIpOnMacOrWithCertificate( MiniYBDaemonType daemonType) throws IOException { List bindIps = new ArrayList<>(); @@ -337,16 +336,39 @@ private String pickFreeBindIpOnlyVaryingLastTwoBytes( // range of x. final int nextToLastByteMax = useIpWithCertificate ? 0 : 3; - // We only use even last bytes of the loopback IP in case we are testing TLS encryption. - final int lastIpByteStep = useIpWithCertificate ? 2 : 1; - for (int nextToLastByte = nextToLastByteMin; - nextToLastByte <= nextToLastByteMax; - ++nextToLastByte) { - for (int lastIpByte = 2; lastIpByte <= 254; lastIpByte += lastIpByteStep) { - String bindIp = getLoopbackIpWithLastTwoBytesByte(nextToLastByte, lastIpByte); - if (!usedBindIPs.contains(bindIp)) { - bindIps.add(bindIp); + if (TestUtils.IS_LINUX) { + // We only use even last bytes of the loopback IP in case we are testing TLS encryption. + final int lastIpByteStep = useIpWithCertificate ? 2 : 1; + for (int nextToLastByte = nextToLastByteMin; + nextToLastByte <= nextToLastByteMax; + ++nextToLastByte) { + for (int lastIpByte = MIN_LAST_IP_BYTE; + lastIpByte <= MAX_LAST_IP_BYTE; + lastIpByte += lastIpByteStep) { + String bindIp = getLoopbackIpWithLastTwoBytes(nextToLastByte, lastIpByte); + if (!usedBindIPs.contains(bindIp)) { + bindIps.add(bindIp); + } + } + } + } else { + List loopbackIps = BindIpUtil.getLoopbackIPs(); + if (useIpWithCertificate) { + // macOS, but we need a 127.0.0.x, where x is even. + for (String loopbackIp : loopbackIps) { + if (loopbackIp.startsWith("127.0.0.")) { + String[] components = loopbackIp.split("[.]"); + int lastIpByte = Integer.valueOf(components[components.length - 1]); + if (lastIpByte >= MIN_LAST_IP_BYTE && + lastIpByte <= MAX_LAST_IP_BYTE && + lastIpByte % 2 == 0) { + bindIps.add(loopbackIp); + } + } } + } else { + // macOS, no requirement that there is a certificate. + bindIps = loopbackIps; } } diff --git a/java/yb-client/src/test/java/org/yb/util/BindIpUtil.java b/java/yb-client/src/test/java/org/yb/util/BindIpUtil.java new file mode 100644 index 000000000000..3e28f1493724 --- /dev/null +++ b/java/yb-client/src/test/java/org/yb/util/BindIpUtil.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) YugaByte, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations + * under the License. + */ + +package org.yb.util; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InterfaceAddress; +import java.net.NetworkInterface; +import java.util.*; + +public class BindIpUtil { + + public static List getLoopbackIPs() throws IOException { + Enumeration nets = NetworkInterface.getNetworkInterfaces(); + List loopbackIps = new ArrayList<>(); + for (NetworkInterface netint : Collections.list(nets)) { + for (InterfaceAddress address : netint.getInterfaceAddresses()) { + InetAddress inetAddress = address.getAddress(); + if (inetAddress.getHostAddress().startsWith("127.")) { + loopbackIps.add(inetAddress.getHostAddress()); + } + } + } + return loopbackIps; + } + +} diff --git a/java/yb-client/src/test/java/org/yb/util/SideBySideDiff.java b/java/yb-client/src/test/java/org/yb/util/SideBySideDiff.java index 2e19e0a6321a..6f9cec0c647e 100644 --- a/java/yb-client/src/test/java/org/yb/util/SideBySideDiff.java +++ b/java/yb-client/src/test/java/org/yb/util/SideBySideDiff.java @@ -92,10 +92,10 @@ private static void sanityCheckLinesMatch( return; } String lineFromFile = lines.get(i); - if (lineFromFile.equals(lineFromDiff)) { + if (!lineFromFile.equals(lineFromDiff)) { LOG.error("SideBySideDiff sanity check failed: line " + (i + 1) + " from the " + - fileDescription + " file is\n" + lineFromFile + - "\nbut the diff tool implies it should be\n" + lineFromDiff); + fileDescription + " file is\n" + lineFromFile + "" + + "\nbut the diff tool implies it should be\n" + lineFromDiff + ""); } } diff --git a/python/yb/aggregate_test_reports.py b/python/yb/aggregate_test_reports.py new file mode 100755 index 000000000000..87c0389a56d2 --- /dev/null +++ b/python/yb/aggregate_test_reports.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python + +""" +Aggregates test reports from JSON-based test report files produced by postprocess_test_result.py. +Takes the set of ..._test_report.json files on standard input -- that is best generated with a +find command. Produces a test_results.json file with the following format: + +{ + "total_errors": 0, + "total_skipped": 296, + "total_failures": 1, + "total_test_instances": 4086, + "total_tests_run": 4062, + "build_root": "build/asan-clang-dynamic-enterprise-ninja", + "aggregation_errors": [], + "compiler_type": "clang", + "build_type": "asan", + "tests": [ + { + "language": "java", + "class_name": "org.yb.cql.TestSelect", + "junit_xml_path": "java/yb-cql/target/...", + "time": 12.804, + "log_path": "java/yb-cql/target/...", + "test_name": "testQualifiedColumnReference" + }, + ... + ] +} + +Also produces a shorter test_failures.json with the following format: + +{ + "total_errors": 0, + "total_skipped": 296, + "total_failures": 1, + "total_test_instances": 4086, + "total_tests_run": 4062, + "build_root": "build/asan-clang-dynamic-enterprise-ninja", + "aggregation_errors": [], + "compiler_type": "clang", + "build_type": "asan", + "successful_tests": [ + [ + "AdminCliTest", + "BlackList" + ], + ... + ], + "skipped_tests": [ + [ + "AdminCliTest", + "DISABLED_TestTLS" + ], + ... + ], + "failures": [ + { + "status": "run", + "language": "cxx", + "class_name": "TestRedisService", + "junit_xml_path": "build/asan-clang-dynamic-enterprise-ninja/...", + "test_name": "TestTimeSeriesTTL", + "extra_error_log_path": "build/asan-clang-dynamic-enterprise-ninja/...", + "log_path": "build/asan-clang-dynamic-enterprise-ninja/yb-test-logs/...", + "cxx_rel_test_binary": "tests-redisserver/redisserver-test", + "num_failures": 1, + "time": 59.223 + }, + ... + ] +} +""" + +import argparse +import json +import logging +import os +import sys +import yugabyte_pycommon + + +def is_test_failure(report): + for key in ['num_errors', 'num_failures']: + value = report.get(key, 0) + if value > 0: + return True + return False + + +def is_test_skipped(report): + return report.get('num_skipped', 0) > 0 or report.get('status') == 'notrun' + + +def get_zero_one_counter(report, key): + if report.get(key, 0) == 0: + return 0 + return 1 + + +def parse_args(): + parser = argparse.ArgumentParser( + description=__doc__) + parser.add_argument( + '--yb-src-root', + help='YugaByte source directory. Needed for making paths relative.', + required=True) + parser.add_argument( + '--output-dir', + help='Output directory to generate aggregated output files in.') + parser.add_argument( + '--build-type', + help='YugaByte build type. Added to test result files.') + parser.add_argument( + '--compiler-type', + help='C/C++ compiler type. Added to test result files.') + parser.add_argument( + '--build-root', + help='Root directory for build artifacts (not including Java). Added to test result files.', + required=True) + + return parser.parse_args() + + +def get_test_set(all_test_reports, predicate): + return sorted(set([ + (report["class_name"], report["test_name"]) + for report in all_test_reports if predicate(report) + ])) + + +def aggregate_test_reports(args): + all_test_reports = [] + failure_reports = [] + errors = [] + + for file_path in sys.stdin: + file_path = os.path.realpath(file_path.strip()) + try: + with open(file_path) as input_file: + report = json.load(input_file) + except IOError, ex: + errors.append("Failed reading file %s: %s" % (file_path, ex)) + + if isinstance(report, list): + all_test_reports.extend(report) + logging.info("JSON test report file has multiple results: %s", file_path) + else: + all_test_reports.append(report) + + logging.info("Collected %d test report files", len(all_test_reports)) + + total_errors = 0 + total_failures = 0 + total_skipped = 0 + total_tests_run = 0 + + for test_report in all_test_reports: + total_errors += get_zero_one_counter(test_report, "num_errors") + total_failures += get_zero_one_counter(test_report, "num_failures") + if is_test_skipped(test_report): + total_skipped += 1 + else: + total_tests_run += 1 + if is_test_failure(test_report): + failure_reports.append(test_report) + + top_level_details = { + "total_errors": total_errors, + "total_failures": total_failures, + "total_skipped": total_skipped, + "total_test_instances": len(all_test_reports), + "total_tests_run": total_tests_run, + "aggregation_errors": errors, + "build_type": args.build_type, + "compiler_type": args.compiler_type, + "build_root": os.path.relpath( + os.path.realpath(args.build_root), + os.path.realpath(args.yb_src_root) + ) + } + + failure_only_report = dict(top_level_details) + failure_only_report["failures"] = failure_reports + failure_only_report["successful_tests"] = get_test_set( + all_test_reports, + lambda test_report: not is_test_failure(test_report) and not is_test_skipped(test_report)) + failure_only_report["skipped_tests"] = get_test_set( + all_test_reports, lambda test_report: is_test_skipped(test_report)) + + full_report = dict(top_level_details) + full_report["tests"] = all_test_reports + + full_report_output_path = os.path.join(args.output_dir, 'test_results.json') + logging.info("Writing full test report to %s", full_report_output_path) + with open(full_report_output_path, 'w') as output_file: + json.dump(full_report, output_file, indent=2) + + failure_only_output_path = os.path.join(args.output_dir, 'test_failures.json') + logging.info("Writing failure-only test report to %s", failure_only_output_path) + with open(failure_only_output_path, 'w') as output_file: + json.dump(failure_only_report, output_file, indent=2) + + logging.info( + "Number of tests seen %d, run: %d, errors: %d, failures: %d, skipped: %d", + len(all_test_reports), + total_tests_run, + total_errors, + total_failures, + total_skipped) + + +if __name__ == '__main__': + yugabyte_pycommon.init_logging() + args = parse_args() + aggregate_test_reports(args), diff --git a/python/yb/postprocess_test_result.py b/python/yb/postprocess_test_result.py new file mode 100755 index 000000000000..1de1dc07743d --- /dev/null +++ b/python/yb/postprocess_test_result.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python + +""" +Post-processes results of running a single YugaByte DB unit test (e.g. a C++ or Java test) and +creates a structured output file with a summary of those results. This includes test running time +and possible causes of test failure. +""" + +import sys +import os +import logging +import yugabyte_pycommon +import argparse +import xml.etree.ElementTree as ET +import json +import signal +import glob + +# Example test failure (from C++) +# +# +# +# +# +# +# +# + +# Example test success (Java): +# +# +# +# +# +# +# + +# Example test failure (Java): +# +# +# +# +# +# +# +# redis.clients.jedis.exceptions.JedisConnectionException: Could not get a resource from the +# pool +# +# ... +# +# + +# Other related resources: +# https://llg.cubic.org/docs/junit/ +# https://stackoverflow.com/questions/442556/spec-for-junit-xml-output/4926073#4926073 +# https://raw.githubusercontent.com/windyroad/JUnit-Schema/master/JUnit.xsd + + +def rename_key(d, key, new_key): + if key in d: + d[new_key] = d[key] + del d[key] + + +def del_default_value(d, key, default_value): + if key in d and d[key] == default_value: + del d[key] + + +def count_subelements(root, el_name): + return len(list(root.iter(el_name))) + + +def set_element_count_property(test_kvs, root, el_name, field_name, increment_by=0): + count = count_subelements(root, el_name) + increment_by + if count != 0: + test_kvs[field_name] = count + + +class Postprocessor: + + def __init__(self): + parser = argparse.ArgumentParser( + description=__doc__) + parser.add_argument( + '--yb-src-root', + help='Root directory of YugaByte source code', + required=True) + parser.add_argument( + '--build-root', + help='Root directory of YugaByte build', + required=True) + parser.add_argument( + '--test-log-path', + help='Main log file of this test', + required=True) + parser.add_argument( + '--junit-xml-path', + help='JUnit-compatible XML result file of this test', + required=True) + parser.add_argument( + '--language', + help='The langugage this unit test is written in', + choices=['cxx', 'java'], + required=True) + parser.add_argument( + '--test-failed', + help='Flag computed by the Bash-based test framework saying whether the test failed', + choices=['true', 'false'] + ) + parser.add_argument( + '--fatal-details-path-prefix', + help='Prefix of fatal failure details files' + ) + parser.add_argument( + '--class-name', + help='Class name (helpful when there is no test result XML file)' + ) + parser.add_argument( + '--test-name', + help='Test name within the class (helpful when there is no test result XML file)' + ) + parser.add_argument( + '--java-module-dir', + help='Java module directory containing this test' + ) + parser.add_argument( + '--cxx-rel-test-binary', + help='C++ test binary path relative to the build directory') + parser.add_argument( + '--extra-error-log-path', + help='Extra error log path (stdout/stderr of the outermost test invocation)') + self.args = parser.parse_args() + self.test_log_path = self.args.test_log_path + if not os.path.exists(self.test_log_path) and os.path.exists(self.test_log_path + '.gz'): + self.test_log_path += '.gz' + logging.info("Log path: %s", self.test_log_path) + logging.info("JUnit XML path: %s", self.args.junit_xml_path) + self.real_build_root = os.path.realpath(self.args.build_root) + self.real_yb_src_root = os.path.realpath(self.args.yb_src_root) + self.rel_test_log_path = self.path_rel_to_src_root(self.test_log_path) + self.rel_junit_xml_path = self.path_rel_to_src_root(self.args.junit_xml_path) + + self.rel_extra_error_log_path = None + if self.args.extra_error_log_path: + self.rel_extra_error_log_path = self.path_rel_to_src_root( + self.args.extra_error_log_path) + + self.fatal_details_paths = None + if self.args.class_name: + logging.info("Externally specified class name: %s", self.args.class_name) + if self.args.test_name: + logging.info("Externally specified test name: %s", self.args.test_name) + + if (not self.args.fatal_details_path_prefix and + self.args.test_name and + self.args.test_log_path.endswith('-output.txt')): + # This mimics the logic in MiniYBCluster.configureAndStartProcess that comes up with a + # prefix like "org.yb.client.TestYBClient.testKeySpace.fatal_failure_details.", while + # the Java test log paths are of the form "org.yb.client.TestYBClient-output.txt". + self.args.fatal_details_path_prefix = '.'.join([ + self.args.test_log_path[:-len('-output.txt')], + self.args.test_name, + 'fatal_failure_details' + ]) + '.' + + if self.args.fatal_details_path_prefix: + self.fatal_details_paths = [ + self.path_rel_to_src_root(path) + for path in glob.glob(self.args.fatal_details_path_prefix + '*') + ] + + self.test_descriptor_str = os.environ.get('YB_TEST_DESCRIPTOR') + + def path_rel_to_build_root(self, path): + return os.path.relpath(os.path.realpath(path), self.real_build_root) + + def path_rel_to_src_root(self, path): + return os.path.relpath(os.path.realpath(path), self.real_yb_src_root) + + def set_common_test_kvs(self, test_kvs): + """ + Set common data for all tests produced by this invocation of the script. In practice there + won't be too much duplication as this script will be invoked for one test at a time. + """ + test_kvs["language"] = self.args.language + if self.args.cxx_rel_test_binary: + test_kvs["cxx_rel_test_binary"] = self.args.cxx_rel_test_binary + test_kvs["log_path"] = self.rel_test_log_path + test_kvs["junit_xml_path"] = self.rel_junit_xml_path + if self.fatal_details_paths: + test_kvs["fatal_details_paths"] = self.fatal_details_paths + if self.test_descriptor_str: + test_kvs["test_descriptor"] = self.test_descriptor_str + + if self.rel_extra_error_log_path: + test_kvs["extra_error_log_path"] = self.rel_extra_error_log_path + + def run(self): + junit_xml = ET.parse(self.args.junit_xml_path) + tests = [] + for test_case_element in junit_xml.iter('testcase'): + test_kvs = dict(test_case_element.attrib) + set_element_count_property(test_kvs, test_case_element, 'error', 'num_errors') + set_element_count_property(test_kvs, test_case_element, 'failure', 'num_failures') + + # In C++ tests, we don't get the "skipped" attribute, but we get the status="notrun" + # attibute. + set_element_count_property( + test_kvs, test_case_element, 'skipped', 'num_skipped', + increment_by=1 if test_kvs.get('status') == 'notrun' else 0) + + parsing_errors = [] + if "time" in test_kvs and isinstance(test_kvs["time"], str): + # Remove commas to parse numbers like "1,275.516". + time_str = test_kvs["time"].replace(",", "") + try: + test_kvs["time"] = float(time_str) + except ValueError, ex: + test_kvs["time"] = None + parsing_errors.append( + "Could not parse time: %s. Error: %s" % (time_str, str(ex)) + ) + if parsing_errors: + test_kvs["parsing_errors"] = parsing_errors + rename_key(test_kvs, 'name', 'test_name') + rename_key(test_kvs, 'classname', 'class_name') + self.set_common_test_kvs(test_kvs) + tests.append(test_kvs) + + output_path = os.path.splitext(self.args.junit_xml_path)[0] + '_test_report.json' + if len(tests) == 1: + tests = tests[0] + with open(output_path, 'w') as output_file: + output_file.write( + json.dumps(tests, indent=2) + ) + logging.info("Wrote JSON test report file: %s", output_path) + + +def main(): + postprocessor = Postprocessor() + postprocessor.run() + + +if __name__ == '__main__': + yugabyte_pycommon.init_logging() + main() diff --git a/python/yb/process_tree_supervisor.py b/python/yb/process_tree_supervisor.py index 7c02bcc4b75c..ca3f555005eb 100755 --- a/python/yb/process_tree_supervisor.py +++ b/python/yb/process_tree_supervisor.py @@ -110,16 +110,23 @@ def run(self): def new_process_found(self, process): pid = process.pid try: - logging.info("Tracking a child pid: %s: %s", pid, process.cmdline()) - return True + cmdline = process.cmdline() except psutil.NoSuchProcess, e: logging.warning("Newly added child process disappeared right away") return False except psutil.AccessDenied, e: logging.warning( - "Access denied trying to get the command line for pid %d, ignoring this process", - pid) + "Access denied trying to get the command line for pid %d (%s), " + "ignoring this process", pid, str(e)) return False + except OSError, e: + logging.warning( + "OSError trying to get the command line for pid %d (%s), ignoring this process", + pid, str(e)) + return False + + logging.info("Tracking a child pid: %s: %s", pid, cmdline) + return True def process_terminated(self, existing_pid): logging.info("Process finished by itself: %d", existing_pid) @@ -217,7 +224,7 @@ def main(): "Supervisor of pid %d's process tree (this script's pid: %d) is terminating%s", args.pid, os.getpid(), ('' if g_signal_caught is None else - ' (caught signal %d / %s' % ( + ' (caught signal %d / %s)' % ( g_signal_caught, SIGNAL_TO_NAME.get(g_signal_caught, 'unknown')))) diff --git a/src/yb/gutil/strings/string_util-test.cc b/src/yb/gutil/strings/string_util-test.cc index 21dfa30ca654..b088cfc78212 100644 --- a/src/yb/gutil/strings/string_util-test.cc +++ b/src/yb/gutil/strings/string_util-test.cc @@ -33,6 +33,7 @@ #include "yb/gutil/strings/util.h" #include "yb/util/string_util.h" +#include "yb/util/test_util.h" #include @@ -40,7 +41,10 @@ using std::string; namespace yb { -TEST(StringUtilTest, MatchPatternTest) { +class StringUtilTest : public YBTest { +}; + +TEST_F(StringUtilTest, MatchPatternTest) { EXPECT_TRUE(MatchPattern("www.google.com", "*.com")); EXPECT_TRUE(MatchPattern("www.google.com", "*")); EXPECT_FALSE(MatchPattern("www.google.com", "www*.g*.org")); @@ -73,7 +77,7 @@ TEST(StringUtilTest, MatchPatternTest) { "He********************************o")); } -TEST(StringUtilTest, TestIsBigInteger) { +TEST_F(StringUtilTest, TestIsBigInteger) { ASSERT_TRUE(IsBigInteger("0")); ASSERT_TRUE(IsBigInteger("1234")); ASSERT_TRUE(IsBigInteger("-1234")); @@ -93,7 +97,7 @@ TEST(StringUtilTest, TestIsBigInteger) { ASSERT_FALSE(IsBigInteger("0 ")); } -TEST(StringUtilTest, TestIsDecimal) { +TEST_F(StringUtilTest, TestIsDecimal) { // Integer cases ASSERT_TRUE(IsDecimal("0")); ASSERT_TRUE(IsDecimal("1234")); @@ -151,7 +155,7 @@ TEST(StringUtilTest, TestIsDecimal) { ASSERT_FALSE(IsDecimal("0e0.")); } -TEST(StringUtilTest, TestIsBoolean) { +TEST_F(StringUtilTest, TestIsBoolean) { ASSERT_TRUE(IsBoolean("true")); ASSERT_TRUE(IsBoolean("TRUE")); ASSERT_TRUE(IsBoolean("fAlSe")); @@ -163,7 +167,7 @@ TEST(StringUtilTest, TestIsBoolean) { ASSERT_FALSE(IsBoolean("false ")); } -TEST(StringUtilTest, TestAppendWithSeparator) { +TEST_F(StringUtilTest, TestAppendWithSeparator) { string s; AppendWithSeparator("foo", &s); ASSERT_EQ(s, "foo"); @@ -181,7 +185,7 @@ TEST(StringUtilTest, TestAppendWithSeparator) { ASSERT_EQ(s, "foo, bar -- foo"); } -TEST(StringUtilTest, TestCollectionToString) { +TEST_F(StringUtilTest, TestCollectionToString) { std::vector v{"foo", "123", "bar", ""}; ASSERT_EQ("[foo, 123, bar, ]", VectorToString(v)); ASSERT_EQ("[foo, 123, bar, ]", RangeToString(v.begin(), v.end())); diff --git a/src/yb/util/debug-util.h b/src/yb/util/debug-util.h index dcc2ba5c8a5b..d29a190dd585 100644 --- a/src/yb/util/debug-util.h +++ b/src/yb/util/debug-util.h @@ -99,10 +99,7 @@ inline std::string GetStackTraceWithoutTopFrame() { std::string GetStackTraceHex(); // This is the same as GetStackTraceHex(), except multi-line in a format that -// looks very similar to GetStackTrace() but without symbols. Because it's in -// that format, the tool stacktrace_addr2line.pl in the yb build-support -// directory can symbolize it automatically (to the extent that addr2line(1) -// is able to find the symbols). +// looks very similar to GetStackTrace() but without symbols. std::string GetLogFormatStackTraceHex(); // Collect the current stack trace in hex form into the given buffer. diff --git a/src/yb/yql/pggate/pg_session.cc b/src/yb/yql/pggate/pg_session.cc index 1d46e35c6ac2..305446265754 100644 --- a/src/yb/yql/pggate/pg_session.cc +++ b/src/yb/yql/pggate/pg_session.cc @@ -46,9 +46,15 @@ using client::YBTable; using client::YBTableName; using client::YBTableType; -// TODO(neil) This should be derived from a GFLAGS. -static MonoDelta kSessionTimeout = 60s; +#if defined(__APPLE__) && !defined(NDEBUG) +// We are experiencing more slowness in tests on macOS in debug mode. +const int kDefaultPgYbSessionTimeoutMs = 120 * 1000; +#else +const int kDefaultPgYbSessionTimeoutMs = 60 * 1000; +#endif +DEFINE_int32(pg_yb_session_timeout_ms, kDefaultPgYbSessionTimeoutMs, + "Timeout for operations between PostgreSQL server and YugaByte DocDB services"); //-------------------------------------------------------------------------------------------------- // Constants used for the sequences data table. @@ -82,7 +88,7 @@ PgSession::PgSession( session_(client_->NewSession()), pg_txn_manager_(std::move(pg_txn_manager)), clock_(std::move(clock)) { - session_->SetTimeout(kSessionTimeout); + session_->SetTimeout(MonoDelta::FromMilliseconds(FLAGS_pg_yb_session_timeout_ms)); session_->SetForceConsistentRead(client::ForceConsistentRead::kTrue); } diff --git a/thirdparty/homebrew_version_for_jenkins.txt b/thirdparty/homebrew_version_for_jenkins.txt index cc410538902d..12bb833b2515 100644 --- a/thirdparty/homebrew_version_for_jenkins.txt +++ b/thirdparty/homebrew_version_for_jenkins.txt @@ -1 +1 @@ -2018-11-14T20_55_23 +2019-03-29T14_22_17 diff --git a/thirdparty/version_for_jenkins_mac.txt b/thirdparty/version_for_jenkins_mac.txt index b375eccda6c5..fe36a609e4d6 100644 --- a/thirdparty/version_for_jenkins_mac.txt +++ b/thirdparty/version_for_jenkins_mac.txt @@ -1 +1 @@ -2019-02-28T00_41_05 \ No newline at end of file +2019-03-29T17_59_31 diff --git a/yb_build.sh b/yb_build.sh index 0d476773279a..b8a5236f4159 100755 --- a/yb_build.sh +++ b/yb_build.sh @@ -700,6 +700,10 @@ while [[ $# -gt 0 ]]; do export YB_GTEST_FILTER=$2 shift ;; + # Support the way of argument passing that is used for gtest test programs themselves. + --gtest-filter=*) + export YB_GTEST_FILTER=${2#--gtest-filter=} + ;; --rebuild-file) ensure_option_has_arg "$@" register_file_to_rebuild "$2" @@ -764,7 +768,8 @@ while [[ $# -gt 0 ]]; do ;; --) if [[ $num_test_repetitions -lt 2 ]]; then - fatal "Forward to arguments to repeat_unit_test.sh without multiple repetitions" + fatal "Trying to forward arguments to repeat_unit_test.sh, but -n not specified, so" \ + "we won't be repeating a test multiple times." fi forward_args_to_repeat_unit_test=true ;;