Optimize circleci build times (#2781)

* Optimize

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

* Faster checkout

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

* Split building of extension to speed build workflow up

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

* Cache cargo packages

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

* Fix ASAN build

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

* Move web tests from build_packages to build

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

* Fix some tests

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

* Increase test_c2php timeout

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

---------

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

Makefile

@@ -26,6 +26,7 @@ ARCHITECTURE = $(shell uname -m)
 QUIET_TESTS := ${CIRCLE_SHA1}
 RUST_DEBUG_BUILD ?= $(shell [ -n "${DD_TRACE_DOCKER_DEBUG}" ] && echo 1)
 EXTRA_CONFIGURE_OPTIONS ?=
+ASSUME_COMPILED := ${DD_TRACE_ASSUME_COMPILED}
 
 VERSION := $(shell cat VERSION)
 
@@ -97,19 +98,22 @@ all: $(BUILD_DIR)/configure $(SO_FILE)
 $(BUILD_DIR)/configure: $(M4_FILES) $(BUILD_DIR)/ddtrace.sym $(BUILD_DIR)/VERSION
 	$(Q) (cd $(BUILD_DIR); phpize && $(SED_I) 's/\/FAILED/\/\\bFAILED/' $(BUILD_DIR)/run-tests.php) # Fix PHP 5.4 exit code bug when running selected tests (FAILED vs XFAILED)
 
+$(BUILD_DIR)/run-tests.php: $(if $(ASSUME_COMPILED),, $(BUILD_DIR)/configure)
+	$(if $(ASSUME_COMPILED), cp $(shell dirname $(shell realpath $(shell which phpize)))/../lib/php/build/run-tests.php $(BUILD_DIR)/run-tests.php)
+
 $(BUILD_DIR)/Makefile: $(BUILD_DIR)/configure
-	$(Q) (cd $(BUILD_DIR); ./configure --$(if $(RUST_DEBUG_BUILD),enable,disable)-ddtrace-rust-debug $(EXTRA_CONFIGURE_OPTIONS))
+	$(Q) (cd $(BUILD_DIR); ./configure --$(if $(RUST_DEBUG_BUILD),enable,disable)-ddtrace-rust-debug $(if $(ASAN), --enable-ddtrace-sanitize) $(EXTRA_CONFIGURE_OPTIONS))
 
 all_object_files: $(C_FILES) $(RUST_FILES) $(BUILD_DIR)/Makefile
 
-$(SO_FILE): all_object_files $(BUILD_DIR)/compile_rust.sh
-	$(Q) $(MAKE) -C $(BUILD_DIR) -j CFLAGS="$(CFLAGS)$(if $(ASAN), -fsanitize=address)" LDFLAGS="$(LDFLAGS)$(if $(ASAN), -fsanitize=address)"
+$(SO_FILE): $(if $(ASSUME_COMPILED),, all_object_files $(BUILD_DIR)/compile_rust.sh)
+	$(if $(ASSUME_COMPILED),,$(Q) $(MAKE) -C $(BUILD_DIR) -j)
 
 $(AR_FILE): all_object_files
-	$(Q) $(MAKE) -C $(BUILD_DIR) -j ./modules/ddtrace.a all CFLAGS="$(CFLAGS)$(if $(ASAN), -fsanitize=address)"
+	$(Q) $(MAKE) -C $(BUILD_DIR) -j ./modules/ddtrace.a all
 
 $(PHP_EXTENSION_DIR)/ddtrace.so: $(SO_FILE)
-	$(Q) $(SUDO) $(MAKE) -C $(BUILD_DIR) install
+	$(Q) $(SUDO) $(if $(ASSUME_COMPILED),cp $(BUILD_DIR)/modules/ddtrace.so $(PHP_EXTENSION_DIR)/ddtrace.so,$(MAKE) -C $(BUILD_DIR) install)
 
 install: $(PHP_EXTENSION_DIR)/ddtrace.so
 
@@ -125,51 +129,51 @@ install_ini: $(INI_FILE)
 
 install_all: install install_ini
 
-run_tests: $(TEST_FILES) $(TEST_STUB_FILES) $(BUILD_DIR)/configure
+run_tests: $(TEST_FILES) $(TEST_STUB_FILES) $(BUILD_DIR)/run-tests.php
 	$(RUN_TESTS_CMD) $(BUILD_DIR)/$(TESTS)
 
-test_c: $(SO_FILE) $(TEST_FILES) $(TEST_STUB_FILES)
+test_c: $(SO_FILE) $(TEST_FILES) $(TEST_STUB_FILES) $(BUILD_DIR)/run-tests.php
 	$(if $(ASAN), USE_ZEND_ALLOC=0 USE_TRACKED_ALLOC=1) DD_TRACE_CLI_ENABLED=1 DD_TRACE_GIT_METADATA_ENABLED=0 $(RUN_TESTS_CMD) -d extension=$(SO_FILE) $(BUILD_DIR)/$(TESTS)
 
 test_c_coverage: dist_clean
 	DD_TRACE_DOCKER_DEBUG=1 EXTRA_CFLAGS="-fprofile-arcs -ftest-coverage" $(MAKE) test_c || exit 0
 
-test_c_disabled: $(SO_FILE) $(TEST_FILES) $(TEST_STUB_FILES)
+test_c_disabled: $(SO_FILE) $(TEST_FILES) $(TEST_STUB_FILES) $(BUILD_DIR)/run-tests.php
 	( \
 	DD_TRACE_CLI_ENABLED=0 DD_TRACE_DEBUG=1 $(RUN_TESTS_CMD) -d extension=$(SO_FILE) $(BUILD_DIR)/$(TESTS) || true; \
 	! grep -E 'Successfully triggered flush with trace of size|=== Total [0-9]+ memory leaks detected ===|Segmentation fault|Assertion ' $$(find $(BUILD_DIR)/$(TESTS) -name "*.out" | grep -v segfault_backtrace_enabled.out); \
 	)
 
-test_c_observer: $(SO_FILE) $(TEST_FILES) $(TEST_STUB_FILES)
+test_c_observer: $(SO_FILE) $(TEST_FILES) $(TEST_STUB_FILES) $(BUILD_DIR)/run-tests.php
 	$(if $(ASAN), USE_ZEND_ALLOC=0 USE_TRACKED_ALLOC=1) DD_TRACE_CLI_ENABLED=1 DD_TRACE_GIT_METADATA_ENABLED=0 $(RUN_TESTS_CMD) -d extension=$(SO_FILE) -d extension=zend_test.so -d zend_test.observer.enabled=1 -d zend_test.observer.observe_all=1 -d zend_test.observer.show_output=0 $(BUILD_DIR)/$(TESTS)
 
-test_opcache: $(SO_FILE) $(TEST_OPCACHE_FILES)
+test_opcache: $(SO_FILE) $(TEST_OPCACHE_FILES) $(BUILD_DIR)/run-tests.php
 	$(if $(ASAN), USE_ZEND_ALLOC=0 USE_TRACKED_ALLOC=1) DD_TRACE_CLI_ENABLED=1 $(RUN_TESTS_CMD) -d extension=$(SO_FILE) -d zend_extension=opcache.so $(BUILD_DIR)/tests/opcache
 
 test_c_mem: export DD_TRACE_CLI_ENABLED=1
-test_c_mem: $(SO_FILE) $(TEST_FILES) $(TEST_STUB_FILES)
+test_c_mem: $(SO_FILE) $(TEST_FILES) $(TEST_STUB_FILES) $(BUILD_DIR)/run-tests.php
 	$(RUN_TESTS_CMD) -d extension=$(SO_FILE) -m $(BUILD_DIR)/$(TESTS)
 
-test_c2php: $(SO_FILE) $(INIT_HOOK_TEST_FILES)
+test_c2php: $(SO_FILE) $(INIT_HOOK_TEST_FILES) $(BUILD_DIR)/run-tests.php
 	( \
 	set -xe; \
+	sed -i 's/stream_socket_accept($$listenSock, 5)/stream_socket_accept($$listenSock, 20)/' $(BUILD_DIR)/run-tests.php; \
 	export DD_TRACE_CLI_ENABLED=1; \
 	export USE_ZEND_ALLOC=0; \
 	export ZEND_DONT_UNLOAD_MODULES=1; \
 	export USE_TRACKED_ALLOC=1; \
-	$(shell grep -Pzo '(?<=--ENV--)(?s).+?(?=--)' $(INIT_HOOK_TEST_FILES)) valgrind -q --tool=memcheck --trace-children=yes --vex-iropt-register-updates=allregs-at-mem-access php -n -d extension=$(SO_FILE) -d datadog.trace.sources_path=$(TRACER_SOURCE_DIR) $(INIT_HOOK_TEST_FILES); \
+	$(shell grep -Pzo '(?<=--ENV--)(?s).+?(?=--)' $(INIT_HOOK_TEST_FILES)) valgrind -q --tool=memcheck --trace-children=yes --vex-iropt-register-updates=allregs-at-mem-access bash -c '$(RUN_TESTS_CMD) -d extension=$(SO_FILE) -d datadog.trace.sources_path=$(TRACER_SOURCE_DIR) -d pcre.jit=0 $(INIT_HOOK_TEST_FILES)'; \
 	)
 
-test_with_init_hook: $(SO_FILE) $(INIT_HOOK_TEST_FILES)
+test_with_init_hook: $(SO_FILE) $(INIT_HOOK_TEST_FILES) $(BUILD_DIR)/run-tests.php
 	$(if $(ASAN), USE_ZEND_ALLOC=0 USE_TRACKED_ALLOC=1) DD_TRACE_CLI_ENABLED=1 $(RUN_TESTS_CMD) -d extension=$(SO_FILE) -d datadog.trace.sources_path=$(TRACER_SOURCE_DIR) $(INIT_HOOK_TEST_FILES);
 
-test_extension_ci: $(SO_FILE) $(TEST_FILES) $(TEST_STUB_FILES)
+test_extension_ci: $(SO_FILE) $(TEST_FILES) $(TEST_STUB_FILES) $(BUILD_DIR)/run-tests.php
 	( \
 	set -xe; \
 	export PATH="$(PROJECT_ROOT)/tests/ext/valgrind:$$PATH"; \
 	export DD_TRACE_CLI_ENABLED=1; \
 	export TEST_PHP_JUNIT=$(JUNIT_RESULTS_DIR)/normal-extension-test.xml; \
-	$(MAKE) -C $(BUILD_DIR) CFLAGS="-g" clean all; \
 	export DD_TRACE_GIT_METADATA_ENABLED=0; \
 	$(RUN_TESTS_CMD) -d extension=$(SO_FILE) $(BUILD_DIR)/$(TESTS); \
 	\
@@ -435,32 +439,41 @@ $(PACKAGES_BUILD_DIR):
 
 # Example .tar.gz.aarch64, .tar.gz.x86_64
 .tar.gz.%: $(PACKAGES_BUILD_DIR)
-	mkdir -p /tmp/$(PACKAGES_BUILD_DIR)
-	rm -rf /tmp/$(PACKAGES_BUILD_DIR)/*
-	fpm -p /tmp/$(PACKAGES_BUILD_DIR)/$(PACKAGE_NAME)-$(VERSION) -t dir $(call FPM_OPTS, $(*)) $(call FPM_FILES, $(*))
-	tar zcf $(PACKAGES_BUILD_DIR)/$(PACKAGE_NAME)-$(VERSION).$(*).tar.gz -C /tmp/$(PACKAGES_BUILD_DIR)/$(PACKAGE_NAME)-$(VERSION) . --owner=0 --group=0
+	mkdir -p /tmp/$(PACKAGES_BUILD_DIR)-$(*)
+	rm -rf /tmp/$(PACKAGES_BUILD_DIR)-$(*)/*
+	fpm -p /tmp/$(PACKAGES_BUILD_DIR)-$(*)/$(PACKAGE_NAME)-$(VERSION) -t dir $(call FPM_OPTS, $(*)) $(call FPM_FILES, $(*))
+	tar -zcf $(PACKAGES_BUILD_DIR)/$(PACKAGE_NAME)-$(VERSION).$(*).tar.gz -C /tmp/$(PACKAGES_BUILD_DIR)-$(*)/$(PACKAGE_NAME)-$(VERSION) . --owner=0 --group=0
 
 bundle.tar.gz: $(PACKAGES_BUILD_DIR)
 	bash ./tooling/bin/generate-final-artifact.sh \
 		$(VERSION) \
 		$(PACKAGES_BUILD_DIR)
+
+$(PACKAGES_BUILD_DIR)/datadog-setup.php: $(PACKAGES_BUILD_DIR)
 	bash ./tooling/bin/generate-installers.sh \
 		$(VERSION) \
 		$(PACKAGES_BUILD_DIR)
-	bash ./tooling/bin/generate-ssi-package.sh \
-		$(VERSION) \
-		$(PACKAGES_BUILD_DIR)
 
 build_pecl_package:
 	BUILD_DIR='$(BUILD_DIR)/'; \
 	FILES="$(C_FILES) $(RUST_FILES) $(TEST_FILES) $(TEST_STUB_FILES) $(M4_FILES) Cargo.lock"; \
 	tooling/bin/pecl-build $${FILES//$${BUILD_DIR}/}
 
 dbgsym.tar.gz:
-	$(if $(DDTRACE_MAKE_PACKAGES_ASAN), , tar zcf $(PACKAGES_BUILD_DIR)/dd-library-php-$(VERSION)_windows_debugsymbols.tar.gz ./extensions_windows_x86_64_debugsymbols --owner=0 --group=0)
+	$(if $(DDTRACE_MAKE_PACKAGES_ASAN), , tar -zcf $(PACKAGES_BUILD_DIR)/dd-library-php-$(VERSION)_windows_debugsymbols.tar.gz ./extensions_windows_x86_64_debugsymbols --owner=0 --group=0)
+
+installer_packages: .apk.x86_64 .apk.aarch64 .rpm.x86_64 .rpm.aarch64 .deb.x86_64 .deb.arm64 .tar.gz.x86_64 .tar.gz.aarch64 bundle.tar.gz dbgsym.tar.gz
+	tar --use-compress-program=pigz --exclude='dd-library-php-ssi-*' -cf packages.tar.gz $(PACKAGES_BUILD_DIR) --owner=0 --group=0
+
+ssi_packages: $(PACKAGES_BUILD_DIR)
+	bash ./tooling/bin/generate-ssi-package.sh \
+		$(VERSION) \
+		$(PACKAGES_BUILD_DIR)
+
+calculate_package_sha256_sums: $(PACKAGES_BUILD_DIR)/datadog-setup.php installer_packages
+	(cd build/packages && find . -type f -exec sha256sum {} + > ../../package_sha256sums)
 
-packages: .apk.x86_64 .apk.aarch64 .rpm.x86_64 .rpm.aarch64 .deb.x86_64 .deb.arm64 .tar.gz.x86_64 .tar.gz.aarch64 bundle.tar.gz dbgsym.tar.gz
-	tar --exclude='dd-library-php-ssi-*' -zcf packages.tar.gz $(PACKAGES_BUILD_DIR) --owner=0 --group=0
+packages: $(PACKAGES_BUILD_DIR)/datadog-setup.php ssi_packages installer_packages
 
 # Generates the src/bridge/_generated_*.php files.
 generate:

compile_rust.sh

@@ -15,4 +15,11 @@ case "${host_os}" in
     ;;
 esac
 
+set -x
+
+if test -n "$COMPILE_ASAN"; then
+  export LDFLAGS="-fsanitize=address"
+  export CFLAGS="-fsanitize=address -fno-omit-frame-pointer"
+fi
+
 SIDECAR_VERSION=$(cat ../VERSION) RUSTFLAGS="$RUSTFLAGS" RUSTC_BOOTSTRAP=1 "${DDTRACE_CARGO:-cargo}" build $(test "${PROFILE:-debug}" = "debug" || echo --profile "$PROFILE") "$@"

config.m4

@@ -1,8 +1,8 @@
 PHP_ARG_ENABLE(ddtrace, whether to enable Datadog tracing support,
   [  --enable-ddtrace   Enable Datadog tracing support])
 
-PHP_ARG_WITH(ddtrace-sanitize, whether to enable AddressSanitizer for ddtrace,
-  [  --with-ddtrace-sanitize Build Datadog tracing with AddressSanitizer support], no, no)
+PHP_ARG_ENABLE(ddtrace-sanitize, whether to enable AddressSanitizer for ddtrace,
+  [  --enable-ddtrace-sanitize Build Datadog tracing with AddressSanitizer support], no, no)
 
 PHP_ARG_WITH(ddtrace-rust-library, the rust library is located; i.e. to compile without cargo,
   [  --with-ddtrace-rust-library Location to rust library for linking against], -, will be compiled)
@@ -325,7 +325,7 @@ EOT
 
     cat <<EOT >> Makefile.fragments
 $ddtrace_rust_lib: $( (find "$ext_srcdir/components-rs" -name "*.rs" -o -name "Cargo.toml"; find "$ext_srcdir/../../libdatadog" -name "*.rs" -not -path "*/target/*"; find "$ext_srcdir/libdatadog" -name "*.rs" -not -path "*/target/*") 2>/dev/null | xargs )
-	(cd "$ext_srcdir"; CARGO_TARGET_DIR=\$(builddir)/target/ SHARED=$(test "$ext_shared" = "yes" && echo 1) PROFILE="$ddtrace_cargo_profile" host_os="$host_os" DDTRACE_CARGO=\$(DDTRACE_CARGO) sh ./compile_rust.sh \$(shell echo "\$(MAKEFLAGS)" | $EGREP -o "[[-]]j[[0-9]]+"))
+	(cd "$ext_srcdir"; CARGO_TARGET_DIR=\$(builddir)/target/ SHARED=$(test "$ext_shared" = "yes" && echo 1) PROFILE="$ddtrace_cargo_profile" host_os="$host_os" DDTRACE_CARGO=\$(DDTRACE_CARGO) $(if test "$PHP_DDTRACE_SANITIZE" != "no"; then echo COMPILE_ASAN=1; fi) sh ./compile_rust.sh \$(shell echo "\$(MAKEFLAGS)" | $EGREP -o "[[-]]j[[0-9]]+"))
 EOT
   fi
 

dockerfiles/packaging/fpm_packaging/Dockerfile

@@ -0,0 +1,23 @@
+FROM cimg/ruby:3.3.4 AS base
+
+USER root
+WORKDIR /home/circleci
+
+RUN apt-get update && apt-get -y install vim less build-essential rpm lintian jq
+
+# Need at least tar 1.35 to work around concurrency problems with atime and inode count checks
+# See also: https://stackoverflow.com/a/77765876
+FROM base as compile-tar
+RUN curl -LO https://ftp.gnu.org/gnu/tar/tar-1.35.tar.gz
+RUN tar xfz tar-1.35.tar.gz
+RUN (cd tar-1.35 && FORCE_UNSAFE_CONFIGURE=1 ./configure --prefix=/usr && make install)
+
+FROM base as final
+ADD fpm_apk_pax_header.patch /tmp
+RUN sudo gem install fpm -v 1.15.1
+RUN (cd /usr/local/lib/ruby/gems/3.3.0/gems/fpm-1.15.1; patch -p 1 < /tmp/fpm_apk_pax_header.patch ) && rm -f /tmp/fpm_apk_pax_header.patch
+
+COPY --from=compile-tar /usr/bin/tar /usr/bin/tar
+
+USER circleci
+ENTRYPOINT [ "bash" ]

dockerfiles/packaging/fpm_packaging/README.md

@@ -0,0 +1,5 @@
+## Build for x86_64
+
+```
+docker buildx build --platform=linux/amd64 -t datadog/dd-trace-ci:php_fpm_packaging --push .
+```

dockerfiles/packaging/fpm_packaging/fpm_apk_pax_header.patch

@@ -0,0 +1,29 @@
+From 18221cceb06860395120b56cca7e270539b74073 Mon Sep 17 00:00:00 2001
+From: Pawel Chojnacki <pawel.chcki@gmail.com>
+Date: Wed, 30 Jan 2019 16:35:50 +0100
+Subject: [PATCH] Add guard against modifying LongLink entry and leave entries
+ above 80 chars alone
+
+---
+ lib/fpm/package/apk.rb | 6 ++++++
+ 1 file changed, 6 insertions(+)
+
+diff --git a/lib/fpm/package/apk.rb b/lib/fpm/package/apk.rb
+index dd69067..bc9ff50 100644
+--- a/lib/fpm/package/apk.rb
++++ b/lib/fpm/package/apk.rb
+@@ -435,6 +435,12 @@ class FPM::Package::APK< FPM::Package
+   # This takes an unchanged directory name and "paxifies" it.
+   def add_paxstring(ret)
+
++    # Check if this is special Tar filename to mark that the next entry file name will be longer than 100 characters.
++    return ret if ret.match? "././@LongLink"
++
++    # adding PaxHeaders prefix to entries over 80 characters is impossible without creating extra entries
++    return ret if ret.length > 80
++
+     pax_slash = ret.rindex('/')
+     if(pax_slash == nil)
+       pax_slash = 0
+--
+2.18.0

tests/ext/background-sender/agent_sampling_sidecar.phpt

@@ -2,6 +2,7 @@
 The sidecar trace flusher sender informs about changes to the agent sample rate
 --SKIPIF--
 <?php include __DIR__ . '/../includes/skipif_no_dev_env.inc'; ?>
+<?php if (getenv('USE_ZEND_ALLOC') === '0' && !getenv("SKIP_ASAN")) die('skip: valgrind reports sendmsg(msg.msg_control) points to uninitialised byte(s), but it is unproblematic and outside our control in rust code'); ?>
 --ENV--
 DD_TRACE_LOG_LEVEL=info,startup=off
 DD_AGENT_HOST=request-replayer

tests/randomized/Makefile

@@ -66,7 +66,7 @@ pull.buster.%:
 library.local:
 	@mkdir -p $(LIBRARY_DOWNLOAD_PATH)
 #	For now we only handle centos in randomized tests
-	@cp ../../build/packages/dd-library-php-*-$(ARCHITECTURE)-linux-gnu.tar.gz $(LIBRARY_DOWNLOAD_PATH)/dd-library-php.tar.gz
+	@cp ../../dd-library-php-*-$(ARCHITECTURE)-linux-gnu.tar.gz $(LIBRARY_DOWNLOAD_PATH)/dd-library-php.tar.gz
 
 library.download:
 	@echo "Downloading library at url: $(LIBRARY_TEST_URL)"