Add appsec benchmarks (#2791)

Add benchmarking with appsec loaded Add logs of all benchmarking executions
DataDog · Sep 25, 2024 · f117e22 · f117e22
1 parent 3746a2f
commit f117e22
Show file tree

Hide file tree

Showing 13 changed files with 232 additions and 171 deletions.
diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml
@@ -63,9 +63,34 @@ benchmarks-tracer:
         compare_to: "master"
       when: on_success
     - when: manual
+  artifacts:
+    name: "logs"
+    paths:
+      - candidate.tar.gz
+      - baseline.tar.gz
+    expire_in: 2 days
   variables:
     SCENARIO: "tracer"
 
+benchmarks-appsec:
+  extends: .microbenchmarks
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
+      changes:
+        paths:
+          - appsec/src/**/*
+        compare_to: "master"
+      when: on_success
+    - when: manual
+  artifacts:
+    name: "logs"
+    paths:
+      - candidate.tar.gz
+      - baseline.tar.gz
+    expire_in: 2 days
+  variables:
+    SCENARIO: "appsec"
+
 
 download_circle_ci_release:
   stage: benchmarks

diff --git a/Makefile b/Makefile
@@ -3,8 +3,10 @@ PROJECT_ROOT := ${PWD}
 TRACER_SOURCE_DIR := $(PROJECT_ROOT)/src/
 ASAN ?= $(shell ldd $(shell which php) 2>/dev/null | grep -q libasan && echo 1)
 SHELL = /bin/bash
+APPSEC_SOURCE_DIR = $(PROJECT_ROOT)/appsec/
 BUILD_SUFFIX = extension
 BUILD_DIR = $(PROJECT_ROOT)/tmp/build_$(BUILD_SUFFIX)
+BUILD_DIR_APPSEC = $(BUILD_DIR)/appsec/
 ZAI_BUILD_DIR = $(PROJECT_ROOT)/tmp/build_zai$(if $(ASAN),_asan)
 TEA_BUILD_DIR = $(PROJECT_ROOT)/tmp/build_tea$(if $(ASAN),_asan)
 TEA_INSTALL_DIR = $(TEA_BUILD_DIR)/opt
@@ -13,7 +15,8 @@ TEA_BUILD_BENCHMARKS ?= OFF
 TEA_BENCHMARK_REPETITIONS ?= 10
 # Note: If the tea benchmark format or output is changed, make changes to ./benchmark/runall.sh
 TEA_BENCHMARK_FORMAT ?= json
-TEA_BENCHMARK_OUTPUT ?= $(PROJECT_ROOT)/tea/benchmarks/reports/tea-bench-results.$(TEA_BENCHMARK_FORMAT)
+TEA_BENCHMARK_OUTPUT ?= $(PROJECT_ROOT)/tea/benchmarks/reports/tracer-tea-bench-results.$(TEA_BENCHMARK_FORMAT)
+BENCHMARK_EXTRA ?=
 COMPONENTS_BUILD_DIR = $(PROJECT_ROOT)/tmp/build_components
 SO_FILE = $(BUILD_DIR)/modules/ddtrace.so
 AR_FILE = $(BUILD_DIR)/modules/ddtrace.a
@@ -127,6 +130,24 @@ $(INI_FILE):
 
 install_ini: $(INI_FILE)
 
+delete_ini:
+	$(SUDO) rm $(INI_FILE)
+
+install_appsec:
+	cmake -S $(APPSEC_SOURCE_DIR) -B $(BUILD_DIR_APPSEC)
+	cd $(BUILD_DIR_APPSEC);make extension ddappsec-helper
+	cp $(BUILD_DIR_APPSEC)/ddappsec.so $(PHP_EXTENSION_DIR)/ddappsec.so
+	cp $(BUILD_DIR_APPSEC)/libddappsec-helper.so $(PHP_EXTENSION_DIR)/libddappsec-helper.so
+	cp $(APPSEC_SOURCE_DIR)/recommended.json /tmp/recommended.json
+	$(Q) echo "extension=ddappsec.so" | $(SUDO) tee -a $(INI_FILE)
+	$(Q) echo "datadog.appsec.cli_start_on_rinit=true" | $(SUDO) tee -a $(INI_FILE)
+	$(Q) echo "datadog.appsec.helper_path=$(PHP_EXTENSION_DIR)/libddappsec-helper.so" | $(SUDO) tee -a $(INI_FILE)
+	$(Q) echo "datadog.appsec.rules=/tmp/recommended.json" | $(SUDO) tee -a $(INI_FILE)
+	$(Q) echo "datadog.appsec.helper_socket_path=/tmp/ddappsec.sock" | $(SUDO) tee -a $(INI_FILE)
+	$(Q) echo "datadog.appsec.helper_lock_path=/tmp/ddappsec.lock" | $(SUDO) tee -a $(INI_FILE)
+	$(Q) echo "datadog.appsec.log_file=/tmp/logs/appsec.log" | $(SUDO) tee -a $(INI_FILE)
+	$(Q) echo "datadog.appsec.helper_log_file=/tmp/logs/helper.log" | $(SUDO) tee -a $(INI_FILE)
+
 install_all: install install_ini
 
 run_tests: $(TEST_FILES) $(TEST_STUB_FILES) $(BUILD_DIR)/run-tests.php
@@ -1040,11 +1061,11 @@ endef
 
 
 define run_benchmarks
-	$(ENV_OVERRIDE) php -d extension=redis-5.3.7.so $(TEST_EXTRA_INI) $(TRACER_SOURCES_INI) $(PHPBENCH) --config=$(1) --filter=$(FILTER) --report=all --output=file --output=console
+	$(ENV_OVERRIDE) php -d extension=redis-5.3.7.so $(TEST_EXTRA_INI) $(TRACER_SOURCES_INI) $(PHPBENCH) --config=$(1) --filter=$(FILTER) --report=all --output=file --output=console $(BENCHMARK_EXTRA)
 endef
 
 define run_benchmarks_with_ddprof
-	$(ENV_OVERRIDE) ddprof -S $(DDPROF_IDENTIFIER) php -d extension=redis-5.3.7.so $(TEST_EXTRA_INI) $(REQUEST_INIT_HOOK) $(PHPBENCH) --config=$(1) --filter=$(FILTER) --report=all --output=file --output=console
+	$(ENV_OVERRIDE) ddprof -S $(DDPROF_IDENTIFIER) php -d extension=redis-5.3.7.so $(TEST_EXTRA_INI) $(REQUEST_INIT_HOOK) $(PHPBENCH) --config=$(1) --filter=$(FILTER) --report=all --output=file --output=console $(BENCHMARK_EXTRA)
 endef
 
 define run_composer_with_lock
@@ -1115,7 +1136,7 @@ test_distributed_tracing_coverage:
 test_metrics: global_test_run_dependencies
 	$(call run_tests,--testsuite=metrics $(TESTS))
 
-benchmarks_run_dependencies: global_test_run_dependencies tests/Frameworks/Symfony/Version_5_2/composer.lock-php$(PHP_MAJOR_MINOR) tests/Frameworks/Laravel/Version_8_x/composer.lock-php$(PHP_MAJOR_MINOR) tests/Benchmarks/composer.lock-php$(PHP_MAJOR_MINOR)
+benchmarks_run_dependencies: global_test_run_dependencies tests/Frameworks/Symfony/Version_5_2/composer.lock-php$(PHP_MAJOR_MINOR) tests/Frameworks/Laravel/Version_10_x/composer.lock-php$(PHP_MAJOR_MINOR) tests/Benchmarks/composer.lock-php$(PHP_MAJOR_MINOR)
 	php tests/Frameworks/Symfony/Version_5_2/bin/console cache:clear --no-warmup --env=prod
 
 call_benchmarks:

diff --git a/benchmark/runall.sh b/benchmark/runall.sh
@@ -30,5 +30,17 @@ elif [ "$SCENARIO" = "tracer" ]; then
 
   ## Request Startup/Shutdown Benchmarks
   make benchmarks_tea
-  cp tea/benchmarks/reports/tea-bench-results.json "$ARTIFACTS_DIR/tracer-tea-bench-results.json"
+  cp tea/benchmarks/reports/tracer-tea-bench-results.json "$ARTIFACTS_DIR/tracer-tea-bench-results.json"
+elif [ "$SCENARIO" = "appsec" ]; then
+  # Run Appsec Benchmarks
+  cd ..
+  make composer_tests_update
+  make benchmarks_run_dependencies
+  make install_appsec
+
+  ## Non-OPCache Benchmarks
+  BENCHMARK_EXTRA="--group=frameworks" make call_benchmarks
+  cp tests/Benchmarks/reports/tracer-bench-results.csv "$ARTIFACTS_DIR/appsec-bench-results.csv"
+
+  make delete_ini
 fi
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -83,6 +83,7 @@ services:
   '8.3-buster': { <<: *linux_php_service, image: 'datadog/dd-trace-ci:php-8.3_buster' }
   'php-master-buster': { <<: *linux_php_service, image: 'datadog/dd-trace-ci:php-master_buster' }
   # --- Bookworm ---
+  '8.2-bookworm': { <<: *linux_php_service, image: 'datadog/dd-trace-ci:php-8.2_bookworm-3' }
   '8.3-bookworm': { <<: *linux_php_service, image: 'datadog/dd-trace-ci:php-8.3_bookworm-3' }
   # --- CentOS 6 ---
   '7.0-centos7': { <<: *linux_php_service, image: 'datadog/dd-trace-ci:php-7.0_centos-7' }

diff --git a/tests/Benchmarks/Integrations/EmptyFileBench.php b/tests/Benchmarks/Integrations/EmptyFileBench.php
@@ -6,11 +6,12 @@
 
 use DDTrace\Tests\Common\WebFrameworkTestCase;
 use DDTrace\Tests\Frameworks\Util\Request\GetSpec;
+use Benchmarks\Integrations\FrameworkBenchmarksCase;
 
-class EmptyFileBench extends WebFrameworkTestCase
+class EmptyFileBench extends FrameworkBenchmarksCase
 {
     /**
-     * @BeforeMethods("disabledTracing")
+     * @BeforeMethods("disableDatadog")
      * @AfterMethods("afterMethod")
      * @Revs(10)
      * @Iterations(10)
@@ -27,7 +28,7 @@ public function benchEmptyFileBaseline()
     }
 
     /**
-     * @BeforeMethods("enableTracing")
+     * @BeforeMethods("enableDatadog")
      * @AfterMethods("afterMethod")
      * @Revs(10)
      * @Iterations(10)
@@ -48,22 +49,8 @@ public static function getAppIndexScript()
         return __DIR__ . '/../../Frameworks/Custom/Version_Not_Autoloaded/index.php';
     }
 
-    public function disabledTracing()
-    {
-        $this->setUpWebServer([
-            'DD_TRACE_ENABLED' => 0,
-        ]);
-    }
-
     public function afterMethod()
     {
         $this->TearDownAfterClass();
     }
-
-    public function enableTracing()
-    {
-        $this->setUpWebServer([
-            'DD_TRACE_ENABLED' => 1,
-        ]);
-    }
 }
diff --git a/tests/Benchmarks/Integrations/FrameworkBenchmarksCase.php b/tests/Benchmarks/Integrations/FrameworkBenchmarksCase.php
@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Benchmarks\Integrations;
+
+use DDTrace\Tests\Common\WebFrameworkTestCase;
+use DDTrace\Tests\Frameworks\Util\Request\GetSpec;
+
+abstract class FrameworkBenchmarksCase extends WebFrameworkTestCase
+{
+    protected function getClassName()
+    {
+        $fullyQualified = get_class($this);
+        $tokens = explode('\\', $fullyQualified);
+        return end($tokens);
+    }
+
+    public function disableDatadog()
+    {
+        $name = $this->getClassName();
+        $this->setUpWebServer([
+            'DD_TRACE_ENABLED' => 0,
+            'DD_APPSEC_ENABLED' => 0,
+        ], ['error_log' => "/tmp/logs/$name.log"]);
+    }
+
+    public function enableDatadog()
+    {
+        $name = $this->getClassName();
+        $this->setUpWebServer([
+            'DD_TRACE_ENABLED' => 1,
+            'DD_APPSEC_ENABLED' => 1,
+        ], ['error_log' => "/tmp/logs/$name.log"]);
+    }
+}
diff --git a/tests/Benchmarks/Integrations/LaravelBench.php b/tests/Benchmarks/Integrations/LaravelBench.php
@@ -7,10 +7,13 @@
 use DDTrace\Tests\Common\WebFrameworkTestCase;
 use DDTrace\Tests\Frameworks\Util\Request\GetSpec;
 
-class LaravelBench extends WebFrameworkTestCase
+/**
+* @Groups({"frameworks"})
+*/
+class LaravelBench extends FrameworkBenchmarksCase
 {
     /**
-     * @BeforeMethods("disableLaravelTracing")
+     * @BeforeMethods("disableDatadog")
      * @AfterMethods("afterMethod")
      * @Revs(10)
      * @Iterations(10)
@@ -27,7 +30,7 @@ public function benchLaravelBaseline()
     }
 
     /**
-     * @BeforeMethods("enableLaravelTracing")
+     * @BeforeMethods("enableDatadog")
      * @AfterMethods("afterMethod")
      * @Revs(10)
      * @Iterations(10)
@@ -48,22 +51,8 @@ public static function getAppIndexScript()
         return __DIR__ . '/../../Frameworks/Laravel/Version_10_x/public/index.php';
     }
 
-    public function disableLaravelTracing()
-    {
-        $this->setUpWebServer([
-                'DD_TRACE_ENABLED' => 0,
-        ]);
-    }
-
     public function afterMethod()
     {
         $this->TearDownAfterClass();
     }
-
-    public function enableLaravelTracing()
-    {
-        $this->setUpWebServer([
-            'DD_TRACE_ENABLED' => 1,
-        ]);
-    }
 }
diff --git a/tests/Benchmarks/Integrations/SymfonyBench.php b/tests/Benchmarks/Integrations/SymfonyBench.php
@@ -7,10 +7,13 @@
 use DDTrace\Tests\Common\WebFrameworkTestCase;
 use DDTrace\Tests\Frameworks\Util\Request\GetSpec;
 
-class SymfonyBench extends WebFrameworkTestCase
+/**
+* @Groups({"frameworks"})
+*/
+class SymfonyBench extends FrameworkBenchmarksCase
 {
     /**
-     * @BeforeMethods("disableSymfonyTracing")
+     * @BeforeMethods("disableDatadog")
      * @AfterMethods("afterMethod")
      * @Revs(10)
      * @Iterations(10)
@@ -27,7 +30,7 @@ public function benchSymfonyBaseline()
     }
 
     /**
-     * @BeforeMethods("enableSymfonyTracing")
+     * @BeforeMethods("enableDatadog")
      * @AfterMethods("afterMethod")
      * @Revs(10)
      * @Iterations(10)
@@ -48,22 +51,8 @@ public static function getAppIndexScript()
         return __DIR__ . '/../../Frameworks/Symfony/Version_5_2/public/index.php';
     }
 
-    public function disableSymfonyTracing()
-    {
-        $this->setUpWebServer([
-            'DD_TRACE_ENABLED' => 0,
-        ]);
-    }
-
     public function afterMethod()
     {
         $this->TearDownAfterClass();
     }
-
-    public function enableSymfonyTracing()
-    {
-        $this->setUpWebServer([
-            'DD_TRACE_ENABLED' => 1,
-        ]);
-    }
 }
diff --git a/tests/Benchmarks/Integrations/WordPressBench.php b/tests/Benchmarks/Integrations/WordPressBench.php
@@ -7,10 +7,13 @@
 use DDTrace\Tests\Common\WebFrameworkTestCase;
 use DDTrace\Tests\Frameworks\Util\Request\GetSpec;
 
-class WordPressBench extends WebFrameworkTestCase
+/**
+* @Groups({"frameworks"})
+*/
+class WordPressBench extends FrameworkBenchmarksCase
 {
     /**
-     * @BeforeMethods("enableWordPressTracing")
+     * @BeforeMethods({"enableDatadog", "createDatabase"})
      * @AfterMethods("afterMethod")
      * @Revs(10)
      * @Iterations(10)
@@ -31,30 +34,20 @@ public static function getAppIndexScript()
         return __DIR__ . '/../../Frameworks/WordPress/Version_6_1/index.php';
     }
 
-    public function disableWordPressTracing()
+    public function createDatabase(): void
     {
         $pdo = new \PDO('mysql:host=mysql_integration', 'test', 'test');
         $pdo->exec('CREATE DATABASE IF NOT EXISTS wp61');
         $pdo->exec(file_get_contents(__DIR__ . '/../../Frameworks/WordPress/Version_6_1/scripts/wp_initdb.sql'));
-        $this->setUpWebServer([
-            'DD_TRACE_ENABLED' => 0,
-        ]);
     }
 
     public function afterMethod()
     {
         $this->TearDownAfterClass();
     }
 
-    public function enableWordPressTracing()
-    {
-        $this->setUpWebServer([
-            'DD_TRACE_ENABLED' => 1
-        ]);
-    }
-
     /**
-     * @BeforeMethods("disableWordPressTracing")
+     * @BeforeMethods({"disableDatadog", "createDatabase"})
      * @AfterMethods("afterMethod")
      * @Revs(10)
      * @Iterations(10)

diff --git a/tests/Benchmarks/composer.json b/tests/Benchmarks/composer.json
@@ -5,5 +5,10 @@
         "phpbench/phpbench": "^1.0",
         "phpunit/phpunit": "<10",
         "symfony/process": "<5"
+    },
+    "autoload-dev": {
+        "psr-4": {
+            "Benchmarks\\": "./"
+        }
     }
 }