Set up regression benchmark for scalar performance

.gitignore

@@ -40,3 +40,5 @@ build/
 result
 result-*
 *.class
+# Exclude rust build directories
+*target/

bindings/rust/s2n-tls/Cargo.toml

@@ -15,8 +15,10 @@ unstable-ktls = ["s2n-tls-sys/unstable-ktls"]
 quic = ["s2n-tls-sys/quic"]
 fips = ["s2n-tls-sys/fips"]
 pq = ["s2n-tls-sys/pq"]
+testing = ["bytes"]
 
 [dependencies]
+bytes = { version = "1", optional = true }
 errno = { version = "0.3" }
 libc = "0.2"
 s2n-tls-sys = { version = "=0.2.8", path = "../s2n-tls-sys", features = ["internal"] }

bindings/rust/s2n-tls/src/lib.rs

@@ -27,5 +27,5 @@ pub mod security;
 
 pub use s2n_tls_sys as ffi;
 
-#[cfg(test)]
-mod testing;
+#[cfg(any(feature = "testing", test))]
+pub mod testing;

tests/regression/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "regression"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+s2n-tls = { path = "../../bindings/rust/s2n-tls", features = ["testing"] }
+bytes = { version = "1", optional = true }
+errno = { version = "0.3" }
+libc = "0.2"
+crabgrind = "0.1"
+futures-test = "0.3.30"

tests/regression/README.md

@@ -0,0 +1,77 @@
+# Regression Testing for s2n-tls
+
+This folder contains regression tests and benchmarking tools for the `s2n-tls` library. The tests focus on various aspects of TLS connections, including handshakes and session resumptions.
+
+## Contents
+
+1. **Regression Harnesses**
+   - **config_create.rs**: Creates a minimal s2n-tls configuration.
+   - **config_configure.rs**: Configures an s2n-tls config with security policies and certificate key pairs.
+
+2. **Cargo.toml**
+   - The configuration file for building and running the regression tests using Cargo.
+
+3. **run_harnesses.sh**
+   - Script to run all harnesses, a specified harness, or a combination of harnesses with Valgrind and store annotated results.
+
+
+## Prerequisites
+
+Ensure you have the following installed:
+- Rust (with Cargo)
+- Valgrind (for crabgrind instrumentation)
+
+## Running the Harnesses with Valgrind
+To run the harnesses with Valgrind and store the annotated results, use the `run_harnesses.sh` script:
+
+### Make the Script Executable
+
+Make the `run_harnesses.sh` script executable:
+
+```
+chmod +x run_harnesses.sh
+```
+
+### Run All Harnesses
+
+To run all harnesses, execute the script without any arguments:
+
+```
+./run_harnesses.sh
+```
+
+### Run a Specific Harness
+
+To run a specific harness, provide the harness name as an argument:
+
+```
+./run_harnesses.sh config_create
+```
+
+### Run Multiple Specified Harnesses
+
+To run multiple specified harnesses, provide the harness names as arguments:
+
+```
+./run_harnesses.sh config_create config_configure
+```
+
+The script will build the harnesses, run each specified harness with Valgrind, store the unformatted output in the root directory and store the annotated output in the `perf_outputs` folder.
+
+## Test Details
+
+### config_create.rs
+
+Creates a minimal s2n-tls configuration and ensures it can be built successfully.
+
+### config_configure.rs
+
+Configures an s2n-tls configuration with a specified security policy and loads a certificate key pair. Ensures the configuration is valid and can be built.
+
+## Contributing to s2n-tls
+
+If you are interested in contributing to s2n-tls, please see our [development guide](https://github.com/aws/s2n-tls/blob/main/docs/DEVELOPMENT-GUIDE.md).
+
+## License
+
+This project is licensed under the Apache License, Version 2.0. See the [LICENSE](LICENSE) file for details.

tests/regression/run_harnesses.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Function to run a harness
+run_harness() {
+    local harness=$1
+    local output_file="cachegrind.out.${harness}"
+    local annotated_output="perf_outputs/${harness}_annotated.txt"
+
+    echo "Running harness: $harness"
+    cargo build
+    valgrind --tool=cachegrind --cachegrind-out-file=$output_file target/debug/$harness
+    cg_annotate $output_file > $annotated_output
+
+    echo "Annotated output saved to: $annotated_output"
+}
+
+# Create the perf_outputs directory if it doesn't exist
+mkdir -p perf_outputs
+
+# Check if any harness is specified
+if [ $# -eq 0 ]; then
+    echo "No harness specified. Running all harnesses..."
+    harnesses=(config_create config_configure)
+else
+    # Use the specified harnesses
+    harnesses=("$@")
+fi
+
+# Run each specified harness
+for harness in "${harnesses[@]}"; do
+    run_harness $harness
+done
+

tests/regression/src/bin/config_configure.rs

@@ -0,0 +1,29 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Harness to build a configured s2n-tls config object.
+//!
+//! This harness only measures the cost of setting the security policy and host callback verification.
+//! Loading and trusting certs is typically also included in this step but for a more fine-grain
+//! performance analysis, it is left out so cert creation can be measured in its own harness
+//!
+
+use crabgrind as cg;
+use s2n_tls::security;
+use regression::{create_empty_config, configure_config};
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    cg::cachegrind::stop_instrumentation();
+
+    let builder = create_empty_config()?;
+
+    cg::cachegrind::start_instrumentation();
+
+    let builder = configure_config(builder, &security::DEFAULT_TLS13)?;
+
+    let _config = builder.build().expect("Failed to build config");
+
+    cg::cachegrind::stop_instrumentation();
+
+    Ok(())
+}

tests/regression/src/bin/config_create.rs

@@ -0,0 +1,24 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Harness to build an empty s2n-tls config object.
+//!
+//! Empty config creation is implemented seperate from a configured, usable config object.
+//! This is to measure the performance of each component seperately.
+//!
+
+use crabgrind as cg;
+use regression::create_empty_config;
+use s2n_tls::config::Builder;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+
+    let builder: Builder = create_empty_config()?;
+
+    builder.build().map_err(|e| Box::new(e) as Box<dyn std::error::Error>)?;
+
+    cg::cachegrind::stop_instrumentation();
+
+    Ok(())
+}
+

tests/regression/src/lib.rs

@@ -0,0 +1,33 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use s2n_tls::{callbacks::VerifyHostNameCallback, config::Builder, security};
+type Error = Box<dyn std::error::Error>;
+
+//Initializes an empty config object without paramter setting
+pub fn create_empty_config() -> Result<s2n_tls::config::Builder, Error> {
+    Ok(Builder::new())
+}
+
+pub struct InsecureAcceptAllCertificatesHandler {}
+
+impl VerifyHostNameCallback for InsecureAcceptAllCertificatesHandler {
+    fn verify_host_name(&self, _host_name: &str) -> bool {
+        true
+    }
+}
+
+
+// Configure the security policy and host call back verification for an s2n_tls config
+pub fn configure_config(
+    mut builder: s2n_tls::config::Builder,
+    cipher_prefs: &security::Policy
+) -> Result<s2n_tls::config::Builder, Error> {
+    builder
+        .set_security_policy(cipher_prefs)
+        .expect("Unable to set config cipher preferences");
+    builder
+        .set_verify_host_callback(InsecureAcceptAllCertificatesHandler {})
+        .expect("Unable to set a host verify callback.");
+    Ok(builder)
+}