Expand ${var} in benchcomp variant env

docs/src/benchcomp-conf.md

@@ -4,6 +4,26 @@
 This page lists the different visualizations that are available.
 
 
+## Variants
+
+A *variant* is a single invocation of a benchmark suite. Benchcomp runs several
+variants, so that their performance can be compared later. A variant consists of
+a command-line argument, working directory, and environment. Benchcomp invokes
+the command using the operating system environment, updated with the keys and
+values in `env`. If any values in `env` contain strings of the form `${var}`,
+Benchcomp expands them to the value of the environment variable `$var`.
+
+```yaml
+variants:
+    variant_1:
+        config:
+            command_line: echo "Hello, world"
+            directory: /tmp
+            env:
+              PATH: /my/local/directory:${PATH}
+```
+
+
 ## Built-in visualizations
 
 The following visualizations are available; these can be added to the `visualize` list of `benchcomp.yaml`.

tools/benchcomp/benchcomp/entry/run.py

@@ -13,6 +13,7 @@
 import logging
 import os
 import pathlib
+import re
 import shutil
 import subprocess
 import typing
@@ -53,9 +54,10 @@ def __post_init__(self):
         else:
             self.working_copy = pathlib.Path(self.directory)
 
+
     def __call__(self):
-        env = dict(os.environ)
-        env.update(self.env)
+        update_environment_with = _EnvironmentUpdater()
+        env = update_environment_with(self.env)
 
         if self.copy_benchmarks_dir:
             shutil.copytree(
@@ -128,6 +130,44 @@ def __call__(self):
         tmp_symlink.rename(self.out_symlink)
 
 
+
+@dataclasses.dataclass
+class _EnvironmentUpdater:
+    """Update the OS environment with keys and values containing variables
+
+    When called, this class returns the operating environment updated with new
+    keys and values. The values can contain variables of the form '${var_name}'.
+    The class evaluates those variables using values already in the environment.
+    """
+
+    os_environment: dict = dataclasses.field(
+        default_factory=lambda : dict(os.environ))
+    pattern: re.Pattern = re.compile(r"\$\{(\w+?)\}")
+
+
+    def _evaluate(self, key, value):
+        """Evaluate all ${var} in value using self.os_environment"""
+        old_value = value
+
+        for variable in re.findall(self.pattern, value):
+            if variable not in self.os_environment:
+                logging.error(
+                    "Couldn't evaluate ${%s} in the value '%s' for environment "
+                    "variable '%s'. Ensure the environment variable $%s is set",
+                    variable, old_value, key, variable)
+                sys.exit(1)
+            value = re.sub(
+                r"\$\{" + variable + "\}", self.os_environment[variable], value)
+        return value
+
+
+    def __call__(self, new_environment):
+        ret = dict(self.os_environment)
+        for key, value in new_environment.items():
+            ret[key] = self._evaluate(key, value)
+        return ret
+
+
 def get_default_out_symlink():
     return "latest"
 

tools/benchcomp/test/test_regression.py

@@ -662,6 +662,43 @@ def test_return_0_on_fail(self):
                 result = yaml.safe_load(handle)
 
 
+    def test_env_expansion(self):
+        """Ensure that config parser expands '${}' in env key"""
+
+        with tempfile.TemporaryDirectory() as tmp:
+            run_bc = Benchcomp({
+                "variants": {
+                    "env_set": {
+                        "config": {
+                            "command_line": 'echo "$__BENCHCOMP_ENV_VAR" > out',
+                            "directory": tmp,
+                            "env": {"__BENCHCOMP_ENV_VAR": "foo:${PATH}"}
+                        }
+                    },
+                },
+                "run": {
+                    "suites": {
+                        "suite_1": {
+                            "parser": {
+                                # The word 'bin' typically appears in $PATH, so
+                                # check that what was echoed contains 'bin'.
+                                "command": textwrap.dedent("""\
+                                    grep bin out && grep '^foo:' out && echo '{
+                                        "benchmarks": {},
+                                        "metrics": {}
+                                    }'
+                                    """)
+                            },
+                            "variants": ["env_set"]
+                        }
+                    }
+                },
+                "visualize": [],
+            })
+            run_bc()
+            self.assertEqual(run_bc.proc.returncode, 0, msg=run_bc.stderr)
+
+
     def test_env(self):
         """Ensure that benchcomp reads the 'env' key of variant config"""
 

tools/benchcomp/test/test_unit.py

@@ -0,0 +1,66 @@
+# Copyright Kani Contributors
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+# Benchcomp regression testing suite. This suite uses Python's stdlib unittest
+# module, but nevertheless actually runs the binary rather than running unit
+# tests.
+
+import unittest
+import uuid
+
+import benchcomp.entry.run
+
+
+
+class TestEnvironmentUpdater(unittest.TestCase):
+    def test_environment_construction(self):
+        """Test that the default constructor reads the OS environment"""
+
+        update_environment = benchcomp.entry.run._EnvironmentUpdater()
+        environment = update_environment({})
+        self.assertIn("PATH", environment)
+
+
+    def test_placeholder_construction(self):
+        """Test that the placeholder constructor reads the placeholder"""
+
+        key, value = [str(uuid.uuid4()) for _ in range(2)]
+        update_environment = benchcomp.entry.run._EnvironmentUpdater({
+            key: value,
+        })
+        environment = update_environment({})
+        self.assertIn(key, environment)
+        self.assertEqual(environment[key], value)
+
+
+    def test_environment_update(self):
+        """Test that the environment is updated"""
+
+        key, value, update = [str(uuid.uuid4()) for _ in range(3)]
+        update_environment = benchcomp.entry.run._EnvironmentUpdater({
+            key: value,
+        })
+        environment = update_environment({
+            key: update
+        })
+        self.assertIn(key, environment)
+        self.assertEqual(environment[key], update)
+
+
+    def test_environment_update_variable(self):
+        """Test that the environment is updated"""
+
+        old_env = {
+            "key1": str(uuid.uuid4()),
+            "key2": str(uuid.uuid4()),
+        }
+
+        actual_update = "${key2}xxx${key1}"
+        expected_update = f"{old_env['key2']}xxx{old_env['key1']}"
+
+        update_environment = benchcomp.entry.run._EnvironmentUpdater(old_env)
+        environment = update_environment({
+            "key1": actual_update,
+        })
+        self.assertIn("key1", environment)
+        self.assertEqual(environment["key1"], expected_update)