Merge #4852

4852: Bench nomad exec r=fmaste a=fmaste

Exec task driver and standalone Nomad Server and Clients (no -dev mode)

- Local/testing environment
- - Bye bye `-dev` mode, now both Nomad Servers and Clients are started independently 
- - Can automate as many Nomad Clients as needed (`wb backend pass nomad server|client start|stop NAME`)
- - Optional stateful Nomad Agents instead of starting and stopping on every run
- - Only when using the Nomad `exec` Task Driver Clients are run as root
- - Create symlinks to every Task's `stdout` and `stderr` file for easier/live monitoring and debugging
- Cloud deployments
- - Add a new mode called `nomadexec` that uses SRE's patched version of Nomad
- - As part of the switch-over to directly-executable derivations, the Nomad Job specification file is self contained. Every script and config file is included as a template stanza only leaving outside the genesis folder.
- - Created an ad-hoc "container-specs.json" file with GitHub references (towards being able to benchmark any cardano-node commit from any workbench version).
- - The workbench checks that the commit being tested is publicly available on GitHub
- - Fetch all Nomad (some dynamically) generated `start.sh` and `config.json` files for log and debugging purposes
- Both local and cloud
- - Control and monitoring of Nomad Jobs was refactored in line with the important role it has
- - Removal of all locality assumptions
- - Removal of all 1-1 `allocation ID`-`task name` relation assumptions
- Others
- - Fixed how `wb backend` calls backend specific subcommands

Co-authored-by: Federico Mastellone <federico.pascual-mastellone@iohk.io>

Makefile

@@ -40,7 +40,7 @@ trace-documentation:
 ###
 ### Workbench
 ###
-workbench-ci: workbench-ci-test ci-test-auto ci-test-autonix ci-test-autonomad
+workbench-ci: workbench-ci-test ci-test-auto ci-test-autonix ci-test-autonomadpodman
 CI_TARGETS := hlint workbench-ci haddock-hoogle
 ci:  ci-report ci-targets
 ci-report:

lib.mk

@@ -27,14 +27,18 @@ endif
 endef
 
 define define_profile_targets
-$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof),            $$(prof),false, true,false,false,false, supervisor)))
-$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-prof,       $$(prof),false, true,false,false, true, supervisor)))
-$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-auto,       $$(prof),false, true, true,false,false, supervisor)))
-$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-autostay,   $$(prof),false, true, true, true,false, supervisor)))
-$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nix,        $$(prof), true,false,false,false,false, supervisor)))
-$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-autonix,    $$(prof), true,false, true,false,false, supervisor)))
-$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nomad,      $$(prof), true,false,false,false,false, nomad)))
-$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-autonomad,  $$(prof), true,false, true,false,false, nomad)))
-$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nixops,     $$(prof), true, true,false,false,false, nixops)))
-$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-autonixops, $$(prof), true, true, true, true,false, nixops)))
+$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof),                  $$(prof),false, true,false,false,false, supervisor)))
+$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-prof,             $$(prof),false, true,false,false, true, supervisor)))
+$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-auto,             $$(prof),false, true, true,false,false, supervisor)))
+$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-autostay,         $$(prof),false, true, true, true,false, supervisor)))
+$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nix,              $$(prof), true,false,false,false,false, supervisor)))
+$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-autonix,          $$(prof), true,false, true,false,false, supervisor)))
+$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nomad,            $$(prof), true,false,false,false,false, nomad)))
+$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nomad-auto,       $$(prof), true,false, true,false,false, nomad)))
+$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nomadpodman,      $$(prof), true,false,false,false,false, nomadpodman)))
+$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nomadpodman-auto, $$(prof), true,false, true,false,false, nomadpodman)))
+$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nomadexec,        $$(prof), true,false,false,false,false, nomadexec)))
+$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nomadexec-auto,   $$(prof), true,false, true,false,false, nomadexec)))
+$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-nixops,           $$(prof), true, true,false,false,false, nixops)))
+$$(foreach prof,$(1),$$(eval $$(call proftgt,$$(prof)-autonixops,       $$(prof), true, true, true, true,false, nixops)))
 endef

nix/pkgs.nix

@@ -12,17 +12,26 @@ let
   # A conveniently-parametrisable workbench preset.
   # See https://input-output-hk.github.io/haskell.nix/user-guide/development/
   # The general idea is:
-  # 1. backendName -> useCabalRun -> backend
-  # 2. stateDir -> batchName -> profileName -> backend -> workbench -> runner
+  # 1. backendName -> stateDir -> basePort -> useCabalRun -> backend
+  # 2. batchName -> profileName -> backend -> workbench -> runner
   # * `workbench` is in case a pinned version of the workbench is needed.
   workbench-runner =
-    let backendRegistry =
-        {
-            nixops     = ./workbench/backend/nixops.nix;
-            nomad      = ./workbench/backend/nomad.nix;
-            supervisor = ./workbench/backend/supervisor.nix;
-        };
-    in
+  let
+    backendRegistry =
+      {
+        nixops      = params:
+          import ./workbench/backend/nixops.nix params;
+        nomad       = params:
+          import ./workbench/backend/nomad.nix (params // {execTaskDriver=false;});
+        nomadexec   = params:
+          import ./workbench/backend/nomad.nix (params // {execTaskDriver=true; });
+        nomadpodman = params:
+          import ./workbench/backend/nomad.nix (params // {execTaskDriver=false;});
+        supervisor  = params:
+          import ./workbench/backend/supervisor.nix params;
+      }
+    ;
+  in
     { stateDir           ? customConfig.localCluster.stateDir
     , batchName          ? customConfig.localCluster.batchName
     , profileNix         ? null
@@ -40,12 +49,12 @@ let
         # The `useCabalRun` flag is set in the backend to allow the backend to
         # override its value. The runner uses the value of `useCabalRun` from
         # the backend to prevent a runner using a different value.
-        backend = import (backendRegistry."${backendName}")
+        backend = (backendRegistry."${backendName}")
                    { inherit pkgs lib stateDir basePort useCabalRun; };
     in import ./workbench/backend/runner.nix
       {
         inherit pkgs lib cardanoNodePackages;
-        inherit stateDir batchName profileName backend;
+        inherit batchName profileName backend;
         inherit cardano-node-rev;
         inherit workbench workbenchDevMode;
       };

nix/workbench/backend/backend.sh

@@ -55,7 +55,7 @@ case "${op}" in
     cleanup-cluster )            backend_$WB_BACKEND "$@";;
 
     ## Handle non-generic calls:
-    passthrough | pass )         backend_$WB_BACKEND "$@";;
+    passthrough | pass )         shift; backend_$WB_BACKEND "$@";;
 
     validate )
         local usage="USAGE: wb run $op"

nix/workbench/backend/nixops.nix

@@ -1,6 +1,8 @@
 { pkgs
 , lib
+, stateDir # Not used here, just returned
 , basePort
+## `useCabalRun` not used here unlike `supervisor.nix`.
 , ...
 }:
 with lib; with pkgs.commonLib;
@@ -265,7 +267,7 @@ let
     };
 
   materialise-profile =
-    { stateDir, profileNix }:
+    { profileNix }:
       let
       in pkgs.runCommand "workbench-backend-output-${profileNix.profileName}-nixops"
         {}
@@ -307,7 +309,7 @@ in
 {
   name = "nixops";
 
-  inherit extraShellPkgs materialise-profile overlay service-modules basePort;
+  inherit extraShellPkgs materialise-profile overlay service-modules stateDir basePort;
 
   useCabalRun = false;
 }