read configs from stdin on '-' (#407) (#420)

* read configs from stdin on '-'
* Reuse k8s instance across feature tests (#392)

Co-authored-by: Benjamin Schimke <benjamin.schimke@canonical.com>

docs/src/_parts/commands/k8s_bootstrap.md

@@ -14,7 +14,7 @@ k8s bootstrap [flags]
 
 ```
       --address string         microcluster address, defaults to the node IP address
-      --file string            path to the YAML file containing your custom cluster bootstrap configuration.
+      --file string            path to the YAML file containing your custom cluster bootstrap configuration. Use '-' to read from stdin.
   -h, --help                   help for bootstrap
       --interactive            interactively configure the most important cluster options
       --name string            node name, defaults to hostname

docs/src/_parts/commands/k8s_join-cluster.md

@@ -10,7 +10,7 @@ k8s join-cluster <join-token> [flags]
 
 ```
       --address string         microcluster address, defaults to the node IP address
-      --file string            path to the YAML file containing your custom cluster join configuration
+      --file string            path to the YAML file containing your custom cluster join configuration. Use '-' to read from stdin.
   -h, --help                   help for join-cluster
       --name string            node name, defaults to hostname
       --output-format string   set the output format to one of plain, json or yaml (default "plain")

src/k8s/cmd/k8s/k8s_bootstrap.go

@@ -88,7 +88,7 @@ func newBootstrapCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
 			case opts.interactive:
 				bootstrapConfig = getConfigInteractively(env.Stdin, env.Stdout, env.Stderr)
 			case opts.configFile != "":
-				bootstrapConfig, err = getConfigFromYaml(opts.configFile)
+				bootstrapConfig, err = getConfigFromYaml(env, opts.configFile)
 				if err != nil {
 					cmd.PrintErrf("Error: Failed to read bootstrap configuration from %q.\n\nThe error was: %v\n", opts.configFile, err)
 					env.Exit(1)
@@ -134,18 +134,28 @@ func newBootstrapCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
 	}
 
 	cmd.Flags().BoolVar(&opts.interactive, "interactive", false, "interactively configure the most important cluster options")
-	cmd.Flags().StringVar(&opts.configFile, "file", "", "path to the YAML file containing your custom cluster bootstrap configuration.")
+	cmd.Flags().StringVar(&opts.configFile, "file", "", "path to the YAML file containing your custom cluster bootstrap configuration. Use '-' to read from stdin.")
 	cmd.Flags().StringVar(&opts.name, "name", "", "node name, defaults to hostname")
 	cmd.Flags().StringVar(&opts.address, "address", "", "microcluster address, defaults to the node IP address")
 	cmd.Flags().StringVar(&opts.outputFormat, "output-format", "plain", "set the output format to one of plain, json or yaml")
 
 	return cmd
 }
 
-func getConfigFromYaml(filePath string) (apiv1.BootstrapConfig, error) {
-	b, err := os.ReadFile(filePath)
-	if err != nil {
-		return apiv1.BootstrapConfig{}, fmt.Errorf("failed to read file: %w", err)
+func getConfigFromYaml(env cmdutil.ExecutionEnvironment, filePath string) (apiv1.BootstrapConfig, error) {
+	var b []byte
+	var err error
+
+	if filePath == "-" {
+		b, err = io.ReadAll(env.Stdin)
+		if err != nil {
+			return apiv1.BootstrapConfig{}, fmt.Errorf("failed to read config from stdin: %w", err)
+		}
+	} else {
+		b, err = os.ReadFile(filePath)
+		if err != nil {
+			return apiv1.BootstrapConfig{}, fmt.Errorf("failed to read file: %w", err)
+		}
 	}
 
 	var config apiv1.BootstrapConfig

src/k8s/cmd/k8s/k8s_bootstrap_test.go

@@ -1,12 +1,15 @@
 package k8s
 
 import (
+	"bytes"
 	_ "embed"
-	"github.com/canonical/k8s/pkg/utils"
 	"os"
 	"path/filepath"
 	"testing"
 
+	cmdutil "github.com/canonical/k8s/cmd/util"
+	"github.com/canonical/k8s/pkg/utils"
+
 	apiv1 "github.com/canonical/k8s/api/v1"
 	. "github.com/onsi/gomega"
 )
@@ -112,7 +115,7 @@ func TestGetConfigYaml(t *testing.T) {
 			mustAddConfigToTestDir(t, configPath, tc.yamlConfig)
 
 			// Get the config from the test directory
-			bootstrapConfig, err := getConfigFromYaml(configPath)
+			bootstrapConfig, err := getConfigFromYaml(cmdutil.DefaultExecutionEnvironment(), configPath)
 
 			if tc.expectedError != "" {
 				g.Expect(err).To(HaveOccurred())
@@ -124,3 +127,20 @@ func TestGetConfigYaml(t *testing.T) {
 		})
 	}
 }
+
+func TestGetConfigFromYaml_Stdin(t *testing.T) {
+	g := NewWithT(t)
+
+	input := `secure-port: 5000`
+
+	// Redirect stdin to the mock input
+	env := cmdutil.DefaultExecutionEnvironment()
+	env.Stdin = bytes.NewBufferString(input)
+
+	// Call the getConfigFromYaml function with "-" as filePath
+	config, err := getConfigFromYaml(env, "-")
+	g.Expect(err).ToNot(HaveOccurred())
+
+	expectedConfig := apiv1.BootstrapConfig{SecurePort: utils.Pointer(5000)}
+	g.Expect(config).To(Equal(expectedConfig))
+}

src/k8s/cmd/k8s/k8s_join_cluster.go

@@ -2,6 +2,7 @@ package k8s
 
 import (
 	"fmt"
+	"io"
 	"os"
 
 	apiv1 "github.com/canonical/k8s/api/v1"
@@ -66,11 +67,23 @@ func newJoinClusterCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
 
 			var joinClusterConfig string
 			if opts.configFile != "" {
-				b, err := os.ReadFile(opts.configFile)
-				if err != nil {
-					cmd.PrintErrf("Error: Failed to read join configuration from %q.\n\nThe error was: %v\n", opts.configFile, err)
-					env.Exit(1)
-					return
+				var b []byte
+				var err error
+
+				if opts.configFile == "-" {
+					b, err = io.ReadAll(os.Stdin)
+					if err != nil {
+						cmd.PrintErrf("Error: Failed to read join configuration from stdin. \n\nThe error was: %v\n", err)
+						env.Exit(1)
+						return
+					}
+				} else {
+					b, err = os.ReadFile(opts.configFile)
+					if err != nil {
+						cmd.PrintErrf("Error: Failed to read join configuration from %q.\n\nThe error was: %v\n", opts.configFile, err)
+						env.Exit(1)
+						return
+					}
 				}
 				joinClusterConfig = string(b)
 			}
@@ -87,7 +100,7 @@ func newJoinClusterCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
 	}
 	cmd.Flags().StringVar(&opts.name, "name", "", "node name, defaults to hostname")
 	cmd.Flags().StringVar(&opts.address, "address", "", "microcluster address, defaults to the node IP address")
-	cmd.Flags().StringVar(&opts.configFile, "file", "", "path to the YAML file containing your custom cluster join configuration")
+	cmd.Flags().StringVar(&opts.configFile, "file", "", "path to the YAML file containing your custom cluster join configuration. Use '-' to read from stdin.")
 	cmd.Flags().StringVar(&opts.outputFormat, "output-format", "plain", "set the output format to one of plain, json or yaml")
 	return cmd
 }

tests/integration/templates/bootstrap-all.yaml

@@ -0,0 +1,15 @@
+cluster-config:
+  network:
+    enabled: true
+  dns:
+    enabled: true
+  ingress:
+    enabled: true
+  load-balancer:
+    enabled: true
+  local-storage:
+    enabled: true
+  gateway:
+    enabled: true
+  metrics-server:
+    enabled: true

tests/integration/tests/conftest.py

@@ -86,7 +86,7 @@ def instances(
     snap_path = (tmp_path / "k8s.snap").as_posix()
 
     LOG.info(f"Creating {node_count} instances")
-    instances: List[util.Instance] = []
+    instances: List[harness.Instance] = []
 
     for _ in range(node_count):
         # Create <node_count> instances and setup the k8s snap in each.
@@ -100,7 +100,43 @@ def instances(
 
     yield instances
 
-    _harness_clean(h)
+    if config.SKIP_CLEANUP:
+        LOG.warning("Skipping clean-up of instances, delete them on your own")
+        return
+
+    # Cleanup after each test.
+    # We cannot execute _harness_clean() here as this would also
+    # remove the session_instance. The harness ensures that everything is cleaned up
+    # at the end of the test session.
+    for instance in instances:
+        h.delete_instance(instance.id)
+
+
+@pytest.fixture(scope="session")
+def session_instance(
+    h: harness.Harness, tmp_path_factory: pytest.TempPathFactory
+) -> Generator[harness.Instance, None, None]:
+    """Constructs and bootstraps an instance that persists over a test session.
+
+    Bootstraps the instance with all k8sd features enabled to reduce testing time.
+    """
+    LOG.info("Setup node and enable all features")
+
+    snap_path = str(tmp_path_factory.mktemp("data") / "k8s.snap")
+    instance = h.new_instance()
+    util.setup_k8s_snap(instance, snap_path)
+
+    bootstrap_config_path = "/home/ubuntu/bootstrap-all-features.yaml"
+    instance.send_file(
+        (config.MANIFESTS_DIR / "bootstrap-all.yaml").as_posix(), bootstrap_config_path
+    )
+
+    instance.exec(["k8s", "bootstrap", "--file", bootstrap_config_path])
+    util.wait_until_k8s_ready(instance, [instance])
+    util.wait_for_network(instance)
+    util.wait_for_dns(instance)
+
+    yield instance
 
 
 @pytest.fixture(scope="function")
@@ -122,5 +158,3 @@ def etcd_cluster(
     cluster = EtcdCluster(h, initial_node_count=etcd_count)
 
     yield cluster
-
-    _harness_clean(h)

tests/integration/tests/test_dns.py

@@ -2,19 +2,14 @@
 # Copyright 2024 Canonical, Ltd.
 #
 import logging
-from typing import List
 
 from test_util import harness, util
 
 LOG = logging.getLogger(__name__)
 
 
-def test_dns(instances: List[harness.Instance]):
-    instance = instances[0]
-    util.wait_for_dns(instance)
-    util.wait_for_network(instance)
-
-    instance.exec(
+def test_dns(session_instance: harness.Instance):
+    session_instance.exec(
         [
             "k8s",
             "kubectl",
@@ -28,7 +23,7 @@ def test_dns(instances: List[harness.Instance]):
         ],
     )
 
-    util.stubbornly(retries=3, delay_s=1).on(instance).exec(
+    util.stubbornly(retries=3, delay_s=1).on(session_instance).exec(
         [
             "k8s",
             "kubectl",
@@ -42,14 +37,14 @@ def test_dns(instances: List[harness.Instance]):
         ]
     )
 
-    result = instance.exec(
+    result = session_instance.exec(
         ["k8s", "kubectl", "exec", "busybox", "--", "nslookup", "kubernetes.default"],
         capture_output=True,
     )
 
     assert "10.152.183.1 kubernetes.default.svc.cluster.local" in result.stdout.decode()
 
-    result = instance.exec(
+    result = session_instance.exec(
         ["k8s", "kubectl", "exec", "busybox", "--", "nslookup", "canonical.com"],
         capture_output=True,
         check=False,

tests/integration/tests/test_gateway.py

@@ -3,32 +3,27 @@
 #
 import logging
 from pathlib import Path
-from typing import List
 
 from test_util import harness, util
 from test_util.config import MANIFESTS_DIR
 
 LOG = logging.getLogger(__name__)
 
 
-def test_gateway(instances: List[harness.Instance]):
-    instance = instances[0]
-    util.wait_for_network(instance)
-    util.wait_for_dns(instance)
-
+def test_gateway(session_instance: harness.Instance):
     manifest = MANIFESTS_DIR / "gateway-test.yaml"
-    instance.exec(
+    session_instance.exec(
         ["k8s", "kubectl", "apply", "-f", "-"],
         input=Path(manifest).read_bytes(),
     )
 
     LOG.info("Waiting for nginx pod to show up...")
-    util.stubbornly(retries=5, delay_s=10).on(instance).until(
+    util.stubbornly(retries=5, delay_s=10).on(session_instance).until(
         lambda p: "my-nginx" in p.stdout.decode()
     ).exec(["k8s", "kubectl", "get", "pod", "-o", "json"])
     LOG.info("Nginx pod showed up.")
 
-    util.stubbornly(retries=3, delay_s=1).on(instance).exec(
+    util.stubbornly(retries=3, delay_s=1).on(session_instance).exec(
         [
             "k8s",
             "kubectl",
@@ -42,11 +37,11 @@ def test_gateway(instances: List[harness.Instance]):
         ]
     )
 
-    util.stubbornly(retries=5, delay_s=2).on(instance).until(
+    util.stubbornly(retries=5, delay_s=2).on(session_instance).until(
         lambda p: "cilium-gateway-my-gateway" in p.stdout.decode()
     ).exec(["k8s", "kubectl", "get", "service", "-o", "json"])
 
-    p = instance.exec(
+    p = session_instance.exec(
         [
             "k8s",
             "kubectl",
@@ -59,6 +54,6 @@ def test_gateway(instances: List[harness.Instance]):
     )
     gateway_http_port = p.stdout.decode().replace("'", "")
 
-    util.stubbornly(retries=5, delay_s=5).on(instance).until(
+    util.stubbornly(retries=5, delay_s=5).on(session_instance).until(
         lambda p: "Welcome to nginx!" in p.stdout.decode()
     ).exec(["curl", f"localhost:{gateway_http_port}"])

tests/integration/tests/test_ingress.py

@@ -11,18 +11,13 @@
 LOG = logging.getLogger(__name__)
 
 
-def test_ingress(instances: List[harness.Instance]):
-    instance = instances[0]
-    util.wait_for_network(instance)
-    util.wait_for_dns(instance)
+def test_ingress(session_instance: List[harness.Instance]):
 
-    instance.exec(["k8s", "enable", "ingress"])
-
-    util.stubbornly(retries=5, delay_s=2).on(instance).until(
+    util.stubbornly(retries=5, delay_s=2).on(session_instance).until(
         lambda p: "cilium-ingress" in p.stdout.decode()
     ).exec(["k8s", "kubectl", "get", "service", "-n", "kube-system", "-o", "json"])
 
-    p = instance.exec(
+    p = session_instance.exec(
         [
             "k8s",
             "kubectl",
@@ -38,18 +33,18 @@ def test_ingress(instances: List[harness.Instance]):
     ingress_http_port = p.stdout.decode().replace("'", "")
 
     manifest = MANIFESTS_DIR / "ingress-test.yaml"
-    instance.exec(
+    session_instance.exec(
         ["k8s", "kubectl", "apply", "-f", "-"],
         input=Path(manifest).read_bytes(),
     )
 
     LOG.info("Waiting for nginx pod to show up...")
-    util.stubbornly(retries=5, delay_s=10).on(instance).until(
+    util.stubbornly(retries=5, delay_s=10).on(session_instance).until(
         lambda p: "my-nginx" in p.stdout.decode()
     ).exec(["k8s", "kubectl", "get", "pod", "-o", "json"])
     LOG.info("Nginx pod showed up.")
 
-    util.stubbornly(retries=3, delay_s=1).on(instance).exec(
+    util.stubbornly(retries=3, delay_s=1).on(session_instance).exec(
         [
             "k8s",
             "kubectl",
@@ -63,6 +58,6 @@ def test_ingress(instances: List[harness.Instance]):
         ]
     )
 
-    util.stubbornly(retries=5, delay_s=5).on(instance).until(
+    util.stubbornly(retries=5, delay_s=5).on(session_instance).until(
         lambda p: "Welcome to nginx!" in p.stdout.decode()
     ).exec(["curl", f"localhost:{ingress_http_port}", "-H", "Host: foo.bar.com"])