add k8s runner config loading from envvars

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/ijt/goparsify v0.0.0-20221203142333-3a5276334b8d
 	github.com/imdario/mergo v0.3.16
 	github.com/joho/godotenv v1.5.1
+	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/klauspost/compress v1.16.7
 	github.com/klauspost/pgzip v1.2.5
 	github.com/korovkin/limiter v0.0.0-20221015170604-22eb1ceceddc

go.sum

@@ -366,6 +366,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
 github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=

pkg/container/kubernetes_runner.go

@@ -33,6 +33,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/imdario/mergo"
+	"github.com/kelseyhightower/envconfig"
 	"go.opentelemetry.io/otel"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -69,8 +70,13 @@ type k8s struct {
 }
 
 func KubernetesRunner(_ context.Context, logger log.Logger) (Runner, error) {
+	cfg, err := NewKubernetesConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure kubernetes runner: %v", err)
+	}
+
 	runner := &k8s{
-		Config: NewKubernetesConfig(),
+		Config: cfg,
 		logger: logger,
 	}
 
@@ -118,7 +124,7 @@ func (k *k8s) StartPod(ctx context.Context, cfg *Config) error {
 	}
 	k.logger.Infof("created builder pod '%s' with UID '%s'", pod.Name, pod.UID)
 
-	if err := wait.PollUntilContextTimeout(ctx, 10*time.Second, k.Config.StartTimeout, true, func(ctx context.Context) (done bool, err error) {
+	if err := wait.PollUntilContextTimeout(ctx, 10*time.Second, k.Config.StartTimeout.Duration, true, func(ctx context.Context) (done bool, err error) {
 		p, err := podclient.Get(ctx, pod.Name, metav1.GetOptions{})
 		if err != nil {
 			return false, err
@@ -437,9 +443,9 @@ type KubernetesRunnerConfig struct {
 	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
 	Labels      map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
 
-	StartTimeout time.Duration `json:"startTimeout" yaml:"startTimeout"`
-	BuildTimeout time.Duration `json:"buildTimeout" yaml:"buildTimeout"`
+	StartTimeout metav1.Duration `json:"startTimeout" yaml:"startTimeout" split_words:"true"`
 
+	// This field and everything below it is ignored by the environment variable parser
 	PodTemplate *KubernetesRunnerConfigPodTemplate `json:"podTemplate,omitempty" yaml:"podTemplate,omitempty" ignored:"true"`
 
 	// A "burstable" QOS is really the only thing that makes sense for ephemeral builder pods
@@ -460,13 +466,12 @@ type KubernetesRunnerConfigPodTemplate struct {
 }
 
 // NewKubernetesConfig returns a default Kubernetes runner config setup
-func NewKubernetesConfig(opt ...KubernetesRunnerConfigOptions) *KubernetesRunnerConfig {
+func NewKubernetesConfig(opt ...KubernetesRunnerConfigOptions) (*KubernetesRunnerConfig, error) {
 	cfg := &KubernetesRunnerConfig{
 		Provider:     "generic",
 		Namespace:    "default",
 		Repo:         "ttl.sh/melange",
-		StartTimeout: 5 * time.Minute,
-		BuildTimeout: 30 * time.Minute,
+		StartTimeout: metav1.Duration{Duration: 10 * time.Minute},
 		Resources: corev1.ResourceList{
 			corev1.ResourceCPU:    resource.MustParse("2"),
 			corev1.ResourceMemory: resource.MustParse("4Gi"),
@@ -479,13 +484,32 @@ func NewKubernetesConfig(opt ...KubernetesRunnerConfigOptions) *KubernetesRunner
 		o(cfg)
 	}
 
-	// We don't care about errors here, the empty value is safe to "merge"
+	// Override the defaults with values obtained from the global config file
 	global := &KubernetesRunnerConfig{}
-	data, _ := os.ReadFile(cfg.baseConfigFile)
-	_ = yaml.Unmarshal(data, global)
-	_ = mergo.Merge(cfg, global, mergo.WithOverride)
+	data, err := os.ReadFile(cfg.baseConfigFile)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("error reading config file %s: %w", cfg.baseConfigFile, err)
+	} else {
+		if err := yaml.Unmarshal(data, global); err != nil {
+			return nil, fmt.Errorf("error parsing config file %s: %w", cfg.baseConfigFile, err)
+		}
+	}
+
+	if err := mergo.Merge(cfg, global, mergo.WithOverride); err != nil {
+		return nil, fmt.Errorf("error merging config file values %s with defaults: %w", cfg.baseConfigFile, err)
+	}
+
+	// Finally, override with the values from the environment
+	var envcfg KubernetesRunnerConfig
+	if err := envconfig.Process("melange", &envcfg); err != nil {
+		return nil, fmt.Errorf("error parsing environment variables: %w", err)
+	}
+
+	if err := mergo.Merge(cfg, envcfg, mergo.WithOverride); err != nil {
+		return nil, fmt.Errorf("error merging environment variables with defaults: %w", err)
+	}
 
-	return cfg
+	return cfg, nil
 }
 
 // escapeRFC1123 escapes a string to be RFC1123 compliant.  We don't worry about

pkg/container/kubernetes_runner_test.go

@@ -5,9 +5,13 @@ import (
 	"fmt"
 	"os"
 	"testing"
+	"time"
 
 	"chainguard.dev/apko/pkg/build/types"
 	"chainguard.dev/apko/pkg/log"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/imdario/mergo"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -18,6 +22,90 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+func TestKubernetesRunnerConfig(t *testing.T) {
+	t.Parallel()
+
+	dwant, _ := NewKubernetesConfig()
+
+	// Intentionally use raw yaml to surface any type marshaling issues
+	writeRaw := func(data []byte) string {
+		f, err := os.CreateTemp("", "config.yaml")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer f.Close()
+
+		if _, err := f.Write(data); err != nil {
+			t.Fatal(err)
+		}
+
+		return f.Name()
+	}
+
+	tests := []struct {
+		name     string
+		rawInput string
+		envs     map[string]string
+		want     *KubernetesRunnerConfig
+	}{
+		{
+			name: "should have default values",
+			want: dwant,
+		},
+		{
+			name: "should override default values with global yaml values",
+			rawInput: `
+namespace: foo
+startTimeout: 10s
+`,
+			want: &KubernetesRunnerConfig{
+				Namespace:    "foo",
+				StartTimeout: metav1.Duration{Duration: 10 * time.Second},
+			},
+		},
+		{
+			name: "should override values with global yaml values and env vars",
+			rawInput: `
+namespace: foo
+startTimeout: 5m
+repo: somewhere
+`,
+			envs: map[string]string{
+				"MELANGE_REPO": "nowhere",
+			},
+			want: &KubernetesRunnerConfig{
+				Namespace:    "foo",
+				StartTimeout: metav1.Duration{Duration: 5 * time.Minute},
+				Repo:         "nowhere",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Pickup the default want values
+			want, _ := NewKubernetesConfig()
+			if err := mergo.Merge(want, tt.want, mergo.WithOverride); err != nil {
+				t.Fatal(err)
+			}
+
+			for k, v := range tt.envs {
+				if err := os.Setenv(k, v); err != nil {
+					t.Fatalf("setting env var %s=%s: %v", k, v, err)
+				}
+			}
+			got, err := NewKubernetesConfig(WithKubernetesRunnerConfigBaseConfigFile(writeRaw([]byte(tt.rawInput))))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if diff := cmp.Diff(got, want, cmpopts.IgnoreUnexported(KubernetesRunnerConfig{})); diff != "" {
+				t.Errorf("KubernetesConfig mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
 func Test_k8s_StartPod(t *testing.T) {
 	t.Parallel()
 
@@ -26,6 +114,7 @@ func Test_k8s_StartPod(t *testing.T) {
 	tests := []struct {
 		name   string
 		pkgCfg *Config
+		envs   map[string]string
 		k8sCfg *KubernetesRunnerConfig
 		wanter func(got corev1.Pod) bool
 	}{
@@ -38,13 +127,39 @@ func Test_k8s_StartPod(t *testing.T) {
 			},
 		},
 		{
-			name:   "should use global namespace instead of default",
+			name:   "should load global configs from yaml",
 			pkgCfg: &Config{PackageName: "donkey", Arch: types.Architecture("amd64")},
 			k8sCfg: &KubernetesRunnerConfig{Namespace: "not-default"},
 			wanter: func(got corev1.Pod) bool {
 				return got.Namespace == "not-default"
 			},
 		},
+		{
+			name:   "should prioritize environment configs",
+			pkgCfg: &Config{PackageName: "donkey", Arch: types.Architecture("amd64")},
+			k8sCfg: &KubernetesRunnerConfig{Namespace: "not-default"},
+			envs: map[string]string{
+				"MELANGE_NAMESPACE": "from-env",
+				"MELANGE_REPO":      "nowhere",
+			},
+			wanter: func(got corev1.Pod) bool {
+				return got.Namespace == "from-env"
+			},
+		},
+		{
+			name:   "should skip environment configs for certain fields",
+			pkgCfg: &Config{PackageName: "donkey", Arch: types.Architecture("amd64")},
+			k8sCfg: &KubernetesRunnerConfig{PodTemplate: &KubernetesRunnerConfigPodTemplate{ServiceAccountName: "foo"}},
+			envs: map[string]string{
+				"MELANGE_SERVICE_ACCOUNT_NAME": "bar",
+				"MELANGE_SERVICEACCOUNTNAME":   "bar",
+				"SERVICE_ACCOUNT_NAME":         "bar",
+				"SERVICEACCOUNTNAME":           "bar",
+			},
+			wanter: func(got corev1.Pod) bool {
+				return got.Spec.ServiceAccountName == "foo"
+			},
+		},
 		{
 			name:   "should support additional labels",
 			pkgCfg: &Config{PackageName: "donkey", Arch: types.Architecture("amd64")},
@@ -165,13 +280,21 @@ func Test_k8s_StartPod(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			wantCfg := NewKubernetesConfig(WithKubernetesRunnerConfigBaseConfigFile(writeYamlToTemp(t, tt.k8sCfg)))
+			for k, v := range tt.envs {
+				if err := os.Setenv(k, v); err != nil {
+					t.Fatalf("setting env var %s=%s: %v", k, v, err)
+				}
+			}
+			gotCfg, err := NewKubernetesConfig(WithKubernetesRunnerConfigBaseConfigFile(writeYamlToTemp(t, tt.k8sCfg)))
+			if err != nil {
+				t.Fatal(err)
+			}
 
 			fc := fake.NewSimpleClientset()
-			fc.PrependReactor("create", "pods", podDefaulterAction(t, tt.pkgCfg, wantCfg))
+			fc.PrependReactor("create", "pods", podDefaulterAction(t, tt.pkgCfg, gotCfg))
 
 			r := &k8s{
-				Config:    wantCfg,
+				Config:    gotCfg,
 				logger:    log.NewLogger(os.Stdout),
 				clientset: fc,
 			}
@@ -180,7 +303,7 @@ func Test_k8s_StartPod(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			gots, err := fc.CoreV1().Pods(wantCfg.Namespace).List(ctx, metav1.ListOptions{})
+			gots, err := fc.CoreV1().Pods(gotCfg.Namespace).List(ctx, metav1.ListOptions{})
 			if err != nil {
 				t.Fatal(err)
 			}