diff --git a/go.mod b/go.mod index aaf59abbe..68fe22451 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/ijt/goparsify v0.0.0-20221203142333-3a5276334b8d github.com/imdario/mergo v0.3.16 github.com/joho/godotenv v1.5.1 + github.com/kelseyhightower/envconfig v1.4.0 github.com/klauspost/compress v1.16.7 github.com/klauspost/pgzip v1.2.5 github.com/korovkin/limiter v0.0.0-20221015170604-22eb1ceceddc diff --git a/go.sum b/go.sum index 3d80b9140..dc3ab9540 100644 --- a/go.sum +++ b/go.sum @@ -366,6 +366,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/pkg/container/kubernetes_runner.go b/pkg/container/kubernetes_runner.go index c36d09632..1c4eab695 100644 --- a/pkg/container/kubernetes_runner.go +++ b/pkg/container/kubernetes_runner.go @@ -33,6 +33,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/imdario/mergo" + "github.com/kelseyhightower/envconfig" "go.opentelemetry.io/otel" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -69,8 +70,13 @@ type k8s struct { } func KubernetesRunner(_ context.Context, logger log.Logger) (Runner, error) { + cfg, err := NewKubernetesConfig() + if err != nil { + return nil, fmt.Errorf("failed to configure kubernetes runner: %v", err) + } + runner := &k8s{ - Config: NewKubernetesConfig(), + Config: cfg, logger: logger, } @@ -118,7 +124,7 @@ func (k *k8s) StartPod(ctx context.Context, cfg *Config) error { } k.logger.Infof("created builder pod '%s' with UID '%s'", pod.Name, pod.UID) - if err := wait.PollUntilContextTimeout(ctx, 10*time.Second, k.Config.StartTimeout, true, func(ctx context.Context) (done bool, err error) { + if err := wait.PollUntilContextTimeout(ctx, 10*time.Second, k.Config.StartTimeout.Duration, true, func(ctx context.Context) (done bool, err error) { p, err := podclient.Get(ctx, pod.Name, metav1.GetOptions{}) if err != nil { return false, err @@ -437,9 +443,9 @@ type KubernetesRunnerConfig struct { Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"` Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` - StartTimeout time.Duration `json:"startTimeout" yaml:"startTimeout"` - BuildTimeout time.Duration `json:"buildTimeout" yaml:"buildTimeout"` + StartTimeout metav1.Duration `json:"startTimeout" yaml:"startTimeout" split_words:"true"` + // This field and everything below it is ignored by the environment variable parser PodTemplate *KubernetesRunnerConfigPodTemplate `json:"podTemplate,omitempty" yaml:"podTemplate,omitempty" ignored:"true"` // A "burstable" QOS is really the only thing that makes sense for ephemeral builder pods @@ -460,13 +466,12 @@ type KubernetesRunnerConfigPodTemplate struct { } // NewKubernetesConfig returns a default Kubernetes runner config setup -func NewKubernetesConfig(opt ...KubernetesRunnerConfigOptions) *KubernetesRunnerConfig { +func NewKubernetesConfig(opt ...KubernetesRunnerConfigOptions) (*KubernetesRunnerConfig, error) { cfg := &KubernetesRunnerConfig{ Provider: "generic", Namespace: "default", Repo: "ttl.sh/melange", - StartTimeout: 5 * time.Minute, - BuildTimeout: 30 * time.Minute, + StartTimeout: metav1.Duration{Duration: 10 * time.Minute}, Resources: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("4Gi"), @@ -479,13 +484,32 @@ func NewKubernetesConfig(opt ...KubernetesRunnerConfigOptions) *KubernetesRunner o(cfg) } - // We don't care about errors here, the empty value is safe to "merge" + // Override the defaults with values obtained from the global config file global := &KubernetesRunnerConfig{} - data, _ := os.ReadFile(cfg.baseConfigFile) - _ = yaml.Unmarshal(data, global) - _ = mergo.Merge(cfg, global, mergo.WithOverride) + data, err := os.ReadFile(cfg.baseConfigFile) + if err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("error reading config file %s: %w", cfg.baseConfigFile, err) + } else { + if err := yaml.Unmarshal(data, global); err != nil { + return nil, fmt.Errorf("error parsing config file %s: %w", cfg.baseConfigFile, err) + } + } + + if err := mergo.Merge(cfg, global, mergo.WithOverride); err != nil { + return nil, fmt.Errorf("error merging config file values %s with defaults: %w", cfg.baseConfigFile, err) + } + + // Finally, override with the values from the environment + var envcfg KubernetesRunnerConfig + if err := envconfig.Process("melange", &envcfg); err != nil { + return nil, fmt.Errorf("error parsing environment variables: %w", err) + } + + if err := mergo.Merge(cfg, envcfg, mergo.WithOverride); err != nil { + return nil, fmt.Errorf("error merging environment variables with defaults: %w", err) + } - return cfg + return cfg, nil } // escapeRFC1123 escapes a string to be RFC1123 compliant. We don't worry about diff --git a/pkg/container/kubernetes_runner_test.go b/pkg/container/kubernetes_runner_test.go index b6ed55c75..e33442941 100644 --- a/pkg/container/kubernetes_runner_test.go +++ b/pkg/container/kubernetes_runner_test.go @@ -5,9 +5,13 @@ import ( "fmt" "os" "testing" + "time" "chainguard.dev/apko/pkg/build/types" "chainguard.dev/apko/pkg/log" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/imdario/mergo" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,6 +22,90 @@ import ( "sigs.k8s.io/yaml" ) +func TestKubernetesRunnerConfig(t *testing.T) { + t.Parallel() + + dwant, _ := NewKubernetesConfig() + + // Intentionally use raw yaml to surface any type marshaling issues + writeRaw := func(data []byte) string { + f, err := os.CreateTemp("", "config.yaml") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + if _, err := f.Write(data); err != nil { + t.Fatal(err) + } + + return f.Name() + } + + tests := []struct { + name string + rawInput string + envs map[string]string + want *KubernetesRunnerConfig + }{ + { + name: "should have default values", + want: dwant, + }, + { + name: "should override default values with global yaml values", + rawInput: ` +namespace: foo +startTimeout: 10s +`, + want: &KubernetesRunnerConfig{ + Namespace: "foo", + StartTimeout: metav1.Duration{Duration: 10 * time.Second}, + }, + }, + { + name: "should override values with global yaml values and env vars", + rawInput: ` +namespace: foo +startTimeout: 5m +repo: somewhere +`, + envs: map[string]string{ + "MELANGE_REPO": "nowhere", + }, + want: &KubernetesRunnerConfig{ + Namespace: "foo", + StartTimeout: metav1.Duration{Duration: 5 * time.Minute}, + Repo: "nowhere", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Pickup the default want values + want, _ := NewKubernetesConfig() + if err := mergo.Merge(want, tt.want, mergo.WithOverride); err != nil { + t.Fatal(err) + } + + for k, v := range tt.envs { + if err := os.Setenv(k, v); err != nil { + t.Fatalf("setting env var %s=%s: %v", k, v, err) + } + } + got, err := NewKubernetesConfig(WithKubernetesRunnerConfigBaseConfigFile(writeRaw([]byte(tt.rawInput)))) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(got, want, cmpopts.IgnoreUnexported(KubernetesRunnerConfig{})); diff != "" { + t.Errorf("KubernetesConfig mismatch (-want +got):\n%s", diff) + } + }) + } +} + func Test_k8s_StartPod(t *testing.T) { t.Parallel() @@ -26,6 +114,7 @@ func Test_k8s_StartPod(t *testing.T) { tests := []struct { name string pkgCfg *Config + envs map[string]string k8sCfg *KubernetesRunnerConfig wanter func(got corev1.Pod) bool }{ @@ -38,13 +127,39 @@ func Test_k8s_StartPod(t *testing.T) { }, }, { - name: "should use global namespace instead of default", + name: "should load global configs from yaml", pkgCfg: &Config{PackageName: "donkey", Arch: types.Architecture("amd64")}, k8sCfg: &KubernetesRunnerConfig{Namespace: "not-default"}, wanter: func(got corev1.Pod) bool { return got.Namespace == "not-default" }, }, + { + name: "should prioritize environment configs", + pkgCfg: &Config{PackageName: "donkey", Arch: types.Architecture("amd64")}, + k8sCfg: &KubernetesRunnerConfig{Namespace: "not-default"}, + envs: map[string]string{ + "MELANGE_NAMESPACE": "from-env", + "MELANGE_REPO": "nowhere", + }, + wanter: func(got corev1.Pod) bool { + return got.Namespace == "from-env" + }, + }, + { + name: "should skip environment configs for certain fields", + pkgCfg: &Config{PackageName: "donkey", Arch: types.Architecture("amd64")}, + k8sCfg: &KubernetesRunnerConfig{PodTemplate: &KubernetesRunnerConfigPodTemplate{ServiceAccountName: "foo"}}, + envs: map[string]string{ + "MELANGE_SERVICE_ACCOUNT_NAME": "bar", + "MELANGE_SERVICEACCOUNTNAME": "bar", + "SERVICE_ACCOUNT_NAME": "bar", + "SERVICEACCOUNTNAME": "bar", + }, + wanter: func(got corev1.Pod) bool { + return got.Spec.ServiceAccountName == "foo" + }, + }, { name: "should support additional labels", pkgCfg: &Config{PackageName: "donkey", Arch: types.Architecture("amd64")}, @@ -165,13 +280,21 @@ func Test_k8s_StartPod(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - wantCfg := NewKubernetesConfig(WithKubernetesRunnerConfigBaseConfigFile(writeYamlToTemp(t, tt.k8sCfg))) + for k, v := range tt.envs { + if err := os.Setenv(k, v); err != nil { + t.Fatalf("setting env var %s=%s: %v", k, v, err) + } + } + gotCfg, err := NewKubernetesConfig(WithKubernetesRunnerConfigBaseConfigFile(writeYamlToTemp(t, tt.k8sCfg))) + if err != nil { + t.Fatal(err) + } fc := fake.NewSimpleClientset() - fc.PrependReactor("create", "pods", podDefaulterAction(t, tt.pkgCfg, wantCfg)) + fc.PrependReactor("create", "pods", podDefaulterAction(t, tt.pkgCfg, gotCfg)) r := &k8s{ - Config: wantCfg, + Config: gotCfg, logger: log.NewLogger(os.Stdout), clientset: fc, } @@ -180,7 +303,7 @@ func Test_k8s_StartPod(t *testing.T) { t.Fatal(err) } - gots, err := fc.CoreV1().Pods(wantCfg.Namespace).List(ctx, metav1.ListOptions{}) + gots, err := fc.CoreV1().Pods(gotCfg.Namespace).List(ctx, metav1.ListOptions{}) if err != nil { t.Fatal(err) }