diff --git a/cmd/kops/BUILD.bazel b/cmd/kops/BUILD.bazel index e5730205a0567..2903a95fc86c9 100644 --- a/cmd/kops/BUILD.bazel +++ b/cmd/kops/BUILD.bazel @@ -41,6 +41,8 @@ go_library( "rollingupdate.go", "rollingupdatecluster.go", "root.go", + "rotate.go", + "rotate_ca.go", "set.go", "set_cluster.go", "set_instancegroups.go", @@ -63,6 +65,7 @@ go_library( "//:go_default_library", "//cmd/kops/util:go_default_library", "//pkg/apis/kops:go_default_library", + "//pkg/apis/kops/model:go_default_library", "//pkg/apis/kops/registry:go_default_library", "//pkg/apis/kops/util:go_default_library", "//pkg/apis/kops/validation:go_default_library", diff --git a/cmd/kops/root.go b/cmd/kops/root.go index 3f301b72ff58c..e2e9c644060e0 100644 --- a/cmd/kops/root.go +++ b/cmd/kops/root.go @@ -149,6 +149,7 @@ func NewCmdRoot(f *util.Factory, out io.Writer) *cobra.Command { cmd.AddCommand(NewCmdReplace(f, out)) cmd.AddCommand(NewCmdRollingUpdate(f, out)) cmd.AddCommand(NewCmdSet(f, out)) + cmd.AddCommand(NewCmdRotate(f, out)) cmd.AddCommand(NewCmdToolbox(f, out)) cmd.AddCommand(NewCmdValidate(f, out)) cmd.AddCommand(NewCmdVersion(f, out)) diff --git a/cmd/kops/rotate.go b/cmd/kops/rotate.go new file mode 100644 index 0000000000000..07ae157d63fe1 --- /dev/null +++ b/cmd/kops/rotate.go @@ -0,0 +1,51 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "io" + + "github.com/spf13/cobra" + "k8s.io/kops/cmd/kops/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +var ( + rotateLong = templates.LongDesc(i18n.T(` + rotates secrets.`)) + + rotateExample = templates.Examples(i18n.T(` + # Rotate the cluster CA + kops rotate ca --name k8s-cluster.example.com + `)) + + rotateShort = i18n.T(`Rotates secrets.`) +) + +func NewCmdRotate(f *util.Factory, out io.Writer) *cobra.Command { + cmd := &cobra.Command{ + Use: "rotate", + Short: rotateShort, + Long: rotateLong, + Example: rotateExample, + } + + cmd.AddCommand(NewCmdRotateCA(f, out)) + + return cmd +} diff --git a/cmd/kops/rotate_ca.go b/cmd/kops/rotate_ca.go new file mode 100644 index 0000000000000..fb56403cee9b9 --- /dev/null +++ b/cmd/kops/rotate_ca.go @@ -0,0 +1,310 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bufio" + "context" + "crypto/x509/pkix" + "encoding/base64" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/spf13/cobra" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + "k8s.io/kops/cmd/kops/util" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/kops/model" + "k8s.io/kops/pkg/pki" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" +) + +var ( + rotateCALong = templates.LongDesc(i18n.T(` + rotates the cluster CA.`)) + + rotateCAExample = templates.Examples(i18n.T(` + # Rotate the cluster CA + kops rotate ca --name k8s-cluster.example.com + `)) + + rotateCAShort = i18n.T(`Rotates the cluster CA.`) +) + +func NewCmdRotateCA(f *util.Factory, out io.Writer) *cobra.Command { + + cmd := &cobra.Command{ + Use: "ca", + Long: rotateCALong, + Short: rotateCAShort, + Example: rotateCAExample, + Run: func(cmd *cobra.Command, args []string) { + ctx := context.Background() + + err := rootCommand.ProcessArgs(args) + if err != nil { + exitWithError(err) + } + + clusterName := rootCommand.ClusterName() + + if err := RunRotateCA(ctx, f, clusterName, out); err != nil { + exitWithError(err) + } + }, + } + + return cmd +} + +func RunRotateCA(ctx context.Context, f *util.Factory, clusterName string, out io.Writer) error { + + cluster, err := rootCommand.Cluster(ctx) + if err != nil { + return err + } + + scanner := bufio.NewScanner(os.Stdin) + + contextName := cluster.ObjectMeta.Name + clientConfig, err := clientcmd.NewDefaultClientConfigLoadingRules().Load() + + if err != nil { + return fmt.Errorf("failed to delete secondary key: %w", err) + } + + if !model.UseKopsControllerForNodeBootstrap(cluster) { + return fmt.Errorf("only clusters using kops-controller for boostrapping nodes are supported") + } + + exportAdmin := clientConfig.Contexts[contextName].AuthInfo == contextName + + fmt.Println("This comamnd will rotate the cluster CA. It is largely safe, but be aware of the following:") + fmt.Println(" * exporting the admin TLS credentials before this command has succeeded will break") + fmt.Println(" your client credentials and the only way to recover is to `kops rolling update --cloudonly --yes --force`") + fmt.Println(" * This command will rotate all nodes multiple times. This will take a while.") + fmt.Println(" * It is safe to restart this command provided you did not export credentials manually") + fmt.Println("") + fmt.Println("Your cluster should be fully updated and rotated before starting this procedure.") + fmt.Println("") + if exportAdmin { + fmt.Println("The admin TLS credentials was detected. We will export the admin credentials after rotating the CA") + } else { + fmt.Println("Could not detect any admin credentials. Assuming admin credentials are not in use.") + } + + fmt.Println("If you understand the above, type 'yes'. Anything else will abort.") + scanner.Scan() + err = scanner.Err() + if err != nil { + exitWithError(fmt.Errorf("unable to interpret input: %w", err)) + } + val := scanner.Text() + val = strings.TrimSpace(val) + val = strings.ToLower(val) + if val != "yes" { + exitWithError(fmt.Errorf("Aborting")) + } + + clientset, err := f.Clientset() + if err != nil { + return err + } + + keyStore, err := clientset.KeyStore(cluster) + if err != nil { + return err + } + + pool, err := keyStore.FindCertificatePool(fi.CertificateIDCA) + if err != nil { + return fmt.Errorf("could not fetch the CA pool: %w", err) + } + + if len(pool.Secondary) > 0 { + klog.Info("Secondary CA cert already in the pool. Not issuing a new CA") + } else { + err := rotateCAIssueCert(keyStore) + if err != nil { + return fmt.Errorf("could not issue new CA: %w", err) + } + + //Update the pool + pool, err = keyStore.FindCertificatePool(fi.CertificateIDCA) + if err != nil { + return fmt.Errorf("could not fetch the CA pool: %w", err) + } + } + + // Update the cluster to trust both CAs + err = rotateCAUpdateCluster(ctx, cluster, pool, f, out, false) + if err != nil { + return fmt.Errorf("failed to update the cluster: %w", err) + } + + //Delete the old key + klog.Info("deleting the old CA") + + keyId := pool.Secondary[0].Certificate.SerialNumber.String() + keyset, err := keyStore.FindCertificateKeyset(fi.CertificateIDCA) + if err != nil { + return fmt.Errorf("failed to load keyset: %w", err) + } + + err = keyStore.DeleteKeysetItem(keyset, keyId) + if err != nil { + return fmt.Errorf("failed to delete secondary key: %w", err) + } + + if exportAdmin { + klog.Info("Detected the admin TLS user. Will also export a new admin certificate") + } else { + klog.Info("Could not detect the admin TLS user. Assuming existing credentials will continue to work") + } + + // Update the cluster one last time to trust only the new CA + err = rotateCAUpdateCluster(ctx, cluster, pool, f, out, exportAdmin) + if err != nil { + return fmt.Errorf("failed to update the cluster: %w", err) + } + + return nil +} + +func rotateCAIssueCert(keyStore fi.CAStore) error { + + klog.Infof("Issuing new certificate") + + serial := pki.BuildPKISerial(time.Now().UnixNano()) + + subjectPkix := &pkix.Name{ + CommonName: "kubernetes", + } + + req := pki.IssueCertRequest{ + Signer: fi.CertificateIDCA, + Type: "ca", + Subject: *subjectPkix, + AlternateNames: []string{}, + Serial: serial, + } + cert, privateKey, _, err := pki.IssueCert(&req, keyStore) + if err != nil { + return err + } + err = keyStore.StoreKeypair(fi.CertificateIDCA, cert, privateKey) + if err != nil { + return err + } + + return nil +} + +func rotateCAUpdateServiceAccounts(ctx context.Context, cluster *kops.Cluster, caBundle []byte) error { + klog.Info("updating ServiceAccounts with a new CA bundle") + + caBundle64 := base64.StdEncoding.EncodeToString(caBundle) + caBundle64Bytes := []byte(caBundle64) + + contextName := cluster.ObjectMeta.Name + configLoadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + configLoadingRules, + &clientcmd.ConfigOverrides{CurrentContext: contextName}).ClientConfig() + if err != nil { + return fmt.Errorf("cannot load kubecfg settings for %q: %v", contextName, err) + } + + k8sClient, err := kubernetes.NewForConfig(config) + + if err != nil { + return fmt.Errorf("cannot build kubernetes api client for %q: %v", contextName, err) + } + + secretClient := k8sClient.CoreV1().Secrets("") + + secrets, err := secretClient.List(ctx, v1.ListOptions{ + FieldSelector: "type=kubernetes.io/service-account-token", + }) + if err != nil { + return err + } + + for _, secret := range secrets.Items { + secret.Data["ca.crt"] = caBundle64Bytes + secretClient.Update(ctx, &secret, v1.UpdateOptions{}) + } + + return nil +} + +func rotateCAUpdateCluster(ctx context.Context, cluster *kops.Cluster, pool *fi.CertificatePool, f *util.Factory, out io.Writer, exportAdmin bool) error { + caBundle, err := pool.AsBytes() + if err != nil { + return fmt.Errorf("failed to encode ca bundle: %w", err) + } + + //Update service accounts to trust old and new CA + err = rotateCAUpdateServiceAccounts(ctx, cluster, caBundle) + if err != nil { + return fmt.Errorf("error updating ServiceAccounts: %v", err) + } + + adminTTL, _ := time.ParseDuration("0") + if exportAdmin { + adminTTL, _ = time.ParseDuration("18h") + } + + //New kubeconfig with bundled CA so we trust both new and old api servers + RunExportKubecfg(ctx, f, out, &ExportKubecfgOptions{admin: adminTTL}, []string{}) + + klog.Info("rotating all nodes") + + //Update nodes first. This will make kubelet trust new and old CA. + ruo := &RollingUpdateOptions{} + ruo.InitDefaults() + ruo.Yes = true + ruo.ClusterName = cluster.ObjectMeta.Name + ruo.Force = true + ruo.InstanceGroupRoles = []string{"node"} + + err = RunRollingUpdateCluster(ctx, f, out, ruo) + if err != nil { + return fmt.Errorf("failed to rotate cluster: %v", err) + } + + klog.Info("rotating the control plane") + + //Update masters. This will issue new certs for k8s using the new CA. + //New nodes, service accounts etc will use new CA + ruo.InstanceGroupRoles = []string{"master", "apiserver"} + + err = RunRollingUpdateCluster(ctx, f, out, ruo) + if err != nil { + return fmt.Errorf("failed to rotate cluster: %v", err) + } + return nil + +} diff --git a/docs/cli/kops.md b/docs/cli/kops.md index 4e11f08982bfc..9d2eb3863a4f9 100644 --- a/docs/cli/kops.md +++ b/docs/cli/kops.md @@ -47,6 +47,7 @@ kOps is Kubernetes Operations. * [kops import](kops_import.md) - Import a cluster. * [kops replace](kops_replace.md) - Replace cluster resources. * [kops rolling-update](kops_rolling-update.md) - Rolling update a cluster. +* [kops rotate](kops_rotate.md) - Rotates secrets. * [kops set](kops_set.md) - Set fields on clusters and other resources. * [kops toolbox](kops_toolbox.md) - Misc infrequently used commands. * [kops update](kops_update.md) - Update a cluster. diff --git a/docs/cli/kops_rotate.md b/docs/cli/kops_rotate.md new file mode 100644 index 0000000000000..cb345769612db --- /dev/null +++ b/docs/cli/kops_rotate.md @@ -0,0 +1,50 @@ + + + +## kops rotate + +Rotates secrets. + +### Synopsis + +rotates secrets. + +### Examples + +``` + # Rotate the cluster CA + kops rotate ca --name k8s-cluster.example.com +``` + +### Options + +``` + -h, --help help for rotate +``` + +### Options inherited from parent commands + +``` + --add_dir_header If true, adds the file directory to the header of the log messages + --alsologtostderr log to standard error as well as files + --config string yaml config file (default is $HOME/.kops.yaml) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_file string If non-empty, use this log file + --log_file_max_size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) + --logtostderr log to standard error instead of files (default true) + --name string Name of cluster. Overrides KOPS_CLUSTER_NAME environment variable + --one_output If true, only write logs to their native severity level (vs also writing to each lower severity level) + --skip_headers If true, avoid header prefixes in the log messages + --skip_log_headers If true, avoid headers when opening log files + --state string Location of state storage (kops 'config' file). Overrides KOPS_STATE_STORE environment variable + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + -v, --v Level number for the log level verbosity + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + +### SEE ALSO + +* [kops](kops.md) - kOps is Kubernetes Operations. +* [kops rotate ca](kops_rotate_ca.md) - Rotates the cluster CA. + diff --git a/docs/cli/kops_rotate_ca.md b/docs/cli/kops_rotate_ca.md new file mode 100644 index 0000000000000..f79630a42c2a0 --- /dev/null +++ b/docs/cli/kops_rotate_ca.md @@ -0,0 +1,53 @@ + + + +## kops rotate ca + +Rotates the cluster CA. + +### Synopsis + +rotates the cluster CA. + +``` +kops rotate ca [flags] +``` + +### Examples + +``` + # Rotate the cluster CA + kops rotate ca --name k8s-cluster.example.com +``` + +### Options + +``` + -h, --help help for ca +``` + +### Options inherited from parent commands + +``` + --add_dir_header If true, adds the file directory to the header of the log messages + --alsologtostderr log to standard error as well as files + --config string yaml config file (default is $HOME/.kops.yaml) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_file string If non-empty, use this log file + --log_file_max_size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) + --logtostderr log to standard error instead of files (default true) + --name string Name of cluster. Overrides KOPS_CLUSTER_NAME environment variable + --one_output If true, only write logs to their native severity level (vs also writing to each lower severity level) + --skip_headers If true, avoid header prefixes in the log messages + --skip_log_headers If true, avoid headers when opening log files + --state string Location of state storage (kops 'config' file). Overrides KOPS_STATE_STORE environment variable + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + -v, --v Level number for the log level verbosity + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + +### SEE ALSO + +* [kops rotate](kops_rotate.md) - Rotates secrets. + diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go index 4c718b600208d..df016c6f324d5 100644 --- a/nodeup/pkg/model/bootstrap_client.go +++ b/nodeup/pkg/model/bootstrap_client.go @@ -55,7 +55,11 @@ func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error { return err } - cert, err := b.GetCert(fi.CertificateIDCA) + pool, err := b.KeyStore.FindCertificatePool(fi.CertificateIDCA) + if err != nil { + return err + } + cert, err := pool.AsBytes() if err != nil { return err } diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index 9c2417712ab8f..fcf2805d32fa4 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -247,7 +247,11 @@ func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.Model if c.UseKopsControllerForNodeBootstrap() { cert, key := c.GetBootstrapCert(name) - ca, err := c.GetCert(fi.CertificateIDCA) + pool, err := c.KeyStore.FindCertificatePool(fi.CertificateIDCA) + if err != nil { + return nil, err + } + ca, err := pool.AsBytes() if err != nil { return nil, err } @@ -457,6 +461,33 @@ func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, n return nil } +// BuildCertificateBundleTask builds a task to create a certificate bundle file. +func (c *NodeupModelContext) BuildCertificateBundleTask(ctx *fi.ModelBuilderContext, name, filename string, owner *string) error { + pool, err := c.KeyStore.FindCertificatePool(name) + if err != nil { + return err + } + + serialized, err := pool.AsString() + if err != nil { + return err + } + p := filename + if !filepath.IsAbs(p) { + p = filepath.Join(c.PathSrvKubernetes(), filename) + } + + ctx.AddTask(&nodetasks.File{ + Path: p, + Contents: fi.NewStringResource(serialized), + Type: nodetasks.FileType_File, + Mode: s("0600"), + Owner: owner, + }) + + return nil +} + // BuildPrivateKeyTask builds a task to create a private key file. func (c *NodeupModelContext) BuildPrivateKeyTask(ctx *fi.ModelBuilderContext, name, filename string, owner *string) error { cert, err := c.KeyStore.FindPrivateKey(name) diff --git a/nodeup/pkg/model/fakes_test.go b/nodeup/pkg/model/fakes_test.go index f126c8cdfe536..084e2044626ea 100644 --- a/nodeup/pkg/model/fakes_test.go +++ b/nodeup/pkg/model/fakes_test.go @@ -61,7 +61,14 @@ type fakeCAStore struct { var _ fi.CAStore = &fakeCAStore{} func (k fakeCAStore) FindCertificatePool(name string) (*fi.CertificatePool, error) { - panic("fakeCAStore does not implement FindCertificatePool") + pool := &fi.CertificatePool{} + + cert, exists := k.certs[name] + + if exists { + pool.Primary = cert + } + return pool, nil } func (k fakeCAStore) FindCertificateKeyset(name string) (*kops.Keyset, error) { diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index 5e847edc5654a..319ae831ee008 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -296,9 +296,9 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { kubeAPIServer.ServiceAccountSigningKeyFile = &s } } - // If clientCAFile is not specified, set it to the default value ${PathSrvKubernetes}/ca.crt + // If clientCAFile is not specified, set it to the default value ${PathSrvKubernetes}/cabundle.crt if kubeAPIServer.ClientCAFile == "" { - kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") + kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "cabundle.crt") } kubeAPIServer.TLSCertFile = filepath.Join(b.PathSrvKubernetes(), "server.crt") kubeAPIServer.TLSPrivateKeyFile = filepath.Join(b.PathSrvKubernetes(), "server.key") diff --git a/nodeup/pkg/model/kube_controller_manager.go b/nodeup/pkg/model/kube_controller_manager.go index a7312cef6b2bd..d180d6a803cd5 100644 --- a/nodeup/pkg/model/kube_controller_manager.go +++ b/nodeup/pkg/model/kube_controller_manager.go @@ -103,7 +103,7 @@ func (b *KubeControllerManagerBuilder) Build(c *fi.ModelBuilderContext) error { func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) { kcm := b.Cluster.Spec.KubeControllerManager - kcm.RootCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") + kcm.RootCAFile = filepath.Join(b.PathSrvKubernetes(), "cabundle.crt") kcm.ServiceAccountPrivateKeyFile = filepath.Join(b.PathSrvKubernetes(), "service-account.key") flags, err := flagbuilder.BuildFlagsList(kcm) diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index 5e356c2ff768f..19e4ab2c1c6fa 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -428,7 +428,7 @@ func (b *KubeletBuilder) buildKubeletConfigSpec() (*kops.KubeletConfigSpec, erro // check if we are using secure kubelet <-> api settings if b.UseSecureKubelet() { - c.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") + c.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "cabundle.crt") } if isMaster { diff --git a/nodeup/pkg/model/secrets.go b/nodeup/pkg/model/secrets.go index 4b9fcc21ecb9f..6a8df124cde7f 100644 --- a/nodeup/pkg/model/secrets.go +++ b/nodeup/pkg/model/secrets.go @@ -54,6 +54,10 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { return err } + if err := b.BuildCertificateBundleTask(c, fi.CertificateIDCA, "cabundle.crt", nil); err != nil { + return err + } + // Write out docker auth secret, if exists if b.SecretStore != nil { key := "dockerconfig" diff --git a/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml index 65a4c0aa01f31..47866c52148a0 100644 --- a/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml @@ -41,7 +41,7 @@ contents: | - --authentication-token-webhook-config-file=/etc/kubernetes/authn.config - --authorization-mode=AlwaysAllow - --bind-address=0.0.0.0 - - --client-ca-file=/srv/kubernetes/ca.crt + - --client-ca-file=/srv/kubernetes/cabundle.crt - --cloud-config=/etc/kubernetes/cloud.config - --cloud-provider=aws - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml index 917e69ef52dd7..91b90d4597589 100644 --- a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml @@ -19,7 +19,7 @@ contents: | - --apiserver-count=1 - --authorization-mode=AlwaysAllow - --bind-address=0.0.0.0 - - --client-ca-file=/srv/kubernetes/ca.crt + - --client-ca-file=/srv/kubernetes/cabundle.crt - --cloud-config=/etc/kubernetes/cloud.config - --cloud-provider=aws - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml index 5ea5af457cdc6..b6cf98b173011 100644 --- a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml +++ b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml @@ -24,7 +24,7 @@ contents: | - --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ - --kubeconfig=/var/lib/kube-controller-manager/kubeconfig - --leader-elect=true - - --root-ca-file=/srv/kubernetes/ca.crt + - --root-ca-file=/srv/kubernetes/cabundle.crt - --service-account-private-key-file=/srv/kubernetes/service-account.key - --use-service-account-credentials=true - --v=2 diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-secret.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-secret.yaml index e0d573956ebb9..f1b007542eb2e 100644 --- a/nodeup/pkg/model/tests/golden/minimal/tasks-secret.yaml +++ b/nodeup/pkg/model/tests/golden/minimal/tasks-secret.yaml @@ -58,6 +58,29 @@ mode: "0600" path: /srv/kubernetes/ca.crt type: file --- +contents: | + -----BEGIN CERTIFICATE----- + MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw + FTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy + MzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA + A4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd + XEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY + OACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1 + YYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC + An8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA + NPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB + /wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r + hzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO + HE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe + rN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt + O6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy + VpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh + 8yGeRx9AbknHh4Ia + -----END CERTIFICATE----- +mode: "0600" +path: /srv/kubernetes/cabundle.crt +type: file +--- contents: task: Name: master diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml index 3ffccbc69ed6d..119e9aa738722 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml @@ -19,7 +19,7 @@ contents: | - --apiserver-count=1 - --authorization-mode=AlwaysAllow - --bind-address=0.0.0.0 - - --client-ca-file=/srv/kubernetes/ca.crt + - --client-ca-file=/srv/kubernetes/cabundle.crt - --cloud-config=/etc/kubernetes/cloud.config - --cloud-provider=aws - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml index 44597ad845de0..ab90c64d95b0b 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml @@ -19,7 +19,7 @@ contents: | - --apiserver-count=1 - --authorization-mode=AlwaysAllow - --bind-address=0.0.0.0 - - --client-ca-file=/srv/kubernetes/ca.crt + - --client-ca-file=/srv/kubernetes/cabundle.crt - --cloud-config=/etc/kubernetes/cloud.config - --cloud-provider=aws - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-amd64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-amd64.yaml index 56e71cece597e..4296fe16ebc62 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-amd64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-amd64.yaml @@ -24,7 +24,7 @@ contents: | - --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ - --kubeconfig=/var/lib/kube-controller-manager/kubeconfig - --leader-elect=true - - --root-ca-file=/srv/kubernetes/ca.crt + - --root-ca-file=/srv/kubernetes/cabundle.crt - --service-account-private-key-file=/srv/kubernetes/service-account.key - --use-service-account-credentials=true - --v=2 diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-arm64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-arm64.yaml index 4316732642cc4..01596e4f8e580 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-arm64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-arm64.yaml @@ -24,7 +24,7 @@ contents: | - --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ - --kubeconfig=/var/lib/kube-controller-manager/kubeconfig - --leader-elect=true - - --root-ca-file=/srv/kubernetes/ca.crt + - --root-ca-file=/srv/kubernetes/cabundle.crt - --service-account-private-key-file=/srv/kubernetes/service-account.key - --use-service-account-credentials=true - --v=2 diff --git a/pkg/kubeconfig/create_kubecfg.go b/pkg/kubeconfig/create_kubecfg.go index 8c5a5ff00ac30..c56b1af659670 100644 --- a/pkg/kubeconfig/create_kubecfg.go +++ b/pkg/kubeconfig/create_kubecfg.go @@ -34,7 +34,7 @@ import ( const DefaultKubecfgAdminLifetime = 18 * time.Hour -func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.SecretStore, status kops.StatusStore, admin time.Duration, configUser string, internal bool, kopsStateStore string, useKopsAuthenticationPlugin bool) (*KubeconfigBuilder, error) { +func BuildKubecfg(cluster *kops.Cluster, keyStore fi.CAStore, secretStore fi.SecretStore, status kops.StatusStore, admin time.Duration, configUser string, internal bool, kopsStateStore string, useKopsAuthenticationPlugin bool) (*KubeconfigBuilder, error) { clusterName := cluster.ObjectMeta.Name var master string @@ -110,12 +110,13 @@ func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.Se // add the CA Cert to the kubeconfig only if we didn't specify a certificate for the LB // or if we're using admin credentials and the secondary port if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" || cluster.Spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork || internal { - cert, _, _, err := keyStore.FindKeypair(fi.CertificateIDCA) + pool, err := keyStore.FindCertificatePool(fi.CertificateIDCA) + if err != nil { return nil, fmt.Errorf("error fetching CA keypair: %v", err) } - if cert != nil { - b.CACert, err = cert.AsBytes() + if pool != nil { + b.CACert, err = pool.AsBytes() if err != nil { return nil, err } diff --git a/pkg/kubeconfig/create_kubecfg_test.go b/pkg/kubeconfig/create_kubecfg_test.go index a6f5394f24040..13ea24cf15504 100644 --- a/pkg/kubeconfig/create_kubecfg_test.go +++ b/pkg/kubeconfig/create_kubecfg_test.go @@ -48,7 +48,9 @@ func (f fakeStatusStore) GetApiIngressStatus(cluster *kops.Cluster) ([]kops.ApiI } // mock a fake key store -type fakeKeyStore struct { +type fakeCAStore struct { + fi.CAStore + FindKeypairFn func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) // StoreKeypair writes the keypair to the store @@ -58,18 +60,29 @@ type fakeKeyStore struct { MirrorToFn func(basedir vfs.Path) error } -func (f fakeKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { +func (f fakeCAStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { return f.FindKeypairFn(name) } -func (f fakeKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { +func (f fakeCAStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { return f.StoreKeypairFn(id, cert, privateKey) } -func (f fakeKeyStore) MirrorTo(basedir vfs.Path) error { +func (f fakeCAStore) MirrorTo(basedir vfs.Path) error { return f.MirrorToFn(basedir) } +func (f fakeCAStore) FindCertificatePool(name string) (*fi.CertificatePool, error) { + pool := &fi.CertificatePool{} + cert, _, _, err := f.FindKeypair(name) + if err != nil { + return pool, err + } + pool.Primary = cert + + return pool, nil +} + // build a generic minimal cluster func buildMinimalCluster(clusterName string, masterPublicName string, lbCert bool, nlb bool) *kops.Cluster { cluster := testutils.BuildMinimalCluster(clusterName) @@ -313,7 +326,7 @@ func TestBuildKubecfg(t *testing.T) { t.Run(tt.name, func(t *testing.T) { kopsStateStore := "memfs://example-state-store" - keyStore := fakeKeyStore{ + keyStore := fakeCAStore{ FindKeypairFn: func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { return fakeCertificate(), fakePrivateKey(), diff --git a/upup/pkg/fi/ca.go b/upup/pkg/fi/ca.go index 4b9402feb761f..112dbb74c1ce5 100644 --- a/upup/pkg/fi/ca.go +++ b/upup/pkg/fi/ca.go @@ -124,6 +124,28 @@ func (c *CertificatePool) All() []*pki.Certificate { return certs } +func (c *CertificatePool) AsBytes() ([]byte, error) { + // Nicer behaviour because this is called from templates + if c == nil { + return nil, fmt.Errorf("AsBytes called on nil CertificatePool") + } + + var data bytes.Buffer + if c.Primary != nil { + _, err := c.Primary.WriteTo(&data) + if err != nil { + return nil, fmt.Errorf("error writing SSL certificate: %v", err) + } + } + for _, cert := range c.Secondary { + _, err := cert.WriteTo(&data) + if err != nil { + return nil, fmt.Errorf("error writing SSL certificate: %v", err) + } + } + return data.Bytes(), nil +} + func (c *CertificatePool) AsString() (string, error) { // Nicer behaviour because this is called from templates if c == nil {