diff --git a/cmd/kops/BUILD.bazel b/cmd/kops/BUILD.bazel index e5730205a0567..c2df8a7bc82e9 100644 --- a/cmd/kops/BUILD.bazel +++ b/cmd/kops/BUILD.bazel @@ -48,6 +48,7 @@ go_library( "toolbox_convert_imported.go", "toolbox_dump.go", "toolbox_instance_selector.go", + "toolbox_rotate.go", "toolbox_template.go", "update.go", "update_cluster.go", @@ -63,6 +64,7 @@ go_library( "//:go_default_library", "//cmd/kops/util:go_default_library", "//pkg/apis/kops:go_default_library", + "//pkg/apis/kops/model:go_default_library", "//pkg/apis/kops/registry:go_default_library", "//pkg/apis/kops/util:go_default_library", "//pkg/apis/kops/validation:go_default_library", diff --git a/cmd/kops/toolbox.go b/cmd/kops/toolbox.go index a755a7d8734a8..b056c14fce036 100644 --- a/cmd/kops/toolbox.go +++ b/cmd/kops/toolbox.go @@ -49,6 +49,7 @@ func NewCmdToolbox(f *util.Factory, out io.Writer) *cobra.Command { cmd.AddCommand(NewCmdToolboxDump(f, out)) cmd.AddCommand(NewCmdToolboxTemplate(f, out)) cmd.AddCommand(NewCmdToolboxInstanceSelector(f, out)) + cmd.AddCommand(NewCmdToolboxRotate(f, out)) return cmd } diff --git a/cmd/kops/toolbox_rotate.go b/cmd/kops/toolbox_rotate.go new file mode 100644 index 0000000000000..19329aaf4b998 --- /dev/null +++ b/cmd/kops/toolbox_rotate.go @@ -0,0 +1,210 @@ +package main + +import ( + "context" + "encoding/base64" + "fmt" + "io" + + "github.com/spf13/cobra" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/kops/cmd/kops/util" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/kops/model" + "k8s.io/kops/pkg/kubeconfig" + "k8s.io/kops/upup/pkg/fi" +) + +func NewCmdToolboxRotate(f *util.Factory, out io.Writer) *cobra.Command { + + cmd := &cobra.Command{ + Use: "rotate", + Run: func(cmd *cobra.Command, args []string) { + ctx := context.Background() + + err := rootCommand.ProcessArgs(args) + if err != nil { + exitWithError(err) + } + + clusterName := rootCommand.ClusterName() + + if err := RunToolboxRotate(ctx, f, clusterName, out); err != nil { + exitWithError(err) + } + }, + } + return cmd +} + +func RunToolboxRotate(ctx context.Context, f *util.Factory, clusterName string, out io.Writer) error { + + cluster, err := rootCommand.Cluster(ctx) + if err != nil { + return err + } + + if !model.UseKopsControllerForNodeBootstrap(cluster) { + return fmt.Errorf("only clusters using kops-controller for boostrapping nodes are supported") + } + + clientset, err := f.Clientset() + if err != nil { + return err + } + + keyStore, err := clientset.KeyStore(cluster) + if err != nil { + return err + } + + oldCAID := "old-ca" + + //Fetch the current CA (which will be old-ca) + oldCert, err := keyStore.FindCert(fi.CertificateIDCA) + if err != nil { + return fmt.Errorf("could not fetch existing ca cert: %v", err) + } + oldKey, err := keyStore.FindPrivateKey(fi.CertificateIDCA) + if err != nil { + return fmt.Errorf("could not fetch existing ca key: %v", err) + } + + //Delete the current CA + ks, err := keyStore.FindCertificateKeyset(fi.CertificateIDCA) + if err != nil { + return err + } + + for _, item := range ks.Spec.Keys { + keyStore.DeleteKeysetItem(ks, item.Id) + } + + //Store the current CA as old-ca + keyStore.StoreKeypair(oldCAID, oldCert, oldKey) + + //Run a cluster update to generate a new CA. This should also update the kubeconfig with the CA bundle + updateClusterOpts := &UpdateClusterOptions{ + Yes: true, + Target: "direct", + } + _, err = RunUpdateCluster(ctx, f, clusterName, out, updateClusterOpts) + if err != nil { + return fmt.Errorf("could not update cluster: %v", err) + } + + //Fetch the new CA + cert, err := keyStore.FindCert(fi.CertificateIDCA) + if err != nil { + return fmt.Errorf("failed to load the new CA cert: %v", err) + } + + //Bundle the old and new CA + oldCertString, _ := oldCert.AsString() + certString, _ := cert.AsString() + + caBundleString := oldCertString + "\n" + certString + + //Update service accounts to trust old and new CA + err = updateServiceAccounts(ctx, cluster, caBundleString) + if err != nil { + return fmt.Errorf("error updating ServiceAccounts: %v", err) + } + + //New kubeconfig with bundled CA + RunExportKubecfg(ctx, f, out, &ExportKubecfgOptions{}, []string{}) + + return nil + + //Update nodes first. This will make kubelet trust new and old CA. + ruo := &RollingUpdateOptions{} + ruo.InitDefaults() + ruo.Yes = true + ruo.ClusterName = clusterName + ruo.Force = true + ruo.InstanceGroupRoles = []string{"node"} + + err = RunRollingUpdateCluster(ctx, f, out, ruo) + if err != nil { + return fmt.Errorf("failed to rotate cluster: %v", err) + } + + //Update masters. This will issue new certs for k8s using the new CA. + //New nodes, service accounts etc will use new CA + ruo.InstanceGroupRoles = []string{"master"} + + err = RunRollingUpdateCluster(ctx, f, out, ruo) + if err != nil { + return fmt.Errorf("failed to rotate cluster: %v", err) + } + + //Delete old-ca + ks, err = keyStore.FindCertificateKeyset(oldCAID) + if err != nil { + return err + } + + for _, item := range ks.Spec.Keys { + keyStore.DeleteKeysetItem(ks, item.Id) + } + + //New kubeconfig with only the new CA + RunExportKubecfg(ctx, f, out, &ExportKubecfgOptions{ + admin: kubeconfig.DefaultKubecfgAdminLifetime, + }, []string{}) + + //Rotating one last time to untrust the old certificate + ruo.InstanceGroupRoles = []string{"node"} + + err = RunRollingUpdateCluster(ctx, f, out, ruo) + if err != nil { + return fmt.Errorf("failed to rotate cluster: %v", err) + } + + ruo.InstanceGroupRoles = []string{"master"} + + err = RunRollingUpdateCluster(ctx, f, out, ruo) + if err != nil { + return fmt.Errorf("failed to rotate cluster: %v", err) + } + + return nil +} + +func updateServiceAccounts(ctx context.Context, cluster *kops.Cluster, caBundleString string) error { + + caBundle64 := base64.StdEncoding.EncodeToString([]byte(caBundleString)) + caBundle64Bytes := []byte(caBundle64) + + contextName := cluster.ObjectMeta.Name + configLoadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + configLoadingRules, + &clientcmd.ConfigOverrides{CurrentContext: contextName}).ClientConfig() + if err != nil { + return fmt.Errorf("cannot load kubecfg settings for %q: %v", contextName, err) + } + + k8sClient, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("cannot build kubernetes api client for %q: %v", contextName, err) + } + + secretClient := k8sClient.CoreV1().Secrets("") + + secrets, err := secretClient.List(ctx, v1.ListOptions{ + FieldSelector: "type=kubernetes.io/service-account-token", + }) + if err != nil { + return err + } + + for _, secret := range secrets.Items { + secret.Data["ca.crt"] = caBundle64Bytes + secretClient.Update(ctx, &secret, v1.UpdateOptions{}) + } + + return nil +} diff --git a/go.sum b/go.sum index 320439165362a..1b2b0923c41ab 100644 --- a/go.sum +++ b/go.sum @@ -326,6 +326,7 @@ github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQo github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go index 4c718b600208d..0e94796cb176c 100644 --- a/nodeup/pkg/model/bootstrap_client.go +++ b/nodeup/pkg/model/bootstrap_client.go @@ -60,6 +60,12 @@ func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error { return err } + oldCert, _ := b.GetCert("old-ca") + if oldCert != nil { + cert = append(cert, []byte("\n")...) + cert = append(cert, oldCert...) + } + baseURL := url.URL{ Scheme: "https", Host: net.JoinHostPort("kops-controller.internal."+b.Cluster.ObjectMeta.Name, strconv.Itoa(wellknownports.KopsControllerPort)), @@ -67,6 +73,7 @@ func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error { } bootstrapClient := &nodetasks.KopsBootstrapClient{ + Authenticator: authenticator, CA: cert, BaseURL: baseURL, diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index accc03258a2f1..e89cbe5ed1c20 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -244,6 +244,12 @@ func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.Model return nil, err } + oldCert, _ := c.GetCert("old-ca") + if oldCert != nil { + ca = append(ca, []byte("\n")...) + ca = append(ca, oldCert...) + } + kubeConfig := &nodetasks.KubeConfig{ Name: name, Cert: cert, diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index 45a90fe33a462..26c14f2f11d5f 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -296,10 +296,12 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { kubeAPIServer.ServiceAccountSigningKeyFile = &s } } + // If clientCAFile is not specified, set it to the default value ${PathSrvKubernetes}/ca.crt if kubeAPIServer.ClientCAFile == "" { - kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") + kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "cabundle.crt") } + kubeAPIServer.TLSCertFile = filepath.Join(b.PathSrvKubernetes(), "server.crt") kubeAPIServer.TLSPrivateKeyFile = filepath.Join(b.PathSrvKubernetes(), "server.key") @@ -336,6 +338,7 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { // @note we are making assumption were using the ones created by the pki model, not custom defined ones kubeAPIServer.KubeletClientCertificate = filepath.Join(b.PathSrvKubernetes(), "kubelet-api.crt") kubeAPIServer.KubeletClientKey = filepath.Join(b.PathSrvKubernetes(), "kubelet-api.key") + //kubeAPIServer.KubeletCertificateAuthority = filepath.Join(b.PathSrvKubernetes(), "cabundle.crt") } { diff --git a/nodeup/pkg/model/kube_controller_manager.go b/nodeup/pkg/model/kube_controller_manager.go index a7312cef6b2bd..d180d6a803cd5 100644 --- a/nodeup/pkg/model/kube_controller_manager.go +++ b/nodeup/pkg/model/kube_controller_manager.go @@ -103,7 +103,7 @@ func (b *KubeControllerManagerBuilder) Build(c *fi.ModelBuilderContext) error { func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) { kcm := b.Cluster.Spec.KubeControllerManager - kcm.RootCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") + kcm.RootCAFile = filepath.Join(b.PathSrvKubernetes(), "cabundle.crt") kcm.ServiceAccountPrivateKeyFile = filepath.Join(b.PathSrvKubernetes(), "service-account.key") flags, err := flagbuilder.BuildFlagsList(kcm) diff --git a/nodeup/pkg/model/kube_scheduler.go b/nodeup/pkg/model/kube_scheduler.go index 21601dba47ace..4135c2e44dcd2 100644 --- a/nodeup/pkg/model/kube_scheduler.go +++ b/nodeup/pkg/model/kube_scheduler.go @@ -210,7 +210,8 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) { sortedStrings(flags), "--logtostderr=false", //https://github.com/kubernetes/klog/issues/60 "--alsologtostderr", - "--log-file=/var/log/kube-scheduler.log") + "--log-file=/var/log/kube-scheduler.log", + ) } else { container.Command = exec.WithTee( "/usr/local/bin/kube-scheduler", diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index e9f52d7a504ba..0f7ef08401662 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -427,7 +427,7 @@ func (b *KubeletBuilder) buildKubeletConfigSpec() (*kops.KubeletConfigSpec, erro // check if we are using secure kubelet <-> api settings if b.UseSecureKubelet() { - c.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") + c.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "cabundle.crt") } if isMaster { diff --git a/nodeup/pkg/model/secrets.go b/nodeup/pkg/model/secrets.go index 3c46014efad74..4a1d4f8818630 100644 --- a/nodeup/pkg/model/secrets.go +++ b/nodeup/pkg/model/secrets.go @@ -54,6 +54,22 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { return err } + certs, _ := b.GetCert("ca") + oldCert, _ := b.GetCert("old-ca") + if oldCert != nil { + certs = append(certs, []byte("\n")...) + certs = append(certs, oldCert...) + + } + + p := filepath.Join(b.PathSrvKubernetes(), "cabundle.crt") + c.AddTask(&nodetasks.File{ + Path: p, + Contents: fi.NewBytesResource(certs), + Type: nodetasks.FileType_File, + Mode: s("0600"), + }) + // Write out docker auth secret, if exists if b.SecretStore != nil { key := "dockerconfig" diff --git a/pkg/kubeconfig/create_kubecfg.go b/pkg/kubeconfig/create_kubecfg.go index 8c5a5ff00ac30..09a04559b8064 100644 --- a/pkg/kubeconfig/create_kubecfg.go +++ b/pkg/kubeconfig/create_kubecfg.go @@ -111,11 +111,19 @@ func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.Se // or if we're using admin credentials and the secondary port if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" || cluster.Spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork || internal { cert, _, _, err := keyStore.FindKeypair(fi.CertificateIDCA) + if err != nil { return nil, fmt.Errorf("error fetching CA keypair: %v", err) } if cert != nil { - b.CACert, err = cert.AsBytes() + caBundle, _ := cert.AsString() + oldCert, _, _, err := keyStore.FindKeypair("old-ca") + if oldCert != nil { + oldCertString, _ := oldCert.AsString() + caBundle = caBundle + "\n" + oldCertString + } + + b.CACert = []byte(caBundle) if err != nil { return nil, err }