-
Notifications
You must be signed in to change notification settings - Fork 4.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Create toolbox command for rotating cluster CA
- Loading branch information
Ole Markus With
committed
Mar 8, 2021
1 parent
fea7589
commit 3848dfd
Showing
12 changed files
with
260 additions
and
5 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,210 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/base64" | ||
| "fmt" | ||
| "io" | ||
|
|
||
| "github.com/spf13/cobra" | ||
| v1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/client-go/kubernetes" | ||
| "k8s.io/client-go/tools/clientcmd" | ||
| "k8s.io/kops/cmd/kops/util" | ||
| "k8s.io/kops/pkg/apis/kops" | ||
| "k8s.io/kops/pkg/apis/kops/model" | ||
| "k8s.io/kops/pkg/kubeconfig" | ||
| "k8s.io/kops/upup/pkg/fi" | ||
| ) | ||
|
|
||
| func NewCmdToolboxRotate(f *util.Factory, out io.Writer) *cobra.Command { | ||
|
|
||
| cmd := &cobra.Command{ | ||
| Use: "rotate", | ||
| Run: func(cmd *cobra.Command, args []string) { | ||
| ctx := context.Background() | ||
|
|
||
| err := rootCommand.ProcessArgs(args) | ||
| if err != nil { | ||
| exitWithError(err) | ||
| } | ||
|
|
||
| clusterName := rootCommand.ClusterName() | ||
|
|
||
| if err := RunToolboxRotate(ctx, f, clusterName, out); err != nil { | ||
| exitWithError(err) | ||
| } | ||
| }, | ||
| } | ||
| return cmd | ||
| } | ||
|
|
||
| func RunToolboxRotate(ctx context.Context, f *util.Factory, clusterName string, out io.Writer) error { | ||
|
|
||
| cluster, err := rootCommand.Cluster(ctx) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if !model.UseKopsControllerForNodeBootstrap(cluster) { | ||
| return fmt.Errorf("only clusters using kops-controller for boostrapping nodes are supported") | ||
| } | ||
|
|
||
| clientset, err := f.Clientset() | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| keyStore, err := clientset.KeyStore(cluster) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| oldCAID := "old-ca" | ||
|
|
||
| //Fetch the current CA (which will be old-ca) | ||
| oldCert, err := keyStore.FindCert(fi.CertificateIDCA) | ||
| if err != nil { | ||
| return fmt.Errorf("could not fetch existing ca cert: %v", err) | ||
| } | ||
| oldKey, err := keyStore.FindPrivateKey(fi.CertificateIDCA) | ||
| if err != nil { | ||
| return fmt.Errorf("could not fetch existing ca key: %v", err) | ||
| } | ||
|
|
||
| //Delete the current CA | ||
| ks, err := keyStore.FindCertificateKeyset(fi.CertificateIDCA) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| for _, item := range ks.Spec.Keys { | ||
| keyStore.DeleteKeysetItem(ks, item.Id) | ||
| } | ||
|
|
||
| //Store the current CA as old-ca | ||
| keyStore.StoreKeypair(oldCAID, oldCert, oldKey) | ||
|
|
||
| //Run a cluster update to generate a new CA. This should also update the kubeconfig with the CA bundle | ||
| updateClusterOpts := &UpdateClusterOptions{ | ||
| Yes: true, | ||
| Target: "direct", | ||
| } | ||
| _, err = RunUpdateCluster(ctx, f, clusterName, out, updateClusterOpts) | ||
| if err != nil { | ||
| return fmt.Errorf("could not update cluster: %v", err) | ||
| } | ||
|
|
||
| //Fetch the new CA | ||
| cert, err := keyStore.FindCert(fi.CertificateIDCA) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to load the new CA cert: %v", err) | ||
| } | ||
|
|
||
| //Bundle the old and new CA | ||
| oldCertString, _ := oldCert.AsString() | ||
| certString, _ := cert.AsString() | ||
|
|
||
| caBundleString := oldCertString + "\n" + certString | ||
|
|
||
| //Update service accounts to trust old and new CA | ||
| err = updateServiceAccounts(ctx, cluster, caBundleString) | ||
| if err != nil { | ||
| return fmt.Errorf("error updating ServiceAccounts: %v", err) | ||
| } | ||
|
|
||
| //New kubeconfig with bundled CA | ||
| RunExportKubecfg(ctx, f, out, &ExportKubecfgOptions{}, []string{}) | ||
|
|
||
| return nil | ||
|
|
||
| //Update nodes first. This will make kubelet trust new and old CA. | ||
| ruo := &RollingUpdateOptions{} | ||
| ruo.InitDefaults() | ||
| ruo.Yes = true | ||
| ruo.ClusterName = clusterName | ||
| ruo.Force = true | ||
| ruo.InstanceGroupRoles = []string{"node"} | ||
|
|
||
| err = RunRollingUpdateCluster(ctx, f, out, ruo) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to rotate cluster: %v", err) | ||
| } | ||
|
|
||
| //Update masters. This will issue new certs for k8s using the new CA. | ||
| //New nodes, service accounts etc will use new CA | ||
| ruo.InstanceGroupRoles = []string{"master"} | ||
|
|
||
| err = RunRollingUpdateCluster(ctx, f, out, ruo) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to rotate cluster: %v", err) | ||
| } | ||
|
|
||
| //Delete old-ca | ||
| ks, err = keyStore.FindCertificateKeyset(oldCAID) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| for _, item := range ks.Spec.Keys { | ||
| keyStore.DeleteKeysetItem(ks, item.Id) | ||
| } | ||
|
|
||
| //New kubeconfig with only the new CA | ||
| RunExportKubecfg(ctx, f, out, &ExportKubecfgOptions{ | ||
| admin: kubeconfig.DefaultKubecfgAdminLifetime, | ||
| }, []string{}) | ||
|
|
||
| //Rotating one last time to untrust the old certificate | ||
| ruo.InstanceGroupRoles = []string{"node"} | ||
|
|
||
| err = RunRollingUpdateCluster(ctx, f, out, ruo) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to rotate cluster: %v", err) | ||
| } | ||
|
|
||
| ruo.InstanceGroupRoles = []string{"master"} | ||
|
|
||
| err = RunRollingUpdateCluster(ctx, f, out, ruo) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to rotate cluster: %v", err) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func updateServiceAccounts(ctx context.Context, cluster *kops.Cluster, caBundleString string) error { | ||
|
|
||
| caBundle64 := base64.StdEncoding.EncodeToString([]byte(caBundleString)) | ||
| caBundle64Bytes := []byte(caBundle64) | ||
|
|
||
| contextName := cluster.ObjectMeta.Name | ||
| configLoadingRules := clientcmd.NewDefaultClientConfigLoadingRules() | ||
| config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( | ||
| configLoadingRules, | ||
| &clientcmd.ConfigOverrides{CurrentContext: contextName}).ClientConfig() | ||
| if err != nil { | ||
| return fmt.Errorf("cannot load kubecfg settings for %q: %v", contextName, err) | ||
| } | ||
|
|
||
| k8sClient, err := kubernetes.NewForConfig(config) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot build kubernetes api client for %q: %v", contextName, err) | ||
| } | ||
|
|
||
| secretClient := k8sClient.CoreV1().Secrets("") | ||
|
|
||
| secrets, err := secretClient.List(ctx, v1.ListOptions{ | ||
| FieldSelector: "type=kubernetes.io/service-account-token", | ||
| }) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| for _, secret := range secrets.Items { | ||
| secret.Data["ca.crt"] = caBundle64Bytes | ||
| secretClient.Update(ctx, &secret, v1.UpdateOptions{}) | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters