Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Delete Consul custom resources with consul-k8s uninstall #1623

Merged
merged 43 commits into from
Nov 3, 2022
Merged
Show file tree
Hide file tree
Changes from 27 commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
9e1c77a
Change uses to point to Git hash
Oct 27, 2022
7fa8bdc
Update Go client dep
Oct 13, 2022
22e8ee3
Rework to patch CRs
Oct 18, 2022
8ec67d2
Delete and patch CRs
Oct 19, 2022
616a253
Patch finalizers after uninstall
Oct 19, 2022
aa9d853
Add function for ignoring 404 errors
Oct 20, 2022
90ebac7
Change to using apiextensions lib
Oct 20, 2022
68863bc
Handle timeout for deletion
Oct 20, 2022
ca93bf4
Remove finalizer obj (took different route)
Oct 20, 2022
8f6cf15
Add DeletionError
Oct 20, 2022
bc7546d
Add some early returns
Oct 20, 2022
d363394
WIP: mocking dynamic client
Oct 20, 2022
ac82890
Test the new functions
Oct 21, 2022
c2ffdf1
Finish mocking the CRs in tests
Oct 21, 2022
573cecc
Polish tests
Oct 21, 2022
b5922e3
Add CR deletion to full uninstall test
Oct 21, 2022
273e63b
go mod tidy
Oct 21, 2022
d200b84
Add output to notify user of deletion/patching process
Oct 21, 2022
2687941
Polish output text
Oct 22, 2022
56ba323
Add mapping of kind to resource name
Oct 26, 2022
d5f6169
Fix tests with line break
Oct 26, 2022
d31baf3
Use uiLogger for consistency
Oct 26, 2022
55c6ada
Fix tests to fetch crds
Oct 26, 2022
2f8686c
Test mapCRKindToResource
Oct 26, 2022
e41d4e6
Use DurationVar for timeout
Oct 26, 2022
1371f45
Add CHANGELOG entry for PR 1623
Oct 26, 2022
862cce6
Remove merge conflict in workflows/test
Oct 31, 2022
12ca57e
Include Deletion of CRs message in TestUninstall
Oct 31, 2022
bc4f07b
Re-unify test strings
Oct 31, 2022
01131c1
Use k8serrors and test CR in another ns
Nov 1, 2022
2581d84
Comment out GHA
Nov 1, 2022
56fdd07
Rename k8s clients
Nov 1, 2022
476e4a1
Remove `IgnoreNotFoundError`
Nov 1, 2022
b4d61a9
Match style guide on CHANGELOG
Nov 1, 2022
8569563
Use schema.ParseGroupVersion
Nov 1, 2022
07d535f
Fix punctuation on initKubernetes
Nov 1, 2022
0d47724
Improve logging and separate out removal of CRs
Nov 1, 2022
3b3c034
Clean up the CR deletion method a bit
Nov 1, 2022
2073acd
Order imports
Nov 1, 2022
63aaf5b
Add doc comment to removeCustomResources
Nov 1, 2022
8c02cf2
Add fakeUILogger
Nov 1, 2022
168450c
Add retry to verifying patch command
Nov 1, 2022
2be231a
Fix output in uninstall test
Nov 2, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ FEATURES:
IMPROVEMENTS:
* CLI
* Update minimum go version for project to 1.19 [[GH-1633](https://github.com/hashicorp/consul-k8s/pull/1633)]
* `consul-k8s uninstall` now deletes custom resources when uninstalling Consul. [[GH-1623](https://github.com/hashicorp/consul-k8s/pull/1623)]
t-eckert marked this conversation as resolved.
Show resolved Hide resolved
* Control Plane
* Update minimum go version for project to 1.19 [[GH-1633](https://github.com/hashicorp/consul-k8s/pull/1633)]
* Remove unneeded `agent:read` ACL permissions from mesh gateway policy. [[GH-1255](https://github.com/hashicorp/consul-k8s/pull/1255)]
Expand Down
2 changes: 1 addition & 1 deletion cli/cmd/install/install.go
Original file line number Diff line number Diff line change
Expand Up @@ -401,7 +401,7 @@ func (c *Command) installConsul(valuesYaml []byte, vals map[string]interface{},
if len(vals) == 0 {
c.UI.Output("\nNo overrides provided, using the default Helm values.", terminal.WithInfoStyle())
} else {
c.UI.Output("\nHelm value overrides\n-------------------\n"+string(valuesYaml), terminal.WithInfoStyle())
c.UI.Output("\nHelm value overrides\n--------------------\n"+string(valuesYaml), terminal.WithInfoStyle())
t-eckert marked this conversation as resolved.
Show resolved Hide resolved
}

// Without informing the user, default global.name to consul if it hasn't been set already. We don't allow setting
Expand Down
6 changes: 3 additions & 3 deletions cli/cmd/install/install_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -521,7 +521,7 @@ func TestInstall(t *testing.T) {
},
messages: []string{
"\n==> Checking if Consul can be installed\n ✓ No existing Consul installations found.\n ✓ No existing Consul persistent volume claims found\n ✓ No existing Consul secrets found.\n ✓ Valid enterprise Consul secret found.\n",
"\n==> Consul Installation Summary\n Name: consul\n Namespace: consul\n \n Helm value overrides\n -------------------\n global:\n enterpriseLicense:\n secretName: consul-license\n \n",
"\n==> Consul Installation Summary\n Name: consul\n Namespace: consul\n \n Helm value overrides\n --------------------\n global:\n enterpriseLicense:\n secretName: consul-license\n \n",
"\n==> Installing Consul\n ✓ Downloaded charts.\n ✓ Consul installed in namespace \"consul\".\n",
},
helmActionsRunner: &helm.MockActionRunner{},
Expand Down Expand Up @@ -558,7 +558,7 @@ func TestInstall(t *testing.T) {
},
messages: []string{
"\n==> Checking if Consul can be installed\n ✓ No existing Consul installations found.\n ✓ No existing Consul persistent volume claims found\n ✓ No existing Consul secrets found.\n",
"\n==> Consul Installation Summary\n Name: consul\n Namespace: consul\n \n Helm value overrides\n -------------------\n connectInject:\n enabled: true\n metrics:\n defaultEnableMerging: true\n defaultEnabled: true\n enableGatewayMetrics: true\n controller:\n enabled: true\n global:\n metrics:\n enableAgentMetrics: true\n enabled: true\n name: consul\n prometheus:\n enabled: true\n server:\n replicas: 1\n ui:\n enabled: true\n service:\n enabled: true\n \n",
"\n==> Consul Installation Summary\n Name: consul\n Namespace: consul\n \n Helm value overrides\n --------------------\n connectInject:\n enabled: true\n metrics:\n defaultEnableMerging: true\n defaultEnabled: true\n enableGatewayMetrics: true\n controller:\n enabled: true\n global:\n metrics:\n enableAgentMetrics: true\n enabled: true\n name: consul\n prometheus:\n enabled: true\n server:\n replicas: 1\n ui:\n enabled: true\n service:\n enabled: true\n \n",
"\n==> Installing Consul\n ✓ Downloaded charts.\n ✓ Consul installed in namespace \"consul\".\n",
},
helmActionsRunner: &helm.MockActionRunner{},
Expand All @@ -574,7 +574,7 @@ func TestInstall(t *testing.T) {
},
messages: []string{
"\n==> Checking if Consul can be installed\n ✓ No existing Consul installations found.\n ✓ No existing Consul persistent volume claims found\n ✓ No existing Consul secrets found.\n",
"\n==> Consul Installation Summary\n Name: consul\n Namespace: consul\n \n Helm value overrides\n -------------------\n connectInject:\n enabled: true\n controller:\n enabled: true\n global:\n acls:\n manageSystemACLs: true\n gossipEncryption:\n autoGenerate: true\n name: consul\n tls:\n enableAutoEncrypt: true\n enabled: true\n server:\n replicas: 1\n \n",
"\n==> Consul Installation Summary\n Name: consul\n Namespace: consul\n \n Helm value overrides\n --------------------\n connectInject:\n enabled: true\n controller:\n enabled: true\n global:\n acls:\n manageSystemACLs: true\n gossipEncryption:\n autoGenerate: true\n name: consul\n tls:\n enableAutoEncrypt: true\n enabled: true\n server:\n replicas: 1\n \n",
"\n==> Installing Consul\n ✓ Downloaded charts.\n ✓ Consul installed in namespace \"consul\".\n",
},
helmActionsRunner: &helm.MockActionRunner{},
Expand Down
228 changes: 202 additions & 26 deletions cli/cmd/uninstall/uninstall.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package uninstall
import (
"fmt"
"os"
"strings"
"sync"
"time"

Expand All @@ -11,12 +12,19 @@ import (
"github.com/hashicorp/consul-k8s/cli/common/flag"
"github.com/hashicorp/consul-k8s/cli/common/terminal"
"github.com/hashicorp/consul-k8s/cli/helm"

"github.com/posener/complete"
"golang.org/x/text/cases"
"golang.org/x/text/language"
"helm.sh/helm/v3/pkg/action"
helmCLI "helm.sh/helm/v3/pkg/cli"
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiext "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
)

Expand All @@ -34,7 +42,7 @@ const (
defaultWipeData = false

flagTimeout = "timeout"
defaultTimeout = "10m"
defaultTimeout = 10 * time.Minute

flagContext = "context"
flagKubeconfig = "kubeconfig"
Expand All @@ -45,16 +53,18 @@ type Command struct {

helmActionsRunner helm.HelmActionsRunner

// Configuration for interacting with Kubernetes.
t-eckert marked this conversation as resolved.
Show resolved Hide resolved
kubernetes kubernetes.Interface
dynamic dynamic.Interface
apiext apiext.Interface
t-eckert marked this conversation as resolved.
Show resolved Hide resolved

set *flag.Sets

flagNamespace string
flagReleaseName string
flagAutoApprove bool
flagWipeData bool
flagTimeout string
timeoutDuration time.Duration
flagTimeout time.Duration
t-eckert marked this conversation as resolved.
Show resolved Hide resolved

flagKubeConfig string
flagKubeContext string
Expand Down Expand Up @@ -90,7 +100,7 @@ func (c *Command) init() {
Default: defaultAnyReleaseName,
Usage: "Name of the installation. This can be used to uninstall and/or delete the resources of a specific Helm release.",
})
f.StringVar(&flag.StringVar{
f.DurationVar(&flag.DurationVar{
t-eckert marked this conversation as resolved.
Show resolved Hide resolved
Name: flagTimeout,
Target: &c.flagTimeout,
Default: defaultTimeout,
Expand Down Expand Up @@ -144,12 +154,6 @@ func (c *Command) Run(args []string) int {
c.UI.Output("Can't set -wipe-data alone. Omit this flag to interactively uninstall, or use it with -auto-approve to wipe all data during the uninstall.", terminal.WithErrorStyle())
return 1
}
duration, err := time.ParseDuration(c.flagTimeout)
if err != nil {
c.UI.Output("unable to parse -%s: %s", flagTimeout, err, terminal.WithErrorStyle())
return 1
}
c.timeoutDuration = duration

// helmCLI.New() will create a settings object which is used by the Helm Go SDK calls.
settings := helmCLI.New()
Expand All @@ -160,20 +164,8 @@ func (c *Command) Run(args []string) int {
settings.KubeContext = c.flagKubeContext
}

// Set up the kubernetes client to use for non Helm SDK calls to the Kubernetes API
// The Helm SDK will use settings.RESTClientGetter for its calls as well, so this will
// use a consistent method to target the right cluster for both Helm SDK and non Helm SDK calls.
if c.kubernetes == nil {
restConfig, err := settings.RESTClientGetter().ToRESTConfig()
if err != nil {
c.UI.Output("retrieving Kubernetes auth: %v", err, terminal.WithErrorStyle())
return 1
}
c.kubernetes, err = kubernetes.NewForConfig(restConfig)
if err != nil {
c.UI.Output("initializing Kubernetes client: %v", err, terminal.WithErrorStyle())
return 1
}
if err := c.initKubernetes(settings); err != nil {
t-eckert marked this conversation as resolved.
Show resolved Hide resolved
c.UI.Output("Could not initialize Kubernetes client: %v", err, terminal.WithErrorStyle())
}

// Setup logger to stream Helm library logs.
Expand All @@ -183,7 +175,7 @@ func (c *Command) Run(args []string) int {
}

actionConfig := new(action.Configuration)
actionConfig, err = helm.InitActionConfig(actionConfig, c.flagNamespace, settings, uiLogger)
actionConfig, err := helm.InitActionConfig(actionConfig, c.flagNamespace, settings, uiLogger)
if err != nil {
c.UI.Output(err.Error(), terminal.WithErrorStyle())
return 1
Expand Down Expand Up @@ -321,6 +313,36 @@ func (c *Command) Run(args []string) int {
return 0
}

// initKubernetes sets up the kubernetes clients to use for non Helm SDK calls to the Kubernetes API
t-eckert marked this conversation as resolved.
Show resolved Hide resolved
// The Helm SDK will use settings.RESTClientGetter for its calls as well, so this will
// use a consistent method to target the right cluster for both Helm SDK and non Helm SDK calls.
func (c *Command) initKubernetes(settings *helmCLI.EnvSettings) error {
restConfig, err := settings.RESTClientGetter().ToRESTConfig()
t-eckert marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return err
}

if c.kubernetes == nil {
if c.kubernetes, err = kubernetes.NewForConfig(restConfig); err != nil {
return err
}
}

if c.dynamic == nil {
if c.dynamic, err = dynamic.NewForConfig(restConfig); err != nil {
return err
}
}

if c.apiext == nil {
if c.apiext, err = apiext.NewForConfig(restConfig); err != nil {
return err
}
}

return nil
}

func (c *Command) uninstallHelmRelease(releaseName, namespace, releaseType string, settings *helmCLI.EnvSettings,
uiLogger action.DebugLog, actionConfig *action.Configuration) error {
c.UI.Output(fmt.Sprintf("Existing %s installation found.", releaseType), terminal.WithSuccessStyle())
Expand All @@ -345,13 +367,60 @@ func (c *Command) uninstallHelmRelease(releaseName, namespace, releaseType strin
}
}

// Delete any custom resources managed by Consul. If they cannot be deleted,
// patch the finalizers to be empty on each one.
if releaseType == common.ReleaseTypeConsul {
uiLogger("Deleting custom resources managed by Consul")
crds, err := c.fetchCustomResourceDefinitions()
if err != nil {
return fmt.Errorf("unable to fetch Custom Resource Definitions for Consul deployment: %v", err)
}
kindToResource := mapCRKindToResourceName(crds)
t-eckert marked this conversation as resolved.
Show resolved Hide resolved
err = backoff.Retry(func() error {
crs, err := c.fetchCustomResources(crds)
if err != nil {
return err
}
if len(crs) == 0 {
return nil
}

if err = c.deleteCustomResources(crs, kindToResource); err != nil {
return err
}

crs, err = c.fetchCustomResources(crds)
if err != nil {
return err
}
if len(crs) != 0 {
return common.NewDeletionError(fmt.Sprintf("%d custom resources remain after deletion request. Retrying deletion", len(crs)))
}
t-eckert marked this conversation as resolved.
Show resolved Hide resolved

return nil
}, backoff.WithMaxRetries(backoff.NewConstantBackOff(time.Second), 5))
if common.IsDeletionError(err) {
uiLogger("Patching finalizers on custom resources managed by Consul")
crs, err := c.fetchCustomResources(crds)
if err != nil {
return err
}

if err = c.patchCustomResources(crs, kindToResource); err != nil {
return err
}
t-eckert marked this conversation as resolved.
Show resolved Hide resolved
} else if err != nil {
return err
}
}

actionConfig, err := helm.InitActionConfig(actionConfig, namespace, settings, uiLogger)
if err != nil {
return err
}

uninstall := action.NewUninstall(actionConfig)
uninstall.Timeout = c.timeoutDuration
uninstall.Timeout = c.flagTimeout

res, err := c.helmActionsRunner.Uninstall(uninstall, releaseName)
if err != nil {
Expand All @@ -361,10 +430,105 @@ func (c *Command) uninstallHelmRelease(releaseName, namespace, releaseType strin
c.UI.Output("Uninstall result: %s", res.Info, terminal.WithInfoStyle())
return nil
}

c.UI.Output(fmt.Sprintf("Successfully uninstalled %s Helm release.", releaseType), terminal.WithSuccessStyle())
return nil
}

// fetchCustomResourceDefinitions fetches all Custom Resource Definitions managed by Consul.
func (c *Command) fetchCustomResourceDefinitions() (*apiextv1.CustomResourceDefinitionList, error) {
return c.apiext.ApiextensionsV1().CustomResourceDefinitions().List(c.Ctx, metav1.ListOptions{
LabelSelector: "app=consul",
})
}

// fetchCustomResources gets a list of all custom resources deployed in the
// cluster that are managed by Consul.
func (c *Command) fetchCustomResources(crds *apiextv1.CustomResourceDefinitionList) ([]unstructured.Unstructured, error) {
crs := make([]unstructured.Unstructured, 0)
for _, crd := range crds.Items {
for _, version := range crd.Spec.Versions {
target := schema.GroupVersionResource{
Group: crd.Spec.Group,
Version: version.Name,
Resource: crd.Spec.Names.Plural,
}

crList, err := c.dynamic.Resource(target).List(c.Ctx, metav1.ListOptions{})
if err != nil {
return nil, err
}
if crList != nil {
crs = append(crs, crList.Items...)
}
}
}

return crs, nil
}

// deleteCustomResources takes a list of unstructured custom resources and
// sends a request to each one to be deleted.
func (c *Command) deleteCustomResources(crs []unstructured.Unstructured, kindToResource map[string]string) error {
for _, cr := range crs {
t-eckert marked this conversation as resolved.
Show resolved Hide resolved
apiVersion := strings.Split(cr.GetAPIVersion(), "/")
group, version := apiVersion[0], apiVersion[1]
if group == "" || version == "" {
return fmt.Errorf("malformed api version: %s", apiVersion)
}
t-eckert marked this conversation as resolved.
Show resolved Hide resolved

target := schema.GroupVersionResource{
Group: group,
Version: version,
Resource: kindToResource[cr.GetKind()],
}

err := c.dynamic.
Resource(target).
Namespace(cr.GetNamespace()).
Delete(c.Ctx, cr.GetName(), metav1.DeleteOptions{})
if common.IgnoreNotFoundError(err) != nil {
t-eckert marked this conversation as resolved.
Show resolved Hide resolved
return err
}
}

return nil
}

// patchCustomResources takes a list of unstructured custom resources and
// sends a request to each one to patch its finalizers to an empty list.
func (c *Command) patchCustomResources(crs []unstructured.Unstructured, kindToResource map[string]string) error {
finalizerPatch := []byte(`[{
"op": "replace",
"path": "/metadata/finalizers",
"value": []
}]`)

for _, cr := range crs {
apiVersion := strings.Split(cr.GetAPIVersion(), "/")
group, version := apiVersion[0], apiVersion[1]
if group == "" || version == "" {
return fmt.Errorf("malformed api version: %s", apiVersion)
}

target := schema.GroupVersionResource{
Group: group,
Version: version,
Resource: kindToResource[cr.GetKind()],
}

_, err := c.dynamic.
Resource(target).
Namespace(cr.GetNamespace()).
Patch(c.Ctx, cr.GetName(), types.JSONPatchType, finalizerPatch, metav1.PatchOptions{})
if common.IgnoreNotFoundError(err) != nil {
t-eckert marked this conversation as resolved.
Show resolved Hide resolved
return err
}
}

return nil
}

func (c *Command) Help() string {
c.once.Do(c.init)
s := "Usage: consul-k8s uninstall [flags]" + "\n" + "Uninstall Consul with options to delete data and resources associated with Consul installation." + "\n\n" + c.help
Expand Down Expand Up @@ -649,3 +813,15 @@ func (c *Command) deleteClusterRoleBindings(foundReleaseName string) error {
}
return nil
}

// mapCRKindToResourceName takes the list of custom resource definitions and
// creates a mapping from the "kind" of the CRD to its "resource" name.
// This is needed for the dynamic API which finds custom resources by their
// lowercase, plural resource name. (e.g. "ingressgateways" for "IngressGateway" kind).
func mapCRKindToResourceName(crds *apiextv1.CustomResourceDefinitionList) map[string]string {
kindToResourceName := make(map[string]string)
for _, crd := range crds.Items {
kindToResourceName[crd.Spec.Names.Kind] = crd.Spec.Names.Plural
}
return kindToResourceName
}
Loading