delete remnant route entries

controllers/provider-gcp/pkg/controller/infrastructure/actuator_delete.go

@@ -16,6 +16,8 @@ package infrastructure
 
 import (
 	"context"
+	"time"
+
 	gcpv1alpha1 "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/apis/gcp/v1alpha1"
 	"github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/internal"
 	gcpclient "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/internal/client"
@@ -24,7 +26,6 @@ import (
 	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
 	"github.com/gardener/gardener/pkg/operation/terraformer"
 	"github.com/gardener/gardener/pkg/utils/flow"
-	"time"
 )
 
 func (a *actuator) cleanupKubernetesFirewallRules(
@@ -45,6 +46,25 @@ func (a *actuator) cleanupKubernetesFirewallRules(
 	return infrastructure.CleanupKubernetesFirewalls(ctx, client, account.ProjectID, state.VPCName)
 }
 
+func (a *actuator) cleanupKubernetesRoutes(
+	ctx context.Context,
+	config *gcpv1alpha1.InfrastructureConfig,
+	client gcpclient.Interface,
+	tf *terraformer.Terraformer,
+	account *internal.ServiceAccount,
+	namespace string,
+) error {
+	state, err := infrastructure.ExtractTerraformState(tf, config)
+	if err != nil {
+		if terraformer.IsVariablesNotFoundError(err) {
+			return nil
+		}
+		return err
+	}
+
+	return infrastructure.CleanupKubernetesRoutes(ctx, client, account.ProjectID, state.VPCName, namespace)
+}
+
 // Delete implements infrastructure.Actuator.
 func (a *actuator) Delete(ctx context.Context, infra *extensionsv1alpha1.Infrastructure, cluster *controller.Cluster) error {
 	config, err := internal.InfrastructureConfigFromInfrastructure(infra)
@@ -72,6 +92,8 @@ func (a *actuator) Delete(ctx context.Context, infra *extensionsv1alpha1.Infrast
 		return err
 	}
 
+	namespace := infra.GetNamespace()
+
 	var (
 		g                              = flow.NewGraph("GCP infrastructure destruction")
 		destroyKubernetesFirewallRules = g.Add(flow.Task{
@@ -83,10 +105,19 @@ func (a *actuator) Delete(ctx context.Context, infra *extensionsv1alpha1.Infrast
 				DoIf(configExists),
 		})
 
+		destroyKubernetesRoutes = g.Add(flow.Task{
+			Name: "Destroying Kubernetes route entries",
+			Fn: flow.TaskFn(func(ctx context.Context) error {
+				return a.cleanupKubernetesRoutes(ctx, config, gcpClient, tf, serviceAccount, namespace)
+			}).
+				RetryUntilTimeout(10*time.Second, 5*time.Minute).
+				DoIf(configExists),
+		})
+
 		_ = g.Add(flow.Task{
 			Name:         "Destroying Shoot infrastructure",
 			Fn:           flow.SimpleTaskFn(tf.Destroy),
-			Dependencies: flow.NewTaskIDs(destroyKubernetesFirewallRules),
+			Dependencies: flow.NewTaskIDs(destroyKubernetesFirewallRules, destroyKubernetesRoutes),
 		})
 
 		f = g.Compile()

controllers/provider-gcp/pkg/internal/client/client.go

@@ -16,6 +16,7 @@ package gcp
 
 import (
 	"context"
+
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/compute/v1"
@@ -30,14 +31,26 @@ type firewallsService struct {
 	firewallsService *compute.FirewallsService
 }
 
+type routesService struct {
+	routesService *compute.RoutesService
+}
+
 type firewallsListCall struct {
 	firewallsListCall *compute.FirewallsListCall
 }
 
+type routesListCall struct {
+	routesListCall *compute.RoutesListCall
+}
+
 type firewallsDeleteCall struct {
 	firewallsDeleteCall *compute.FirewallsDeleteCall
 }
 
+type routesDeleteCall struct {
+	routesDeleteCall *compute.RoutesDeleteCall
+}
+
 // NewFromServiceAccount creates a new client from the given service account.
 func NewFromServiceAccount(ctx context.Context, serviceAccount []byte) (Interface, error) {
 	jwt, err := google.JWTConfigFromJSON(serviceAccount, compute.CloudPlatformScope)
@@ -64,27 +77,57 @@ func (c *client) Firewalls() FirewallsService {
 	return &firewallsService{c.service.Firewalls}
 }
 
+// Routes implements Interface.
+func (c *client) Routes() RoutesService {
+	return &routesService{c.service.Routes}
+}
+
 // List implements FirewallsService.
 func (f *firewallsService) List(projectID string) FirewallsListCall {
 	return &firewallsListCall{f.firewallsService.List(projectID)}
 }
 
+// List implements RoutesService.
+func (r *routesService) List(projectID string) RoutesListCall {
+	return &routesListCall{r.routesService.List(projectID)}
+}
+
 // Pages implements FirewallsListCall.
 func (c *firewallsListCall) Pages(ctx context.Context, f func(*compute.FirewallList) error) error {
 	return c.firewallsListCall.Pages(ctx, f)
 }
 
+// Pages implements RoutesListCall.
+func (c *routesListCall) Pages(ctx context.Context, f func(*compute.RouteList) error) error {
+	return c.routesListCall.Pages(ctx, f)
+}
+
 // Delete implements FirewallsService.
 func (f *firewallsService) Delete(projectID, firewall string) FirewallsDeleteCall {
 	return &firewallsDeleteCall{f.firewallsService.Delete(projectID, firewall)}
 }
 
+// Delete implements RoutesService.
+func (r *routesService) Delete(projectID, route string) RoutesDeleteCall {
+	return &routesDeleteCall{r.routesService.Delete(projectID, route)}
+}
+
 // Context implements FirewallsDeleteCall.
 func (c *firewallsDeleteCall) Context(ctx context.Context) FirewallsDeleteCall {
 	return &firewallsDeleteCall{c.firewallsDeleteCall.Context(ctx)}
 }
 
+// Context implements RoutesDeleteCall.
+func (c *routesDeleteCall) Context(ctx context.Context) RoutesDeleteCall {
+	return &routesDeleteCall{c.routesDeleteCall.Context(ctx)}
+}
+
 // Do implements FirewallsDeleteCall.
 func (c *firewallsDeleteCall) Do(opts ...googleapi.CallOption) (*compute.Operation, error) {
 	return c.firewallsDeleteCall.Do(opts...)
 }
+
+// Do implements RoutesDeleteCall.
+func (c *routesDeleteCall) Do(opts ...googleapi.CallOption) (*compute.Operation, error) {
+	return c.routesDeleteCall.Do(opts...)
+}

controllers/provider-gcp/pkg/internal/client/types.go

@@ -16,6 +16,7 @@ package gcp
 
 import (
 	"context"
+
 	"google.golang.org/api/compute/v1"
 	"google.golang.org/api/googleapi"
 )
@@ -24,6 +25,8 @@ import (
 type Interface interface {
 	// Firewalls retrieves the GCP firewalls service.
 	Firewalls() FirewallsService
+	// Routes retrieves the GCP routes service.
+	Routes() RoutesService
 }
 
 // FirewallsService is the interface for the GCP firewalls service.
@@ -34,16 +37,38 @@ type FirewallsService interface {
 	Delete(projectID, firewall string) FirewallsDeleteCall
 }
 
+// RoutesService is the interface for the GCP routes service.
+type RoutesService interface {
+	// List initiates a RoutesListCall.
+	List(projectID string) RoutesListCall
+	// Delete initiates a RoutesDeleteCall.
+	Delete(projectID, route string) RoutesDeleteCall
+}
+
 // FirewallsListCall is a list call to the firewalls service.
 type FirewallsListCall interface {
 	// Pages runs the given function on the paginated result of listing the firewalls.
 	Pages(context.Context, func(*compute.FirewallList) error) error
 }
 
+// RoutesListCall is a list call to the routes service.
+type RoutesListCall interface {
+	// Pages runs the given function on the paginated result of listing the routes.
+	Pages(context.Context, func(*compute.RouteList) error) error
+}
+
 // FirewallsDeleteCall is a delete call to the firewalls service.
 type FirewallsDeleteCall interface {
 	// Do executes the deletion call.
 	Do(opts ...googleapi.CallOption) (*compute.Operation, error)
 	// Context sets the context for the deletion call.
 	Context(context.Context) FirewallsDeleteCall
 }
+
+// RoutesDeleteCall is a delete call to the routes service.
+type RoutesDeleteCall interface {
+	// Do executes the deletion call.
+	Do(opts ...googleapi.CallOption) (*compute.Operation, error)
+	// Context sets the context for the deletion call.
+	Context(context.Context) RoutesDeleteCall
+}

controllers/provider-gcp/pkg/internal/infrastructure/infrastructure.go

@@ -16,12 +16,13 @@ package infrastructure
 
 import (
 	"context"
+	"strings"
+
 	"github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/internal"
 	gcpclient "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/internal/client"
 	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
 	"google.golang.org/api/compute/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"strings"
 )
 
 // KubernetesFirewallNamePrefix is the name prefix that Kubernetes related firewall rules have.
@@ -44,6 +45,22 @@ func ListKubernetesFirewalls(ctx context.Context, client gcpclient.Interface, pr
 	return names, nil
 }
 
+// ListKubernetesRoutes returns a list of all routes within the shoot network which have the namespace as prefix.
+func ListKubernetesRoutes(ctx context.Context, client gcpclient.Interface, projectID, network, namespace string) ([]string, error) {
+	var routes []string
+	if err := client.Routes().List(projectID).Pages(ctx, func(page *compute.RouteList) error {
+		for _, route := range page.Items {
+			if strings.HasPrefix(route.Name, namespace) {
+				routes = append(routes, route.Name)
+			}
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return routes, nil
+}
+
 // DeleteFirewalls deletes the firewalls with the given names in the given project.
 //
 // If a deletion fails, it immediately returns the error of that deletion.
@@ -56,6 +73,18 @@ func DeleteFirewalls(ctx context.Context, client gcpclient.Interface, projectID
 	return nil
 }
 
+// DeleteRoutes deletes the route entries with the given names in the given project.
+//
+// If a deletion fails, it immediately returns the error of that deletion.
+func DeleteRoutes(ctx context.Context, client gcpclient.Interface, projectID string, routes []string) error {
+	for _, route := range routes {
+		if _, err := client.Routes().Delete(projectID, route).Context(ctx).Do(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // CleanupKubernetesFirewalls lists all Kubernetes firewall rules and then deletes them one after another.
 //
 // If a deletion fails, this method returns immediately with the encountered error.
@@ -68,6 +97,18 @@ func CleanupKubernetesFirewalls(ctx context.Context, client gcpclient.Interface,
 	return DeleteFirewalls(ctx, client, projectID, firewallNames)
 }
 
+// CleanupKubernetesRoutes lists all Kubernetes route rules and then deletes them one after another.
+//
+// If a deletion fails, this method returns immediately with the encountered error.
+func CleanupKubernetesRoutes(ctx context.Context, client gcpclient.Interface, projectID, network, namespace string) error {
+	routeNames, err := ListKubernetesRoutes(ctx, client, projectID, network, namespace)
+	if err != nil {
+		return err
+	}
+
+	return DeleteRoutes(ctx, client, projectID, routeNames)
+}
+
 // GetServiceAccountFromInfrastructure retrieves the ServiceAccount from the Secret referenced in the given Infrastructure.
 func GetServiceAccountFromInfrastructure(ctx context.Context, c client.Client, config *extensionsv1alpha1.Infrastructure) (*internal.ServiceAccount, error) {
 	return internal.GetServiceAccount(ctx, c, config.Spec.SecretRef.Namespace, config.Spec.SecretRef.Name)

controllers/provider-gcp/pkg/internal/infrastructure/infrastructure_test.go

@@ -17,12 +17,13 @@ package infrastructure
 import (
 	"context"
 	"fmt"
+	"testing"
+
 	mockgcpclient "github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/internal/mock/client"
 	"github.com/golang/mock/gomock"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"google.golang.org/api/compute/v1"
-	"testing"
 )
 
 func TestActuator(t *testing.T) {
@@ -76,6 +77,42 @@ var _ = Describe("Infrastructure", func() {
 		})
 	})
 
+	Describe("#ListKubernetesRoutes", func() {
+		It("should list all kubernetes related route names with the shoot namespace as prefix", func() {
+			var (
+				ctx       = context.TODO()
+				projectID = "foo"
+				network   = "shoot--foo--bar"
+				namespace = "shoot--foo--bar"
+
+				routeName  = fmt.Sprintf("shoot--foo--bar-2690fa98-450f-11e9-8ebe-ce2a79d67b14")
+				routeNames = []string{routeName}
+
+				client         = mockgcpclient.NewMockInterface(ctrl)
+				routes         = mockgcpclient.NewMockRoutesService(ctrl)
+				routesListCall = mockgcpclient.NewMockRoutesListCall(ctrl)
+			)
+
+			gomock.InOrder(
+				client.EXPECT().Routes().Return(routes),
+				routes.EXPECT().List(projectID).Return(routesListCall),
+				routesListCall.EXPECT().Pages(ctx, gomock.AssignableToTypeOf(func(*compute.RouteList) error { return nil })).
+					DoAndReturn(func(_ context.Context, f func(*compute.RouteList) error) error {
+						return f(&compute.RouteList{
+							Items: []*compute.Route{
+								{Name: routeName, Network: network},
+							},
+						})
+					}),
+			)
+
+			actual, err := ListKubernetesRoutes(ctx, client, projectID, network, namespace)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(actual).To(Equal(routeNames))
+		})
+	})
+
 	Describe("#DeleteFirewalls", func() {
 		It("should delete all firewalls", func() {
 			var (
@@ -100,4 +137,29 @@ var _ = Describe("Infrastructure", func() {
 			Expect(DeleteFirewalls(ctx, client, projectID, firewallNames)).To(Succeed())
 		})
 	})
+
+	Describe("#DeleteRoutes", func() {
+		It("should delete all routess", func() {
+			var (
+				ctx       = context.TODO()
+				projectID = "foo"
+
+				routeName  = fmt.Sprintf("shoot--foo--bar-2690fa98-450f-11e9-8ebe-ce2a79d67b14")
+				routeNames = []string{routeName}
+
+				client           = mockgcpclient.NewMockInterface(ctrl)
+				routes           = mockgcpclient.NewMockRoutesService(ctrl)
+				routesDeleteCall = mockgcpclient.NewMockRoutesDeleteCall(ctrl)
+			)
+
+			gomock.InOrder(
+				client.EXPECT().Routes().Return(routes),
+				routes.EXPECT().Delete(projectID, routeName).Return(routesDeleteCall),
+				routesDeleteCall.EXPECT().Context(ctx).Return(routesDeleteCall),
+				routesDeleteCall.EXPECT().Do(),
+			)
+
+			Expect(DeleteRoutes(ctx, client, projectID, routeNames)).To(Succeed())
+		})
+	})
 })

controllers/provider-gcp/pkg/internal/mock/client/doc.go

@@ -1,3 +1,3 @@
-//go:generate mockgen -package=client -destination=mocks.go github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/internal/client Interface,FirewallsService,FirewallsListCall,FirewallsDeleteCall
+//go:generate mockgen -package=client -destination=mocks.go github.com/gardener/gardener-extensions/controllers/provider-gcp/pkg/internal/client Interface,FirewallsService,RoutesService,FirewallsListCall,RoutesListCall,FirewallsDeleteCall,RoutesDeleteCall
 
 package client