Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

APIGW4CONK8s: Add Consul Cache #2118

Merged
merged 13 commits into from
May 15, 2023
Merged

APIGW4CONK8s: Add Consul Cache #2118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
dcdd52e
Added basic cache functionality with most tests, todo: add get method
jm96441n May 8, 2023
3d216f9
Updated tests for Cache.Run function, removed tests of unexported
jm96441n May 9, 2023
446bd88
Moved translation function def to translation package, added translate
jm96441n May 9, 2023
39fb469
Add translation for consul config entries to k8s namespaced name meta
jm96441n May 10, 2023
00f97d0
Added Get method to cache
jm96441n May 10, 2023
9f7b0fe
Add watch for contoller and setup in inject command
jm96441n May 10, 2023
215891c
Updated comments, renamed TranslateConsulInlineSecret method to
jm96441n May 10, 2023
56575d9
Updates from PR review
jm96441n May 12, 2023
c2d5921
Parallelize tests
jm96441n May 12, 2023
c074b0f
Bump consul api version
jm96441n May 12, 2023
7dcbe18
Set api timeout for cache calls
jm96441n May 15, 2023
9fa1dfe
Revert "Bump consul api version"
jm96441n May 15, 2023
d17db20
Linting fun
jm96441n May 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 57 additions & 8 deletions control-plane/api-gateway/controllers/gateway_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package controllers

import (
"context"

"github.com/go-logr/logr"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand All @@ -16,6 +17,11 @@ import (
"sigs.k8s.io/controller-runtime/pkg/source"
gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"

"github.com/hashicorp/consul-k8s/control-plane/api-gateway/translation"
"github.com/hashicorp/consul-k8s/control-plane/cache"
"github.com/hashicorp/consul-k8s/control-plane/consul"
"github.com/hashicorp/consul/api"
)

const (
Expand All @@ -24,10 +30,19 @@ const (
kindGateway = "Gateway"
)

// GatewayControllerConfig holds the values necessary for configuring the GatewayController.
type GatewayControllerConfig struct {
ConsulClientConfig *consul.Config
ConsulServerConnMgr consul.ServerConnectionManager
NamespacesEnabled bool
Partition string
}

// GatewayController reconciles a Gateway object.
// The Gateway is responsible for defining the behavior of API gateways.
type GatewayController struct {
Log logr.Logger
cache *cache.Cache
Log logr.Logger
client.Client
}

Expand Down Expand Up @@ -78,7 +93,7 @@ func (r *GatewayController) Reconcile(ctx context.Context, req ctrl.Request) (ct
return ctrl.Result{}, nil
}

//TODO: serialize gatewayClassConfig onto Gateway.
// TODO: serialize gatewayClassConfig onto Gateway.

didUpdate, err := EnsureFinalizer(ctx, r.Client, gw, gatewayFinalizer)
if err != nil {
Expand All @@ -90,14 +105,30 @@ func (r *GatewayController) Reconcile(ctx context.Context, req ctrl.Request) (ct
return ctrl.Result{}, nil
}

//TODO: Handle reconciliation.
// TODO: Handle reconciliation.

return ctrl.Result{}, nil
}

// SetupWithManager registers the controller with the given manager.
func (r *GatewayController) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
// SetupWithGatewayControllerManager registers the controller with the given manager.
func SetupGatewayControllerWithManager(ctx context.Context, mgr ctrl.Manager, config GatewayControllerConfig) (*cache.Cache, error) {
c := cache.New(cache.Config{
ConsulClientConfig: config.ConsulClientConfig,
ConsulServerConnMgr: config.ConsulServerConnMgr,
NamespacesEnabled: config.NamespacesEnabled,
Partition: config.Partition,
Logger: mgr.GetLogger(),
})

r := &GatewayController{
Client: mgr.GetClient(),
cache: c,
Log: mgr.GetLogger(),
}

translator := translation.NewConsulToNamespaceNameTranslator(c)

return c, ctrl.NewControllerManagedBy(mgr).
For(&gwv1beta1.Gateway{}).
Owns(&appsv1.Deployment{}).
Owns(&corev1.Service{}).
Expand All @@ -122,8 +153,26 @@ func (r *GatewayController) SetupWithManager(ctx context.Context, mgr ctrl.Manag
source.NewKindWithCache(&gwv1beta1.ReferenceGrant{}, mgr.GetCache()),
handler.EnqueueRequestsFromMapFunc(r.transformReferenceGrant(ctx)),
).
// TODO: Watches for consul resources.
Complete(r)
Watches(
// Subscribe to changes from Consul for APIGateways
&source.Channel{Source: c.Subscribe(ctx, api.APIGateway, translator.BuildConsulGatewayTranslator(ctx)).Events()},
&handler.EnqueueRequestForObject{},
).
Watches(
// Subscribe to changes from Consul for HTTPRoutes
&source.Channel{Source: c.Subscribe(ctx, api.APIGateway, translator.BuildConsulHTTPRouteTranslator(ctx)).Events()},
&handler.EnqueueRequestForObject{},
).
Watches(
// Subscribe to changes from Consul for TCPRoutes
&source.Channel{Source: c.Subscribe(ctx, api.APIGateway, translator.BuildConsulTCPRouteTranslator(ctx)).Events()},
&handler.EnqueueRequestForObject{},
).
Watches(
// Subscribe to changes from Consul for InlineCertificates
&source.Channel{Source: c.Subscribe(ctx, api.InlineCertificate, translator.BuildConsulInlineCertificateTranslator(ctx, r.transformSecret)).Events()},
&handler.EnqueueRequestForObject{},
).Complete(r)
}

// transformGatewayClass will check the list of GatewayClass objects for a matching
Expand Down
32 changes: 23 additions & 9 deletions control-plane/api-gateway/translation/config_entry_translation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ const (
metaValueManagedBy = "consul-k8s-gateway-controller"
metaKeyKubeNS = "k8s-namespace"
metaKeyKubeServiceName = "k8s-service-name"
metaKeyKubeName = "k8s-name"
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this came from the POC implementation

consul-k8s/control-plane/controller/apigateway_controller.go

Line 416 in 408114a

metaKeyKubeName = "k8s-name"
, I'm not 100% on which we use as metaKeyKubeService name elsewhere in this repo (or do we use both keys in the meta?)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we can keep it around and if we get to the end of the implementation and find that we aren't using it, we can toss it.


// AnnotationGateway is the annotation used to override the gateway name.
AnnotationGateway = "consul.hashicorp.com/gateway"
Expand All @@ -37,8 +38,8 @@ type consulIdentifier struct {
partition string
}

// Translator handles translating K8s resources into Consul config entries.
type Translator struct {
// K8sToConsulTranslator handles translating K8s resources into Consul config entries.
type K8sToConsulTranslator struct {
EnableConsulNamespaces bool
ConsulDestNamespace string
EnableK8sMirroring bool
Expand All @@ -47,7 +48,7 @@ type Translator struct {
}

// GatewayToAPIGateway translates a kuberenetes API gateway into a Consul APIGateway Config Entry.
func (t Translator) GatewayToAPIGateway(k8sGW gwv1beta1.Gateway, certs map[types.NamespacedName]consulIdentifier) capi.APIGatewayConfigEntry {
func (t K8sToConsulTranslator) GatewayToAPIGateway(k8sGW gwv1beta1.Gateway, certs map[types.NamespacedName]consulIdentifier) capi.APIGatewayConfigEntry {
listeners := make([]capi.APIGatewayListener, 0, len(k8sGW.Spec.Listeners))
for _, listener := range k8sGW.Spec.Listeners {
certificates := make([]capi.ResourceReference, 0, len(listener.TLS.CertificateRefs))
Expand Down Expand Up @@ -96,6 +97,7 @@ func (t Translator) GatewayToAPIGateway(k8sGW gwv1beta1.Gateway, certs map[types
metaKeyManagedBy: metaValueManagedBy,
metaKeyKubeNS: k8sGW.GetObjectMeta().GetNamespace(),
metaKeyKubeServiceName: k8sGW.GetObjectMeta().GetName(),
metaKeyKubeName: k8sGW.GetObjectMeta().GetName(),
},
Listeners: listeners,
Partition: t.ConsulPartition,
Expand All @@ -104,7 +106,7 @@ func (t Translator) GatewayToAPIGateway(k8sGW gwv1beta1.Gateway, certs map[types
}

// HTTPRouteToHTTPRoute translates a k8s HTTPRoute into a Consul HTTPRoute Config Entry.
func (t Translator) HTTPRouteToHTTPRoute(k8sHTTPRoute gwv1beta1.HTTPRoute, parentRefs map[types.NamespacedName]consulIdentifier) capi.HTTPRouteConfigEntry {
func (t K8sToConsulTranslator) HTTPRouteToHTTPRoute(k8sHTTPRoute gwv1beta1.HTTPRoute, parentRefs map[types.NamespacedName]consulIdentifier) capi.HTTPRouteConfigEntry {
routeName := k8sHTTPRoute.Name
if routeNameFromAnnotation, ok := k8sHTTPRoute.Annotations[AnnotationHTTPRoute]; ok && routeNameFromAnnotation != "" && !strings.Contains(routeNameFromAnnotation, ",") {
routeName = routeNameFromAnnotation
Expand All @@ -117,6 +119,7 @@ func (t Translator) HTTPRouteToHTTPRoute(k8sHTTPRoute gwv1beta1.HTTPRoute, paren
metaKeyManagedBy: metaValueManagedBy,
metaKeyKubeNS: k8sHTTPRoute.GetObjectMeta().GetNamespace(),
metaKeyKubeServiceName: k8sHTTPRoute.GetObjectMeta().GetName(),
metaKeyKubeName: k8sHTTPRoute.GetObjectMeta().GetName(),
},
Partition: t.ConsulPartition,

Expand Down Expand Up @@ -170,7 +173,7 @@ func isRefAPIGateway(ref gwv1beta1.ParentReference) bool {
}

// translate the rules portion of a HTTPRoute.
func (t Translator) translateHTTPRouteRules(k8sRules []gwv1beta1.HTTPRouteRule) []capi.HTTPRouteRule {
func (t K8sToConsulTranslator) translateHTTPRouteRules(k8sRules []gwv1beta1.HTTPRouteRule) []capi.HTTPRouteRule {
rules := make([]capi.HTTPRouteRule, 0, len(k8sRules))
for _, k8sRule := range k8sRules {
rule := capi.HTTPRouteRule{}
Expand Down Expand Up @@ -282,7 +285,7 @@ func translateHTTPFilters(k8sFilters []gwv1beta1.HTTPRouteFilter) capi.HTTPFilte
}

// translate the backendrefs into services.
func (t Translator) translateHTTPServices(k8sBackendRefs []gwv1beta1.HTTPBackendRef) []capi.HTTPService {
func (t K8sToConsulTranslator) translateHTTPServices(k8sBackendRefs []gwv1beta1.HTTPBackendRef) []capi.HTTPService {
services := make([]capi.HTTPService, 0, len(k8sBackendRefs))

for _, k8sRef := range k8sBackendRefs {
Expand All @@ -299,7 +302,7 @@ func (t Translator) translateHTTPServices(k8sBackendRefs []gwv1beta1.HTTPBackend
}

// TCPRouteToTCPRoute translates a Kuberenetes TCPRoute into a Consul TCPRoute Config Entry.
func (t Translator) TCPRouteToTCPRoute(k8sRoute gwv1alpha2.TCPRoute, parentRefs map[types.NamespacedName]consulIdentifier) capi.TCPRouteConfigEntry {
func (t K8sToConsulTranslator) TCPRouteToTCPRoute(k8sRoute gwv1alpha2.TCPRoute, parentRefs map[types.NamespacedName]consulIdentifier) capi.TCPRouteConfigEntry {
routeName := k8sRoute.Name
if routeNameFromAnnotation, ok := k8sRoute.Annotations[AnnotationTCPRoute]; ok && routeNameFromAnnotation != "" && !strings.Contains(routeNameFromAnnotation, ",") {
routeName = routeNameFromAnnotation
Expand All @@ -312,6 +315,7 @@ func (t Translator) TCPRouteToTCPRoute(k8sRoute gwv1alpha2.TCPRoute, parentRefs
metaKeyManagedBy: metaValueManagedBy,
metaKeyKubeNS: k8sRoute.GetObjectMeta().GetNamespace(),
metaKeyKubeServiceName: k8sRoute.GetObjectMeta().GetName(),
metaKeyKubeName: k8sRoute.GetObjectMeta().GetName(),
},
Partition: t.ConsulPartition,

Expand Down Expand Up @@ -342,7 +346,7 @@ func (t Translator) TCPRouteToTCPRoute(k8sRoute gwv1alpha2.TCPRoute, parentRefs
}

// SecretToInlineCertificate translates a Kuberenetes Secret into a Consul Inline Certificate Config Entry.
func (t Translator) SecretToInlineCertificate(k8sSecret gwv1beta1.SecretObjectReference, certs map[types.NamespacedName]consulIdentifier) capi.InlineCertificateConfigEntry {
func (t K8sToConsulTranslator) SecretToInlineCertificate(k8sSecret gwv1beta1.SecretObjectReference, certs map[types.NamespacedName]consulIdentifier) capi.InlineCertificateConfigEntry {
inlineCert := capi.InlineCertificateConfigEntry{Kind: capi.InlineCertificate}

for namespaceName, consulIdentifier := range certs {
Expand All @@ -359,17 +363,27 @@ func (t Translator) SecretToInlineCertificate(k8sSecret gwv1beta1.SecretObjectRe
metaKeyManagedBy: metaValueManagedBy,
metaKeyKubeNS: k8sSecretNS,
metaKeyKubeServiceName: string(k8sSecret.Name),
metaKeyKubeName: string(k8sSecret.Name),
}
return inlineCert
}
}
return inlineCert
}

func (t Translator) getConsulNamespace(k8sNS string) string {
func (t K8sToConsulTranslator) getConsulNamespace(k8sNS string) string {
return namespaces.ConsulNamespace(k8sNS, t.EnableK8sMirroring, t.ConsulDestNamespace, t.EnableK8sMirroring, t.MirroringPrefix)
}

func EntryToReference(entry capi.ConfigEntry) capi.ResourceReference {
return capi.ResourceReference{
Kind: entry.GetKind(),
Name: entry.GetName(),
Partition: entry.GetPartition(),
Namespace: entry.GetNamespace(),
}
}

func ptrTo[T any](v T) *T {
return &v
}
21 changes: 17 additions & 4 deletions control-plane/api-gateway/translation/config_entry_translation_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
)

func TestTranslator_GatewayToAPIGateway(t *testing.T) {
t.Parallel()
k8sObjectName := "my-k8s-gw"
k8sNamespace := "my-k8s-namespace"

Expand Down Expand Up @@ -182,6 +183,7 @@ func TestTranslator_GatewayToAPIGateway(t *testing.T) {
metaKeyManagedBy: metaValueManagedBy,
metaKeyKubeNS: k8sNamespace,
metaKeyKubeServiceName: k8sObjectName,
metaKeyKubeName: k8sObjectName,
},
Listeners: []capi.APIGatewayListener{
{
Expand Down Expand Up @@ -218,7 +220,7 @@ func TestTranslator_GatewayToAPIGateway(t *testing.T) {
Status: capi.ConfigEntryStatus{},
Namespace: k8sNamespace,
}
translator := Translator{
translator := K8sToConsulTranslator{
EnableConsulNamespaces: true,
ConsulDestNamespace: "",
EnableK8sMirroring: true,
Expand Down Expand Up @@ -246,6 +248,7 @@ func TestTranslator_GatewayToAPIGateway(t *testing.T) {
}

func TestTranslator_HTTPRouteToHTTPRoute(t *testing.T) {
t.Parallel()
type args struct {
k8sHTTPRoute gwv1beta1.HTTPRoute
parentRefs map[types.NamespacedName]consulIdentifier
Expand Down Expand Up @@ -465,6 +468,7 @@ func TestTranslator_HTTPRouteToHTTPRoute(t *testing.T) {
metaKeyManagedBy: metaValueManagedBy,
metaKeyKubeNS: "k8s-ns",
metaKeyKubeServiceName: "k8s-http-route",
metaKeyKubeName: "k8s-http-route",
},
Namespace: "k8s-ns",
},
Expand Down Expand Up @@ -682,6 +686,7 @@ func TestTranslator_HTTPRouteToHTTPRoute(t *testing.T) {
metaKeyManagedBy: metaValueManagedBy,
metaKeyKubeNS: "k8s-ns",
metaKeyKubeServiceName: "k8s-http-route",
metaKeyKubeName: "k8s-http-route",
},
Namespace: "k8s-ns",
},
Expand Down Expand Up @@ -899,6 +904,7 @@ func TestTranslator_HTTPRouteToHTTPRoute(t *testing.T) {
metaKeyManagedBy: metaValueManagedBy,
metaKeyKubeNS: "k8s-ns",
metaKeyKubeServiceName: "k8s-http-route",
metaKeyKubeName: "k8s-http-route",
},
Namespace: "k8s-ns",
},
Expand Down Expand Up @@ -1122,6 +1128,7 @@ func TestTranslator_HTTPRouteToHTTPRoute(t *testing.T) {
metaKeyManagedBy: metaValueManagedBy,
metaKeyKubeNS: "k8s-ns",
metaKeyKubeServiceName: "k8s-http-route",
metaKeyKubeName: "k8s-http-route",
},
Namespace: "k8s-ns",
},
Expand Down Expand Up @@ -1336,14 +1343,15 @@ func TestTranslator_HTTPRouteToHTTPRoute(t *testing.T) {
metaKeyManagedBy: metaValueManagedBy,
metaKeyKubeNS: "k8s-ns",
metaKeyKubeServiceName: "k8s-http-route",
metaKeyKubeName: "k8s-http-route",
},
Namespace: "k8s-ns",
},
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
tr := Translator{
tr := K8sToConsulTranslator{
EnableConsulNamespaces: true,
EnableK8sMirroring: true,
}
Expand All @@ -1356,6 +1364,7 @@ func TestTranslator_HTTPRouteToHTTPRoute(t *testing.T) {
}

func TestTranslator_TCPRouteToTCPRoute(t *testing.T) {
t.Parallel()
type args struct {
k8sRoute gwv1alpha2.TCPRoute
parentRefs map[types.NamespacedName]consulIdentifier
Expand Down Expand Up @@ -1448,6 +1457,7 @@ func TestTranslator_TCPRouteToTCPRoute(t *testing.T) {
metaKeyManagedBy: metaValueManagedBy,
metaKeyKubeNS: "k8s-ns",
metaKeyKubeServiceName: "tcp-route",
metaKeyKubeName: "tcp-route",
},
},
},
Expand Down Expand Up @@ -1539,13 +1549,14 @@ func TestTranslator_TCPRouteToTCPRoute(t *testing.T) {
metaKeyManagedBy: metaValueManagedBy,
metaKeyKubeNS: "k8s-ns",
metaKeyKubeServiceName: "tcp-route",
metaKeyKubeName: "tcp-route",
},
},
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
tr := Translator{
tr := K8sToConsulTranslator{
EnableConsulNamespaces: true,
EnableK8sMirroring: true,
}
Expand All @@ -1559,6 +1570,7 @@ func TestTranslator_TCPRouteToTCPRoute(t *testing.T) {
}

func TestTranslator_SecretToInlineCertificate(t *testing.T) {
t.Parallel()
type args struct {
k8sSecret gwv1beta1.SecretObjectReference
certs map[types.NamespacedName]consulIdentifier
Expand Down Expand Up @@ -1591,14 +1603,15 @@ func TestTranslator_SecretToInlineCertificate(t *testing.T) {
metaKeyManagedBy: metaValueManagedBy,
metaKeyKubeNS: "k8s-ns",
metaKeyKubeServiceName: "my-secret",
metaKeyKubeName: "my-secret",
},
Namespace: "my-ns",
},
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
tr := Translator{
tr := K8sToConsulTranslator{
EnableConsulNamespaces: true,
EnableK8sMirroring: true,
}
Expand Down
Loading