diff --git a/.changelog/3767.txt b/.changelog/3767.txt new file mode 100644 index 0000000000..1801c88739 --- /dev/null +++ b/.changelog/3767.txt @@ -0,0 +1,3 @@ +```release-note:feature +gateways: api-gateway now uses the Consul file-system-certificate by default for TLS +``` \ No newline at end of file diff --git a/acceptance/go.mod b/acceptance/go.mod index 967e7fdab7..67e3aa7790 100644 --- a/acceptance/go.mod +++ b/acceptance/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/uuid v1.3.0 github.com/gruntwork-io/terratest v0.46.7 github.com/hashicorp/consul-k8s/control-plane v0.0.0-20240226161840-f3842c41cb2b - github.com/hashicorp/consul/api v1.28.2 + github.com/hashicorp/consul/api v1.10.1-0.20240422130714-057ad7e95280 github.com/hashicorp/consul/proto-public v0.6.0 github.com/hashicorp/consul/sdk v0.16.0 github.com/hashicorp/go-multierror v1.1.1 diff --git a/acceptance/go.sum b/acceptance/go.sum index aa5ac02746..b01ab85378 100644 --- a/acceptance/go.sum +++ b/acceptance/go.sum @@ -290,8 +290,8 @@ github.com/hashicorp/consul-k8s/control-plane v0.0.0-20240226161840-f3842c41cb2b github.com/hashicorp/consul-k8s/control-plane v0.0.0-20240226161840-f3842c41cb2b/go.mod h1:TVaSJM7vYM/mtKGpVc/Lch53lrqLI9XAXJgy/gY8v4A= github.com/hashicorp/consul-server-connection-manager v0.1.6 h1:ktj8Fi+dRXn9hhM+FXsfEJayhzzgTqfH08Ne5M6Fmug= github.com/hashicorp/consul-server-connection-manager v0.1.6/go.mod h1:HngMIv57MT+pqCVeRQMa1eTB5dqnyMm8uxjyv+Hn8cs= -github.com/hashicorp/consul/api v1.28.2 h1:mXfkRHrpHN4YY3RqL09nXU1eHKLNiuAN4kHvDQ16k/8= -github.com/hashicorp/consul/api v1.28.2/go.mod h1:KyzqzgMEya+IZPcD65YFoOVAgPpbfERu4I/tzG6/ueE= +github.com/hashicorp/consul/api v1.10.1-0.20240422130714-057ad7e95280 h1:LDbQAc0Zx9vBChuwHxzegb+6Dyx9udRjpng0MyAfnSc= +github.com/hashicorp/consul/api v1.10.1-0.20240422130714-057ad7e95280/go.mod h1:cb1Sh6DxIGZ+5XBwpWmJF6ImxxV0vP8S5NMFRXsNN+8= github.com/hashicorp/consul/proto-public v0.6.0 h1:9qrBujmoTB5gQQ84kQO+YWvhjgYoYBNrOoHdo4cpHHM= github.com/hashicorp/consul/proto-public v0.6.0/go.mod h1:JF6983XNCzvw4wDNOLEwLqOq2IPw7iyT+pkswHSz08U= github.com/hashicorp/consul/sdk v0.16.0 h1:SE9m0W6DEfgIVCJX7xU+iv/hUl4m/nxqMTnCdMxDpJ8= diff --git a/acceptance/tests/api-gateway/api_gateway_external_servers_test.go b/acceptance/tests/api-gateway/api_gateway_external_servers_test.go index aa0934dc65..43755f5b40 100644 --- a/acceptance/tests/api-gateway/api_gateway_external_servers_test.go +++ b/acceptance/tests/api-gateway/api_gateway_external_servers_test.go @@ -76,8 +76,19 @@ func TestAPIGateway_ExternalServers(t *testing.T) { require.NoError(t, err) logger.Log(t, "set consul config entry") + // Create certificate secret, we do this separately since + // applying the secret will make an invalid certificate that breaks other tests + logger.Log(t, "creating certificate secret") + out, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "apply", "-f", "../fixtures/bases/api-gateway/certificate.yaml") + require.NoError(t, err, out) + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + // Ignore errors here because if the test ran as expected + // the custom resources will have been deleted. + k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "delete", "-f", "../fixtures/bases/api-gateway/certificate.yaml") + }) + logger.Log(t, "creating api-gateway resources") - out, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "apply", "-k", "../fixtures/bases/api-gateway") + out, err = k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "apply", "-k", "../fixtures/bases/api-gateway") require.NoError(t, err, out) helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { // Ignore errors here because if the test ran as expected diff --git a/acceptance/tests/api-gateway/api_gateway_lifecycle_test.go b/acceptance/tests/api-gateway/api_gateway_lifecycle_test.go index f6f66ed995..5c4f070060 100644 --- a/acceptance/tests/api-gateway/api_gateway_lifecycle_test.go +++ b/acceptance/tests/api-gateway/api_gateway_lifecycle_test.go @@ -279,7 +279,7 @@ func TestAPIGateway_Lifecycle(t *testing.T) { // make sure our certificate exists logger.Log(t, "checking that the certificate is synchronized to Consul") - checkConsulExists(t, consulClient, api.InlineCertificate, certificateName) + checkConsulExists(t, consulClient, api.FileSystemCertificate, certificateName) // delete the certificate in Kubernetes logger.Log(t, "deleting the certificate in Kubernetes") @@ -288,7 +288,7 @@ func TestAPIGateway_Lifecycle(t *testing.T) { // make sure the certificate no longer exists in Consul logger.Log(t, "checking that the certificate is deleted from Consul") - checkConsulNotExists(t, consulClient, api.InlineCertificate, certificateName) + checkConsulNotExists(t, consulClient, api.FileSystemCertificate, certificateName) } func checkConsulNotExists(t *testing.T, client *api.Client, kind, name string, namespace ...string) { diff --git a/acceptance/tests/api-gateway/api_gateway_tenancy_test.go b/acceptance/tests/api-gateway/api_gateway_tenancy_test.go index f7b0ac6d79..461fde1ecf 100644 --- a/acceptance/tests/api-gateway/api_gateway_tenancy_test.go +++ b/acceptance/tests/api-gateway/api_gateway_tenancy_test.go @@ -168,7 +168,7 @@ func TestAPIGateway_Tenancy(t *testing.T) { }) // we only sync validly referenced certificates over, so check to make sure it is not created. - checkConsulNotExists(t, consulClient, api.InlineCertificate, "certificate", namespaceForConsul(c.namespaceMirroring, certificateNamespace)) + checkConsulNotExists(t, consulClient, api.FileSystemCertificate, "certificate", namespaceForConsul(c.namespaceMirroring, certificateNamespace)) // now create reference grants createReferenceGrant(t, k8sClient, "gateway-certificate", gatewayNamespace, certificateNamespace) @@ -237,11 +237,11 @@ func TestAPIGateway_Tenancy(t *testing.T) { // and check to make sure that the certificate exists retryCheck(t, 30, func(r *retry.R) { - entry, _, err := consulClient.ConfigEntries().Get(api.InlineCertificate, "certificate", &api.QueryOptions{ + entry, _, err := consulClient.ConfigEntries().Get(api.FileSystemCertificate, "certificate", &api.QueryOptions{ Namespace: namespaceForConsul(c.namespaceMirroring, certificateNamespace), }) require.NoError(r, err) - certificate := entry.(*api.InlineCertificateConfigEntry) + certificate := entry.(*api.FileSystemCertificateConfigEntry) require.EqualValues(r, "certificate", certificate.Meta["k8s-name"]) require.EqualValues(r, certificateNamespace, certificate.Meta["k8s-namespace"]) diff --git a/acceptance/tests/partitions/partitions_gateway_test.go b/acceptance/tests/partitions/partitions_gateway_test.go index a90a790cb6..acf9226715 100644 --- a/acceptance/tests/partitions/partitions_gateway_test.go +++ b/acceptance/tests/partitions/partitions_gateway_test.go @@ -219,8 +219,19 @@ func TestPartitions_Gateway(t *testing.T) { logger.Log(t, "creating static-client pod in secondary partition cluster") k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-client") + // Create certificate secret, we do this separately since + // applying the secret will make an invalid certificate that breaks other tests + logger.Log(t, "creating certificate secret") + out, err := k8s.RunKubectlAndGetOutputE(t, secondaryPartitionClusterStaticServerOpts, "apply", "-f", "../fixtures/bases/api-gateway/certificate.yaml") + require.NoError(t, err, out) + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + // Ignore errors here because if the test ran as expected + // the custom resources will have been deleted. + k8s.RunKubectlAndGetOutputE(t, secondaryPartitionClusterStaticServerOpts, "delete", "-f", "../fixtures/bases/api-gateway/certificate.yaml") + }) + logger.Log(t, "creating api-gateway resources") - out, err := k8s.RunKubectlAndGetOutputE(t, secondaryPartitionClusterStaticServerOpts, "apply", "-k", "../fixtures/bases/api-gateway") + out, err = k8s.RunKubectlAndGetOutputE(t, secondaryPartitionClusterStaticServerOpts, "apply", "-k", "../fixtures/bases/api-gateway") require.NoError(t, err, out) helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { // Ignore errors here because if the test ran as expected diff --git a/control-plane/api-gateway/binding/binder.go b/control-plane/api-gateway/binding/binder.go index c704a6b04e..cbd46dd0e8 100644 --- a/control-plane/api-gateway/binding/binder.go +++ b/control-plane/api-gateway/binding/binder.go @@ -172,10 +172,7 @@ func (b *Binder) Snapshot() *Snapshot { for secret := range gatewaySecrets.Iter() { // ignore the error if the certificate cannot be processed and just don't add it into the final // sync set - if err := b.config.Resources.TranslateInlineCertificate(secret.(types.NamespacedName)); err != nil { - b.config.Logger.Error(err, "error parsing referenced secret, ignoring") - continue - } + b.config.Resources.TranslateFileSystemCertificate(secret.(types.NamespacedName)) } } diff --git a/control-plane/api-gateway/binding/binder_test.go b/control-plane/api-gateway/binding/binder_test.go index b4a274c8fa..c975a6e791 100644 --- a/control-plane/api-gateway/binding/binder_test.go +++ b/control-plane/api-gateway/binding/binder_test.go @@ -55,19 +55,19 @@ var ( ) type resourceMapResources struct { - grants []gwv1beta1.ReferenceGrant - secrets []corev1.Secret - gateways []gwv1beta1.Gateway - httpRoutes []gwv1beta1.HTTPRoute - tcpRoutes []gwv1alpha2.TCPRoute - meshServices []v1alpha1.MeshService - services []types.NamespacedName - jwtProviders []*v1alpha1.JWTProvider - gatewayPolicies []*v1alpha1.GatewayPolicy - externalAuthFilters []*v1alpha1.RouteAuthFilter - consulInlineCertificates []api.InlineCertificateConfigEntry - consulHTTPRoutes []api.HTTPRouteConfigEntry - consulTCPRoutes []api.TCPRouteConfigEntry + grants []gwv1beta1.ReferenceGrant + secrets []corev1.Secret + gateways []gwv1beta1.Gateway + httpRoutes []gwv1beta1.HTTPRoute + tcpRoutes []gwv1alpha2.TCPRoute + meshServices []v1alpha1.MeshService + services []types.NamespacedName + jwtProviders []*v1alpha1.JWTProvider + gatewayPolicies []*v1alpha1.GatewayPolicy + externalAuthFilters []*v1alpha1.RouteAuthFilter + consulFileSystemCertificates []api.FileSystemCertificateConfigEntry + consulHTTPRoutes []api.HTTPRouteConfigEntry + consulTCPRoutes []api.TCPRouteConfigEntry } func newTestResourceMap(t *testing.T, resources resourceMapResources) *common.ResourceMap { @@ -282,7 +282,7 @@ func TestBinder_Lifecycle(t *testing.T) { Protocol: "http", TLS: api.APIGatewayTLSConfiguration{ Certificates: []api.ResourceReference{{ - Kind: api.InlineCertificate, + Kind: api.FileSystemCertificate, Name: "secret-one", }}, }, @@ -644,7 +644,7 @@ func TestBinder_Lifecycle(t *testing.T) { }, }, }, - consulInlineCertificates: []api.InlineCertificateConfigEntry{ + consulFileSystemCertificates: []api.FileSystemCertificateConfigEntry{ *certificateOne, *certificateTwo, }, @@ -771,7 +771,7 @@ func TestBinder_Lifecycle(t *testing.T) { expectedConsulDeletions: []api.ResourceReference{ {Kind: api.HTTPRoute, Name: "http-route-one"}, {Kind: api.TCPRoute, Name: "tcp-route-one"}, - {Kind: api.InlineCertificate, Name: "secret-two"}, + {Kind: api.FileSystemCertificate, Name: "secret-two"}, {Kind: api.APIGateway, Name: "gateway-deleted"}, }, }, @@ -3133,7 +3133,7 @@ func controlledBinder(config BinderConfig) BinderConfig { return config } -func generateTestCertificate(t *testing.T, namespace, name string) (*api.InlineCertificateConfigEntry, corev1.Secret) { +func generateTestCertificate(t *testing.T, namespace, name string) (*api.FileSystemCertificateConfigEntry, corev1.Secret) { privateKey, err := rsa.GenerateKey(rand.Reader, common.MinKeyLength) require.NoError(t, err) @@ -3180,8 +3180,7 @@ func generateTestCertificate(t *testing.T, namespace, name string) (*api.InlineC }, } - certificate, err := (common.ResourceTranslator{}).ToInlineCertificate(secret) - require.NoError(t, err) + certificate := (common.ResourceTranslator{}).ToFileSystemCertificate(secret) return certificate, secret } diff --git a/control-plane/api-gateway/cache/consul.go b/control-plane/api-gateway/cache/consul.go index 984f6db7b4..f773a45b97 100644 --- a/control-plane/api-gateway/cache/consul.go +++ b/control-plane/api-gateway/cache/consul.go @@ -59,7 +59,7 @@ const ( apiTimeout = 5 * time.Minute ) -var Kinds = []string{api.APIGateway, api.HTTPRoute, api.TCPRoute, api.InlineCertificate, api.JWTProvider} +var Kinds = []string{api.APIGateway, api.HTTPRoute, api.TCPRoute, api.FileSystemCertificate, api.JWTProvider} type Config struct { ConsulClientConfig *consul.Config diff --git a/control-plane/api-gateway/cache/consul_test.go b/control-plane/api-gateway/cache/consul_test.go index d206c0a8a8..609ebd8d29 100644 --- a/control-plane/api-gateway/cache/consul_test.go +++ b/control-plane/api-gateway/cache/consul_test.go @@ -1539,9 +1539,9 @@ func Test_Run(t *testing.T) { tcpRoute := setupTCPRoute() tcpRoutes := []*api.TCPRouteConfigEntry{tcpRoute} - // setup inline certs - inlineCert := setupInlineCertificate() - certs := []*api.InlineCertificateConfigEntry{inlineCert} + // setup file-system certs + fileSystemCert := setupFileSystemCertificate() + certs := []*api.FileSystemCertificateConfigEntry{fileSystemCert} // setup jwt providers jwtProvider := setupJWTProvider() @@ -1573,7 +1573,7 @@ func Test_Run(t *testing.T) { return } fmt.Fprintln(w, string(val)) - case "/v1/config/inline-certificate": + case "/v1/config/file-system-certificate": val, err := json.Marshal(certs) if err != nil { w.WriteHeader(500) @@ -1627,7 +1627,7 @@ func Test_Run(t *testing.T) { } expectedCache := loadedReferenceMaps([]api.ConfigEntry{ - gw, tcpRoute, httpRouteOne, httpRouteTwo, inlineCert, jwtProvider, + gw, tcpRoute, httpRouteOne, httpRouteTwo, fileSystemCert, jwtProvider, }) ctx, cancelFn := context.WithCancel(context.Background()) @@ -1677,11 +1677,11 @@ func Test_Run(t *testing.T) { }) certNsn := types.NamespacedName{ - Name: inlineCert.Name, - Namespace: inlineCert.Namespace, + Name: fileSystemCert.Name, + Namespace: fileSystemCert.Namespace, } - certSubscriber := c.Subscribe(ctx, api.InlineCertificate, func(cfe api.ConfigEntry) []types.NamespacedName { + certSubscriber := c.Subscribe(ctx, api.FileSystemCertificate, func(cfe api.ConfigEntry) []types.NamespacedName { return []types.NamespacedName{ {Name: cfe.GetName(), Namespace: cfe.GetNamespace()}, } @@ -1968,10 +1968,10 @@ func setupTCPRoute() *api.TCPRouteConfigEntry { } } -func setupInlineCertificate() *api.InlineCertificateConfigEntry { - return &api.InlineCertificateConfigEntry{ - Kind: api.InlineCertificate, - Name: "inline-cert", +func setupFileSystemCertificate() *api.FileSystemCertificateConfigEntry { + return &api.FileSystemCertificateConfigEntry{ + Kind: api.FileSystemCertificate, + Name: "file-system-cert", Certificate: "cert", PrivateKey: "super secret", Meta: map[string]string{ diff --git a/control-plane/api-gateway/common/diff.go b/control-plane/api-gateway/common/diff.go index 7db86807b7..6f6b20b4cc 100644 --- a/control-plane/api-gateway/common/diff.go +++ b/control-plane/api-gateway/common/diff.go @@ -70,8 +70,8 @@ func EntriesEqual(a, b api.ConfigEntry) bool { if bCast, ok := b.(*api.TCPRouteConfigEntry); ok { return tcpRoutesEqual(aCast, bCast) } - case *api.InlineCertificateConfigEntry: - if bCast, ok := b.(*api.InlineCertificateConfigEntry); ok { + case *api.FileSystemCertificateConfigEntry: + if bCast, ok := b.(*api.FileSystemCertificateConfigEntry); ok { return certificatesEqual(aCast, bCast) } } @@ -323,7 +323,7 @@ func (e entryComparator) tcpRouteServicesEqual(a, b api.TCPService) bool { orDefault(a.Partition, e.partitionA) == orDefault(b.Partition, e.partitionB) } -func certificatesEqual(a, b *api.InlineCertificateConfigEntry) bool { +func certificatesEqual(a, b *api.FileSystemCertificateConfigEntry) bool { if a == nil || b == nil { return false } @@ -336,7 +336,7 @@ func certificatesEqual(a, b *api.InlineCertificateConfigEntry) bool { }).certificatesEqual(*a, *b) } -func (e entryComparator) certificatesEqual(a, b api.InlineCertificateConfigEntry) bool { +func (e entryComparator) certificatesEqual(a, b api.FileSystemCertificateConfigEntry) bool { return a.Kind == b.Kind && a.Name == b.Name && e.namespaceA == e.namespaceB && diff --git a/control-plane/api-gateway/common/resources.go b/control-plane/api-gateway/common/resources.go index 051c914ae7..514bb3c92a 100644 --- a/control-plane/api-gateway/common/resources.go +++ b/control-plane/api-gateway/common/resources.go @@ -191,7 +191,7 @@ func (s *ResourceMap) Certificate(key types.NamespacedName) *corev1.Secret { if !s.certificates.Contains(key) { return nil } - consulKey := NormalizeMeta(s.toConsulReference(api.InlineCertificate, key)) + consulKey := NormalizeMeta(s.toConsulReference(api.FileSystemCertificate, key)) if secret, ok := s.certificateGateways[consulKey]; ok { return secret.secret } @@ -201,7 +201,7 @@ func (s *ResourceMap) Certificate(key types.NamespacedName) *corev1.Secret { func (s *ResourceMap) ReferenceCountCertificate(secret corev1.Secret) { key := client.ObjectKeyFromObject(&secret) s.certificates.Add(key) - consulKey := NormalizeMeta(s.toConsulReference(api.InlineCertificate, key)) + consulKey := NormalizeMeta(s.toConsulReference(api.FileSystemCertificate, key)) if _, ok := s.certificateGateways[consulKey]; !ok { s.certificateGateways[consulKey] = &certificate{ secret: &secret, @@ -231,7 +231,7 @@ func (s *ResourceMap) ReferenceCountGateway(gateway gwv1beta1.Gateway) { set.certificates.Add(certificateKey) - consulCertificateKey := s.toConsulReference(api.InlineCertificate, certificateKey) + consulCertificateKey := s.toConsulReference(api.FileSystemCertificate, certificateKey) certificate, ok := s.certificateGateways[NormalizeMeta(consulCertificateKey)] if ok { certificate.gateways.Add(key) @@ -270,7 +270,7 @@ func (s *ResourceMap) ResourcesToGC(key types.NamespacedName) []api.ResourceRefe // the route altogether toGC = append(toGC, id) } - case api.InlineCertificate: + case api.FileSystemCertificate: if s.processedCertificates.Contains(id) { continue } @@ -323,7 +323,7 @@ func (s *ResourceMap) ReferenceCountConsulTCPRoute(route api.TCPRouteConfigEntry s.consulTCPRoutes[NormalizeMeta(key)] = set } -func (s *ResourceMap) ReferenceCountConsulCertificate(cert api.InlineCertificateConfigEntry) { +func (s *ResourceMap) ReferenceCountConsulCertificate(cert api.FileSystemCertificateConfigEntry) { key := s.objectReference(&cert) var referenced *certificate @@ -644,36 +644,29 @@ func (s *ResourceMap) CanGCTCPRouteOnUnbind(id api.ResourceReference) bool { return true } -func (s *ResourceMap) TranslateInlineCertificate(key types.NamespacedName) error { - consulKey := s.toConsulReference(api.InlineCertificate, key) +func (s *ResourceMap) TranslateFileSystemCertificate(key types.NamespacedName) { + consulKey := s.toConsulReference(api.FileSystemCertificate, key) certificate, ok := s.certificateGateways[NormalizeMeta(consulKey)] if !ok { - return nil + return } if certificate.secret == nil { - return nil - } - - consulCertificate, err := s.translator.ToInlineCertificate(*certificate.secret) - if err != nil { - return err + return } - // add to the processed set so we don't GC it. + // add to the processed set so that we don't GC it. s.processedCertificates.Add(consulKey) s.consulMutations = append(s.consulMutations, &ConsulUpdateOperation{ - Entry: consulCertificate, + Entry: s.translator.ToFileSystemCertificate(*certificate.secret), // just swallow the error and log it since we can't propagate status back on a certificate. - OnUpdate: func(error) { + OnUpdate: func(err error) { if err != nil { s.logger.Error(err, "error syncing certificate to Consul") } }, }) - - return nil } func (s *ResourceMap) Mutations() []*ConsulUpdateOperation { diff --git a/control-plane/api-gateway/common/translation.go b/control-plane/api-gateway/common/translation.go index 5161e6b033..86e637d585 100644 --- a/control-plane/api-gateway/common/translation.go +++ b/control-plane/api-gateway/common/translation.go @@ -4,6 +4,7 @@ package common import ( + "fmt" "strings" corev1 "k8s.io/api/core/v1" @@ -109,7 +110,7 @@ func (t ResourceTranslator) toAPIGatewayListener(gateway gwv1beta1.Gateway, list ref := IndexedNamespacedNameWithDefault(ref.Name, ref.Namespace, namespace) if resources.Certificate(ref) != nil { - certificates = append(certificates, t.NonNormalizedConfigEntryReference(api.InlineCertificate, ref)) + certificates = append(certificates, t.NonNormalizedConfigEntryReference(api.FileSystemCertificate, ref)) } } } @@ -529,31 +530,19 @@ func (t ResourceTranslator) translateTCPRouteRule(route gwv1alpha2.TCPRoute, ref return api.TCPService{}, false } -func (t ResourceTranslator) ToInlineCertificate(secret corev1.Secret) (*api.InlineCertificateConfigEntry, error) { - certificate, privateKey, err := ParseCertificateData(secret) - if err != nil { - return nil, err - } - - err = ValidateKeyLength(privateKey) - if err != nil { - return nil, err - } - - namespace := t.Namespace(secret.Namespace) - - return &api.InlineCertificateConfigEntry{ - Kind: api.InlineCertificate, +func (t ResourceTranslator) ToFileSystemCertificate(secret corev1.Secret) *api.FileSystemCertificateConfigEntry { + return &api.FileSystemCertificateConfigEntry{ + Kind: api.FileSystemCertificate, Name: secret.Name, - Namespace: namespace, + Namespace: t.Namespace(secret.Namespace), Partition: t.ConsulPartition, - Certificate: strings.TrimSpace(certificate), - PrivateKey: strings.TrimSpace(privateKey), + Certificate: fmt.Sprintf("/consul/gateway-certificates/%s/%s/tls.crt", secret.Namespace, secret.Name), + PrivateKey: fmt.Sprintf("/consul/gateway-certificates/%s/%s/tls.key", secret.Namespace, secret.Name), Meta: t.addDatacenterToMeta(map[string]string{ constants.MetaKeyKubeNS: secret.Namespace, constants.MetaKeyKubeName: secret.Name, }), - }, nil + } } func EntryToNamespacedName(entry api.ConfigEntry) types.NamespacedName { diff --git a/control-plane/api-gateway/common/translation_test.go b/control-plane/api-gateway/common/translation_test.go index 4331e2b77a..e841464b9a 100644 --- a/control-plane/api-gateway/common/translation_test.go +++ b/control-plane/api-gateway/common/translation_test.go @@ -303,7 +303,7 @@ func TestTranslator_ToAPIGateway(t *testing.T) { TLS: api.APIGatewayTLSConfiguration{ Certificates: []api.ResourceReference{ { - Kind: api.InlineCertificate, + Kind: api.FileSystemCertificate, Name: listenerOneCertName, Namespace: listenerOneCertConsulNamespace, }, @@ -321,7 +321,7 @@ func TestTranslator_ToAPIGateway(t *testing.T) { TLS: api.APIGatewayTLSConfiguration{ Certificates: []api.ResourceReference{ { - Kind: api.InlineCertificate, + Kind: api.FileSystemCertificate, Name: listenerTwoCertName, Namespace: listenerTwoCertConsulNamespace, }, diff --git a/control-plane/api-gateway/controllers/gateway_controller.go b/control-plane/api-gateway/controllers/gateway_controller.go index 537100fd70..cb2a26ed8e 100644 --- a/control-plane/api-gateway/controllers/gateway_controller.go +++ b/control-plane/api-gateway/controllers/gateway_controller.go @@ -136,10 +136,10 @@ func (r *GatewayController) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, err } - // fetch our inline certificates from cache, this needs to happen + // fetch our file-system-certificates from cache, this needs to happen // here since the certificates need to be reference counted before // the gateways. - r.fetchConsulInlineCertificates(resources) + r.fetchConsulFileSystemCertificates(resources) // add our current gateway even if it's not controlled by us so we // can garbage collect any resources for it. @@ -471,8 +471,8 @@ func SetupGatewayControllerWithManager(ctx context.Context, mgr ctrl.Manager, co &handler.EnqueueRequestForObject{}, ). Watches( - // Subscribe to changes from Consul for InlineCertificates - &source.Channel{Source: c.Subscribe(ctx, api.InlineCertificate, r.transformConsulInlineCertificate(ctx)).Events()}, + // Subscribe to changes from Consul for FileSystemCertificates + &source.Channel{Source: c.Subscribe(ctx, api.FileSystemCertificate, r.transformConsulFileSystemCertificate(ctx)).Events()}, &handler.EnqueueRequestForObject{}, ). Watches( @@ -674,7 +674,7 @@ func (r *GatewayController) transformConsulTCPRoute(ctx context.Context) func(en } } -func (r *GatewayController) transformConsulInlineCertificate(ctx context.Context) func(entry api.ConfigEntry) []types.NamespacedName { +func (r *GatewayController) transformConsulFileSystemCertificate(ctx context.Context) func(entry api.ConfigEntry) []types.NamespacedName { return func(entry api.ConfigEntry) []types.NamespacedName { certificateKey := api.ResourceReference{ Kind: entry.GetKind(), @@ -1230,8 +1230,8 @@ func (c *GatewayController) fetchConsulTCPRoutes(ref api.ResourceReference, reso } } -func (c *GatewayController) fetchConsulInlineCertificates(resources *common.ResourceMap) { - for _, cert := range configEntriesTo[*api.InlineCertificateConfigEntry](c.cache.List(api.InlineCertificate)) { +func (c *GatewayController) fetchConsulFileSystemCertificates(resources *common.ResourceMap) { + for _, cert := range configEntriesTo[*api.FileSystemCertificateConfigEntry](c.cache.List(api.FileSystemCertificate)) { resources.ReferenceCountConsulCertificate(*cert) } } diff --git a/control-plane/api-gateway/controllers/gateway_controller_integration_test.go b/control-plane/api-gateway/controllers/gateway_controller_integration_test.go index ee8d8240f6..e266f841c8 100644 --- a/control-plane/api-gateway/controllers/gateway_controller_integration_test.go +++ b/control-plane/api-gateway/controllers/gateway_controller_integration_test.go @@ -151,7 +151,7 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { gwSub := resourceCache.Subscribe(ctx, api.APIGateway, gwCtrl.transformConsulGateway) httpRouteSub := resourceCache.Subscribe(ctx, api.HTTPRoute, gwCtrl.transformConsulHTTPRoute(ctx)) tcpRouteSub := resourceCache.Subscribe(ctx, api.TCPRoute, gwCtrl.transformConsulTCPRoute(ctx)) - inlineCertSub := resourceCache.Subscribe(ctx, api.InlineCertificate, gwCtrl.transformConsulInlineCertificate(ctx)) + fileSystemCertSub := resourceCache.Subscribe(ctx, api.FileSystemCertificate, gwCtrl.transformConsulFileSystemCertificate(ctx)) cert := tc.certFn(t, ctx, k8sClient, tc.namespace) k8sGWObj := tc.gwFn(t, ctx, k8sClient, tc.namespace) @@ -225,7 +225,7 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { tcpRouteDone = true w.Done() } - case <-inlineCertSub.Events(): + case <-fileSystemCertSub.Events(): } } }(wg) @@ -255,7 +255,7 @@ func TestControllerDoesNotInfinitelyReconcile(t *testing.T) { gwRef := gwCtrl.Translator.ConfigEntryReference(api.APIGateway, gwNamespaceName) httpRouteRef := gwCtrl.Translator.ConfigEntryReference(api.HTTPRoute, httpRouteNamespaceName) tcpRouteRef := gwCtrl.Translator.ConfigEntryReference(api.TCPRoute, tcpRouteNamespaceName) - certRef := gwCtrl.Translator.ConfigEntryReference(api.InlineCertificate, certNamespaceName) + certRef := gwCtrl.Translator.ConfigEntryReference(api.FileSystemCertificate, certNamespaceName) curGWModifyIndex := resourceCache.Get(gwRef).GetModifyIndex() curHTTPRouteModifyIndex := resourceCache.Get(httpRouteRef).GetModifyIndex() diff --git a/control-plane/api-gateway/gatekeeper/dataplane.go b/control-plane/api-gateway/gatekeeper/dataplane.go index 16839cbb09..a2f4c8a0f3 100644 --- a/control-plane/api-gateway/gatekeeper/dataplane.go +++ b/control-plane/api-gateway/gatekeeper/dataplane.go @@ -23,10 +23,10 @@ const ( consulDataplaneDNSBindHost = "127.0.0.1" consulDataplaneDNSBindPort = 8600 defaultEnvoyProxyConcurrency = 1 - volumeName = "consul-connect-inject-data" + volumeNameForConnectInject = "consul-connect-inject-data" ) -func consulDataplaneContainer(metrics common.MetricsConfig, config common.HelmConfig, gcc v1alpha1.GatewayClassConfig, name, namespace string) (corev1.Container, error) { +func consulDataplaneContainer(metrics common.MetricsConfig, config common.HelmConfig, gcc v1alpha1.GatewayClassConfig, name, namespace string, mounts []corev1.VolumeMount) (corev1.Container, error) { // Extract the service account token's volume mount. var ( err error @@ -77,12 +77,7 @@ func consulDataplaneContainer(metrics common.MetricsConfig, config common.HelmCo Value: "$(NODE_NAME)-virtual", }, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: volumeName, - MountPath: "/consul/connect-inject", - }, - }, + VolumeMounts: mounts, Args: args, ReadinessProbe: probe, } diff --git a/control-plane/api-gateway/gatekeeper/deployment.go b/control-plane/api-gateway/gatekeeper/deployment.go index d3bb56a69f..61ef1fd318 100644 --- a/control-plane/api-gateway/gatekeeper/deployment.go +++ b/control-plane/api-gateway/gatekeeper/deployment.go @@ -107,7 +107,9 @@ func (g *Gatekeeper) deployment(gateway gwv1beta1.Gateway, gcc v1alpha1.GatewayC annotations[constants.AnnotationPrometheusPort] = strconv.Itoa(metrics.Port) } - container, err := consulDataplaneContainer(metrics, config, gcc, gateway.Name, gateway.Namespace) + volumes, mounts := volumesAndMounts(gateway) + + container, err := consulDataplaneContainer(metrics, config, gcc, gateway.Name, gateway.Namespace, mounts) if err != nil { return nil, err } @@ -129,14 +131,7 @@ func (g *Gatekeeper) deployment(gateway gwv1beta1.Gateway, gcc v1alpha1.GatewayC Annotations: annotations, }, Spec: corev1.PodSpec{ - Volumes: []corev1.Volume{ - { - Name: volumeName, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}, - }, - }, - }, + Volumes: volumes, InitContainers: []corev1.Container{ initContainer, }, diff --git a/control-plane/api-gateway/gatekeeper/init.go b/control-plane/api-gateway/gatekeeper/init.go index 1cd616bfc9..3b6147f164 100644 --- a/control-plane/api-gateway/gatekeeper/init.go +++ b/control-plane/api-gateway/gatekeeper/init.go @@ -47,7 +47,7 @@ func initContainer(config common.HelmConfig, name, namespace string) (corev1.Con // Create expected volume mounts volMounts := []corev1.VolumeMount{ { - Name: volumeName, + Name: volumeNameForConnectInject, MountPath: "/consul/connect-inject", }, } diff --git a/control-plane/api-gateway/gatekeeper/volumes.go b/control-plane/api-gateway/gatekeeper/volumes.go new file mode 100644 index 0000000000..ba80084623 --- /dev/null +++ b/control-plane/api-gateway/gatekeeper/volumes.go @@ -0,0 +1,61 @@ +package gatekeeper + +import ( + "fmt" + "github.com/hashicorp/consul-k8s/control-plane/api-gateway/common" + "golang.org/x/exp/maps" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +// volumesAndMounts generates the list of volumes for the Deployment and the list of volume +// mounts for the primary container in the Deployment. In addition to the "default" volume +// containing connect-inject data, there will be one volume + mount per unique Secret +// referenced in the Gateway's listener TLS configurations. The volume references the Secret +// directly. +func volumesAndMounts(gateway v1beta1.Gateway) ([]corev1.Volume, []corev1.VolumeMount) { + volumes := map[string]corev1.Volume{ + volumeNameForConnectInject: { + Name: volumeNameForConnectInject, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}, + }, + }, + } + mounts := map[string]corev1.VolumeMount{ + volumeNameForConnectInject: { + Name: volumeNameForConnectInject, + MountPath: "/consul/connect-inject", + }, + } + + for i, listener := range gateway.Spec.Listeners { + if listener.TLS == nil { + continue + } + + for j, ref := range listener.TLS.CertificateRefs { + refNamespace := common.ValueOr(ref.Namespace, gateway.Namespace) + + volumeName := fmt.Sprintf("listener-%d-cert-%d-volume", i, j) + + volumes[volumeName] = corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: string(ref.Name), + DefaultMode: common.PointerTo(int32(444)), + Optional: common.PointerTo(false), + }, + }, + } + + mounts[volumeName] = corev1.VolumeMount{ + Name: volumeName, + MountPath: fmt.Sprintf("/consul/gateway-certificates/%s/%s", refNamespace, ref.Name), + } + } + } + + return maps.Values(volumes), maps.Values(mounts) +} diff --git a/control-plane/go.mod b/control-plane/go.mod index 9d23705050..4541272f02 100644 --- a/control-plane/go.mod +++ b/control-plane/go.mod @@ -1,6 +1,6 @@ module github.com/hashicorp/consul-k8s/control-plane -replace github.com/hashicorp/consul/api => github.com/hashicorp/consul/api v1.10.1-0.20240312203720-262f4358003f +replace github.com/hashicorp/consul/api => github.com/hashicorp/consul/api v1.10.1-0.20240422130714-057ad7e95280 require ( github.com/cenkalti/backoff v2.2.1+incompatible @@ -13,7 +13,7 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20240226161840-f3842c41cb2b github.com/hashicorp/consul-server-connection-manager v0.1.6 - github.com/hashicorp/consul/api v1.28.2 + github.com/hashicorp/consul/api v1.10.1-0.20240422130714-057ad7e95280 github.com/hashicorp/consul/proto-public v0.6.0 github.com/hashicorp/consul/sdk v0.16.0 github.com/hashicorp/go-bexpr v0.1.11 diff --git a/control-plane/go.sum b/control-plane/go.sum index b09b04cbf7..08561c9c18 100644 --- a/control-plane/go.sum +++ b/control-plane/go.sum @@ -275,8 +275,8 @@ github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20240226161840-f3842c41 github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20240226161840-f3842c41cb2b/go.mod h1:9NKJHOcgmz/6P2y6MegNIOXhIKE/0ils/mHWd5sZgoU= github.com/hashicorp/consul-server-connection-manager v0.1.6 h1:ktj8Fi+dRXn9hhM+FXsfEJayhzzgTqfH08Ne5M6Fmug= github.com/hashicorp/consul-server-connection-manager v0.1.6/go.mod h1:HngMIv57MT+pqCVeRQMa1eTB5dqnyMm8uxjyv+Hn8cs= -github.com/hashicorp/consul/api v1.10.1-0.20240312203720-262f4358003f h1:8clIrMnJtO5ab5Kd1qF19s9s581cyGYhQxfPLVRaFZs= -github.com/hashicorp/consul/api v1.10.1-0.20240312203720-262f4358003f/go.mod h1:JnWx0qZd1Ffeoa42yVAxzv7/v7eaZyptkw0dG9F/gF4= +github.com/hashicorp/consul/api v1.10.1-0.20240422130714-057ad7e95280 h1:LDbQAc0Zx9vBChuwHxzegb+6Dyx9udRjpng0MyAfnSc= +github.com/hashicorp/consul/api v1.10.1-0.20240422130714-057ad7e95280/go.mod h1:cb1Sh6DxIGZ+5XBwpWmJF6ImxxV0vP8S5NMFRXsNN+8= github.com/hashicorp/consul/proto-public v0.6.0 h1:9qrBujmoTB5gQQ84kQO+YWvhjgYoYBNrOoHdo4cpHHM= github.com/hashicorp/consul/proto-public v0.6.0/go.mod h1:JF6983XNCzvw4wDNOLEwLqOq2IPw7iyT+pkswHSz08U= github.com/hashicorp/consul/sdk v0.16.0 h1:SE9m0W6DEfgIVCJX7xU+iv/hUl4m/nxqMTnCdMxDpJ8=