chore(*) ExternalServices add CA and Client certificate support (#1094)

* chore(*) add CA and Client certificate support code * feat(*) push ExternalService TLS config to Envoy * chore(*) move tls.go * chore(*) update UpstreamTlsContextOutsideMesh * test(*) e2e for TLS certs * fix(*) load certificates at BuildEndpointMap Signed-off-by: Nikolay Nikolaev <nikolay.nikolaev@konghq.com>
kumahq · Nov 12, 2020 · 1cf6d0d · 1cf6d0d
1 parent 1e9d36b
commit 1cf6d0d
Show file tree

Hide file tree

Showing 30 changed files with 414 additions and 95 deletions.
diff --git a/api/mesh/v1alpha1/externalservice.pb.go b/api/mesh/v1alpha1/externalservice.pb.go
diff --git a/api/mesh/v1alpha1/externalservice.pb.validate.go b/api/mesh/v1alpha1/externalservice.pb.validate.go
diff --git a/api/mesh/v1alpha1/externalservice.proto b/api/mesh/v1alpha1/externalservice.proto
@@ -6,6 +6,8 @@ option go_package = "v1alpha1";
 
 import "validate/validate.proto";
 
+import "system/v1alpha1/datasource.proto";
+
 // ExternalService defines configuration of the externaly accessible service
 message ExternalService {
 
@@ -18,6 +20,15 @@ message ExternalService {
     message TLS {
       // denotes that the external service uses TLS
       bool enabled = 1;
+
+      // Data source for the certificate of CA
+      kuma.system.v1alpha1.DataSource ca_cert = 2;
+
+      // Data source for the authentication
+      kuma.system.v1alpha1.DataSource client_cert = 3;
+
+      // Data source for the authentication
+      kuma.system.v1alpha1.DataSource client_key = 4;
     }
 
     TLS tls = 2;
@@ -27,5 +38,5 @@ message ExternalService {
 
   // Tags associated with the external service,
   // e.g. kuma.io/service=web, kuma.io/protocol, version=1.0.
-  map<string, string> tags = 3 [ (validate.rules).map.min_pairs = 1 ];
+  map<string, string> tags = 2 [ (validate.rules).map.min_pairs = 1 ];
 }
diff --git a/pkg/core/bootstrap/bootstrap.go b/pkg/core/bootstrap/bootstrap.go
@@ -70,7 +70,7 @@ func buildRuntime(cfg kuma_cp.Config) (core_runtime.Runtime, error) {
 		return nil, err
 	}
 
-	builder.WithDataSourceLoader(datasource.NewDataSourceLoader(builder.ResourceManager()))
+	builder.WithDataSourceLoader(datasource.NewDataSourceLoader(builder.ReadOnlyResourceManager()))
 
 	if err := initializeCaManagers(builder); err != nil {
 		return nil, err

diff --git a/pkg/core/datasource/loader.go b/pkg/core/datasource/loader.go
@@ -18,12 +18,12 @@ type Loader interface {
 }
 
 type loader struct {
-	secretManager manager.ResourceManager
+	secretManager manager.ReadOnlyResourceManager
 }
 
 var _ Loader = &loader{}
 
-func NewDataSourceLoader(secretManager manager.ResourceManager) Loader {
+func NewDataSourceLoader(secretManager manager.ReadOnlyResourceManager) Loader {
 	return &loader{
 		secretManager: secretManager,
 	}

diff --git a/pkg/core/runtime/builder.go b/pkg/core/runtime/builder.go
@@ -200,6 +200,7 @@ func (b *Builder) Build() (Runtime, error) {
 			ss:       b.ss,
 			cam:      b.cam,
 			xds:      b.xds,
+			dsl:      b.dsl,
 			ext:      b.ext,
 			dns:      b.dns,
 			configm:  b.configm,

diff --git a/pkg/core/runtime/runtime.go b/pkg/core/runtime/runtime.go
@@ -4,6 +4,8 @@ import (
 	"context"
 	"sync"
 
+	"github.com/kumahq/kuma/pkg/core/datasource"
+
 	"github.com/kumahq/kuma/pkg/core/dns/lookup"
 	"github.com/kumahq/kuma/pkg/core/secrets/store"
 	"github.com/kumahq/kuma/pkg/metrics"
@@ -37,6 +39,7 @@ type RuntimeInfo interface {
 type RuntimeContext interface {
 	Config() kuma_cp.Config
 	XDS() core_xds.XdsContext
+	DataSourceLoader() datasource.Loader
 	ResourceManager() core_manager.ResourceManager
 	ResourceStore() core_store.ResourceStore
 	ReadOnlyResourceManager() core_manager.ReadOnlyResourceManager
@@ -95,6 +98,7 @@ type runtimeContext struct {
 	rom      core_manager.ReadOnlyResourceManager
 	cam      ca.Managers
 	xds      core_xds.XdsContext
+	dsl      datasource.Loader
 	ext      context.Context
 	dns      dns.DNSResolver
 	configm  config_manager.ConfigManager
@@ -116,6 +120,9 @@ func (rc *runtimeContext) Config() kuma_cp.Config {
 func (rc *runtimeContext) XDS() core_xds.XdsContext {
 	return rc.xds
 }
+func (rc *runtimeContext) DataSourceLoader() datasource.Loader {
+	return rc.dsl
+}
 func (rc *runtimeContext) ResourceManager() core_manager.ResourceManager {
 	return rc.rm
 }

diff --git a/pkg/core/xds/types.go b/pkg/core/xds/types.go
@@ -44,6 +44,9 @@ type DestinationMap map[ServiceName]TagSelectorSet
 
 type ExternalService struct {
 	TLSEnabled bool
+	CaCert     *envoy_core.DataSource
+	ClientCert *envoy_core.DataSource
+	ClientKey  *envoy_core.DataSource
 }
 
 type Locality struct {

diff --git a/pkg/sds/server/reconciler.go b/pkg/sds/server/reconciler.go
@@ -10,6 +10,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/kumahq/kuma/pkg/xds/envoy/tls"
+
 	"github.com/pkg/errors"
 
 	envoy_auth "github.com/envoyproxy/go-control-plane/envoy/api/v2/auth"
@@ -27,7 +29,6 @@ import (
 	core_store "github.com/kumahq/kuma/pkg/core/resources/store"
 	core_xds "github.com/kumahq/kuma/pkg/core/xds"
 	sds_provider "github.com/kumahq/kuma/pkg/sds/provider"
-	"github.com/kumahq/kuma/pkg/xds/envoy"
 )
 
 // DataplaneReconciler keeps the state of the Cache for SDS consistent
@@ -143,15 +144,15 @@ func (d *DataplaneReconciler) generateSnapshot(dataplane *mesh_core.DataplaneRes
 		Services: dataplane.Spec.TagSet().Values(mesh_proto.ServiceTag),
 		Mesh:     dataplane.GetMeta().GetMesh(),
 	}
-	identitySecret, err := d.identityProvider.Get(context.Background(), envoy.IdentityCertResource, requestor)
+	identitySecret, err := d.identityProvider.Get(context.Background(), tls.IdentityCertResource, requestor)
 	if err != nil {
 		return envoy_cache.Snapshot{}, errors.Wrap(err, "could not get Dataplane cert pair")
 	}
 
 	requestor = sds_provider.Identity{
 		Mesh: dataplane.GetMeta().GetMesh(),
 	}
-	caSecret, err := d.meshCaProvider.Get(context.Background(), envoy.MeshCaResource, requestor)
+	caSecret, err := d.meshCaProvider.Get(context.Background(), tls.MeshCaResource, requestor)
 	if err != nil {
 		return envoy_cache.Snapshot{}, errors.Wrap(err, "could not get mesh CA cert")
 	}
@@ -161,14 +162,14 @@ func (d *DataplaneReconciler) generateSnapshot(dataplane *mesh_core.DataplaneRes
 		Resources: [envoy_types.UnknownType]envoy_cache.Resources{},
 	}
 	snap.Resources[envoy_types.Secret] = envoy_cache.NewResources(version, []envoy_types.Resource{
-		identitySecret.ToResource(envoy.IdentityCertResource),
-		caSecret.ToResource(envoy.MeshCaResource),
+		identitySecret.ToResource(tls.IdentityCertResource),
+		caSecret.ToResource(tls.MeshCaResource),
 	})
 	return snap, nil
 }
 
 func (d *DataplaneReconciler) updateInsights(dataplaneId core_model.ResourceKey, snapshot envoy_cache.Snapshot) error {
-	secret := snapshot.Resources[envoy_types.Secret].Items[envoy.IdentityCertResource].(*envoy_auth.Secret)
+	secret := snapshot.Resources[envoy_types.Secret].Items[tls.IdentityCertResource].(*envoy_auth.Secret)
 	certPEM := secret.GetTlsCertificate().CertificateChain.GetInlineBytes()
 	block, _ := pem.Decode(certPEM)
 	cert, err := x509.ParseCertificate(block.Bytes)

diff --git a/pkg/sds/server/server_test.go b/pkg/sds/server/server_test.go
@@ -7,6 +7,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/kumahq/kuma/pkg/xds/envoy/tls"
+
 	envoy_api "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	envoy_api_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	envoy_discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2"
@@ -15,6 +17,9 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/metadata"
 
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
 	mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
 	kuma_cp "github.com/kumahq/kuma/pkg/config/app/kuma-cp"
 	dp_server_cfg "github.com/kumahq/kuma/pkg/config/dp-server"
@@ -30,10 +35,6 @@ import (
 	"github.com/kumahq/kuma/pkg/test/runtime"
 	tokens_builtin "github.com/kumahq/kuma/pkg/tokens/builtin"
 	tokens_issuer "github.com/kumahq/kuma/pkg/tokens/builtin/issuer"
-	"github.com/kumahq/kuma/pkg/xds/envoy"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
 )
 
 var _ = Describe("SDS Server", func() {
@@ -171,7 +172,7 @@ var _ = Describe("SDS Server", func() {
 			Node: &envoy_api_core.Node{
 				Id: "default.backend-01",
 			},
-			ResourceNames: []string{envoy.MeshCaResource, envoy.IdentityCertResource},
+			ResourceNames: []string{tls.MeshCaResource, tls.IdentityCertResource},
 			TypeUrl:       envoy_resource.SecretType,
 		}
 	}

diff --git a/pkg/xds/cache/cla/cache.go b/pkg/xds/cache/cla/cache.go
@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/kumahq/kuma/pkg/core/datasource"
+
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/kumahq/kuma/pkg/core/resources/model"
@@ -33,6 +35,7 @@ var (
 type Cache struct {
 	cache   *cache.Cache
 	rm      manager.ReadOnlyResourceManager
+	dsl     datasource.Loader
 	ipFunc  lookup.LookupIPFunc
 	zone    string
 	onceMap *once.Map
@@ -41,6 +44,7 @@ type Cache struct {
 
 func NewCache(
 	rm manager.ReadOnlyResourceManager,
+	dsl datasource.Loader,
 	zone string, expirationTime time.Duration,
 	ipFunc lookup.LookupIPFunc,
 	metrics metrics.Metrics,
@@ -55,6 +59,7 @@ func NewCache(
 	return &Cache{
 		cache:   cache.New(expirationTime, time.Duration(int64(float64(expirationTime)*0.9))),
 		rm:      rm,
+		dsl:     dsl,
 		zone:    zone,
 		ipFunc:  ipFunc,
 		onceMap: once.NewMap(),
@@ -86,7 +91,7 @@ func (c *Cache) GetCLA(ctx context.Context, meshName, service string) (*envoy_ap
 		if err := c.rm.List(ctx, externalServices, core_store.ListByMesh(meshName)); err != nil {
 			return nil, err
 		}
-		endpointMap := topology.BuildEndpointMap(dataplanes.Items, c.zone, mesh, externalServices.Items)
+		endpointMap := topology.BuildEndpointMap(mesh, c.zone, dataplanes.Items, externalServices.Items, c.dsl)
 		cla := endpoints.CreateClusterLoadAssignment(service, endpointMap[service])
 		c.cache.SetDefault(key, cla)
 		c.onceMap.Delete(key)