Refresh client CA certificate if changed

cmd/allocator/main.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	"agones.dev/agones/pkg"
@@ -43,6 +44,7 @@ import (
 	"github.com/spf13/viper"
 	"go.opencensus.io/plugin/ochttp"
 	"go.opencensus.io/stats/view"
+	"gopkg.in/fsnotify.v1"
 	k8serror "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sruntime "k8s.io/apimachinery/pkg/runtime"
@@ -104,15 +106,51 @@ func main() {
 	httpsMux := http.NewServeMux()
 	httpsMux.HandleFunc("/v1alpha1/gameserverallocation", h.postOnly(h.allocateHandler))
 
-	caCertPool, err := getCACertPool(certDir)
+	// creates a new file watcher for client certificate folder
+	watcher, _ := fsnotify.NewWatcher()
+	defer watcher.Close() // nolint: errcheck
+	if err := watcher.Add(certDir); err != nil {
+		logger.WithError(err).Fatalf("cannot watch folder %s for secret changes", certDir)
+	}
+
+	tlsCer, err := tls.LoadX509KeyPair(tlsDir+"tls.crt", tlsDir+"tls.key")
 	if err != nil {
-		logger.WithError(err).Fatal("could not get CA certs")
+		logger.WithError(err).Fatal("server TLS could not be loaded")
 	}
+	caCertPool := loadCACertPool()
+
+	// Watching for the events in certificate directory for updating certificates, when there is a change
+	go func() {
+		for {
+			select {
+			// watch for events
+			case event := <-watcher.Events:
+				h.certMutex.Lock()
+				caCertPool = loadCACertPool()
+				logger.Infof("Certificate directory change event %v", event)
+				h.certMutex.Unlock()
+
+				// watch for errors
+			case err := <-watcher.Errors:
+				logger.WithError(err).Error("error watching for certificate directory")
+			}
+		}
+	}()
 
 	cfg := &tls.Config{
-		ClientAuth: tls.RequireAndVerifyClientCert,
-		ClientCAs:  caCertPool,
+		Certificates: []tls.Certificate{tlsCer},
+		ClientAuth:   tls.RequireAndVerifyClientCert,
+		GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) {
+			h.certMutex.RLock()
+			defer h.certMutex.RUnlock()
+			return &tls.Config{
+				Certificates: []tls.Certificate{tlsCer},
+				ClientAuth:   tls.RequireAndVerifyClientCert,
+				ClientCAs:    caCertPool,
+			}, nil
+		},
 	}
+
 	srv := &http.Server{
 		Addr:      ":" + sslPort,
 		TLSConfig: cfg,
@@ -124,7 +162,8 @@ func main() {
 
 	// listen on https to serve allocations
 	go func() {
-		err := srv.ListenAndServeTLS(tlsDir+"tls.crt", tlsDir+"tls.key")
+		// The certs are set on the config so passing empty as the cert path
+		err := srv.ListenAndServeTLS("", "")
 		logger.WithError(err).Fatal("allocation service crashed")
 		os.Exit(1)
 	}()
@@ -135,6 +174,14 @@ func main() {
 	logger.WithError(err).Fatal("allocation service crashed")
 }
 
+func loadCACertPool() *x509.CertPool {
+	caCertPool, err := getCACertPool(certDir)
+	if err != nil {
+		logger.WithError(err).Fatal("could not get CA certs")
+	}
+	return caCertPool
+}
+
 func newServiceHandler(kubeClient kubernetes.Interface, agonesClient versioned.Interface, health healthcheck.Handler) *httpHandler {
 	defaultResync := 30 * time.Second
 	agonesInformerFactory := externalversions.NewSharedInformerFactory(agonesClient, defaultResync)
@@ -202,7 +249,8 @@ func getCACertPool(path string) (*x509.CertPool, error) {
 				return nil, fmt.Errorf("ca cert is not readable or missing: %s", err.Error())
 			}
 			if !caCertPool.AppendCertsFromPEM(caCert) {
-				return nil, fmt.Errorf("client cert %s cannot be installed", certFile)
+				logger.Errorf("client cert %s cannot be installed", certFile)
+				continue
 			}
 			logger.Infof("client cert %s is installed", certFile)
 		}
@@ -224,6 +272,7 @@ func (h *httpHandler) postOnly(in handler) handler {
 
 type httpHandler struct {
 	allocationCallback func(*allocationv1.GameServerAllocation) (k8sruntime.Object, error)
+	certMutex          sync.RWMutex
 }
 
 func (h *httpHandler) allocateHandler(w http.ResponseWriter, r *http.Request) {

go.mod

@@ -44,6 +44,8 @@ require (
 	google.golang.org/api v0.0.0-20190117000611-43037ff31f69 // indirect
 	google.golang.org/genproto v0.0.0-20190111180523-db91494dd46c
 	google.golang.org/grpc v1.17.0
+	gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect
+	gopkg.in/fsnotify.v1 v1.4.7
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0-20170531160350-a96e63847dc3
 	gopkg.in/yaml.v2 v2.2.2

go.sum

@@ -266,6 +266,8 @@ google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmE
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
+gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY=
+gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=

test/e2e/allocator_test.go

@@ -68,8 +68,6 @@ func TestAllocator(t *testing.T) {
 	clientSecretName := fmt.Sprintf("allocator-client-%s", uuid.NewUUID())
 	genClientSecret(t, tlsCA, namespace, clientSecretName)
 
-	restartAllocator(t)
-
 	flt, err := createFleet(namespace)
 	if !assert.Nil(t, err) {
 		return
@@ -128,6 +126,7 @@ func TestAllocatorCrossNamespace(t *testing.T) {
 	ip, port := getAllocatorEndpoint(t)
 	requestURL := fmt.Sprintf(allocatorReqURLFmt, ip, port)
 	tlsCA := refreshAllocatorTLSCerts(t, ip)
+	restartAllocator(t)
 
 	// Create namespaces A and B
 	namespaceA := fmt.Sprintf("allocator-a-%s", uuid.NewUUID())
@@ -141,8 +140,6 @@ func TestAllocatorCrossNamespace(t *testing.T) {
 	clientSecretNameA := fmt.Sprintf("allocator-client-%s", uuid.NewUUID())
 	genClientSecret(t, tlsCA, namespaceA, clientSecretNameA)
 
-	restartAllocator(t)
-
 	policyName := fmt.Sprintf("a-to-b-%s", uuid.NewUUID())
 	p := &multiclusterv1alpha1.GameServerAllocationPolicy{
 		ObjectMeta: metav1.ObjectMeta{
@@ -366,6 +363,8 @@ func refreshAllocatorTLSCerts(t *testing.T, host string) []byte {
 		t.Fatalf("updating secrets failed: %s", err)
 	}
 
+	restartAllocator(t)
+
 	t.Logf("Allocator TLS is refreshed with public CA: %s for endpoint %s", string(pub), host)
 	return pub
 }