feat(kuma-cp) removed dataplane token server

app/kumactl/pkg/tokens/client_test.go

@@ -3,8 +3,9 @@ package tokens_test
 import (
 	"fmt"
 	"github.com/Kong/kuma/app/kumactl/pkg/tokens"
+	admin_server "github.com/Kong/kuma/pkg/admin-server"
+	admin_server_config "github.com/Kong/kuma/pkg/config/admin-server"
 	config_kumactl "github.com/Kong/kuma/pkg/config/app/kumactl/v1alpha1"
-	token_server "github.com/Kong/kuma/pkg/config/token-server"
 	"github.com/Kong/kuma/pkg/core/xds"
 	"github.com/Kong/kuma/pkg/sds/auth"
 	"github.com/Kong/kuma/pkg/test"
@@ -45,23 +46,23 @@ var _ = Describe("Tokens Client", func() {
 		Expect(err).ToNot(HaveOccurred())
 		publicPort = p
 
-		srv := server.DataplaneTokenServer{
-			Config: &token_server.DataplaneTokenServerConfig{
+		adminCfg := admin_server_config.AdminServerConfig{
+			DataplaneTokenWs: &admin_server_config.DataplaneTokenWsConfig{
 				Enabled: true,
-				Local: &token_server.LocalDataplaneTokenServerConfig{
-					Port: uint32(port),
-				},
-				Public: &token_server.PublicDataplaneTokenServerConfig{
-					Enabled:        true,
-					Port:           uint32(publicPort),
-					Interface:      "localhost",
-					TlsCertFile:    filepath.Join("..", "..", "..", "..", "pkg", "tokens", "builtin", "server", "testdata", "server-cert.pem"),
-					TlsKeyFile:     filepath.Join("..", "..", "..", "..", "pkg", "tokens", "builtin", "server", "testdata", "server-key.pem"),
-					ClientCertsDir: filepath.Join("..", "..", "..", "..", "pkg", "tokens", "builtin", "server", "testdata", "authorized-clients"),
-				},
 			},
-			Issuer: &staticTokenIssuer{},
+			Local: &admin_server_config.LocalAdminServerConfig{
+				Port: uint32(port),
+			},
+			Public: &admin_server_config.PublicAdminServerConfig{
+				Enabled:        true,
+				Port:           uint32(publicPort),
+				Interface:      "localhost",
+				TlsCertFile:    filepath.Join("..", "..", "..", "..", "pkg", "admin-server", "testdata", "server-cert.pem"),
+				TlsKeyFile:     filepath.Join("..", "..", "..", "..", "pkg", "admin-server", "testdata", "server-key.pem"),
+				ClientCertsDir: filepath.Join("..", "..", "..", "..", "pkg", "admin-server", "testdata", "authorized-clients"),
+			},
 		}
+		srv := admin_server.NewAdminServer(adminCfg, server.NewWebservice(&staticTokenIssuer{}))
 
 		ch := make(chan struct{})
 		errCh := make(chan error)
@@ -105,8 +106,8 @@ var _ = Describe("Tokens Client", func() {
 				return fmt.Sprintf("https://localhost:%d", publicPort)
 			},
 			config: &config_kumactl.Context_DataplaneTokenApiCredentials{
-				ClientCert: filepath.Join("..", "..", "..", "..", "pkg", "tokens", "builtin", "server", "testdata", "authorized-client-cert.pem"),
-				ClientKey:  filepath.Join("..", "..", "..", "..", "pkg", "tokens", "builtin", "server", "testdata", "authorized-client-key.pem"),
+				ClientCert: filepath.Join("..", "..", "..", "..", "pkg", "admin-server", "testdata", "authorized-client-cert.pem"),
+				ClientKey:  filepath.Join("..", "..", "..", "..", "pkg", "admin-server", "testdata", "authorized-client-key.pem"),
 			},
 		}),
 	)

pkg/admin-server/server.go

@@ -2,6 +2,8 @@ package admin_server
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	admin_server "github.com/Kong/kuma/pkg/config/admin-server"
 	config_core "github.com/Kong/kuma/pkg/config/core"
@@ -13,7 +15,10 @@ import (
 	"github.com/emicklei/go-restful"
 	"github.com/pkg/errors"
 	"go.uber.org/multierr"
+	"io/ioutil"
 	"net/http"
+	"path/filepath"
+	"strings"
 )
 
 var (
@@ -39,16 +44,31 @@ func NewAdminServer(cfg admin_server.AdminServerConfig, services ...*restful.Web
 func (a *AdminServer) Start(stop <-chan struct{}) error {
 	httpServer, httpErrChan := a.startHttpServer()
 
+	var httpsServer *http.Server
+	var httpsErrChan chan error
+	if a.cfg.Public.Enabled {
+		httpsServer, httpsErrChan = a.startHttpsServer()
+	} else {
+		httpsErrChan = make(chan error)
+	}
+
 	select {
 	case <-stop:
 		log.Info("stopping")
 		var multiErr error
 		if err := httpServer.Shutdown(context.Background()); err != nil {
 			multiErr = multierr.Combine(err)
 		}
+		if httpsServer != nil {
+			if err := httpsServer.Shutdown(context.Background()); err != nil {
+				multiErr = multierr.Combine(err)
+			}
+		}
 		return multiErr
 	case err := <-httpErrChan:
 		return err
+	case err := <-httpsErrChan:
+		return err
 	}
 }
 
@@ -75,6 +95,63 @@ func (a *AdminServer) startHttpServer() (*http.Server, chan error) {
 	return server, errChan
 }
 
+func (a *AdminServer) startHttpsServer() (*http.Server, chan error) {
+	errChan := make(chan error)
+
+	tlsConfig, err := requireClientCerts(a.cfg.Public.ClientCertsDir)
+	if err != nil {
+		errChan <- err
+	}
+
+	server := &http.Server{
+		Addr:      fmt.Sprintf("%s:%d", a.cfg.Public.Interface, a.cfg.Public.Port),
+		Handler:   a.container,
+		TLSConfig: tlsConfig,
+	}
+
+	go func() {
+		defer close(errChan)
+		if err := server.ListenAndServeTLS(a.cfg.Public.TlsCertFile, a.cfg.Public.TlsKeyFile); err != nil {
+			if err != http.ErrServerClosed {
+				log.Error(err, "https server terminated with an error")
+				errChan <- err
+				return
+			}
+		}
+		log.Info("https server terminated normally")
+	}()
+	log.Info("starting server", "interface", a.cfg.Public.Interface, "port", a.cfg.Public.Port, "tls", true)
+	return server, errChan
+}
+
+func requireClientCerts(certsDir string) (*tls.Config, error) {
+	files, err := ioutil.ReadDir(certsDir)
+	if err != nil {
+		return nil, err
+	}
+	clientCertPool := x509.NewCertPool()
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+		if !strings.HasSuffix(file.Name(), ".pem") {
+			continue
+		}
+		path := filepath.Join(certsDir, file.Name())
+		caCert, err := ioutil.ReadFile(path)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not read certificate %s", path)
+		}
+		clientCertPool.AppendCertsFromPEM(caCert)
+	}
+	tlsConfig := &tls.Config{
+		ClientCAs:  clientCertPool,
+		ClientAuth: tls.RequireAndVerifyClientCert,
+	}
+	tlsConfig.BuildNameToCertificate()
+	return tlsConfig, nil
+}
+
 func SetupServer(rt runtime.Runtime) error {
 	var webservices []*restful.WebService
 
@@ -95,7 +172,7 @@ func SetupServer(rt runtime.Runtime) error {
 
 func dataplaneTokenWs(rt runtime.Runtime) (*restful.WebService, error) {
 	if !rt.Config().AdminServer.DataplaneTokenWs.Enabled {
-		log.Info("Dataplane Token Webservice disabled")
+		log.Info("Dataplane Token Webservice is disabled. Dataplane Tokens won't be verified.")
 		return nil, nil
 	}
 

pkg/admin-server/server_suite_test.go

@@ -0,0 +1,13 @@
+package admin_server_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestAdminServer(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Admin Server Suite")
+}

pkg/admin-server/server_test.go

@@ -0,0 +1,146 @@
+package admin_server_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"path/filepath"
+
+	admin_server "github.com/Kong/kuma/pkg/admin-server"
+	admin_server_config "github.com/Kong/kuma/pkg/config/admin-server"
+	"github.com/Kong/kuma/pkg/test"
+	util_http "github.com/Kong/kuma/pkg/util/http"
+	"github.com/emicklei/go-restful"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Admin Server", func() {
+
+	httpsClient := func(name string) *http.Client {
+		httpClient := &http.Client{}
+		err := util_http.ConfigureTls(
+			httpClient,
+			filepath.Join("testdata", "server-cert.pem"),
+			filepath.Join("testdata", fmt.Sprintf("%s-cert.pem", name)),
+			filepath.Join("testdata", fmt.Sprintf("%s-key.pem", name)),
+		)
+		Expect(err).ToNot(HaveOccurred())
+		return httpClient
+	}
+
+	var port int
+	var publicPort int
+
+	BeforeEach(func() {
+		// setup server
+		p, err := test.GetFreePort()
+		Expect(err).ToNot(HaveOccurred())
+		port = p
+		p, err = test.GetFreePort()
+		Expect(err).ToNot(HaveOccurred())
+		publicPort = p
+
+		cfg := admin_server_config.AdminServerConfig{
+			Local: &admin_server_config.LocalAdminServerConfig{
+				Port: uint32(port),
+			},
+			Public: &admin_server_config.PublicAdminServerConfig{
+				Enabled:        true,
+				Port:           uint32(publicPort),
+				Interface:      "localhost",
+				TlsCertFile:    filepath.Join("testdata", "server-cert.pem"),
+				TlsKeyFile:     filepath.Join("testdata", "server-key.pem"),
+				ClientCertsDir: filepath.Join("testdata", "authorized-clients"),
+			},
+		}
+
+		srv := admin_server.NewAdminServer(cfg, pingWs())
+
+		ch := make(chan struct{})
+		go func() {
+			defer GinkgoRecover()
+			Expect(srv.Start(ch)).ToNot(HaveOccurred())
+		}()
+
+		// wait for server to start
+		Eventually(func() error {
+			req, err := http.NewRequest("GET", fmt.Sprintf("http://localhost:%d/ping", port), nil)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = http.DefaultClient.Do(req)
+			return err
+		}, "5s", "100ms").ShouldNot(HaveOccurred())
+
+		// and should response on public port
+		Eventually(func() error {
+			req, err := http.NewRequest("GET", fmt.Sprintf("https://localhost:%d/ping", publicPort), nil)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = httpsClient("authorized-client").Do(req)
+			return err
+		}, "5s", "100ms").ShouldNot(HaveOccurred())
+	})
+
+	It("should serve on http", func() {
+		// given
+		req, err := http.NewRequest("GET", fmt.Sprintf("http://localhost:%d/ping", port), nil)
+		Expect(err).ToNot(HaveOccurred())
+
+		// when
+		resp, err := http.DefaultClient.Do(req)
+
+		// then
+		Expect(err).ToNot(HaveOccurred())
+
+		// when
+		bytes, err := ioutil.ReadAll(resp.Body)
+
+		// then
+		Expect(err).ToNot(HaveOccurred())
+		Expect(string(bytes)).To(Equal("pong"))
+	})
+
+	It("should serve on https for authorized client", func() {
+		// given
+		req, err := http.NewRequest("GET", fmt.Sprintf("https://localhost:%d/ping", publicPort), nil)
+		Expect(err).ToNot(HaveOccurred())
+
+		// when
+		resp, err := httpsClient("authorized-client").Do(req)
+
+		// then
+		Expect(err).ToNot(HaveOccurred())
+
+		// when
+		bytes, err := ioutil.ReadAll(resp.Body)
+
+		// then
+		Expect(err).ToNot(HaveOccurred())
+		Expect(string(bytes)).To(Equal("pong"))
+	})
+
+	It("should not let unauthorized to use an https server", func() {
+		// given
+		req, err := http.NewRequest("GET", fmt.Sprintf("https://localhost:%d/ping", publicPort), nil)
+		Expect(err).ToNot(HaveOccurred())
+
+		// when
+		_, err = httpsClient("unauthorized-client").Do(req)
+
+		// then
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(HaveSuffix("tls: bad certificate"))
+	})
+})
+
+func pingWs() *restful.WebService {
+	ws := new(restful.WebService).
+		Consumes(restful.MIME_JSON).
+		Produces(restful.MIME_JSON)
+	ws.Path("/").
+		Route(ws.GET("/ping").To(func(request *restful.Request, response *restful.Response) {
+			_, _ = response.Write([]byte(`pong`))
+		}),
+	)
+	return ws
+}

pkg/api-server/config_ws_test.go

@@ -79,7 +79,6 @@ var _ = Describe("Config WS", func() {
             }
           },
           "adminServer": {
-            "enabled": true,
             "local": {
               "port": 5679
             },
@@ -91,7 +90,7 @@ var _ = Describe("Config WS", func() {
               "tlsCertFile": "",
               "tlsKeyFile": ""
             },
-			"dataplaneTokenWs": {
+            "dataplaneTokenWs": {
               "enabled": true
             }
           },

pkg/config/app/kuma-cp/kuma-cp.defaults.yaml

@@ -79,8 +79,6 @@ dataplaneTokenServer:
 
 # Admin server configuration
 adminServer:
-  # If true then Admin Server and token verification is enabled
-  enabled: true
   # Local configuration of server that is available only on localhost
   local:
     # Port on which the server will be exposed

pkg/core/bootstrap/autoconfig.go

@@ -73,7 +73,7 @@ func autoconfigureSds(cfg *kuma_cp.Config) error {
 func autoconfigureAdminServer(cfg *kuma_cp.Config) {
 	// use old dataplane token server config values for admin server
 	if !reflect.DeepEqual(cfg.DataplaneTokenServer, token_server.DefaultDataplaneTokenServerConfig()) {
-		autoconfigureLog.Info("Deprecated config DataplaneTokenServer is used. Use AdminServer config instead.")
+		autoconfigureLog.Info("Deprecated DataplaneTokenServer config is used. It will be removed in the next major version of Kuma - use AdminServer config instead.")
 		cfg.AdminServer = &admin_server.AdminServerConfig{
 			DataplaneTokenWs: &admin_server.DataplaneTokenWsConfig{
 				Enabled: cfg.DataplaneTokenServer.Enabled,