feat(kuma-dp) retry connection to the CP and for fetching bootstrap (#…

…982)

Signed-off-by: Jakub Dyszkiewicz <jakub.dyszkiewicz@gmail.com>

app/kuma-dp/cmd/run.go

@@ -1,30 +1,31 @@
 package cmd
 
 import (
+	"context"
 	"crypto/tls"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
 	"time"
 
-	kuma_version "github.com/kumahq/kuma/pkg/version"
-
-	"github.com/kumahq/kuma/pkg/catalog/client"
-
 	"github.com/pkg/errors"
+	"github.com/sethvargo/go-retry"
 	"github.com/spf13/cobra"
 
 	kumadp_config "github.com/kumahq/kuma/app/kuma-dp/pkg/config"
 	"github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/accesslogs"
 	"github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/envoy"
+	"github.com/kumahq/kuma/pkg/catalog"
+	"github.com/kumahq/kuma/pkg/catalog/client"
 	"github.com/kumahq/kuma/pkg/config"
 	kuma_dp "github.com/kumahq/kuma/pkg/config/app/kuma-dp"
 	config_types "github.com/kumahq/kuma/pkg/config/types"
 	"github.com/kumahq/kuma/pkg/core"
 	"github.com/kumahq/kuma/pkg/core/runtime/component"
 	leader_memory "github.com/kumahq/kuma/pkg/plugins/leader/memory"
 	util_net "github.com/kumahq/kuma/pkg/util/net"
+	kuma_version "github.com/kumahq/kuma/pkg/version"
 )
 
 type CatalogClientFactory func(string) (client.CatalogClient, error)
@@ -52,20 +53,16 @@ func newRunCmd() *cobra.Command {
 				runLog.Error(err, "unable to load configuration")
 				return err
 			}
-			if conf, err := config.ToYAML(&cfg); err == nil {
+			if conf, err := config.ToJson(&cfg); err == nil {
 				runLog.Info("effective configuration", "config", string(conf))
 			} else {
 				runLog.Error(err, "unable to format effective configuration", "config", cfg)
 				return err
 			}
 
-			catalogClient, err := catalogClientFactory(cfg.ControlPlane.ApiServer.URL)
-			if err != nil {
-				return errors.Wrap(err, "could not create catalog client")
-			}
-			catalog, err := catalogClient.Catalog()
+			catalog, err := fetchCatalog(cfg)
 			if err != nil {
-				return errors.Wrap(err, "could retrieve catalog")
+				return err
 			}
 			if catalog.Apis.DataplaneToken.Enabled() {
 				if cfg.DataplaneRuntime.TokenPath == "" && cfg.DataplaneRuntime.Token == "" {
@@ -114,7 +111,7 @@ func newRunCmd() *cobra.Command {
 			}
 
 			dataplane, err := envoy.New(envoy.Opts{
-				Catalog:   catalog,
+				Catalog:   *catalog,
 				Config:    cfg,
 				Generator: bootstrapGenerator,
 				Stdout:    cmd.OutOrStdout(),
@@ -157,3 +154,34 @@ func writeFile(filename string, data []byte, perm os.FileMode) error {
 	}
 	return ioutil.WriteFile(filename, data, perm)
 }
+
+// fetchCatalog tries to fetch Kuma CP catalog several times
+// The main reason for introducing retries here is situation when DP is deployed in the same time as CP (ex. Ingress for Remote CP)
+func fetchCatalog(cfg kuma_dp.Config) (*catalog.Catalog, error) {
+	runLog.Info("connecting to the Control Plane API for Bootstrap API location")
+	catalogClient, err := catalogClientFactory(cfg.ControlPlane.ApiServer.URL)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create catalog client")
+	}
+
+	backoff, err := retry.NewConstant(cfg.ControlPlane.ApiServer.Retry.Backoff)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create retry backoff")
+	}
+	backoff = retry.WithMaxDuration(cfg.ControlPlane.ApiServer.Retry.MaxDuration, backoff)
+	var c catalog.Catalog
+	err = retry.Do(context.Background(), backoff, func(ctx context.Context) error {
+		c, err = catalogClient.Catalog()
+		if err != nil {
+			runLog.Info("could not connect to the Control Plane API. Retrying.", "backoff", cfg.ControlPlane.ApiServer.Retry.Backoff, "err", err.Error())
+			return retry.RetryableError(err)
+		}
+		return nil
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "could not retrieve catalog")
+	}
+	runLog.Info("connection successful", "catalog", c)
+	return &c, nil
+}

app/kuma-dp/pkg/dataplane/envoy/envoy.go

@@ -100,6 +100,7 @@ func lookupEnvoyPath(configuredPath string) (string, error) {
 }
 
 func (e *Envoy) Start(stop <-chan struct{}) error {
+	runLog.Info("generating bootstrap configuration")
 	bootstrapConfig, err := e.opts.Generator(e.opts.Catalog.Apis.Bootstrap.Url, e.opts.Config)
 	if err != nil {
 		return errors.Errorf("Failed to generate Envoy bootstrap config. %v", err)
@@ -108,6 +109,7 @@ func (e *Envoy) Start(stop <-chan struct{}) error {
 	if err != nil {
 		return err
 	}
+	runLog.Info("bootstrap configuration saved to a file", "file", configFile)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go

@@ -2,6 +2,7 @@ package envoy
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"io/ioutil"
 	"net/http"
@@ -10,8 +11,10 @@ import (
 	envoy_bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2"
 	"github.com/golang/protobuf/proto"
 	"github.com/pkg/errors"
+	"github.com/sethvargo/go-retry"
 
 	kuma_dp "github.com/kumahq/kuma/pkg/config/app/kuma-dp"
+	"github.com/kumahq/kuma/pkg/core"
 	util_proto "github.com/kumahq/kuma/pkg/util/proto"
 	"github.com/kumahq/kuma/pkg/xds/bootstrap/types"
 )
@@ -25,12 +28,51 @@ func NewRemoteBootstrapGenerator(client *http.Client) BootstrapConfigFactoryFunc
 	return rb.Generate
 }
 
+var (
+	log           = core.Log.WithName("dataplane")
+	DpNotFoundErr = errors.New("Dataplane entity not found. If you are running on Universal please create a Dataplane entity on kuma-cp before starting kuma-dp. If you are running on Kubernetes, please check the kuma-cp logs to determine why the Dataplane entity could not be created by the automatic sidecar injection.")
+)
+
 func (b *remoteBootstrap) Generate(url string, cfg kuma_dp.Config) (proto.Message, error) {
 	bootstrapUrl, err := net_url.Parse(url)
 	if err != nil {
 		return nil, err
 	}
-	bootstrapUrl.Path = "/bootstrap"
+
+	backoff, err := retry.NewConstant(cfg.ControlPlane.BootstrapServer.Retry.Backoff)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create retry backoff")
+	}
+	backoff = retry.WithMaxDuration(cfg.ControlPlane.BootstrapServer.Retry.MaxDuration, backoff)
+	var respBytes []byte
+	err = retry.Do(context.Background(), backoff, func(ctx context.Context) error {
+		log.Info("trying to fetch bootstrap configuration from the Control Plane")
+		respBytes, err = b.requestForBootstrap(bootstrapUrl, cfg)
+		if err == nil {
+			return nil
+		}
+		switch err {
+		case DpNotFoundErr:
+			log.Info("Dataplane entity is not yet found in the Control Plane. If you are running on Kubernetes, CP is most likely still in the process of converting Pod to Dataplane. Retrying.", "backoff", cfg.ControlPlane.ApiServer.Retry.Backoff)
+		default:
+			log.Info("could not fetch bootstrap configuration. Retrying.", "backoff", cfg.ControlPlane.BootstrapServer.Retry.Backoff, "err", err.Error())
+		}
+		return retry.RetryableError(err)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	bootstrap := envoy_bootstrap.Bootstrap{}
+	if err := util_proto.FromYAML(respBytes, &bootstrap); err != nil {
+		return nil, errors.Wrap(err, "could not parse the bootstrap configuration")
+	}
+
+	return &bootstrap, nil
+}
+
+func (b *remoteBootstrap) requestForBootstrap(url *net_url.URL, cfg kuma_dp.Config) ([]byte, error) {
+	url.Path = "/bootstrap"
 	request := types.BootstrapRequest{
 		Mesh: cfg.Dataplane.Mesh,
 		Name: cfg.Dataplane.Name,
@@ -43,14 +85,14 @@ func (b *remoteBootstrap) Generate(url string, cfg kuma_dp.Config) (proto.Messag
 	if err != nil {
 		return nil, errors.Wrap(err, "could not marshal request to json")
 	}
-	resp, err := b.client.Post(bootstrapUrl.String(), "application/json", bytes.NewReader(jsonBytes))
+	resp, err := b.client.Post(url.String(), "application/json", bytes.NewReader(jsonBytes))
 	if err != nil {
 		return nil, errors.Wrap(err, "request to bootstrap server failed")
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
 		if resp.StatusCode == http.StatusNotFound {
-			return nil, errors.New("Dataplane entity not found. If you are running on Universal please create a Dataplane entity on kuma-cp before starting kuma-dp. If you are running on Kubernetes, please check the kuma-cp logs to determine why the Dataplane entity could not be created by the automatic sidecar injection.")
+			return nil, DpNotFoundErr
 		}
 		if resp.StatusCode == http.StatusUnprocessableEntity {
 			bodyBytes, err := ioutil.ReadAll(resp.Body)
@@ -61,15 +103,9 @@ func (b *remoteBootstrap) Generate(url string, cfg kuma_dp.Config) (proto.Messag
 		}
 		return nil, errors.Errorf("unexpected status code: %d", resp.StatusCode)
 	}
-
-	bootstrap := envoy_bootstrap.Bootstrap{}
 	respBytes, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not read the body of the response")
 	}
-	if err := util_proto.FromYAML(respBytes, &bootstrap); err != nil {
-		return nil, errors.Wrap(err, "could not parse the bootstrap configuration")
-	}
-
-	return &bootstrap, nil
+	return respBytes, nil
 }

app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/ginkgo/extensions/table"
@@ -115,6 +116,40 @@ var _ = Describe("Remote Bootstrap", func() {
 			}()),
 	)
 
+	It("should retry when DP is not found", func() {
+		// given
+		mux := http.NewServeMux()
+		server := httptest.NewServer(mux)
+		defer server.Close()
+		i := 0
+		mux.HandleFunc("/bootstrap", func(writer http.ResponseWriter, req *http.Request) {
+			defer GinkgoRecover()
+			if i < 2 {
+				writer.WriteHeader(404)
+				i++
+			} else {
+				response, err := ioutil.ReadFile(filepath.Join("testdata", "remote-bootstrap-config.golden.yaml"))
+				Expect(err).ToNot(HaveOccurred())
+				_, err = writer.Write(response)
+				Expect(err).ToNot(HaveOccurred())
+			}
+		})
+		port, err := strconv.Atoi(strings.Split(server.Listener.Addr().String(), ":")[1])
+		Expect(err).ToNot(HaveOccurred())
+
+		// and
+		generator := NewRemoteBootstrapGenerator(http.DefaultClient)
+
+		// when
+		cfg := kuma_dp.DefaultConfig()
+		cfg.ControlPlane.BootstrapServer.Retry.Backoff = 10 * time.Millisecond
+		_, err = generator(fmt.Sprintf("http://localhost:%d", port), cfg)
+
+		// then
+		Expect(err).ToNot(HaveOccurred())
+		Expect(cfg).ToNot(BeNil())
+	})
+
 	It("should return error when DP is not found", func() {
 		// given
 		mux := http.NewServeMux()
@@ -131,9 +166,12 @@ var _ = Describe("Remote Bootstrap", func() {
 		generator := NewRemoteBootstrapGenerator(http.DefaultClient)
 
 		// when
-		_, err = generator(fmt.Sprintf("http://localhost:%d", port), kuma_dp.DefaultConfig())
+		config := kuma_dp.DefaultConfig()
+		config.ControlPlane.BootstrapServer.Retry.Backoff = 10 * time.Millisecond
+		config.ControlPlane.BootstrapServer.Retry.MaxDuration = 100 * time.Millisecond
+		_, err = generator(fmt.Sprintf("http://localhost:%d", port), config)
 
 		// then
-		Expect(err).To(MatchError("Dataplane entity not found. If you are running on Universal please create a Dataplane entity on kuma-cp before starting kuma-dp. If you are running on Kubernetes, please check the kuma-cp logs to determine why the Dataplane entity could not be created by the automatic sidecar injection."))
+		Expect(err).To(MatchError("retryable: Dataplane entity not found. If you are running on Universal please create a Dataplane entity on kuma-cp before starting kuma-dp. If you are running on Kubernetes, please check the kuma-cp logs to determine why the Dataplane entity could not be created by the automatic sidecar injection."))
 	})
 })

go.mod

@@ -36,6 +36,7 @@ require (
 	github.com/pkg/errors v0.9.0
 	github.com/prometheus/common v0.9.1
 	github.com/prometheus/prometheus v0.0.0-00010101000000-000000000000
+	github.com/sethvargo/go-retry v0.1.0
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749
 	github.com/spf13/cobra v1.0.0
 	github.com/spiffe/go-spiffe v0.0.0-20190820222348-6adcf1eecbcc

go.sum

@@ -498,6 +498,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sethvargo/go-retry v0.1.0 h1:8sPqlWannzcReEcYjHSNw9becsiYudcwTD7CasGjQaI=
+github.com/sethvargo/go-retry v0.1.0/go.mod h1:JzIOdZqQDNpPkQDmcqgtteAcxFLtYpNF/zJCM1ysDg8=
 github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=