From 8e1fa7812ca2d41089b0c1465a3b34af20f63de5 Mon Sep 17 00:00:00 2001 From: Iryna Shustava Date: Tue, 7 Apr 2020 23:51:25 -0700 Subject: [PATCH] Support external servers * Add -bootstrap-token-file to provide your own bootstrap token. If provided, server-acl-init will skip ACL bootstrapping of the servers and will not update server policies and set tokens. * The -server-address flag now can also be a cloud auto-join string. This enables us to re-use the same string you would use for retry-join. --- helper/go-discover/discover.go | 61 ++++++ subcommand/get-consul-client-ca/command.go | 44 +---- subcommand/server-acl-init/anonymous_token.go | 2 +- subcommand/server-acl-init/command.go | 138 +++++++++----- subcommand/server-acl-init/command_test.go | 177 ++++++++++++++++++ .../server-acl-init/create_or_update.go | 8 +- subcommand/server-acl-init/servers.go | 12 +- subcommand/sync-catalog/command_ent_test.go | 2 - 8 files changed, 342 insertions(+), 102 deletions(-) create mode 100644 helper/go-discover/discover.go diff --git a/helper/go-discover/discover.go b/helper/go-discover/discover.go new file mode 100644 index 0000000000..f94621158a --- /dev/null +++ b/helper/go-discover/discover.go @@ -0,0 +1,61 @@ +package godiscover + +import ( + "fmt" + "strings" + + "github.com/hashicorp/consul-k8s/version" + "github.com/hashicorp/go-discover" + discoverk8s "github.com/hashicorp/go-discover/provider/k8s" + "github.com/hashicorp/go-hclog" +) + +// ConsulServerAddresses uses go-discover to discover Consul servers +// provided by the 'discoverString' and returns them. +func ConsulServerAddresses(discoverString string, providers map[string]discover.Provider, logger hclog.Logger) ([]string, error) { + // If it's a cloud-auto join string, discover server addresses through the cloud provider. + // This code was adapted from + // https://github.com/hashicorp/consul/blob/c5fe112e59f6e8b03159ec8f2dbe7f4a026ce823/agent/retry_join.go#L55-L89. + disco, err := newDiscover(providers) + if err != nil { + return nil, err + } + logger.Debug("using cloud auto-join", "server-addr", discoverString) + servers, err := disco.Addrs(discoverString, logger.StandardLogger(&hclog.StandardLoggerOptions{ + InferLevels: true, + })) + if err != nil { + return nil, err + } + + // check if we discovered any servers + if len(servers) == 0 { + return nil, fmt.Errorf("could not discover any Consul servers with %q", discoverString) + } + + logger.Debug("discovered servers", "servers", strings.Join(servers, " ")) + + return servers, nil +} + +// newDiscover initializes the new Discover object +// set up with all predefined providers, as well as +// the k8s provider. +// This code was adapted from +// https://github.com/hashicorp/consul/blob/c5fe112e59f6e8b03159ec8f2dbe7f4a026ce823/agent/retry_join.go#L42-L53 +func newDiscover(providers map[string]discover.Provider) (*discover.Discover, error) { + if providers == nil { + providers = make(map[string]discover.Provider) + } + + for k, v := range discover.Providers { + providers[k] = v + } + providers["k8s"] = &discoverk8s.Provider{} + + userAgent := fmt.Sprintf("consul-k8s/%s (https://www.consul.io/)", version.GetHumanVersion()) + return discover.New( + discover.WithUserAgent(userAgent), + discover.WithProviders(providers), + ) +} diff --git a/subcommand/get-consul-client-ca/command.go b/subcommand/get-consul-client-ca/command.go index 36c1012c47..4453f48a8e 100644 --- a/subcommand/get-consul-client-ca/command.go +++ b/subcommand/get-consul-client-ca/command.go @@ -10,11 +10,10 @@ import ( "time" "github.com/cenkalti/backoff" - "github.com/hashicorp/consul-k8s/version" + godiscover "github.com/hashicorp/consul-k8s/helper/go-discover" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/command/flags" "github.com/hashicorp/go-discover" - discoverk8s "github.com/hashicorp/go-discover/provider/k8s" "github.com/hashicorp/go-hclog" "github.com/mitchellh/cli" ) @@ -175,27 +174,10 @@ func (c *Command) consulServerAddr(logger hclog.Logger) (string, error) { return fmt.Sprintf("%s:%s", c.flagServerAddr, c.flagServerPort), nil } - // If it's a cloud-auto join string, discover server addresses through the cloud provider. - // This code was adapted from - // https://github.com/hashicorp/consul/blob/c5fe112e59f6e8b03159ec8f2dbe7f4a026ce823/agent/retry_join.go#L55-L89. - disco, err := c.newDiscover() + servers, err := godiscover.ConsulServerAddresses(c.flagServerAddr, c.providers, logger) if err != nil { return "", err } - logger.Debug("using cloud auto-join", "server-addr", c.flagServerAddr) - servers, err := disco.Addrs(c.flagServerAddr, logger.StandardLogger(&hclog.StandardLoggerOptions{ - InferLevels: true, - })) - if err != nil { - return "", err - } - - // check if we discovered any servers - if len(servers) == 0 { - return "", fmt.Errorf("could not discover any Consul servers with %q", c.flagServerAddr) - } - - logger.Debug("discovered servers", "servers", strings.Join(servers, " ")) // Pick the first server from the list, // ignoring the port since we need to use HTTP API @@ -204,28 +186,6 @@ func (c *Command) consulServerAddr(logger hclog.Logger) (string, error) { return fmt.Sprintf("%s:%s", firstServer, c.flagServerPort), nil } -// newDiscover initializes the new Discover object -// set up with all predefined providers, as well as -// the k8s provider. -// This code was adapted from -// https://github.com/hashicorp/consul/blob/c5fe112e59f6e8b03159ec8f2dbe7f4a026ce823/agent/retry_join.go#L42-L53 -func (c *Command) newDiscover() (*discover.Discover, error) { - if c.providers == nil { - c.providers = make(map[string]discover.Provider) - } - - for k, v := range discover.Providers { - c.providers[k] = v - } - c.providers["k8s"] = &discoverk8s.Provider{} - - userAgent := fmt.Sprintf("consul-k8s/%s (https://www.consul.io/)", version.GetHumanVersion()) - return discover.New( - discover.WithUserAgent(userAgent), - discover.WithProviders(c.providers), - ) -} - // getActiveRoot returns the currently active root // from the roots list, otherwise returns error. func getActiveRoot(roots *api.CARootList) (string, error) { diff --git a/subcommand/server-acl-init/anonymous_token.go b/subcommand/server-acl-init/anonymous_token.go index 43d8e4e456..3423ee78da 100644 --- a/subcommand/server-acl-init/anonymous_token.go +++ b/subcommand/server-acl-init/anonymous_token.go @@ -9,7 +9,7 @@ import ( func (c *Command) configureAnonymousPolicy(consulClient *api.Client) error { anonRules, err := c.anonymousTokenRules() if err != nil { - c.Log.Error("Error templating anonymous token rules", "err", err) + c.log.Error("Error templating anonymous token rules", "err", err) return err } diff --git a/subcommand/server-acl-init/command.go b/subcommand/server-acl-init/command.go index 9cd5d51ef2..6eae5f4658 100644 --- a/subcommand/server-acl-init/command.go +++ b/subcommand/server-acl-init/command.go @@ -11,6 +11,7 @@ import ( "sync" "time" + godiscover "github.com/hashicorp/consul-k8s/helper/go-discover" "github.com/hashicorp/consul-k8s/subcommand" k8sflags "github.com/hashicorp/consul-k8s/subcommand/flags" "github.com/hashicorp/consul/api" @@ -26,8 +27,8 @@ import ( type Command struct { UI cli.Ui - flags *flag.FlagSet - k8s *k8sflags.K8SFlags + flags *flag.FlagSet + k8s *k8sflags.K8SFlags flagResourcePrefix string flagK8sNamespace string @@ -43,7 +44,7 @@ type Command struct { flagCreateMeshGatewayToken bool // Flags to configure Consul client - flagServerAddresses []string + flagServerAddresses []string flagServerPort uint flagConsulCACert string flagConsulTLSServerName string @@ -62,16 +63,20 @@ type Command struct { flagEnableInjectK8SNSMirroring bool // Enables mirroring of k8s namespaces into Consul for Connect inject flagInjectK8SNSMirroringPrefix string // Prefix added to Consul namespaces created when mirroring injected services + // Flag to support a custom bootstrap token + flagBootstrapTokenFile string + flagLogLevel string flagTimeout time.Duration clientset kubernetes.Interface + // cmdTimeout is cancelled when the command timeout is reached. cmdTimeout context.Context retryDuration time.Duration - // Log - Log hclog.Logger + // log + log hclog.Logger once sync.Once help string @@ -84,13 +89,14 @@ func (c *Command) init() { c.flags.StringVar(&c.flagResourcePrefix, "resource-prefix", "", "Prefix to use for Kubernetes resources. If not set, the \"-consul\" prefix is used, where is the value set by the -release-name flag.") c.flags.StringVar(&c.flagK8sNamespace, "k8s-namespace", "", - "Name of Kubernetes namespace where the servers are deployed") + "Name of Kubernetes namespace where Consul and consul-k8s components are deployed.") + c.flags.BoolVar(&c.flagAllowDNS, "allow-dns", false, "Toggle for updating the anonymous token to allow DNS queries to work") c.flags.BoolVar(&c.flagCreateClientToken, "create-client-token", true, - "Toggle for creating a client agent token") + "Toggle for creating a client agent token. Default is true.") c.flags.BoolVar(&c.flagCreateSyncToken, "create-sync-token", false, - "Toggle for creating a catalog sync token") + "Toggle for creating a catalog sync token.") c.flags.BoolVar(&c.flagCreateInjectToken, "create-inject-namespace-token", false, "Toggle for creating a connect injector token. Only required when namespaces are enabled.") c.flags.BoolVar(&c.flagCreateInjectAuthMethod, "create-inject-auth-method", false, @@ -100,15 +106,15 @@ func (c *Command) init() { c.flags.StringVar(&c.flagBindingRuleSelector, "acl-binding-rule-selector", "", "Selector string for connectInject ACL Binding Rule") c.flags.BoolVar(&c.flagCreateEntLicenseToken, "create-enterprise-license-token", false, - "Toggle for creating a token for the enterprise license job") + "Toggle for creating a token for the enterprise license job.") c.flags.BoolVar(&c.flagCreateSnapshotAgentToken, "create-snapshot-agent-token", false, - "Toggle for creating a token for the Consul snapshot agent deployment (enterprise only)") + "[Enterprise Only] Toggle for creating a token for the Consul snapshot agent deployment.") c.flags.BoolVar(&c.flagCreateMeshGatewayToken, "create-mesh-gateway-token", false, "Toggle for creating a token for a Connect mesh gateway") c.flags.Var((*flags.AppendSliceValue)(&c.flagServerAddresses), "server-address", - "The IP, DNS name or cloud auto-join string of the Consul server(s), may be provided multiple times." + - "At least one value is required.") + "The IP, DNS name or the cloud auto-join string of the Consul server(s), may be provided multiple times."+ + "At least one value is required.") c.flags.UintVar(&c.flagServerPort, "server-port", 8500, "The HTTP or HTTPS port of the Consul server. Defaults to 8500.") c.flags.StringVar(&c.flagConsulCACert, "consul-ca-cert", "", "Path to the PEM-encoded CA certificate of the Consul cluster.") @@ -140,10 +146,15 @@ func (c *Command) init() { "Toggle for creating a token for ACL replication between datacenters") c.flags.StringVar(&c.flagACLReplicationTokenFile, "acl-replication-token-file", "", "Path to file containing ACL token to be used for ACL replication. If set, ACL replication is enabled.") + + c.flags.StringVar(&c.flagBootstrapTokenFile, "bootstrap-token-file", "", + "Path to file containing ACL token for creating policies and tokens. This token must have 'acl:write' permissions."+ + "When provided, servers will not be bootstrapped and their policies and tokens will not be updated.") + c.flags.DurationVar(&c.flagTimeout, "timeout", 10*time.Minute, "How long we'll try to bootstrap ACLs for before timing out, e.g. 1ms, 2s, 3m") c.flags.StringVar(&c.flagLogLevel, "log-level", "info", - "Log verbosity level. Supported values (in order of detail) are \"trace\", "+ + "log verbosity level. Supported values (in order of detail) are \"trace\", "+ "\"debug\", \"info\", \"warn\", and \"error\".") c.k8s = &k8sflags.K8SFlags{} @@ -200,6 +211,21 @@ func (c *Command) Run(args []string) int { aclReplicationToken = strings.TrimSpace(string(tokenBytes)) } + var providedBootstrapToken string + if c.flagBootstrapTokenFile != "" { + // Load the bootstrap token from file. + tokenBytes, err := ioutil.ReadFile(c.flagBootstrapTokenFile) + if err != nil { + c.UI.Error(fmt.Sprintf("Unable to read ACL replication token from file %q: %s", c.flagBootstrapTokenFile, err)) + return 1 + } + if len(tokenBytes) == 0 { + c.UI.Error(fmt.Sprintf("ACL replication token file %q is empty", c.flagBootstrapTokenFile)) + return 1 + } + providedBootstrapToken = strings.TrimSpace(string(tokenBytes)) + } + var cancel context.CancelFunc c.cmdTimeout, cancel = context.WithTimeout(context.Background(), c.flagTimeout) // The context will only ever be intentionally ended by the timeout. @@ -211,15 +237,26 @@ func (c *Command) Run(args []string) int { c.UI.Error(fmt.Sprintf("Unknown log level: %s", c.flagLogLevel)) return 1 } - c.Log = hclog.New(&hclog.LoggerOptions{ + c.log = hclog.New(&hclog.LoggerOptions{ Level: level, Output: os.Stderr, }) + serverAddresses := c.flagServerAddresses + // check if the provided addresses is a cloud-auto join string + if len(c.flagServerAddresses) == 1 && strings.Contains(c.flagServerAddresses[0], "provider=") { + var err error + serverAddresses, err = godiscover.ConsulServerAddresses(c.flagServerAddresses[0], c.providers, c.log) + if err != nil { + c.UI.Error(fmt.Sprintf("Unable to discover any Consul addresses from %q: %s", c.flagServerAddresses[0], err)) + return 1 + } + } + // The ClientSet might already be set if we're in a test. if c.clientset == nil { if err := c.configureKubeClient(); err != nil { - c.Log.Error(err.Error()) + c.log.Error(err.Error()) return 1 } } @@ -232,12 +269,19 @@ func (c *Command) Run(args []string) int { var updateServerPolicy bool var bootstrapToken string - if c.flagACLReplicationTokenFile != "" { + if c.flagBootstrapTokenFile != "" { + // If bootstrap token is provided, + // we skip server bootstrapping and use + // the provided token to create policies and tokens + // for the rest of the components. + c.log.Info("Bootstrap token is provided so skipping ACL bootstrapping") + bootstrapToken = providedBootstrapToken + } else if c.flagACLReplicationTokenFile != "" { // If ACL replication is enabled, we don't need to ACL bootstrap the servers // since they will be performing replication. // We can use the replication token as our bootstrap token because it // has permissions to create policies and tokens. - c.Log.Info("ACL replication is enabled so skipping ACL bootstrapping") + c.log.Info("ACL replication is enabled so skipping ACL bootstrapping") bootstrapToken = aclReplicationToken } else { // Check if we've already been bootstrapped. @@ -245,12 +289,12 @@ func (c *Command) Run(args []string) int { bootTokenSecretName := c.withPrefix("bootstrap-acl-token") bootstrapToken, err = c.getBootstrapToken(bootTokenSecretName) if err != nil { - c.Log.Error(fmt.Sprintf("Unexpected error looking for preexisting bootstrap Secret: %s", err)) + c.log.Error(fmt.Sprintf("Unexpected error looking for preexisting bootstrap Secret: %s", err)) return 1 } if bootstrapToken != "" { - c.Log.Info(fmt.Sprintf("ACLs already bootstrapped - retrieved bootstrap token from Secret %q", bootTokenSecretName)) + c.log.Info(fmt.Sprintf("ACLs already bootstrapped - retrieved bootstrap token from Secret %q", bootTokenSecretName)) // Mark that we should update the server ACL policy in case // there are namespace related config changes. Because of the @@ -258,17 +302,17 @@ func (c *Command) Run(args []string) int { // otherwise won't be updated. updateServerPolicy = true } else { - c.Log.Info("No bootstrap token from previous installation found, continuing on to bootstrapping") - bootstrapToken, err = c.bootstrapServers(bootTokenSecretName, scheme) + c.log.Info("No bootstrap token from previous installation found, continuing on to bootstrapping") + bootstrapToken, err = c.bootstrapServers(serverAddresses, bootTokenSecretName, scheme) if err != nil { - c.Log.Error(err.Error()) + c.log.Error(err.Error()) return 1 } } } // For all of the next operations we'll need a Consul client. - serverAddr := fmt.Sprintf("%s:%d", c.flagServerAddresses[0], c.flagServerPort) + serverAddr := fmt.Sprintf("%s:%d", serverAddresses[0], c.flagServerPort) consulClient, err := api.NewClient(&api.Config{ Address: serverAddr, Scheme: scheme, @@ -279,16 +323,16 @@ func (c *Command) Run(args []string) int { }, }) if err != nil { - c.Log.Error(fmt.Sprintf("Error creating Consul client for addr %q: %s", serverAddr, err)) + c.log.Error(fmt.Sprintf("Error creating Consul client for addr %q: %s", serverAddr, err)) return 1 } consulDC, err := c.consulDatacenter(consulClient) if err != nil { - c.Log.Error("Error getting datacenter name", "err", err) + c.log.Error("Error getting datacenter name", "err", err) return 1 } - c.Log.Info("Current datacenter", "datacenter", consulDC) + c.log.Info("Current datacenter", "datacenter", consulDC) // With the addition of namespaces, the ACL policies associated // with the server tokens may need to be updated if Enterprise Consul @@ -297,7 +341,7 @@ func (c *Command) Run(args []string) int { if updateServerPolicy { _, err = c.setServerPolicy(consulClient) if err != nil { - c.Log.Error("Error updating the server ACL policy", "err", err) + c.log.Error("Error updating the server ACL policy", "err", err) return 1 } } @@ -319,7 +363,7 @@ func (c *Command) Run(args []string) int { return c.createOrUpdateACLPolicy(policyTmpl, consulClient) }) if err != nil { - c.Log.Error("Error creating or updating the cross namespace policy", "err", err) + c.log.Error("Error creating or updating the cross namespace policy", "err", err) return 1 } @@ -335,7 +379,7 @@ func (c *Command) Run(args []string) int { } _, _, err = consulClient.Namespaces().Update(&consulNamespace, &api.WriteOptions{}) if err != nil { - c.Log.Error("Error updating the default namespace to include the cross namespace policy", "err", err) + c.log.Error("Error updating the default namespace to include the cross namespace policy", "err", err) return 1 } } @@ -343,13 +387,13 @@ func (c *Command) Run(args []string) int { if c.flagCreateClientToken { agentRules, err := c.agentRules() if err != nil { - c.Log.Error("Error templating client agent rules", "err", err) + c.log.Error("Error templating client agent rules", "err", err) return 1 } err = c.createLocalACL("client", agentRules, consulDC, consulClient) if err != nil { - c.Log.Error(err.Error()) + c.log.Error(err.Error()) return 1 } } @@ -357,7 +401,7 @@ func (c *Command) Run(args []string) int { if c.createAnonymousPolicy() { err := c.configureAnonymousPolicy(consulClient) if err != nil { - c.Log.Error(err.Error()) + c.log.Error(err.Error()) return 1 } } @@ -365,7 +409,7 @@ func (c *Command) Run(args []string) int { if c.flagCreateSyncToken { syncRules, err := c.syncRules() if err != nil { - c.Log.Error("Error templating sync rules", "err", err) + c.log.Error("Error templating sync rules", "err", err) return 1 } @@ -377,7 +421,7 @@ func (c *Command) Run(args []string) int { err = c.createLocalACL("catalog-sync", syncRules, consulDC, consulClient) } if err != nil { - c.Log.Error(err.Error()) + c.log.Error(err.Error()) return 1 } } @@ -385,7 +429,7 @@ func (c *Command) Run(args []string) int { if c.flagCreateInjectToken { injectRules, err := c.injectRules() if err != nil { - c.Log.Error("Error templating inject rules", "err", err) + c.log.Error("Error templating inject rules", "err", err) return 1 } @@ -398,7 +442,7 @@ func (c *Command) Run(args []string) int { } if err != nil { - c.Log.Error(err.Error()) + c.log.Error(err.Error()) return 1 } } @@ -406,7 +450,7 @@ func (c *Command) Run(args []string) int { if c.flagCreateEntLicenseToken { err := c.createLocalACL("enterprise-license", entLicenseRules, consulDC, consulClient) if err != nil { - c.Log.Error(err.Error()) + c.log.Error(err.Error()) return 1 } } @@ -414,7 +458,7 @@ func (c *Command) Run(args []string) int { if c.flagCreateSnapshotAgentToken { err := c.createLocalACL("client-snapshot-agent", snapshotAgentRules, consulDC, consulClient) if err != nil { - c.Log.Error(err.Error()) + c.log.Error(err.Error()) return 1 } } @@ -422,7 +466,7 @@ func (c *Command) Run(args []string) int { if c.flagCreateMeshGatewayToken { meshGatewayRules, err := c.meshGatewayRules() if err != nil { - c.Log.Error("Error templating dns rules", "err", err) + c.log.Error("Error templating dns rules", "err", err) return 1 } @@ -430,7 +474,7 @@ func (c *Command) Run(args []string) int { // discover services in other datacenters. err = c.createGlobalACL("mesh-gateway", meshGatewayRules, consulDC, consulClient) if err != nil { - c.Log.Error(err.Error()) + c.log.Error(err.Error()) return 1 } } @@ -438,7 +482,7 @@ func (c *Command) Run(args []string) int { if c.flagCreateInjectAuthMethod { err := c.configureConnectInject(consulClient) if err != nil { - c.Log.Error(err.Error()) + c.log.Error(err.Error()) return 1 } } @@ -446,19 +490,19 @@ func (c *Command) Run(args []string) int { if c.flagCreateACLReplicationToken { rules, err := c.aclReplicationRules() if err != nil { - c.Log.Error("Error templating acl replication token rules", "err", err) + c.log.Error("Error templating acl replication token rules", "err", err) return 1 } // Policy must be global because it replicates from the primary DC // and so the primary DC needs to be able to accept the token. err = c.createGlobalACL("acl-replication", rules, consulDC, consulClient) if err != nil { - c.Log.Error(err.Error()) + c.log.Error(err.Error()) return 1 } } - c.Log.Info("server-acl-init completed successfully") + c.log.Info("server-acl-init completed successfully") return 0 } @@ -498,11 +542,11 @@ func (c *Command) untilSucceeds(opName string, op func() error) error { for { err := op() if err == nil { - c.Log.Info(fmt.Sprintf("Success: %s", opName)) + c.log.Info(fmt.Sprintf("Success: %s", opName)) break } - c.Log.Error(fmt.Sprintf("Failure: %s", opName), "err", err) - c.Log.Info("Retrying in " + c.retryDuration.String()) + c.log.Error(fmt.Sprintf("Failure: %s", opName), "err", err) + c.log.Info("Retrying in " + c.retryDuration.String()) // Wait on either the retry duration (in which case we continue) or the // overall command timeout. select { diff --git a/subcommand/server-acl-init/command_test.go b/subcommand/server-acl-init/command_test.go index 907b88e890..8db991a6b4 100644 --- a/subcommand/server-acl-init/command_test.go +++ b/subcommand/server-acl-init/command_test.go @@ -341,6 +341,104 @@ func TestRun_TokensReplicatedDC(t *testing.T) { } } +// Test creating each token type when the bootstrap token is provided. +func TestRun_TokensProvidedBootstrapToken(t *testing.T) { + t.Parallel() + + cases := []struct { + TokenFlag string + PolicyName string + SecretName string + }{ + { + TokenFlag: "-create-client-token", + PolicyName: "client-token", + SecretName: "my-prefix-client-acl-token", + }, + { + TokenFlag: "-create-sync-token", + PolicyName: "catalog-sync-token", + SecretName: "my-prefix-catalog-sync-acl-token", + }, + { + TokenFlag: "-create-inject-namespace-token", + PolicyName: "connect-inject-token", + SecretName: "my-prefix-connect-inject-acl-token", + }, + { + TokenFlag: "-create-enterprise-license-token", + PolicyName: "enterprise-license-token", + SecretName: "my-prefix-enterprise-license-acl-token", + }, + { + TokenFlag: "-create-snapshot-agent-token", + PolicyName: "client-snapshot-agent-token", + SecretName: "my-prefix-client-snapshot-agent-acl-token", + }, + { + TokenFlag: "-create-mesh-gateway-token", + PolicyName: "mesh-gateway-token", + SecretName: "my-prefix-mesh-gateway-acl-token", + }, + { + TokenFlag: "-create-acl-replication-token", + PolicyName: "acl-replication-token", + SecretName: "my-prefix-acl-replication-acl-token", + }, + } + for _, c := range cases { + t.Run(c.TokenFlag, func(t *testing.T) { + bootToken := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + tokenFile, fileCleanup := writeTempFile(t, bootToken) + defer fileCleanup() + + k8s, testAgent := completeBootstrappedSetup(t, bootToken) + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmdArgs := []string{ + "-k8s-namespace=" + ns, + "-bootstrap-token-file", tokenFile, + "-server-address", strings.Split(testAgent.HTTPAddr, ":")[0], + "-server-port", strings.Split(testAgent.HTTPAddr, ":")[1], + "-resource-prefix=my-prefix", + c.TokenFlag, + } + responseCode := cmd.Run(cmdArgs) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + + consul, err := api.NewClient(&api.Config{ + Address: testAgent.HTTPAddr, + Token: bootToken, + }) + require.NoError(t, err) + + // Check that the expected policy was created. + retry.Run(t, func(r *retry.R) { + policyExists(r, c.PolicyName, consul) + }) + + retry.Run(t, func(r *retry.R) { + // Test that the token was created as a Kubernetes Secret. + tokenSecret, err := k8s.CoreV1().Secrets(ns).Get(c.SecretName, metav1.GetOptions{}) + require.NoError(r, err) + require.NotNil(r, tokenSecret) + token, ok := tokenSecret.Data["token"] + require.True(r, ok) + + // Test that the token has the expected policies in Consul. + tokenData, _, err := consul.ACL().TokenReadSelf(&api.QueryOptions{Token: string(token)}) + require.NoError(r, err) + require.Equal(r, c.PolicyName, tokenData.Policies[0].Name) + }) + }) + } +} + // Test the conditions under which we should create the anonymous token // policy. func TestRun_AnonymousTokenPolicy(t *testing.T) { @@ -1010,6 +1108,71 @@ func TestRun_AlreadyBootstrapped(t *testing.T) { }, consulAPICalls) } +// Test if there is a provided bootstrap we skip bootstrapping of the servers +// and continue on to the next step. +func TestRun_SkipBootstrapping_WhenBootstrapTokenIsProvided(t *testing.T) { + t.Parallel() + require := require.New(t) + k8s := fake.NewSimpleClientset() + + bootToken := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + tokenFile, fileCleanup := writeTempFile(t, bootToken) + defer fileCleanup() + + type APICall struct { + Method string + Path string + } + var consulAPICalls []APICall + + // Start the Consul server. + consulServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Record all the API calls made. + consulAPICalls = append(consulAPICalls, APICall{ + Method: r.Method, + Path: r.URL.Path, + }) + switch r.URL.Path { + case "/v1/agent/self": + fmt.Fprintln(w, `{"Config": {"Datacenter": "dc1"}}`) + default: + // Send an empty JSON response with code 200 to all calls. + fmt.Fprintln(w, "{}") + } + })) + defer consulServer.Close() + + // Create the Server Pods. + serverURL, err := url.Parse(consulServer.URL) + require.NoError(err) + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + + responseCode := cmd.Run([]string{ + "-resource-prefix=" + resourcePrefix, + "-k8s-namespace=" + ns, + "-server-address=" + serverURL.Hostname(), + "-server-port=" + serverURL.Port(), + "-bootstrap-token-file=" + tokenFile, + "-create-client-token=false", // disable client token, so there are less calls + }) + require.Equal(0, responseCode, ui.ErrorWriter.String()) + + // Test that the expected API calls were made. + require.Equal([]APICall{ + // We only expect the calls to get the datacenter + { + "GET", + "/v1/agent/self", + }, + }, consulAPICalls) +} + // Test that we exit after timeout. func TestRun_Timeout(t *testing.T) { t.Parallel() @@ -1228,6 +1391,20 @@ func completeSetup(t *testing.T) (*fake.Clientset, *testutil.TestServer) { return k8s, svr } +// Set up test consul agent and kubernetes cluster. +// The consul agent is bootstrapped with the master token. +func completeBootstrappedSetup(t *testing.T, masterToken string) (*fake.Clientset, *testutil.TestServer) { + k8s := fake.NewSimpleClientset() + + svr, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { + c.ACL.Enabled = true + c.ACL.Tokens.Master = masterToken + }) + require.NoError(t, err) + + return k8s, svr +} + // completeReplicatedSetup sets up two Consul servers with ACL replication // using the server-acl-init command to start the replication. // Returns the Kubernetes client for the secondary DC, diff --git a/subcommand/server-acl-init/create_or_update.go b/subcommand/server-acl-init/create_or_update.go index 65f55dfc6f..ae890c7b07 100644 --- a/subcommand/server-acl-init/create_or_update.go +++ b/subcommand/server-acl-init/create_or_update.go @@ -58,7 +58,7 @@ func (c *Command) createACL(name, rules string, localToken bool, dc string, cons secretName := c.withPrefix(name + "-acl-token") _, err = c.clientset.CoreV1().Secrets(c.flagK8sNamespace).Get(secretName, metav1.GetOptions{}) if err == nil { - c.Log.Info(fmt.Sprintf("Secret %q already exists", secretName)) + c.log.Info(fmt.Sprintf("Secret %q already exists", secretName)) return nil } @@ -107,7 +107,7 @@ func (c *Command) createOrUpdateACLPolicy(policy api.ACLPolicy, consulClient *ap // updated to be namespace aware. if isPolicyExistsErr(err, policy.Name) { if c.flagEnableNamespaces { - c.Log.Info(fmt.Sprintf("Policy %q already exists, updating", policy.Name)) + c.log.Info(fmt.Sprintf("Policy %q already exists, updating", policy.Name)) // The policy ID is required in any PolicyUpdate call, so first we need to // get the existing policy to extract its ID. @@ -134,7 +134,7 @@ func (c *Command) createOrUpdateACLPolicy(policy api.ACLPolicy, consulClient *ap _, _, err = consulClient.ACL().PolicyUpdate(&policy, &api.WriteOptions{}) return err } else { - c.Log.Info(fmt.Sprintf("Policy %q already exists, skipping update", policy.Name)) + c.log.Info(fmt.Sprintf("Policy %q already exists, skipping update", policy.Name)) return nil } } @@ -169,7 +169,7 @@ func (c *Command) checkAndCreateNamespace(ns string, consulClient *api.Client) e if err != nil { return err } - c.Log.Info("created consul namespace", "name", consulNamespace.Name) + c.log.Info("created consul namespace", "name", consulNamespace.Name) } return nil diff --git a/subcommand/server-acl-init/servers.go b/subcommand/server-acl-init/servers.go index 14f81fd7e8..c280225a4c 100644 --- a/subcommand/server-acl-init/servers.go +++ b/subcommand/server-acl-init/servers.go @@ -11,9 +11,9 @@ import ( ) // bootstrapServers bootstraps ACLs and ensures each server has an ACL token. -func (c *Command) bootstrapServers(bootTokenSecretName, scheme string) (string, error) { +func (c *Command) bootstrapServers(serverAddresses []string, bootTokenSecretName, scheme string) (string, error) { // Pick the first server address to connect to for bootstrapping and set up connection. - firstServerAddr := fmt.Sprintf("%s:%d", c.flagServerAddresses[0], c.flagServerPort) + firstServerAddr := fmt.Sprintf("%s:%d", serverAddresses[0], c.flagServerPort) consulClient, err := api.NewClient(&api.Config{ Address: firstServerAddr, Scheme: scheme, @@ -93,7 +93,7 @@ func (c *Command) bootstrapServers(bootTokenSecretName, scheme string) (string, } // Create new tokens for each server and apply them. - if err := c.setServerTokens(consulClient, string(bootstrapToken), scheme); err != nil { + if err := c.setServerTokens(consulClient, serverAddresses, string(bootstrapToken), scheme); err != nil { return "", err } return string(bootstrapToken), nil @@ -101,14 +101,14 @@ func (c *Command) bootstrapServers(bootTokenSecretName, scheme string) (string, // setServerTokens creates policies and associated ACL token for each server // and then provides the token to the server. -func (c *Command) setServerTokens(consulClient *api.Client, bootstrapToken, scheme string) error { +func (c *Command) setServerTokens(consulClient *api.Client, serverAddresses []string, bootstrapToken, scheme string) error { agentPolicy, err := c.setServerPolicy(consulClient) if err != nil { return err } // Create agent token for each server agent. - for _, host := range c.flagServerAddresses { + for _, host := range serverAddresses { var token *api.ACLToken // We create a new client for each server because we need to call each @@ -156,7 +156,7 @@ func (c *Command) setServerTokens(consulClient *api.Client, bootstrapToken, sche func (c *Command) setServerPolicy(consulClient *api.Client) (api.ACLPolicy, error) { agentRules, err := c.agentRules() if err != nil { - c.Log.Error("Error templating server agent rules", "err", err) + c.log.Error("Error templating server agent rules", "err", err) return api.ACLPolicy{}, err } diff --git a/subcommand/sync-catalog/command_ent_test.go b/subcommand/sync-catalog/command_ent_test.go index bbd09e6b2b..fe696e294c 100644 --- a/subcommand/sync-catalog/command_ent_test.go +++ b/subcommand/sync-catalog/command_ent_test.go @@ -723,8 +723,6 @@ func TestRun_ToConsulNamespacesACLs(t *testing.T) { } // Set up test consul agent and fake kubernetes cluster client -// todo: use this setup method everywhere. The old one (completeSetup) uses -// the test agent instead of the testserver. func completeSetupEnterprise(t *testing.T) (*fake.Clientset, *testutil.TestServer) { k8s := fake.NewSimpleClientset() svr, err := testutil.NewTestServerT(t)