diff --git a/subcommand/server-acl-init/command.go b/subcommand/server-acl-init/command.go index 15f031c924..afe1ab1754 100644 --- a/subcommand/server-acl-init/command.go +++ b/subcommand/server-acl-init/command.go @@ -5,7 +5,9 @@ import ( "errors" "flag" "fmt" + "io/ioutil" "os" + "strings" "sync" "time" @@ -40,6 +42,8 @@ type Command struct { flagCreateSnapshotAgentToken bool flagCreateMeshGatewayToken bool flagCreateACLReplicationToken bool + flagEnableACLReplication bool + flagACLReplicationTokenFile string flagConsulCACert string flagConsulTLSServerName string flagUseHTTPS bool @@ -126,6 +130,10 @@ func (c *Command) init() { c.flags.StringVar(&c.flagInjectK8SNSMirroringPrefix, "inject-k8s-namespace-mirroring-prefix", "", "[Enterprise Only] Prefix that will be added to all k8s namespaces mirrored into Consul by Connect inject "+ "if mirroring is enabled.") + c.flags.BoolVar(&c.flagEnableACLReplication, "enable-acl-replication", false, + "Enables ACL replication. If true, -acl-replication-token-file must be set") + c.flags.StringVar(&c.flagACLReplicationTokenFile, "acl-replication-token-file", "", + "Path to file containing ACL token to be used for ACL replication.") c.flags.StringVar(&c.flagTimeout, "timeout", "10m", "How long we'll try to bootstrap ACLs for before timing out, e.g. 1ms, 2s, 3m") c.flags.StringVar(&c.flagLogLevel, "log-level", "info", @@ -183,6 +191,21 @@ func (c *Command) Run(args []string) int { if c.flagReleaseName != "" { c.flagServerLabelSelector = fmt.Sprintf("app=consul,component=server,release=%s", c.flagReleaseName) } + var aclReplicationToken string + if c.flagEnableACLReplication { + if c.flagACLReplicationTokenFile == "" { + c.UI.Error("if -enable-acl-replication is true, -acl-replication-token-file must be set") + return 1 + } + + // Load the ACL replication token from file. + tokenBytes, err := ioutil.ReadFile(c.flagACLReplicationTokenFile) + if err != nil { + c.UI.Error(fmt.Sprintf("Unable to read acl replication token from file %q: %s", c.flagACLReplicationTokenFile, err)) + return 1 + } + aclReplicationToken = strings.TrimSpace(string(tokenBytes)) + } var cancel context.CancelFunc c.cmdTimeout, cancel = context.WithTimeout(context.Background(), timeout) @@ -234,30 +257,41 @@ func (c *Command) Run(args []string) int { return 1 } - // Check if we've already been bootstrapped. - bootTokenSecretName := c.withPrefix("bootstrap-acl-token") - bootstrapToken, err := c.getBootstrapToken(bootTokenSecretName) - if err != nil { - c.Log.Error(fmt.Sprintf("Unexpected error looking for preexisting bootstrap Secret: %s", err)) - return 1 - } - var updateServerPolicy bool - if bootstrapToken != "" { - c.Log.Info(fmt.Sprintf("ACLs already bootstrapped - retrieved bootstrap token from Secret %q", bootTokenSecretName)) - - // Mark that we should update the server ACL policy in case - // there are namespace related config changes. Because of the - // organization of the server token creation code, the policy - // otherwise won't be updated. - updateServerPolicy = true + var bootstrapToken string + + if c.flagEnableACLReplication { + // If ACL replication is enabled, we don't need to ACL bootstrap the servers + // since they will be performing replication. + // We can use the replication token as our bootstrap token because it + // has permissions to create policies and tokens. + c.Log.Info("ACL replication is enabled so skipping ACL bootstrapping") + bootstrapToken = aclReplicationToken } else { - c.Log.Info("No bootstrap token from previous installation found, continuing on to bootstrapping") - bootstrapToken, err = c.bootstrapServers(bootTokenSecretName, scheme) + // Check if we've already been bootstrapped. + bootTokenSecretName := c.withPrefix("bootstrap-acl-token") + bootstrapToken, err = c.getBootstrapToken(bootTokenSecretName) if err != nil { - c.Log.Error(err.Error()) + c.Log.Error(fmt.Sprintf("Unexpected error looking for preexisting bootstrap Secret: %s", err)) return 1 } + + if bootstrapToken != "" { + c.Log.Info(fmt.Sprintf("ACLs already bootstrapped - retrieved bootstrap token from Secret %q", bootTokenSecretName)) + + // Mark that we should update the server ACL policy in case + // there are namespace related config changes. Because of the + // organization of the server token creation code, the policy + // otherwise won't be updated. + updateServerPolicy = true + } else { + c.Log.Info("No bootstrap token from previous installation found, continuing on to bootstrapping") + bootstrapToken, err = c.bootstrapServers(bootTokenSecretName, scheme) + if err != nil { + c.Log.Error(err.Error()) + return 1 + } + } } // For all of the next operations we'll need a Consul client. @@ -281,6 +315,13 @@ func (c *Command) Run(args []string) int { return 1 } + consulDC, err := c.consulDatacenter(consulClient) + if err != nil { + c.Log.Error("Error getting datacenter name", "err", err) + return 1 + } + c.Log.Info("Current datacenter", "datacenter", consulDC) + // With the addition of namespaces, the ACL policies associated // with the server tokens may need to be updated if Enterprise Consul // users upgrade to 1.7+. This updates the policy if the bootstrap @@ -338,14 +379,20 @@ func (c *Command) Run(args []string) int { return 1 } - err = c.createACL("client", agentRules, consulClient) + err = c.createLocalACL("client", agentRules, consulDC, consulClient) if err != nil { c.Log.Error(err.Error()) return 1 } } - if c.flagAllowDNS { + // The DNS policy is attached to the anonymous token. + // If performing ACL replication, we assume that the primary datacenter + // has already created the DNS policy and attached it to the anonymous + // token. We don't want to modify the DNS policy in secondary datacenters + // because it is global and we can't create separate tokens for each + // secondary datacenter because the anonymous token is global. + if c.flagAllowDNS && !c.flagEnableACLReplication { err := c.configureDNSPolicies(consulClient) if err != nil { c.Log.Error(err.Error()) @@ -360,7 +407,7 @@ func (c *Command) Run(args []string) int { return 1 } - err = c.createACL("catalog-sync", syncRules, consulClient) + err = c.createLocalACL("catalog-sync", syncRules, consulDC, consulClient) if err != nil { c.Log.Error(err.Error()) return 1 @@ -374,7 +421,7 @@ func (c *Command) Run(args []string) int { return 1 } - err = c.createACL("connect-inject", injectRules, consulClient) + err = c.createLocalACL("connect-inject", injectRules, consulDC, consulClient) if err != nil { c.Log.Error(err.Error()) return 1 @@ -382,7 +429,7 @@ func (c *Command) Run(args []string) int { } if c.flagCreateEntLicenseToken { - err := c.createACL("enterprise-license", entLicenseRules, consulClient) + err := c.createLocalACL("enterprise-license", entLicenseRules, consulDC, consulClient) if err != nil { c.Log.Error(err.Error()) return 1 @@ -390,7 +437,7 @@ func (c *Command) Run(args []string) int { } if c.flagCreateSnapshotAgentToken { - err := c.createACL("client-snapshot-agent", snapshotAgentRules, consulClient) + err := c.createLocalACL("client-snapshot-agent", snapshotAgentRules, consulDC, consulClient) if err != nil { c.Log.Error(err.Error()) return 1 @@ -404,7 +451,9 @@ func (c *Command) Run(args []string) int { return 1 } - err = c.createACL("mesh-gateway", meshGatewayRules, consulClient) + // Mesh gateways require a global policy/token because they must + // discover services in other datacenters. + err = c.createGlobalACL("mesh-gateway", meshGatewayRules, consulDC, consulClient) if err != nil { c.Log.Error(err.Error()) return 1 @@ -425,7 +474,7 @@ func (c *Command) Run(args []string) int { c.Log.Error("Error templating acl replication token rules", "err", err) return 1 } - err = c.createACL("acl-replication", rules, consulClient) + err = c.createGlobalACL("acl-replication", rules, consulDC, consulClient) if err != nil { c.Log.Error(err.Error()) return 1 @@ -511,3 +560,32 @@ Usage: consul-k8s server-acl-init [options] and safe to run multiple times. ` + +// consulDatacenter returns the current datacenter name using the +// /agent/self API endpoint. +func (c *Command) consulDatacenter(client *api.Client) (string, error) { + var agentCfg map[string]map[string]interface{} + err := c.untilSucceeds("calling /agent/self to get datacenter", + func() error { + var opErr error + agentCfg, opErr = client.Agent().Self() + return opErr + }) + if err != nil { + return "", err + } + if _, ok := agentCfg["Config"]; !ok { + return "", fmt.Errorf("/agent/self response did not contain Config key: %s", agentCfg) + } + if _, ok := agentCfg["Config"]["Datacenter"]; !ok { + return "", fmt.Errorf("/agent/self response did not contain Config.Datacenter key: %s", agentCfg) + } + dc, ok := agentCfg["Config"]["Datacenter"].(string) + if !ok { + return "", fmt.Errorf("could not cast Config.Datacenter as string: %s", agentCfg) + } + if dc == "" { + return "", fmt.Errorf("value of Config.Datacenter was empty string: %s", agentCfg) + } + return dc, nil +} diff --git a/subcommand/server-acl-init/command_test.go b/subcommand/server-acl-init/command_test.go index ec42a3a8da..757ced36ad 100644 --- a/subcommand/server-acl-init/command_test.go +++ b/subcommand/server-acl-init/command_test.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/consul/agent" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/sdk/testutil" + "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/hashicorp/consul/tlsutil" "github.com/mitchellh/cli" "github.com/stretchr/testify/require" @@ -48,6 +49,14 @@ func TestRun_FlagValidation(t *testing.T) { Flags: []string{"-server-label-selector=hi"}, ExpErr: "if -server-label-selector is set -resource-prefix must also be set", }, + { + Flags: []string{"-enable-acl-replication", "-server-label-selector=hi", "-resource-prefix=prefix"}, + ExpErr: "if -enable-acl-replication is true, -acl-replication-token-file must be set", + }, + { + Flags: []string{"-enable-acl-replication", "-acl-replication-token-file=/notexist", "-server-label-selector=hi", "-resource-prefix=prefix"}, + ExpErr: "Unable to read acl replication token from file \"/notexist\": open /notexist: no such file or directory", + }, } for _, c := range cases { @@ -99,23 +108,17 @@ func TestRun_Defaults(t *testing.T) { // Check that it has the right policies. consul, err := api.NewClient(&api.Config{ Address: testSvr.HTTPAddr, + Token: bootToken, }) require.NoError(err) - tokenData, _, err := consul.ACL().TokenReadSelf(&api.QueryOptions{Token: bootToken}) + tokenData, _, err := consul.ACL().TokenReadSelf(nil) require.NoError(err) require.Equal("global-management", tokenData.Policies[0].Name) // Check that the agent policy was created. - policies, _, err := consul.ACL().PolicyList(&api.QueryOptions{Token: bootToken}) - require.NoError(err) - found := false - for _, p := range policies { - if p.Name == "agent-token" { - found = true - break - } - } - require.True(found, "agent-token policy was not found") + agentPolicy := policyExists(t, "agent-token", consul) + // Should be a global policy. + require.Len(agentPolicy.Datacenters, 0) // We should also test that the server's token was updated, however I // couldn't find a way to test that with the test agent. Instead we test @@ -129,95 +132,139 @@ func TestRun_Defaults(t *testing.T) { // Test the different flags that should create tokens and save them as // Kubernetes secrets. We test using the -release-name flag vs using the // -resource-prefix flag. -func TestRun_Tokens(t *testing.T) { +func TestRun_TokensPrimaryDC(t *testing.T) { t.Parallel() cases := map[string]struct { TokenFlag string ResourcePrefixFlag string ReleaseNameFlag string - TokenName string + PolicyName string + PolicyDCs []string SecretName string + LocalToken bool }{ "client token -release-name": { TokenFlag: "-create-client-token", ResourcePrefixFlag: "", ReleaseNameFlag: "release-name", - TokenName: "client", + PolicyName: "client-token", + PolicyDCs: []string{"dc1"}, SecretName: "release-name-consul-client-acl-token", + LocalToken: true, }, "client token -resource-prefix": { TokenFlag: "-create-client-token", ResourcePrefixFlag: "my-prefix", - TokenName: "client", + PolicyName: "client-token", + PolicyDCs: []string{"dc1"}, SecretName: "my-prefix-client-acl-token", + LocalToken: true, }, "catalog-sync token -release-name": { TokenFlag: "-create-sync-token", ResourcePrefixFlag: "", ReleaseNameFlag: "release-name", - TokenName: "catalog-sync", + PolicyName: "catalog-sync-token", + PolicyDCs: []string{"dc1"}, SecretName: "release-name-consul-catalog-sync-acl-token", + LocalToken: true, }, "catalog-sync token -resource-prefix": { TokenFlag: "-create-sync-token", ResourcePrefixFlag: "my-prefix", - TokenName: "catalog-sync", + PolicyName: "catalog-sync-token", + PolicyDCs: []string{"dc1"}, SecretName: "my-prefix-catalog-sync-acl-token", + LocalToken: true, }, "connect-inject-namespace token -release-name": { TokenFlag: "-create-inject-namespace-token", ResourcePrefixFlag: "", ReleaseNameFlag: "release-name", - TokenName: "connect-inject", + PolicyName: "connect-inject-token", + PolicyDCs: []string{"dc1"}, SecretName: "release-name-consul-connect-inject-acl-token", + LocalToken: true, }, "connect-inject-namespace token -resource-prefix": { TokenFlag: "-create-inject-namespace-token", ResourcePrefixFlag: "my-prefix", - TokenName: "connect-inject", + PolicyName: "connect-inject-token", + PolicyDCs: []string{"dc1"}, SecretName: "my-prefix-connect-inject-acl-token", + LocalToken: true, }, "enterprise-license token -release-name": { TokenFlag: "-create-enterprise-license-token", ResourcePrefixFlag: "", ReleaseNameFlag: "release-name", - TokenName: "enterprise-license", + PolicyName: "enterprise-license-token", + PolicyDCs: []string{"dc1"}, SecretName: "release-name-consul-enterprise-license-acl-token", + LocalToken: true, }, "enterprise-license token -resource-prefix": { TokenFlag: "-create-enterprise-license-token", ResourcePrefixFlag: "my-prefix", - TokenName: "enterprise-license", + PolicyName: "enterprise-license-token", + PolicyDCs: []string{"dc1"}, SecretName: "my-prefix-enterprise-license-acl-token", + LocalToken: true, + }, + "client-snapshot-agent token -release-name": { + TokenFlag: "-create-snapshot-agent-token", + ResourcePrefixFlag: "", + ReleaseNameFlag: "release-name", + PolicyName: "client-snapshot-agent-token", + PolicyDCs: []string{"dc1"}, + SecretName: "release-name-consul-client-snapshot-agent-acl-token", + LocalToken: true, + }, + "client-snapshot-agent token -resource-prefix": { + TokenFlag: "-create-snapshot-agent-token", + ResourcePrefixFlag: "my-prefix", + ReleaseNameFlag: "release-name", + PolicyName: "client-snapshot-agent-token", + PolicyDCs: []string{"dc1"}, + SecretName: "my-prefix-client-snapshot-agent-acl-token", + LocalToken: true, }, "mesh-gateway token -release-name": { TokenFlag: "-create-mesh-gateway-token", ResourcePrefixFlag: "", ReleaseNameFlag: "release-name", - TokenName: "mesh-gateway", + PolicyName: "mesh-gateway-token", + PolicyDCs: nil, SecretName: "release-name-consul-mesh-gateway-acl-token", + LocalToken: false, }, "mesh-gateway token -resource-prefix": { TokenFlag: "-create-mesh-gateway-token", ResourcePrefixFlag: "my-prefix", ReleaseNameFlag: "release-name", - TokenName: "mesh-gateway", + PolicyName: "mesh-gateway-token", + PolicyDCs: nil, SecretName: "my-prefix-mesh-gateway-acl-token", + LocalToken: false, }, "acl-replication token -release-name": { TokenFlag: "-create-acl-replication-token", ResourcePrefixFlag: "", ReleaseNameFlag: "release-name", - TokenName: "acl-replication", + PolicyName: "acl-replication-token", + PolicyDCs: nil, SecretName: "release-name-consul-acl-replication-acl-token", + LocalToken: false, }, "acl-replication token -resource-prefix": { TokenFlag: "-create-acl-replication-token", ResourcePrefixFlag: "my-prefix", ReleaseNameFlag: "release-name", - TokenName: "acl-replication", + PolicyName: "acl-replication-token", + PolicyDCs: nil, SecretName: "my-prefix-acl-replication-acl-token", + LocalToken: false, }, } for testName, c := range cases { @@ -253,22 +300,15 @@ func TestRun_Tokens(t *testing.T) { responseCode := cmd.Run(cmdArgs) require.Equal(0, responseCode, ui.ErrorWriter.String()) - // Check that the client policy was created. + // Check that the expected policy was created. bootToken := getBootToken(t, k8s, prefix, ns) consul, err := api.NewClient(&api.Config{ Address: testSvr.HTTPAddr, + Token: bootToken, }) require.NoError(err) - policies, _, err := consul.ACL().PolicyList(&api.QueryOptions{Token: bootToken}) - require.NoError(err) - found := false - for _, p := range policies { - if p.Name == c.TokenName+"-token" { - found = true - break - } - } - require.True(found, "%s-token policy was not found", c.TokenName) + policy := policyExists(t, c.PolicyName, consul) + require.Equal(c.PolicyDCs, policy.Datacenters) // Test that the token was created as a Kubernetes Secret. tokenSecret, err := k8s.CoreV1().Secrets(ns).Get(c.SecretName, metav1.GetOptions{}) @@ -280,7 +320,8 @@ func TestRun_Tokens(t *testing.T) { // Test that the token has the expected policies in Consul. tokenData, _, err := consul.ACL().TokenReadSelf(&api.QueryOptions{Token: string(token)}) require.NoError(err) - require.Equal(c.TokenName+"-token", tokenData.Policies[0].Name) + require.Equal(c.PolicyName, tokenData.Policies[0].Name) + require.Equal(c.LocalToken, tokenData.Local) // Test that if the same command is run again, it doesn't error. t.Run(testName+"-retried", func(t *testing.T) { @@ -297,6 +338,112 @@ func TestRun_Tokens(t *testing.T) { } } +// Test creating each token type when replication is enabled. +func TestRun_TokensReplicatedDC(t *testing.T) { + t.Parallel() + + cases := []struct { + TokenFlag string + PolicyName string + PolicyDCs []string + SecretName string + LocalToken bool + }{ + { + TokenFlag: "-create-client-token", + PolicyName: "client-token-dc2", + PolicyDCs: []string{"dc2"}, + SecretName: "release-name-consul-client-acl-token", + LocalToken: true, + }, + { + TokenFlag: "-create-sync-token", + PolicyName: "catalog-sync-token-dc2", + PolicyDCs: []string{"dc2"}, + SecretName: "release-name-consul-catalog-sync-acl-token", + LocalToken: true, + }, + { + TokenFlag: "-create-inject-namespace-token", + PolicyName: "connect-inject-token-dc2", + PolicyDCs: []string{"dc2"}, + SecretName: "release-name-consul-connect-inject-acl-token", + LocalToken: true, + }, + { + TokenFlag: "-create-enterprise-license-token", + PolicyName: "enterprise-license-token-dc2", + PolicyDCs: []string{"dc2"}, + SecretName: "release-name-consul-enterprise-license-acl-token", + LocalToken: true, + }, + { + TokenFlag: "-create-snapshot-agent-token", + PolicyName: "client-snapshot-agent-token-dc2", + PolicyDCs: []string{"dc2"}, + SecretName: "release-name-consul-client-snapshot-agent-acl-token", + LocalToken: true, + }, + { + TokenFlag: "-create-mesh-gateway-token", + PolicyName: "mesh-gateway-token-dc2", + PolicyDCs: nil, + SecretName: "release-name-consul-mesh-gateway-acl-token", + LocalToken: false, + }, + } + for _, c := range cases { + t.Run(c.TokenFlag, func(t *testing.T) { + bootToken := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + tokenFile, fileCleanup := writeTempFile(t, bootToken) + defer fileCleanup() + + k8s, consul, cleanup := completeReplicatedSetup(t, resourcePrefix, bootToken) + defer cleanup() + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmd.init() + cmdArgs := []string{ + "-k8s-namespace=" + ns, + "-expected-replicas=1", + "-acl-replication-token-file", tokenFile, + "-enable-acl-replication", + "-server-label-selector=component=server,app=consul,release=" + releaseName, + "-resource-prefix=" + resourcePrefix, + c.TokenFlag, + } + responseCode := cmd.Run(cmdArgs) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + + // Check that the expected policy was created. + retry.Run(t, func(r *retry.R) { + policy := policyExists(r, c.PolicyName, consul) + require.Equal(r, c.PolicyDCs, policy.Datacenters) + }) + + retry.Run(t, func(r *retry.R) { + // Test that the token was created as a Kubernetes Secret. + tokenSecret, err := k8s.CoreV1().Secrets(ns).Get(c.SecretName, metav1.GetOptions{}) + require.NoError(r, err) + require.NotNil(r, tokenSecret) + token, ok := tokenSecret.Data["token"] + require.True(r, ok) + + // Test that the token has the expected policies in Consul. + tokenData, _, err := consul.ACL().TokenReadSelf(&api.QueryOptions{Token: string(token)}) + require.NoError(r, err) + require.Equal(r, c.PolicyName, tokenData.Policies[0].Name) + require.Equal(r, c.LocalToken, tokenData.Local) + }) + }) + } +} + func TestRun_AllowDNS(t *testing.T) { t.Parallel() k8s, testSvr := completeSetup(t, resourcePrefix) @@ -324,18 +471,12 @@ func TestRun_AllowDNS(t *testing.T) { bootToken := getBootToken(t, k8s, resourcePrefix, ns) consul, err := api.NewClient(&api.Config{ Address: testSvr.HTTPAddr, + Token: bootToken, }) require.NoError(err) - policies, _, err := consul.ACL().PolicyList(&api.QueryOptions{Token: bootToken}) - require.NoError(err) - found := false - for _, p := range policies { - if p.Name == "dns-policy" { - found = true - break - } - } - require.True(found, "Did not find dns-policy") + policy := policyExists(t, "dns-policy", consul) + // Should be a global policy. + require.Len(policy.Datacenters, 0) // Check that the anonymous token has the DNS policy. tokenData, _, err := consul.ACL().TokenReadSelf(&api.QueryOptions{Token: "anonymous"}) @@ -533,8 +674,13 @@ func TestRun_DelayedServerPods(t *testing.T) { Path: r.URL.Path, }) - // Send an empty JSON response with code 200 to all calls. - fmt.Fprintln(w, "{}") + switch r.URL.Path { + case "/v1/agent/self": + fmt.Fprintln(w, `{"Config": {"Datacenter": "dc1"}}`) + default: + // Send an empty JSON response with code 200 to all calls. + fmt.Fprintln(w, "{}") + } })) defer consulServer.Close() serverURL, err := url.Parse(consulServer.URL) @@ -644,6 +790,10 @@ func TestRun_DelayedServerPods(t *testing.T) { "PUT", "/v1/agent/token/agent", }, + { + "GET", + "/v1/agent/self", + }, { "PUT", "/v1/acl/policy", @@ -672,9 +822,13 @@ func TestRun_InProgressDeployment(t *testing.T) { Method: r.Method, Path: r.URL.Path, }) - - // Send an empty JSON response with code 200 to all calls. - fmt.Fprintln(w, "{}") + switch r.URL.Path { + case "/v1/agent/self": + fmt.Fprintln(w, `{"Config": {"Datacenter": "dc1"}}`) + default: + // Send an empty JSON response with code 200 to all calls. + fmt.Fprintln(w, "{}") + } })) defer consulServer.Close() serverURL, err := url.Parse(consulServer.URL) @@ -800,6 +954,10 @@ func TestRun_InProgressDeployment(t *testing.T) { "PUT", "/v1/agent/token/agent", }, + { + "GET", + "/v1/agent/self", + }, { "PUT", "/v1/acl/policy", @@ -843,6 +1001,8 @@ func TestRun_NoLeader(t *testing.T) { fmt.Fprintln(w, "{}") } numACLBootCalls++ + case "/v1/agent/self": + fmt.Fprintln(w, `{"Config": {"Datacenter": "dc1"}}`) default: fmt.Fprintln(w, "{}") } @@ -955,6 +1115,10 @@ func TestRun_NoLeader(t *testing.T) { "PUT", "/v1/agent/token/agent", }, + { + "GET", + "/v1/agent/self", + }, { "PUT", "/v1/acl/policy", @@ -998,6 +1162,8 @@ func TestRun_ClientTokensRetry(t *testing.T) { fmt.Fprintln(w, "{}") } numPolicyCalls++ + case "/v1/agent/self": + fmt.Fprintln(w, `{"Config": {"Datacenter": "dc1"}}`) default: fmt.Fprintln(w, "{}") } @@ -1087,6 +1253,10 @@ func TestRun_ClientTokensRetry(t *testing.T) { "PUT", "/v1/agent/token/agent", }, + { + "GET", + "/v1/agent/self", + }, // This call should happen twice since the first will fail. { "PUT", @@ -1123,9 +1293,11 @@ func TestRun_AlreadyBootstrapped(t *testing.T) { Method: r.Method, Path: r.URL.Path, }) - switch r.URL.Path { + case "/v1/agent/self": + fmt.Fprintln(w, `{"Config": {"Datacenter": "dc1"}}`) default: + // Send an empty JSON response with code 200 to all calls. fmt.Fprintln(w, "{}") } })) @@ -1217,6 +1389,10 @@ func TestRun_AlreadyBootstrapped(t *testing.T) { require.Equal([]APICall{ // We only expect the calls for creating client tokens // and updating the server policy. + { + "GET", + "/v1/agent/self", + }, { "PUT", "/v1/acl/policy", @@ -1312,6 +1488,156 @@ func TestRun_HTTPS(t *testing.T) { require.True(ok) } +// Test that the ACL replication token created from the primary DC can be used +// for replication in the secondary DC. +func TestRun_ACLReplicationTokenValid(t *testing.T) { + t.Parallel() + + // Set up the primary DC first. + primarySvr, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { + c.ACL.Enabled = true + }) + require.NoError(t, err) + defer primarySvr.Stop() + + primaryK8s := fake.NewSimpleClientset() + createTestK8SResources(t, primaryK8s, primarySvr.HTTPAddr, resourcePrefix, "http", ns) + require.NoError(t, err) + + // Run the command to bootstrap ACLs + primaryUI := cli.NewMockUi() + primaryCmd := Command{ + UI: primaryUI, + clientset: primaryK8s, + } + primaryCmd.init() + primaryCmdArgs := []string{ + "-k8s-namespace=" + ns, + "-expected-replicas=1", + "-server-label-selector=component=server,app=consul,release=" + releaseName, + "-resource-prefix=" + resourcePrefix, + "-create-acl-replication-token", + } + responseCode := primaryCmd.Run(primaryCmdArgs) + require.Equal(t, 0, responseCode, primaryUI.ErrorWriter.String()) + + // Retrieve the replication ACL token from the kubernetes secret. + tokenSecret, err := primaryK8s.CoreV1().Secrets(ns).Get("release-name-consul-acl-replication-acl-token", metav1.GetOptions{}) + require.NoError(t, err) + require.NotNil(t, tokenSecret) + aclReplicationTokenBytes, ok := tokenSecret.Data["token"] + require.True(t, ok) + aclReplicationToken := string(aclReplicationTokenBytes) + + // Now we can set up the secondary DC. + secondarySvr, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { + c.Datacenter = "dc2" + c.ACL.Enabled = true + c.ACL.TokenReplication = true + c.PrimaryDatacenter = "dc1" + // When running via the Helm chart these are passed in to the servers. + c.ACL.Tokens.Agent = aclReplicationToken + c.ACL.Tokens.Replication = aclReplicationToken + }) + require.NoError(t, err) + defer secondarySvr.Stop() + + // When running via Helm, the WAN join config is set in the server config + // however that config isn't exposed by the testutil server so instead we need to + // make the join API call. This requires the bootstrap token permissions. + bootToken := getBootToken(t, primaryK8s, resourcePrefix, ns) + secondaryConsulClient, err := api.NewClient(&api.Config{ + Address: secondarySvr.HTTPAddr, + Token: bootToken, + }) + err = secondaryConsulClient.Agent().Join(primarySvr.WANAddr, true) + require.NoError(t, err) + + secondaryK8s := fake.NewSimpleClientset() + createTestK8SResources(t, secondaryK8s, secondarySvr.HTTPAddr, resourcePrefix, "http", ns) + + // Now we're ready to run the command on in our secondary dc. + tokenFile, cleanup := writeTempFile(t, aclReplicationToken) + defer cleanup() + secondaryUI := cli.NewMockUi() + secondaryCmd := Command{ + UI: secondaryUI, + clientset: secondaryK8s, + } + secondaryCmd.init() + secondaryCmdArgs := []string{ + "-k8s-namespace=" + ns, + "-expected-replicas=1", + "-server-label-selector=component=server,app=consul,release=" + releaseName, + "-resource-prefix=" + resourcePrefix, + "-enable-acl-replication", + "-acl-replication-token-file", tokenFile, + "-create-client-token", + "-create-mesh-gateway-token", + } + responseCode = secondaryCmd.Run(secondaryCmdArgs) + require.Equal(t, 0, responseCode, secondaryUI.ErrorWriter.String()) + + // Test that replication was successful. + retry.Run(t, func(r *retry.R) { + replicationStatus, _, err := secondaryConsulClient.ACL().Replication(nil) + require.NoError(t, err) + require.True(t, replicationStatus.Enabled) + var uintZero uint64 = 0 + require.Greater(t, replicationStatus.ReplicatedIndex, uintZero) + }) + + // Test that the client policy was created. + retry.Run(t, func(r *retry.R) { + p := policyExists(r, "client-token-dc2", secondaryConsulClient) + require.Equal(r, []string{"dc2"}, p.Datacenters) + }) + + // Test that the mesh-gateway policy was created. This is a global policy + // so replication has to have worked for it to exist. + retry.Run(t, func(r *retry.R) { + p := policyExists(r, "mesh-gateway-token-dc2", secondaryConsulClient) + require.Len(r, p.Datacenters, 0) + }) +} + +// Test that if acl replication is enabled, we don't create a dns policy. +func TestRun_DNSFlagIgnoredReplication(t *testing.T) { + bootToken := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + tokenFile, fileCleanup := writeTempFile(t, bootToken) + defer fileCleanup() + k8s, consul, cleanup := completeReplicatedSetup(t, resourcePrefix, bootToken) + defer cleanup() + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmd.init() + cmdArgs := []string{ + "-k8s-namespace=" + ns, + "-expected-replicas=1", + "-enable-acl-replication", + "-acl-replication-token-file", tokenFile, + "-server-label-selector=component=server,app=consul,release=" + releaseName, + "-resource-prefix=" + resourcePrefix, + "-allow-dns", + } + responseCode := cmd.Run(cmdArgs) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + + // The DNS policy should not have been created. + policies, _, err := consul.ACL().PolicyList(nil) + require.NoError(t, err) + for _, p := range policies { + if p.Name == "dns-policy" { + require.Fail(t, "dns-policy exists") + } + } +} + // Set up test consul agent and kubernetes cluster. func completeSetup(t *testing.T, prefix string) (*fake.Clientset, *testutil.TestServer) { k8s := fake.NewSimpleClientset() @@ -1326,6 +1652,52 @@ func completeSetup(t *testing.T, prefix string) (*fake.Clientset, *testutil.Test return k8s, svr } +// Set up two Consul servers with ACL replication. +// Returns the Kubernetes client for the secondary DC, +// a Consul API client initialized for the secondary DC and a +// cleanup function that should be called at the end of the test that cleans +// up resources. +func completeReplicatedSetup(t *testing.T, prefix string, bootToken string) (*fake.Clientset, *api.Client, func()) { + t.Helper() + + primarySvr, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { + c.ACL.Enabled = true + c.ACL.Tokens.Master = bootToken + }) + require.NoError(t, err) + + // Set up the secondary server that will federate with the primary. + secondarySvr, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { + c.Datacenter = "dc2" + c.ACL.Enabled = true + c.ACL.TokenReplication = true + c.ACL.Tokens.Agent = bootToken + c.ACL.Tokens.Replication = bootToken + c.PrimaryDatacenter = "dc1" + }) + require.NoError(t, err) + + // Our consul client will use the secondary dc. + consul, err := api.NewClient(&api.Config{ + Address: secondarySvr.HTTPAddr, + Token: bootToken, + }) + require.NoError(t, err) + + // WAN join the primary and secondary. + err = consul.Agent().Join(primarySvr.WANAddr, true) + require.NoError(t, err) + + // Finally, set up our kube cluster. It will use the secondary dc. + k8s := fake.NewSimpleClientset() + createTestK8SResources(t, k8s, secondarySvr.HTTPAddr, prefix, "http", ns) + + return k8s, consul, func() { + primarySvr.Stop() + secondarySvr.Stop() + } +} + // Create test k8s resources (server pods and server stateful set) func createTestK8SResources(t *testing.T, k8s *fake.Clientset, consulHTTPAddr, prefix, scheme, k8sNamespace string) { require := require.New(t) @@ -1496,5 +1868,32 @@ func setUpK8sServiceAccount(t *testing.T, k8s *fake.Clientset) (string, string) return string(caCertBytes), string(tokenBytes) } +// policyExists asserts that policy with name exists. Returns the policy +// if it does, otherwise fails the test. +func policyExists(t require.TestingT, name string, client *api.Client) *api.ACLPolicyListEntry { + policies, _, err := client.ACL().PolicyList(nil) + require.NoError(t, err) + var policy *api.ACLPolicyListEntry + for _, p := range policies { + if p.Name == name { + policy = p + break + } + } + require.NotNil(t, policy, "policy %s not found", name) + return policy +} + +func writeTempFile(t *testing.T, contents string) (string, func()) { + t.Helper() + file, err := ioutil.TempFile("", "") + require.NoError(t, err) + err = ioutil.WriteFile(file.Name(), []byte(contents), 0600) + require.NoError(t, err) + return file.Name(), func() { + os.Remove(file.Name()) + } +} + var serviceAccountCACert = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURDekNDQWZPZ0F3SUJBZ0lRS3pzN05qbDlIczZYYzhFWG91MjVoekFOQmdrcWhraUc5dzBCQVFzRkFEQXYKTVMwd0t3WURWUVFERXlRMU9XVTJaR00wTVMweU1EaG1MVFF3T1RVdFlUSTRPUzB4Wm1NM01EQmhZekZqWXpndwpIaGNOTVRrd05qQTNNVEF4TnpNeFdoY05NalF3TmpBMU1URXhOek14V2pBdk1TMHdLd1lEVlFRREV5UTFPV1UyClpHTTBNUzB5TURobUxUUXdPVFV0WVRJNE9TMHhabU0zTURCaFl6RmpZemd3Z2dFaU1BMEdDU3FHU0liM0RRRUIKQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUURaakh6d3FvZnpUcEdwYzBNZElDUzdldXZmdWpVS0UzUEMvYXBmREFnQgo0anpFRktBNzgvOStLVUd3L2MvMFNIZVNRaE4rYThnd2xIUm5BejFOSmNmT0lYeTRkd2VVdU9rQWlGeEg4cGh0CkVDd2tlTk83ejhEb1Y4Y2VtaW5DUkhHamFSbW9NeHBaN2cycFpBSk5aZVB4aTN5MWFOa0ZBWGU5Z1NVU2RqUloKUlhZa2E3d2gyQU85azJkbEdGQVlCK3Qzdld3SjZ0d2pHMFR0S1FyaFlNOU9kMS9vTjBFMDFMekJjWnV4a04xawo4Z2ZJSHk3Yk9GQ0JNMldURURXLzBhQXZjQVByTzhETHFESis2TWpjM3I3K3psemw4YVFzcGIwUzA4cFZ6a2k1CkR6Ly84M2t5dTBwaEp1aWo1ZUI4OFY3VWZQWHhYRi9FdFY2ZnZyTDdNTjRmQWdNQkFBR2pJekFoTUE0R0ExVWQKRHdFQi93UUVBd0lDQkRBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCdgpRc2FHNnFsY2FSa3RKMHpHaHh4SjUyTm5SVjJHY0lZUGVOM1p2MlZYZTNNTDNWZDZHMzJQVjdsSU9oangzS21BCi91TWg2TmhxQnpzZWtrVHowUHVDM3dKeU0yT0dvblZRaXNGbHF4OXNGUTNmVTJtSUdYQ2Ezd0M4ZS9xUDhCSFMKdzcvVmVBN2x6bWozVFFSRS9XMFUwWkdlb0F4bjliNkp0VDBpTXVjWXZQMGhYS1RQQldsbnpJaWphbVU1MHIyWQo3aWEwNjVVZzJ4VU41RkxYL3Z4T0EzeTRyanBraldvVlFjdTFwOFRaclZvTTNkc0dGV3AxMGZETVJpQUhUdk9ICloyM2pHdWs2cm45RFVIQzJ4UGozd0NUbWQ4U0dFSm9WMzFub0pWNWRWZVE5MHd1c1h6M3ZURzdmaWNLbnZIRlMKeHRyNVBTd0gxRHVzWWZWYUdIMk8KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=" var serviceAccountToken = "ZXlKaGJHY2lPaUpTVXpJMU5pSXNJbXRwWkNJNklpSjkuZXlKcGMzTWlPaUpyZFdKbGNtNWxkR1Z6TDNObGNuWnBZMlZoWTJOdmRXNTBJaXdpYTNWaVpYSnVaWFJsY3k1cGJ5OXpaWEoyYVdObFlXTmpiM1Z1ZEM5dVlXMWxjM0JoWTJVaU9pSmtaV1poZFd4MElpd2lhM1ZpWlhKdVpYUmxjeTVwYnk5elpYSjJhV05sWVdOamIzVnVkQzl6WldOeVpYUXVibUZ0WlNJNkltdG9ZV3RwTFdGeVlXTm9ibWxrTFdOdmJuTjFiQzFqYjI1dVpXTjBMV2x1YW1WamRHOXlMV0YxZEdodFpYUm9iMlF0YzNaakxXRmpZMjlvYm1SaWRpSXNJbXQxWW1WeWJtVjBaWE11YVc4dmMyVnlkbWxqWldGalkyOTFiblF2YzJWeWRtbGpaUzFoWTJOdmRXNTBMbTVoYldVaU9pSnJhR0ZyYVMxaGNtRmphRzVwWkMxamIyNXpkV3d0WTI5dWJtVmpkQzFwYm1wbFkzUnZjaTFoZFhSb2JXVjBhRzlrTFhOMll5MWhZMk52ZFc1MElpd2lhM1ZpWlhKdVpYUmxjeTVwYnk5elpYSjJhV05sWVdOamIzVnVkQzl6WlhKMmFXTmxMV0ZqWTI5MWJuUXVkV2xrSWpvaU4yVTVOV1V4TWprdFpUUTNNeTB4TVdVNUxUaG1ZV0V0TkRJd01UQmhPREF3TVRJeUlpd2ljM1ZpSWpvaWMzbHpkR1Z0T25ObGNuWnBZMlZoWTJOdmRXNTBPbVJsWm1GMWJIUTZhMmhoYTJrdFlYSmhZMmh1YVdRdFkyOXVjM1ZzTFdOdmJtNWxZM1F0YVc1cVpXTjBiM0l0WVhWMGFHMWxkR2h2WkMxemRtTXRZV05qYjNWdWRDSjkuWWk2M01NdHpoNU1CV0tLZDNhN2R6Q0pqVElURTE1aWtGeV9UbnBka19Bd2R3QTlKNEFNU0dFZUhONXZXdEN1dUZqb19sTUpxQkJQSGtLMkFxYm5vRlVqOW01Q29wV3lxSUNKUWx2RU9QNGZVUS1SYzBXMVBfSmpVMXJaRVJIRzM5YjVUTUxnS1BRZ3V5aGFpWkVKNkNqVnRtOXdVVGFncmdpdXFZVjJpVXFMdUY2U1lObTZTckt0a1BTLWxxSU8tdTdDMDZ3Vms1bTV1cXdJVlFOcFpTSUNfNUxzNWFMbXlaVTNuSHZILVY3RTNIbUJoVnlaQUI3NmpnS0IwVHlWWDFJT3NrdDlQREZhck50VTNzdVp5Q2p2cUMtVUpBNnNZZXlTZTRkQk5Lc0tsU1o2WXV4VVVtbjFSZ3YzMllNZEltbnNXZzhraGYtekp2cWdXazdCNUVB" diff --git a/subcommand/server-acl-init/create_or_update.go b/subcommand/server-acl-init/create_or_update.go index 1e75cde679..c2978206bb 100644 --- a/subcommand/server-acl-init/create_or_update.go +++ b/subcommand/server-acl-init/create_or_update.go @@ -10,14 +10,40 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// createACL creates a policy with rules and name, creates an ACL token for that -// policy and then writes the token to a Kubernetes secret. -func (c *Command) createACL(name, rules string, consulClient *api.Client) error { +// createLocalACL creates a policy and acl token for this dc (datacenter), i.e. +// the policy is only valid for this datacenter and the token is a local token. +func (c *Command) createLocalACL(name, rules, dc string, consulClient *api.Client) error { + return c.createACL(name, rules, true, dc, consulClient) +} + +// createGlobalACL creates a global policy and acl token. The policy is valid +// for all datacenters and the token is global. dc must be passed because the +// policy may have the datacenter name appended. +func (c *Command) createGlobalACL(name, rules, dc string, consulClient *api.Client) error { + return c.createACL(name, rules, false, dc, consulClient) +} + +// createACL creates a policy with rules and name. If localToken is true then +// the token will be a local token. If dc is not empty, the policy will +// be scoped to just that datacenter, otherwise it will be global. +// The token will be written to a Kubernetes secret. +func (c *Command) createACL(name, rules string, localToken bool, dc string, consulClient *api.Client) error { // Create policy with the given rules. + policyName := fmt.Sprintf("%s-token", name) + if c.flagEnableACLReplication { + // If performing ACL replication, we must ensure policy names are + // globally unique so we append the datacenter name. + policyName += fmt.Sprintf("-%s", dc) + } + var datacenters []string + if localToken { + datacenters = append(datacenters, dc) + } policyTmpl := api.ACLPolicy{ - Name: fmt.Sprintf("%s-token", name), - Description: fmt.Sprintf("%s Token Policy", name), + Name: policyName, + Description: fmt.Sprintf("%s Token Policy", policyName), Rules: rules, + Datacenters: datacenters, } err := c.untilSucceeds(fmt.Sprintf("creating %s policy", policyTmpl.Name), func() error { @@ -38,8 +64,9 @@ func (c *Command) createACL(name, rules string, consulClient *api.Client) error // Create token for the policy if the secret did not exist previously. tokenTmpl := api.ACLToken{ - Description: fmt.Sprintf("%s Token", name), + Description: fmt.Sprintf("%s Token", policyTmpl.Name), Policies: []*api.ACLTokenPolicyLink{{Name: policyTmpl.Name}}, + Local: localToken, } var token string err = c.untilSucceeds(fmt.Sprintf("creating token for policy %s", policyTmpl.Name), diff --git a/subcommand/server-acl-init/rules.go b/subcommand/server-acl-init/rules.go index 518e17cd1b..bbce5746a3 100644 --- a/subcommand/server-acl-init/rules.go +++ b/subcommand/server-acl-init/rules.go @@ -149,14 +149,19 @@ operator = "write" } func (c *Command) aclReplicationRules() (string, error) { - // NOTE: The node_prefix rule is not required for ACL replication. It's - // added so that this token can be used as an ACL replication token *and* - // as an agent token. This allows us to only pass one token between + // NOTE: The node_prefix and agent_prefix rules are not required for ACL + // replication. They're added so that this token can be used as an ACL + // replication token, an agent token and for the server-acl-init command + // where we need agent:read to get the current datacenter. + // This allows us to only pass one token between // datacenters during federation since in order to start ACL replication, // we need a token with both replication and agent permissions. aclReplicationRulesTpl := ` acl = "write" operator = "write" +agent_prefix "" { + policy = "read" +} {{- if .EnableNamespaces }} namespace_prefix "" { {{- end }} diff --git a/subcommand/server-acl-init/rules_test.go b/subcommand/server-acl-init/rules_test.go index f8c15a55f8..2a575946da 100644 --- a/subcommand/server-acl-init/rules_test.go +++ b/subcommand/server-acl-init/rules_test.go @@ -309,6 +309,9 @@ func TestReplicationTokenRules(t *testing.T) { false, `acl = "write" operator = "write" +agent_prefix "" { + policy = "read" +} node_prefix "" { policy = "write" } @@ -322,6 +325,9 @@ operator = "write" true, `acl = "write" operator = "write" +agent_prefix "" { + policy = "read" +} namespace_prefix "" { node_prefix "" { policy = "write"