Merge remote-tracking branch 'oss/master' into db-cred-rotate

* oss/master: (178 commits)
  Cut version 0.9.4
  Remove netbsd/arm as it won't compile
  Bump files for new version
  Update plugins
  Update go-plugin
  changelog++
  Handling nomad maxTokenNameLength = 64 (#4009)
  Remove unneeded looping since Go 1.10 cover it already (#4010)
  Fix test statement with formatting in fatal call
  Fix PKI tests by generating on-demand
  Sanitize pem encoding to Go default of a newline at the end rather than break backwards compat
  Remove now-unneeded PKCS8 code and update certutil tests for Go 1.10
  Kick Travis
  Bump Travis to Go 1.10
  Fix bug with vault cli when reading an individual field containing a Printf formatting verb (#4005)
  Adding path roles test coverage for storing PKIX fields (#4003)
  Add test coverage for recently-added PKIX fields. (#4002)
  Fix missing CommonName in subject generation
  changelog++
  Handle missed error case in seal status output format (#4001)
  ...

.travis.yml

@@ -7,7 +7,7 @@ services:
   - docker
 
 go:
-  - 1.9.1
+  - "1.10"
 
 matrix:
   allow_failures:

CHANGELOG.md

@@ -1,3 +1,93 @@
+## 0.9.4 (February 20th, 2018)
+
+SECURITY:
+
+ * Role Tags used with the EC2 style of AWS auth were being improperly parsed;
+   as a result they were not being used to properly restrict values.
+   Implementations following our suggestion of using these as defense-in-depth
+   rather than the only source of restriction should not have significant
+   impact.
+
+FEATURES:
+
+ * **ChaCha20-Poly1305 support in `transit`**: You can now encrypt and decrypt
+   with ChaCha20-Poly1305 in `transit`. Key derivation and convergent
+   encryption is also supported.
+ * **Okta Push support in Okta Auth Backend**: If a user account has MFA
+   required within Okta, an Okta Push MFA flow can be used to successfully
+   finish authentication.
+ * **PKI Improvements**: Custom OID subject alternate names can now be set,
+   subject to allow restrictions that support globbing. Additionally, Country,
+   Locality, Province, Street Address, and Postal Code can now be set in
+   certificate subjects.
+ * **Manta Storage**: Joyent Triton Manta can now be used for Vault storage
+ * **Google Cloud Spanner Storage**: Google Cloud Spanner can now be used for
+   Vault storage
+
+IMPROVEMENTS:
+
+ * auth/centrify: Add CLI helper
+ * audit: Always log failure metrics, even if zero, to ensure the values appear
+   on dashboards [GH-3937]
+ * cli: Disable color when output is not a TTY [GH-3897]
+ * cli: Add `-format` flag to all subcommands [GH-3897]
+ * cli: Do not display deprecation warnings when the format is not table
+   [GH-3897]
+ * core: If over a predefined lease count (256k), log a warning not more than
+   once a minute. Too many leases can be problematic for many of the storage
+   backends and often this number of leases is indicative of a need for
+   workflow improvements. [GH-3957]
+ * secret/nomad: Have generated ACL tokens cap out at 64 characters [GH-4009]
+ * secret/pki: Country, Locality, Province, Street Address, and Postal Code can
+   now be set on certificates [GH-3992]
+ * secret/pki: UTF-8 Other Names can now be set in Subject Alternate Names in
+   issued certs; allowed values can be set per role and support globbing
+   [GH-3889]
+ * secret/pki: Add a flag to make the common name optional on certs [GH-3940]
+ * secret/pki: Ensure only DNS-compatible names go into DNS SANs; additionally,
+   properly handle IDNA transformations for these DNS names [GH-3953]
+ * secret/ssh: Add `valid-principles` flag to CLI for CA mode [GH-3922]
+ * storage/manta: Add Manta storage [GH-3270]
+ * ui (Enterprise): Support for ChaCha20-Poly1305 keys in the transit engine.
+
+BUG FIXES:
+
+ * api/renewer: Honor increment value in renew auth calls [GH-3904]
+ * auth/approle: Fix inability to use limited-use-count secret IDs on
+   replication performance secondaries
+ * auth/approle: Cleanup of secret ID accessors during tidy and removal of
+   dangling accessor entries [GH-3924]
+ * auth/aws-ec2: Avoid masking of role tag response [GH-3941]
+ * auth/cert: Verify DNS SANs in the authenticating certificate [GH-3982]
+ * auth/okta: Return configured durations as seconds, not nanoseconds [GH-3871]
+ * auth/token: Token creation via the CLI no longer forces periodic token
+   creation. Passing an explicit zero value for the period no longer create
+   periodic tokens. [GH-3880]
+ * command/rekey: Re-add lost `stored-shares` parameter [GH-3974]
+ * command/ssh: Create and reuse the api client [GH-3909]
+ * command/status: Fix panic when status returns 500 from leadership lookup
+   [GH-3998]
+ * identity: Fix race when creating entities [GH-3932]
+ * plugin/gRPC: Fixed an issue with list requests and raw responses coming from 
+   plugins using gRPC transport [GH-3881]
+ * plugin/gRPC: Fix panic when special paths are not set [GH-3946]
+ * secret/pki: Verify a name is a valid hostname before adding to DNS SANs
+   [GH-3918]
+ * secret/transit: Fix auditing when reading a key after it has been backed up
+   or restored [GH-3919]
+ * secret/transit: Fix storage/memory consistency when persistence fails
+   [GH-3959]
+ * storage/consul: Validate that service names are RFC 1123 compliant [GH-3960]
+ * storage/etcd3: Fix memory ballooning with standby instances [GH-3798]
+ * storage/etcd3: Fix large lists (like token loading at startup) not being
+   handled [GH-3772]
+ * storage/postgresql: Fix compatibility with versions using custom string
+   version tags [GH-3949]
+ * storage/zookeeper: Update vendoring to fix freezing issues [GH-3896]
+ * ui (Enterprise): Decoding the replication token should no longer error and
+   prevent enabling of a secondary replication cluster via the ui.
+ * plugin/gRPC: Add connection info to the request object [GH-3997]
+
 ## 0.9.3 (January 28th, 2018)
 
 A regression from a feature merge disabled the Nomad secrets backend in 0.9.2.
@@ -302,10 +392,12 @@ IMPROVEMENTS:
  * api: Add ability to set custom headers on each call [GH-3394]
  * command/server: Add config option to disable requesting client certificates
    [GH-3373]
+ * auth/aws: Max retries can now be customized for the AWS client [GH-3965]
  * core: Disallow mounting underneath an existing path, not just over [GH-2919]
  * physical/file: Use `700` as permissions when creating directories. The files
    themselves were `600` and are all encrypted, but this doesn't hurt.
  * secret/aws: Add ability to use custom IAM/STS endpoints [GH-3416]
+ * secret/aws: Max retries can now be customized for the AWS client [GH-3965]
  * secret/cassandra: Work around Cassandra ignoring consistency levels for a
    user listing query [GH-3469]
  * secret/pki: Private keys can now be marshalled as PKCS#8 [GH-3518]

Makefile

@@ -11,7 +11,7 @@ EXTERNAL_TOOLS=\
 BUILD_TAGS?=vault
 GOFMT_FILES?=$$(find . -name '*.go' | grep -v vendor)
 
-GO_VERSION_MIN=1.9.1
+GO_VERSION_MIN=1.10
 
 default: dev
 

README.md

@@ -59,7 +59,7 @@ Developing Vault
 --------------------
 
 If you wish to work on Vault itself or any of its built-in systems, you'll
-first need [Go](https://www.golang.org) installed on your machine (version 1.9+
+first need [Go](https://www.golang.org) installed on your machine (version 1.10+
 is *required*).
 
 For local dev first make sure Go is properly installed, including setting up a

api/renewer.go

@@ -195,7 +195,7 @@ func (r *Renewer) renewAuth() error {
 		}
 
 		// Renew the auth.
-		renewal, err := client.Auth().Token().RenewTokenAsSelf(token, 0)
+		renewal, err := client.Auth().Token().RenewTokenAsSelf(token, r.increment)
 		if err != nil {
 			return err
 		}

api/sys_generate_root.go

@@ -99,11 +99,11 @@ func (c *Sys) generateRootUpdateCommon(path, shard, nonce string) (*GenerateRoot
 }
 
 type GenerateRootStatusResponse struct {
-	Nonce            string
-	Started          bool
-	Progress         int
-	Required         int
-	Complete         bool
+	Nonce            string `json:"nonce"`
+	Started          bool   `json:"started"`
+	Progress         int    `json:"progress"`
+	Required         int    `json:"required"`
+	Complete         bool   `json:"complete"`
 	EncodedToken     string `json:"encoded_token"`
 	EncodedRootToken string `json:"encoded_root_token"`
 	PGPFingerprint   string `json:"pgp_fingerprint"`

api/sys_rekey.go

@@ -177,27 +177,27 @@ type RekeyInitRequest struct {
 }
 
 type RekeyStatusResponse struct {
-	Nonce           string
-	Started         bool
-	T               int
-	N               int
-	Progress        int
-	Required        int
+	Nonce           string   `json:"nonce"`
+	Started         bool     `json:"started"`
+	T               int      `json:"t"`
+	N               int      `json:"n"`
+	Progress        int      `json:"progress"`
+	Required        int      `json:"required"`
 	PGPFingerprints []string `json:"pgp_fingerprints"`
-	Backup          bool
+	Backup          bool     `json:"backup"`
 }
 
 type RekeyUpdateResponse struct {
-	Nonce           string
-	Complete        bool
-	Keys            []string
+	Nonce           string   `json:"nonce"`
+	Complete        bool     `json:"complete"`
+	Keys            []string `json:"keys"`
 	KeysB64         []string `json:"keys_base64"`
 	PGPFingerprints []string `json:"pgp_fingerprints"`
-	Backup          bool
+	Backup          bool     `json:"backup"`
 }
 
 type RekeyRetrieveResponse struct {
-	Nonce   string
-	Keys    map[string][]string
+	Nonce   string              `json:"nonce"`
+	Keys    map[string][]string `json:"keys"`
 	KeysB64 map[string][]string `json:"keys_base64"`
 }

builtin/credential/approle/backend.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"sync"
 
+	"github.com/hashicorp/vault/helper/consts"
 	"github.com/hashicorp/vault/helper/locksutil"
 	"github.com/hashicorp/vault/helper/salt"
 	"github.com/hashicorp/vault/logical"
@@ -142,7 +143,9 @@ func (b *backend) invalidate(_ context.Context, key string) {
 // to delay the removal of SecretIDs by a minute.
 func (b *backend) periodicFunc(ctx context.Context, req *logical.Request) error {
 	// Initiate clean-up of expired SecretID entries
-	b.tidySecretID(ctx, req.Storage)
+	if b.System().LocalMount() || !b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary) {
+		b.tidySecretID(ctx, req.Storage)
+	}
 	return nil
 }
 

builtin/credential/approle/path_login.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/vault/logical"
 	"github.com/hashicorp/vault/logical/framework"
 )
@@ -51,9 +52,14 @@ func (b *backend) pathLoginUpdateAliasLookahead(ctx context.Context, req *logica
 // Returns the Auth object indicating the authentication and authorization information
 // if the credentials provided are validated by the backend.
 func (b *backend) pathLoginUpdate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
-	role, roleName, metadata, _, err := b.validateCredentials(ctx, req, data)
-	if err != nil || role == nil {
-		return logical.ErrorResponse(fmt.Sprintf("failed to validate credentials: %v", err)), nil
+	role, roleName, metadata, _, userErr, intErr := b.validateCredentials(ctx, req, data)
+	switch {
+	case intErr != nil:
+		return nil, errwrap.Wrapf("failed to validate credentials: {{err}}", intErr)
+	case userErr != nil:
+		return logical.ErrorResponse(fmt.Sprintf("failed to validate credentials: %v", userErr)), nil
+	case role == nil:
+		return logical.ErrorResponse("failed to validate credentials; could not find role"), nil
 	}
 
 	// Always include the role name, for later filtering

builtin/credential/approle/path_tidy_user_id.go

@@ -38,6 +38,16 @@ func (b *backend) tidySecretID(ctx context.Context, s logical.Storage) error {
 		return err
 	}
 
+	// List all the accessors and add them all to a map
+	accessorHashes, err := s.List(ctx, "accessor/")
+	if err != nil {
+		return err
+	}
+	accessorMap := make(map[string]bool, len(accessorHashes))
+	for _, accessorHash := range accessorHashes {
+		accessorMap[accessorHash] = true
+	}
+
 	var result error
 	for _, roleNameHMAC := range roleNameHMACs {
 		// roleNameHMAC will already have a '/' suffix. Don't append another one.
@@ -77,14 +87,46 @@ func (b *backend) tidySecretID(ctx context.Context, s logical.Storage) error {
 
 			// ExpirationTime not being set indicates non-expiring SecretIDs
 			if !result.ExpirationTime.IsZero() && time.Now().After(result.ExpirationTime) {
+				// Clean up the accessor of the secret ID first
+				err = b.deleteSecretIDAccessorEntry(ctx, s, result.SecretIDAccessor)
+				if err != nil {
+					lock.Unlock()
+					return err
+				}
+
 				if err := s.Delete(ctx, entryIndex); err != nil {
 					lock.Unlock()
 					return fmt.Errorf("error deleting SecretID %s from storage: %s", secretIDHMAC, err)
 				}
 			}
+
+			// At this point, the secret ID is not expired and is valid. Delete
+			// the corresponding accessor from the accessorMap. This will leave
+			// only the dangling accessors in the map which can then be cleaned
+			// up later.
+			salt, err := b.Salt()
+			if err != nil {
+				lock.Unlock()
+				return err
+			}
+			delete(accessorMap, salt.SaltID(result.SecretIDAccessor))
+
 			lock.Unlock()
 		}
 	}
+
+	// Accessor indexes were not getting cleaned up until 0.9.3. This is a fix
+	// to clean up the dangling accessor entries.
+	for accessorHash, _ := range accessorMap {
+		// Ideally, locking should be performed here. But for that, accessors
+		// are required in plaintext, which are not available. Hence performing
+		// a racy cleanup.
+		err = s.Delete(ctx, "accessor/"+accessorHash)
+		if err != nil {
+			return err
+		}
+	}
+
 	return result
 }
 

builtin/credential/approle/path_tidy_user_id_test.go

@@ -0,0 +1,79 @@
+package approle
+
+import (
+	"context"
+	"testing"
+
+	"github.com/hashicorp/vault/logical"
+)
+
+func TestAppRole_TidyDanglingAccessors(t *testing.T) {
+	var resp *logical.Response
+	var err error
+	b, storage := createBackendWithStorage(t)
+
+	// Create a role
+	createRole(t, b, storage, "role1", "a,b,c")
+
+	// Create a secret-id
+	roleSecretIDReq := &logical.Request{
+		Operation: logical.UpdateOperation,
+		Path:      "role/role1/secret-id",
+		Storage:   storage,
+	}
+	resp, err = b.HandleRequest(context.Background(), roleSecretIDReq)
+	if err != nil || (resp != nil && resp.IsError()) {
+		t.Fatalf("err:%v resp:%#v", err, resp)
+	}
+
+	accessorHashes, err := storage.List(context.Background(), "accessor/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(accessorHashes) != 1 {
+		t.Fatalf("bad: len(accessorHashes); expect 1, got %d", len(accessorHashes))
+	}
+
+	entry1, err := logical.StorageEntryJSON(
+		"accessor/invalid1",
+		&secretIDAccessorStorageEntry{
+			SecretIDHMAC: "samplesecretidhmac",
+		},
+	)
+	err = storage.Put(context.Background(), entry1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	entry2, err := logical.StorageEntryJSON(
+		"accessor/invalid2",
+		&secretIDAccessorStorageEntry{
+			SecretIDHMAC: "samplesecretidhmac2",
+		},
+	)
+	err = storage.Put(context.Background(), entry2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	accessorHashes, err = storage.List(context.Background(), "accessor/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(accessorHashes) != 3 {
+		t.Fatalf("bad: len(accessorHashes); expect 3, got %d", len(accessorHashes))
+	}
+
+	err = b.tidySecretID(context.Background(), storage)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	accessorHashes, err = storage.List(context.Background(), "accessor/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(accessorHashes) != 1 {
+		t.Fatalf("bad: len(accessorHashes); expect 1, got %d", len(accessorHashes))
+	}
+}