diff --git a/command/server.go b/command/server.go index f07b6d129e09..9389570a0d98 100644 --- a/command/server.go +++ b/command/server.go @@ -741,7 +741,7 @@ CLUSTER_SYNTHESIS_COMPLETE: // Initialize the core core, newCoreError := vault.NewCore(coreConfig) if newCoreError != nil { - if !errwrap.ContainsType(newCoreError, new(vault.NonFatalError)) { + if vault.IsFatalError(newCoreError) { c.UI.Error(fmt.Sprintf("Error initializing core: %s", newCoreError)) return 1 } @@ -937,12 +937,31 @@ CLUSTER_SYNTHESIS_COMPLETE: return 1 } - if err := core.UnsealWithStoredKeys(context.Background()); err != nil { - if !errwrap.ContainsType(err, new(vault.NonFatalError)) { - c.UI.Error(fmt.Sprintf("Error initializing core: %s", err)) - return 1 + // Attempt unsealing in a background goroutine. This is needed for when a + // Vault cluster with multiple servers is configured with auto-unseal but is + // uninitialized. Once one server initializes the storage backend, this + // goroutine will pick up the unseal keys and unseal this instance. + go func() { + for { + err := core.UnsealWithStoredKeys(context.Background()) + if err == nil { + return + } + + if vault.IsFatalError(err) { + c.logger.Error(fmt.Sprintf("Error unsealing core", "error", err)) + return + } else { + c.logger.Warn(fmt.Sprintf("Failed to unseal core", "error" err)) + } + + select { + case <-c.ShutdownCh: + return + case <-time.After(5 * time.Second): + } } - } + }() // Perform service discovery registrations and initialization of // HTTP server after the verifyOnly check. diff --git a/http/sys_init.go b/http/sys_init.go index 39e2c5551d5e..3d5a8b7e5b69 100644 --- a/http/sys_init.go +++ b/http/sys_init.go @@ -7,7 +7,6 @@ import ( "fmt" "net/http" - "github.com/hashicorp/errwrap" "github.com/hashicorp/vault/vault" ) @@ -104,7 +103,7 @@ func handleSysInitPut(core *vault.Core, w http.ResponseWriter, r *http.Request) result, initErr := core.Initialize(ctx, initParams) if initErr != nil { - if !errwrap.ContainsType(initErr, new(vault.NonFatalError)) { + if vault.IsFatalError(initErr) { respondError(w, http.StatusBadRequest, initErr) return } else { @@ -136,7 +135,10 @@ func handleSysInitPut(core *vault.Core, w http.ResponseWriter, r *http.Request) } } - core.UnsealWithStoredKeys(ctx) + if err := core.UnsealWithStoredKeys(ctx); err != nil { + respondError(w, http.StatusInternalServerError, err) + return + } respondOk(w, resp) } diff --git a/vault/core.go b/vault/core.go index 5c8a4db0e98e..aac28fbd160b 100644 --- a/vault/core.go +++ b/vault/core.go @@ -107,6 +107,16 @@ func (e *NonFatalError) Error() string { return e.Err.Error() } +// NewNonFatalError returns a new non-fatal error. +func NewNonFatalError(err error) *NonFatalError { + return &NonFatalError{Err: err} +} + +// IsFatalError returns true if the given error is a non-fatal error. +func IsFatalError(err error) bool { + return !errwrap.ContainsType(err, new(NonFatalError)) +} + // ErrInvalidKey is returned if there is a user-based error with a provided // unseal key. This will be shown to the user, so should not contain // information that is sensitive. @@ -386,6 +396,10 @@ type Core struct { // Stores the sealunwrapper for downgrade needs sealUnwrapper physical.Backend + // unsealwithStoredKeysLock is a mutex that prevents multiple processes from + // unsealing with stored keys are the same time. + unsealWithStoredKeysLock sync.Mutex + // Stores any funcs that should be run on successful postUnseal postUnsealFuncs []func() diff --git a/vault/init.go b/vault/init.go index 426cc62a942a..c4ad07ecf028 100644 --- a/vault/init.go +++ b/vault/init.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "encoding/hex" + "errors" "fmt" "github.com/hashicorp/errwrap" @@ -270,54 +271,63 @@ func (c *Core) Initialize(ctx context.Context, initParams *InitParams) (*InitRes return results, nil } -// UnsealWithStoredKeys performs auto-unseal using stored keys. +// UnsealWithStoredKeys performs auto-unseal using stored keys. An error +// return value of "nil" implies the Vault instance is unsealed. +// +// Callers should attempt to retry any NOnFatalErrors. Callers should +// not re-attempt fatal errors. func (c *Core) UnsealWithStoredKeys(ctx context.Context) error { + c.unsealWithStoredKeysLock.Lock() + defer c.unsealWithStoredKeysLock.Unlock() + if !c.seal.StoredKeysSupported() { return nil } // Disallow auto-unsealing when migrating if c.IsInSealMigration() { - return nil + return NewNonFatalError(errors.New("cannot auto-unseal during seal migration")) } sealed := c.Sealed() if !sealed { + c.Logger().Warn("attempted unseal with stored keys, but vault is already unsealed") return nil } - c.logger.Info("stored unseal keys supported, attempting fetch") + c.Logger().Info("stored unseal keys supported, attempting fetch") keys, err := c.seal.GetStoredKeys(ctx) if err != nil { - c.logger.Error("fetching stored unseal keys failed", "error", err) - return &NonFatalError{Err: errwrap.Wrapf("fetching stored unseal keys failed: {{err}}", err)} + return NewNonFatalError(errwrap.Wrapf("fetching stored unseal keys failed: {{err}}", err)) } + + // This usually happens when auto-unseal is configured, but the servers have + // not been initialized yet. if len(keys) == 0 { - c.logger.Warn("stored unseal key(s) supported but none found") - } else { - unsealed := false - keysUsed := 0 - for _, key := range keys { - unsealed, err = c.Unseal(key) - if err != nil { - c.logger.Error("unseal with stored unseal key failed", "error", err) - return &NonFatalError{Err: errwrap.Wrapf("unseal with stored key failed: {{err}}", err)} - } - keysUsed += 1 - if unsealed { - break - } + return NewNonFatalError(errors.New("stored unseal keys are supported, but none were found")) + } + + unsealed := false + keysUsed := 0 + for _, key := range keys { + unsealed, err = c.Unseal(key) + if err != nil { + return NewNonFatalError(errwrap.Wrapf("unseal with stored key failed: {{err}}", err)) } - if !unsealed { - if c.logger.IsWarn() { - c.logger.Warn("stored unseal key(s) used but Vault not unsealed yet", "stored_keys_used", keysUsed) - } - } else { - if c.logger.IsInfo() { - c.logger.Info("successfully unsealed with stored key(s)", "stored_keys_used", keysUsed) - } + keysUsed++ + if unsealed { + break } } + if !unsealed { + // This most likely means that the user configured Vault to only store a + // subset of the required threshold of keys. We still consider this a + // "success", since trying again would yield the same result. + c.Logger().Warn("vault still sealed after using stored unseal keys", "stored_keys_used", keysUsed) + } else { + c.Logger().Info("unsealed with stored keys", "stored_keys_used", keysUsed) + } + return nil }