Skip to content

Commit

Permalink
Add new data structure to track seal config generation (#4492)
Browse files Browse the repository at this point in the history 
* Add new data structure to track seal config generation

* Remove import cycle

* Fix undefined variable errors

* update comment

* Update setSeal response

* Fix setSealResponse in operator_diagnose
  • Loading branch information
@divyapola5 @victorr
divyapola5 authored and victorr committed Aug 24, 2023
1 parent 1310b82 commit d76cd87
Show file tree
Hide file tree
Showing 4 changed files with 157 additions and 24 deletions.
21 changes: 16 additions & 5 deletions command/operator_diagnose.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -431,10 +431,15 @@ func (c *OperatorDiagnoseCommand) offlineDiagnostics(ctx context.Context) error
})

sealcontext, sealspan := diagnose.StartSpan(ctx, "Create Vault Server Configuration Seals")
var seals []vault.Seal
var sealConfigError error

barrierSeal, barrierWrapper, unwrapSeal, seals, sealConfigError, err := setSeal(server, config, make([]string, 0), make(map[string]string))
setSealResponse := setSeal(server, config, make([]string, 0), make(map[string]string))
barrierSeal := setSealResponse.barrierSeal
barrierWrapper := setSealResponse.barrierWrapper
unwrapSeal := setSealResponse.unwrapSeal
createdSeals := setSealResponse.createdSeals
allSeals := setSealResponse.allSeals
sealConfigError := setSealResponse.sealConfigError
err := setSealResponse.err
// Check error here
if err != nil {
diagnose.Advise(ctx, "For assistance with the seal stanza, see the Vault configuration documentation.")
Expand All @@ -446,7 +451,7 @@ func (c *OperatorDiagnoseCommand) offlineDiagnostics(ctx context.Context) error
goto SEALFAIL
}

for _, seal := range seals {
for _, seal := range createdSeals {
// There is always one nil seal. We need to skip it so we don't start an empty Finalize-Seal-Shamir
// section.
if seal == nil {
Expand Down Expand Up @@ -525,6 +530,12 @@ SEALFAIL:
return nil
})

sealGenInfo := vault.SealGenerationInfo{
Generation: 1,
Seals: allSeals,
Rewrapped: false,
}

var coreConfig vault.CoreConfig
diagnose.Test(ctx, "Create Core Configuration", func(ctx context.Context) error {
var secureRandomReader io.Reader
Expand All @@ -539,7 +550,7 @@ SEALFAIL:
return diagnose.SpotError(ctx, randReaderTestName, fmt.Errorf("Could not initialize randomness for core: %w.", err))
}
diagnose.SpotOk(ctx, randReaderTestName, "")
coreConfig = createCoreConfig(server, config, *backend, configSR, barrierSeal, unwrapSeal, metricsHelper, metricSink, secureRandomReader)
coreConfig = createCoreConfig(server, config, *backend, configSR, barrierSeal, unwrapSeal, sealGenInfo, metricsHelper, metricSink, secureRandomReader)
return nil
})

Expand Down
93 changes: 74 additions & 19 deletions command/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import (
"sync"
"time"

"github.com/google/go-cmp/cmp"

systemd "github.com/coreos/go-systemd/daemon"
"github.com/hashicorp/errwrap"
"github.com/hashicorp/go-hclog"
Expand Down Expand Up @@ -1250,45 +1252,51 @@ func (c *ServerCommand) Run(args []string) int {
infoKeys = append(infoKeys, expKey)
}

barrierSeal, _, unwrapSeal, _, sealConfigError, err := setSeal(c, config, infoKeys, info)
setSealResponse := setSeal(c, config, infoKeys, info)
// Check error here
if err != nil {
c.UI.Error(err.Error())
return 1
}

if barrierSeal != nil {
if setSealResponse.barrierSeal != nil {
defer func(seal *vault.Seal) {
err = (*seal).Finalize(context.Background())
if err != nil {
c.UI.Error(fmt.Sprintf("Error finalizing seals: %v", err))
}
}(&barrierSeal)
}(&setSealResponse.barrierSeal)
}

if unwrapSeal != nil {
if setSealResponse.unwrapSeal != nil {
defer func(seal *vault.Seal) {
err = (*seal).Finalize(context.Background())
if err != nil {
c.UI.Error(fmt.Sprintf("Error finalizing seals: %v", err))
}
}(&unwrapSeal)
}(&setSealResponse.unwrapSeal)
}

if barrierSeal == nil {
if setSealResponse.barrierSeal == nil {
c.UI.Error("Could not create barrier seal! Most likely proper Seal configuration information was not set, but no error was generated.")
return 1
}

// prepare a secure random reader for core
entropyAugLogger := c.logger.Named("entropy-augmentation")
secureRandomReader, err := configutil.CreateSecureRandomReaderFunc(config.SharedConfig, barrierSeal.GetAccess().GetSealInfoByPriority(), entropyAugLogger)
secureRandomReader, err := configutil.CreateSecureRandomReaderFunc(config.SharedConfig, setSealResponse.barrierSeal.GetAccess().GetSealInfoByPriority(), entropyAugLogger)
if err != nil {
c.UI.Error(err.Error())
return 1
}

coreConfig := createCoreConfig(c, config, backend, configSR, barrierSeal, unwrapSeal, metricsHelper, metricSink, secureRandomReader)
sealGenInfo := vault.SealGenerationInfo{
Generation: 1,
Seals: setSealResponse.allSeals,
Rewrapped: false,
}

coreConfig := createCoreConfig(c, config, backend, configSR, setSealResponse.barrierSeal, setSealResponse.unwrapSeal, sealGenInfo, metricsHelper, metricSink, secureRandomReader)
if c.flagDevThreeNode {
return c.enableThreeNodeDevCluster(&coreConfig, info, infoKeys, c.flagDevListenAddr, os.Getenv("VAULT_DEV_TEMP_DIR"))
}
Expand Down Expand Up @@ -1384,6 +1392,19 @@ func (c *ServerCommand) Run(args []string) int {

}

existingSealGenInfo, err := core.PhysicalSealGenInfo(context.Background())
if err != nil {
c.UI.Error(fmt.Sprintf("Error getting seal generation info: %v", err))
return 1
}

if existingSealGenInfo == nil {
sealGenInfo.Generation = 1
} else if !cmp.Equal(existingSealGenInfo.Seals, setSealResponse.allSeals) { // If the stored copy doesn't match the current configuration, we introduce a new generation which keeps track if a rewrap of all CSPs and seal wrapped values has completed (initially false).
sealGenInfo.Generation++
c.logger.Info("incrementing seal config gen, new generation: ", "sealGenInfo.Generation", sealGenInfo.Generation)
}

// Now we can use the core SubloggerHook to add any new subloggers to
// core.allLoggers
c.SubloggerAdder = core
Expand Down Expand Up @@ -1561,7 +1582,7 @@ func (c *ServerCommand) Run(args []string) int {
return 0
}

if sealConfigError != nil {
if setSealResponse.sealConfigError != nil {
init, err := core.InitializedLocally(context.Background())
if err != nil {
c.UI.Error(fmt.Sprintf("Error checking if core is initialized: %v", err))
Expand Down Expand Up @@ -2531,23 +2552,36 @@ func CheckStorageMigration(b physical.Backend) (*StorageMigrationStatus, error)
return &status, nil
}

// setSeal return barrierSeal, barrierWrapper, unwrapSeal, and all the created seals from the configs so we can close them in Run
type SetSealResponse struct {
barrierSeal vault.Seal
barrierWrapper wrapping.Wrapper
unwrapSeal vault.Seal
createdSeals []vault.Seal
allSeals []*configutil.KMS
sealConfigError error
err error
}

// setSeal return barrierSeal, barrierWrapper, unwrapSeal, all the created seals, and all the provided seals from the configs so we can close them in Run
// The two errors are the sealConfigError and the regular error
func setSeal(c *ServerCommand, config *server.Config, infoKeys []string, info map[string]string) (vault.Seal, wrapping.Wrapper, vault.Seal, []vault.Seal, error, error) {
func setSeal(c *ServerCommand, config *server.Config, infoKeys []string, info map[string]string) (setSealResponse SetSealResponse) {
var barrierSeal vault.Seal
var unwrapSeal vault.Seal

var sealConfigError error
var wrapper wrapping.Wrapper
var barrierWrapper wrapping.Wrapper

if c.flagDevAutoSeal {
var err error
access, _ := vaultseal.NewTestSeal(nil)
barrierSeal, err = vault.NewAutoSeal(access)
if err != nil {
return nil, nil, nil, nil, nil, err
setSealResponse.err = err
return setSealResponse
}
return barrierSeal, nil, nil, nil, nil, nil
setSealResponse.barrierSeal = barrierSeal
return setSealResponse
}

// Handle the case where no seal is provided
Expand All @@ -2573,6 +2607,7 @@ func setSeal(c *ServerCommand, config *server.Config, infoKeys []string, info ma
var createdSeals []vault.Seal = make([]vault.Seal, 0)
enabledSeals := make([]vaultseal.SealInfo, 0)
disabledSeals := make([]vaultseal.SealInfo, 0)
allSeals := make([]*configutil.KMS, 0)

var enabledSealType string
var disabledSealType string
Expand All @@ -2599,8 +2634,16 @@ func setSeal(c *ServerCommand, config *server.Config, infoKeys []string, info ma
wrapper, sealConfigError = configutil.ConfigureWrapper(configSeal, &sealInfoKeys, &sealInfoMap, sealLogger)
if sealConfigError != nil {
if !errwrap.ContainsType(sealConfigError, new(logical.KeyNotFoundError)) {
return barrierSeal, barrierWrapper, unwrapSeal, createdSeals, sealConfigError, fmt.Errorf(
"Error parsing Seal configuration: %s", sealConfigError)
return SetSealResponse{
barrierSeal: barrierSeal,
barrierWrapper: barrierWrapper,
unwrapSeal: unwrapSeal,
createdSeals: createdSeals,
allSeals: allSeals,
sealConfigError: sealConfigError,
err: fmt.Errorf(
"Error parsing Seal configuration: %s", sealConfigError),
}
}
}
if wrapper == nil {
Expand Down Expand Up @@ -2634,6 +2677,7 @@ func setSeal(c *ServerCommand, config *server.Config, infoKeys []string, info ma
infoKeys = append(infoKeys, infoPrefix+k)
info[infoPrefix+k] = sealInfoMap[k]
}
allSeals = append(allSeals, configSeal)
}

var err error
Expand All @@ -2642,7 +2686,8 @@ func setSeal(c *ServerCommand, config *server.Config, infoKeys []string, info ma
} else {
barrierSeal, err = vault.NewAutoSeal(vaultseal.NewAccess(enabledSeals))
if err != nil {
return nil, nil, nil, nil, nil, err
setSealResponse.err = err
return setSealResponse
}
}

Expand All @@ -2651,12 +2696,21 @@ func setSeal(c *ServerCommand, config *server.Config, infoKeys []string, info ma
} else if disabledSealType != "" {
unwrapSeal, err = vault.NewAutoSeal(vaultseal.NewAccess(disabledSeals))
if err != nil {
return nil, nil, nil, nil, nil, err
setSealResponse.err = err
return setSealResponse
}
}
createdSeals = append(createdSeals, barrierSeal)

return barrierSeal, barrierWrapper, unwrapSeal, createdSeals, sealConfigError, nil
return SetSealResponse{
barrierSeal: barrierSeal,
barrierWrapper: barrierWrapper,
unwrapSeal: unwrapSeal,
createdSeals: createdSeals,
allSeals: allSeals,
sealConfigError: sealConfigError,
err: nil,
}
}

func initHaBackend(c *ServerCommand, config *server.Config, coreConfig *vault.CoreConfig, backend physical.Backend) (bool, error) {
Expand Down Expand Up @@ -2847,7 +2901,7 @@ func runUnseal(c *ServerCommand, core *vault.Core, ctx context.Context) {
}

func createCoreConfig(c *ServerCommand, config *server.Config, backend physical.Backend, configSR sr.ServiceRegistration, barrierSeal, unwrapSeal vault.Seal,
metricsHelper *metricsutil.MetricsHelper, metricSink *metricsutil.ClusterMetricSink, secureRandomReader io.Reader,
sealGenInfo vault.SealGenerationInfo, metricsHelper *metricsutil.MetricsHelper, metricSink *metricsutil.ClusterMetricSink, secureRandomReader io.Reader,
) vault.CoreConfig {
coreConfig := &vault.CoreConfig{
RawConfig: config,
Expand Down Expand Up @@ -2892,6 +2946,7 @@ func createCoreConfig(c *ServerCommand, config *server.Config, backend physical.
DisableSSCTokens: config.DisableSSCTokens,
Experiments: config.Experiments,
AdministrativeNamespacePath: config.AdministrativeNamespacePath,
SealGenInfo: sealGenInfo,
}

if c.flagDev {
Expand Down
14 changes: 14 additions & 0 deletions vault/core.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,9 @@ type Core struct {
// seal is our seal, for seal configuration information
seal Seal

// sealGenInfo tracks the generation of seals
sealGenInfo SealGenerationInfo

// raftJoinDoneCh is used by the raft retry join routine to inform unseal process
// that the join is complete
raftJoinDoneCh chan struct{}
Expand Down Expand Up @@ -748,6 +751,9 @@ type CoreConfig struct {
// seal in migration scenarios.
UnwrapSeal Seal

// SealGenInfo tracks the generation of seals
SealGenInfo SealGenerationInfo

SecureRandomReader io.Reader

LogLevel string
Expand Down Expand Up @@ -1029,6 +1035,7 @@ func CreateCore(conf *CoreConfig) (*Core, error) {
pendingRemovalMountsAllowed: conf.PendingRemovalMountsAllowed,
expirationRevokeRetryBase: conf.ExpirationRevokeRetryBase,
rollbackMountPathMetrics: conf.MetricSink.TelemetryConsts.RollbackMetricsIncludeMountPoint,
sealGenInfo: conf.SealGenInfo,
}

c.standbyStopCh.Store(make(chan struct{}))
Expand Down Expand Up @@ -2382,6 +2389,13 @@ func (s standardUnsealStrategy) unseal(ctx context.Context, logger log.Logger, c
if err := c.setupPluginReload(); err != nil {
return err
}

// store the sealGenInfo
err := c.SetPhysicalSealGenInfo(context.Background(), c.sealGenInfo)
if err != nil {
c.logger.Error("failed to save seal generation info", "error", err)
return err
}
}

if c.getClusterListener() != nil && (c.ha != nil || shouldStartClusterListener(c)) {
Expand Down
53 changes: 53 additions & 0 deletions vault/seal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import (
"fmt"
"sync/atomic"

"github.com/hashicorp/vault/internalshared/configutil"

"github.com/hashicorp/vault/sdk/helper/jsonutil"
"github.com/hashicorp/vault/sdk/physical"

Expand Down Expand Up @@ -45,6 +47,12 @@ const (

// hsmStoredIVPath is the path to the initialization vector for stored keys
hsmStoredIVPath = "core/hsm/iv"

// SealGenInfoPath is the path used to store our seal generation info.
// This is required so that we can detect any seal config changes and introduce
// a new generation which keeps track if a rewrap of all CSPs and seal wrapped
// values has completed .
SealGenInfoPath = "core/seal-gen-info"
)

const (
Expand Down Expand Up @@ -476,3 +484,48 @@ func readStoredKeys(ctx context.Context, storage physical.Backend, encryptor sea

return UnsealWrapStoredBarrierKeys(ctx, encryptor, pe)
}

type SealGenerationInfo struct {
Generation int
Seals []*configutil.KMS
Rewrapped bool
}

func (c *Core) SetPhysicalSealGenInfo(ctx context.Context, sealGenInfo SealGenerationInfo) error {
// Encode the seal generation info
buf, err := json.Marshal(sealGenInfo)
if err != nil {
return fmt.Errorf("failed to encode seal generation info: %w", err)
}

// Store the seal generation info
pe := &physical.Entry{
Key: SealGenInfoPath,
Value: buf,
}

if err := c.physical.Put(ctx, pe); err != nil {
c.logger.Error("failed to write seal generation info", "error", err)
return fmt.Errorf("failed to write seal generation info: %w", err)
}

return nil
}

func (c *Core) PhysicalSealGenInfo(ctx context.Context) (*SealGenerationInfo, error) {
pe, err := c.physical.Get(ctx, SealGenInfoPath)
if err != nil {
return nil, fmt.Errorf("failed to fetch seal generation info: %w", err)
}
if pe == nil {
return nil, nil
}

sealGenInfo := new(SealGenerationInfo)

if err := jsonutil.DecodeJSON(pe.Value, sealGenInfo); err != nil {
return nil, fmt.Errorf("failed to decode seal generation info: %w", err)
}

return sealGenInfo, nil
}

0 comments on commit d76cd87

Please sign in to comment.