From bf2891de03c1a6388239ce6bc76c984f40173d2d Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Tue, 31 Oct 2023 13:57:53 -0400 Subject: [PATCH] identity: version check multiple and implicit identities (#18926) Job submitters cannot set multiple identities prior to Nomad 1.7, and cluster administrators should not set the identity configurations for their `consul` and `vault` configuration blocks until all servers have been upgraded. Validate these cases during job submission so as to prevent state store corruption when jobs are submitting in the middle of a cluster upgrade. --- nomad/job_endpoint_hooks.go | 34 +++++++++++++++++++++++++++++----- nomad/leader.go | 5 +++++ version/version.go | 2 +- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/nomad/job_endpoint_hooks.go b/nomad/job_endpoint_hooks.go index 4d703c2a62dd..be11169d4661 100644 --- a/nomad/job_endpoint_hooks.go +++ b/nomad/job_endpoint_hooks.go @@ -381,19 +381,26 @@ func (v *jobValidate) Validate(job *structs.Job) (warnings []error, err error) { multierror.Append(validationErrors, fmt.Errorf("job priority must be between [%d, %d]", structs.JobMinPriority, v.srv.config.JobMaxPriority)) } + okForIdentity := v.isEligibleForMultiIdentity() + for _, tg := range job.TaskGroups { for _, s := range tg.Services { - serviceErrs := v.validateServiceIdentity(s, fmt.Sprintf("task group %s", tg.Name)) + serviceErrs := v.validateServiceIdentity( + s, fmt.Sprintf("task group %s", tg.Name), okForIdentity) multierror.Append(validationErrors, serviceErrs) } for _, t := range tg.Tasks { + if len(t.Identities) > 1 && !okForIdentity { + multierror.Append(validationErrors, fmt.Errorf("tasks can only have 1 identity block until all servers are upgraded to %s or later", minVersionMultiIdentities)) + } for _, s := range t.Services { - serviceErrs := v.validateServiceIdentity(s, fmt.Sprintf("task %s", t.Name)) + serviceErrs := v.validateServiceIdentity( + s, fmt.Sprintf("task %s", t.Name), okForIdentity) multierror.Append(validationErrors, serviceErrs) } - vaultWarns, vaultErrs := v.validateVaultIdentity(t) + vaultWarns, vaultErrs := v.validateVaultIdentity(t, okForIdentity) multierror.Append(validationErrors, vaultErrs) warnings = append(warnings, vaultWarns...) } @@ -402,7 +409,19 @@ func (v *jobValidate) Validate(job *structs.Job) (warnings []error, err error) { return warnings, validationErrors.ErrorOrNil() } -func (v *jobValidate) validateServiceIdentity(s *structs.Service, parent string) error { +func (v *jobValidate) isEligibleForMultiIdentity() bool { + if v.srv == nil || v.srv.serf == nil { + return true // handle tests w/o real servers safely + } + return ServersMeetMinimumVersion( + v.srv.Members(), v.srv.Region(), minVersionMultiIdentities, true) +} + +func (v *jobValidate) validateServiceIdentity(s *structs.Service, parent string, okForIdentity bool) error { + if s.Identity != nil && !okForIdentity { + return fmt.Errorf("Service %s in %s cannot have an identity until all servers are upgraded to %s or later", + s.Name, parent, minVersionMultiIdentities) + } if s.Identity != nil && s.Identity.Name == "" { return fmt.Errorf("Service %s in %s has an identity with an empty name", s.Name, parent) } @@ -415,7 +434,7 @@ func (v *jobValidate) validateServiceIdentity(s *structs.Service, parent string) // // It assumes the jobImplicitIdentitiesHook mutator hook has been called to // inject task identities if necessary. -func (v *jobValidate) validateVaultIdentity(t *structs.Task) ([]error, error) { +func (v *jobValidate) validateVaultIdentity(t *structs.Task, okForIdentity bool) ([]error, error) { var warnings []error if t.Vault == nil { @@ -430,6 +449,11 @@ func (v *jobValidate) validateVaultIdentity(t *structs.Task) ([]error, error) { vaultWIDName := t.Vault.IdentityName() vaultWID := t.GetIdentity(vaultWIDName) + + if vaultWID != nil && !okForIdentity { + return warnings, fmt.Errorf("Task %s cannot have an identity for Vault until all servers are upgraded to %s or later", t.Name, minVersionMultiIdentities) + } + if vaultWID == nil { // Tasks using non-default clusters are required to have an identity. if t.Vault.Cluster != structs.VaultDefaultCluster { diff --git a/nomad/leader.go b/nomad/leader.go index f85bfc8ce5a2..ac846880bcb1 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -79,6 +79,11 @@ var minNomadServiceRegistrationVersion = version.Must(version.NewVersion("1.3.0" // prevent older versions of the server from crashing. var minNodePoolsVersion = version.Must(version.NewVersion("1.6.0")) +// minVersionMultiIdentities is the Nomad version at which users can add +// multiple identity blocks to tasks and workload identities can be +// automatically added to jobs that need access to Consul or Vault +var minVersionMultiIdentities = version.Must(version.NewVersion("1.7.0")) + // monitorLeadership is used to monitor if we acquire or lose our role // as the leader in the Raft cluster. There is some work the leader is // expected to do, so we must react to changes diff --git a/version/version.go b/version/version.go index dd94b345e567..b4352b8753fe 100644 --- a/version/version.go +++ b/version/version.go @@ -19,7 +19,7 @@ var ( GitDescribe string // The main version number that is being run at the moment. - Version = "1.6.4" + Version = "1.7.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release