identity: version check multiple and implicit identities

Job submitters cannot set multiple identities prior to Nomad 1.7, and cluster
administrators should not set the identity configurations for their `consul` and
`vault` configuration blocks until all servers have been upgraded. Validate
these cases during job submission so as to prevent state store corruption when
jobs are submitting in the middle of a cluster upgrade.

nomad/job_endpoint_hook_implicit_identities.go

@@ -18,6 +18,7 @@ func (jobImplicitIdentitiesHook) Name() string {
 }
 
 func (h jobImplicitIdentitiesHook) Mutate(job *structs.Job) (*structs.Job, []error, error) {
+
 	for _, tg := range job.TaskGroups {
 		for _, s := range tg.Services {
 			h.handleConsulService(s)

nomad/job_endpoint_hooks.go

@@ -381,19 +381,27 @@ func (v *jobValidate) Validate(job *structs.Job) (warnings []error, err error) {
 		multierror.Append(validationErrors, fmt.Errorf("job priority must be between [%d, %d]", structs.JobMinPriority, v.srv.config.JobMaxPriority))
 	}
 
+	okForIdentity := ServersMeetMinimumVersion(
+		v.srv.Members(), v.srv.Region(), minVersionMultiIdentities, true)
+
 	for _, tg := range job.TaskGroups {
 		for _, s := range tg.Services {
-			serviceErrs := v.validateServiceIdentity(s, fmt.Sprintf("task group %s", tg.Name))
+			serviceErrs := v.validateServiceIdentity(
+				s, fmt.Sprintf("task group %s", tg.Name), okForIdentity)
 			multierror.Append(validationErrors, serviceErrs)
 		}
 
 		for _, t := range tg.Tasks {
+			if len(t.Identities) > 1 && !okForIdentity {
+				multierror.Append(validationErrors, fmt.Errorf("tasks can only have 1 identity block until all servers are upgraded to %s", minVersionMultiIdentities))
+			}
 			for _, s := range t.Services {
-				serviceErrs := v.validateServiceIdentity(s, fmt.Sprintf("task %s", t.Name))
+				serviceErrs := v.validateServiceIdentity(
+					s, fmt.Sprintf("task %s", t.Name), okForIdentity)
 				multierror.Append(validationErrors, serviceErrs)
 			}
 
-			vaultWarns, vaultErrs := v.validateVaultIdentity(t)
+			vaultWarns, vaultErrs := v.validateVaultIdentity(t, okForIdentity)
 			multierror.Append(validationErrors, vaultErrs)
 			warnings = append(warnings, vaultWarns...)
 		}
@@ -402,7 +410,11 @@ func (v *jobValidate) Validate(job *structs.Job) (warnings []error, err error) {
 	return warnings, validationErrors.ErrorOrNil()
 }
 
-func (v *jobValidate) validateServiceIdentity(s *structs.Service, parent string) error {
+func (v *jobValidate) validateServiceIdentity(s *structs.Service, parent string, okForIdentity bool) error {
+	if s.Identity != nil && !okForIdentity {
+		return fmt.Errorf("Service %s in %s cannot have an identity until all servers are upgraded to %s",
+			s.Name, parent, minVersionMultiIdentities)
+	}
 	if s.Identity != nil && s.Identity.Name == "" {
 		return fmt.Errorf("Service %s in %s has an identity with an empty name", s.Name, parent)
 	}
@@ -415,7 +427,7 @@ func (v *jobValidate) validateServiceIdentity(s *structs.Service, parent string)
 //
 // It assumes the jobImplicitIdentitiesHook mutator hook has been called to
 // inject task identities if necessary.
-func (v *jobValidate) validateVaultIdentity(t *structs.Task) ([]error, error) {
+func (v *jobValidate) validateVaultIdentity(t *structs.Task, okForIdentity bool) ([]error, error) {
 	var warnings []error
 
 	if t.Vault == nil {
@@ -430,6 +442,11 @@ func (v *jobValidate) validateVaultIdentity(t *structs.Task) ([]error, error) {
 
 	vaultWIDName := t.Vault.IdentityName()
 	vaultWID := t.GetIdentity(vaultWIDName)
+
+	if vaultWID != nil && !okForIdentity {
+		return warnings, fmt.Errorf("Task %s cannot have an identity for Vault until all servers are upgraded to %s", t.Name, minVersionMultiIdentities)
+	}
+
 	if vaultWID == nil {
 		// Tasks using non-default clusters are required to have an identity.
 		if t.Vault.Cluster != structs.VaultDefaultCluster {

nomad/leader.go

@@ -79,6 +79,11 @@ var minNomadServiceRegistrationVersion = version.Must(version.NewVersion("1.3.0"
 // prevent older versions of the server from crashing.
 var minNodePoolsVersion = version.Must(version.NewVersion("1.6.0"))
 
+// minVersionMultiIdentities is the Nomad version at which users can add
+// multiple identity blocks to tasks and workload identities can be
+// automatically added to jobs that need access to Consul or Vault
+var minVersionMultiIdentities = version.Must(version.NewVersion("1.7.0"))
+
 // monitorLeadership is used to monitor if we acquire or lose our role
 // as the leader in the Raft cluster. There is some work the leader is
 // expected to do, so we must react to changes

version/version.go

@@ -19,7 +19,7 @@ var (
 	GitDescribe string
 
 	// The main version number that is being run at the moment.
-	Version = "1.6.4"
+	Version = "1.7.0"
 
 	// A pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release