Merge pull request #1012 from stgraber/main

Relax cluster upgrade requirements around API extensions

cmd/incusd/api_1.0.go

@@ -223,7 +223,7 @@ func api10Get(d *Daemon, r *http.Request) response.Response {
 	}
 
 	srv := api.ServerUntrusted{
-		APIExtensions: version.APIExtensions,
+		APIExtensions: version.APIExtensions[:d.apiExtensions],
 		APIStatus:     "stable",
 		APIVersion:    version.APIVersion,
 		Public:        false,

cmd/incusd/daemon.go

@@ -168,6 +168,9 @@ type Daemon struct {
 	// OVN clients.
 	ovnnb *ovn.NB
 	ovnsb *ovn.SB
+
+	// API info.
+	apiExtensions int
 }
 
 // DaemonConfig holds configuration values for Daemon.
@@ -197,6 +200,7 @@ func newDaemon(config *DaemonConfig, os *sys.OS) *Daemon {
 		shutdownCtx:    shutdownCtx,
 		shutdownCancel: shutdownCancel,
 		shutdownDoneCh: make(chan error),
+		apiExtensions:  len(version.APIExtensions),
 	}
 
 	d.serverCert = func() *localtls.CertInfo { return d.serverCertInt }
@@ -2384,6 +2388,10 @@ func (d *Daemon) nodeRefreshTask(heartbeatData *cluster.APIHeartbeat, isLeader b
 		return
 	}
 
+	if heartbeatData.Version.MinAPIExtensions > 0 && heartbeatData.Version.MinAPIExtensions != d.apiExtensions {
+		d.apiExtensions = heartbeatData.Version.MinAPIExtensions
+	}
+
 	// If the max version of the cluster has changed, check whether we need to upgrade.
 	if d.lastNodeList == nil || d.lastNodeList.Version.APIExtensions != heartbeatData.Version.APIExtensions || d.lastNodeList.Version.Schema != heartbeatData.Version.Schema {
 		err := cluster.MaybeUpdate(s)

internal/server/cluster/heartbeat.go

@@ -46,8 +46,9 @@ type APIHeartbeatMember struct {
 
 // APIHeartbeatVersion contains max versions for all nodes in cluster.
 type APIHeartbeatVersion struct {
-	Schema        int
-	APIExtensions int
+	Schema           int
+	APIExtensions    int
+	MinAPIExtensions int
 }
 
 // NewAPIHearbeat returns initialized APIHeartbeat.
@@ -74,7 +75,7 @@ type APIHeartbeat struct {
 // Update updates an existing APIHeartbeat struct with the raft and all node states supplied.
 // If allNodes provided is an empty set then this is considered a non-full state list.
 func (hbState *APIHeartbeat) Update(fullStateList bool, raftNodes []db.RaftNode, allNodes []db.NodeInfo, offlineThreshold time.Duration) {
-	var maxSchemaVersion, maxAPIExtensionsVersion int
+	var maxSchemaVersion, maxAPIExtensionsVersion, minAPIExtensionsVersion int
 
 	if hbState.Members == nil {
 		hbState.Members = make(map[int64]APIHeartbeatMember)
@@ -115,14 +116,19 @@ func (hbState *APIHeartbeat) Update(fullStateList bool, raftNodes []db.RaftNode,
 			maxAPIExtensionsVersion = node.APIExtensions
 		}
 
+		if minAPIExtensionsVersion == 0 || node.APIExtensions < minAPIExtensionsVersion {
+			minAPIExtensionsVersion = node.APIExtensions
+		}
+
 		if node.Schema > maxSchemaVersion {
 			maxSchemaVersion = node.Schema
 		}
 	}
 
 	hbState.Version = APIHeartbeatVersion{
-		Schema:        maxSchemaVersion,
-		APIExtensions: maxAPIExtensionsVersion,
+		Schema:           maxSchemaVersion,
+		APIExtensions:    maxAPIExtensionsVersion,
+		MinAPIExtensions: minAPIExtensionsVersion,
 	}
 
 	if len(raftNodeMap) > 0 && hbState.cluster != nil {

internal/server/db/cluster/open.go

@@ -250,7 +250,8 @@ func checkClusterIsUpgradable(ctx context.Context, tx *sql.Tx, target [2]int) er
 	}
 
 	for _, version := range versions {
-		n, err := daemonUtil.CompareVersions(target, version)
+		// Compare schema versions only.
+		n, err := daemonUtil.CompareVersions(target, version, false)
 		if err != nil {
 			return err
 		}

internal/server/db/cluster/open_test.go

@@ -55,8 +55,8 @@ func TestEnsureSchema_ClusterNotUpgradable(t *testing.T) {
 				addNode(t, db, "1", schema, apiExtensions)
 				addNode(t, db, "2", schema, apiExtensions-1)
 			},
-			false, // The schema was not updated
-			"",    // No error is returned
+			true, // The schema was not updated
+			"",   // No error is returned
 		},
 		{
 			`this node's schema is behind`,
@@ -73,8 +73,8 @@ func TestEnsureSchema_ClusterNotUpgradable(t *testing.T) {
 				addNode(t, db, "1", schema, apiExtensions)
 				addNode(t, db, "2", schema, apiExtensions+1)
 			},
-			false,
-			"This cluster member's version is behind, please upgrade",
+			true,
+			"",
 		},
 		{
 			`inconsistent schema version and API extensions number`,
@@ -83,7 +83,7 @@ func TestEnsureSchema_ClusterNotUpgradable(t *testing.T) {
 				addNode(t, db, "2", schema+1, apiExtensions-1)
 			},
 			false,
-			"Cluster members have inconsistent versions",
+			"This cluster member's version is behind, please upgrade",
 		},
 	}
 

internal/server/db/node.go

@@ -91,7 +91,6 @@ type NodeInfoArgs struct {
 // ToAPI returns an API entry.
 func (n NodeInfo) ToAPI(ctx context.Context, tx *ClusterTx, args NodeInfoArgs) (*api.ClusterMember, error) {
 	var err error
-	var maxVersion [2]int
 	var failureDomain string
 
 	domainID := args.MemberFailureDomains[n.Address]
@@ -155,13 +154,19 @@ func (n NodeInfo) ToAPI(ctx context.Context, tx *ClusterTx, args NodeInfoArgs) (
 		result.Status = "Offline"
 		result.Message = fmt.Sprintf("No heartbeat for %s (%s)", time.Since(n.Heartbeat), n.Heartbeat)
 	} else {
+		// Check for max DB schema and API extensions.
+		maxVersion, err := tx.GetNodeMaxVersion(ctx)
+		if err != nil {
+			return nil, err
+		}
+
 		// Check if up to date.
-		n, err := localUtil.CompareVersions(maxVersion, n.Version())
+		ret, err := localUtil.CompareVersions(maxVersion, n.Version(), true)
 		if err != nil {
 			return nil, err
 		}
 
-		if n == 1 {
+		if ret == 1 {
 			result.Status = "Blocked"
 			result.Message = "Needs updating to newer version"
 		}
@@ -336,7 +341,7 @@ func (c *ClusterTx) NodeIsOutdated(ctx context.Context) (bool, error) {
 			continue
 		}
 
-		n, err := localUtil.CompareVersions(node.Version(), version)
+		n, err := localUtil.CompareVersions(node.Version(), version, true)
 		if err != nil {
 			return false, fmt.Errorf("Failed to compare with version of member %s: %w", node.Name, err)
 		}

internal/server/util/version.go

@@ -15,10 +15,16 @@ import (
 // Return an error if inconsistent versions are detected, for example the first
 // member's schema is greater than the second's, but the number of extensions is
 // smaller.
-func CompareVersions(version1, version2 [2]int) (int, error) {
+func CompareVersions(version1, version2 [2]int, checkExtensions bool) (int, error) {
 	schema1, extensions1 := version1[0], version1[1]
 	schema2, extensions2 := version2[0], version2[1]
 
+	if !checkExtensions {
+		// Don't compare API extensions.
+		extensions1 = 0
+		extensions2 = 0
+	}
+
 	if schema1 == schema2 && extensions1 == extensions2 {
 		return 0, nil
 	}