From 6333ed9d68c74202710a7f5c32a620a3e5048682 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Thu, 18 Jul 2024 13:51:39 -0400 Subject: [PATCH 1/4] incusd/db/node: Fix version check in GetAPI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/db/node.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/server/db/node.go b/internal/server/db/node.go index a20fb53485f..37bf03645dc 100644 --- a/internal/server/db/node.go +++ b/internal/server/db/node.go @@ -91,7 +91,6 @@ type NodeInfoArgs struct { // ToAPI returns an API entry. func (n NodeInfo) ToAPI(ctx context.Context, tx *ClusterTx, args NodeInfoArgs) (*api.ClusterMember, error) { var err error - var maxVersion [2]int var failureDomain string domainID := args.MemberFailureDomains[n.Address] @@ -155,6 +154,12 @@ func (n NodeInfo) ToAPI(ctx context.Context, tx *ClusterTx, args NodeInfoArgs) ( result.Status = "Offline" result.Message = fmt.Sprintf("No heartbeat for %s (%s)", time.Since(n.Heartbeat), n.Heartbeat) } else { + // Check for max DB schema and API extensions. + maxVersion, err := tx.GetNodeMaxVersion(ctx) + if err != nil { + return nil, err + } + // Check if up to date. n, err := localUtil.CompareVersions(maxVersion, n.Version()) if err != nil { From 1bbcc630558a52c4bdef1b0bfe99ae1abea03713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Thu, 18 Jul 2024 13:52:21 -0400 Subject: [PATCH 2/4] incusd/db: Allow cluster startup with differing API extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #999 Signed-off-by: Stéphane Graber --- internal/server/db/cluster/open.go | 3 ++- internal/server/db/node.go | 6 +++--- internal/server/util/version.go | 8 +++++++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/internal/server/db/cluster/open.go b/internal/server/db/cluster/open.go index ad7b2e2ee66..7ef761f57ba 100644 --- a/internal/server/db/cluster/open.go +++ b/internal/server/db/cluster/open.go @@ -250,7 +250,8 @@ func checkClusterIsUpgradable(ctx context.Context, tx *sql.Tx, target [2]int) er } for _, version := range versions { - n, err := daemonUtil.CompareVersions(target, version) + // Compare schema versions only. + n, err := daemonUtil.CompareVersions(target, version, false) if err != nil { return err } diff --git a/internal/server/db/node.go b/internal/server/db/node.go index 37bf03645dc..b7e09f6ad50 100644 --- a/internal/server/db/node.go +++ b/internal/server/db/node.go @@ -161,12 +161,12 @@ func (n NodeInfo) ToAPI(ctx context.Context, tx *ClusterTx, args NodeInfoArgs) ( } // Check if up to date. - n, err := localUtil.CompareVersions(maxVersion, n.Version()) + ret, err := localUtil.CompareVersions(maxVersion, n.Version(), true) if err != nil { return nil, err } - if n == 1 { + if ret == 1 { result.Status = "Blocked" result.Message = "Needs updating to newer version" } @@ -341,7 +341,7 @@ func (c *ClusterTx) NodeIsOutdated(ctx context.Context) (bool, error) { continue } - n, err := localUtil.CompareVersions(node.Version(), version) + n, err := localUtil.CompareVersions(node.Version(), version, true) if err != nil { return false, fmt.Errorf("Failed to compare with version of member %s: %w", node.Name, err) } diff --git a/internal/server/util/version.go b/internal/server/util/version.go index 3f5e880c765..8245ede3ea0 100644 --- a/internal/server/util/version.go +++ b/internal/server/util/version.go @@ -15,10 +15,16 @@ import ( // Return an error if inconsistent versions are detected, for example the first // member's schema is greater than the second's, but the number of extensions is // smaller. -func CompareVersions(version1, version2 [2]int) (int, error) { +func CompareVersions(version1, version2 [2]int, checkExtensions bool) (int, error) { schema1, extensions1 := version1[0], version1[1] schema2, extensions2 := version2[0], version2[1] + if !checkExtensions { + // Don't compare API extensions. + extensions1 = 0 + extensions2 = 0 + } + if schema1 == schema2 && extensions1 == extensions2 { return 0, nil } From 14a9f807de6b0759d2c4c282beddacf09f93f66b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Thu, 18 Jul 2024 14:13:29 -0400 Subject: [PATCH 3/4] incusd: Extend heartbeat data for minimum API extension count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This exposes not only the maximum API extension count as done today to detect needed upgrades, but also the minimum value so newer servers can restrict what they expose to clients while waiting for the cluster to be consistent. Signed-off-by: Stéphane Graber --- cmd/incusd/api_1.0.go | 2 +- cmd/incusd/daemon.go | 8 ++++++++ internal/server/cluster/heartbeat.go | 16 +++++++++++----- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/cmd/incusd/api_1.0.go b/cmd/incusd/api_1.0.go index 160e55ae2a0..99c4a0c0e4a 100644 --- a/cmd/incusd/api_1.0.go +++ b/cmd/incusd/api_1.0.go @@ -223,7 +223,7 @@ func api10Get(d *Daemon, r *http.Request) response.Response { } srv := api.ServerUntrusted{ - APIExtensions: version.APIExtensions, + APIExtensions: version.APIExtensions[:d.apiExtensions], APIStatus: "stable", APIVersion: version.APIVersion, Public: false, diff --git a/cmd/incusd/daemon.go b/cmd/incusd/daemon.go index d7575b22ce1..8302c2c817f 100644 --- a/cmd/incusd/daemon.go +++ b/cmd/incusd/daemon.go @@ -168,6 +168,9 @@ type Daemon struct { // OVN clients. ovnnb *ovn.NB ovnsb *ovn.SB + + // API info. + apiExtensions int } // DaemonConfig holds configuration values for Daemon. @@ -197,6 +200,7 @@ func newDaemon(config *DaemonConfig, os *sys.OS) *Daemon { shutdownCtx: shutdownCtx, shutdownCancel: shutdownCancel, shutdownDoneCh: make(chan error), + apiExtensions: len(version.APIExtensions), } d.serverCert = func() *localtls.CertInfo { return d.serverCertInt } @@ -2384,6 +2388,10 @@ func (d *Daemon) nodeRefreshTask(heartbeatData *cluster.APIHeartbeat, isLeader b return } + if heartbeatData.Version.MinAPIExtensions > 0 && heartbeatData.Version.MinAPIExtensions != d.apiExtensions { + d.apiExtensions = heartbeatData.Version.MinAPIExtensions + } + // If the max version of the cluster has changed, check whether we need to upgrade. if d.lastNodeList == nil || d.lastNodeList.Version.APIExtensions != heartbeatData.Version.APIExtensions || d.lastNodeList.Version.Schema != heartbeatData.Version.Schema { err := cluster.MaybeUpdate(s) diff --git a/internal/server/cluster/heartbeat.go b/internal/server/cluster/heartbeat.go index 5b1e2a2cd8c..31439dd7663 100644 --- a/internal/server/cluster/heartbeat.go +++ b/internal/server/cluster/heartbeat.go @@ -46,8 +46,9 @@ type APIHeartbeatMember struct { // APIHeartbeatVersion contains max versions for all nodes in cluster. type APIHeartbeatVersion struct { - Schema int - APIExtensions int + Schema int + APIExtensions int + MinAPIExtensions int } // NewAPIHearbeat returns initialized APIHeartbeat. @@ -74,7 +75,7 @@ type APIHeartbeat struct { // Update updates an existing APIHeartbeat struct with the raft and all node states supplied. // If allNodes provided is an empty set then this is considered a non-full state list. func (hbState *APIHeartbeat) Update(fullStateList bool, raftNodes []db.RaftNode, allNodes []db.NodeInfo, offlineThreshold time.Duration) { - var maxSchemaVersion, maxAPIExtensionsVersion int + var maxSchemaVersion, maxAPIExtensionsVersion, minAPIExtensionsVersion int if hbState.Members == nil { hbState.Members = make(map[int64]APIHeartbeatMember) @@ -115,14 +116,19 @@ func (hbState *APIHeartbeat) Update(fullStateList bool, raftNodes []db.RaftNode, maxAPIExtensionsVersion = node.APIExtensions } + if minAPIExtensionsVersion == 0 || node.APIExtensions < minAPIExtensionsVersion { + minAPIExtensionsVersion = node.APIExtensions + } + if node.Schema > maxSchemaVersion { maxSchemaVersion = node.Schema } } hbState.Version = APIHeartbeatVersion{ - Schema: maxSchemaVersion, - APIExtensions: maxAPIExtensionsVersion, + Schema: maxSchemaVersion, + APIExtensions: maxAPIExtensionsVersion, + MinAPIExtensions: minAPIExtensionsVersion, } if len(raftNodeMap) > 0 && hbState.cluster != nil { From 897049ee8a0f76f50fe87b6507bc3fa310426ea2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Thu, 18 Jul 2024 17:38:29 -0400 Subject: [PATCH 4/4] incusd/db/cluster: Update tests for relaxed API extensions checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/db/cluster/open_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/server/db/cluster/open_test.go b/internal/server/db/cluster/open_test.go index f6f79b51d66..476db90b7f0 100644 --- a/internal/server/db/cluster/open_test.go +++ b/internal/server/db/cluster/open_test.go @@ -55,8 +55,8 @@ func TestEnsureSchema_ClusterNotUpgradable(t *testing.T) { addNode(t, db, "1", schema, apiExtensions) addNode(t, db, "2", schema, apiExtensions-1) }, - false, // The schema was not updated - "", // No error is returned + true, // The schema was not updated + "", // No error is returned }, { `this node's schema is behind`, @@ -73,8 +73,8 @@ func TestEnsureSchema_ClusterNotUpgradable(t *testing.T) { addNode(t, db, "1", schema, apiExtensions) addNode(t, db, "2", schema, apiExtensions+1) }, - false, - "This cluster member's version is behind, please upgrade", + true, + "", }, { `inconsistent schema version and API extensions number`, @@ -83,7 +83,7 @@ func TestEnsureSchema_ClusterNotUpgradable(t *testing.T) { addNode(t, db, "2", schema+1, apiExtensions-1) }, false, - "Cluster members have inconsistent versions", + "This cluster member's version is behind, please upgrade", }, }