Skip to content

Commit

Permalink
Only update zen2 settings if there is no v6 master node
Browse files Browse the repository at this point in the history 
When doing any zen2 API call (voting config exclusions), we checked if
there is at least one v7+ master in the cluster in order to proceed.
This is wrong because if the current master is a v6 node, the call will
fail.
In a cluster with mixed v6 -> v7 versions, the current master is
supposed to be a v6 node. Zen2 APIs can be called once there is no more
v6 node in the cluster, at which point the current master is a v7 node.

For more context, see
elastic#1281 (comment)
  • Loading branch information
@sebgl
sebgl committed Aug 19, 2019
1 parent 633adf7 commit 0cbb157
Show file tree
Hide file tree
Showing 8 changed files with 200 additions and 82 deletions.
2 changes: 1 addition & 1 deletion operators/pkg/controller/elasticsearch/driver/downscale.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ func updateZenSettingsForDownscale(ctx downscaleContext, downscale ssetDownscale
}

// Update zen2 settings to exclude leaving master nodes from voting.
if err := zen2.AddToVotingConfigExclusions(ctx.esClient, downscale.statefulSet, downscale.leavingNodeNames()); err != nil {
if err := zen2.AddToVotingConfigExclusions(ctx.k8sClient, ctx.esClient, ctx.es, downscale.leavingNodeNames()); err != nil {
return err
}

Expand Down
112 changes: 69 additions & 43 deletions operators/pkg/controller/elasticsearch/driver/downscale_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/elastic/cloud-on-k8s/operators/pkg/controller/common"
"github.com/elastic/cloud-on-k8s/operators/pkg/controller/common/reconciler"
esclient "github.com/elastic/cloud-on-k8s/operators/pkg/controller/elasticsearch/client"
"github.com/elastic/cloud-on-k8s/operators/pkg/controller/elasticsearch/label"
"github.com/elastic/cloud-on-k8s/operators/pkg/controller/elasticsearch/nodespec"
"github.com/elastic/cloud-on-k8s/operators/pkg/controller/elasticsearch/observer"
"github.com/elastic/cloud-on-k8s/operators/pkg/controller/elasticsearch/reconcile"
Expand All @@ -30,53 +31,61 @@ import (

// Sample StatefulSets to use in tests
var (
clusterName = "cluster"
ssetMaster3Replicas = nodespec.CreateTestSset("ssetMaster3Replicas", "7.2.0", 3, true, false)
podsSsetMaster3Replicas = []corev1.Pod{
{
ObjectMeta: metav1.ObjectMeta{
Namespace: ssetMaster3Replicas.Namespace,
Name: sset.PodName(ssetMaster3Replicas.Name, 0),
},
},
{
ObjectMeta: metav1.ObjectMeta{
Namespace: ssetMaster3Replicas.Namespace,
Name: sset.PodName(ssetMaster3Replicas.Name, 1),
},
},
{
ObjectMeta: metav1.ObjectMeta{
Namespace: ssetMaster3Replicas.Namespace,
Name: sset.PodName(ssetMaster3Replicas.Name, 2),
},
},
nodespec.TestPod{
Namespace: ssetMaster3Replicas.Namespace,
Name: sset.PodName(ssetMaster3Replicas.Name, 0),
ClusterName: clusterName,
Version: "7.2.0",
Master: true,
}.Build(),
nodespec.TestPod{
Namespace: ssetMaster3Replicas.Namespace,
Name: sset.PodName(ssetMaster3Replicas.Name, 1),
ClusterName: clusterName,
Version: "7.2.0",
Master: true,
}.Build(),
nodespec.TestPod{
Namespace: ssetMaster3Replicas.Namespace,
Name: sset.PodName(ssetMaster3Replicas.Name, 2),
ClusterName: clusterName,
Version: "7.2.0",
Master: true,
}.Build(),
}
ssetData4Replicas = nodespec.CreateTestSset("ssetData4Replicas", "7.2.0", 4, false, true)
podsSsetData4Replicas = []corev1.Pod{
{
ObjectMeta: metav1.ObjectMeta{
Namespace: ssetMaster3Replicas.Namespace,
Name: sset.PodName(ssetData4Replicas.Name, 0),
},
},
{
ObjectMeta: metav1.ObjectMeta{
Namespace: ssetMaster3Replicas.Namespace,
Name: sset.PodName(ssetData4Replicas.Name, 1),
},
},
{
ObjectMeta: metav1.ObjectMeta{
Namespace: ssetMaster3Replicas.Namespace,
Name: sset.PodName(ssetData4Replicas.Name, 2),
},
},
{
ObjectMeta: metav1.ObjectMeta{
Namespace: ssetMaster3Replicas.Namespace,
Name: sset.PodName(ssetData4Replicas.Name, 3),
},
},
nodespec.TestPod{
Namespace: ssetData4Replicas.Namespace,
Name: sset.PodName(ssetData4Replicas.Name, 0),
ClusterName: clusterName,
Version: "7.2.0",
Data: true,
}.Build(),
nodespec.TestPod{
Namespace: ssetData4Replicas.Namespace,
Name: sset.PodName(ssetData4Replicas.Name, 1),
ClusterName: clusterName,
Version: "7.2.0",
Data: true,
}.Build(),
nodespec.TestPod{
Namespace: ssetData4Replicas.Namespace,
Name: sset.PodName(ssetData4Replicas.Name, 2),
ClusterName: clusterName,
Version: "7.2.0",
Data: true,
}.Build(),
nodespec.TestPod{
Namespace: ssetData4Replicas.Namespace,
Name: sset.PodName(ssetData4Replicas.Name, 3),
ClusterName: clusterName,
Version: "7.2.0",
Data: true,
}.Build(),
}
runtimeObjs = []runtime.Object{&ssetMaster3Replicas, &ssetData4Replicas,
&podsSsetMaster3Replicas[0], &podsSsetMaster3Replicas[1], &podsSsetMaster3Replicas[2],
Expand Down Expand Up @@ -757,7 +766,24 @@ func Test_doDownscale_zen2VotingConfigExclusions(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
k8sClient := k8s.WrapClient(fake.NewFakeClient(&ssetMasters, &ssetData))
es := v1alpha1.Elasticsearch{
ObjectMeta: metav1.ObjectMeta{
Namespace: ssetMasters.Namespace,
Name: "es",
},
}
// simulate an existing v7 master for zen2 to be called
v7Pod := corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: es.Namespace,
Labels: map[string]string{
label.ClusterNameLabelName: es.Name,
string(label.NodeTypesMasterLabelName): "true",
label.VersionLabelName: "7.1.0",
},
},
}
k8sClient := k8s.WrapClient(fake.NewFakeClient(&ssetMasters, &ssetData, &v7Pod))
esClient := &fakeESClient{}
downscaleCtx := downscaleContext{
k8sClient: k8sClient,
Expand Down
32 changes: 32 additions & 0 deletions operators/pkg/controller/elasticsearch/nodespec/fixtures.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,35 @@ func CreateTestSset(name string, esversion string, replicas int32, master bool,
},
}
}

type TestPod struct {
Namespace string
Name string
ClusterName string
StatefulSetName string
Version string
Master bool
Data bool
}

func (t TestPod) Build() corev1.Pod {
labels := map[string]string{
label.VersionLabelName: t.Version,
label.ClusterNameLabelName: t.ClusterName,
label.StatefulSetNameLabelName: t.StatefulSetName,
}
label.NodeTypesMasterLabelName.Set(t.Master, labels)
label.NodeTypesDataLabelName.Set(t.Data, labels)
return corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: t.Namespace,
Name: t.Name,
Labels: labels,
},
}
}

func (t TestPod) BuildPtr() *corev1.Pod {
pod := t.Build()
return &pod
}
9 changes: 9 additions & 0 deletions operators/pkg/controller/elasticsearch/sset/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,15 @@ func GetActualPodsForCluster(c k8s.Client, es v1alpha1.Elasticsearch) ([]corev1.
return pods.Items, nil
}

// GetActualMastersForCluster returns the list of existing master-eligible pods for the cluster.
func GetActualMastersForCluster(c k8s.Client, es v1alpha1.Elasticsearch) ([]corev1.Pod, error) {
pods, err := GetActualPodsForCluster(c, es)
if err != nil {
return nil, err
}
return label.FilterMasterNodePods(pods), nil
}

// ScheduledUpgradesDone returns true if all pods scheduled for upgrade have been upgraded.
// This is done by checking the revision of pods whose ordinal is higher or equal than the StatefulSet
// rollingUpdate.Partition index.
Expand Down
28 changes: 24 additions & 4 deletions operators/pkg/controller/elasticsearch/version/zen2/compatibility.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,11 @@ package zen2
import (
appsv1 "k8s.io/api/apps/v1"

"github.com/elastic/cloud-on-k8s/operators/pkg/apis/elasticsearch/v1alpha1"
"github.com/elastic/cloud-on-k8s/operators/pkg/controller/common/version"
"github.com/elastic/cloud-on-k8s/operators/pkg/controller/elasticsearch/label"
"github.com/elastic/cloud-on-k8s/operators/pkg/controller/elasticsearch/sset"
"github.com/elastic/cloud-on-k8s/operators/pkg/utils/k8s"
)

// zen2VersionMatch returns true if the given Elasticsearch version is compatible with zen2.
Expand All @@ -21,8 +24,25 @@ func IsCompatibleWithZen2(statefulSet appsv1.StatefulSet) bool {
return sset.ESVersionMatch(statefulSet, zen2VersionMatch)
}

// AtLeastOneNodeCompatibleWithZen2 returns true if the given StatefulSetList contains
// at least one StatefulSet compatible with zen2.
func AtLeastOneNodeCompatibleWithZen2(statefulSets sset.StatefulSetList) bool {
return sset.AtLeastOneESVersionMatch(statefulSets, zen2VersionMatch)
// AllMastersCompatibleWithZen2 returns true if all master nodes in the given cluster can use zen2 APIs.
// During a v6 -> v7 rolling upgrade, we can only call zen2 APIs once the current master is using v7,
// which would happen only if there is no more v6 master-eligible nodes in the cluster.
func AllMastersCompatibleWithZen2(c k8s.Client, es v1alpha1.Elasticsearch) (bool, error) {
masters, err := sset.GetActualMastersForCluster(c, es)
if err != nil {
return false, err
}
if len(masters) == 0 {
return false, nil
}
for _, pod := range masters {
v, err := label.ExtractVersion(pod.Labels)
if err != nil {
return false, err
}
if !zen2VersionMatch(*v) {
return false, nil
}
}
return true, nil
}
63 changes: 43 additions & 20 deletions operators/pkg/controller/elasticsearch/version/zen2/compatibility_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,17 @@ package zen2
import (
"testing"

"github.com/stretchr/testify/require"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client/fake"

"github.com/elastic/cloud-on-k8s/operators/pkg/apis/elasticsearch/v1alpha1"
"github.com/elastic/cloud-on-k8s/operators/pkg/controller/elasticsearch/label"
"github.com/elastic/cloud-on-k8s/operators/pkg/controller/elasticsearch/sset"
"github.com/elastic/cloud-on-k8s/operators/pkg/controller/elasticsearch/nodespec"
"github.com/elastic/cloud-on-k8s/operators/pkg/utils/k8s"
)

func createStatefulSetWithESVersion(version string) appsv1.StatefulSet {
Expand All @@ -26,7 +31,6 @@ func createStatefulSetWithESVersion(version string) appsv1.StatefulSet {
}

func TestIsCompatibleWithZen2(t *testing.T) {

tests := []struct {
name string
sset appsv1.StatefulSet
Expand Down Expand Up @@ -57,37 +61,56 @@ func TestIsCompatibleWithZen2(t *testing.T) {
}
}

func TestAtLeastOneNodeCompatibleWithZen2(t *testing.T) {
func TestAllMastersCompatibleWithZen2(t *testing.T) {
es := v1alpha1.Elasticsearch{
ObjectMeta: metav1.ObjectMeta{
Namespace: "ns",
Name: "cluster",
},
}
tests := []struct {
name string
statefulSets sset.StatefulSetList
want bool
name string
pods []runtime.Object
want bool
}{
{
name: "no sset",
statefulSets: nil,
want: false,
name: "only v7 master nodes",
pods: []runtime.Object{
nodespec.TestPod{Namespace: es.Namespace, Name: "node0", ClusterName: es.Name, Version: "7.2.0", Master: true}.BuildPtr(),
nodespec.TestPod{Namespace: es.Namespace, Name: "node1", ClusterName: es.Name, Version: "7.2.0", Master: true}.BuildPtr(),
nodespec.TestPod{Namespace: es.Namespace, Name: "node2", ClusterName: es.Name, Version: "7.2.0", Data: true}.BuildPtr(),
},
want: true,
},
{
name: "none compatible",
statefulSets: sset.StatefulSetList{createStatefulSetWithESVersion("6.8.0"), createStatefulSetWithESVersion("6.8.1")},
want: false,
name: "only v6 master nodes (with v7 data nodes)",
pods: []runtime.Object{
nodespec.TestPod{Namespace: es.Namespace, Name: "node0", ClusterName: es.Name, Version: "6.8.0", Master: true}.BuildPtr(),
nodespec.TestPod{Namespace: es.Namespace, Name: "node1", ClusterName: es.Name, Version: "6.8.0", Master: true}.BuildPtr(),
nodespec.TestPod{Namespace: es.Namespace, Name: "node2", ClusterName: es.Name, Version: "7.2.0", Data: true}.BuildPtr(),
},
want: false,
},
{
name: "one compatible",
statefulSets: sset.StatefulSetList{createStatefulSetWithESVersion("6.8.0"), createStatefulSetWithESVersion("7.1.0")},
want: true,
name: "mixed v6/v7 masters",
pods: []runtime.Object{
nodespec.TestPod{Namespace: es.Namespace, Name: "node0", ClusterName: es.Name, Version: "7.2.0", Master: true}.BuildPtr(),
nodespec.TestPod{Namespace: es.Namespace, Name: "node1", ClusterName: es.Name, Version: "6.8.0", Master: true}.BuildPtr(),
},
want: false,
},
{
name: "all compatible",
statefulSets: sset.StatefulSetList{createStatefulSetWithESVersion("7.1.0"), createStatefulSetWithESVersion("7.2.0")},
want: true,
name: "no pods",
pods: []runtime.Object{},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := AtLeastOneNodeCompatibleWithZen2(tt.statefulSets); got != tt.want {
t.Errorf("AtLeastOneNodeCompatibleWithZen2() = %v, want %v", got, tt.want)
got, err := AllMastersCompatibleWithZen2(k8s.WrapClient(fake.NewFakeClient(tt.pods...)), es)
require.NoError(t, err)
if got != tt.want {
t.Errorf("AllMastersCompatibleWithZen2() got = %v, want %v", got, tt.want)
}
})
}
Expand Down
19 changes: 14 additions & 5 deletions operators/pkg/controller/elasticsearch/version/zen2/voting_exclusions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ package zen2
import (
"context"

appsv1 "k8s.io/api/apps/v1"
logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"

"github.com/elastic/cloud-on-k8s/operators/pkg/apis/elasticsearch/v1alpha1"
Expand All @@ -21,11 +20,15 @@ var (
)

// AddToVotingConfigExclusions adds the given node names to exclude from voting config exclusions.
func AddToVotingConfigExclusions(esClient client.Client, sset appsv1.StatefulSet, excludeNodes []string) error {
if !IsCompatibleWithZen2(sset) {
func AddToVotingConfigExclusions(c k8s.Client, esClient client.Client, es v1alpha1.Elasticsearch, excludeNodes []string) error {
compatible, err := AllMastersCompatibleWithZen2(c, es)
if err != nil {
return err
}
if !compatible {
return nil
}
log.Info("Setting voting config exclusions", "namespace", sset.Namespace, "nodes", excludeNodes)
log.Info("Setting voting config exclusions", "namespace", es.Namespace, "nodes", excludeNodes)
ctx, cancel := context.WithTimeout(context.Background(), client.DefaultReqTimeout)
defer cancel()
if err := esClient.AddVotingConfigExclusions(ctx, excludeNodes, ""); err != nil {
Expand All @@ -50,9 +53,15 @@ func canClearVotingConfigExclusions(c k8s.Client, es v1alpha1.Elasticsearch, act
// ClearVotingConfigExclusions resets the voting config exclusions if all excluded nodes are properly removed.
// It returns true if this should be retried later (re-queued).
func ClearVotingConfigExclusions(es v1alpha1.Elasticsearch, c k8s.Client, esClient client.Client, actualStatefulSets sset.StatefulSetList) (bool, error) {
if !AtLeastOneNodeCompatibleWithZen2(actualStatefulSets) {
compatible, err := AllMastersCompatibleWithZen2(c, es)
if err != nil {
return false, err
}
if !compatible {
// nothing to do
return false, nil
}

canClear, err := canClearVotingConfigExclusions(c, es, actualStatefulSets)
if err != nil {
return false, err
Expand Down
Loading

0 comments on commit 0cbb157

Please sign in to comment.