-
Notifications
You must be signed in to change notification settings - Fork 9.7k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[3.5] member replace e2e test #17123
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
// Copyright 2023 The etcd Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
//go:build !cluster_proxy | ||
|
||
package e2e | ||
|
||
import ( | ||
"context" | ||
"math/rand" | ||
"os" | ||
"testing" | ||
"time" | ||
|
||
"github.com/stretchr/testify/require" | ||
|
||
"go.etcd.io/etcd/server/v3/etcdserver" | ||
"go.etcd.io/etcd/tests/v3/framework/e2e" | ||
) | ||
|
||
func TestMemberReplace(t *testing.T) { | ||
e2e.BeforeTest(t) | ||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) | ||
defer cancel() | ||
|
||
epc, err := e2e.NewEtcdProcessCluster(t, &e2e.EtcdProcessClusterConfig{ | ||
ClusterSize: 3, | ||
KeepDataDir: true, | ||
CorruptCheckTime: time.Second, | ||
}) | ||
require.NoError(t, err) | ||
defer epc.Close() | ||
|
||
memberIdx := rand.Int() % len(epc.Procs) | ||
member := epc.Procs[memberIdx] | ||
memberName := member.Config().Name | ||
var endpoints []string | ||
for i := 1; i < len(epc.Procs); i++ { | ||
endpoints = append(endpoints, epc.Procs[(memberIdx+i)%len(epc.Procs)].EndpointsGRPC()...) | ||
} | ||
cc := NewEtcdctl(endpoints, e2e.ClientNonTLS, false, false) | ||
|
||
memberID, found, err := getMemberIdByName(ctx, cc, memberName) | ||
require.NoError(t, err) | ||
require.Equal(t, found, true, "Member not found") | ||
|
||
// Need to wait health interval for cluster to accept member changes | ||
time.Sleep(etcdserver.HealthInterval) | ||
|
||
t.Logf("Removing member %s", memberName) | ||
_, err = cc.MemberRemove(memberID) | ||
require.NoError(t, err) | ||
_, found, err = getMemberIdByName(ctx, cc, memberName) | ||
require.NoError(t, err) | ||
require.Equal(t, found, false, "Expected member to be removed") | ||
for member.IsRunning() { | ||
member.Close() | ||
time.Sleep(10 * time.Millisecond) | ||
} | ||
|
||
t.Logf("Removing member %s data", memberName) | ||
err = os.RemoveAll(member.Config().DataDirPath) | ||
require.NoError(t, err) | ||
|
||
t.Logf("Adding member %s back", memberName) | ||
removedMemberPeerUrl := member.Config().Purl.String() | ||
_, err = cc.MemberAdd(memberName, []string{removedMemberPeerUrl}) | ||
require.NoError(t, err) | ||
member.Config().Args = patchArgs(member.Config().Args, "initial-cluster-state", "existing") | ||
require.NoError(t, err) | ||
|
||
// Sleep 100ms to bypass the known issue https://github.com/etcd-io/etcd/issues/16687. | ||
time.Sleep(100 * time.Millisecond) | ||
t.Logf("Starting member %s", memberName) | ||
err = member.Start() | ||
require.NoError(t, err) | ||
e2e.ExecuteUntil(ctx, t, func() { | ||
for { | ||
_, found, err := getMemberIdByName(ctx, cc, memberName) | ||
if err != nil || !found { | ||
time.Sleep(10 * time.Millisecond) | ||
continue | ||
} | ||
break | ||
} | ||
}) | ||
} |
Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
@@ -17,6 +17,7 @@ package e2e | |||||||||||||||||||||||||||||||
import ( | ||||||||||||||||||||||||||||||||
"context" | ||||||||||||||||||||||||||||||||
"fmt" | ||||||||||||||||||||||||||||||||
"strings" | ||||||||||||||||||||||||||||||||
"testing" | ||||||||||||||||||||||||||||||||
"time" | ||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||
|
@@ -114,3 +115,29 @@ func fillEtcdWithData(ctx context.Context, c *clientv3.Client, dbSize int) error | |||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||
return g.Wait() | ||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||
func getMemberIdByName(ctx context.Context, c *Etcdctl, name string) (id uint64, found bool, err error) { | ||||||||||||||||||||||||||||||||
resp, err := c.MemberList() | ||||||||||||||||||||||||||||||||
if err != nil { | ||||||||||||||||||||||||||||||||
return 0, false, err | ||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||
for _, member := range resp.Members { | ||||||||||||||||||||||||||||||||
if name == member.Name { | ||||||||||||||||||||||||||||||||
return member.ID, true, nil | ||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||
return 0, false, nil | ||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||
// Different implementations here since 3.5 e2e test framework does not have "initial-cluster-state" as a default argument | ||||||||||||||||||||||||||||||||
// Append new flag if not exist, otherwise replace the value | ||||||||||||||||||||||||||||||||
func patchArgs(args []string, flag, newValue string) []string { | ||||||||||||||||||||||||||||||||
for i, arg := range args { | ||||||||||||||||||||||||||||||||
if strings.Contains(arg, flag) { | ||||||||||||||||||||||||||||||||
args[i] = fmt.Sprintf("--%s=%s", flag, newValue) | ||||||||||||||||||||||||||||||||
return args | ||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||
args = append(args, fmt.Sprintf("--%s=%s", flag, newValue)) | ||||||||||||||||||||||||||||||||
return args | ||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||
Comment on lines
+134
to
+143
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why the implementation is different from the main branch? Lines 139 to 147 in 93530f6
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 3.5 does not have "initial-cluster-state" as a default argument in e2e framework. So, we need to append it in etcd/tests/framework/e2e/cluster.go Lines 468 to 473 in 93530f6
First introduced in commit 6f63f4b There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We should backport #16707 to 3.5 if possible. Please add a comment to explain the difference for now. |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For 3.5 release, there is no background goroutine as zombie process's reaper.
I think we can just export
Close()
here. Otherwise, zombie process is running andmember.IsRunning
is true until timeout.https://github.com/etcd-io/etcd/blob/5b572f15162d9b61979fb5eed65e65b026917464/pkg/expect/expect.go#L88C8-L88C23