From 8d057ea2b832251648e69af026407b22bcbfe03f Mon Sep 17 00:00:00 2001 From: Chao Chen Date: Wed, 10 Aug 2022 16:43:21 -0700 Subject: [PATCH] common tests framework: cluster client creation fail with invalid auth Signed-off-by: Chao Chen --- client/v3/config.go | 4 + tests/common/alarm_test.go | 23 ++--- tests/common/compact_test.go | 13 +-- tests/common/defrag_test.go | 9 +- tests/common/endpoint_test.go | 11 ++- tests/common/kv_test.go | 7 +- tests/common/lease_test.go | 11 +-- tests/common/member_test.go | 4 +- tests/common/role_test.go | 11 +-- tests/common/status_test.go | 4 +- tests/common/txn_test.go | 5 +- tests/common/user_test.go | 12 +-- tests/common/watch_test.go | 4 +- tests/framework/config/cluster.go | 1 + tests/framework/e2e.go | 28 ++++++- tests/framework/e2e/etcdctl.go | 48 +++++++++++ tests/framework/integration.go | 84 ++++++++++++++++++- tests/framework/integration/cluster.go | 64 +++++++------- tests/framework/interface.go | 11 ++- tests/framework/util.go | 22 +++++ .../integration/clientv3/maintenance_test.go | 2 +- tests/integration/corrupt_test.go | 8 +- tests/integration/hashkv_test.go | 3 +- tests/integration/v3_lease_test.go | 8 +- 24 files changed, 298 insertions(+), 99 deletions(-) create mode 100644 tests/framework/util.go diff --git a/client/v3/config.go b/client/v3/config.go index 59370e3e3d2..6d76bb9c23e 100644 --- a/client/v3/config.go +++ b/client/v3/config.go @@ -121,6 +121,10 @@ type AuthConfig struct { Password string `json:"password"` } +func (cfg AuthConfig) Empty() bool { + return cfg.Username == "" && cfg.Password == "" +} + // NewClientConfig creates a Config based on the provided ConfigSpec. func NewClientConfig(confSpec *ConfigSpec, lg *zap.Logger) (*Config, error) { tlsCfg, err := newTLSConfig(confSpec.Secure, lg) diff --git a/tests/common/alarm_test.go b/tests/common/alarm_test.go index d9218277d4b..e6b163a61d1 100644 --- a/tests/common/alarm_test.go +++ b/tests/common/alarm_test.go @@ -22,6 +22,7 @@ import ( "time" clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -32,17 +33,18 @@ func TestAlarm(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, config.ClusterConfig{ClusterSize: 1, QuotaBackendBytes: int64(13 * os.Getpagesize())}) defer clus.Close() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { // test small put still works smallbuf := strings.Repeat("a", 64) - if err := clus.Client().Put(ctx, "1st_test", smallbuf, config.PutOptions{}); err != nil { + if err := cc.Put(ctx, "1st_test", smallbuf, config.PutOptions{}); err != nil { t.Fatalf("alarmTest: put kv error (%v)", err) } // write some chunks to fill up the database buf := strings.Repeat("b", os.Getpagesize()) for { - if err := clus.Client().Put(ctx, "2nd_test", buf, config.PutOptions{}); err != nil { + if err := cc.Put(ctx, "2nd_test", buf, config.PutOptions{}); err != nil { if !strings.Contains(err.Error(), "etcdserver: mvcc: database space exceeded") { t.Fatal(err) } @@ -51,20 +53,20 @@ func TestAlarm(t *testing.T) { } // quota alarm should now be on - alarmResp, err := clus.Client().AlarmList(ctx) + alarmResp, err := cc.AlarmList(ctx) if err != nil { t.Fatalf("alarmTest: Alarm error (%v)", err) } // check that Put is rejected when alarm is on - if err := clus.Client().Put(ctx, "3rd_test", smallbuf, config.PutOptions{}); err != nil { + if err := cc.Put(ctx, "3rd_test", smallbuf, config.PutOptions{}); err != nil { if !strings.Contains(err.Error(), "etcdserver: mvcc: database space exceeded") { t.Fatal(err) } } // get latest revision to compact - sresp, err := clus.Client().Status(ctx) + sresp, err := cc.Status(ctx) if err != nil { t.Fatalf("get endpoint status error: %v", err) } @@ -77,12 +79,12 @@ func TestAlarm(t *testing.T) { } // make some space - _, err = clus.Client().Compact(ctx, rvs, config.CompactOption{Physical: true, Timeout: 10 * time.Second}) + _, err = cc.Compact(ctx, rvs, config.CompactOption{Physical: true, Timeout: 10 * time.Second}) if err != nil { t.Fatalf("alarmTest: Compact error (%v)", err) } - if err = clus.Client().Defragment(ctx, config.DefragOption{Timeout: 10 * time.Second}); err != nil { + if err = cc.Defragment(ctx, config.DefragOption{Timeout: 10 * time.Second}); err != nil { t.Fatalf("alarmTest: defrag error (%v)", err) } @@ -92,14 +94,14 @@ func TestAlarm(t *testing.T) { MemberID: alarm.MemberID, Alarm: alarm.Alarm, } - _, err = clus.Client().AlarmDisarm(ctx, alarmMember) + _, err = cc.AlarmDisarm(ctx, alarmMember) if err != nil { t.Fatalf("alarmTest: Alarm error (%v)", err) } } // put one more key below quota - if err := clus.Client().Put(ctx, "4th_test", smallbuf, config.PutOptions{}); err != nil { + if err := cc.Put(ctx, "4th_test", smallbuf, config.PutOptions{}); err != nil { t.Fatal(err) } }) @@ -115,10 +117,11 @@ func TestAlarmlistOnMemberRestart(t *testing.T) { SnapshotCount: 5, }) defer clus.Close() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { for i := 0; i < 6; i++ { - if _, err := clus.Client().AlarmList(ctx); err != nil { + if _, err := cc.AlarmList(ctx); err != nil { t.Fatalf("Unexpected error: %v", err) } } diff --git a/tests/common/compact_test.go b/tests/common/compact_test.go index b1a4848d5da..19883fffb98 100644 --- a/tests/common/compact_test.go +++ b/tests/common/compact_test.go @@ -21,6 +21,8 @@ import ( "time" "github.com/stretchr/testify/assert" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -47,14 +49,15 @@ func TestCompact(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, config.ClusterConfig{ClusterSize: 3}) defer clus.Close() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { var kvs = []testutils.KV{{Key: "key", Val: "val1"}, {Key: "key", Val: "val2"}, {Key: "key", Val: "val3"}} for i := range kvs { - if err := clus.Client().Put(ctx, kvs[i].Key, kvs[i].Val, config.PutOptions{}); err != nil { + if err := cc.Put(ctx, kvs[i].Key, kvs[i].Val, config.PutOptions{}); err != nil { t.Fatalf("compactTest #%d: put kv error (%v)", i, err) } } - get, err := clus.Client().Get(ctx, "key", config.GetOptions{Revision: 3}) + get, err := cc.Get(ctx, "key", config.GetOptions{Revision: 3}) if err != nil { t.Fatalf("compactTest: Get kv by revision error (%v)", err) } @@ -62,12 +65,12 @@ func TestCompact(t *testing.T) { getkvs := testutils.KeyValuesFromGetResponse(get) assert.Equal(t, kvs[1:2], getkvs) - _, err = clus.Client().Compact(ctx, 4, tc.options) + _, err = cc.Compact(ctx, 4, tc.options) if err != nil { t.Fatalf("compactTest: Compact error (%v)", err) } - get, err = clus.Client().Get(ctx, "key", config.GetOptions{Revision: 3}) + get, err = cc.Get(ctx, "key", config.GetOptions{Revision: 3}) if err != nil { if !strings.Contains(err.Error(), "required revision has been compacted") { t.Fatalf("compactTest: Get compact key error (%v)", err) @@ -76,7 +79,7 @@ func TestCompact(t *testing.T) { t.Fatalf("expected '...has been compacted' error, got ") } - _, err = clus.Client().Compact(ctx, 2, tc.options) + _, err = cc.Compact(ctx, 2, tc.options) if err != nil { if !strings.Contains(err.Error(), "required revision has been compacted") { t.Fatal(err) diff --git a/tests/common/defrag_test.go b/tests/common/defrag_test.go index e6bac207084..56922653282 100644 --- a/tests/common/defrag_test.go +++ b/tests/common/defrag_test.go @@ -19,6 +19,8 @@ import ( "testing" "time" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -29,20 +31,21 @@ func TestDefragOnline(t *testing.T) { defer cancel() options := config.DefragOption{Timeout: 10 * time.Second} clus := testRunner.NewCluster(ctx, t, config.ClusterConfig{ClusterSize: 3}) + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { defer clus.Close() var kvs = []testutils.KV{{Key: "key", Val: "val1"}, {Key: "key", Val: "val2"}, {Key: "key", Val: "val3"}} for i := range kvs { - if err := clus.Client().Put(ctx, kvs[i].Key, kvs[i].Val, config.PutOptions{}); err != nil { + if err := cc.Put(ctx, kvs[i].Key, kvs[i].Val, config.PutOptions{}); err != nil { t.Fatalf("compactTest #%d: put kv error (%v)", i, err) } } - _, err := clus.Client().Compact(ctx, 4, config.CompactOption{Physical: true, Timeout: 10 * time.Second}) + _, err := cc.Compact(ctx, 4, config.CompactOption{Physical: true, Timeout: 10 * time.Second}) if err != nil { t.Fatalf("defrag_test: compact with revision error (%v)", err) } - if err = clus.Client().Defragment(ctx, options); err != nil { + if err = cc.Defragment(ctx, options); err != nil { t.Fatalf("defrag_test: defrag error (%v)", err) } }) diff --git a/tests/common/endpoint_test.go b/tests/common/endpoint_test.go index 963340addf6..a23648fbe39 100644 --- a/tests/common/endpoint_test.go +++ b/tests/common/endpoint_test.go @@ -19,6 +19,8 @@ import ( "testing" "time" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -29,8 +31,9 @@ func TestEndpointStatus(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, config.ClusterConfig{ClusterSize: 3}) defer clus.Close() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { - _, err := clus.Client().Status(ctx) + _, err := cc.Status(ctx) if err != nil { t.Fatalf("get endpoint status error: %v", err) } @@ -43,8 +46,9 @@ func TestEndpointHashKV(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, config.ClusterConfig{ClusterSize: 3}) defer clus.Close() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { - _, err := clus.Client().HashKV(ctx, 0) + _, err := cc.HashKV(ctx, 0) if err != nil { t.Fatalf("get endpoint hashkv error: %v", err) } @@ -57,8 +61,9 @@ func TestEndpointHealth(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, config.ClusterConfig{ClusterSize: 3}) defer clus.Close() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { - if err := clus.Client().Health(ctx); err != nil { + if err := cc.Health(ctx); err != nil { t.Fatalf("get endpoint health error: %v", err) } }) diff --git a/tests/common/kv_test.go b/tests/common/kv_test.go index 0be00ba964a..8ae74b4b7bd 100644 --- a/tests/common/kv_test.go +++ b/tests/common/kv_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/assert" clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -33,7 +34,7 @@ func TestKVPut(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { key, value := "foo", "bar" @@ -67,7 +68,7 @@ func TestKVGet(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { var ( @@ -127,7 +128,7 @@ func TestKVDelete(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { kvs := []string{"a", "b", "c", "c/abc", "d"} tests := []struct { diff --git a/tests/common/lease_test.go b/tests/common/lease_test.go index 0e36faa5e90..848b36ce07b 100644 --- a/tests/common/lease_test.go +++ b/tests/common/lease_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -59,7 +60,7 @@ func TestLeaseGrantTimeToLive(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { ttl := int64(10) @@ -103,7 +104,7 @@ func TestLeaseGrantAndList(t *testing.T) { t.Logf("Creating cluster...") clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) t.Logf("Created cluster and client") testutils.ExecuteUntil(ctx, t, func() { var createdLeases []clientv3.LeaseID @@ -150,7 +151,7 @@ func TestLeaseGrantTimeToLiveExpired(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { leaseResp, err := cc.Grant(ctx, 2) @@ -187,7 +188,7 @@ func TestLeaseGrantKeepAliveOnce(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { leaseResp, err := cc.Grant(ctx, 2) @@ -216,7 +217,7 @@ func TestLeaseGrantRevoke(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { leaseResp, err := cc.Grant(ctx, 20) diff --git a/tests/common/member_test.go b/tests/common/member_test.go index 581cce689a0..a4e8137a19c 100644 --- a/tests/common/member_test.go +++ b/tests/common/member_test.go @@ -35,7 +35,7 @@ func TestMemberList(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { resp, err := cc.MemberList(ctx) @@ -109,7 +109,7 @@ func TestMemberAdd(t *testing.T) { c.DisableStrictReconfigCheck = !quorumTc.strictReconfigCheck clus := testRunner.NewCluster(ctx, t, c) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { var addResp *clientv3.MemberAddResponse diff --git a/tests/common/role_test.go b/tests/common/role_test.go index 6f116fb5be9..b31a9c54f79 100644 --- a/tests/common/role_test.go +++ b/tests/common/role_test.go @@ -22,6 +22,7 @@ import ( "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -34,7 +35,7 @@ func TestRoleAdd_Simple(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.RoleAdd(ctx, "root") @@ -52,7 +53,7 @@ func TestRoleAdd_Error(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, config.ClusterConfig{ClusterSize: 1}) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.RoleAdd(ctx, "test-role") if err != nil { @@ -75,7 +76,7 @@ func TestRootRole(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, config.ClusterConfig{ClusterSize: 1}) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.RoleAdd(ctx, "root") if err != nil { @@ -105,7 +106,7 @@ func TestRoleGrantRevokePermission(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, config.ClusterConfig{ClusterSize: 1}) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.RoleAdd(ctx, "role1") if err != nil { @@ -140,7 +141,7 @@ func TestRoleDelete(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, config.ClusterConfig{ClusterSize: 1}) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.RoleAdd(ctx, "role1") if err != nil { diff --git a/tests/common/status_test.go b/tests/common/status_test.go index 746d091a2b6..8d910b11f3b 100644 --- a/tests/common/status_test.go +++ b/tests/common/status_test.go @@ -19,6 +19,8 @@ import ( "testing" "time" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -32,7 +34,7 @@ func TestStatus(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { rs, err := cc.Status(ctx) diff --git a/tests/common/txn_test.go b/tests/common/txn_test.go index 9bdd47a8243..edab66876c4 100644 --- a/tests/common/txn_test.go +++ b/tests/common/txn_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/assert" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -60,7 +61,7 @@ func TestTxnSucc(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, cfg.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { if err := cc.Put(ctx, "key1", "value1", config.PutOptions{}); err != nil { t.Fatalf("could not create key:%s, value:%s", "key1", "value1") @@ -104,7 +105,7 @@ func TestTxnFail(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, cfg.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { if err := cc.Put(ctx, "key1", "value1", config.PutOptions{}); err != nil { t.Fatalf("could not create key:%s, value:%s", "key1", "value1") diff --git a/tests/common/user_test.go b/tests/common/user_test.go index bdd785943ad..338d5ebff70 100644 --- a/tests/common/user_test.go +++ b/tests/common/user_test.go @@ -20,6 +20,8 @@ import ( "time" "github.com/stretchr/testify/assert" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -68,7 +70,7 @@ func TestUserAdd_Simple(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { resp, err := cc.UserAdd(ctx, nc.username, nc.password, config.UserAddOptions{NoPassword: nc.noPassword}) @@ -102,7 +104,7 @@ func TestUserAdd_DuplicateUserNotAllowed(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { user := "barb" @@ -131,7 +133,7 @@ func TestUserList(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { // No Users Yet @@ -172,7 +174,7 @@ func TestUserDelete(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { user := "barb" @@ -224,7 +226,7 @@ func TestUserChangePassword(t *testing.T) { defer cancel() clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { user := "barb" diff --git a/tests/common/watch_test.go b/tests/common/watch_test.go index e7b9cac9d41..b759ae24abe 100644 --- a/tests/common/watch_test.go +++ b/tests/common/watch_test.go @@ -6,6 +6,8 @@ import ( "time" "github.com/stretchr/testify/assert" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -20,7 +22,7 @@ func TestWatch(t *testing.T) { clus := testRunner.NewCluster(ctx, t, tc.config) defer clus.Close() - cc := clus.Client() + cc := framework.MustClient(clus.Client(clientv3.AuthConfig{})) testutils.ExecuteUntil(ctx, t, func() { tests := []struct { puts []testutils.KV diff --git a/tests/framework/config/cluster.go b/tests/framework/config/cluster.go index 0af2cc1e8c3..5b79e5ff429 100644 --- a/tests/framework/config/cluster.go +++ b/tests/framework/config/cluster.go @@ -32,5 +32,6 @@ type ClusterConfig struct { ClientTLS TLSConfig QuotaBackendBytes int64 DisableStrictReconfigCheck bool + AuthToken string SnapshotCount int } diff --git a/tests/framework/e2e.go b/tests/framework/e2e.go index da694091612..1935cbe0f6d 100644 --- a/tests/framework/e2e.go +++ b/tests/framework/e2e.go @@ -22,8 +22,11 @@ import ( "time" "go.etcd.io/etcd/client/pkg/v3/testutil" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" + "go.etcd.io/etcd/tests/v3/framework/integration" + "google.golang.org/grpc" ) type e2eRunner struct{} @@ -47,6 +50,7 @@ func (e e2eRunner) NewCluster(ctx context.Context, t testing.TB, cfg config.Clus ClusterSize: cfg.ClusterSize, QuotaBackendBytes: cfg.QuotaBackendBytes, DisableStrictReconfigCheck: cfg.DisableStrictReconfigCheck, + AuthTokenOpts: cfg.AuthToken, SnapshotCount: cfg.SnapshotCount, } switch cfg.ClientTLS { @@ -78,15 +82,31 @@ func (e e2eRunner) NewCluster(ctx context.Context, t testing.TB, cfg config.Clus if err != nil { t.Fatalf("could not start etcd integrationCluster: %s", err) } - return &e2eCluster{*epc} + return &e2eCluster{t, *epc} } type e2eCluster struct { + t testing.TB e2e.EtcdProcessCluster } -func (c *e2eCluster) Client() Client { - return e2eClient{e2e.NewEtcdctl(c.Cfg, c.EndpointsV3())} +func (c *e2eCluster) Client(cfg clientv3.AuthConfig) (Client, error) { + etcdctl := e2e.NewEtcdctl(c.Cfg, c.EndpointsV3()) + if !cfg.Empty() { + // use integration test client to validate if permissions are authorized + _, err := integration.NewClient(c.t, clientv3.Config{ + Endpoints: c.EndpointsV3(), + DialTimeout: 5 * time.Second, + DialOptions: []grpc.DialOption{grpc.WithBlock()}, + Username: cfg.Username, + Password: cfg.Password, + }) + if err != nil { + return nil, err + } + etcdctl = etcdctl.WithAuth(cfg.Username, cfg.Password) + } + return e2eClient{etcdctl}, nil } func (c *e2eCluster) Members() (ms []Member) { @@ -107,7 +127,7 @@ func (c *e2eCluster) WaitLeader(t testing.TB) int { // WaitMembersForLeader waits until given members agree on the same leader, // and returns its 'index' in the 'membs' list func (c *e2eCluster) WaitMembersForLeader(ctx context.Context, t testing.TB, membs []Member) int { - cc := c.Client() + cc := MustClient(c.Client(clientv3.AuthConfig{})) // ensure leader is up via linearizable get for { diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index e29c1c27324..1c114e524dc 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -31,6 +31,8 @@ import ( type EtcdctlV3 struct { cfg *EtcdProcessClusterConfig endpoints []string + userName string + password string } func NewEtcdctl(cfg *EtcdProcessClusterConfig, endpoints []string) *EtcdctlV3 { @@ -40,6 +42,12 @@ func NewEtcdctl(cfg *EtcdProcessClusterConfig, endpoints []string) *EtcdctlV3 { } } +func (ctl *EtcdctlV3) WithAuth(userName, password string) *EtcdctlV3 { + ctl.userName = userName + ctl.password = password + return ctl +} + func (ctl *EtcdctlV3) DowngradeEnable(ctx context.Context, version string) error { _, err := SpawnWithExpectLines(ctx, ctl.cmdArgs("downgrade", "enable", version), nil, "Downgrade enable success") return err @@ -234,6 +242,7 @@ func AddTxnResponse(resp *clientv3.TxnResponse, jsonData string) { } } } + func (ctl *EtcdctlV3) MemberList(ctx context.Context) (*clientv3.MemberListResponse, error) { var resp clientv3.MemberListResponse err := ctl.spawnJsonCmd(ctx, &resp, "member", "list") @@ -283,6 +292,9 @@ func (ctl *EtcdctlV3) flags() map[string]string { } } fmap["endpoints"] = strings.Join(ctl.endpoints, ",") + if ctl.userName != "" && ctl.password != "" { + fmap["user"] = ctl.userName + ":" + ctl.password + } return fmap } @@ -453,6 +465,24 @@ func (ctl *EtcdctlV3) AlarmDisarm(ctx context.Context, _ *clientv3.AlarmMember) return &resp, err } +func (ctl *EtcdctlV3) AuthEnable(ctx context.Context) (*clientv3.AuthEnableResponse, error) { + var resp clientv3.AuthEnableResponse + err := ctl.spawnJsonCmd(ctx, &resp, "auth", "enable") + return &resp, err +} + +func (ctl *EtcdctlV3) AuthDisable(ctx context.Context) (*clientv3.AuthDisableResponse, error) { + var resp clientv3.AuthDisableResponse + err := ctl.spawnJsonCmd(ctx, &resp, "auth", "disable") + return &resp, err +} + +func (ctl *EtcdctlV3) AuthStatus(ctx context.Context) (*clientv3.AuthStatusResponse, error) { + var resp clientv3.AuthStatusResponse + err := ctl.spawnJsonCmd(ctx, &resp, "auth", "status") + return &resp, err +} + func (ctl *EtcdctlV3) UserAdd(ctx context.Context, name, password string, opts config.UserAddOptions) (*clientv3.AuthUserAddResponse, error) { args := ctl.cmdArgs() args = append(args, "user", "add") @@ -492,6 +522,12 @@ func (ctl *EtcdctlV3) UserAdd(ctx context.Context, name, password string, opts c return &resp, err } +func (ctl *EtcdctlV3) UserGet(ctx context.Context, name string) (*clientv3.AuthUserGetResponse, error) { + var resp clientv3.AuthUserGetResponse + err := ctl.spawnJsonCmd(ctx, &resp, "user", "get", name) + return &resp, err +} + func (ctl *EtcdctlV3) UserList(ctx context.Context) (*clientv3.AuthUserListResponse, error) { var resp clientv3.AuthUserListResponse err := ctl.spawnJsonCmd(ctx, &resp, "user", "list") @@ -521,6 +557,18 @@ func (ctl *EtcdctlV3) UserChangePass(ctx context.Context, user, newPass string) return err } +func (ctl *EtcdctlV3) UserGrantRole(ctx context.Context, user string, role string) (*clientv3.AuthUserGrantRoleResponse, error) { + var resp clientv3.AuthUserGrantRoleResponse + err := ctl.spawnJsonCmd(ctx, &resp, "user", "grant-role", user, role) + return &resp, err +} + +func (ctl *EtcdctlV3) UserRevokeRole(ctx context.Context, user string, role string) (*clientv3.AuthUserRevokeRoleResponse, error) { + var resp clientv3.AuthUserRevokeRoleResponse + err := ctl.spawnJsonCmd(ctx, &resp, "user", "revoke-role", user, role) + return &resp, err +} + func (ctl *EtcdctlV3) RoleAdd(ctx context.Context, name string) (*clientv3.AuthRoleAddResponse, error) { var resp clientv3.AuthRoleAddResponse err := ctl.spawnJsonCmd(ctx, &resp, "role", "add", name) diff --git a/tests/framework/integration.go b/tests/framework/integration.go index c5517168f7f..63b670f7018 100644 --- a/tests/framework/integration.go +++ b/tests/framework/integration.go @@ -48,6 +48,7 @@ func (e integrationRunner) NewCluster(ctx context.Context, t testing.TB, cfg con Size: cfg.ClusterSize, QuotaBackendBytes: cfg.QuotaBackendBytes, DisableStrictReconfigCheck: cfg.DisableStrictReconfigCheck, + AuthToken: cfg.AuthToken, SnapshotCount: uint64(cfg.SnapshotCount), } integrationCfg.ClientTLS, err = tlsInfo(t, cfg.ClientTLS) @@ -117,12 +118,19 @@ func (c *integrationCluster) Close() error { return nil } -func (c *integrationCluster) Client() Client { - cc, err := c.ClusterClient() +func (c *integrationCluster) Client(cfg clientv3.AuthConfig) (Client, error) { + option := func(_ *clientv3.Config) {} + if !cfg.Empty() { + option = func(clientCfg *clientv3.Config) { + clientCfg.Username = cfg.Username + clientCfg.Password = cfg.Password + } + } + cc, err := c.ClusterClient(c.t, option) if err != nil { - c.t.Fatal(err) + return nil, err } - return integrationClient{Client: cc} + return integrationClient{Client: cc}, nil } type integrationClient struct { @@ -261,17 +269,85 @@ func (c integrationClient) TimeToLive(ctx context.Context, id clientv3.LeaseID, return c.Client.TimeToLive(ctx, id, leaseOpts...) } +func (c integrationClient) Leases(ctx context.Context) (*clientv3.LeaseLeasesResponse, error) { + return c.Client.Leases(ctx) +} + +func (c integrationClient) KeepAliveOnce(ctx context.Context, id clientv3.LeaseID) (*clientv3.LeaseKeepAliveResponse, error) { + return c.Client.KeepAliveOnce(ctx, id) +} + +func (c integrationClient) Revoke(ctx context.Context, id clientv3.LeaseID) (*clientv3.LeaseRevokeResponse, error) { + return c.Client.Revoke(ctx, id) +} + +func (c integrationClient) AuthEnable(ctx context.Context) (*clientv3.AuthEnableResponse, error) { + return c.Client.AuthEnable(ctx) +} + +func (c integrationClient) AuthDisable(ctx context.Context) (*clientv3.AuthDisableResponse, error) { + return c.Client.AuthDisable(ctx) +} + +func (c integrationClient) AuthStatus(ctx context.Context) (*clientv3.AuthStatusResponse, error) { + return c.Client.AuthStatus(ctx) +} + func (c integrationClient) UserAdd(ctx context.Context, name, password string, opts config.UserAddOptions) (*clientv3.AuthUserAddResponse, error) { return c.Client.UserAddWithOptions(ctx, name, password, &clientv3.UserAddOptions{ NoPassword: opts.NoPassword, }) } +func (c integrationClient) UserGet(ctx context.Context, name string) (*clientv3.AuthUserGetResponse, error) { + return c.Client.UserGet(ctx, name) +} + +func (c integrationClient) UserList(ctx context.Context) (*clientv3.AuthUserListResponse, error) { + return c.Client.UserList(ctx) +} + +func (c integrationClient) UserDelete(ctx context.Context, name string) (*clientv3.AuthUserDeleteResponse, error) { + return c.Client.UserDelete(ctx, name) +} + func (c integrationClient) UserChangePass(ctx context.Context, user, newPass string) error { _, err := c.Client.UserChangePassword(ctx, user, newPass) return err } +func (c integrationClient) UserGrantRole(ctx context.Context, user string, role string) (*clientv3.AuthUserGrantRoleResponse, error) { + return c.Client.UserGrantRole(ctx, user, role) +} + +func (c integrationClient) UserRevokeRole(ctx context.Context, user string, role string) (*clientv3.AuthUserRevokeRoleResponse, error) { + return c.Client.UserRevokeRole(ctx, user, role) +} + +func (c integrationClient) RoleAdd(ctx context.Context, name string) (*clientv3.AuthRoleAddResponse, error) { + return c.Client.RoleAdd(ctx, name) +} + +func (c integrationClient) RoleGrantPermission(ctx context.Context, name string, key, rangeEnd string, permType clientv3.PermissionType) (*clientv3.AuthRoleGrantPermissionResponse, error) { + return c.Client.RoleGrantPermission(ctx, name, key, rangeEnd, permType) +} + +func (c integrationClient) RoleGet(ctx context.Context, role string) (*clientv3.AuthRoleGetResponse, error) { + return c.Client.RoleGet(ctx, role) +} + +func (c integrationClient) RoleList(ctx context.Context) (*clientv3.AuthRoleListResponse, error) { + return c.Client.RoleList(ctx) +} + +func (c integrationClient) RoleRevokePermission(ctx context.Context, role string, key, rangeEnd string) (*clientv3.AuthRoleRevokePermissionResponse, error) { + return c.Client.RoleRevokePermission(ctx, role, key, rangeEnd) +} + +func (c integrationClient) RoleDelete(ctx context.Context, role string) (*clientv3.AuthRoleDeleteResponse, error) { + return c.Client.RoleDelete(ctx, role) +} + func (c integrationClient) Txn(ctx context.Context, compares, ifSucess, ifFail []string, o config.TxnOptions) (*clientv3.TxnResponse, error) { txn := c.Client.Txn(ctx) var cmps []clientv3.Cmp diff --git a/tests/framework/integration/cluster.go b/tests/framework/integration/cluster.go index 32cb20e91b4..bbbe6290d39 100644 --- a/tests/framework/integration/cluster.go +++ b/tests/framework/integration/cluster.go @@ -179,8 +179,7 @@ type Cluster struct { Members []*Member LastMemberNum int - mu sync.Mutex - clusterClient *clientv3.Client + mu sync.Mutex } func SchemeFromTLSInfo(tls *transport.TLSInfo) string { @@ -441,7 +440,7 @@ func (c *Cluster) waitMembersForLeader(ctx context.Context, t testing.TB, membs for _, m := range membs { possibleLead[uint64(m.Server.MemberId())] = true } - cc, err := c.ClusterClient() + cc, err := c.ClusterClient(t) if err != nil { t.Fatal(err) } @@ -1378,13 +1377,6 @@ func (c *Cluster) Terminate(t testutil.TB) { if t != nil { t.Logf("========= Cluster termination started =====================") } - c.mu.Lock() - if c.clusterClient != nil { - if err := c.clusterClient.Close(); err != nil { - t.Error(err) - } - } - c.mu.Unlock() for _, m := range c.Members { if m.Client != nil { m.Client.Close() @@ -1420,32 +1412,40 @@ func (c *Cluster) Endpoints() []string { return endpoints } -func (c *Cluster) ClusterClient() (client *clientv3.Client, err error) { - if c.clusterClient == nil { - var endpoints []string - for _, m := range c.Members { - endpoints = append(endpoints, m.GrpcURL) - } - cfg := clientv3.Config{ - Endpoints: endpoints, - DialTimeout: 5 * time.Second, - DialOptions: []grpc.DialOption{grpc.WithBlock()}, - MaxCallSendMsgSize: c.Cfg.ClientMaxCallSendMsgSize, - MaxCallRecvMsgSize: c.Cfg.ClientMaxCallRecvMsgSize, - } - if c.Cfg.ClientTLS != nil { - tls, err := c.Cfg.ClientTLS.ClientConfig() - if err != nil { - return nil, err - } - cfg.TLS = tls - } - c.clusterClient, err = newClientV3(cfg) +func (c *Cluster) ClusterClient(t testing.TB, opts ...func(*clientv3.Config)) (client *clientv3.Client, err error) { + cfg, err := c.newClientCfg() + if err != nil { + return nil, err + } + for _, opt := range opts { + opt(cfg) + } + client, err = newClientV3(*cfg) + if err != nil { + return nil, err + } + t.Cleanup(func() { + client.Close() + }) + return client, nil +} + +func (c *Cluster) newClientCfg() (*clientv3.Config, error) { + cfg := &clientv3.Config{ + Endpoints: c.Endpoints(), + DialTimeout: 5 * time.Second, + DialOptions: []grpc.DialOption{grpc.WithBlock()}, + MaxCallSendMsgSize: c.Cfg.ClientMaxCallSendMsgSize, + MaxCallRecvMsgSize: c.Cfg.ClientMaxCallRecvMsgSize, + } + if c.Cfg.ClientTLS != nil { + tls, err := c.Cfg.ClientTLS.ClientConfig() if err != nil { return nil, err } + cfg.TLS = tls } - return c.clusterClient, nil + return cfg, nil } // NewClientV3 creates a new grpc client connection to the member diff --git a/tests/framework/interface.go b/tests/framework/interface.go index f3ff33de00f..696879be9f2 100644 --- a/tests/framework/interface.go +++ b/tests/framework/interface.go @@ -30,7 +30,7 @@ type testRunner interface { type Cluster interface { Members() []Member - Client() Client + Client(cfg clientv3.AuthConfig) (Client, error) WaitLeader(t testing.TB) int Close() error } @@ -57,18 +57,23 @@ type Client interface { Leases(context context.Context) (*clientv3.LeaseLeasesResponse, error) KeepAliveOnce(context context.Context, id clientv3.LeaseID) (*clientv3.LeaseKeepAliveResponse, error) Revoke(context context.Context, id clientv3.LeaseID) (*clientv3.LeaseRevokeResponse, error) + + AuthEnable(context context.Context) (*clientv3.AuthEnableResponse, error) + AuthDisable(context context.Context) (*clientv3.AuthDisableResponse, error) + AuthStatus(context context.Context) (*clientv3.AuthStatusResponse, error) UserAdd(context context.Context, name, password string, opts config.UserAddOptions) (*clientv3.AuthUserAddResponse, error) + UserGet(context context.Context, name string) (*clientv3.AuthUserGetResponse, error) UserList(context context.Context) (*clientv3.AuthUserListResponse, error) UserDelete(context context.Context, name string) (*clientv3.AuthUserDeleteResponse, error) UserChangePass(context context.Context, user, newPass string) error - + UserGrantRole(context context.Context, user string, role string) (*clientv3.AuthUserGrantRoleResponse, error) + UserRevokeRole(context context.Context, user string, role string) (*clientv3.AuthUserRevokeRoleResponse, error) RoleAdd(context context.Context, name string) (*clientv3.AuthRoleAddResponse, error) RoleGrantPermission(context context.Context, name string, key, rangeEnd string, permType clientv3.PermissionType) (*clientv3.AuthRoleGrantPermissionResponse, error) RoleGet(context context.Context, role string) (*clientv3.AuthRoleGetResponse, error) RoleList(context context.Context) (*clientv3.AuthRoleListResponse, error) RoleRevokePermission(context context.Context, role string, key, rangeEnd string) (*clientv3.AuthRoleRevokePermissionResponse, error) RoleDelete(context context.Context, role string) (*clientv3.AuthRoleDeleteResponse, error) - Txn(context context.Context, compares, ifSucess, ifFail []string, o config.TxnOptions) (*clientv3.TxnResponse, error) MemberList(context context.Context) (*clientv3.MemberListResponse, error) diff --git a/tests/framework/util.go b/tests/framework/util.go new file mode 100644 index 00000000000..98da1653db9 --- /dev/null +++ b/tests/framework/util.go @@ -0,0 +1,22 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package framework + +func MustClient(c Client, err error) Client { + if err != nil { + panic(err) + } + return c +} diff --git a/tests/integration/clientv3/maintenance_test.go b/tests/integration/clientv3/maintenance_test.go index 8292319daf3..a6290823f96 100644 --- a/tests/integration/clientv3/maintenance_test.go +++ b/tests/integration/clientv3/maintenance_test.go @@ -77,7 +77,7 @@ func TestCompactionHash(t *testing.T) { clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 1}) defer clus.Terminate(t) - cc, err := clus.ClusterClient() + cc, err := clus.ClusterClient(t) if err != nil { t.Fatal(err) } diff --git a/tests/integration/corrupt_test.go b/tests/integration/corrupt_test.go index b9cb374f3eb..5bcd00ae1e7 100644 --- a/tests/integration/corrupt_test.go +++ b/tests/integration/corrupt_test.go @@ -34,7 +34,7 @@ func TestPeriodicCheck(t *testing.T) { clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) - cc, err := clus.ClusterClient() + cc, err := clus.ClusterClient(t) require.NoError(t, err) ctx := context.Background() @@ -70,7 +70,7 @@ func TestPeriodicCheckDetectsCorruption(t *testing.T) { clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) - cc, err := clus.ClusterClient() + cc, err := clus.ClusterClient(t) require.NoError(t, err) ctx := context.Background() @@ -106,7 +106,7 @@ func TestCompactHashCheck(t *testing.T) { clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) - cc, err := clus.ClusterClient() + cc, err := clus.ClusterClient(t) require.NoError(t, err) ctx := context.Background() @@ -143,7 +143,7 @@ func TestCompactHashCheckDetectCorruption(t *testing.T) { clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) - cc, err := clus.ClusterClient() + cc, err := clus.ClusterClient(t) require.NoError(t, err) ctx := context.Background() diff --git a/tests/integration/hashkv_test.go b/tests/integration/hashkv_test.go index 00d571a1692..b623bd8aa4e 100644 --- a/tests/integration/hashkv_test.go +++ b/tests/integration/hashkv_test.go @@ -34,11 +34,10 @@ func TestCompactionHash(t *testing.T) { clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 1}) defer clus.Terminate(t) - cc, err := clus.ClusterClient() + cc, err := clus.ClusterClient(t) if err != nil { t.Fatal(err) } - client := &http.Client{ Transport: &http.Transport{ DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { diff --git a/tests/integration/v3_lease_test.go b/tests/integration/v3_lease_test.go index 5899a606abf..49a2e48e1f0 100644 --- a/tests/integration/v3_lease_test.go +++ b/tests/integration/v3_lease_test.go @@ -523,11 +523,11 @@ func testLeaseStress(t *testing.T, stresser func(context.Context, pb.LeaseClient errc := make(chan error) if useClusterClient { + clusterClient, err := clus.ClusterClient(t) + if err != nil { + t.Fatal(err) + } for i := 0; i < 300; i++ { - clusterClient, err := clus.ClusterClient() - if err != nil { - t.Fatal(err) - } go func(i int) { errc <- stresser(ctx, integration.ToGRPC(clusterClient).Lease) }(i) } } else {