diff --git a/clientv3/integration/network_partition_test.go b/clientv3/integration/network_partition_test.go new file mode 100644 index 000000000000..1742edb8b310 --- /dev/null +++ b/clientv3/integration/network_partition_test.go @@ -0,0 +1,102 @@ +// Copyright 2017 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !cluster_proxy + +package integration + +import ( + "context" + "testing" + "time" + + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/integration" + "github.com/coreos/etcd/pkg/testutil" +) + +// TestNetworkPartitionBalancerWithPut tests when one member becomes isolated, +// first Put requests fail and following retry succeeds with client balancer +// switching to others. +func TestNetworkPartitionBalancerWithPut(t *testing.T) { + testNetworkPartitionBalancer(t, func(cli *clientv3.Client) { + var err error + for i := 0; i < 2; i++ { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + _, err = cli.Put(ctx, "a", "b") + cancel() + if err == nil { + break + } + if err != context.DeadlineExceeded { + t.Fatalf("#%d: expected %v, got %v", i, context.DeadlineExceeded, err) + } + // give enough time for endpoint switch + // TODO: sync directly from client balancer without random sleep + if i == 0 { + time.Sleep(5 * time.Second) + } + } + if err != nil { + t.Fatalf("balancer did not switch in time (%v)", err) + } + }) +} + +// TestNetworkPartitionBalancerWithGet tests when one member becomes isolated, +// quorum Get request succeeds via internal retry function, while triggering +// endpoint switch out of network partition. +func TestNetworkPartitionBalancerWithGet(t *testing.T) { + testNetworkPartitionBalancer(t, func(cli *clientv3.Client) { + // give enough timeout for balancer endpoint switch + ctx, cancel := context.WithTimeout(context.Background(), 7*time.Second) + _, err := cli.Get(ctx, "a") + cancel() + if err != nil { + t.Errorf("want err == nil, got %v", err) + } + }) +} + +func testNetworkPartitionBalancer(t *testing.T, op func(*clientv3.Client)) { + defer testutil.AfterTest(t) + + clus := integration.NewClusterV3(t, &integration.ClusterConfig{ + Size: 3, + GRPCKeepAliveMinTime: time.Millisecond, // avoid too_many_pings + }) + defer clus.Terminate(t) + + // expect pin ep[0] + ccfg := clientv3.Config{ + Endpoints: []string{clus.Members[0].GRPCAddr()}, + DialTimeout: 3 * time.Second, + DialKeepAliveTime: 2 * time.Second, + DialKeepAliveTimeout: 2 * time.Second, + } + cli, err := clientv3.New(ccfg) + if err != nil { + t.Fatal(err) + } + defer cli.Close() + + // add other endpoints for later endpoint switch + cli.SetEndpoints(clus.Members[0].GRPCAddr(), clus.Members[1].GRPCAddr(), clus.Members[1].GRPCAddr()) + + time.Sleep(3 * time.Second) + clus.Members[0].InjectPartition(t, clus.Members[1:]) + defer clus.Members[0].RecoverPartition(t, clus.Members[1:]) + + op(cli) +}