etcdserver: adjust election ticks on restart

Adjust advance Raft election ticks on restart.

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

etcdserver/raft.go

@@ -379,16 +379,6 @@ func (r *raftNode) resumeSending() {
 	p.Resume()
 }
 
-// advanceTicksForElection advances ticks to the node for fast election.
-// This reduces the time to wait for first leader election if bootstrapping the whole
-// cluster, while leaving at least 1 heartbeat for possible existing leader
-// to contact it.
-func advanceTicksForElection(n raft.Node, electionTicks int) {
-	for i := 0; i < electionTicks-1; i++ {
-		n.Tick()
-	}
-}
-
 func startNode(cfg ServerConfig, cl *membership.RaftCluster, ids []types.ID) (id types.ID, n raft.Node, s *raft.MemoryStorage, w *wal.WAL) {
 	var err error
 	member := cl.MemberByName(cfg.Name)
@@ -426,7 +416,6 @@ func startNode(cfg ServerConfig, cl *membership.RaftCluster, ids []types.ID) (id
 	raftStatusMu.Lock()
 	raftStatus = n.Status
 	raftStatusMu.Unlock()
-	advanceTicksForElection(n, c.ElectionTick)
 	return id, n, s, w
 }
 
@@ -460,7 +449,6 @@ func restartNode(cfg ServerConfig, snapshot *raftpb.Snapshot) (types.ID, *member
 	raftStatusMu.Lock()
 	raftStatus = n.Status
 	raftStatusMu.Unlock()
-	advanceTicksForElection(n, c.ElectionTick)
 	return id, cl, n, s, w
 }
 

etcdserver/server.go

@@ -179,6 +179,12 @@ type EtcdServer struct {
 	consistIndex consistentIndex // must use atomic operations to access; keep 64-bit aligned.
 	r            raftNode        // uses 64-bit atomics; keep 64-bit aligned.
 
+	// advanceRaftTicks advances ticks of Raft node.
+	// This can be used for fast-forwarding election
+	// ticks in multi data-center deployments, thus
+	// speeding up election process.
+	advanceRaftTicks func(ticks int)
+
 	readych chan struct{}
 	Cfg     ServerConfig
 
@@ -439,6 +445,12 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) {
 		hostWhitelist: cfg.HostWhitelist,
 	}
 
+	srv.advanceRaftTicks = func(ticks int) {
+		for i := 0; i < ticks; i++ {
+			srv.r.tick()
+		}
+	}
+
 	srv.applyV2 = &applierV2store{store: srv.v2store, cluster: srv.cluster}
 
 	srv.be = be
@@ -521,6 +533,29 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) {
 	}
 	srv.r.transport = tr
 
+	// fresh start, or single node cluster
+	if !haveWAL || len(cl.Members()) == 1 {
+		ticks := cfg.ElectionTicks - 1
+		plog.Infof("%s fast-forwarding %d ticks (election ticks %d) with %d found member(s)", srv.ID(), ticks, cfg.ElectionTicks, len(cl.Members()))
+		srv.advanceRaftTicks(ticks)
+	} else {
+		// wait for peer connection reports,
+		// then advance ticks conditionally
+		srv.goAttach(func() {
+			// TODO: handle restarted single-node cluster (no peer to be notified from)
+			select {
+			case <-time.After(rafthttp.ConnReadTimeout):
+				// connection failed, or no active peers
+				plog.Infof("%s waited %s but no active peer found (or restarted 1-node cluster)", srv.ID(), rafthttp.ConnReadTimeout)
+			case <-tr.InitialPeerNotify():
+				// found active peers
+				ticks := cfg.ElectionTicks - 2
+				plog.Infof("%s fast-forwarding %d ticks (election ticks %d) with %d found member(s)", srv.ID(), ticks, cfg.ElectionTicks, len(cl.Members()))
+				srv.advanceRaftTicks(ticks)
+			}
+		})
+	}
+
 	return srv, nil
 }