vault: recover from standby losing etcd lease (#3031) (#3511)

This change makes these errors transient instead of permanent:

[ERROR] core: failed to acquire lock: error=etcdserver: requested lease not found

After this change, there can still be one of these errors when a
standby vault that lost its lease tries to become leader, but on the
next lock acquisition attempt a new session will be created. With this
new session, the standby will be able to become the leader.

physical/etcd/etcd3.go

@@ -267,6 +267,21 @@ func (c *EtcdLock) Lock(stopCh <-chan struct{}) (<-chan struct{}, error) {
 		return nil, EtcdLockHeldError
 	}
 
+	select {
+	case _, ok := <-c.etcdSession.Done():
+		if !ok {
+			// The session's done channel is closed, so the session is over,
+			// and we need a new one
+			session, err := concurrency.NewSession(c.etcd, concurrency.WithTTL(etcd3LockTimeoutInSeconds))
+			if err != nil {
+				return nil, err
+			}
+			c.etcdSession = session
+			c.etcdMu = concurrency.NewMutex(session, c.prefix)
+		}
+	default:
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {
 		<-stopCh