core: Record the time a node became active

api/sys_leader.go

@@ -1,6 +1,9 @@
 package api
 
-import "context"
+import (
+	"context"
+	"time"
+)
 
 func (c *Sys) Leader() (*LeaderResponse, error) {
 	r := c.c.NewRequest("GET", "/v1/sys/leader")
@@ -19,13 +22,14 @@ func (c *Sys) Leader() (*LeaderResponse, error) {
 }
 
 type LeaderResponse struct {
-	HAEnabled                bool   `json:"ha_enabled"`
-	IsSelf                   bool   `json:"is_self"`
-	LeaderAddress            string `json:"leader_address"`
-	LeaderClusterAddress     string `json:"leader_cluster_address"`
-	PerfStandby              bool   `json:"performance_standby"`
-	PerfStandbyLastRemoteWAL uint64 `json:"performance_standby_last_remote_wal"`
-	LastWAL                  uint64 `json:"last_wal"`
-	RaftCommittedIndex       uint64 `json:"raft_committed_index,omitempty"`
-	RaftAppliedIndex         uint64 `json:"raft_applied_index,omitempty"`
+	HAEnabled                bool      `json:"ha_enabled"`
+	IsSelf                   bool      `json:"is_self"`
+	ActiveTime               time.Time `json:"active_time"`
+	LeaderAddress            string    `json:"leader_address"`
+	LeaderClusterAddress     string    `json:"leader_cluster_address"`
+	PerfStandby              bool      `json:"performance_standby"`
+	PerfStandbyLastRemoteWAL uint64    `json:"performance_standby_last_remote_wal"`
+	LastWAL                  uint64    `json:"last_wal"`
+	RaftCommittedIndex       uint64    `json:"raft_committed_index,omitempty"`
+	RaftAppliedIndex         uint64    `json:"raft_applied_index,omitempty"`
 }

changelog/10489.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core: Added active since timestamp to the status output of active nodes.
+```

command/format.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/ghodss/yaml"
 	"github.com/hashicorp/vault/api"
@@ -191,6 +192,9 @@ func (t TableFormatter) OutputSealStatusStruct(ui cli.Ui, secret *api.Secret, da
 			}
 			out = append(out, fmt.Sprintf("HA Mode | %s", mode))
 
+			if status.IsSelf && !status.ActiveTime.IsZero() {
+				out = append(out, fmt.Sprintf("Active Since | %s", status.ActiveTime.Format(time.RFC3339Nano)))
+			}
 			// This is down here just to keep ordering consistent
 			if showLeaderAddr {
 				out = append(out, fmt.Sprintf("Active Node Address | %s", status.LeaderAddress))
@@ -405,6 +409,7 @@ func OutputSealStatus(ui cli.Ui, client *api.Client, status *api.SealStatusRespo
 	// copy leaderStatus fields into sealStatusOutput for display later
 	sealStatusOutput.HAEnabled = leaderStatus.HAEnabled
 	sealStatusOutput.IsSelf = leaderStatus.IsSelf
+	sealStatusOutput.ActiveTime = leaderStatus.ActiveTime
 	sealStatusOutput.LeaderAddress = leaderStatus.LeaderAddress
 	sealStatusOutput.LeaderClusterAddress = leaderStatus.LeaderClusterAddress
 	sealStatusOutput.PerfStandby = leaderStatus.PerfStandby
@@ -431,13 +436,14 @@ func looksLikeDuration(k string) bool {
 // Currently we are adding the fields from api.LeaderResponse
 type SealStatusOutput struct {
 	api.SealStatusResponse
-	HAEnabled                bool   `json:"ha_enabled"`
-	IsSelf                   bool   `json:"is_self,omitempty""`
-	LeaderAddress            string `json:"leader_address,omitempty"`
-	LeaderClusterAddress     string `json:"leader_cluster_address,omitempty"`
-	PerfStandby              bool   `json:"performance_standby,omitempty"`
-	PerfStandbyLastRemoteWAL uint64 `json:"performance_standby_last_remote_wal,omitempty"`
-	LastWAL                  uint64 `json:"last_wal,omitempty"`
-	RaftCommittedIndex       uint64 `json:"raft_committed_index,omitempty"`
-	RaftAppliedIndex         uint64 `json:"raft_applied_index,omitempty"`
+	HAEnabled                bool      `json:"ha_enabled"`
+	IsSelf                   bool      `json:"is_self,omitempty""`
+	ActiveTime               time.Time `json:"active_time,omitempty""`
+	LeaderAddress            string    `json:"leader_address,omitempty"`
+	LeaderClusterAddress     string    `json:"leader_cluster_address,omitempty"`
+	PerfStandby              bool      `json:"performance_standby,omitempty"`
+	PerfStandbyLastRemoteWAL uint64    `json:"performance_standby_last_remote_wal,omitempty"`
+	LastWAL                  uint64    `json:"last_wal,omitempty"`
+	RaftCommittedIndex       uint64    `json:"raft_committed_index,omitempty"`
+	RaftAppliedIndex         uint64    `json:"raft_applied_index,omitempty"`
 }

command/format_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/ghodss/yaml"
 	"github.com/hashicorp/vault/api"
@@ -178,6 +179,7 @@ func getMockStatusData(emptyFields bool) SealStatusOutput {
 			sealStatusResponseMock,
 			true,                     // HAEnabled
 			true,                     // IsSelf
+			time.Time{}.UTC(),        // ActiveTime
 			"leader address",         // LeaderAddress
 			"leader cluster address", // LeaderClusterAddress
 			true,                     // PerfStandby
@@ -206,15 +208,16 @@ func getMockStatusData(emptyFields bool) SealStatusOutput {
 		// must initialize this struct without explicit field names due to embedding
 		status = SealStatusOutput{
 			sealStatusResponseMock,
-			false, // HAEnabled
-			false, // IsSelf
-			"",    // LeaderAddress
-			"",    // LeaderClusterAddress
-			false, // PerfStandby
-			0,     // PerfStandbyLastRemoteWAL
-			0,     // LastWAL
-			0,     // RaftCommittedIndex
-			0,     // RaftAppliedIndex
+			false,             // HAEnabled
+			false,             // IsSelf
+			time.Time{}.UTC(), // ActiveTime
+			"",                // LeaderAddress
+			"",                // LeaderClusterAddress
+			false,             // PerfStandby
+			0,                 // PerfStandbyLastRemoteWAL
+			0,                 // LastWAL
+			0,                 // RaftCommittedIndex
+			0,                 // RaftAppliedIndex
 		}
 	}
 	return status

http/sys_leader.go

@@ -2,6 +2,7 @@ package http
 
 import (
 	"net/http"
+	"time"
 
 	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/vault/vault"
@@ -36,6 +37,9 @@ func handleSysLeaderGet(core *vault.Core, w http.ResponseWriter, r *http.Request
 		LeaderClusterAddress: clusterAddr,
 		PerfStandby:          core.PerfStandby(),
 	}
+	if isLeader {
+		resp.ActiveTime = core.ActiveTime()
+	}
 	if resp.PerfStandby {
 		resp.PerfStandbyLastRemoteWAL = vault.LastRemoteWAL(core)
 	} else if isLeader || !haEnabled {
@@ -48,13 +52,14 @@ func handleSysLeaderGet(core *vault.Core, w http.ResponseWriter, r *http.Request
 }
 
 type LeaderResponse struct {
-	HAEnabled                bool   `json:"ha_enabled"`
-	IsSelf                   bool   `json:"is_self"`
-	LeaderAddress            string `json:"leader_address"`
-	LeaderClusterAddress     string `json:"leader_cluster_address"`
-	PerfStandby              bool   `json:"performance_standby"`
-	PerfStandbyLastRemoteWAL uint64 `json:"performance_standby_last_remote_wal"`
-	LastWAL                  uint64 `json:"last_wal,omitempty"`
+	HAEnabled                bool      `json:"ha_enabled"`
+	IsSelf                   bool      `json:"is_self"`
+	ActiveTime               time.Time `json:"active_time,omitempty"`
+	LeaderAddress            string    `json:"leader_address"`
+	LeaderClusterAddress     string    `json:"leader_cluster_address"`
+	PerfStandby              bool      `json:"performance_standby"`
+	PerfStandbyLastRemoteWAL uint64    `json:"performance_standby_last_remote_wal"`
+	LastWAL                  uint64    `json:"last_wal,omitempty"`
 
 	// Raft Indexes for this node
 	RaftCommittedIndex uint64 `json:"raft_committed_index,omitempty"`

http/sys_leader_test.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/vault/vault"
 )
@@ -27,10 +28,11 @@ func TestSysLeader_get(t *testing.T) {
 		"leader_cluster_address":              "",
 		"performance_standby":                 false,
 		"performance_standby_last_remote_wal": json.Number("0"),
+		"active_time":                         time.Time{}.UTC().Format(time.RFC3339),
 	}
 	testResponseStatus(t, resp, 200)
 	testResponseBody(t, resp, &actual)
 	if !reflect.DeepEqual(actual, expected) {
-		t.Fatalf("bad: %#v", actual)
+		t.Fatalf("bad: %#v \n%#v", actual, expected)
 	}
 }

vault/core.go

@@ -535,6 +535,10 @@ type Core struct {
 
 	activityLogConfig ActivityLogCoreConfig
 
+	// activeTime is set on active nodes indicating the time at which this node
+	// became active.
+	activeTime time.Time
+
 	// KeyRotateGracePeriod is how long we allow an upgrade path
 	// for standby instances before we delete the upgrade keys
 	keyRotateGracePeriod *int64
@@ -1856,6 +1860,10 @@ func (s standardUnsealStrategy) unseal(ctx context.Context, logger log.Logger, c
 	c.clearForwardingClients()
 	c.requestForwardingConnectionLock.Unlock()
 
+	// Mark the active time. We do this first so it can be correlated to the logs
+	// for the active startup.
+	c.activeTime = time.Now().UTC()
+
 	if err := postUnsealPhysical(c); err != nil {
 		return err
 	}
@@ -2047,6 +2055,7 @@ func (c *Core) preSeal() error {
 
 	// Clear any pending funcs
 	c.postUnsealFuncs = nil
+	c.activeTime = time.Time{}
 
 	// Clear any rekey progress
 	c.barrierRekeyConfig = nil

vault/ha.go

@@ -70,6 +70,13 @@ func (c *Core) PerfStandby() bool {
 	return perfStandby
 }
 
+func (c *Core) ActiveTime() time.Time {
+	c.stateLock.RLock()
+	activeTime := c.activeTime
+	c.stateLock.RUnlock()
+	return activeTime
+}
+
 // StandbyStates is meant as a way to avoid some extra locking on the very
 // common sys/health check.
 func (c *Core) StandbyStates() (standby, perfStandby bool) {