From fe7f0cefa91f86a8b9f3f5750709f8de5658e46b Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Tue, 6 Jun 2023 09:51:24 +0000 Subject: [PATCH 1/7] backport of commit 2f94024b356f774d0032ad069babb1a3fce6cfbd --- api/operator_raft.go | 17 +++++ api/operator_raft_test.go | 19 +++++ .../raft/listpeers/operator_raft_list.go | 71 +++++++++++++++++-- .../raft/listpeers/operator_raft_list_test.go | 27 +++++++ 4 files changed, 129 insertions(+), 5 deletions(-) diff --git a/api/operator_raft.go b/api/operator_raft.go index 393d6fb3c5f7..6ee75c4d8bb0 100644 --- a/api/operator_raft.go +++ b/api/operator_raft.go @@ -122,3 +122,20 @@ func (op *Operator) RaftRemovePeerByID(id string, q *WriteOptions) error { } return nil } + +// GetAutoPilotHealth is used to query the autopilot health. +func (op *Operator) GetAutoPilotHealth(q *QueryOptions) (*OperatorHealthReply, error) { + r := op.c.newRequest("GET", "/v1/operator/autopilot/health") + r.setQueryOptions(q) + _, resp, err := op.c.doRequest(r) + if err != nil { + return nil, err + } + defer closeResponseBody(resp) + + var out OperatorHealthReply + if err := decodeBody(resp, &out); err != nil { + return nil, err + } + return &out, nil +} diff --git a/api/operator_raft_test.go b/api/operator_raft_test.go index 6e3b7fc0e310..57b50b745269 100644 --- a/api/operator_raft_test.go +++ b/api/operator_raft_test.go @@ -57,3 +57,22 @@ func TestAPI_OperatorRaftLeaderTransfer(t *testing.T) { t.Fatalf("err:%v", transfer) } } + +func TestAPI_GetAutoPilotHealth(t *testing.T) { + t.Parallel() + c, s := makeClient(t) + defer s.Stop() + + operator := c.Operator() + out, err := operator.GetAutoPilotHealth(nil) + if err != nil { + t.Fatalf("err: %v", err) + } + + if len(out.Servers) != 1 || + !out.Servers[0].Leader || + !out.Servers[0].Voter || + out.Servers[0].LastIndex <= 0 { + t.Fatalf("bad: %v", out) + } +} diff --git a/command/operator/raft/listpeers/operator_raft_list.go b/command/operator/raft/listpeers/operator_raft_list.go index 47bd161fed48..9cc4db67250a 100644 --- a/command/operator/raft/listpeers/operator_raft_list.go +++ b/command/operator/raft/listpeers/operator_raft_list.go @@ -24,10 +24,16 @@ type cmd struct { flags *flag.FlagSet http *flags.HTTPFlags help string + + // flags + detailed bool } func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) + c.flags.BoolVar(&c.detailed, "detailed", false, + "Outputs additional information 'commit_index' which is "+ + "the index of the server's last committed Raft log entry.") c.http = &flags.HTTPFlags{} flags.Merge(c.flags, c.http.ClientFlags()) flags.Merge(c.flags, c.http.ServerFlags()) @@ -51,13 +57,22 @@ func (c *cmd) Run(args []string) int { } // Fetch the current configuration. - result, err := raftListPeers(client, c.http.Stale()) - if err != nil { - c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) - return 1 + if c.detailed { + result, err := raftListPeersDetailed(client, c.http.Stale()) + if err != nil { + c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) + return 1 + } + c.UI.Output(result) + } else { + result, err := raftListPeers(client, c.http.Stale()) + if err != nil { + c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) + return 1 + } + c.UI.Output(result) } - c.UI.Output(result) return 0 } @@ -89,6 +104,52 @@ func raftListPeers(client *api.Client, stale bool) (string, error) { return columnize.Format(result, &columnize.Config{Delim: string([]byte{0x1f})}), nil } +func raftListPeersDetailed(client *api.Client, stale bool) (string, error) { + q := &api.QueryOptions{ + AllowStale: stale, + } + reply, err := client.Operator().RaftGetConfiguration(q) + if err != nil { + return "", fmt.Errorf("Failed to retrieve raft configuration: %v", err) + } + + autoPilotReply, err := client.Operator().GetAutoPilotHealth(q) + if err != nil { + return "", fmt.Errorf("Failed to retrieve autopilot health: %v", err) + } + + serverHealthDataMap := make(map[string]api.ServerHealth) + + for _, serverHealthData := range autoPilotReply.Servers { + serverHealthDataMap[serverHealthData.ID] = serverHealthData + } + + // Format it as a nice table. + result := []string{"Node\x1fID\x1fAddress\x1fState\x1fVoter\x1fRaftProtocol\x1fCommitIndex"} + for _, s := range reply.Servers { + raftProtocol := s.ProtocolVersion + + if raftProtocol == "" { + raftProtocol = "<=1" + } + state := "follower" + if s.Leader { + state = "leader" + } + + serverHealthData, ok := serverHealthDataMap[s.ID] + if ok { + result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", + s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, serverHealthData.LastIndex)) + } else { + result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", + s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, "")) + } + } + + return columnize.Format(result, &columnize.Config{Delim: string([]byte{0x1f})}), nil +} + func (c *cmd) Synopsis() string { return synopsis } diff --git a/command/operator/raft/listpeers/operator_raft_list_test.go b/command/operator/raft/listpeers/operator_raft_list_test.go index 15bd1bfbe34f..a610c099955f 100644 --- a/command/operator/raft/listpeers/operator_raft_list_test.go +++ b/command/operator/raft/listpeers/operator_raft_list_test.go @@ -46,6 +46,33 @@ func TestOperatorRaftListPeersCommand(t *testing.T) { } } +func TestOperatorRaftListPeersCommandDetailed(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + a := agent.NewTestAgent(t, ``) + defer a.Shutdown() + + expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3", + a.Config.NodeName, a.Config.NodeID, a.Config.ServerPort) + + // Test the list-peers subcommand directly + ui := cli.NewMockUi() + c := New(ui) + args := []string{"-http-addr=" + a.HTTPAddr(), "-detailed"} + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + output := strings.TrimSpace(ui.OutputWriter.String()) + if !strings.Contains(output, expected) { + t.Fatalf("bad: %q, %q", output, expected) + } +} + func TestOperatorRaftListPeersCommand_verticalBar(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") From f7b3d0d1dff75b9c11bb54af30bd5172465fadf1 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Tue, 6 Jun 2023 09:56:50 +0000 Subject: [PATCH 2/7] backport of commit 7626d0992dd59f424d893dd462bbde3a9cb0646c --- command/operator/raft/listpeers/operator_raft_list_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/operator/raft/listpeers/operator_raft_list_test.go b/command/operator/raft/listpeers/operator_raft_list_test.go index a610c099955f..f60cad21e279 100644 --- a/command/operator/raft/listpeers/operator_raft_list_test.go +++ b/command/operator/raft/listpeers/operator_raft_list_test.go @@ -55,7 +55,7 @@ func TestOperatorRaftListPeersCommandDetailed(t *testing.T) { a := agent.NewTestAgent(t, ``) defer a.Shutdown() - expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3", + expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3 1", a.Config.NodeName, a.Config.NodeID, a.Config.ServerPort) // Test the list-peers subcommand directly From 54bf16c92281ef3c0623f13d6f8d896736da94aa Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Tue, 6 Jun 2023 11:01:54 +0000 Subject: [PATCH 3/7] backport of commit 79aabc958aed60b2bd97c836529d53a627262124 --- website/content/commands/operator/raft.mdx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/content/commands/operator/raft.mdx b/website/content/commands/operator/raft.mdx index 857a9ee1ac12..7c948c1e14e3 100644 --- a/website/content/commands/operator/raft.mdx +++ b/website/content/commands/operator/raft.mdx @@ -73,6 +73,10 @@ configuration. we recommend setting this option to `true. Default is `false`. +- `-detailed` - Outputs additional information 'commit_index' which is +the index of the server's last committed Raft log entry. +Default is `false`. + ## remove-peer Corresponding HTTP API Endpoint: [\[DELETE\] /v1/operator/raft/peer](/consul/api-docs/operator/raft#delete-raft-peer) From e9b8c9399bf14759f054d3ce7aa80e54f0946b39 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Tue, 6 Jun 2023 11:14:01 +0000 Subject: [PATCH 4/7] backport of commit 44eee4168b5a0d405636dbcb5b14ee7959334550 --- .changelog/17582.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/17582.txt diff --git a/.changelog/17582.txt b/.changelog/17582.txt new file mode 100644 index 000000000000..3beae9d726a0 --- /dev/null +++ b/.changelog/17582.txt @@ -0,0 +1,3 @@ +```release-note:feature +improved consul operator raft list-peers command added -detailed flag which will print CommitIndex in the table in response +``` From fa1bb6f650ba7467d2dec5e35c5bb4644586e511 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut <134911583+absolutelightning@users.noreply.github.com> Date: Thu, 15 Jun 2023 20:11:04 +0530 Subject: [PATCH 5/7] Updated docs added explanation. (#17751) * init * fix tests * added -detailed in docs * added change log * fix doc * checking for entry in map * fix tests * removed detailed flag * removed detailed flag * revert unwanted changes * removed unwanted changes * updated change log * pr review comment changes * pr comment changes single API instead of two * fix change log * fix tests * fix tests * fix test operator raft endpoint test * Update .changelog/17582.txt Co-authored-by: Semir Patel * nits * updated docs * explanation added --------- Co-authored-by: Semir Patel --- website/content/commands/operator/raft.mdx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/content/commands/operator/raft.mdx b/website/content/commands/operator/raft.mdx index 7c948c1e14e3..511f04bdfc98 100644 --- a/website/content/commands/operator/raft.mdx +++ b/website/content/commands/operator/raft.mdx @@ -66,6 +66,10 @@ Raft configuration. `Voter` is "true" or "false", indicating if the server has a vote in the Raft configuration. +`Commit Index` is the last log index the server has a record of in its Raft log. + +`Trails Leader By` is the difference in commit index the server has from the leader. + #### Command Options - `-stale` - Enables non-leader servers to provide cluster state information. From 993d4c407a1c079f79698520bf90c7ceb5f60c08 Mon Sep 17 00:00:00 2001 From: absolutelightning Date: Wed, 21 Jun 2023 11:40:32 +0530 Subject: [PATCH 6/7] fix diff --- api/operator_raft.go | 17 ----- api/operator_raft_test.go | 19 ------ .../raft/listpeers/operator_raft_list.go | 68 ++----------------- .../raft/listpeers/operator_raft_list_test.go | 27 -------- website/content/commands/operator/raft.mdx | 4 -- 5 files changed, 5 insertions(+), 130 deletions(-) diff --git a/api/operator_raft.go b/api/operator_raft.go index df579d959a44..d72c00c97b93 100644 --- a/api/operator_raft.go +++ b/api/operator_raft.go @@ -125,20 +125,3 @@ func (op *Operator) RaftRemovePeerByID(id string, q *WriteOptions) error { } return nil } - -// GetAutoPilotHealth is used to query the autopilot health. -func (op *Operator) GetAutoPilotHealth(q *QueryOptions) (*OperatorHealthReply, error) { - r := op.c.newRequest("GET", "/v1/operator/autopilot/health") - r.setQueryOptions(q) - _, resp, err := op.c.doRequest(r) - if err != nil { - return nil, err - } - defer closeResponseBody(resp) - - var out OperatorHealthReply - if err := decodeBody(resp, &out); err != nil { - return nil, err - } - return &out, nil -} diff --git a/api/operator_raft_test.go b/api/operator_raft_test.go index 57b50b745269..6e3b7fc0e310 100644 --- a/api/operator_raft_test.go +++ b/api/operator_raft_test.go @@ -57,22 +57,3 @@ func TestAPI_OperatorRaftLeaderTransfer(t *testing.T) { t.Fatalf("err:%v", transfer) } } - -func TestAPI_GetAutoPilotHealth(t *testing.T) { - t.Parallel() - c, s := makeClient(t) - defer s.Stop() - - operator := c.Operator() - out, err := operator.GetAutoPilotHealth(nil) - if err != nil { - t.Fatalf("err: %v", err) - } - - if len(out.Servers) != 1 || - !out.Servers[0].Leader || - !out.Servers[0].Voter || - out.Servers[0].LastIndex <= 0 { - t.Fatalf("bad: %v", out) - } -} diff --git a/command/operator/raft/listpeers/operator_raft_list.go b/command/operator/raft/listpeers/operator_raft_list.go index b44c858b7373..356897c85eb7 100644 --- a/command/operator/raft/listpeers/operator_raft_list.go +++ b/command/operator/raft/listpeers/operator_raft_list.go @@ -24,9 +24,6 @@ type cmd struct { flags *flag.FlagSet http *flags.HTTPFlags help string - - // flags - detailed bool } func (c *cmd) init() { @@ -57,22 +54,13 @@ func (c *cmd) Run(args []string) int { } // Fetch the current configuration. - if c.detailed { - result, err := raftListPeersDetailed(client, c.http.Stale()) - if err != nil { - c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) - return 1 - } - c.UI.Output(result) - } else { - result, err := raftListPeers(client, c.http.Stale()) - if err != nil { - c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) - return 1 - } - c.UI.Output(result) + result, err := raftListPeers(client, c.http.Stale()) + if err != nil { + c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) + return 1 } + c.UI.Output(result) return 0 } @@ -132,52 +120,6 @@ func raftListPeers(client *api.Client, stale bool) (string, error) { return columnize.Format(result, &columnize.Config{Delim: string([]byte{0x1f})}), nil } -func raftListPeersDetailed(client *api.Client, stale bool) (string, error) { - q := &api.QueryOptions{ - AllowStale: stale, - } - reply, err := client.Operator().RaftGetConfiguration(q) - if err != nil { - return "", fmt.Errorf("Failed to retrieve raft configuration: %v", err) - } - - autoPilotReply, err := client.Operator().GetAutoPilotHealth(q) - if err != nil { - return "", fmt.Errorf("Failed to retrieve autopilot health: %v", err) - } - - serverHealthDataMap := make(map[string]api.ServerHealth) - - for _, serverHealthData := range autoPilotReply.Servers { - serverHealthDataMap[serverHealthData.ID] = serverHealthData - } - - // Format it as a nice table. - result := []string{"Node\x1fID\x1fAddress\x1fState\x1fVoter\x1fRaftProtocol\x1fCommitIndex"} - for _, s := range reply.Servers { - raftProtocol := s.ProtocolVersion - - if raftProtocol == "" { - raftProtocol = "<=1" - } - state := "follower" - if s.Leader { - state = "leader" - } - - serverHealthData, ok := serverHealthDataMap[s.ID] - if ok { - result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", - s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, serverHealthData.LastIndex)) - } else { - result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", - s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, "")) - } - } - - return columnize.Format(result, &columnize.Config{Delim: string([]byte{0x1f})}), nil -} - func (c *cmd) Synopsis() string { return synopsis } diff --git a/command/operator/raft/listpeers/operator_raft_list_test.go b/command/operator/raft/listpeers/operator_raft_list_test.go index ddba48266cf2..0694e0dd1046 100644 --- a/command/operator/raft/listpeers/operator_raft_list_test.go +++ b/command/operator/raft/listpeers/operator_raft_list_test.go @@ -46,33 +46,6 @@ func TestOperatorRaftListPeersCommand(t *testing.T) { } } -func TestOperatorRaftListPeersCommandDetailed(t *testing.T) { - if testing.Short() { - t.Skip("too slow for testing.Short") - } - - t.Parallel() - a := agent.NewTestAgent(t, ``) - defer a.Shutdown() - - expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3 1", - a.Config.NodeName, a.Config.NodeID, a.Config.ServerPort) - - // Test the list-peers subcommand directly - ui := cli.NewMockUi() - c := New(ui) - args := []string{"-http-addr=" + a.HTTPAddr(), "-detailed"} - - code := c.Run(args) - if code != 0 { - t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) - } - output := strings.TrimSpace(ui.OutputWriter.String()) - if !strings.Contains(output, expected) { - t.Fatalf("bad: %q, %q", output, expected) - } -} - func TestOperatorRaftListPeersCommand_verticalBar(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") diff --git a/website/content/commands/operator/raft.mdx b/website/content/commands/operator/raft.mdx index 3b695c5e456f..b89c0385ac88 100644 --- a/website/content/commands/operator/raft.mdx +++ b/website/content/commands/operator/raft.mdx @@ -77,10 +77,6 @@ configuration. we recommend setting this option to `true`. Default is `false`. -- `-detailed` - Outputs additional information 'commit_index' which is -the index of the server's last committed Raft log entry. -Default is `false`. - ## remove-peer Corresponding HTTP API Endpoint: [\[DELETE\] /v1/operator/raft/peer](/consul/api-docs/operator/raft#delete-raft-peer) From 1104cc32388eb64380e24a73953ce77440ce04bc Mon Sep 17 00:00:00 2001 From: absolutelightning Date: Wed, 21 Jun 2023 11:40:56 +0530 Subject: [PATCH 7/7] fix diff --- command/operator/raft/listpeers/operator_raft_list.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/command/operator/raft/listpeers/operator_raft_list.go b/command/operator/raft/listpeers/operator_raft_list.go index 356897c85eb7..29643a87cf33 100644 --- a/command/operator/raft/listpeers/operator_raft_list.go +++ b/command/operator/raft/listpeers/operator_raft_list.go @@ -28,9 +28,6 @@ type cmd struct { func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) - c.flags.BoolVar(&c.detailed, "detailed", false, - "Outputs additional information 'commit_index' which is "+ - "the index of the server's last committed Raft log entry.") c.http = &flags.HTTPFlags{} flags.Merge(c.flags, c.http.ClientFlags()) flags.Merge(c.flags, c.http.ServerFlags())