Skip to content

Commit

Permalink
Log envoy config for mesh gateways on test failures (hashicorp#718)
Browse files Browse the repository at this point in the history
If the mesh gateway tests fail, the acceptance tests will now write two
files <PODNAME>-envoy-configdump.json and <PODNAME>envoy-clusters.json
which will have information from the envoy admin /config_dump endpoint
and /clusters endpoint.

Currently we write the outputs of `kubectl logs POD` and `kubectl describe
POD` for the acceptance tests, but while debugging EKS acceptance test
failures, it was hard to know from our existing logs whether the mesh
gateway itself is unreachable or if the consul servers behind it are
unhealthy. We'd be able to tell this from looking at the envoy config
dump and cluster config to see what envoy's view of the consul servers
are.
  • Loading branch information
ndhanushkodi authored Dec 2, 2020
1 parent e00c696 commit 146833a
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 5 deletions.
34 changes: 30 additions & 4 deletions test/acceptance/framework/k8s/debug.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,25 +31,51 @@ func WritePodsDebugInfoIfFailed(t *testing.T, kubectlOptions *k8s.KubectlOptions
testDebugDirectory := filepath.Join(debugDirectory, t.Name(), contextName)
require.NoError(t, os.MkdirAll(testDebugDirectory, 0755))

logger.Logf(t, "dumping logs and pod info for %s to %s", labelSelector, testDebugDirectory)
logger.Logf(t, "dumping logs, pod info, and envoy config for %s to %s", labelSelector, testDebugDirectory)

// Describe and get logs for any pods.
pods, err := client.CoreV1().Pods(kubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector})
require.NoError(t, err)

for _, pod := range pods.Items {
// Get logs for each pod, passing the discard logger to make sure secrets aren't printed to test logs.
logs, err := RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "logs", "--all-containers=true", pod.Name)

// Write logs or err to file name <pod.Name>.log
logFilename := filepath.Join(testDebugDirectory, fmt.Sprintf("%s.log", pod.Name))
if err != nil {
logs = fmt.Sprintf("Error getting logs: %s: %s", err, logs)
}

// Write logs or err to file name <pod.Name>.log
logFilename := filepath.Join(testDebugDirectory, fmt.Sprintf("%s.log", pod.Name))
require.NoError(t, ioutil.WriteFile(logFilename, []byte(logs), 0600))

// Describe pod and write it to a file.
writeResourceInfoToFile(t, pod.Name, "pod", testDebugDirectory, kubectlOptions)
}

// Get envoy configuration from the mesh gateways, if there are any.
meshGatewayPods, err := client.CoreV1().Pods(kubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "component=mesh-gateway"})
require.NoError(t, err)

for _, mpod := range meshGatewayPods.Items {
// Get configdump from mesh gateway, passing the discard logger since we only need these logs written to the file (below).
configDump, err := RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "exec", mpod.Name, "-c", "lifecycle-sidecar", "--", "curl", "-s", "localhost:19000/config_dump?format=json")
if err != nil {
configDump = fmt.Sprintf("Error getting config_dump: %s: %s", err, configDump)
}
// Get cluster config from mesh gateway, passing the discard logger since we only need these logs written to the file (below).
clusters, err := RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "exec", mpod.Name, "-c", "lifecycle-sidecar", "--", "curl", "-s", "localhost:19000/clusters?format=json")
if err != nil {
clusters = fmt.Sprintf("Error getting clusters: %s: %s", err, clusters)
}

// Write config/clusters or err to file name <pod.Name>-envoy-[configdump/clusters].json
configDumpFilename := filepath.Join(testDebugDirectory, fmt.Sprintf("%s-envoy-configdump.json", mpod.Name))
clustersFilename := filepath.Join(testDebugDirectory, fmt.Sprintf("%s-envoy-clusters.json", mpod.Name))
require.NoError(t, ioutil.WriteFile(configDumpFilename, []byte(configDump), 0600))
require.NoError(t, ioutil.WriteFile(clustersFilename, []byte(clusters), 0600))

}

// Describe any stateful sets.
statefulSets, err := client.AppsV1().StatefulSets(kubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector})
for _, statefulSet := range statefulSets.Items {
Expand Down
1 change: 0 additions & 1 deletion test/acceptance/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,6 @@ github.com/gruntwork-io/gruntwork-cli v0.7.0 h1:YgSAmfCj9c61H+zuvHwKfYUwlMhu5arn
github.com/gruntwork-io/gruntwork-cli v0.7.0/go.mod h1:jp6Z7NcLF2avpY8v71fBx6hds9eOFPELSuD/VPv7w00=
github.com/gruntwork-io/terratest v0.30.15 h1:/OAnHKEERSsy03wYi8dyEwnpYy2XV8wpzwq0c4BHvoc=
github.com/gruntwork-io/terratest v0.30.15/go.mod h1:vl/YEB2AEqVZOv9zg0nlBX+fgfqcqNcpqNfjvhRFLXo=
github.com/hashicorp/consul-helm v0.25.0 h1:aXYK733Z7lIWdiVNczND+bBwrheGNtp6wvLe7N/8TgU=
github.com/hashicorp/consul/api v1.4.1-0.20201015173526-812fe06d6c64 h1:8/zXC/mxIXKRqcRACIu/G016QKE/XWWBXsuNnJO3O9s=
github.com/hashicorp/consul/api v1.4.1-0.20201015173526-812fe06d6c64/go.mod h1:duA0cMzNQkvdlj8Ayz6LmyaMUta7iVW8qY2w2bic/Iw=
github.com/hashicorp/consul/sdk v0.6.0 h1:FfhMEkwvQl57CildXJyGHnwGGM4HMODGyfjGwNM1Vdw=
Expand Down

0 comments on commit 146833a

Please sign in to comment.