From 24fdf4d887ea48cd31ca1198469823a9ac1a674c Mon Sep 17 00:00:00 2001 From: Wanlin Du Date: Mon, 25 Apr 2022 15:06:12 -0700 Subject: [PATCH] Save logs from all containers and add url link properties. This commit updates the logics to save logs from all containers to the original directory. The name of the logs follows the format: pod-name-container-name.log. This commit also add log url to the properties if the log url prefix is provided. --- tools/cmd/runner/main.go | 12 +++++- tools/runner/logsaver.go | 79 +++++++++++++++++++++++++++------------- tools/runner/runner.go | 9 ++++- 3 files changed, 72 insertions(+), 28 deletions(-) diff --git a/tools/cmd/runner/main.go b/tools/cmd/runner/main.go index 851a8802..e2d090e2 100644 --- a/tools/cmd/runner/main.go +++ b/tools/cmd/runner/main.go @@ -34,6 +34,7 @@ func main() { var c runner.ConcurrencyLevels var a string var p time.Duration + var u string var retries uint var deleteSuccessfulTests bool @@ -41,6 +42,7 @@ func main() { flag.StringVar(&o, "o", "", "name of the output file for xunit xml report") flag.Var(&c, "c", "concurrency level, in the form [:]") flag.StringVar(&a, "annotation-key", "pool", "annotation key to parse for queue assignment") + flag.StringVar(&u, "log-url-prefix", "", "prefix for log urls") flag.DurationVar(&p, "polling-interval", 20*time.Second, "polling interval for load test status") flag.UintVar(&retries, "polling-retries", 2, "Maximum retries in case of communication failure") flag.BoolVar(&deleteSuccessfulTests, "delete-successful-tests", false, "Delete tests immediately in case of successful termination") @@ -69,14 +71,22 @@ func main() { outputDirMap[qName] = outputDir } + logURLPrefix := "" + if u != "" { + logURLPrefix = "http://cnsviewer2/placer/prod/home/kokoro-dedicated/build_artifacts/" + u + "/github/grpc/" + } + log.Printf("Annotation key for queue assignment: %s", a) log.Printf("Polling interval: %v", p) log.Printf("Polling retries: %d", retries) log.Printf("Test counts per queue: %v", runner.CountConfigs(configQueueMap)) log.Printf("Queue concurrency levels: %v", c) log.Printf("Output directories: %v", outputDirMap) + if logURLPrefix != "" { + log.Printf("Prefix for url for saved logs: %s", logURLPrefix) + } - r := runner.NewRunner(runner.NewLoadTestGetter(), runner.AfterIntervalFunction(p), retries, deleteSuccessfulTests, runner.NewLogSaver(runner.NewPodsGetter())) + r := runner.NewRunner(runner.NewLoadTestGetter(), runner.AfterIntervalFunction(p), retries, deleteSuccessfulTests, runner.NewLogSaver(runner.NewPodsGetter(), logURLPrefix)) logPrefixFmt := runner.LogPrefixFmt(configQueueMap) diff --git a/tools/runner/logsaver.go b/tools/runner/logsaver.go index 06884b23..e2c4fd53 100644 --- a/tools/runner/logsaver.go +++ b/tools/runner/logsaver.go @@ -11,7 +11,6 @@ import ( "strings" grpcv1 "github.com/grpc/test-infra/api/v1" - "github.com/grpc/test-infra/config" "github.com/grpc/test-infra/status" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -20,17 +19,20 @@ import ( // LogSaver provides functionality to save pod logs to files. type LogSaver struct { - podsGetter corev1types.PodsGetter + podsGetter corev1types.PodsGetter + logURLPrefix string } // NewLogSaver creates a new LogSaver object. -func NewLogSaver(podsGetter corev1types.PodsGetter) *LogSaver { +func NewLogSaver(podsGetter corev1types.PodsGetter, logURLPrefix string) *LogSaver { return &LogSaver{ - podsGetter: podsGetter, + podsGetter: podsGetter, + logURLPrefix: logURLPrefix, } } -// SavePodLogs saves pod logs to files with same name as pod. +// SavePodLogs saves container logs to files with name in format +// pod-name-container-name.log. // This function returns a map where pods are keys and values are the filepath // of the saved log. func (ls *LogSaver) SavePodLogs(ctx context.Context, loadTest *grpcv1.LoadTest, podLogDir string) (*SavedLogs, error) { @@ -50,16 +52,21 @@ func (ls *LogSaver) SavePodLogs(ctx context.Context, loadTest *grpcv1.LoadTest, // Write logs to files for _, pod := range pods { - logFilePath := filepath.Join(podLogDir, pod.Name+".log") - buffer, err := ls.getPodLogBuffer(ctx, pod) + containerNametoLogPathMap := make(map[string]string) + containerNamesToLogMap, err := ls.getPodLogBuffers(ctx, pod) if err != nil { return savedLogs, fmt.Errorf("could not get log from pod: %s", err) } - err = ls.writeBufferToFile(buffer, logFilePath) - if err != nil { - return savedLogs, fmt.Errorf("could not write pod log buffer to file: %s", err) + for containerName, buffer := range containerNamesToLogMap { + logFilePath := filepath.Join(podLogDir, fmt.Sprintf("%s-%s.log", pod.Name, containerName)) + err = ls.writeBufferToFile(buffer, logFilePath) + if err != nil { + return savedLogs, fmt.Errorf("could not write %s container in %s pod log buffer to file: %s", containerName, pod.Name, err) + } + containerNametoLogPathMap[containerName] = logFilePath } - savedLogs.podToPathMap[pod] = logFilePath + + savedLogs.podToPathMap[pod] = containerNametoLogPathMap } return savedLogs, nil } @@ -79,18 +86,24 @@ func (ls *LogSaver) getTestPods(ctx context.Context, loadTest *grpcv1.LoadTest) return testPods, nil } -func (ls *LogSaver) getPodLogBuffer(ctx context.Context, pod *corev1.Pod) (*bytes.Buffer, error) { - req := ls.podsGetter.Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Container: config.RunContainerName}) - driverLogs, err := req.Stream(ctx) - if err != nil { - return nil, err +// getPodLogBuffers retrieves logs from all existing containers +// from the pod and save the log buffers in a map, the key of +// the map is the container name and the value of the map is the +// log buffers. +func (ls *LogSaver) getPodLogBuffers(ctx context.Context, pod *corev1.Pod) (map[string]*bytes.Buffer, error) { + containerNamesToLogMap := make(map[string]*bytes.Buffer) + for _, container := range pod.Spec.Containers { + req := ls.podsGetter.Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Container: container.Name}) + containerLogs, err := req.Stream(ctx) + if err != nil { + return nil, err + } + defer containerLogs.Close() + logBuffer := new(bytes.Buffer) + logBuffer.ReadFrom(containerLogs) + containerNamesToLogMap[container.Name] = logBuffer } - defer driverLogs.Close() - - logBuffer := new(bytes.Buffer) - logBuffer.ReadFrom(driverLogs) - - return logBuffer, nil + return containerNamesToLogMap, nil } func (ls *LogSaver) writeBufferToFile(buffer *bytes.Buffer, filePath string) error { @@ -118,18 +131,18 @@ func (ls *LogSaver) writeBufferToFile(buffer *bytes.Buffer, filePath string) err // SavedLogs adds functions to get information about saved pod logs. type SavedLogs struct { - podToPathMap map[*corev1.Pod]string + podToPathMap map[*corev1.Pod]map[string]string } // NewSavedLogs creates a new SavedLogs object. func NewSavedLogs() *SavedLogs { return &SavedLogs{ - podToPathMap: make(map[*corev1.Pod]string), + podToPathMap: make(map[*corev1.Pod]map[string]string), } } -// GenerateProperties creates pod-log related properties. -func (sl *SavedLogs) GenerateProperties(loadTest *grpcv1.LoadTest) map[string]string { +// GenerateNameProperties creates pod-name related properties. +func (sl *SavedLogs) GenerateNameProperties(loadTest *grpcv1.LoadTest) map[string]string { properties := make(map[string]string) for pod := range sl.podToPathMap { name := sl.podToPropertyName(pod.Name, loadTest.Name, "name") @@ -138,6 +151,20 @@ func (sl *SavedLogs) GenerateProperties(loadTest *grpcv1.LoadTest) map[string]st return properties } +// GenerateLogProperties creates pod-log related properties. +func (sl *SavedLogs) GenerateLogProperties(loadTest *grpcv1.LoadTest, logURLPrefix string) map[string]string { + properties := make(map[string]string) + for pod, buffers := range sl.podToPathMap { + for containerName, logFilePath := range buffers { + elementName := "log." + containerName + name := sl.podToPropertyName(pod.Name, loadTest.Name, elementName) + url := logURLPrefix + logFilePath + properties[name] = url + } + } + return properties +} + func (sl *SavedLogs) podToPropertyName(podName, loadTestName, elementName string) string { prefix := fmt.Sprintf("%s-", loadTestName) podNameSuffix := strings.TrimPrefix(podName, prefix) diff --git a/tools/runner/runner.go b/tools/runner/runner.go index d84f3028..9333f617 100644 --- a/tools/runner/runner.go +++ b/tools/runner/runner.go @@ -146,9 +146,16 @@ func (r *Runner) runTest(ctx context.Context, config *grpcv1.LoadTest, reporter reporter.Error("Could not save pod logs: %s", err) } reporter.AddProperty("name", loadTest.Name) - for property, value := range savedLogs.GenerateProperties(loadTest) { + for property, value := range savedLogs.GenerateNameProperties(loadTest) { reporter.AddProperty(property, value) } + + if r.logSaver.logURLPrefix != "" { + for property, value := range savedLogs.GenerateLogProperties(loadTest, r.logSaver.logURLPrefix) { + reporter.AddProperty(property, value) + } + } + if status != "Succeeded" { reporter.Error("Test failed with reason %q: %v", loadTest.Status.Reason, loadTest.Status.Message) } else {