Save logs from all containers and add url link properties.

This commit updates the logics to save logs from all containers to
the original directory. The name of the logs follows the format:
pod-name-container-name.log. This commit also add log url to the
properties if the log url prefix is provided.

tools/cmd/runner/main.go

@@ -34,13 +34,15 @@ func main() {
 	var c runner.ConcurrencyLevels
 	var a string
 	var p time.Duration
+	var u string
 	var retries uint
 	var deleteSuccessfulTests bool
 
 	flag.Var(&i, "i", "input files containing load test configurations")
 	flag.StringVar(&o, "o", "", "name of the output file for xunit xml report")
 	flag.Var(&c, "c", "concurrency level, in the form [<queue name>:]<concurrency level>")
 	flag.StringVar(&a, "annotation-key", "pool", "annotation key to parse for queue assignment")
+	flag.StringVar(&u, "log-url-prefix", "", "prefix for log urls")
 	flag.DurationVar(&p, "polling-interval", 20*time.Second, "polling interval for load test status")
 	flag.UintVar(&retries, "polling-retries", 2, "Maximum retries in case of communication failure")
 	flag.BoolVar(&deleteSuccessfulTests, "delete-successful-tests", false, "Delete tests immediately in case of successful termination")
@@ -69,14 +71,22 @@ func main() {
 		outputDirMap[qName] = outputDir
 	}
 
+	logURLPrefix := ""
+	if u != "" {
+		logURLPrefix = "http://cnsviewer2/placer/prod/home/kokoro-dedicated/build_artifacts/" + u + "/github/grpc/"
+	}
+
 	log.Printf("Annotation key for queue assignment: %s", a)
 	log.Printf("Polling interval: %v", p)
 	log.Printf("Polling retries: %d", retries)
 	log.Printf("Test counts per queue: %v", runner.CountConfigs(configQueueMap))
 	log.Printf("Queue concurrency levels: %v", c)
 	log.Printf("Output directories: %v", outputDirMap)
+	if logURLPrefix != "" {
+		log.Printf("Prefix for url for saved logs: %s", logURLPrefix)
+	}
 
-	r := runner.NewRunner(runner.NewLoadTestGetter(), runner.AfterIntervalFunction(p), retries, deleteSuccessfulTests, runner.NewLogSaver(runner.NewPodsGetter()))
+	r := runner.NewRunner(runner.NewLoadTestGetter(), runner.AfterIntervalFunction(p), retries, deleteSuccessfulTests, runner.NewLogSaver(runner.NewPodsGetter(), logURLPrefix))
 
 	logPrefixFmt := runner.LogPrefixFmt(configQueueMap)
 

tools/runner/logsaver.go

@@ -11,7 +11,6 @@ import (
 	"strings"
 
 	grpcv1 "github.com/grpc/test-infra/api/v1"
-	"github.com/grpc/test-infra/config"
 	"github.com/grpc/test-infra/status"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -20,17 +19,20 @@ import (
 
 // LogSaver provides functionality to save pod logs to files.
 type LogSaver struct {
-	podsGetter corev1types.PodsGetter
+	podsGetter   corev1types.PodsGetter
+	logURLPrefix string
 }
 
 // NewLogSaver creates a new LogSaver object.
-func NewLogSaver(podsGetter corev1types.PodsGetter) *LogSaver {
+func NewLogSaver(podsGetter corev1types.PodsGetter, logURLPrefix string) *LogSaver {
 	return &LogSaver{
-		podsGetter: podsGetter,
+		podsGetter:   podsGetter,
+		logURLPrefix: logURLPrefix,
 	}
 }
 
-// SavePodLogs saves pod logs to files with same name as pod.
+// SavePodLogs saves container logs to files with name in format
+// pod-name-container-name.log.
 // This function returns a map where pods are keys and values are the filepath
 // of the saved log.
 func (ls *LogSaver) SavePodLogs(ctx context.Context, loadTest *grpcv1.LoadTest, podLogDir string) (*SavedLogs, error) {
@@ -50,16 +52,21 @@ func (ls *LogSaver) SavePodLogs(ctx context.Context, loadTest *grpcv1.LoadTest,
 
 	// Write logs to files
 	for _, pod := range pods {
-		logFilePath := filepath.Join(podLogDir, pod.Name+".log")
-		buffer, err := ls.getPodLogBuffer(ctx, pod)
+		containerNametoLogPathMap := make(map[string]string)
+		containerNamesToLogMap, err := ls.getPodLogBuffers(ctx, pod)
 		if err != nil {
 			return savedLogs, fmt.Errorf("could not get log from pod: %s", err)
 		}
-		err = ls.writeBufferToFile(buffer, logFilePath)
-		if err != nil {
-			return savedLogs, fmt.Errorf("could not write pod log buffer to file: %s", err)
+		for containerName, buffer := range containerNamesToLogMap {
+			logFilePath := filepath.Join(podLogDir, fmt.Sprintf("%s-%s.log", pod.Name, containerName))
+			err = ls.writeBufferToFile(buffer, logFilePath)
+			if err != nil {
+				return savedLogs, fmt.Errorf("could not write %s container in %s pod log buffer to file: %s", containerName, pod.Name, err)
+			}
+			containerNametoLogPathMap[containerName] = logFilePath
 		}
-		savedLogs.podToPathMap[pod] = logFilePath
+
+		savedLogs.podToPathMap[pod] = containerNametoLogPathMap
 	}
 	return savedLogs, nil
 }
@@ -79,18 +86,24 @@ func (ls *LogSaver) getTestPods(ctx context.Context, loadTest *grpcv1.LoadTest)
 	return testPods, nil
 }
 
-func (ls *LogSaver) getPodLogBuffer(ctx context.Context, pod *corev1.Pod) (*bytes.Buffer, error) {
-	req := ls.podsGetter.Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Container: config.RunContainerName})
-	driverLogs, err := req.Stream(ctx)
-	if err != nil {
-		return nil, err
+// getPodLogBuffers retrieves logs from all existing containers
+// from the pod and save the log buffers in a map, the key of
+// the map is the container name and the value of the map is the
+// log buffers.
+func (ls *LogSaver) getPodLogBuffers(ctx context.Context, pod *corev1.Pod) (map[string]*bytes.Buffer, error) {
+	containerNamesToLogMap := make(map[string]*bytes.Buffer)
+	for _, container := range pod.Spec.Containers {
+		req := ls.podsGetter.Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Container: container.Name})
+		containerLogs, err := req.Stream(ctx)
+		if err != nil {
+			return nil, err
+		}
+		defer containerLogs.Close()
+		logBuffer := new(bytes.Buffer)
+		logBuffer.ReadFrom(containerLogs)
+		containerNamesToLogMap[container.Name] = logBuffer
 	}
-	defer driverLogs.Close()
-
-	logBuffer := new(bytes.Buffer)
-	logBuffer.ReadFrom(driverLogs)
-
-	return logBuffer, nil
+	return containerNamesToLogMap, nil
 }
 
 func (ls *LogSaver) writeBufferToFile(buffer *bytes.Buffer, filePath string) error {
@@ -118,18 +131,18 @@ func (ls *LogSaver) writeBufferToFile(buffer *bytes.Buffer, filePath string) err
 
 // SavedLogs adds functions to get information about saved pod logs.
 type SavedLogs struct {
-	podToPathMap map[*corev1.Pod]string
+	podToPathMap map[*corev1.Pod]map[string]string
 }
 
 // NewSavedLogs creates a new SavedLogs object.
 func NewSavedLogs() *SavedLogs {
 	return &SavedLogs{
-		podToPathMap: make(map[*corev1.Pod]string),
+		podToPathMap: make(map[*corev1.Pod]map[string]string),
 	}
 }
 
-// GenerateProperties creates pod-log related properties.
-func (sl *SavedLogs) GenerateProperties(loadTest *grpcv1.LoadTest) map[string]string {
+// GenerateNameProperties creates pod-name related properties.
+func (sl *SavedLogs) GenerateNameProperties(loadTest *grpcv1.LoadTest) map[string]string {
 	properties := make(map[string]string)
 	for pod := range sl.podToPathMap {
 		name := sl.podToPropertyName(pod.Name, loadTest.Name, "name")
@@ -138,6 +151,20 @@ func (sl *SavedLogs) GenerateProperties(loadTest *grpcv1.LoadTest) map[string]st
 	return properties
 }
 
+// GenerateLogProperties creates pod-log related properties.
+func (sl *SavedLogs) GenerateLogProperties(loadTest *grpcv1.LoadTest, logURLPrefix string) map[string]string {
+	properties := make(map[string]string)
+	for pod, buffers := range sl.podToPathMap {
+		for containerName, logFilePath := range buffers {
+			elementName := "log." + containerName
+			name := sl.podToPropertyName(pod.Name, loadTest.Name, elementName)
+			url := logURLPrefix + logFilePath
+			properties[name] = url
+		}
+	}
+	return properties
+}
+
 func (sl *SavedLogs) podToPropertyName(podName, loadTestName, elementName string) string {
 	prefix := fmt.Sprintf("%s-", loadTestName)
 	podNameSuffix := strings.TrimPrefix(podName, prefix)

tools/runner/runner.go

@@ -146,9 +146,16 @@ func (r *Runner) runTest(ctx context.Context, config *grpcv1.LoadTest, reporter
 				reporter.Error("Could not save pod logs: %s", err)
 			}
 			reporter.AddProperty("name", loadTest.Name)
-			for property, value := range savedLogs.GenerateProperties(loadTest) {
+			for property, value := range savedLogs.GenerateNameProperties(loadTest) {
 				reporter.AddProperty(property, value)
 			}
+
+			if r.logSaver.logURLPrefix != "" {
+				for property, value := range savedLogs.GenerateLogProperties(loadTest, r.logSaver.logURLPrefix) {
+					reporter.AddProperty(property, value)
+				}
+			}
+
 			if status != "Succeeded" {
 				reporter.Error("Test failed with reason %q: %v", loadTest.Status.Reason, loadTest.Status.Message)
 			} else {