Collect logs from all containers and add log to test properties.

tools/cmd/runner/main.go

@@ -36,6 +36,7 @@ func main() {
 	var p time.Duration
 	var retries uint
 	var deleteSuccessfulTests bool
+	var logURLPrefix string
 
 	flag.Var(&i, "i", "input files containing load test configurations")
 	flag.StringVar(&o, "o", "", "name of the output file for xunit xml report")
@@ -44,6 +45,7 @@ func main() {
 	flag.DurationVar(&p, "polling-interval", 20*time.Second, "polling interval for load test status")
 	flag.UintVar(&retries, "polling-retries", 2, "Maximum retries in case of communication failure")
 	flag.BoolVar(&deleteSuccessfulTests, "delete-successful-tests", false, "Delete tests immediately in case of successful termination")
+	flag.StringVar(&logURLPrefix, "log-url-prefix", "", "prefix for log urls")
 	flag.Parse()
 
 	inputConfigs, err := runner.DecodeFromFiles(i)
@@ -75,8 +77,11 @@ func main() {
 	log.Printf("Test counts per queue: %v", runner.CountConfigs(configQueueMap))
 	log.Printf("Queue concurrency levels: %v", c)
 	log.Printf("Output directories: %v", outputDirMap)
+	if logURLPrefix != "" {
+		log.Printf("Prefix for log urls: %s", logURLPrefix)
+	}
 
-	r := runner.NewRunner(runner.NewLoadTestGetter(), runner.AfterIntervalFunction(p), retries, deleteSuccessfulTests, runner.NewLogSaver(runner.NewPodsGetter()))
+	r := runner.NewRunner(runner.NewLoadTestGetter(), runner.NewPodsGetter(), runner.AfterIntervalFunction(p), retries, deleteSuccessfulTests, logURLPrefix)
 
 	logPrefixFmt := runner.LogPrefixFmt(configQueueMap)
 

tools/runner/client.go

@@ -17,6 +17,8 @@ limitations under the License.
 package runner
 
 import (
+	"context"
+	"fmt"
 	"log"
 	"os"
 	"strings"
@@ -36,6 +38,7 @@ import (
 
 	grpcv1 "github.com/grpc/test-infra/api/v1"
 	clientset "github.com/grpc/test-infra/clientset"
+	"github.com/grpc/test-infra/status"
 )
 
 // NewLoadTestGetter returns a client to interact with LoadTest resources. The
@@ -85,6 +88,21 @@ func NewPodsGetter() corev1types.PodsGetter {
 	return clientset.CoreV1()
 }
 
+// GetTestPods retrieves the pods associated with a LoadTest.
+func GetTestPods(ctx context.Context, loadTest *grpcv1.LoadTest, podsGetter corev1types.PodsGetter) ([]*corev1.Pod, error) {
+	podLister := podsGetter.Pods(metav1.NamespaceAll)
+
+	// Get a list of all pods.
+	podList, err := podLister.List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch list of pods: %v", err)
+	}
+
+	// Get pods just for this specific test.
+	testPods := status.PodsForLoadTest(loadTest, podList.Items)
+	return testPods, nil
+}
+
 // getKubernetesConfig retrieves the kubernetes configuration.
 func getKubernetesConfig() *rest.Config {
 	config, err := rest.InClusterConfig()

tools/runner/logsaver.go

@@ -1,145 +1,109 @@
+/*
+Copyright 2021 gRPC authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package runner
 
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
-	"strings"
 
 	grpcv1 "github.com/grpc/test-infra/api/v1"
-	"github.com/grpc/test-infra/status"
 	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	corev1types "k8s.io/client-go/kubernetes/typed/core/v1"
 )
 
-// LogSaver provides functionality to save pod logs to files.
-type LogSaver struct {
-	podsGetter corev1types.PodsGetter
-}
-
-// NewLogSaver creates a new LogSaver object.
-func NewLogSaver(podsGetter corev1types.PodsGetter) *LogSaver {
-	return &LogSaver{
-		podsGetter: podsGetter,
-	}
-}
-
-// SavePodLogs saves pod logs to files with same name as pod.
-// This function returns a map where pods are keys and values are the filepath
-// of the saved log.
-func (ls *LogSaver) SavePodLogs(ctx context.Context, loadTest *grpcv1.LoadTest, podLogDir string) (*SavedLogs, error) {
-	savedLogs := NewSavedLogs()
+// SaveAllLogs saves all container logs to files under a given directory.
+// This function goes through every container in every pods and writes
+// its log to a file, if it is not empty. Information about each saved log
+// is returned as a pointer to a LogInfo object.
+func SaveAllLogs(ctx context.Context, loadTest *grpcv1.LoadTest, podsGetter corev1types.PodsGetter, pods []*corev1.Pod, podLogDir string) ([]*LogInfo, error) {
+	var logInfos []*LogInfo
 
-	// Get pods for this test
-	pods, err := ls.getTestPods(ctx, loadTest)
+	// Attempts to create directory. Will not error if directory already exists.
+	err := os.MkdirAll(podLogDir, os.ModePerm)
 	if err != nil {
-		return savedLogs, err
+		return logInfos, fmt.Errorf("failed to create pod log output directory %s: %v", podLogDir, err)
 	}
 
-	// Attempt to create directory. Will not error if directory already exists
-	err = os.MkdirAll(podLogDir, os.ModePerm)
-	if err != nil {
-		return savedLogs, fmt.Errorf("Failed to create pod log output directory %s: %v", podLogDir, err)
-	}
-
-	// Write logs to files
+	// Write logs to files.
 	for _, pod := range pods {
-		logFilePath := filepath.Join(podLogDir, pod.Name+".log")
-		buffer, err := ls.getPodLogBuffer(ctx, pod)
-		if err != nil {
-			return savedLogs, fmt.Errorf("could not get log from pod: %s", err)
-		}
-		err = ls.writeBufferToFile(buffer, logFilePath)
-		if err != nil {
-			return savedLogs, fmt.Errorf("could not write pod log buffer to file: %s", err)
-		}
-		savedLogs.podToPathMap[pod] = logFilePath
-	}
-	return savedLogs, nil
-}
+		for _, container := range pod.Spec.Containers {
 
-// getTestPods retrieves the pods associated with a LoadTest.
-func (ls *LogSaver) getTestPods(ctx context.Context, loadTest *grpcv1.LoadTest) ([]*corev1.Pod, error) {
-	podLister := ls.podsGetter.Pods(metav1.NamespaceAll)
+			logInfo, err := SaveLog(ctx, loadTest, podsGetter, pod, container.Name, podLogDir)
+			if err != nil {
+				return logInfos, fmt.Errorf("could not get log from container: %v", err)
+			}
 
-	// Get a list of all pods
-	podList, err := podLister.List(ctx, metav1.ListOptions{})
-	if err != nil {
-		return nil, errors.New("Failed to fetch list of pods")
+			if logInfo != nil {
+				logInfos = append(logInfos, logInfo)
+			}
+		}
 	}
-
-	// Get pods just for this specific test
-	testPods := status.PodsForLoadTest(loadTest, podList.Items)
-	return testPods, nil
+	return logInfos, nil
 }
 
-func (ls *LogSaver) getPodLogBuffer(ctx context.Context, pod *corev1.Pod) (*bytes.Buffer, error) {
-	req := ls.podsGetter.Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{})
-	driverLogs, err := req.Stream(ctx)
+// SaveLog retrieves and saves logs for a specific container.
+// This function retrieves the log for a single container within a given
+// pod, and writes it to a file, if it is not empty. Information about
+// the saved log is returned as a pointer to a LogInfo object.
+func SaveLog(ctx context.Context, loadTest *grpcv1.LoadTest, podsGetter corev1types.PodsGetter, pod *corev1.Pod, containerName string, podLogDir string) (*LogInfo, error) {
+	req := podsGetter.Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Container: containerName})
+	containerLogs, err := req.Stream(ctx)
 	if err != nil {
 		return nil, err
 	}
-	defer driverLogs.Close()
+	defer containerLogs.Close()
 
 	logBuffer := new(bytes.Buffer)
-	logBuffer.ReadFrom(driverLogs)
-
-	return logBuffer, nil
-}
+	logBuffer.ReadFrom(containerLogs)
 
-func (ls *LogSaver) writeBufferToFile(buffer *bytes.Buffer, filePath string) error {
-	// Don't write empty buffers
-	if buffer.Len() == 0 {
-		return nil
+	// Don't write empty buffers,
+	if logBuffer.Len() == 0 {
+		return nil, nil
 	}
 
-	// Open output file
+	// Open output file,
+	filePath := filepath.Join(podLogDir, LogFileName(pod.Name, containerName))
 	file, err := os.Create(filePath)
 	if err != nil {
-		return fmt.Errorf("could not open %s for writing", filePath)
+		return nil, fmt.Errorf("could not open %s for writing", filePath)
 	}
 	defer file.Close()
 
-	// Write log to output file
-	_, err = io.Copy(file, buffer)
+	// Write log to output file,
+	_, err = io.Copy(file, logBuffer)
 	file.Sync()
 	if err != nil {
-		return fmt.Errorf("error writing to %s: %v", filePath, err)
+		return nil, fmt.Errorf("error writing to %s: %v", filePath, err)
 	}
 
-	return nil
-}
-
-// SavedLogs adds functions to get information about saved pod logs.
-type SavedLogs struct {
-	podToPathMap map[*corev1.Pod]string
-}
-
-// NewSavedLogs creates a new SavedLogs object.
-func NewSavedLogs() *SavedLogs {
-	return &SavedLogs{
-		podToPathMap: make(map[*corev1.Pod]string),
+	logInfo := &LogInfo{
+		PodNameElem:   PodNameElem(pod.Name, loadTest.Name),
+		ContainerName: containerName,
+		LogPath:       filePath,
 	}
-}
 
-// GenerateProperties creates pod-log related properties.
-func (sl *SavedLogs) GenerateProperties(loadTest *grpcv1.LoadTest) map[string]string {
-	properties := make(map[string]string)
-	for pod := range sl.podToPathMap {
-		name := sl.podToPropertyName(pod.Name, loadTest.Name, "name")
-		properties[name] = pod.Name
-	}
-	return properties
+	return logInfo, nil
 }
 
-func (sl *SavedLogs) podToPropertyName(podName, loadTestName, elementName string) string {
-	prefix := fmt.Sprintf("%s-", loadTestName)
-	podNameSuffix := strings.TrimPrefix(podName, prefix)
-	propertyName := fmt.Sprintf("pod.%s.%s", podNameSuffix, elementName)
-	return propertyName
+// LogFileName constructs a log file name from pod and container names.
+func LogFileName(podName string, containerName string) string {
+	return fmt.Sprintf("%s-%s.log", podName, containerName)
 }

tools/runner/properties.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2022 gRPC authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package runner
+
+import (
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+// LogInfo contains infomation for each log file.
+type LogInfo struct {
+	// PodNameElem is the element added to the LoadTest name to
+	// construct the pod name. Examples of PodNameElem are client-0,
+	// driver-0 and server-0.
+	PodNameElem string
+	// ContainerName is the container's name where the log comes from.
+	ContainerName string
+	// LogPath is the path pointing to the log file.
+	LogPath string
+}
+
+// PodLogProperties creates a map of log property keys to log path urls.
+func PodLogProperties(logInfos []*LogInfo, logURLPrefix string, prefix ...string) map[string]string {
+	properties := make(map[string]string)
+	for _, logInfo := range logInfos {
+		podLogPropertyKey := PodLogPropertyKey(logInfo, prefix...)
+		logURL := logURLPrefix + logInfo.LogPath
+		properties[podLogPropertyKey] = logURL
+	}
+	return properties
+}
+
+// PodLogPropertyKey generates the key for a pod log property.
+func PodLogPropertyKey(logInfo *LogInfo, prefix ...string) string {
+	key := strings.Join(append(prefix, logInfo.PodNameElem, "log", logInfo.ContainerName), ".")
+	return key
+}
+
+// PodNameProperties creates a map of pod name property keys to pod names.
+func PodNameProperties(pods []*corev1.Pod, loadTestName string, prefix ...string) map[string]string {
+	properties := make(map[string]string)
+	for _, pod := range pods {
+		podNamePropertyKey := PodNamePropertyKey(PodNameElem(pod.Name, loadTestName), prefix...)
+		properties[podNamePropertyKey] = pod.Name
+	}
+
+	return properties
+}
+
+// PodNameElem returns the pod name element used to construct a pod name.
+// Pods within a LoadTest are distinguished by elements attached to the
+// LoadTest name, such as client-0, driver-0, server-0.
+func PodNameElem(podName, loadTestName string) string {
+	prefix := fmt.Sprintf("%s-", loadTestName)
+	podNameElem := strings.TrimPrefix(podName, prefix)
+	return podNameElem
+}
+
+// PodNamePropertyKey generates the key for a pod name property.
+func PodNamePropertyKey(podNameElem string, prefix ...string) string {
+	key := strings.Join(append(prefix, podNameElem, "name"), ".")
+	return key
+}