diff --git a/tools/cmd/runner/main.go b/tools/cmd/runner/main.go index 851a8802..d374739d 100644 --- a/tools/cmd/runner/main.go +++ b/tools/cmd/runner/main.go @@ -36,6 +36,7 @@ func main() { var p time.Duration var retries uint var deleteSuccessfulTests bool + var logURLPrefix string flag.Var(&i, "i", "input files containing load test configurations") flag.StringVar(&o, "o", "", "name of the output file for xunit xml report") @@ -44,6 +45,7 @@ func main() { flag.DurationVar(&p, "polling-interval", 20*time.Second, "polling interval for load test status") flag.UintVar(&retries, "polling-retries", 2, "Maximum retries in case of communication failure") flag.BoolVar(&deleteSuccessfulTests, "delete-successful-tests", false, "Delete tests immediately in case of successful termination") + flag.StringVar(&logURLPrefix, "log-url-prefix", "", "prefix for log urls") flag.Parse() inputConfigs, err := runner.DecodeFromFiles(i) @@ -75,8 +77,11 @@ func main() { log.Printf("Test counts per queue: %v", runner.CountConfigs(configQueueMap)) log.Printf("Queue concurrency levels: %v", c) log.Printf("Output directories: %v", outputDirMap) + if logURLPrefix != "" { + log.Printf("Prefix for log urls: %s", logURLPrefix) + } - r := runner.NewRunner(runner.NewLoadTestGetter(), runner.AfterIntervalFunction(p), retries, deleteSuccessfulTests, runner.NewLogSaver(runner.NewPodsGetter())) + r := runner.NewRunner(runner.NewLoadTestGetter(), runner.NewPodsGetter(), runner.AfterIntervalFunction(p), retries, deleteSuccessfulTests, logURLPrefix) logPrefixFmt := runner.LogPrefixFmt(configQueueMap) diff --git a/tools/runner/client.go b/tools/runner/client.go index 0eb3241e..fd484373 100644 --- a/tools/runner/client.go +++ b/tools/runner/client.go @@ -17,6 +17,8 @@ limitations under the License. package runner import ( + "context" + "fmt" "log" "os" "strings" @@ -36,6 +38,7 @@ import ( grpcv1 "github.com/grpc/test-infra/api/v1" clientset "github.com/grpc/test-infra/clientset" + "github.com/grpc/test-infra/status" ) // NewLoadTestGetter returns a client to interact with LoadTest resources. The @@ -85,6 +88,21 @@ func NewPodsGetter() corev1types.PodsGetter { return clientset.CoreV1() } +// GetTestPods retrieves the pods associated with a LoadTest. +func GetTestPods(ctx context.Context, loadTest *grpcv1.LoadTest, podsGetter corev1types.PodsGetter) ([]*corev1.Pod, error) { + podLister := podsGetter.Pods(metav1.NamespaceAll) + + // Get a list of all pods. + podList, err := podLister.List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to fetch list of pods: %v", err) + } + + // Get pods just for this specific test. + testPods := status.PodsForLoadTest(loadTest, podList.Items) + return testPods, nil +} + // getKubernetesConfig retrieves the kubernetes configuration. func getKubernetesConfig() *rest.Config { config, err := rest.InClusterConfig() diff --git a/tools/runner/logsaver.go b/tools/runner/logsaver.go index 30c5adf7..0ff95fe5 100644 --- a/tools/runner/logsaver.go +++ b/tools/runner/logsaver.go @@ -1,145 +1,109 @@ +/* +Copyright 2021 gRPC authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package runner import ( "bytes" "context" - "errors" "fmt" "io" "os" "path/filepath" - "strings" grpcv1 "github.com/grpc/test-infra/api/v1" - "github.com/grpc/test-infra/status" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1types "k8s.io/client-go/kubernetes/typed/core/v1" ) -// LogSaver provides functionality to save pod logs to files. -type LogSaver struct { - podsGetter corev1types.PodsGetter -} - -// NewLogSaver creates a new LogSaver object. -func NewLogSaver(podsGetter corev1types.PodsGetter) *LogSaver { - return &LogSaver{ - podsGetter: podsGetter, - } -} - -// SavePodLogs saves pod logs to files with same name as pod. -// This function returns a map where pods are keys and values are the filepath -// of the saved log. -func (ls *LogSaver) SavePodLogs(ctx context.Context, loadTest *grpcv1.LoadTest, podLogDir string) (*SavedLogs, error) { - savedLogs := NewSavedLogs() +// SaveAllLogs saves all container logs to files under a given directory. +// This function goes through every container in every pods and writes +// its log to a file, if it is not empty. Information about each saved log +// is returned as a pointer to a LogInfo object. +func SaveAllLogs(ctx context.Context, loadTest *grpcv1.LoadTest, podsGetter corev1types.PodsGetter, pods []*corev1.Pod, podLogDir string) ([]*LogInfo, error) { + var logInfos []*LogInfo - // Get pods for this test - pods, err := ls.getTestPods(ctx, loadTest) + // Attempts to create directory. Will not error if directory already exists. + err := os.MkdirAll(podLogDir, os.ModePerm) if err != nil { - return savedLogs, err + return logInfos, fmt.Errorf("failed to create pod log output directory %s: %v", podLogDir, err) } - // Attempt to create directory. Will not error if directory already exists - err = os.MkdirAll(podLogDir, os.ModePerm) - if err != nil { - return savedLogs, fmt.Errorf("Failed to create pod log output directory %s: %v", podLogDir, err) - } - - // Write logs to files + // Write logs to files. for _, pod := range pods { - logFilePath := filepath.Join(podLogDir, pod.Name+".log") - buffer, err := ls.getPodLogBuffer(ctx, pod) - if err != nil { - return savedLogs, fmt.Errorf("could not get log from pod: %s", err) - } - err = ls.writeBufferToFile(buffer, logFilePath) - if err != nil { - return savedLogs, fmt.Errorf("could not write pod log buffer to file: %s", err) - } - savedLogs.podToPathMap[pod] = logFilePath - } - return savedLogs, nil -} + for _, container := range pod.Spec.Containers { -// getTestPods retrieves the pods associated with a LoadTest. -func (ls *LogSaver) getTestPods(ctx context.Context, loadTest *grpcv1.LoadTest) ([]*corev1.Pod, error) { - podLister := ls.podsGetter.Pods(metav1.NamespaceAll) + logInfo, err := SaveLog(ctx, loadTest, podsGetter, pod, container.Name, podLogDir) + if err != nil { + return logInfos, fmt.Errorf("could not get log from container: %v", err) + } - // Get a list of all pods - podList, err := podLister.List(ctx, metav1.ListOptions{}) - if err != nil { - return nil, errors.New("Failed to fetch list of pods") + if logInfo != nil { + logInfos = append(logInfos, logInfo) + } + } } - - // Get pods just for this specific test - testPods := status.PodsForLoadTest(loadTest, podList.Items) - return testPods, nil + return logInfos, nil } -func (ls *LogSaver) getPodLogBuffer(ctx context.Context, pod *corev1.Pod) (*bytes.Buffer, error) { - req := ls.podsGetter.Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{}) - driverLogs, err := req.Stream(ctx) +// SaveLog retrieves and saves logs for a specific container. +// This function retrieves the log for a single container within a given +// pod, and writes it to a file, if it is not empty. Information about +// the saved log is returned as a pointer to a LogInfo object. +func SaveLog(ctx context.Context, loadTest *grpcv1.LoadTest, podsGetter corev1types.PodsGetter, pod *corev1.Pod, containerName string, podLogDir string) (*LogInfo, error) { + req := podsGetter.Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Container: containerName}) + containerLogs, err := req.Stream(ctx) if err != nil { return nil, err } - defer driverLogs.Close() + defer containerLogs.Close() logBuffer := new(bytes.Buffer) - logBuffer.ReadFrom(driverLogs) - - return logBuffer, nil -} + logBuffer.ReadFrom(containerLogs) -func (ls *LogSaver) writeBufferToFile(buffer *bytes.Buffer, filePath string) error { - // Don't write empty buffers - if buffer.Len() == 0 { - return nil + // Don't write empty buffers, + if logBuffer.Len() == 0 { + return nil, nil } - // Open output file + // Open output file, + filePath := filepath.Join(podLogDir, LogFileName(pod.Name, containerName)) file, err := os.Create(filePath) if err != nil { - return fmt.Errorf("could not open %s for writing", filePath) + return nil, fmt.Errorf("could not open %s for writing", filePath) } defer file.Close() - // Write log to output file - _, err = io.Copy(file, buffer) + // Write log to output file, + _, err = io.Copy(file, logBuffer) file.Sync() if err != nil { - return fmt.Errorf("error writing to %s: %v", filePath, err) + return nil, fmt.Errorf("error writing to %s: %v", filePath, err) } - return nil -} - -// SavedLogs adds functions to get information about saved pod logs. -type SavedLogs struct { - podToPathMap map[*corev1.Pod]string -} - -// NewSavedLogs creates a new SavedLogs object. -func NewSavedLogs() *SavedLogs { - return &SavedLogs{ - podToPathMap: make(map[*corev1.Pod]string), + logInfo := &LogInfo{ + PodNameElem: PodNameElem(pod.Name, loadTest.Name), + ContainerName: containerName, + LogPath: filePath, } -} -// GenerateProperties creates pod-log related properties. -func (sl *SavedLogs) GenerateProperties(loadTest *grpcv1.LoadTest) map[string]string { - properties := make(map[string]string) - for pod := range sl.podToPathMap { - name := sl.podToPropertyName(pod.Name, loadTest.Name, "name") - properties[name] = pod.Name - } - return properties + return logInfo, nil } -func (sl *SavedLogs) podToPropertyName(podName, loadTestName, elementName string) string { - prefix := fmt.Sprintf("%s-", loadTestName) - podNameSuffix := strings.TrimPrefix(podName, prefix) - propertyName := fmt.Sprintf("pod.%s.%s", podNameSuffix, elementName) - return propertyName +// LogFileName constructs a log file name from pod and container names. +func LogFileName(podName string, containerName string) string { + return fmt.Sprintf("%s-%s.log", podName, containerName) } diff --git a/tools/runner/properties.go b/tools/runner/properties.go new file mode 100644 index 00000000..836c065a --- /dev/null +++ b/tools/runner/properties.go @@ -0,0 +1,79 @@ +/* +Copyright 2022 gRPC authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runner + +import ( + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" +) + +// LogInfo contains infomation for each log file. +type LogInfo struct { + // PodNameElem is the element added to the LoadTest name to + // construct the pod name. Examples of PodNameElem are client-0, + // driver-0 and server-0. + PodNameElem string + // ContainerName is the container's name where the log comes from. + ContainerName string + // LogPath is the path pointing to the log file. + LogPath string +} + +// PodLogProperties creates a map of log property keys to log path urls. +func PodLogProperties(logInfos []*LogInfo, logURLPrefix string, prefix ...string) map[string]string { + properties := make(map[string]string) + for _, logInfo := range logInfos { + podLogPropertyKey := PodLogPropertyKey(logInfo, prefix...) + logURL := logURLPrefix + logInfo.LogPath + properties[podLogPropertyKey] = logURL + } + return properties +} + +// PodLogPropertyKey generates the key for a pod log property. +func PodLogPropertyKey(logInfo *LogInfo, prefix ...string) string { + key := strings.Join(append(prefix, logInfo.PodNameElem, "log", logInfo.ContainerName), ".") + return key +} + +// PodNameProperties creates a map of pod name property keys to pod names. +func PodNameProperties(pods []*corev1.Pod, loadTestName string, prefix ...string) map[string]string { + properties := make(map[string]string) + for _, pod := range pods { + podNamePropertyKey := PodNamePropertyKey(PodNameElem(pod.Name, loadTestName), prefix...) + properties[podNamePropertyKey] = pod.Name + } + + return properties +} + +// PodNameElem returns the pod name element used to construct a pod name. +// Pods within a LoadTest are distinguished by elements attached to the +// LoadTest name, such as client-0, driver-0, server-0. +func PodNameElem(podName, loadTestName string) string { + prefix := fmt.Sprintf("%s-", loadTestName) + podNameElem := strings.TrimPrefix(podName, prefix) + return podNameElem +} + +// PodNamePropertyKey generates the key for a pod name property. +func PodNamePropertyKey(podNameElem string, prefix ...string) string { + key := strings.Join(append(prefix, podNameElem, "name"), ".") + return key +} diff --git a/tools/runner/runner.go b/tools/runner/runner.go index d84f3028..1189aba5 100644 --- a/tools/runner/runner.go +++ b/tools/runner/runner.go @@ -26,6 +26,7 @@ import ( grpcv1 "github.com/grpc/test-infra/api/v1" clientset "github.com/grpc/test-infra/clientset" + corev1types "k8s.io/client-go/kubernetes/typed/core/v1" ) // AfterIntervalFunction returns a function that stops for a time interval. @@ -41,6 +42,9 @@ type Runner struct { // loadTestGetter interacts with the cluster to create, get and delete // LoadTests. loadTestGetter clientset.LoadTestGetter + // podsGetter has a method to return a PodInterface which provide access + // to work with Pod resources. + podsGetter corev1types.PodsGetter // afterInterval stops for a set time interval before returning. // It is used to set a polling interval. afterInterval func() @@ -50,18 +54,19 @@ type Runner struct { // deleteSuccessfulTests determines whether tests that terminate without // errors should be deleted immediately. deleteSuccessfulTests bool - // logSaver saves pod log files - logSaver *LogSaver + // logURLPrefix is a prefix to be added to log path urls. + logURLPrefix string } // NewRunner creates a new Runner object. -func NewRunner(loadTestGetter clientset.LoadTestGetter, afterInterval func(), retries uint, deleteSuccessfulTests bool, logSaver *LogSaver) *Runner { +func NewRunner(loadTestGetter clientset.LoadTestGetter, podsGetter corev1types.PodsGetter, afterInterval func(), retries uint, deleteSuccessfulTests bool, logURLPrefix string) *Runner { return &Runner{ loadTestGetter: loadTestGetter, + podsGetter: podsGetter, afterInterval: afterInterval, retries: retries, deleteSuccessfulTests: deleteSuccessfulTests, - logSaver: logSaver, + logURLPrefix: logURLPrefix, } } @@ -141,14 +146,23 @@ func (r *Runner) runTest(ctx context.Context, config *grpcv1.LoadTest, reporter status = statusString(config) switch { case loadTest.Status.State.IsTerminated(): - savedLogs, err := r.logSaver.SavePodLogs(ctx, loadTest, outputDir) + pods, err := GetTestPods(ctx, loadTest, r.podsGetter) if err != nil { - reporter.Error("Could not save pod logs: %s", err) + reporter.Error("Could not list all pods: %v", err) + } + savedLogInfos, err := SaveAllLogs(ctx, loadTest, r.podsGetter, pods, outputDir) + if err != nil { + reporter.Error("Could not save pod logs: %v", err) } reporter.AddProperty("name", loadTest.Name) - for property, value := range savedLogs.GenerateProperties(loadTest) { + for property, value := range PodNameProperties(pods, loadTest.Name, "pod") { + reporter.AddProperty(property, value) + } + + for property, value := range PodLogProperties(savedLogInfos, r.logURLPrefix, "pod") { reporter.AddProperty(property, value) } + if status != "Succeeded" { reporter.Error("Test failed with reason %q: %v", loadTest.Status.Reason, loadTest.Status.Message) } else {