Dump CSI logs if E2E fails

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

e2e/log.go

@@ -0,0 +1,64 @@
+package e2e
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
+)
+
+func logsCSIPods(label string, c clientset.Interface) {
+	ns := "default"
+	opt := metav1.ListOptions{
+		LabelSelector: label,
+	}
+	podList, err := c.CoreV1().Pods(ns).List(opt)
+	if err != nil {
+		e2elog.Logf("failed to list pods with selector %s %v", label, err)
+		return
+	}
+
+	e2elog.Logf("Running kubectl logs on ceph-csi containers in %v", ns)
+	for i := range podList.Items {
+		kubectlLogPod(c, &podList.Items[i])
+	}
+}
+
+func kubectlLogPod(c clientset.Interface, pod *v1.Pod) {
+	container := pod.Spec.Containers
+	for i := range container {
+		logs, err := framework.GetPodLogs(c, pod.Namespace, pod.Name, container[i].Name)
+		if err != nil {
+			logs, err = getPreviousPodLogs(c, pod.Namespace, pod.Name, container[i].Name)
+			if err != nil {
+				e2elog.Logf("Failed to get logs of pod %v, container %v, err: %v", pod.Name, container[i].Name, err)
+			}
+		}
+		e2elog.Logf("Logs of %v/%v:%v on node %v\n", pod.Namespace, pod.Name, container[i].Name, pod.Spec.NodeName)
+		e2elog.Logf("STARTLOG\n\n%s\n\nENDLOG for container %v:%v:%v", logs, pod.Namespace, pod.Name, container[i].Name)
+	}
+
+}
+
+func getPreviousPodLogs(c clientset.Interface, namespace, podName, containerName string) (string, error) {
+	logs, err := c.CoreV1().RESTClient().Get().
+		Resource("pods").
+		Namespace(namespace).
+		Name(podName).SubResource("log").
+		Param("container", containerName).
+		Param("previous", strconv.FormatBool(true)).
+		Do().
+		Raw()
+	if err != nil {
+		return "", err
+	}
+	if strings.Contains(string(logs), "Internal Error") {
+		return "", fmt.Errorf("fetched log contains \"Internal Error\": %q", string(logs))
+	}
+	return string(logs), err
+}

e2e/rbd.go

@@ -74,8 +74,10 @@ func deleteRBDPlugin() {
 
 var _ = Describe("RBD", func() {
 	f := framework.NewDefaultFramework("rbd")
+	var c clientset.Interface
 	// deploy RBD CSI
 	BeforeEach(func() {
+		c = f.ClientSet
 		updaterbdDirPath(f.ClientSet)
 		createRBDPool()
 		createConfigMap(rbdDirPath, f.ClientSet, f)
@@ -86,6 +88,12 @@ var _ = Describe("RBD", func() {
 	})
 
 	AfterEach(func() {
+		if CurrentGinkgoTestDescription().Failed {
+			// log provisoner
+			logsCSIPods("app=csi-rbdplugin-provisioner", c)
+			// log node plugin
+			logsCSIPods("app=csi-rbdplugin", c)
+		}
 		deleteRBDPlugin()
 		deleteConfiMap(rbdDirPath)
 		deleteRBDPool()
@@ -141,7 +149,7 @@ var _ = Describe("RBD", func() {
 
 			By("create a PVC clone and Bind it to an app", func() {
 				createRBDSnapshotClass(f)
-				validateCloneFromSnapshot(pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, 10, true, false, f)
+				validateCloneFromSnapshot(pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, 1, true, false, f)
 			})
 
 			// skipped raw pvc test in travis

e2e/snapshot.go

@@ -83,7 +83,7 @@ func validateCloneFromSnapshot(pvcPath, appPath, snapshotPath, pvcClonePath, app
 		}
 		// check any stale snapshots present on parent volume
 		if len(snapList) != 0 {
-			e2elog.Logf("stale backend snapshot count = %v %d", len(snapList))
+			e2elog.Logf("stale snapshot count = %v %d on parent image %s", snapList, len(snapList), images[0])
 			Fail("validate backend snapshot failed")
 		}
 
@@ -93,7 +93,7 @@ func validateCloneFromSnapshot(pvcPath, appPath, snapshotPath, pvcClonePath, app
 		images = listRBDImages(f)
 		count := 1 + total
 		if len(images) != count {
-			e2elog.Logf("backend image creation not matching pvc count, image count = %d pvc count %d", len(images), count)
+			e2elog.Logf("backend image creation not matching pvc count, image count = %d pvc count %d : image %v", len(images), count, images)
 			Fail("validate multiple snapshot failed")
 		}
 	}

e2e/utils.go

@@ -632,7 +632,7 @@ func validatePVCAndAppBinding(pvcPath, appPath string, totalCount int, validateR
 		// validate created backend rbd images
 		images := listRBDImages(f)
 		if len(images) != totalCount {
-			e2elog.Logf("backend image creation not matching pvc count, image count = % pvc count %d", len(images), totalCount)
+			e2elog.Logf("backend image creation not matching pvc count, image count = % pvc count %d : images", len(images), totalCount, images)
 			Fail("validate multiple pvc failed")
 		}
 	}