diff --git a/e2e/node.go b/e2e/node.go index 9a4546af226f..992c6c371a4f 100644 --- a/e2e/node.go +++ b/e2e/node.go @@ -2,7 +2,9 @@ package e2e import ( "context" + "errors" + core "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" @@ -42,3 +44,20 @@ func checkNodeHasLabel(c kubernetes.Interface, labelKey, labelValue string) erro } return nil } + +// List all nodes in the cluster (we have one), and return the IP-address. +// Possibly need to add a selector, pick the node where a Pod is running? +func getKubeletIP(c kubernetes.Interface) (string, error) { + nodes, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return "", err + } + + for _, address := range nodes.Items[0].Status.Addresses { + if address.Type == core.NodeInternalIP { + return address.Address, nil + } + } + + return "", errors.New("could not find internal IP for node") +} diff --git a/e2e/pvc.go b/e2e/pvc.go index 64dc866f0d19..3dd7c83f98a2 100644 --- a/e2e/pvc.go +++ b/e2e/pvc.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "time" v1 "k8s.io/api/core/v1" @@ -244,3 +245,44 @@ func checkPVSelectorValuesForPVC(f *framework.Framework, pvc *v1.PersistentVolum } return nil } + +func getMetricsForPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, t int) error { + kubelet, err := getKubeletIP(f.ClientSet) + if err != nil { + return err + } + + // kubelet needs to be started with --read-only-port=10255 + cmd := fmt.Sprintf("curl --silent 'http://%s:10255/metrics'", kubelet) + + // retry as kubelet does not immediately have the metrics available + timeout := time.Duration(t) * time.Minute + return wait.PollImmediate(poll, timeout, func() (bool, error) { + stdOut, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace) + if err != nil { + e2elog.Logf("failed to get metrics for pvc %q (%v): %v", pvc.Name, err, stdErr) + return false, nil + } + if stdOut == "" { + e2elog.Logf("no metrics received from kublet on IP %s", kubelet) + return false, nil + } + + namespace := fmt.Sprintf("namespace=%q", pvc.Namespace) + name := fmt.Sprintf("persistentvolumeclaim=%q", pvc.Name) + + for _, line := range strings.Split(stdOut, "\n") { + if !strings.HasPrefix(line, "kubelet_volume_stats_") { + continue + } + if strings.Contains(line, namespace) && strings.Contains(line, name) { + // TODO: validate metrics if possible + e2elog.Logf("found metrics for pvc %s/%s: %s", pvc.Namespace, pvc.Name, line) + return true, nil + } + } + + e2elog.Logf("no metrics found for pvc %s/%s", pvc.Namespace, pvc.Name) + return false, nil + }) +} diff --git a/e2e/utils.go b/e2e/utils.go index 6d7640d2d2f3..f822e285cd66 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -302,6 +302,19 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error { if stdErr != "" { return fmt.Errorf("failed to touch a file as non-root user %v", stdErr) } + + // metrics for BlockMode was added in Kubernetes 1.22 + isBlockMode := false + if pvc.Spec.VolumeMode != nil { + isBlockMode = (*pvc.Spec.VolumeMode == v1.PersistentVolumeBlock) + } + if !isBlockMode || k8sVersionGreaterEquals(f.ClientSet, 1, 22) { + err = getMetricsForPVC(f, pvc, deployTimeout) + if err != nil { + return err + } + } + err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) if err != nil { return err diff --git a/scripts/minikube.sh b/scripts/minikube.sh index 23175d4df5b9..8e0abd2977b0 100755 --- a/scripts/minikube.sh +++ b/scripts/minikube.sh @@ -165,7 +165,7 @@ fi K8S_FEATURE_GATES=${K8S_FEATURE_GATES:-"ExpandCSIVolumes=true"} #extra-config for kube https://minikube.sigs.k8s.io/docs/reference/configuration/kubernetes/ -EXTRA_CONFIG=${EXTRA_CONFIG:-"--extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy"} +EXTRA_CONFIG_PSP="--extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy" # kubelet.resolv-conf needs to point to a file, not a symlink # the default minikube VM has /etc/resolv.conf -> /run/systemd/resolve/resolv.conf @@ -182,6 +182,9 @@ EXTRA_CONFIG="${EXTRA_CONFIG} --extra-config=kubelet.resolv-conf=${RESOLV_CONF}" #extra Rook configuration ROOK_BLOCK_POOL_NAME=${ROOK_BLOCK_POOL_NAME:-"newrbdpool"} +# enable read-only anonymous access to kubelet metrics +EXTRA_CONFIG="${EXTRA_CONFIG} --extra-config=kubelet.read-only-port=10255" + if [[ "${KUBE_VERSION}" == "latest" ]]; then # update the version string from latest with the real version KUBE_VERSION=$(curl -L https://storage.googleapis.com/kubernetes-release/release/stable.txt 2> /dev/null) @@ -205,16 +208,16 @@ up) if minikube_supports_psp; then enable_psp # shellcheck disable=SC2086 - ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" --cni="${CNI}" ${EXTRA_CONFIG} + ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" --cni="${CNI}" ${EXTRA_CONFIG} ${EXTRA_CONFIG_PSP} else # This is a workaround to fix psp issues in minikube >1.6.2 and <1.11.0 # shellcheck disable=SC2086 - ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" --cni="${CNI}" + ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" --cni="${CNI}" ${EXTRA_CONFIG} DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" ${minikube} kubectl -- apply -f "$DIR"/psp.yaml ${minikube} stop # shellcheck disable=SC2086 - ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" --cni="${CNI}" ${EXTRA_CONFIG} + ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" --cni="${CNI}" ${EXTRA_CONFIG} ${EXTRA_CONFIG_PSP} fi # create a link so the default dataDirHostPath will work for this