Added pre & post checks around exec pods (#16)

* feat: Pre/post checks for Exec pods * feat: Remove reduntant clientset inits
Kavinraja-G · Sep 23, 2023 · c54584f · c54584f
1 parent 57a676e
commit c54584f
Show file tree

Hide file tree

Showing 8 changed files with 88 additions and 66 deletions.
diff --git a/cmd/kubectl-ng.go b/cmd/kubectl-ng.go
@@ -2,8 +2,14 @@ package cmd
 
 import (
 	"github.com/Kavinraja-G/node-gizmo/pkg/cmd"
+	"github.com/Kavinraja-G/node-gizmo/utils"
 )
 
+func init() {
+	// inits the clientset and other generic configs if any
+	utils.InitConfig()
+}
+
 // Execute drives the root 'nodegizmo' command
 func Execute() error {
 	root := cmd.NewCmdRoot()

diff --git a/pkg/cmd/nodepool/nodepool.go b/pkg/cmd/nodepool/nodepool.go
@@ -2,13 +2,12 @@ package nodepool
 
 import (
 	"context"
-	"log"
+	"github.com/Kavinraja-G/node-gizmo/utils"
 	"strings"
 
 	"github.com/Kavinraja-G/node-gizmo/pkg/outputs"
 
 	"github.com/Kavinraja-G/node-gizmo/pkg"
-	"github.com/Kavinraja-G/node-gizmo/pkg/auth"
 	"github.com/spf13/cobra"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -31,13 +30,11 @@ func NewCmdNodepoolInfo() *cobra.Command {
 func showNodePoolInfo(cmd *cobra.Command, args []string) error {
 	var genericNodepoolInfos = make(map[string]pkg.GenericNodepoolInfo)
 
-	clientset, err := auth.K8sAuth()
+	nodes, err := utils.Cfg.Clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
-		log.Fatalf("Error while authenticating to kubernetes: %v", err)
 		return err
 	}
 
-	nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
 	for _, node := range nodes.Items {
 		var genericNodepoolInfo pkg.GenericNodepoolInfo
 

diff --git a/pkg/cmd/nodes/capacity.go b/pkg/cmd/nodes/capacity.go
@@ -2,13 +2,10 @@ package nodes
 
 import (
 	"context"
-	"log"
-
-	"github.com/Kavinraja-G/node-gizmo/pkg/outputs"
-	"github.com/Kavinraja-G/node-gizmo/pkg/utils"
+	"github.com/Kavinraja-G/node-gizmo/utils"
 
 	"github.com/Kavinraja-G/node-gizmo/pkg"
-	"github.com/Kavinraja-G/node-gizmo/pkg/auth"
+	"github.com/Kavinraja-G/node-gizmo/pkg/outputs"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -31,13 +28,7 @@ func NewCmdNodeCapacityInfo() *cobra.Command {
 func showNodeCapacities(cmd *cobra.Command, args []string) error {
 	var nodeCapacityInfo []pkg.NodeCapacities
 
-	clientset, err := auth.K8sAuth()
-	if err != nil {
-		log.Fatalf("Error while authenticating to kubernetes: %v", err)
-		return err
-	}
-
-	nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+	nodes, err := utils.Cfg.Clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
 		return err
 	}

diff --git a/pkg/cmd/nodes/exec.go b/pkg/cmd/nodes/exec.go
@@ -4,13 +4,17 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/Kavinraja-G/node-gizmo/utils"
 	"log"
 	"os"
+	"time"
+
+	k8errors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
 
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/remotecommand"
 
-	"github.com/Kavinraja-G/node-gizmo/pkg/auth"
 	"github.com/spf13/cobra"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,9 +38,17 @@ func NewCmdNodeExec() *cobra.Command {
 			if len(args) < 1 {
 				return errors.New("please provide a nodeName to exec")
 			}
-			if !isValidNode(args[0]) {
+			nodeName := args[0]
+
+			if !isValidNode(nodeName) {
 				return errors.New(fmt.Sprintf("%v is not a valid node", args[0]))
 			}
+
+			err := createExecPodInTargetedNode(nodeName)
+			if err != nil {
+				return err
+			}
+
 			return execIntoNode(cmd, args[0])
 		},
 		PostRunE: func(cmd *cobra.Command, args []string) error {
@@ -48,13 +60,7 @@ func NewCmdNodeExec() *cobra.Command {
 
 // isValidNode validates the given node is available in the cluster or not
 func isValidNode(nodeName string) bool {
-	//TODO: Remove repeated clientset initialisation
-	clientset, err := auth.K8sAuth()
-	if err != nil {
-		log.Fatalf("Error while authenticating to kubernetes: %v", err)
-	}
-
-	nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+	nodes, err := utils.Cfg.Clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
 		log.Fatalf("Error while listing the nodes in the cluster: %v", err)
 	}
@@ -111,34 +117,56 @@ func createExecPodInTargetedNode(nodeName string) error {
 		},
 	}
 
-	//TODO: Remove repeated clientset initialisation
-	clientset, err := auth.K8sAuth()
+	_, err := utils.Cfg.Clientset.CoreV1().Pods(nodeshellPodNamespace).Create(context.TODO(), pod, metav1.CreateOptions{})
 	if err != nil {
-		log.Fatalf("Error while authenticating to kubernetes: %v", err)
+		// validates if the pod already exists
+		if k8errors.IsAlreadyExists(err) {
+			return nil
+		}
 		return err
 	}
 
-	_, err = clientset.CoreV1().Pods(nodeshellPodNamespace).Create(context.TODO(), pod, metav1.CreateOptions{})
+	// wait for exec pod to get RUNNING
+	checkExecPodRunningStatus := func() (bool, error) {
+		pod, err := utils.Cfg.Clientset.CoreV1().Pods(nodeshellPodNamespace).Get(context.TODO(), nodeshellPodName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		if pod.Status.Phase == corev1.PodRunning {
+			return true, nil
+		}
+
+		return false, nil
+	}
+
+	backoff := wait.Backoff{
+		Steps:    5,                // Number of retry steps.
+		Duration: 10 * time.Second, // Initial backoff duration.
+		Factor:   1.0,              // Multiplier for each step's duration.
+		Jitter:   0.1,              // Jitter to randomize the duration slightly.
+	}
+
+	// waits exponentially for exec pod to be RUNNING
+	startTime := time.Now()
+	err = wait.ExponentialBackoff(backoff, func() (bool, error) {
+		elapsedWaitTime := time.Since(startTime)
+		log.Printf("Waiting for exec pod %s to be RUNNING. Elapsed time: %v", nodeshellPodName, elapsedWaitTime)
+		return checkExecPodRunningStatus()
+	})
+	if err != nil {
+		log.Fatalf("exec pod did not reached the RUNNING state: %v", err)
+	}
+
 	return err
 }
 
 // execIntoNode is the driver function used to exec into the nsenter pod deployed in the targeted node
 func execIntoNode(cmd *cobra.Command, nodeName string) error {
 	var nodeshellPodName = nodeshellPodNamePrefix + nodeName
-	err := createExecPodInTargetedNode(nodeName)
-	if err != nil {
-		return err
-	}
-
-	//TODO: Remove repeated clientset initialisation
-	clientset, err := auth.K8sAuth()
-	if err != nil {
-		log.Fatalf("Error while authenticating to kubernetes: %v", err)
-		return err
-	}
 
 	var podExecCmd = []string{"sh", "-c", "(bash || ash || sh)"}
-	req := clientset.CoreV1().RESTClient().Post().Resource("pods").Name(nodeshellPodName).Namespace(nodeshellPodNamespace).SubResource("exec")
+	req := utils.Cfg.Clientset.CoreV1().RESTClient().Post().Resource("pods").Name(nodeshellPodName).Namespace(nodeshellPodNamespace).SubResource("exec")
 	opts := &corev1.PodExecOptions{
 		Command: podExecCmd,
 		Stdin:   true,
@@ -150,7 +178,7 @@ func execIntoNode(cmd *cobra.Command, nodeName string) error {
 	req.VersionedParams(opts, scheme.ParameterCodec)
 
 	//TODO: Check if there is any way we can fetch the config from the clientset itself
-	k8sConfig, err := auth.GetKubeConfig()
+	k8sConfig, err := utils.GetKubeConfig()
 	if err != nil {
 		log.Fatalf("Error while getting Kubeconfig: %v", err)
 		return err
@@ -176,14 +204,7 @@ func execIntoNode(cmd *cobra.Command, nodeName string) error {
 func cleanUpNodeshellPods(cmd *cobra.Command, nodeName string) error {
 	var nodeshellPodName = nodeshellPodNamePrefix + nodeName
 
-	//TODO: Remove repeated clientset initialisation
-	clientset, err := auth.K8sAuth()
-	if err != nil {
-		log.Fatalf("Error while authenticating to kubernetes: %v", err)
-		return err
-	}
-
-	err = clientset.CoreV1().Pods(nodeshellPodNamespace).Delete(context.TODO(), nodeshellPodName, metav1.DeleteOptions{})
+	err := utils.Cfg.Clientset.CoreV1().Pods(nodeshellPodNamespace).Delete(context.TODO(), nodeshellPodName, metav1.DeleteOptions{})
 	if err != nil {
 		log.Fatalf("Error while creating the nodeshell pod: %v", err)
 		return err

diff --git a/pkg/cmd/nodes/nodes.go b/pkg/cmd/nodes/nodes.go
@@ -3,13 +3,12 @@ package nodes
 import (
 	"context"
 	"fmt"
-	"log"
+	"github.com/Kavinraja-G/node-gizmo/utils"
 	"strings"
 
 	"github.com/Kavinraja-G/node-gizmo/pkg/outputs"
 
 	"github.com/Kavinraja-G/node-gizmo/pkg"
-	"github.com/Kavinraja-G/node-gizmo/pkg/auth"
 	"github.com/spf13/cobra"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -53,13 +52,11 @@ func showNodeInfo(cmd *cobra.Command, args []string) error {
 		ShowNodeTopologyInfo: showNodeTopologyInfo,
 	}
 
-	clientset, err := auth.K8sAuth()
+	nodes, err := utils.Cfg.Clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
-		log.Fatalf("Error while authenticating to kubernetes: %v", err)
 		return err
 	}
 
-	nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
 	for _, node := range nodes.Items {
 		genericNodeInfo := pkg.GenericNodeInfo{
 			NodeName:   node.Name,

diff --git a/pkg/auth/k8s.go → utils/config.go b/pkg/auth/k8s.go → utils/config.go
@@ -1,35 +1,36 @@
-package auth
+package utils
 
 import (
 	"log"
 	"path/filepath"
 
-	"github.com/Kavinraja-G/node-gizmo/pkg/utils"
-
+	k8s "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/util/homedir"
+)
 
-	k8s "k8s.io/client-go/kubernetes"
+type Config struct {
+	Clientset *k8s.Clientset
+}
 
-	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
-	"k8s.io/client-go/tools/clientcmd"
-)
+var Cfg Config
 
 // GetKubeConfig is used to fetch the kubeConfig based on the KUBECONFIG env or '~/.kube/config' location
 func GetKubeConfig() (*rest.Config, error) {
 	var kubeConfigPath string
 	if home := homedir.HomeDir(); home != "" {
 		kubeConfigPath = filepath.Join(home, ".kube", "config")
 	} else {
-		kubeConfigPath = utils.GetEnv("KUBECONFIG", "~/.kube/config")
+		kubeConfigPath = GetEnv("KUBECONFIG", "~/.kube/config")
 	}
 
 	k8sConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
 	return k8sConfig, err
 }
 
-// K8sAuth is used to get the Kubernetes clientset from the config
-func K8sAuth() (*k8s.Clientset, error) {
+// k8sAuth is used to get the Kubernetes clientset from the config
+func k8sAuth() (*k8s.Clientset, error) {
 	k8sConfig, err := GetKubeConfig()
 	clientset, err := k8s.NewForConfig(k8sConfig)
 	if err != nil {
@@ -38,3 +39,12 @@ func K8sAuth() (*k8s.Clientset, error) {
 
 	return clientset, err
 }
+
+// InitConfig initiates a kubernetes clientset & other generic configs with the current context
+func InitConfig() {
+	var err error
+	Cfg.Clientset, err = k8sAuth()
+	if err != nil {
+		log.Fatalf("Error while authenticating to kubernetes: %v", err)
+	}
+}
diff --git a/pkg/utils/utils.go → utils/utils.go b/pkg/utils/utils.go → utils/utils.go
diff --git a/pkg/utils/utils_test.go → utils/utils_test.go b/pkg/utils/utils_test.go → utils/utils_test.go