Timeout and reconcile when checking API server connectivity (#1943)

pkg/k8sapi/k8sutils.go

@@ -2,10 +2,13 @@ package k8sapi
 
 import (
 	"fmt"
+	"os"
+	"time"
 
 	eniconfigscheme "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
@@ -85,15 +88,25 @@ func CheckAPIServerConnectivity() error {
 	if err != nil {
 		return err
 	}
-	clientSet, _ := kubernetes.NewForConfig(restCfg)
-
-	log.Infof("Testing communication with server")
-	version, err := clientSet.Discovery().ServerVersion()
+	restCfg.Timeout = 5 * time.Second
+	clientSet, err := kubernetes.NewForConfig(restCfg)
 	if err != nil {
-		return fmt.Errorf("error communicating with apiserver: %v", err)
+		return fmt.Errorf("creating kube config, %w", err)
 	}
-	log.Infof("Successful communication with the Cluster! Cluster Version is: v%s.%s. git version: %s. git tree state: %s. commit: %s. platform: %s",
-		version.Major, version.Minor, version.GitVersion, version.GitTreeState, version.GitCommit, version.Platform)
-
-	return nil
+	log.Infof("Testing communication with server")
+	// Reconcile the API server query after waiting for a second, as the request
+	// times out in one second if it fails to connect to the server
+	return wait.PollInfinite(2*time.Second, func() (bool, error) {
+		version, err := clientSet.Discovery().ServerVersion()
+		if err != nil {
+			// When times out return no error, so the PollInfinite will retry with the given interval
+			if os.IsTimeout(err) {
+				return false, nil
+			}
+			return false, fmt.Errorf("error communicating with apiserver: %v", err)
+		}
+		log.Infof("Successful communication with the Cluster! Cluster Version is: v%s.%s. git version: %s. git tree state: %s. commit: %s. platform: %s",
+			version.Major, version.Minor, version.GitVersion, version.GitTreeState, version.GitCommit, version.Platform)
+		return true, nil
+	})
 }