From 50bcbfcbf346e9c45499b2899eb31b62d7f55fa0 Mon Sep 17 00:00:00 2001
From: Nikhil Malik <nikhil.malik@netlox.io>
Date: Tue, 5 Mar 2024 14:25:27 +0900
Subject: [PATCH] Upgraded K8s cicd to k8s v1.29.2

---
 cicd/k8s-calico-ipvs/node_scripts/common.sh   | 16 ++++++------
 cicd/k8s-calico-ipvs/yaml/kubeadm-config.yaml |  2 +-
 cicd/k8s-calico-ipvs/yaml/settings.yaml       |  3 ++-
 cicd/k8s-calico-ipvs2-ha-ka-sync/config.sh    | 25 +++++++++++++++++++
 .../host_validation.sh                        |  2 +-
 .../node_scripts/common.sh                    | 16 ++++++------
 .../node_scripts/loxilb1.sh                   |  2 +-
 .../node_scripts/loxilb2.sh                   |  2 +-
 .../yaml/kubeadm-config.yaml                  |  2 +-
 .../yaml/settings.yaml                        |  3 ++-
 cicd/k8s-calico-ipvs2/config.sh               | 25 +++++++++++++++++++
 cicd/k8s-calico-ipvs2/node_scripts/common.sh  | 16 ++++++------
 cicd/k8s-calico-ipvs2/validation.sh           |  2 +-
 .../k8s-calico-ipvs2/yaml/kubeadm-config.yaml |  2 +-
 cicd/k8s-calico-ipvs2/yaml/settings.yaml      |  3 ++-
 .../node_scripts/common.sh                    | 16 ++++++------
 .../yaml/kubeadm-config.yaml                  |  2 +-
 cicd/k8s-calico-ipvs3-ha/yaml/settings.yaml   |  3 ++-
 cicd/k8s-calico-ipvs3/node_scripts/common.sh  | 16 ++++++------
 .../k8s-calico-ipvs3/yaml/kubeadm-config.yaml |  2 +-
 cicd/k8s-calico-ipvs3/yaml/settings.yaml      |  3 ++-
 cicd/k8s-calico/config.sh                     |  1 +
 cicd/k8s-calico/node_scripts/common.sh        |  4 +--
 23 files changed, 112 insertions(+), 56 deletions(-)

diff --git a/cicd/k8s-calico-ipvs/node_scripts/common.sh b/cicd/k8s-calico-ipvs/node_scripts/common.sh
index b8634194f..654da6b7c 100644
--- a/cicd/k8s-calico-ipvs/node_scripts/common.sh
+++ b/cicd/k8s-calico-ipvs/node_scripts/common.sh
@@ -26,7 +26,7 @@ sudo apt-get update -y
 # Install CRI-O Runtime
 
 VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')"
-
+CRIO_VERSION=1.27
 # Create the .conf file to load the modules at bootup
 cat <<EOF | sudo tee /etc/modules-load.d/crio.conf
 overlay
@@ -57,11 +57,11 @@ sudo sysctl --system
 cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
 deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /
 EOF
-cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list
-deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /
+cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$CRIO_VERSION.list
+deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/ /
 EOF
 
-curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
+curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
 curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
 
 sudo apt-get update
@@ -76,12 +76,12 @@ sudo systemctl enable crio --now
 echo "CRI runtime installed successfully"
 
 sudo apt-get update
-sudo apt-get install -y apt-transport-https ca-certificates curl
-curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg
+sudo apt-get install -y apt-transport-https ca-certificates curl gpg
+curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v'$VERSION'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
 
-echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
 sudo apt-get update -y
-sudo apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION"
+sudo apt-get install -y kubelet kubectl kubeadm
 sudo apt-get update -y
 sudo apt-get install -y jq
 sudo apt-get install -y ipvsadm
diff --git a/cicd/k8s-calico-ipvs/yaml/kubeadm-config.yaml b/cicd/k8s-calico-ipvs/yaml/kubeadm-config.yaml
index 4acccfac8..79d05055f 100644
--- a/cicd/k8s-calico-ipvs/yaml/kubeadm-config.yaml
+++ b/cicd/k8s-calico-ipvs/yaml/kubeadm-config.yaml
@@ -30,7 +30,7 @@ etcd:
   local:
     dataDir: /var/lib/etcd
 imageRepository: registry.k8s.io
-kubernetesVersion: v1.27.5
+kubernetesVersion: v1.29.2
 networking:
   dnsDomain: cluster.local
   podSubnet: 172.16.1.0/16
diff --git a/cicd/k8s-calico-ipvs/yaml/settings.yaml b/cicd/k8s-calico-ipvs/yaml/settings.yaml
index ffa7fe692..b24cbda3d 100644
--- a/cicd/k8s-calico-ipvs/yaml/settings.yaml
+++ b/cicd/k8s-calico-ipvs/yaml/settings.yaml
@@ -38,7 +38,8 @@ software:
      version: 0.7.1
   cluster:
     box: bento/ubuntu-22.04
+    version: 202401.31.0
   calico: 3.26.0
   # To skip the dashboard installation, set its version to an empty value or comment it out:
-  kubernetes: 1.27.1-00
+  kubernetes: 1.29.2
   os: xUbuntu_22.04
diff --git a/cicd/k8s-calico-ipvs2-ha-ka-sync/config.sh b/cicd/k8s-calico-ipvs2-ha-ka-sync/config.sh
index 2d0399dd9..58bbbe7cc 100755
--- a/cicd/k8s-calico-ipvs2-ha-ka-sync/config.sh
+++ b/cicd/k8s-calico-ipvs2-ha-ka-sync/config.sh
@@ -36,3 +36,28 @@ sudo sysctl net.ipv4.conf.vboxnet1.arp_accept=1
 vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/tcp_fullnat.yml' 2> /dev/null
 vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp_fullnat.yml' 2> /dev/null
 #vagrant ssh master -c 'kubectl apply -f https://raw.githubusercontent.com/loxilb-io/kube-loxilb/main/manifest/mesh/loxilb-mesh.yml' 2> /dev/null
+for((i=1; i<=60; i++))
+do
+    fin=1
+    pods=$(vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null | grep -v "NAMESPACE")
+
+    while IFS= read -a pods; do
+        read -a pod <<< "$pods"
+        if [[ ${pod[3]} != *"Running"* ]]; then
+            echo "${pod[1]} is not UP yet"
+            fin=0
+        fi
+    done <<< "$pods"
+    if [ $fin == 1 ];
+    then
+        echo "Cluster is ready"
+        break;
+    fi
+    echo "Will try after 10s"
+    sleep 10
+done
+
+if [[ $fin == 0 ]]; then
+    echo "Cluster is not ready"
+    exit 1
+fi
diff --git a/cicd/k8s-calico-ipvs2-ha-ka-sync/host_validation.sh b/cicd/k8s-calico-ipvs2-ha-ka-sync/host_validation.sh
index 371ee3827..3eeb9aadb 100755
--- a/cicd/k8s-calico-ipvs2-ha-ka-sync/host_validation.sh
+++ b/cicd/k8s-calico-ipvs2-ha-ka-sync/host_validation.sh
@@ -17,6 +17,6 @@ echo "Testing Service"
 echo "*********************************************"
 
 # iperf client accessing fullnat service
-stdbuf -oL nohup iperf -c 192.168.80.5 -p 56002 -t 60 -i 1 -b 100M &> iperff.out &
+stdbuf -oL nohup iperf -c 192.168.80.5 -p 56002 -t 100 -i 1 -b 100M &> iperff.out &
 echo "iperf client started"
 echo "phase-1 done"
diff --git a/cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/common.sh b/cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/common.sh
index 63b64c101..f0fe98697 100644
--- a/cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/common.sh
+++ b/cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/common.sh
@@ -29,7 +29,7 @@ sudo apt-get update -y
 # Install CRI-O Runtime
 
 VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')"
-
+CRIO_VERSION=1.27
 # Create the .conf file to load the modules at bootup
 cat <<EOF | sudo tee /etc/modules-load.d/crio.conf
 overlay
@@ -60,11 +60,11 @@ sudo sysctl --system
 cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
 deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /
 EOF
-cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list
-deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /
+cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$CRIO_VERSION.list
+deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/ /
 EOF
 
-curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
+curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
 curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
 
 sudo apt-get update
@@ -79,12 +79,12 @@ sudo systemctl enable crio --now
 echo "CRI runtime installed successfully"
 
 sudo apt-get update
-sudo apt-get install -y apt-transport-https ca-certificates curl
-curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg
+sudo apt-get install -y apt-transport-https ca-certificates curl gpg
+curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v'$VERSION'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
 
-echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
 sudo apt-get update -y
-sudo apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION"
+sudo apt-get install -y kubelet kubectl kubeadm
 sudo apt-get update -y
 sudo apt-get install -y jq
 sudo apt-get install -y ipvsadm
diff --git a/cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/loxilb1.sh b/cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/loxilb1.sh
index 1a99395b1..e9dac5b3d 100644
--- a/cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/loxilb1.sh
+++ b/cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/loxilb1.sh
@@ -6,4 +6,4 @@ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
 add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
 apt-get update
 apt-get install -y docker-ce
-docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest --cluster=192.168.80.253 --self=0 --ka=192.168.80.253
+docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest --cluster=192.168.80.253 --self=0 --ka=192.168.80.253:192.168.80.252
diff --git a/cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/loxilb2.sh b/cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/loxilb2.sh
index c09d17bc0..6df1db273 100644
--- a/cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/loxilb2.sh
+++ b/cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/loxilb2.sh
@@ -6,4 +6,4 @@ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
 add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
 apt-get update
 apt-get install -y docker-ce
-docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest --cluster=192.168.80.252 --self=1 --ka=192.168.80.252
+docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest --cluster=192.168.80.252 --self=1 --ka=192.168.80.252:192.168.80.253
diff --git a/cicd/k8s-calico-ipvs2-ha-ka-sync/yaml/kubeadm-config.yaml b/cicd/k8s-calico-ipvs2-ha-ka-sync/yaml/kubeadm-config.yaml
index 31afe601c..245a62553 100644
--- a/cicd/k8s-calico-ipvs2-ha-ka-sync/yaml/kubeadm-config.yaml
+++ b/cicd/k8s-calico-ipvs2-ha-ka-sync/yaml/kubeadm-config.yaml
@@ -30,7 +30,7 @@ etcd:
   local:
     dataDir: /var/lib/etcd
 imageRepository: registry.k8s.io
-kubernetesVersion: v1.27.5
+kubernetesVersion: v1.29.2
 networking:
   dnsDomain: cluster.local
   podSubnet: 172.16.1.0/16
diff --git a/cicd/k8s-calico-ipvs2-ha-ka-sync/yaml/settings.yaml b/cicd/k8s-calico-ipvs2-ha-ka-sync/yaml/settings.yaml
index 11a3ba42d..172ab421f 100644
--- a/cicd/k8s-calico-ipvs2-ha-ka-sync/yaml/settings.yaml
+++ b/cicd/k8s-calico-ipvs2-ha-ka-sync/yaml/settings.yaml
@@ -36,7 +36,8 @@ software:
      version: 0.7.1
   cluster:
     box: bento/ubuntu-22.04
+    version: 202401.31.0
   calico: 3.26.0
   # To skip the dashboard installation, set its version to an empty value or comment it out:
-  kubernetes: 1.27.1-00
+  kubernetes: 1.29.2
   os: xUbuntu_22.04
diff --git a/cicd/k8s-calico-ipvs2/config.sh b/cicd/k8s-calico-ipvs2/config.sh
index ed986fbfb..071830cad 100755
--- a/cicd/k8s-calico-ipvs2/config.sh
+++ b/cicd/k8s-calico-ipvs2/config.sh
@@ -35,3 +35,28 @@ sudo sysctl net.ipv4.conf.vboxnet1.arp_accept=1
 #Create fullnat Service
 vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/tcp_fullnat.yml' 2> /dev/null
 vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/udp_fullnat.yml' 2> /dev/null
+for((i=1; i<=60; i++))
+do
+    fin=1
+    pods=$(vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null | grep -v "NAMESPACE")
+
+    while IFS= read -a pods; do
+        read -a pod <<< "$pods"
+        if [[ ${pod[3]} != *"Running"* ]]; then
+            echo "${pod[1]} is not UP yet"
+            fin=0
+        fi
+    done <<< "$pods"
+    if [ $fin == 1 ];
+    then
+        echo "Cluster is ready"
+        break;
+    fi
+    echo "Will try after 10s"
+    sleep 10
+done
+
+if [[ $fin == 0 ]]; then
+    echo "Cluster is not ready"
+    exit 1
+fi
diff --git a/cicd/k8s-calico-ipvs2/node_scripts/common.sh b/cicd/k8s-calico-ipvs2/node_scripts/common.sh
index b8634194f..654da6b7c 100644
--- a/cicd/k8s-calico-ipvs2/node_scripts/common.sh
+++ b/cicd/k8s-calico-ipvs2/node_scripts/common.sh
@@ -26,7 +26,7 @@ sudo apt-get update -y
 # Install CRI-O Runtime
 
 VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')"
-
+CRIO_VERSION=1.27
 # Create the .conf file to load the modules at bootup
 cat <<EOF | sudo tee /etc/modules-load.d/crio.conf
 overlay
@@ -57,11 +57,11 @@ sudo sysctl --system
 cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
 deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /
 EOF
-cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list
-deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /
+cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$CRIO_VERSION.list
+deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/ /
 EOF
 
-curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
+curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
 curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
 
 sudo apt-get update
@@ -76,12 +76,12 @@ sudo systemctl enable crio --now
 echo "CRI runtime installed successfully"
 
 sudo apt-get update
-sudo apt-get install -y apt-transport-https ca-certificates curl
-curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg
+sudo apt-get install -y apt-transport-https ca-certificates curl gpg
+curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v'$VERSION'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
 
-echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
 sudo apt-get update -y
-sudo apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION"
+sudo apt-get install -y kubelet kubectl kubeadm
 sudo apt-get update -y
 sudo apt-get install -y jq
 sudo apt-get install -y ipvsadm
diff --git a/cicd/k8s-calico-ipvs2/validation.sh b/cicd/k8s-calico-ipvs2/validation.sh
index ad5bb5a5e..40a19d2a2 100755
--- a/cicd/k8s-calico-ipvs2/validation.sh
+++ b/cicd/k8s-calico-ipvs2/validation.sh
@@ -28,7 +28,7 @@ do
 done
 
 ## Any routing updates  ??
-sleep 30
+#sleep 30
 
 echo Service IP : $extIP
 echo -e "\nEnd Points List"
diff --git a/cicd/k8s-calico-ipvs2/yaml/kubeadm-config.yaml b/cicd/k8s-calico-ipvs2/yaml/kubeadm-config.yaml
index 31afe601c..245a62553 100644
--- a/cicd/k8s-calico-ipvs2/yaml/kubeadm-config.yaml
+++ b/cicd/k8s-calico-ipvs2/yaml/kubeadm-config.yaml
@@ -30,7 +30,7 @@ etcd:
   local:
     dataDir: /var/lib/etcd
 imageRepository: registry.k8s.io
-kubernetesVersion: v1.27.5
+kubernetesVersion: v1.29.2
 networking:
   dnsDomain: cluster.local
   podSubnet: 172.16.1.0/16
diff --git a/cicd/k8s-calico-ipvs2/yaml/settings.yaml b/cicd/k8s-calico-ipvs2/yaml/settings.yaml
index e5b02a60b..492519dce 100644
--- a/cicd/k8s-calico-ipvs2/yaml/settings.yaml
+++ b/cicd/k8s-calico-ipvs2/yaml/settings.yaml
@@ -38,7 +38,8 @@ software:
      version: 0.7.1
   cluster:
     box: bento/ubuntu-22.04
+    version: 202401.31.0
   calico: 3.26.0
   # To skip the dashboard installation, set its version to an empty value or comment it out:
-  kubernetes: 1.27.1-00
+  kubernetes: 1.29.2
   os: xUbuntu_22.04
diff --git a/cicd/k8s-calico-ipvs3-ha/node_scripts/common.sh b/cicd/k8s-calico-ipvs3-ha/node_scripts/common.sh
index b8634194f..654da6b7c 100644
--- a/cicd/k8s-calico-ipvs3-ha/node_scripts/common.sh
+++ b/cicd/k8s-calico-ipvs3-ha/node_scripts/common.sh
@@ -26,7 +26,7 @@ sudo apt-get update -y
 # Install CRI-O Runtime
 
 VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')"
-
+CRIO_VERSION=1.27
 # Create the .conf file to load the modules at bootup
 cat <<EOF | sudo tee /etc/modules-load.d/crio.conf
 overlay
@@ -57,11 +57,11 @@ sudo sysctl --system
 cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
 deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /
 EOF
-cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list
-deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /
+cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$CRIO_VERSION.list
+deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/ /
 EOF
 
-curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
+curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
 curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
 
 sudo apt-get update
@@ -76,12 +76,12 @@ sudo systemctl enable crio --now
 echo "CRI runtime installed successfully"
 
 sudo apt-get update
-sudo apt-get install -y apt-transport-https ca-certificates curl
-curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg
+sudo apt-get install -y apt-transport-https ca-certificates curl gpg
+curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v'$VERSION'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
 
-echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
 sudo apt-get update -y
-sudo apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION"
+sudo apt-get install -y kubelet kubectl kubeadm
 sudo apt-get update -y
 sudo apt-get install -y jq
 sudo apt-get install -y ipvsadm
diff --git a/cicd/k8s-calico-ipvs3-ha/yaml/kubeadm-config.yaml b/cicd/k8s-calico-ipvs3-ha/yaml/kubeadm-config.yaml
index 31afe601c..245a62553 100644
--- a/cicd/k8s-calico-ipvs3-ha/yaml/kubeadm-config.yaml
+++ b/cicd/k8s-calico-ipvs3-ha/yaml/kubeadm-config.yaml
@@ -30,7 +30,7 @@ etcd:
   local:
     dataDir: /var/lib/etcd
 imageRepository: registry.k8s.io
-kubernetesVersion: v1.27.5
+kubernetesVersion: v1.29.2
 networking:
   dnsDomain: cluster.local
   podSubnet: 172.16.1.0/16
diff --git a/cicd/k8s-calico-ipvs3-ha/yaml/settings.yaml b/cicd/k8s-calico-ipvs3-ha/yaml/settings.yaml
index e5b02a60b..492519dce 100644
--- a/cicd/k8s-calico-ipvs3-ha/yaml/settings.yaml
+++ b/cicd/k8s-calico-ipvs3-ha/yaml/settings.yaml
@@ -38,7 +38,8 @@ software:
      version: 0.7.1
   cluster:
     box: bento/ubuntu-22.04
+    version: 202401.31.0
   calico: 3.26.0
   # To skip the dashboard installation, set its version to an empty value or comment it out:
-  kubernetes: 1.27.1-00
+  kubernetes: 1.29.2
   os: xUbuntu_22.04
diff --git a/cicd/k8s-calico-ipvs3/node_scripts/common.sh b/cicd/k8s-calico-ipvs3/node_scripts/common.sh
index b8634194f..654da6b7c 100644
--- a/cicd/k8s-calico-ipvs3/node_scripts/common.sh
+++ b/cicd/k8s-calico-ipvs3/node_scripts/common.sh
@@ -26,7 +26,7 @@ sudo apt-get update -y
 # Install CRI-O Runtime
 
 VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')"
-
+CRIO_VERSION=1.27
 # Create the .conf file to load the modules at bootup
 cat <<EOF | sudo tee /etc/modules-load.d/crio.conf
 overlay
@@ -57,11 +57,11 @@ sudo sysctl --system
 cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
 deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /
 EOF
-cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list
-deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /
+cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$CRIO_VERSION.list
+deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/ /
 EOF
 
-curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
+curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
 curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
 
 sudo apt-get update
@@ -76,12 +76,12 @@ sudo systemctl enable crio --now
 echo "CRI runtime installed successfully"
 
 sudo apt-get update
-sudo apt-get install -y apt-transport-https ca-certificates curl
-curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg
+sudo apt-get install -y apt-transport-https ca-certificates curl gpg
+curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v'$VERSION'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
 
-echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
 sudo apt-get update -y
-sudo apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION"
+sudo apt-get install -y kubelet kubectl kubeadm
 sudo apt-get update -y
 sudo apt-get install -y jq
 sudo apt-get install -y ipvsadm
diff --git a/cicd/k8s-calico-ipvs3/yaml/kubeadm-config.yaml b/cicd/k8s-calico-ipvs3/yaml/kubeadm-config.yaml
index 31afe601c..245a62553 100644
--- a/cicd/k8s-calico-ipvs3/yaml/kubeadm-config.yaml
+++ b/cicd/k8s-calico-ipvs3/yaml/kubeadm-config.yaml
@@ -30,7 +30,7 @@ etcd:
   local:
     dataDir: /var/lib/etcd
 imageRepository: registry.k8s.io
-kubernetesVersion: v1.27.5
+kubernetesVersion: v1.29.2
 networking:
   dnsDomain: cluster.local
   podSubnet: 172.16.1.0/16
diff --git a/cicd/k8s-calico-ipvs3/yaml/settings.yaml b/cicd/k8s-calico-ipvs3/yaml/settings.yaml
index e5b02a60b..492519dce 100644
--- a/cicd/k8s-calico-ipvs3/yaml/settings.yaml
+++ b/cicd/k8s-calico-ipvs3/yaml/settings.yaml
@@ -38,7 +38,8 @@ software:
      version: 0.7.1
   cluster:
     box: bento/ubuntu-22.04
+    version: 202401.31.0
   calico: 3.26.0
   # To skip the dashboard installation, set its version to an empty value or comment it out:
-  kubernetes: 1.27.1-00
+  kubernetes: 1.29.2
   os: xUbuntu_22.04
diff --git a/cicd/k8s-calico/config.sh b/cicd/k8s-calico/config.sh
index 1a81f9f30..b36068d75 100755
--- a/cicd/k8s-calico/config.sh
+++ b/cicd/k8s-calico/config.sh
@@ -68,5 +68,6 @@ do
 done
 
 if [[ $fin == 0 ]]; then
+    echo "Cluster is not ready"
     exit 1
 fi
diff --git a/cicd/k8s-calico/node_scripts/common.sh b/cicd/k8s-calico/node_scripts/common.sh
index 1e68fa833..f1f19041d 100644
--- a/cicd/k8s-calico/node_scripts/common.sh
+++ b/cicd/k8s-calico/node_scripts/common.sh
@@ -68,8 +68,8 @@ echo "CRI runtime installed successfully"
 
 sudo apt-get update
 sudo apt-get install -y apt-transport-https ca-certificates curl gpg
-curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.29/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
-echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.29/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
+curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v'$VERSION'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
 
 sudo apt-get update -y
 sudo apt-get install -y kubelet kubectl kubeadm