Merge pull request #793 from nik-netlox/main

K8s-calico-incluster CICD test added

.github/workflows/k8s-calico-incluster.yml

@@ -0,0 +1,36 @@
+name: K8s-Calico-Incluster-Sanity-CI
+on:
+ schedule:
+ # Runs "At 11:00 UTC every day-of-week"
+ - cron: '0 11 * * *'
+ workflow_dispatch:
+    inputs:
+      testName:
+        description: 'Test Run-Name'     
+        required: true
+        default: 'k8s-calico-incluster'
+jobs:
+  test-runner:
+    name: k8s-calico-incluster-sanity
+    runs-on: [self-hosted, large]
+    if: github.repository == 'loxilb-io/loxilb' 
+          &&  github.event.inputs.tagName == ''
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+          submodules: recursive
+
+    - name: Run the test
+      run: |
+         cd cicd/k8s-calico-incluster
+         ./config.sh
+         ./validation.sh
+         cd -
+         
+    - name: Clean test-bed
+      if: success() || failure()
+      run: |
+        cd cicd/k8s-calico-incluster || true
+        ./rmconfig.sh
+        cd -

cicd/k8s-calico-incluster/Vagrantfile

@@ -5,32 +5,29 @@ require "yaml"
 settings = YAML.load_file "yaml/settings.yaml"
 
 workers = settings["nodes"]["workers"]["count"]
+loxilbs = (ENV['LOXILBS'] || "2").to_i
 
 Vagrant.configure("2") do |config|
 
   if Vagrant.has_plugin?("vagrant-vbguest")
     config.vbguest.auto_update = false
   end
-
-  config.vm.box = settings["software"]["cluster"]["box"]["name"]
-  config.vm.box_version = settings["software"]["cluster"]["box"]["version"]
-
-  config.vm.define "host" do |host|
+   config.vm.define "host" do |host|
     host.vm.hostname = 'host1'
-    host.vm.network :private_network, ip: settings["network"]["client_ip"], :netmask => "255.255.255.0"
+    host.vm.box = settings["software"]["cluster"]["box"]
     host.vm.network :private_network, ip: "192.168.80.9", :netmask => "255.255.255.0"
     host.vm.network :private_network, ip: "192.168.90.9", :netmask => "255.255.255.0"
     host.vm.provision :shell, :path => "node_scripts/host.sh"
     host.vm.provider :virtualbox do |vbox|
         vbox.customize ["modifyvm", :id, "--memory", 2048]
-        vbox.customize ["modifyvm", :id, "--cpus", 1]
+        vbox.customize ["modifyvm", :id, "--cpus", 2]
     end
   end
 
   config.vm.define "master" do |master|
-    master.vm.hostname = 'master1'
+    master.vm.box = settings["software"]["cluster"]["box"]
+    master.vm.hostname = 'master'
     master.vm.network :private_network, ip: settings["network"]["control_ip"], :netmask => "255.255.255.0"
-    master.vm.network :private_network, ip: settings["network"]["control_ip2"], :netmask => "255.255.255.0"
     master.vm.provision "shell",
       env: {
         "DNS_SERVERS" => settings["network"]["dns_servers"].join(" "),
@@ -50,14 +47,16 @@ Vagrant.configure("2") do |config|
 
     master.vm.provider :virtualbox do |vbox|
         vbox.customize ["modifyvm", :id, "--memory", 4096]
-        vbox.customize ["modifyvm", :id, "--cpus", 3]
+        vbox.customize ["modifyvm", :id, "--cpus", 2]
+        vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
     end
   end
 
   (1..workers).each do |node_number|
     config.vm.define "worker#{node_number}" do |worker|
+      worker.vm.box = settings["software"]["cluster"]["box"]
       worker.vm.hostname = "worker#{node_number}"
-      ip = node_number + 100
+      ip = node_number + 200
       worker.vm.network :private_network, ip: "192.168.80.#{ip}", :netmask => "255.255.255.0"
       worker.vm.provision "shell",
         env: {
@@ -70,8 +69,9 @@ Vagrant.configure("2") do |config|
       worker.vm.provision "shell", path: "node_scripts/worker.sh"
 
       worker.vm.provider :virtualbox do |vbox|
-          vbox.customize ["modifyvm", :id, "--memory", 2048]
+          vbox.customize ["modifyvm", :id, "--memory", 4096]
           vbox.customize ["modifyvm", :id, "--cpus", 2]
+          vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
       end
     end
   end

cicd/k8s-calico-incluster/config.sh

@@ -30,8 +30,38 @@ do
     sleep 10
 done
 
-# Create fullnat Services
+sudo sysctl net.ipv4.conf.vboxnet1.arp_accept=1
+
+#Create fullnat Service
+vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/tcp_onearm.yml' 2> /dev/null
+vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/udp_onearm.yml' 2> /dev/null
+vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp_onearm.yml' 2> /dev/null
 vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/tcp_fullnat.yml' 2> /dev/null
 vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/udp_fullnat.yml' 2> /dev/null
 vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp_fullnat.yml' 2> /dev/null
-vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp.yml' 2> /dev/null
+
+for((i=1; i<=60; i++))
+do
+    fin=1
+    pods=$(vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null | grep -v "NAMESPACE")
+
+    while IFS= read -a pods; do
+        read -a pod <<< "$pods"
+        if [[ ${pod[3]} != *"Running"* ]]; then
+            echo "${pod[1]} is not UP yet"
+            fin=0
+        fi
+    done <<< "$pods"
+    if [ $fin == 1 ];
+    then
+        echo "Cluster is ready"
+        break;
+    fi
+    echo "Will try after 10s"
+    sleep 10
+done
+
+if [[ $fin == 0 ]]; then
+    echo "Cluster is not ready"
+    exit 1
+fi

cicd/k8s-calico-incluster/node_scripts/common.sh

@@ -26,7 +26,7 @@ sudo apt-get update -y
 # Install CRI-O Runtime
 
 VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')"
-
+CRIO_VERSION=1.27
 # Create the .conf file to load the modules at bootup
 cat <<EOF | sudo tee /etc/modules-load.d/crio.conf
 overlay
@@ -36,6 +36,15 @@ EOF
 sudo modprobe overlay
 sudo modprobe br_netfilter
 
+# Install ipvs related modules
+sudo modprobe ip_vs
+sudo modprobe ip_vs_rr
+sudo modprobe ip_vs_wrr
+sudo modprobe ip_vs_sh
+#sudo modprobe nf_conntrack_ipv4
+
+sudo sysctl net.ipv4.vs.sloppy_sctp=1
+
 # Set up required sysctl params, these persist across reboots.
 cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
 net.bridge.bridge-nf-call-iptables  = 1
@@ -46,14 +55,14 @@ EOF
 sudo sysctl --system
 
 cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
-deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /
+deb https://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /
 EOF
-cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list
-deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /
+cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$CRIO_VERSION.list
+deb http://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/ /
 EOF
 
-curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
-curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
+curl -L https://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
+curl -L https://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
 
 sudo apt-get update
 sudo apt-get install cri-o cri-o-runc -y
@@ -67,14 +76,15 @@ sudo systemctl enable crio --now
 echo "CRI runtime installed successfully"
 
 sudo apt-get update
-sudo apt-get install -y apt-transport-https ca-certificates curl
-curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg
+sudo apt-get install -y apt-transport-https ca-certificates curl gpg
+curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v'$VERSION'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
 
-echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
 sudo apt-get update -y
-sudo apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION"
+sudo apt-get install -y kubelet kubectl kubeadm
 sudo apt-get update -y
 sudo apt-get install -y jq
+sudo apt-get install -y ipvsadm
 
 local_ip="$(ip --json a s | jq -r '.[] | if .ifname == "eth1" then .addr_info[] | if .family == "inet" then .local else empty end else empty end')"
 cat > /etc/default/kubelet << EOF

cicd/k8s-calico-incluster/node_scripts/host.sh

@@ -1,10 +1,5 @@
-sudo su
-sudo apt-get install -y lksctp-tools socat
-wget https://github.com/loxilb-io/loxilb/raw/main/cicd/common/sctp_client
-wget https://github.com/loxilb-io/loxilb/raw/main/cicd/common/udp_client
-chmod 777 sctp_client
-chmod 777 udp_client
-echo "123.123.123.1 k8s-svc" >> /etc/hosts
-ifconfig eth2 mtu 1450
-ip route add 123.123.123.0/24 via 192.168.90.10
+# Install Bird to work with k3s
+sudo apt-get update
+sudo apt-get -y install socat lksctp-tools
+
 echo "Host is up"

cicd/k8s-calico-incluster/node_scripts/loxilb.sh

@@ -7,7 +7,3 @@ add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu
 apt-get update
 apt-get install -y docker-ce
 docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest
-echo alias loxicmd=\"sudo docker exec -it loxilb loxicmd\" >> ~/.bashrc
-echo alias loxilb=\"sudo docker exec -it loxilb \" >> ~/.bashrc
-
-echo $LOXILB_IP > /vagrant/loxilb-ip

cicd/k8s-calico-incluster/node_scripts/master.sh

@@ -10,14 +10,13 @@ sudo kubeadm config images pull
 
 echo "Preflight Check Passed: Downloaded All Required Images"
 
-sudo kubeadm init --apiserver-advertise-address=$CONTROL_IP --apiserver-cert-extra-sans=$CONTROL_IP --pod-network-cidr=$POD_CIDR --service-cidr=$SERVICE_CIDR --node-name "$NODENAME" --ignore-preflight-errors Swap
+#sudo kubeadm init --apiserver-advertise-address=$CONTROL_IP --apiserver-cert-extra-sans=$CONTROL_IP --pod-network-cidr=$POD_CIDR --service-cidr=$SERVICE_CIDR --node-name "$NODENAME" --ignore-preflight-errors Swap
+sudo kubeadm init --ignore-preflight-errors Swap --config /vagrant/yaml/kubeadm-config.yaml
 
 mkdir -p "$HOME"/.kube
 sudo cp -i /etc/kubernetes/admin.conf "$HOME"/.kube/config
 sudo chown "$(id -u)":"$(id -g)" "$HOME"/.kube/config
 
-curl -sfL https://github.com/loxilb-io/loxilb-ebpf/raw/main/kprobe/install.sh | sh -
-
 # Save Configs to shared /Vagrant location
 
 # For Vagrant re-runs, check if there is existing configs in the location and delete it for saving new configuration.
@@ -54,6 +53,5 @@ EOF
 kubectl apply -f https://raw.githubusercontent.com/techiescamp/kubeadm-scripts/main/manifests/metrics-server.yaml
 
 # Install loxilb
-kubectl apply -f /vagrant/yaml/loxilb.yml
-kubectl apply -f /vagrant/yaml/loxilb-peer.yml
-kubectl apply -f /vagrant/yaml/kube-loxilb.yml
+kubectl apply -f /vagrant/yaml/kube-loxilb.yaml
+kubectl apply -f /vagrant/yaml/loxilb.yaml

cicd/k8s-calico-incluster/validation.sh

@@ -8,7 +8,7 @@ fi
 
 # Set space as the delimiter
 IFS=' '
-
+alloc=0
 for((i=0; i<120; i++))
 do
   extLB=$(vagrant ssh master -c 'kubectl get svc' 2> /dev/null | grep "tcp-lb-fullnat")
@@ -19,16 +19,32 @@ do
     sleep 1
     continue
   fi 
-  if [[ ${strarr[3]} != *"none"* ]]; then
+  if [[ ${strarr[3]} != *"none"* || ${strarr[3]} != *"pending"* ]]; then
     extIP="$(cut -d'-' -f2 <<<${strarr[3]})"
+    alloc=1
     break
   fi
   echo "No external LB allocated"
   sleep 1
 done
 
+if [[ $alloc != 1 ]]; then
+  echo "No external LB allocated. Check kube-loxilb and loxilb logs"
+  echo "******************************************************************************"
+  vagrant ssh master -c 'kubectl get endpoints -A' 2> /dev/null
+  echo "******************************************************************************"
+  echo -e "\nSVC List"
+  echo "******************************************************************************"
+  vagrant ssh master -c 'kubectl get svc' 2> /dev/null
+  echo "******************************************************************************"
+  echo -e "\nPod List"
+  echo "******************************************************************************"
+  vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null
+  exit 1
+fi
+
 ## Any routing updates  ??
-sleep 30
+#sleep 30
 
 echo Service IP : $extIP
 echo -e "\nEnd Points List"
@@ -45,30 +61,33 @@ vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null
 
 echo -e "\nTEST RESULTS"
 echo "******************************************************************************"
-mode=( "fullnat" )
-tcp_port=( 57002 )
-udp_port=( 57003 )
-sctp_port=( 57004 )
+mode=( "onearm" "fullnat" )
+tcp_port=( 56002 57002 )
+udp_port=( 56003 57003 )
+sctp_port=( 56004 57004 )
 code=0
-for ((i=0;i<1;i++)); do
-out=$(vagrant ssh host -c "curl -s --connect-timeout 10 http://$extIP:${tcp_port[i]}")
-echo $out
+
+for ((i=0;i<=1;i++)); do
+out=$(vagrant ssh host -c "curl -s --connect-timeout 10 http://$extIP:${tcp_port[i]}" 2> /dev/null)
+#echo $out
 if [[ ${out} == *"nginx"* ]]; then
   echo -e "K8s-calico-incluster TCP\t(${mode[i]})\t[OK]"
 else
   echo -e "K8s-calico-incluster TCP\t(${mode[i]})\t[FAILED]"
   code=1
 fi
 
-out=$(vagrant ssh host -c "timeout 5 ./udp_client $extIP ${udp_port[i]}")
+out=$(vagrant ssh host -c "timeout 5 /vagrant/udp_client $extIP ${udp_port[i]}" 2> /dev/null)
+#echo $out
 if [[ ${out} == *"Client"* ]]; then
   echo -e "K8s-calico-incluster UDP\t(${mode[i]})\t[OK]"
 else
   echo -e "K8s-calico-incluster UDP\t(${mode[i]})\t[FAILED]"
   code=1
 fi
 
-out=$(vagrant ssh host -c "socat -T10 - SCTP:$extIP:${sctp_port[i]},bind=192.168.90.9")
+out=$(vagrant ssh host -c "socat -T10 - SCTP:$extIP:${sctp_port[i]}" 2> /dev/null)
+#echo $out
 if [[ ${out} == *"server"* ]]; then
   echo -e "K8s-calico-incluster SCTP\t(${mode[i]})\t[OK]"
 else
@@ -77,16 +96,4 @@ else
 fi
 done
 
-mode=( "default" )
-sctp_port=( 55004 )
-code=0
-
-out=$(vagrant ssh host -c "socat -T10 - SCTP:$extIP:${sctp_port[0]},bind=192.168.90.9")
-if [[ ${out} == *"server"* ]]; then
-  echo -e "K8s-calico-incluster SCTP\t(${mode[0]})\t[OK]"
-else
-  echo -e "K8s-calico-incluster SCTP\t(${mode[0]})\t[FAILED]"
-  code=1
-fi
-
 exit $code