loxilb-io · UltraInstinct14 · Nov 9, 2023 · Nov 9, 2023 · Nov 9, 2023 · Nov 9, 2023
diff --git a/.github/workflows/k3s-sctpmh-2.yml b/.github/workflows/k3s-sctpmh-2.yml
@@ -0,0 +1,39 @@
+name: K3s-SCTPMH-2-Sanity-CI
+on:
+ schedule:
+ # Runs "At 11:00 UTC every day-of-week"
+ - cron: '0 11 * * *'
+ workflow_dispatch:
+    inputs:
+      testName:
+        description: 'Test Run-Name'     
+        required: true
+        default: 'k3s-sctpmh-2'
+jobs:
+  build:
+    name: k3s-sctpmh-2-sanity
+    runs-on: ubuntu-20.04
+    steps:
+      - run: sudo apt -y install bridge-utils iproute2 lksctp-tools iputils-ping
+      - run: |
+         echo "KUBECONFIG=--kubeconfig=/etc/rancher/k3s/k3s.yaml" >> $GITHUB_ENV
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - run: curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh -
+      - run: |
+         sleep 10
+         kubectl "${{ env.KUBECONFIG }}" taint nodes --all node.cloudprovider.kubernetes.io/uninitialized=false:NoSchedule-
+         sleep 60
+         kubectl "${{ env.KUBECONFIG }}" get nodes
+         kubectl "${{ env.KUBECONFIG }}" get pods -A
+         wget https://github.com/loxilb-io/loxi-ccm/raw/master/manifests/loxi-ccm-k3s.yaml
+         kubectl "${{ env.KUBECONFIG }}" apply -f ./loxi-ccm-k3s.yaml
+         sleep 60
+         kubectl "${{ env.KUBECONFIG }}" get pods -A
+      - run: |
+         cd cicd/k3s-sctpmh-2/
+         ./config.sh "${{ env.KUBECONFIG }}"
+         ./validation.sh "${{ env.KUBECONFIG }}"
+         ./rmconfig.sh "${{ env.KUBECONFIG }}"
+         cd -
diff --git a/cicd/k3s-sctpmh-2/config.sh b/cicd/k3s-sctpmh-2/config.sh
@@ -111,6 +111,10 @@ $hexec r1 ip route add 20.20.20.1/32 via 11.11.11.11
 
 # Route back to user
 sudo ip route add 11.11.11.0/24 via 12.12.12.1
+sudo ip route add 1.1.1.1 via 12.12.12.1
+sudo ip route add 123.123.123.1 via 12.12.12.1
+sudo ip route add 133.133.133.1 via 12.12.12.1
+sudo ip route add 133.133.133.2 via 12.12.12.1
 
 # Change default route in llb1
 $hexec llb1 ip route del default 
@@ -123,6 +127,8 @@ $hexec llb2 ip route add default via 14.14.14.254
 # Backup paths in user
 $hexec user ip route add 124.124.124.1/32 via 2.2.2.254
 $hexec user ip route add 125.125.125.1/32 via 2.2.2.254
+$hexec user ip route add 134.134.134.1/32 via 2.2.2.254
+$hexec user ip route add 135.135.135.1/32 via 2.2.2.254
 
 sleep 1
 ##Create LB rule
@@ -190,6 +196,7 @@ kubectl $KUBECONFIG apply -f nginx-svc-lb1.yml
 #docker save --output sctp-server.tar loxilb-io/sctp-server
 #sudo k3s ctr images import ./sctp-server.tar
 kubectl $KUBECONFIG apply -f sctp-svc-lb.yml
+kubectl $KUBECONFIG apply -f sctp-svc-lb2.yml
 
 sleep 50
 
@@ -210,11 +217,11 @@ $dexec llb1 loxicmd get lb -o wide
 echo "llb1: loxicmd get ep -o wide"
 echo "****************************"
 $dexec llb1 loxicmd get ep -o wide
-$dexec llb1 cat /etc/shared/keepalive.state
+#$dexec llb1 cat /etc/shared/keepalive.state
 echo "llb2: loxicmd get lb -o wide"
 echo "****************************"
 $dexec llb2 loxicmd get lb -o wide
 echo "llb2: loxicmd get ep -o wide"
 echo "****************************"
 $dexec llb2 loxicmd get ep -o wide
-$dexec llb2 cat /etc/shared/keepalive.state
+#$dexec llb2 cat /etc/shared/keepalive.state
diff --git a/cicd/k3s-sctpmh-2/sctp-svc-lb2.yml b/cicd/k3s-sctpmh-2/sctp-svc-lb2.yml
@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: sctp-lb2
+  annotations:
+    loxilb.io/lbmode: "fullnat"
+    loxilb.io/liveness: "yes"
+    loxilb.io/secondaryIPs: "134.134.134.1,135.135.135.1"
+spec:
+  externalTrafficPolicy: Local
+  loadBalancerClass: loxilb.io/loxilb
+  selector:
+    what: sctp-test2
+  ports:
+    - port: 55004
+      protocol: SCTP
+      targetPort: 9999
+  type: LoadBalancer
+  externalIPs: 
+    - 133.133.133.1
+    - 133.133.133.2
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sctp-test2
+  labels:
+    what: sctp-test2
+spec:
+  tolerations:
+  - key: "node.kubernetes.io/disk-pressure"
+    operator: "Exists"
+    effect: "NoSchedule"
+  containers:
+    - name: sctp-test2
+      image: loxilbio/sctp-darn:latest
+      imagePullPolicy: Always
+      #command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 6 ; done"]
+      command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"]
+      ports:
+        - containerPort: 9999
diff --git a/cicd/k3s-sctpmh-2/validation.sh b/cicd/k3s-sctpmh-2/validation.sh
@@ -2,7 +2,7 @@
 source ../common.sh
 source ../k3s_common.sh
 
-echo "cluster-k3s: TCP & SCTP Multihoming combined"
+echo -e "cluster-k3s: TCP & SCTP Multihoming combined\n"
 
 if [ "$1" ]; then
   KUBECONFIG="$1"
@@ -32,7 +32,8 @@ done
 
 ## Any routing updates  ??
 #sleep 30
-echo $extIP
+echo "TCP service tcp-lb1 -> $extIP:55002"
+echo -e "------------------------------------------------------------------------------------\n"
 
 out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) 
 if [[ ${out} == *"Welcome to nginx"* ]]; then
@@ -52,6 +53,7 @@ else
   $dexec r1 ip route
   exit 1
 fi
+echo -e "------------------------------------------------------------------------------------\n\n\n"
 
 for((i=0; i<120; i++))
 do
@@ -73,6 +75,7 @@ do
 done
 
 echo "SCTP Multihoming service sctp-lb1 -> $extIP:$port"
+echo -e "------------------------------------------------------------------------------------\n"
 
 $hexec user sctp_darn -H 1.1.1.1 -h 123.123.123.1 -p 55003 -s < input > output
 sleep 5
@@ -115,7 +118,103 @@ else
     echo "BFP trace -- "
     exit 1
 fi
+echo -e "------------------------------------------------------------------------------------\n\n\n"
 
+echo "SCTP Multihoming service sctp-lb2 -> 133.133.133.1:55004"
+echo -e "------------------------------------------------------------------------------------\n"
+
+$hexec user timeout 10 sctp_darn -H 1.1.1.1 -h 133.133.133.1 -p 55004 -s < input > output
+sleep 5
+exp="New connection, peer addresses
+133.133.133.1:55004
+134.134.134.1:55004
+135.135.135.1:55004"
+
+res=`cat output | grep -A 3 "New connection, peer addresses"`
+sudo rm -rf output
+if [[ "$res" == "$exp" ]]; then
+    echo $res
+    echo "cluster-k3s SCTP Multihoming service sctp-lb2 (kube-loxilb) [OK]"
+else
+    echo "cluster-k3s SCTP Multihoming service sctp-lb2 (kube-loxilb) [NOK]"
+    echo "Expected : $exp"
+    echo "Received : $res"
+    ## Dump some debug info
+    echo "system route-info"
+    ip route
+    echo "system ipables"
+    sudo iptables -n -t nat -L -v  |grep sctp
+    echo "llb1 lb-info"
+    $dexec llb1 loxicmd get lb
+    echo "llb1 ep-info"
+    $dexec llb1 loxicmd get ep
+    echo "llb1 bpf-info"
+    $dexec llb1 ntc filter show dev eth0 ingress
+    echo "llb1 route-info"
+    $dexec llb1 ip route
+    echo "llb2 lb-info"
+    $dexec llb2 loxicmd get lb
+    echo "llb2 route-info"
+    $dexec llb2 ip route
+    echo "r1 route-info"
+    $dexec r1 ip route
+    echo "BFP trace -- "
+    sudo timeout 5 cat  /sys/kernel/debug/tracing/trace_pipe
+    sudo killall -9 cat
+    echo "BFP trace -- "
+    exit 1
+fi
+echo -e "------------------------------------------------------------------------------------\n\n\n"
+
+echo "SCTP Multihoming service sctp-lb2 -> 133.133.133.2:55004"
+echo -e "------------------------------------------------------------------------------------\n"
+
+$hexec user timeout 10 sctp_darn -H 1.1.1.1 -h 133.133.133.2 -p 55004 -s < input > output
+sleep 5
+exp="New connection, peer addresses
+133.133.133.2:55004
+134.134.134.1:55004
+135.135.135.1:55004"
+
+res=`cat output | grep -A 3 "New connection, peer addresses"`
+sudo rm -rf output
+if [[ "$res" == "$exp" ]]; then
+    echo $res
+    echo "cluster-k3s SCTP Multihoming service sctp-lb2 (kube-loxilb) [OK]"
+else
+    echo "cluster-k3s SCTP Multihoming service sctp-lb2 (kube-loxilb) [NOK]"
+    echo "Expected : $exp"
+    echo "Received : $res"
+    ## Dump some debug info
+    echo "system route-info"
+    ip route
+    echo "system ipables"
+    sudo iptables -n -t nat -L -v  |grep sctp
+    echo "llb1 lb-info"
+    $dexec llb1 loxicmd get lb
+    echo "llb1 ep-info"
+    $dexec llb1 loxicmd get ep
+    echo "llb1 bpf-info"
+    $dexec llb1 ntc filter show dev eth0 ingress
+    echo "llb1 route-info"
+    $dexec llb1 ip route
+    echo "llb2 lb-info"
+    $dexec llb2 loxicmd get lb
+    echo "llb2 route-info"
+    $dexec llb2 ip route
+    echo "r1 route-info"
+    $dexec r1 ip route
+    echo "BFP trace -- "
+    sudo timeout 5 cat  /sys/kernel/debug/tracing/trace_pipe
+    sudo killall -9 cat
+    echo "BFP trace -- "
+    exit 1
+fi
+echo -e "------------------------------------------------------------------------------------\n\n\n"
+
+
+echo "TCP service tcp-lb1 -> $extIP:55002(del+add)"
+echo -e "------------------------------------------------------------------------------------\n"
 ## Check delete and readd service
 kubectl $KUBECONFIG delete -f nginx-svc-lb1.yml
 sleep 10
@@ -143,4 +242,5 @@ else
   $dexec r1 ip route
   exit 1
 fi
+echo -e "------------------------------------------------------------------------------------\n\n\n"
 
diff --git a/loxinet/dpbroker.go b/loxinet/dpbroker.go
@@ -818,7 +818,7 @@ func (dp *DpH) DpMapGetCt4() []cmn.CtInfo {
 	switch r := ret.(type) {
 	case map[string]*DpCtInfo:
 		for _, dCti := range r {
-
+			servName = "-"
 			mh.mtx.Lock()
 			rule := mh.zr.Rules.GetNatLbRuleByID(dCti.RuleID)
 			mh.mtx.Unlock()