From c193bd1b4767235009a796651b356aec82772e80 Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Thu, 9 Nov 2023 13:19:21 +0900 Subject: [PATCH 1/3] external SCTP secondary IPs CICD updated --- cicd/k3s-sctpmh-2/config.sh | 7 ++ cicd/k3s-sctpmh-2/sctp-svc-lb2.yml | 41 ++++++++++++ cicd/k3s-sctpmh-2/validation.sh | 104 ++++++++++++++++++++++++++++- loxinet/dpbroker.go | 2 +- 4 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 cicd/k3s-sctpmh-2/sctp-svc-lb2.yml diff --git a/cicd/k3s-sctpmh-2/config.sh b/cicd/k3s-sctpmh-2/config.sh index a7959f231..5ede7003a 100755 --- a/cicd/k3s-sctpmh-2/config.sh +++ b/cicd/k3s-sctpmh-2/config.sh @@ -111,6 +111,10 @@ $hexec r1 ip route add 20.20.20.1/32 via 11.11.11.11 # Route back to user sudo ip route add 11.11.11.0/24 via 12.12.12.1 +sudo ip route add 1.1.1.1 via 12.12.12.1 +sudo ip route add 123.123.123.1 via 12.12.12.1 +sudo ip route add 133.133.133.1 via 12.12.12.1 +sudo ip route add 133.133.133.2 via 12.12.12.1 # Change default route in llb1 $hexec llb1 ip route del default @@ -123,6 +127,8 @@ $hexec llb2 ip route add default via 14.14.14.254 # Backup paths in user $hexec user ip route add 124.124.124.1/32 via 2.2.2.254 $hexec user ip route add 125.125.125.1/32 via 2.2.2.254 +$hexec user ip route add 134.134.134.1/32 via 2.2.2.254 +$hexec user ip route add 135.135.135.1/32 via 2.2.2.254 sleep 1 ##Create LB rule @@ -190,6 +196,7 @@ kubectl $KUBECONFIG apply -f nginx-svc-lb1.yml #docker save --output sctp-server.tar loxilb-io/sctp-server #sudo k3s ctr images import ./sctp-server.tar kubectl $KUBECONFIG apply -f sctp-svc-lb.yml +kubectl $KUBECONFIG apply -f sctp-svc-lb2.yml sleep 50 diff --git a/cicd/k3s-sctpmh-2/sctp-svc-lb2.yml b/cicd/k3s-sctpmh-2/sctp-svc-lb2.yml new file mode 100644 index 000000000..378b00df2 --- /dev/null +++ b/cicd/k3s-sctpmh-2/sctp-svc-lb2.yml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: sctp-lb2 + annotations: + loxilb.io/lbmode: "fullnat" + loxilb.io/liveness: "yes" + loxilb.io/secondaryIPs: "134.134.134.1,135.135.135.1" +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: sctp-test2 + ports: + - port: 55004 + protocol: SCTP + targetPort: 9999 + type: LoadBalancer + externalIPs: + - 133.133.133.1 + - 133.133.133.2 +--- +apiVersion: v1 +kind: Pod +metadata: + name: sctp-test2 + labels: + what: sctp-test2 +spec: + tolerations: + - key: "node.kubernetes.io/disk-pressure" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: sctp-test2 + image: loxilbio/sctp-darn:latest + imagePullPolicy: Always + #command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 6 ; done"] + command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"] + ports: + - containerPort: 9999 diff --git a/cicd/k3s-sctpmh-2/validation.sh b/cicd/k3s-sctpmh-2/validation.sh index 0c9071a90..4f5769943 100755 --- a/cicd/k3s-sctpmh-2/validation.sh +++ b/cicd/k3s-sctpmh-2/validation.sh @@ -2,7 +2,7 @@ source ../common.sh source ../k3s_common.sh -echo "cluster-k3s: TCP & SCTP Multihoming combined" +echo -e "cluster-k3s: TCP & SCTP Multihoming combined\n" if [ "$1" ]; then KUBECONFIG="$1" @@ -32,7 +32,8 @@ done ## Any routing updates ?? #sleep 30 -echo $extIP +echo "TCP service tcp-lb1 -> $extIP:55002" +echo -e "------------------------------------------------------------------------------------\n" out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) if [[ ${out} == *"Welcome to nginx"* ]]; then @@ -52,6 +53,7 @@ else $dexec r1 ip route exit 1 fi +echo -e "------------------------------------------------------------------------------------\n\n\n" for((i=0; i<120; i++)) do @@ -73,6 +75,7 @@ do done echo "SCTP Multihoming service sctp-lb1 -> $extIP:$port" +echo -e "------------------------------------------------------------------------------------\n" $hexec user sctp_darn -H 1.1.1.1 -h 123.123.123.1 -p 55003 -s < input > output sleep 5 @@ -115,7 +118,103 @@ else echo "BFP trace -- " exit 1 fi +echo -e "------------------------------------------------------------------------------------\n\n\n" +echo "SCTP Multihoming service sctp-lb2 -> 133.133.133.1:55004" +echo -e "------------------------------------------------------------------------------------\n" + +$hexec user timeout 10 sctp_darn -H 1.1.1.1 -h 133.133.133.1 -p 55004 -s < input > output +sleep 5 +exp="New connection, peer addresses +133.133.133.1:55004 +134.134.134.1:55004 +135.135.135.1:55004" + +res=`cat output | grep -A 3 "New connection, peer addresses"` +sudo rm -rf output +if [[ "$res" == "$exp" ]]; then + echo $res + echo "cluster-k3s SCTP Multihoming service sctp-lb2 (kube-loxilb) [OK]" +else + echo "cluster-k3s SCTP Multihoming service sctp-lb2 (kube-loxilb) [NOK]" + echo "Expected : $exp" + echo "Received : $res" + ## Dump some debug info + echo "system route-info" + ip route + echo "system ipables" + sudo iptables -n -t nat -L -v |grep sctp + echo "llb1 lb-info" + $dexec llb1 loxicmd get lb + echo "llb1 ep-info" + $dexec llb1 loxicmd get ep + echo "llb1 bpf-info" + $dexec llb1 ntc filter show dev eth0 ingress + echo "llb1 route-info" + $dexec llb1 ip route + echo "llb2 lb-info" + $dexec llb2 loxicmd get lb + echo "llb2 route-info" + $dexec llb2 ip route + echo "r1 route-info" + $dexec r1 ip route + echo "BFP trace -- " + sudo timeout 5 cat /sys/kernel/debug/tracing/trace_pipe + sudo killall -9 cat + echo "BFP trace -- " + exit 1 +fi +echo -e "------------------------------------------------------------------------------------\n\n\n" + +echo "SCTP Multihoming service sctp-lb2 -> 133.133.133.2:55004" +echo -e "------------------------------------------------------------------------------------\n" + +$hexec user timeout 10 sctp_darn -H 1.1.1.1 -h 133.133.133.2 -p 55004 -s < input > output +sleep 5 +exp="New connection, peer addresses +133.133.133.2:55004 +134.134.134.1:55004 +135.135.135.1:55004" + +res=`cat output | grep -A 3 "New connection, peer addresses"` +sudo rm -rf output +if [[ "$res" == "$exp" ]]; then + echo $res + echo "cluster-k3s SCTP Multihoming service sctp-lb2 (kube-loxilb) [OK]" +else + echo "cluster-k3s SCTP Multihoming service sctp-lb2 (kube-loxilb) [NOK]" + echo "Expected : $exp" + echo "Received : $res" + ## Dump some debug info + echo "system route-info" + ip route + echo "system ipables" + sudo iptables -n -t nat -L -v |grep sctp + echo "llb1 lb-info" + $dexec llb1 loxicmd get lb + echo "llb1 ep-info" + $dexec llb1 loxicmd get ep + echo "llb1 bpf-info" + $dexec llb1 ntc filter show dev eth0 ingress + echo "llb1 route-info" + $dexec llb1 ip route + echo "llb2 lb-info" + $dexec llb2 loxicmd get lb + echo "llb2 route-info" + $dexec llb2 ip route + echo "r1 route-info" + $dexec r1 ip route + echo "BFP trace -- " + sudo timeout 5 cat /sys/kernel/debug/tracing/trace_pipe + sudo killall -9 cat + echo "BFP trace -- " + exit 1 +fi +echo -e "------------------------------------------------------------------------------------\n\n\n" + + +echo "TCP service tcp-lb1 -> $extIP:55002(del+add)" +echo -e "------------------------------------------------------------------------------------\n" ## Check delete and readd service kubectl $KUBECONFIG delete -f nginx-svc-lb1.yml sleep 10 @@ -143,4 +242,5 @@ else $dexec r1 ip route exit 1 fi +echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/loxinet/dpbroker.go b/loxinet/dpbroker.go index c29285ee7..e10abd6b6 100644 --- a/loxinet/dpbroker.go +++ b/loxinet/dpbroker.go @@ -817,7 +817,7 @@ func (dp *DpH) DpMapGetCt4() []cmn.CtInfo { switch r := ret.(type) { case map[string]*DpCtInfo: for _, dCti := range r { - + servName = "-" mh.mtx.Lock() rule := mh.zr.Rules.GetNatLbRuleByID(dCti.RuleID) mh.mtx.Unlock() From fae7ea965a656de482ee25afe938d6db38293b6a Mon Sep 17 00:00:00 2001 From: nik-netlox <66406124+nik-netlox@users.noreply.github.com> Date: Thu, 9 Nov 2023 13:29:58 +0900 Subject: [PATCH 2/3] Create k3s-sctpmh-2.yml --- .github/workflows/k3s-sctpmh-2.yml | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/k3s-sctpmh-2.yml diff --git a/.github/workflows/k3s-sctpmh-2.yml b/.github/workflows/k3s-sctpmh-2.yml new file mode 100644 index 000000000..c17ff60d7 --- /dev/null +++ b/.github/workflows/k3s-sctpmh-2.yml @@ -0,0 +1,39 @@ +name: K3s-SCTPMH-2-Sanity-CI +on: + schedule: + # Runs "At 11:00 UTC every day-of-week" + - cron: '0 11 * * *' + workflow_dispatch: + inputs: + testName: + description: 'Test Run-Name' + required: true + default: 'k3s-sctpmh-2' +jobs: + build: + name: k3s-sctpmh-2-sanity + runs-on: ubuntu-20.04 + steps: + - run: sudo apt -y install bridge-utils iproute2 lksctp-tools iputils-ping + - run: | + echo "KUBECONFIG=--kubeconfig=/etc/rancher/k3s/k3s.yaml" >> $GITHUB_ENV + - uses: actions/checkout@v2 + with: + submodules: recursive + - run: curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh - + - run: | + sleep 10 + kubectl "${{ env.KUBECONFIG }}" taint nodes --all node.cloudprovider.kubernetes.io/uninitialized=false:NoSchedule- + sleep 60 + kubectl "${{ env.KUBECONFIG }}" get nodes + kubectl "${{ env.KUBECONFIG }}" get pods -A + wget https://github.com/loxilb-io/loxi-ccm/raw/master/manifests/loxi-ccm-k3s.yaml + kubectl "${{ env.KUBECONFIG }}" apply -f ./loxi-ccm-k3s.yaml + sleep 60 + kubectl "${{ env.KUBECONFIG }}" get pods -A + - run: | + cd cicd/k3s-sctpmh-2/ + ./config.sh "${{ env.KUBECONFIG }}" + ./validation.sh "${{ env.KUBECONFIG }}" + ./rmconfig.sh "${{ env.KUBECONFIG }}" + cd - From 992460964e5b1311e9db5f31aea3aed9b8e8af62 Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Thu, 9 Nov 2023 14:38:13 +0900 Subject: [PATCH 3/3] Removed unnecessary commands --- cicd/k3s-sctpmh-2/config.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cicd/k3s-sctpmh-2/config.sh b/cicd/k3s-sctpmh-2/config.sh index 5ede7003a..78ebc59c6 100755 --- a/cicd/k3s-sctpmh-2/config.sh +++ b/cicd/k3s-sctpmh-2/config.sh @@ -217,11 +217,11 @@ $dexec llb1 loxicmd get lb -o wide echo "llb1: loxicmd get ep -o wide" echo "****************************" $dexec llb1 loxicmd get ep -o wide -$dexec llb1 cat /etc/shared/keepalive.state +#$dexec llb1 cat /etc/shared/keepalive.state echo "llb2: loxicmd get lb -o wide" echo "****************************" $dexec llb2 loxicmd get lb -o wide echo "llb2: loxicmd get ep -o wide" echo "****************************" $dexec llb2 loxicmd get ep -o wide -$dexec llb2 cat /etc/shared/keepalive.state +#$dexec llb2 cat /etc/shared/keepalive.state