From 5259d7022c667adcdc9825a29459522bf4b7b0c7 Mon Sep 17 00:00:00 2001
From: Trekkie Coder <trekkie@netlox.io>
Date: Tue, 15 Aug 2023 11:50:01 +0900
Subject: [PATCH 01/11] updated cicd for sctp-multihoming

---
 cicd/k3s-sctpmh-2/Dockerfile                  |  14 ++
 cicd/k3s-sctpmh-2/bird_config/bird.conf       | 225 ++++++++++++++++++
 cicd/k3s-sctpmh-2/config.sh                   | 216 +++++++++++++++++
 cicd/k3s-sctpmh-2/input                       |   6 +
 .../keepalived_config1/keepalived.conf        |  22 ++
 .../k3s-sctpmh-2/keepalived_config1/notify.sh |   5 +
 .../keepalived_config2/keepalived.conf        |  22 ++
 .../k3s-sctpmh-2/keepalived_config2/notify.sh |   5 +
 cicd/k3s-sctpmh-2/kube-loxilb.yml             | 132 ++++++++++
 .../k3s-sctpmh-2/llb1_gobgp_config/gobgp.conf |  51 ++++
 .../k3s-sctpmh-2/llb2_gobgp_config/gobgp.conf |  50 ++++
 cicd/k3s-sctpmh-2/nginx-svc-lb1.yml           |  29 +++
 cicd/k3s-sctpmh-2/r1_config/bgpd.conf         |  10 +
 cicd/k3s-sctpmh-2/r1_config/daemons           |   8 +
 cicd/k3s-sctpmh-2/r1_config/zebra.conf        |   6 +
 cicd/k3s-sctpmh-2/r2_config/bgpd.conf         |  10 +
 cicd/k3s-sctpmh-2/r2_config/daemons           |   8 +
 cicd/k3s-sctpmh-2/r2_config/zebra.conf        |   6 +
 cicd/k3s-sctpmh-2/r3_config/bgpd.conf         |  10 +
 cicd/k3s-sctpmh-2/r3_config/daemons           |   8 +
 cicd/k3s-sctpmh-2/r3_config/zebra.conf        |   6 +
 cicd/k3s-sctpmh-2/rmconfig.sh                 |  57 +++++
 cicd/k3s-sctpmh-2/sctp-svc-lb.yml             |  38 +++
 cicd/k3s-sctpmh-2/validation.sh               | 146 ++++++++++++
 24 files changed, 1090 insertions(+)
 create mode 100644 cicd/k3s-sctpmh-2/Dockerfile
 create mode 100644 cicd/k3s-sctpmh-2/bird_config/bird.conf
 create mode 100755 cicd/k3s-sctpmh-2/config.sh
 create mode 100644 cicd/k3s-sctpmh-2/input
 create mode 100644 cicd/k3s-sctpmh-2/keepalived_config1/keepalived.conf
 create mode 100755 cicd/k3s-sctpmh-2/keepalived_config1/notify.sh
 create mode 100644 cicd/k3s-sctpmh-2/keepalived_config2/keepalived.conf
 create mode 100755 cicd/k3s-sctpmh-2/keepalived_config2/notify.sh
 create mode 100644 cicd/k3s-sctpmh-2/kube-loxilb.yml
 create mode 100644 cicd/k3s-sctpmh-2/llb1_gobgp_config/gobgp.conf
 create mode 100644 cicd/k3s-sctpmh-2/llb2_gobgp_config/gobgp.conf
 create mode 100644 cicd/k3s-sctpmh-2/nginx-svc-lb1.yml
 create mode 100755 cicd/k3s-sctpmh-2/r1_config/bgpd.conf
 create mode 100644 cicd/k3s-sctpmh-2/r1_config/daemons
 create mode 100755 cicd/k3s-sctpmh-2/r1_config/zebra.conf
 create mode 100755 cicd/k3s-sctpmh-2/r2_config/bgpd.conf
 create mode 100644 cicd/k3s-sctpmh-2/r2_config/daemons
 create mode 100755 cicd/k3s-sctpmh-2/r2_config/zebra.conf
 create mode 100755 cicd/k3s-sctpmh-2/r3_config/bgpd.conf
 create mode 100644 cicd/k3s-sctpmh-2/r3_config/daemons
 create mode 100755 cicd/k3s-sctpmh-2/r3_config/zebra.conf
 create mode 100755 cicd/k3s-sctpmh-2/rmconfig.sh
 create mode 100644 cicd/k3s-sctpmh-2/sctp-svc-lb.yml
 create mode 100755 cicd/k3s-sctpmh-2/validation.sh

diff --git a/cicd/k3s-sctpmh-2/Dockerfile b/cicd/k3s-sctpmh-2/Dockerfile
new file mode 100644
index 000000000..505f2d718
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/Dockerfile
@@ -0,0 +1,14 @@
+# Use an official Python runtime as a parent image
+FROM ubuntu:20.04
+
+WORKDIR /
+
+# Install any needed packages specified in requirements.txt
+RUN apt update
+RUN apt -y install lksctp-tools
+
+# Make port 9999 available to the world outside this container
+EXPOSE 9999
+
+# Run app.py when the container launches
+CMD ["sctp_darn", "-H 0.0.0.0 -P 9999 -l"]
diff --git a/cicd/k3s-sctpmh-2/bird_config/bird.conf b/cicd/k3s-sctpmh-2/bird_config/bird.conf
new file mode 100644
index 000000000..1d244c989
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/bird_config/bird.conf
@@ -0,0 +1,225 @@
+# This is a basic configuration file, which contains boilerplate options and
+# some basic examples. It allows the BIRD daemon to start but will not cause
+# anything else to happen.
+#
+# Please refer to the BIRD User's Guide documentation, which is also available
+# online at http://bird.network.cz/ in HTML format, for more information on
+# configuring BIRD and adding routing protocols.
+
+# Configure logging
+#log syslog all;
+log "/var/log/bird.log" { debug, trace, info, remote, warning, error, auth, fatal, bug };
+
+# Set router ID. It is a unique identification of your router, usually one of
+# IPv4 addresses of the router. It is recommended to configure it explicitly.
+router id 12.12.12.254;
+
+# Turn on global debugging of all protocols (all messages or just selected classes)
+# debug protocols all;
+# debug protocols { events, states };
+
+# Turn on internal watchdog
+# watchdog warning 5 s;
+# watchdog timeout 30 s;
+
+# You can define your own constants
+# define my_asn = 65000;
+# define my_addr = 198.51.100.1;
+
+# Tables master4 and master6 are defined by default
+# ipv4 table master4;
+# ipv6 table master6;
+
+# Define more tables, e.g. for policy routing or as MRIB
+# ipv4 table mrib4;
+# ipv6 table mrib6;
+
+# The Device protocol is not a real routing protocol. It does not generate any
+# routes and it only serves as a module for getting information about network
+# interfaces from the kernel. It is necessary in almost any configuration.
+protocol device {
+}
+
+# The direct protocol is not a real routing protocol. It automatically generates
+# direct routes to all network interfaces. Can exist in as many instances as you
+# wish if you want to populate multiple routing tables with direct routes.
+protocol direct {
+	#disabled;		# Disable by default
+	ipv4;			# Connect to default IPv4 table
+	#ipv6;			# ... and to default IPv6 table
+}
+
+# The Kernel protocol is not a real routing protocol. Instead of communicating
+# with other routers in the network, it performs synchronization of BIRD
+# routing tables with the OS kernel. One instance per table.
+protocol kernel {
+	ipv4 {			# Connect protocol to IPv4 table by channel
+#	      table master4;	# Default IPv4 table is master4
+#	      import all;	# Import to table, default is import all
+	      export all;	# Export to protocol. default is export none
+	};
+#	learn;			# Learn alien routes from the kernel
+#	kernel table 10;	# Kernel table to synchronize with (default: main)
+}
+
+# Another instance for IPv6, skipping default options
+protocol kernel {
+	ipv6 { export all; };
+}
+
+# Static routes (Again, there can be multiple instances, for different address
+# families and to disable/enable various groups of static routes on the fly).
+protocol static {
+	ipv4;			# Again, IPv4 channel with default options
+
+#	route 0.0.0.0/0 via 198.51.100.10;
+#	route 192.0.2.0/24 blackhole;
+#	route 10.0.0.0/8 unreachable;
+#	route 10.2.0.0/24 via "eth0";
+#	# Static routes can be defined with optional attributes
+#	route 10.1.1.0/24 via 198.51.100.3 { rip_metric = 3; };
+#	route 10.1.2.0/24 via 198.51.100.3 { ospf_metric1 = 100; };
+#	route 10.1.3.0/24 via 198.51.100.4 { ospf_metric2 = 100; };
+}
+
+# Pipe protocol connects two routing tables. Beware of loops.
+# protocol pipe {
+#	table master4;		# No ipv4/ipv6 channel definition like in other protocols
+#	peer table mrib4;
+#	import all;		# Direction peer table -> table
+#	export all;		# Direction table -> peer table
+# }
+
+# RIP example, both RIP and RIPng are supported
+# protocol rip {
+#	ipv4 {
+#		# Export direct, static routes and ones from RIP itself
+#		import all;
+#		export where source ~ [ RTS_DEVICE, RTS_STATIC, RTS_RIP ];
+#	};
+#	interface "eth*" {
+#	  	update time 10;			# Default period is 30
+#		timeout time 60;		# Default timeout is 180
+#		authentication cryptographic;	# No authentication by default
+#		password "hello" { algorithm hmac sha256; }; # Default is MD5
+#	};
+# }
+
+# OSPF example, both OSPFv2 and OSPFv3 are supported
+# protocol ospf v3 {
+#  	ipv6 {
+#		import all;
+#		export where source = RTS_STATIC;
+#	};
+#	area 0 {
+#		interface "eth*" {
+#			type broadcast;		# Detected by default
+#			cost 10;		# Interface metric
+#			hello 5;		# Default hello perid 10 is too long
+#		};
+#		interface "tun*" {
+#			type ptp;		# PtP mode, avoids DR selection
+#			cost 100;		# Interface metric
+#			hello 5;		# Default hello perid 10 is too long
+#		};
+#		interface "dummy0" {
+#			stub;			# Stub interface, just propagate it
+#		};
+#	};
+#}
+
+# Define simple filter as an example for BGP import filter
+# See https://gitlab.labs.nic.cz/labs/bird/wikis/BGP_filtering for more examples
+# filter rt_import
+# {
+#	if bgp_path.first != 64496 then accept;
+#	if bgp_path.len > 64 then accept;
+#	if bgp_next_hop != from then accept;
+#	reject;
+# }
+
+# BGP example, explicit name 'uplink1' is used instead of default 'bgp1'
+# protocol bgp uplink1 {
+#	description "My BGP uplink";
+#	local 198.51.100.1 as 65000;
+#	neighbor 198.51.100.10 as 64496;
+#	hold time 90;		# Default is 240
+#	password "secret";	# Password used for MD5 authentication
+#
+#	ipv4 {			# regular IPv4 unicast (1/1)
+#		import filter rt_import;
+#		export where source ~ [ RTS_STATIC, RTS_BGP ];
+#	};
+#
+#	ipv6 {			# regular IPv6 unicast (2/1)
+#		import filter rt_import;
+#		export filter {	# The same as 'where' expression above
+#			if source ~ [ RTS_STATIC, RTS_BGP ]
+#			then accept;
+#			else reject;
+#		};
+#	};
+#
+#	ipv4 multicast {	# IPv4 multicast topology (1/2)
+#		table mrib4;	# explicit IPv4 table
+#		import filter rt_import;
+#		export all;
+#	};
+#
+#	ipv6 multicast {	# IPv6 multicast topology (2/2)
+#		table mrib6;	# explicit IPv6 table
+#		import filter rt_import;
+#		export all;
+#	};
+#}
+
+# Template example. Using templates to define IBGP route reflector clients.
+# template bgp rr_clients {
+#	local 10.0.0.1 as 65000;
+#	neighbor as 65000;
+#	rr client;
+#	rr cluster id 1.0.0.1;
+#
+#	ipv4 {
+#		import all;
+#		export where source = RTS_BGP;
+#	};
+#
+#	ipv6 {
+#		import all;
+#		export where source = RTS_BGP;
+#	};
+# }
+#
+# protocol bgp client1 from rr_clients {
+#	neighbor 10.0.1.1;
+# }
+#
+# protocol bgp client2 from rr_clients {
+#	neighbor 10.0.2.1;
+# }
+#
+# protocol bgp client3 from rr_clients {
+#	neighbor 10.0.3.1;
+# }
+#
+protocol bgp llb1 {
+  local as 64512;
+  neighbor 12.12.12.1 as 64512;
+
+  ipv4 {
+    import all;
+    export all;
+  };
+}
+
+protocol bgp llb2 {
+  local 14.14.14.254 as 64512;
+  neighbor 14.14.14.1 as 64512;
+
+  ipv4 {
+    import all;
+    export all;
+  };
+}
+
diff --git a/cicd/k3s-sctpmh-2/config.sh b/cicd/k3s-sctpmh-2/config.sh
new file mode 100755
index 000000000..9aaf8fd6f
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/config.sh
@@ -0,0 +1,216 @@
+#!/bin/bash
+
+source ../common.sh
+source ../k3s_common.sh
+
+echo "#########################################"
+echo "Spawning all hosts"
+echo "#########################################"
+
+spawn_docker_host --dock-type loxilb --dock-name llb1 --with-bgp yes --bgp-config $(pwd)/llb1_gobgp_config
+spawn_docker_host --dock-type loxilb --dock-name llb2 --with-bgp yes --bgp-config $(pwd)/llb2_gobgp_config
+spawn_docker_host --dock-type host --dock-name ep1
+spawn_docker_host --dock-type host --dock-name ep2
+spawn_docker_host --dock-type host --dock-name ep3
+spawn_docker_host --dock-type host --dock-name r1 --with-bgp yes --bgp-config $(pwd)/r1_config
+spawn_docker_host --dock-type host --dock-name r2 --with-bgp yes --bgp-config $(pwd)/r2_config
+spawn_docker_host --dock-type host --dock-name r3 --with-bgp yes --bgp-config $(pwd)/r3_config
+spawn_docker_host --dock-type host --dock-name sw1
+spawn_docker_host --dock-type host --dock-name user
+
+echo "#########################################"
+echo "Connecting and configuring  hosts"
+echo "#########################################"
+
+connect_docker_hosts user r1
+connect_docker_hosts r1 sw1
+connect_docker_hosts sw1 llb1
+connect_docker_hosts user r3
+connect_docker_hosts r3 sw1
+connect_docker_hosts sw1 llb2
+connect_docker_hosts llb1 r2
+connect_docker_hosts llb2 r2
+connect_docker_hosts r2 ep1
+connect_docker_hosts r2 ep2
+connect_docker_hosts r2 ep3
+
+create_docker_host_cnbridge --host1 sw1 --host2 llb1
+create_docker_host_cnbridge --host1 sw1 --host2 llb2
+create_docker_host_cnbridge --host1 sw1 --host2 r1
+create_docker_host_cnbridge --host1 sw1 --host2 r3
+
+## Make network for k3s connectivity
+sudo ip link add ellb1sys type veth peer name esysllb1
+sleep 3
+sudo ip link set ellb1sys netns llb1
+sleep 3
+sudo ip -n llb1 link set ellb1sys up
+sudo ip -n llb1 addr add 12.12.12.1/24 dev ellb1sys
+sudo ip link set esysllb1 up
+sudo ip addr add 12.12.12.254/24 dev esysllb1
+
+sudo ip link add ellb2sys type veth peer name esysllb2
+sleep 3
+sudo ip link set ellb2sys netns llb2
+sleep 3
+sudo ip -n llb2 link set ellb2sys up
+sudo ip -n llb2 addr add 14.14.14.1/24 dev ellb2sys
+sudo ip link set esysllb2 up
+sudo ip addr add 14.14.14.254/24 dev esysllb2
+
+#node1 config
+config_docker_host --host1 user --host2 r1 --ptype phy --addr 1.1.1.1/24 --gw 1.1.1.254
+config_docker_host --host1 r1 --host2 user --ptype phy --addr 1.1.1.254/24
+create_docker_host_vlan --host1 llb1 --host2 sw1 --id 11 --ptype untagged
+create_docker_host_vlan --host1 llb2 --host2 sw1 --id 11 --ptype untagged
+create_docker_host_vlan --host1 r1 --host2 sw1 --id 11 --ptype untagged
+create_docker_host_vlan --host1 r3 --host2 sw1 --id 11 --ptype untagged
+config_docker_host --host1 r1 --host2 sw1 --ptype vlan --id 11 --addr 11.11.11.254/24 --gw 11.11.11.11
+config_docker_host --host1 r3 --host2 sw1 --ptype vlan --id 11 --addr 11.11.11.253/24 --gw 11.11.11.11
+config_docker_host --host1 llb1 --host2 sw1 --ptype vlan --id 11 --addr 11.11.11.1/24 --gw 11.11.11.254
+config_docker_host --host1 llb2 --host2 sw1 --ptype vlan --id 11 --addr 11.11.11.2/24 --gw 11.11.11.254
+config_docker_host --host1 user --host2 r3 --ptype phy --addr 2.2.2.1/24 --gw 2.2.2.254
+config_docker_host --host1 r3 --host2 user --ptype phy --addr 2.2.2.254/24
+
+create_docker_host_vlan --host1 llb1 --host2 r2 --id 10 --ptype untagged
+config_docker_host --host1 llb1 --host2 r2 --ptype vlan --id 10 --addr 10.10.10.1/24
+create_docker_host_vlan --host1 llb2 --host2 r2 --id 10 --ptype untagged
+config_docker_host --host1 llb2 --host2 r2 --ptype vlan --id 10 --addr 10.10.10.2/24
+
+create_docker_host_vlan --host1 r2 --host2 llb1 --id 10 --ptype untagged
+create_docker_host_vlan --host1 r2 --host2 llb2 --id 10 --ptype untagged
+create_docker_host_vlan --host1 r2 --host2 ep1 --id 10 --ptype untagged
+create_docker_host_vlan --host1 r2 --host2 ep2 --id 10 --ptype untagged
+create_docker_host_vlan --host1 r2 --host2 ep3 --id 10 --ptype untagged
+config_docker_host --host1 r2 --host2 llb1 --ptype vlan --id 10 --addr 10.10.10.254/24
+
+create_docker_host_vlan --host1 r2 --host2 ep1 --id 31 --ptype untagged
+config_docker_host --host1 r2 --host2 ep1 --ptype vlan --id 31 --addr 31.31.31.254/24
+
+create_docker_host_vlan --host1 ep1 --host2 r2 --id 31 --ptype untagged
+config_docker_host --host1 ep1 --host2 r2 --ptype vlan --id 31 --addr 31.31.31.1/24 --gw 31.31.31.254
+
+create_docker_host_vlan --host1 r2 --host2 ep2 --id 32 --ptype untagged
+config_docker_host --host1 r2 --host2 ep2 --ptype vlan --id 32 --addr 32.32.32.254/24
+
+create_docker_host_vlan --host1 ep2 --host2 r2 --id 32 --ptype untagged
+config_docker_host --host1 ep2 --host2 r2 --ptype vlan --id 32 --addr 32.32.32.1/24 --gw 32.32.32.254
+
+create_docker_host_vlan --host1 r2 --host2 ep3 --id 33 --ptype untagged
+config_docker_host --host1 r2 --host2 ep3 --ptype vlan --id 33 --addr 33.33.33.254/24
+
+create_docker_host_vlan --host1 ep3 --host2 r2 --id 33 --ptype untagged
+config_docker_host --host1 ep3 --host2 r2 --ptype vlan --id 33 --addr 33.33.33.1/24 --gw 33.33.33.254
+
+$hexec user ip route change default via 1.1.1.254
+
+##Pod networks
+$hexec r1 ip route add 20.20.20.1/32 via 11.11.11.11
+#add_route llb1 1.1.1.0/24 11.11.11.254
+#add_route llb2 1.1.1.0/24 11.11.11.254
+
+# Route back to user
+sudo ip route add 11.11.11.0/24 via 12.12.12.1
+
+# Change default route in llb1
+$hexec llb1 ip route del default 
+$hexec llb1 ip route add default via 12.12.12.254
+
+# Change default route in llb2
+$hexec llb2 ip route del default 
+$hexec llb2 ip route add default via 14.14.14.254
+
+sleep 1
+##Create LB rule
+create_lb_rule llb1 20.20.20.1 --tcp=2020:8080 --endpoints=31.31.31.1:1,32.32.32.1:1,33.33.33.1:1 --mode=fullnat --bgp
+create_lb_rule llb2 20.20.20.1 --tcp=2020:8080 --endpoints=31.31.31.1:1,32.32.32.1:1,33.33.33.1:1 --mode=fullnat --bgp
+
+# keepalive will take few seconds to be UP and running with valid states
+sleep 60
+
+# K3s setup
+
+if [ "$1" ]; then
+  KUBECONFIG="$1"
+fi
+
+# If k3s setup exists, skip installation
+if [[ -f "/usr/local/bin/k3s-uninstall.sh" ]]; then
+  echo "K3s exists"
+  sleep 10
+else
+  echo "Start K3s installation"
+
+  # Install k3s without external cloud-manager and disabled servicelb
+  curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh -
+  #curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh -
+
+  sleep 10
+
+  # Check kubectl works
+  kubectl $KUBECONFIG get pods -A
+
+  # Remove taints in k3s if any (usually happens if started without cloud-manager)
+  kubectl $KUBECONFIG taint nodes --all node.cloudprovider.kubernetes.io/uninitialized=false:NoSchedule-
+
+  # Start loxi-ccm as k3s daemonset
+  kubectl $KUBECONFIG apply -f https://github.com/loxilb-io/loxi-ccm/raw/master/manifests/loxi-ccm-k3s.yaml
+
+  echo "End K3s installation"
+fi
+
+# Install Bird to work with k3s
+sudo apt install bird2 --yes
+
+sleep 5
+
+sudo cp -f bird_config/bird.conf /etc/bird/bird.conf
+if [ ! -f  /var/log/bird.log ]; then
+  sudo touch /var/log/bird.log
+fi
+sudo chown bird:bird /var/log/bird.log
+sudo systemctl restart bird
+
+sleep 10
+
+# Wait for cluster to be ready
+wait_cluster_ready_full
+
+# Start nginx pods and services for test(using kube-loxilb)
+kubectl $KUBECONFIG apply -f kube-loxilb.yml
+sleep 15
+kubectl $KUBECONFIG apply -f nginx-svc-lb1.yml
+
+#Build sctp-server image locally
+#docker build -t loxilb-io/sctp-server .
+#docker save --output sctp-server.tar loxilb-io/sctp-server
+#sudo k3s ctr images import ./sctp-server.tar
+kubectl $KUBECONFIG apply -f sctp-svc-lb.yml
+
+sleep 50
+
+# Wait for cluster to be ready
+wait_cluster_ready_full
+
+# External LB service must be created by now
+echo "kubectl $KUBECONFIG get svc"
+echo "****************************"
+kubectl $KUBECONFIG get svc
+echo "kubectl $KUBECONFIG get pods -A"
+echo "****************************"
+kubectl $KUBECONFIG get pods -A
+
+echo "llb1: loxicmd get lb -o wide"
+echo "****************************"
+$dexec llb1 loxicmd get lb -o wide
+echo "llb1: loxicmd get ep -o wide"
+echo "****************************"
+$dexec llb1 loxicmd get ep -o wide
+$dexec llb1 cat /etc/shared/keepalive.state
+echo "llb2: loxicmd get lb -o wide"
+echo "****************************"
+$dexec llb2 loxicmd get lb -o wide
+echo "llb2: loxicmd get ep -o wide"
+echo "****************************"
+$dexec llb2 loxicmd get ep -o wide
+$dexec llb2 cat /etc/shared/keepalive.state
diff --git a/cicd/k3s-sctpmh-2/input b/cicd/k3s-sctpmh-2/input
new file mode 100644
index 000000000..6fb66a5e2
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/input
@@ -0,0 +1,6 @@
+
+
+
+
+
+
diff --git a/cicd/k3s-sctpmh-2/keepalived_config1/keepalived.conf b/cicd/k3s-sctpmh-2/keepalived_config1/keepalived.conf
new file mode 100644
index 000000000..af33f93e8
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/keepalived_config1/keepalived.conf
@@ -0,0 +1,22 @@
+! Configuration File for keepalived
+
+global_defs {
+   smtp_server localhost
+   smtp_connect_timeout 30
+}
+
+vrrp_instance default {
+    state MASTER
+    interface vlan11
+    virtual_router_id 101
+    priority 200
+    advert_int 1
+    authentication {
+        auth_type PASS
+        auth_pass 1111
+    }
+    virtual_ipaddress {
+        11.11.11.11
+    }
+    notify "/etc/keepalived/notify.sh"
+}
diff --git a/cicd/k3s-sctpmh-2/keepalived_config1/notify.sh b/cicd/k3s-sctpmh-2/keepalived_config1/notify.sh
new file mode 100755
index 000000000..c716b39f6
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/keepalived_config1/notify.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+declare -A vip
+vip["default"]="11.11.11.11"
+echo $1 $2 is in $3 state vip ${vip[$2]}> /etc/shared/keepalive.state
+curl -X 'POST' 'http://0.0.0.0:11111/netlox/v1/config/cistate' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{ "instance": "'$2'", "state" : "'$3'", "vip" : "'${vip[$2]}'" }'
diff --git a/cicd/k3s-sctpmh-2/keepalived_config2/keepalived.conf b/cicd/k3s-sctpmh-2/keepalived_config2/keepalived.conf
new file mode 100644
index 000000000..5d06dc663
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/keepalived_config2/keepalived.conf
@@ -0,0 +1,22 @@
+! Configuration File for keepalived
+
+global_defs {
+   smtp_server localhost
+   smtp_connect_timeout 30
+}
+
+vrrp_instance default {
+    state MASTER
+    interface vlan11
+    virtual_router_id 101
+    priority 100
+    advert_int 1
+    authentication {
+        auth_type PASS
+        auth_pass 1111
+    }
+    virtual_ipaddress {
+        11.11.11.11
+    }
+    notify "/etc/keepalived/notify.sh"
+}
diff --git a/cicd/k3s-sctpmh-2/keepalived_config2/notify.sh b/cicd/k3s-sctpmh-2/keepalived_config2/notify.sh
new file mode 100755
index 000000000..c716b39f6
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/keepalived_config2/notify.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+declare -A vip
+vip["default"]="11.11.11.11"
+echo $1 $2 is in $3 state vip ${vip[$2]}> /etc/shared/keepalive.state
+curl -X 'POST' 'http://0.0.0.0:11111/netlox/v1/config/cistate' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{ "instance": "'$2'", "state" : "'$3'", "vip" : "'${vip[$2]}'" }'
diff --git a/cicd/k3s-sctpmh-2/kube-loxilb.yml b/cicd/k3s-sctpmh-2/kube-loxilb.yml
new file mode 100644
index 000000000..1c78456a7
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/kube-loxilb.yml
@@ -0,0 +1,132 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-loxilb
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-loxilb
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+      - services
+      - services/status
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+      - update
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-loxilb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-loxilb
+subjects:
+  - kind: ServiceAccount
+    name: kube-loxilb
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-loxilb
+  namespace: kube-system
+  labels:
+    app: loxilb
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: loxilb
+  template:
+    metadata:
+      labels:
+        app: loxilb
+    spec:
+      hostNetwork: true
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        # Mark the pod as a critical add-on for rescheduling.
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+      priorityClassName: system-node-critical
+      serviceAccountName: kube-loxilb
+      terminationGracePeriodSeconds: 0
+      containers:
+      - name: kube-loxilb
+        image: ghcr.io/loxilb-io/kube-loxilb:latest
+        imagePullPolicy: Always
+        command:
+        - /bin/kube-loxilb
+        args:
+        - --loxiURL=http://12.12.12.1:11111,http://14.14.14.1:11111
+        - --externalCIDR=123.123.123.1/24
+        - --externalSecondaryCIDRs=124.124.124.1/24,125.125.125.1/24
+        #- --monitor
+        - --setBGP=64511
+        #- --extBGPPeers=50.50.50.1:65101,51.51.51.1:65102
+        - --setRoles
+        #- --setLBMode=2
+        #- --config=/opt/loxilb/agent/kube-loxilb.conf
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+          capabilities:
+            add: ["NET_ADMIN", "NET_RAW"]
diff --git a/cicd/k3s-sctpmh-2/llb1_gobgp_config/gobgp.conf b/cicd/k3s-sctpmh-2/llb1_gobgp_config/gobgp.conf
new file mode 100644
index 000000000..ced38fdcc
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/llb1_gobgp_config/gobgp.conf
@@ -0,0 +1,51 @@
+[global.config]
+  as = 64512
+  router-id = "10.10.10.1"
+
+[[neighbors]]
+  [neighbors.config]
+    neighbor-address = "10.10.10.254"
+    peer-as = 64512
+
+[[neighbors]]
+  [neighbors.config]
+    neighbor-address = "12.12.12.254"
+    peer-as = 64512
+
+[[neighbors]]
+  [neighbors.config]
+    neighbor-address = "11.11.11.254"
+    peer-as = 64511
+
+[[neighbors]]
+  [neighbors.config]
+    neighbor-address = "11.11.11.253"
+    peer-as = 64511
+
+[global.apply-policy.config]
+  export-policy-list = [ "set-next-hop-self", "ext" ]
+
+[[defined-sets.neighbor-sets]]
+  neighbor-set-name = "k3s"
+  neighbor-info-list = [ "12.12.12.254", "10.10.10.254" ]
+
+[[defined-sets.neighbor-sets]]
+  neighbor-set-name = "ebgpn"
+  neighbor-info-list = [ "11.11.11.254", "11.11.11.253" ]
+
+[[policy-definitions]]
+  name = "set-next-hop-self"
+  [[policy-definitions.statements]]
+    [policy-definitions.statements.conditions.match-neighbor-set]
+      neighbor-set = "k3s"
+    [policy-definitions.statements.actions.bgp-actions]
+      set-next-hop = "12.12.12.1"
+      set-local-pref = 200
+
+[[policy-definitions]]
+  name = "ext"
+  [[policy-definitions.statements]]
+    [policy-definitions.statements.conditions.match-neighbor-set]
+      neighbor-set = "ebgpn"
+    [policy-definitions.statements.actions.bgp-actions]
+      set-next-hop = "11.11.11.1"
diff --git a/cicd/k3s-sctpmh-2/llb2_gobgp_config/gobgp.conf b/cicd/k3s-sctpmh-2/llb2_gobgp_config/gobgp.conf
new file mode 100644
index 000000000..25b273eca
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/llb2_gobgp_config/gobgp.conf
@@ -0,0 +1,50 @@
+[global.config]
+  as = 64512
+  router-id = "10.10.10.2"
+
+[[neighbors]]
+  [neighbors.config]
+    neighbor-address = "10.10.10.254"
+    peer-as = 64512
+
+[[neighbors]]
+  [neighbors.config]
+    neighbor-address = "14.14.14.254"
+    peer-as = 64512
+
+[[neighbors]]
+  [neighbors.config]
+    neighbor-address = "11.11.11.254"
+    peer-as = 64511
+
+[[neighbors]]
+  [neighbors.config]
+    neighbor-address = "11.11.11.253"
+    peer-as = 64511
+
+[global.apply-policy.config]
+  export-policy-list = [ "set-next-hop-self", "ext" ]
+
+[[defined-sets.neighbor-sets]]
+  neighbor-set-name = "k3s"
+  neighbor-info-list = [ "14.14.14.254", "10.10.10.254" ]
+
+[[defined-sets.neighbor-sets]]
+  neighbor-set-name = "ebgpn"
+  neighbor-info-list = [ "11.11.11.254", "11.11.11.253" ]
+
+[[policy-definitions]]
+  name = "set-next-hop-self"
+  [[policy-definitions.statements]]
+    [policy-definitions.statements.conditions.match-neighbor-set]
+      neighbor-set = "k3s"
+    [policy-definitions.statements.actions.bgp-actions]
+      set-next-hop = "14.14.14.1"
+
+[[policy-definitions]]
+  name = "ext"
+  [[policy-definitions.statements]]
+    [policy-definitions.statements.conditions.match-neighbor-set]
+      neighbor-set = "ebgpn"
+    [policy-definitions.statements.actions.bgp-actions]
+      set-next-hop = "11.11.11.2"
diff --git a/cicd/k3s-sctpmh-2/nginx-svc-lb1.yml b/cicd/k3s-sctpmh-2/nginx-svc-lb1.yml
new file mode 100644
index 000000000..4f9c2edf1
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/nginx-svc-lb1.yml
@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-lb1
+spec:
+  externalTrafficPolicy: Local
+  loadBalancerClass: loxilb.io/loxilb
+  selector:
+    what: nginx-test
+  ports:
+    - port: 55002
+      targetPort: 80 
+  type: LoadBalancer
+  #externalIPs:
+  #  - 1.2.4.120
+  #  - 1.2.4.121
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-test
+  labels:
+    what: nginx-test
+spec:
+  containers:
+    - name: nginx-test
+      image: nginx:stable
+      ports:
+        - containerPort: 80
diff --git a/cicd/k3s-sctpmh-2/r1_config/bgpd.conf b/cicd/k3s-sctpmh-2/r1_config/bgpd.conf
new file mode 100755
index 000000000..14d1cd77e
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/r1_config/bgpd.conf
@@ -0,0 +1,10 @@
+!
+password zebra
+enable password zebra
+log file /var/log/quagga/bgpd.log
+router bgp 64511
+bgp router-id 11.11.11.254
+neighbor 11.11.11.1 remote-as 64512
+neighbor 11.11.11.2 remote-as 64512
+redistribute connected
+!
diff --git a/cicd/k3s-sctpmh-2/r1_config/daemons b/cicd/k3s-sctpmh-2/r1_config/daemons
new file mode 100644
index 000000000..1a7224103
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/r1_config/daemons
@@ -0,0 +1,8 @@
+zebra=yes
+bgpd=yes
+ospfd=no
+ospf6d=no
+ripd=no
+ripngd=no
+isisd=no
+babeld=no
diff --git a/cicd/k3s-sctpmh-2/r1_config/zebra.conf b/cicd/k3s-sctpmh-2/r1_config/zebra.conf
new file mode 100755
index 000000000..71b440df1
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/r1_config/zebra.conf
@@ -0,0 +1,6 @@
+!
+hostname Router
+password zebra
+enable password zebra
+log file /var/log/quagga/zebra.log
+!
diff --git a/cicd/k3s-sctpmh-2/r2_config/bgpd.conf b/cicd/k3s-sctpmh-2/r2_config/bgpd.conf
new file mode 100755
index 000000000..f20a16857
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/r2_config/bgpd.conf
@@ -0,0 +1,10 @@
+!
+password zebra
+enable password zebra
+log file /var/log/quagga/bgpd.log
+router bgp 64512
+bgp router-id 10.10.10.254
+neighbor 10.10.10.1 remote-as 64512
+neighbor 10.10.10.2 remote-as 64512
+redistribute connected
+!
diff --git a/cicd/k3s-sctpmh-2/r2_config/daemons b/cicd/k3s-sctpmh-2/r2_config/daemons
new file mode 100644
index 000000000..1a7224103
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/r2_config/daemons
@@ -0,0 +1,8 @@
+zebra=yes
+bgpd=yes
+ospfd=no
+ospf6d=no
+ripd=no
+ripngd=no
+isisd=no
+babeld=no
diff --git a/cicd/k3s-sctpmh-2/r2_config/zebra.conf b/cicd/k3s-sctpmh-2/r2_config/zebra.conf
new file mode 100755
index 000000000..71b440df1
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/r2_config/zebra.conf
@@ -0,0 +1,6 @@
+!
+hostname Router
+password zebra
+enable password zebra
+log file /var/log/quagga/zebra.log
+!
diff --git a/cicd/k3s-sctpmh-2/r3_config/bgpd.conf b/cicd/k3s-sctpmh-2/r3_config/bgpd.conf
new file mode 100755
index 000000000..54f9686dc
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/r3_config/bgpd.conf
@@ -0,0 +1,10 @@
+!
+password zebra
+enable password zebra
+log file /var/log/quagga/bgpd.log
+router bgp 64511
+bgp router-id 11.11.11.253
+neighbor 11.11.11.1 remote-as 64512
+neighbor 11.11.11.2 remote-as 64512
+redistribute connected
+!
diff --git a/cicd/k3s-sctpmh-2/r3_config/daemons b/cicd/k3s-sctpmh-2/r3_config/daemons
new file mode 100644
index 000000000..1a7224103
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/r3_config/daemons
@@ -0,0 +1,8 @@
+zebra=yes
+bgpd=yes
+ospfd=no
+ospf6d=no
+ripd=no
+ripngd=no
+isisd=no
+babeld=no
diff --git a/cicd/k3s-sctpmh-2/r3_config/zebra.conf b/cicd/k3s-sctpmh-2/r3_config/zebra.conf
new file mode 100755
index 000000000..71b440df1
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/r3_config/zebra.conf
@@ -0,0 +1,6 @@
+!
+hostname Router
+password zebra
+enable password zebra
+log file /var/log/quagga/zebra.log
+!
diff --git a/cicd/k3s-sctpmh-2/rmconfig.sh b/cicd/k3s-sctpmh-2/rmconfig.sh
new file mode 100755
index 000000000..3ab1017ee
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/rmconfig.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+echo "#########################################"
+echo "Removing testbed"
+echo "#########################################"
+
+if [ "$1" ]; then
+  KUBECONFIG="$1"
+fi
+
+source ../common.sh
+
+sudo kubectl $KUBECONFIG delete -f nginx-svc-lb1.yml >> /dev/null 2>&1
+sudo kubectl $KUBECONFIG delete -f sctp-svc-lb.yml >> /dev/null 2>&1
+sudo kubectl $KUBECONFIG delete -f kube-loxilb.yml >> /dev/null 2>&1
+sudo kubectl $KUBECONFIG delete -f nginx-svc-lb.yml >> /dev/null 2>&1
+sudo kubectl $KUBECONFIG delete -f nginx.yml >> /dev/null 2>&1
+sudo kubectl $KUBECONFIG delete -f https://github.com/loxilb-io/loxi-ccm/raw/master/manifests/loxi-ccm-k3s.yaml >> /dev/null 2>&1
+
+disconnect_docker_hosts user r1
+disconnect_docker_hosts user r3
+disconnect_docker_hosts r1 sw1
+disconnect_docker_hosts r3 sw1 
+disconnect_docker_hosts sw1 llb1
+disconnect_docker_hosts sw1 llb2
+disconnect_docker_hosts llb1 r2
+disconnect_docker_hosts llb2 r2
+disconnect_docker_hosts r2 ep1
+disconnect_docker_hosts r2 ep2
+disconnect_docker_hosts r2 ep3
+
+delete_docker_host ka_llb1
+delete_docker_host ka_llb2
+delete_docker_host llb1
+delete_docker_host llb2
+delete_docker_host user
+delete_docker_host r1
+delete_docker_host r2
+delete_docker_host r3
+delete_docker_host sw1
+delete_docker_host ep1
+delete_docker_host ep2
+delete_docker_host ep3
+sudo ip link del esysllb1
+sudo ip link del esysllb2
+
+# If k3s setup exists, remove it
+if [[ -f "/usr/local/bin/k3s-uninstall.sh" ]]; then
+  /usr/local/bin/k3s-uninstall.sh
+fi
+
+sudo apt-get remove bird2 --yes
+#docker image rm loxilb-io/sctp-server
+#docker images -a | grep "loxilb-io" | awk '{print $3}' | xargs docker rmi
+echo "#########################################"
+echo "Removed testbed"
+echo "#########################################"
diff --git a/cicd/k3s-sctpmh-2/sctp-svc-lb.yml b/cicd/k3s-sctpmh-2/sctp-svc-lb.yml
new file mode 100644
index 000000000..32b749bcc
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/sctp-svc-lb.yml
@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: sctp-lb1
+  annotations:
+    loxilb.io/num-secondary-networks: "2"
+    loxilb.io/lbmode: "fullnat"
+    loxilb.io/liveness: "yes"
+spec:
+  externalTrafficPolicy: Local
+  loadBalancerClass: loxilb.io/loxilb
+  selector:
+    what: sctp-test
+  ports:
+    - port: 55003
+      protocol: SCTP
+      targetPort: 9999
+  type: LoadBalancer
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sctp-test
+  labels:
+    what: sctp-test
+spec:
+  tolerations:
+  - key: "node.kubernetes.io/disk-pressure"
+    operator: "Exists"
+    effect: "NoSchedule"
+  containers:
+    - name: sctp-test
+      image: loxilbio/sctp-darn:latest
+      imagePullPolicy: Always
+      #command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 6 ; done"]
+      command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"]
+      ports:
+        - containerPort: 9999
diff --git a/cicd/k3s-sctpmh-2/validation.sh b/cicd/k3s-sctpmh-2/validation.sh
new file mode 100755
index 000000000..b5391e36d
--- /dev/null
+++ b/cicd/k3s-sctpmh-2/validation.sh
@@ -0,0 +1,146 @@
+#!/bin/bash
+source ../common.sh
+source ../k3s_common.sh
+
+echo "cluster-k3s: TCP & SCTP Multihoming combined"
+
+if [ "$1" ]; then
+  KUBECONFIG="$1"
+fi
+
+# Set space as the delimiter
+IFS=' '
+
+for((i=0; i<120; i++))
+do
+  extLB=$(sudo kubectl $KUBECONFIG get svc | grep "nginx-lb1")
+  read -a strarr <<< "$extLB"
+  len=${#strarr[*]}
+  if [[ $((len)) -lt 6 ]]; then
+    echo "Can't find nginx-lb service"
+    sleep 1
+    continue
+  fi 
+  if [[ ${strarr[3]} != *"none"* ]]; then
+    extIP=${strarr[3]}
+    port=${strarr[4]}
+    break
+  fi
+  echo "No external LB allocated"
+  sleep 1
+done
+
+## Any routing updates  ??
+#sleep 30
+echo $extIP
+
+out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) 
+if [[ ${out} == *"Welcome to nginx"* ]]; then
+  echo "cluster-k3s TCP service nginx-lb (kube-loxilb) [OK]"
+else
+  echo "cluster-k3s TCP service nginx-lb (kube-loxilb) [FAILED]"
+  ## Dump some debug info
+  echo "llb1 lb-info"
+  $dexec llb1 loxicmd get lb
+  echo "llb1 route-info"
+  $dexec llb1 ip route
+  echo "llb2 lb-info"
+  $dexec llb2 loxicmd get lb
+  echo "llb2 route-info"
+  $dexec llb2 ip route
+  echo "r1 route-info"
+  $dexec r1 ip route
+  exit 1
+fi
+
+for((i=0; i<120; i++))
+do
+  extLB=$(sudo kubectl $KUBECONFIG get svc | grep "sctp-lb1")
+  read -a strarr <<< "$extLB"
+  len=${#strarr[*]}
+  if [[ $((len)) -lt 6 ]]; then
+    echo "Can't find sctp-lb1 service"
+    sleep 1
+    continue
+  fi 
+  if [[ ${strarr[3]} != *"none"* ]]; then
+    extIP=${strarr[3]}
+    port=${strarr[4]}
+    break
+  fi
+  echo "No external LB allocated"
+  sleep 1
+done
+
+echo "SCTP Multihoming service sctp-lb1 -> $extIP:$port"
+
+$hexec user sctp_darn -H 1.1.1.1 -h 123.123.123.1 -p 55003 -s < input > output
+sleep 5
+exp="New connection, peer addresses
+123.123.123.1:55003
+124.124.124.1:55003
+125.125.125.1:55003"
+
+res=`cat output | grep -A 3 "New connection, peer addresses"`
+sudo rm -rf output
+if [[ "$res" == "$exp" ]]; then
+    echo $res
+    echo "cluster-k3s SCTP Multihoming service sctp-lb1 (kube-loxilb) [OK]"
+else
+    echo "cluster-k3s SCTP Multihoming service sctp-lb1 (kube-loxilb) [NOK]"
+    echo "Expected : $exp"
+    echo "Received : $res"
+    ## Dump some debug info
+    echo "system route-info"
+    ip route
+    echo "system ipables"
+    sudo iptables -n -t nat -L -v  |grep sctp
+    echo "llb1 lb-info"
+    $dexec llb1 loxicmd get lb
+    echo "llb1 ep-info"
+    $dexec llb1 loxicmd get ep
+    echo "llb1 bpf-info"
+    $dexec llb1 ntc filter show dev eth0 ingress
+    echo "llb1 route-info"
+    $dexec llb1 ip route
+    echo "llb2 lb-info"
+    $dexec llb2 loxicmd get lb
+    echo "llb2 route-info"
+    $dexec llb2 ip route
+    echo "r1 route-info"
+    $dexec r1 ip route
+    echo "BFP trace -- "
+    sudo timeout 5 cat  /sys/kernel/debug/tracing/trace_pipe
+    sudo killall -9 cat
+    echo "BFP trace -- "
+    exit 1
+fi
+
+## Check delete and readd service
+kubectl $KUBECONFIG delete -f nginx-svc-lb1.yml
+sleep 10
+kubectl $KUBECONFIG apply -f nginx-svc-lb1.yml
+sleep 10
+
+# Wait for cluster to be ready
+wait_cluster_ready_full
+
+out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002)
+if [[ ${out} == *"Welcome to nginx"* ]]; then
+  echo "cluster-k3s TCP service nginx-lb del+add (kube-loxilb) [OK]"
+else
+  echo "cluster-k3s TCP service nginx-lb del+add (kube-loxilb) [FAILED]"
+  ## Dump some debug info
+  echo "llb1 lb-info"
+  $dexec llb1 loxicmd get lb
+  echo "llb1 route-info"
+  $dexec llb1 ip route
+  echo "llb2 lb-info"
+  $dexec llb2 loxicmd get lb
+  echo "llb2 route-info"
+  $dexec llb2 ip route
+  echo "r1 route-info"
+  $dexec r1 ip route
+  exit 1
+fi
+

From 2945e85e98e25a841a285c410b2167221ea1e559 Mon Sep 17 00:00:00 2001
From: Trekkie Coder <trekkie@netlox.io>
Date: Tue, 15 Aug 2023 11:52:58 +0900
Subject: [PATCH 02/11] updated cicd for sctp-multihoming

---
 .../keepalived_config1/keepalived.conf        | 22 -------------------
 .../k3s-sctpmh-2/keepalived_config1/notify.sh |  5 -----
 .../keepalived_config2/keepalived.conf        | 22 -------------------
 .../k3s-sctpmh-2/keepalived_config2/notify.sh |  5 -----
 4 files changed, 54 deletions(-)
 delete mode 100644 cicd/k3s-sctpmh-2/keepalived_config1/keepalived.conf
 delete mode 100755 cicd/k3s-sctpmh-2/keepalived_config1/notify.sh
 delete mode 100644 cicd/k3s-sctpmh-2/keepalived_config2/keepalived.conf
 delete mode 100755 cicd/k3s-sctpmh-2/keepalived_config2/notify.sh

diff --git a/cicd/k3s-sctpmh-2/keepalived_config1/keepalived.conf b/cicd/k3s-sctpmh-2/keepalived_config1/keepalived.conf
deleted file mode 100644
index af33f93e8..000000000
--- a/cicd/k3s-sctpmh-2/keepalived_config1/keepalived.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-! Configuration File for keepalived
-
-global_defs {
-   smtp_server localhost
-   smtp_connect_timeout 30
-}
-
-vrrp_instance default {
-    state MASTER
-    interface vlan11
-    virtual_router_id 101
-    priority 200
-    advert_int 1
-    authentication {
-        auth_type PASS
-        auth_pass 1111
-    }
-    virtual_ipaddress {
-        11.11.11.11
-    }
-    notify "/etc/keepalived/notify.sh"
-}
diff --git a/cicd/k3s-sctpmh-2/keepalived_config1/notify.sh b/cicd/k3s-sctpmh-2/keepalived_config1/notify.sh
deleted file mode 100755
index c716b39f6..000000000
--- a/cicd/k3s-sctpmh-2/keepalived_config1/notify.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-declare -A vip
-vip["default"]="11.11.11.11"
-echo $1 $2 is in $3 state vip ${vip[$2]}> /etc/shared/keepalive.state
-curl -X 'POST' 'http://0.0.0.0:11111/netlox/v1/config/cistate' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{ "instance": "'$2'", "state" : "'$3'", "vip" : "'${vip[$2]}'" }'
diff --git a/cicd/k3s-sctpmh-2/keepalived_config2/keepalived.conf b/cicd/k3s-sctpmh-2/keepalived_config2/keepalived.conf
deleted file mode 100644
index 5d06dc663..000000000
--- a/cicd/k3s-sctpmh-2/keepalived_config2/keepalived.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-! Configuration File for keepalived
-
-global_defs {
-   smtp_server localhost
-   smtp_connect_timeout 30
-}
-
-vrrp_instance default {
-    state MASTER
-    interface vlan11
-    virtual_router_id 101
-    priority 100
-    advert_int 1
-    authentication {
-        auth_type PASS
-        auth_pass 1111
-    }
-    virtual_ipaddress {
-        11.11.11.11
-    }
-    notify "/etc/keepalived/notify.sh"
-}
diff --git a/cicd/k3s-sctpmh-2/keepalived_config2/notify.sh b/cicd/k3s-sctpmh-2/keepalived_config2/notify.sh
deleted file mode 100755
index c716b39f6..000000000
--- a/cicd/k3s-sctpmh-2/keepalived_config2/notify.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-declare -A vip
-vip["default"]="11.11.11.11"
-echo $1 $2 is in $3 state vip ${vip[$2]}> /etc/shared/keepalive.state
-curl -X 'POST' 'http://0.0.0.0:11111/netlox/v1/config/cistate' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{ "instance": "'$2'", "state" : "'$3'", "vip" : "'${vip[$2]}'" }'

From 2f68891370306fa4341bb9cc58318bfd7939e1f2 Mon Sep 17 00:00:00 2001
From: Trekkie Coder <trekkie@netlox.io>
Date: Wed, 16 Aug 2023 12:15:10 +0900
Subject: [PATCH 03/11] updated cicd for sctp-multihoming

---
 cicd/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cicd/common.sh b/cicd/common.sh
index 5e8861b8b..b82f6e0e4 100644
--- a/cicd/common.sh
+++ b/cicd/common.sh
@@ -128,7 +128,7 @@ spawn_docker_host() {
       fi
     else
       docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log $loxilb_config --name $dname $lxdocker $bgp_opts
-      docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts
+      docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts
     fi
   elif [[ "$dtype" == "host" ]]; then
     if [[ ! -z "$bpath" ]]; then

From bd5e0d2a8d31423961b4f3521fb525c12bfc2a95 Mon Sep 17 00:00:00 2001
From: Trekkie Coder <trekkie@netlox.io>
Date: Wed, 16 Aug 2023 14:38:44 +0900
Subject: [PATCH 04/11] updated cicd for sctp-multihoming

---
 cicd/k3s-sctpmh-2/config.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cicd/k3s-sctpmh-2/config.sh b/cicd/k3s-sctpmh-2/config.sh
index 9aaf8fd6f..cde82295e 100755
--- a/cicd/k3s-sctpmh-2/config.sh
+++ b/cicd/k3s-sctpmh-2/config.sh
@@ -39,6 +39,9 @@ create_docker_host_cnbridge --host1 sw1 --host2 llb2
 create_docker_host_cnbridge --host1 sw1 --host2 r1
 create_docker_host_cnbridge --host1 sw1 --host2 r3
 
+$hexec user ip route add 124.124.124.1/32 via 2.2.2.254
+$hexec user ip route add 125.125.125.1/32 via 2.2.2.254
+
 ## Make network for k3s connectivity
 sudo ip link add ellb1sys type veth peer name esysllb1
 sleep 3

From aff17c63bf6f2c2966ec97995bb21136ca4d365b Mon Sep 17 00:00:00 2001
From: Trekkie Coder <trekkie@netlox.io>
Date: Wed, 16 Aug 2023 22:11:30 +0900
Subject: [PATCH 05/11] Updated to latest submodule

---
 loxilb-ebpf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/loxilb-ebpf b/loxilb-ebpf
index c716e58cd..ea174aca9 160000
--- a/loxilb-ebpf
+++ b/loxilb-ebpf
@@ -1 +1 @@
-Subproject commit c716e58cd01b5943e30db2579bda75890685d2b6
+Subproject commit ea174aca9b419252fb6461b2fe423c98e32a4246

From 50cfb491688eeff626101e6526946bee178babcd Mon Sep 17 00:00:00 2001
From: TrekkieCoder <111065900+TrekkieCoder@users.noreply.github.com>
Date: Wed, 16 Aug 2023 22:18:10 +0900
Subject: [PATCH 06/11] Update eks.yaml

---
 .github/workflows/eks.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/eks.yaml b/.github/workflows/eks.yaml
index d9ff52dd1..5a023658a 100644
--- a/.github/workflows/eks.yaml
+++ b/.github/workflows/eks.yaml
@@ -36,7 +36,7 @@ jobs:
     # Run this only on main repo
     - name: Check Repo owner
       if: github.repository != 'loxilb-io/loxilb'
-      run: exit 0
+      run: exit 1
 
     # AWS check 
     - name: Configure AWS credentials

From 44c75edcc345c1f8c9fd962731a61d48b457de1f Mon Sep 17 00:00:00 2001
From: TrekkieCoder <111065900+TrekkieCoder@users.noreply.github.com>
Date: Wed, 16 Aug 2023 22:28:55 +0900
Subject: [PATCH 07/11] Update eks.yaml

---
 .github/workflows/eks.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/eks.yaml b/.github/workflows/eks.yaml
index 5a023658a..d8ca2b3b6 100644
--- a/.github/workflows/eks.yaml
+++ b/.github/workflows/eks.yaml
@@ -36,7 +36,9 @@ jobs:
     # Run this only on main repo
     - name: Check Repo owner
       if: github.repository != 'loxilb-io/loxilb'
-      run: exit 1
+      run: |
+          gh run cancel ${{ github.run_id }}
+          gh run watch ${{ github.run_id }}
 
     # AWS check 
     - name: Configure AWS credentials

From e723f0a2ac08dd7466f5ef2dfa9cde220e6c85f5 Mon Sep 17 00:00:00 2001
From: TrekkieCoder <111065900+TrekkieCoder@users.noreply.github.com>
Date: Wed, 16 Aug 2023 22:31:25 +0900
Subject: [PATCH 08/11] Update eks.yaml

---
 .github/workflows/eks.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/eks.yaml b/.github/workflows/eks.yaml
index d8ca2b3b6..9ec43936d 100644
--- a/.github/workflows/eks.yaml
+++ b/.github/workflows/eks.yaml
@@ -39,6 +39,8 @@ jobs:
       run: |
           gh run cancel ${{ github.run_id }}
           gh run watch ${{ github.run_id }}
+      env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     # AWS check 
     - name: Configure AWS credentials

From 8eac370977b23e133a0d8de929d28c74cdf8d4b4 Mon Sep 17 00:00:00 2001
From: TrekkieCoder <111065900+TrekkieCoder@users.noreply.github.com>
Date: Wed, 16 Aug 2023 22:44:18 +0900
Subject: [PATCH 09/11] Update eks.yaml

---
 .github/workflows/eks.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/eks.yaml b/.github/workflows/eks.yaml
index 9ec43936d..082ceb44d 100644
--- a/.github/workflows/eks.yaml
+++ b/.github/workflows/eks.yaml
@@ -12,7 +12,7 @@ on:
         default: 'EKS'
   
 permissions:
-  contents: read
+  contents: write-all
 env: 
   AWS_REGION: ap-northeast-3
 
@@ -140,6 +140,8 @@ jobs:
         kubectl delete -f kube-loxilb.yaml
         
     - name : delete testbed
-      if: ${{ always() }}
+      if: | 
+        ${{ always() && 
+            github.repository == 'loxilb-io/loxilb' }}
       run: |
         terraform destroy  -auto-approve -input=false

From 5ca8308bc794aa7511a74e7a7a073236d46ecf2c Mon Sep 17 00:00:00 2001
From: TrekkieCoder <111065900+TrekkieCoder@users.noreply.github.com>
Date: Wed, 16 Aug 2023 22:48:02 +0900
Subject: [PATCH 10/11] Update eks.yaml

---
 .github/workflows/eks.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/eks.yaml b/.github/workflows/eks.yaml
index 082ceb44d..5e36dbaa0 100644
--- a/.github/workflows/eks.yaml
+++ b/.github/workflows/eks.yaml
@@ -12,12 +12,13 @@ on:
         default: 'EKS'
   
 permissions:
-  contents: write-all
+  contents: read
 env: 
   AWS_REGION: ap-northeast-3
 
 jobs:
   terraform:
+    permissions: write-all
     name: 'Terraform with eks'
     runs-on: ubuntu-latest
     environment: production

From f5ca49c2a6d8fb6ef1cac01556af3f3a3a496ff2 Mon Sep 17 00:00:00 2001
From: TrekkieCoder <111065900+TrekkieCoder@users.noreply.github.com>
Date: Wed, 16 Aug 2023 22:54:48 +0900
Subject: [PATCH 11/11] Update eks.yaml

---
 .github/workflows/eks.yaml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/eks.yaml b/.github/workflows/eks.yaml
index 5e36dbaa0..51e24141e 100644
--- a/.github/workflows/eks.yaml
+++ b/.github/workflows/eks.yaml
@@ -141,8 +141,6 @@ jobs:
         kubectl delete -f kube-loxilb.yaml
         
     - name : delete testbed
-      if: | 
-        ${{ always() && 
-            github.repository == 'loxilb-io/loxilb' }}
+      if: ${{ ! cancelled() }}
       run: |
         terraform destroy  -auto-approve -input=false