From 9fdb97a60f45f228d682ec135d6a20c7e9b2a737 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 22 Jun 2023 23:29:41 -0400 Subject: [PATCH 1/2] k3s-calico-dual-stack added in CICD --- cicd/common.sh | 1 + cicd/k3s-calico-dual-stack/config.sh | 262 +++++ .../custom-resources.yaml | 32 + .../custom-resources.yaml.bk | 32 + cicd/k3s-calico-dual-stack/install_k3s.sh | 37 + cicd/k3s-calico-dual-stack/k3s.sh | 1047 +++++++++++++++++ cicd/k3s-calico-dual-stack/kube-loxilb.yml | 130 ++ cicd/k3s-calico-dual-stack/nginx-svc-lb.yml | 11 + .../nginx-svc-lb1-ipv6.yml | 35 + cicd/k3s-calico-dual-stack/nginx-svc-lb1.yml | 26 + cicd/k3s-calico-dual-stack/nginx.yml | 25 + cicd/k3s-calico-dual-stack/rmconfig.sh | 49 + cicd/k3s-calico-dual-stack/sctp-svc-lb.yml | 41 + cicd/k3s-calico-dual-stack/udp-svc-lb.yml | 30 + cicd/k3s-calico-dual-stack/validation.sh | 99 ++ 15 files changed, 1857 insertions(+) create mode 100755 cicd/k3s-calico-dual-stack/config.sh create mode 100644 cicd/k3s-calico-dual-stack/custom-resources.yaml create mode 100644 cicd/k3s-calico-dual-stack/custom-resources.yaml.bk create mode 100755 cicd/k3s-calico-dual-stack/install_k3s.sh create mode 100644 cicd/k3s-calico-dual-stack/k3s.sh create mode 100644 cicd/k3s-calico-dual-stack/kube-loxilb.yml create mode 100644 cicd/k3s-calico-dual-stack/nginx-svc-lb.yml create mode 100644 cicd/k3s-calico-dual-stack/nginx-svc-lb1-ipv6.yml create mode 100644 cicd/k3s-calico-dual-stack/nginx-svc-lb1.yml create mode 100644 cicd/k3s-calico-dual-stack/nginx.yml create mode 100755 cicd/k3s-calico-dual-stack/rmconfig.sh create mode 100644 cicd/k3s-calico-dual-stack/sctp-svc-lb.yml create mode 100644 cicd/k3s-calico-dual-stack/udp-svc-lb.yml create mode 100755 cicd/k3s-calico-dual-stack/validation.sh diff --git a/cicd/common.sh b/cicd/common.sh index c59d4cf1f..5e8861b8b 100644 --- a/cicd/common.sh +++ b/cicd/common.sh @@ -147,6 +147,7 @@ spawn_docker_host() { get_docker_pid $dname echo $pid if [ ! -f "$hexist/$dname" -a "$pid" != "" ]; then + sudo mkdir -p /var/run/netns sudo touch /var/run/netns/$dname #echo "sudo mount -o bind /proc/$pid/ns/net /var/run/netns/$2" sudo mount -o bind /proc/$pid/ns/net /var/run/netns/$dname diff --git a/cicd/k3s-calico-dual-stack/config.sh b/cicd/k3s-calico-dual-stack/config.sh new file mode 100755 index 000000000..a2a711ff0 --- /dev/null +++ b/cicd/k3s-calico-dual-stack/config.sh @@ -0,0 +1,262 @@ +#!/bin/bash + +source ../common.sh +source ../k3s_common.sh + +sudo sysctl net.ipv6.conf.all.disable_ipv6=0 2>&1 >> /dev/null +sudo sysctl net.ipv6.conf.default.disable_ipv6=0 2>&1 >> /dev/null +sudo sysctl net.ipv6.conf.all.forwarding=1 2>&1 >> /dev/null + +echo "#########################################" +echo "Spawning all hosts" +echo "#########################################" + +spawn_docker_host --dock-type loxilb --dock-name llb1 +spawn_docker_host --dock-type host --dock-name ep1 +spawn_docker_host --dock-type host --dock-name ep2 +spawn_docker_host --dock-type host --dock-name ep3 +spawn_docker_host --dock-type host --dock-name r1 +spawn_docker_host --dock-type host --dock-name r2 +spawn_docker_host --dock-type host --dock-name user + +echo "#########################################" +echo "Connecting and configuring hosts" +echo "#########################################" + +connect_docker_hosts user r1 +connect_docker_hosts r1 llb1 +connect_docker_hosts llb1 r2 +connect_docker_hosts r2 ep1 +connect_docker_hosts r2 ep2 +connect_docker_hosts r2 ep3 + +$hexec user sysctl net.ipv6.conf.all.disable_ipv6=0 2>&1 >> /dev/null +$hexec user sysctl net.ipv6.conf.default.disable_ipv6=0 2>&1 >> /dev/null +$hexec user sysctl net.ipv6.conf.all.forwarding=1 2>&1 >> /dev/null + +$hexec r1 sysctl net.ipv6.conf.all.disable_ipv6=0 2>&1 >> /dev/null +$hexec r1 sysctl net.ipv6.conf.default.disable_ipv6=0 2>&1 >> /dev/null +$hexec r1 sysctl net.ipv6.conf.all.forwarding=1 2>&1 >> /dev/null + +$hexec llb1 sysctl net.ipv6.conf.all.disable_ipv6=0 2>&1 >> /dev/null +$hexec llb1 sysctl net.ipv6.conf.default.disable_ipv6=0 2>&1 >> /dev/null +$hexec llb1 sysctl net.ipv6.conf.all.forwarding=1 2>&1 >> /dev/null + +$hexec r2 sysctl net.ipv6.conf.all.disable_ipv6=0 2>&1 >> /dev/null +$hexec r2 sysctl net.ipv6.conf.default.disable_ipv6=0 2>&1 >> /dev/null +$hexec r2 sysctl net.ipv6.conf.all.forwarding=1 2>&1 >> /dev/null + +$hexec ep1 sysctl net.ipv6.conf.all.disable_ipv6=0 2>&1 >> /dev/null +$hexec ep1 sysctl net.ipv6.conf.default.disable_ipv6=0 2>&1 >> /dev/null +$hexec ep1 sysctl net.ipv6.conf.all.forwarding=1 2>&1 >> /dev/null + +$hexec ep2 sysctl net.ipv6.conf.all.disable_ipv6=0 2>&1 >> /dev/null +$hexec ep2 sysctl net.ipv6.conf.default.disable_ipv6=0 2>&1 >> /dev/null +$hexec ep2 sysctl net.ipv6.conf.all.forwarding=1 2>&1 >> /dev/null + +$hexec ep3 sysctl net.ipv6.conf.all.disable_ipv6=0 2>&1 >> /dev/null +$hexec ep3 sysctl net.ipv6.conf.default.disable_ipv6=0 2>&1 >> /dev/null +$hexec ep3 sysctl net.ipv6.conf.all.forwarding=1 2>&1 >> /dev/null + +## Make network for k3s connectivity +sudo ip link add ellb1sys type veth peer name esysllb1 +sleep 3 +sudo ip link set ellb1sys netns llb1 +sleep 3 +sudo ip -n llb1 link set ellb1sys up +sudo ip -n llb1 addr add 12.12.12.1/24 dev ellb1sys +$hexec llb1 ip -6 addr add 8ffe::1/96 dev ellb1sys + +# Node-IP +sudo ip link set esysllb1 up +sudo ip addr add 12.12.12.254/24 dev esysllb1 +sudo ip -6 addr add 8ffe::2/96 dev esysllb1 + +#node1 config +config_docker_host --host1 user --host2 r1 --ptype phy --addr 1.1.1.1/24 --gw 1.1.1.254 +config_docker_host --host1 r1 --host2 user --ptype phy --addr 1.1.1.254/24 + +config_docker_host --host1 r1 --host2 llb1 --ptype phy --addr 11.11.11.254/24 --gw 11.11.11.1 +config_docker_host --host1 llb1 --host2 r1 --ptype phy --addr 11.11.11.1/24 + +config_docker_host --host1 llb1 --host2 r2 --ptype phy --addr 10.10.10.1/24 + +config_docker_host --host1 r2 --host2 llb1 --ptype phy --addr 10.10.10.254/24 + +config_docker_host --host1 r2 --host2 ep1 --ptype phy --addr 31.31.31.254/24 + +config_docker_host --host1 ep1 --host2 r2 --ptype phy --addr 31.31.31.1/24 --gw 31.31.31.254 + +config_docker_host --host1 r2 --host2 ep2 --ptype phy --addr 32.32.32.254/24 + +config_docker_host --host1 ep2 --host2 r2 --ptype phy --addr 32.32.32.1/24 --gw 32.32.32.254 + +config_docker_host --host1 r2 --host2 ep3 --ptype phy --addr 33.33.33.254/24 + +config_docker_host --host1 ep3 --host2 r2 --ptype phy --addr 33.33.33.1/24 --gw 33.33.33.254 + +##Pod networks +$hexec r1 ip route add 20.20.20.1/32 via 11.11.11.1 +#add_route llb1 1.1.1.0/24 11.11.11.254 + +sleep 1 +##Create LB rule +create_lb_rule llb1 20.20.20.1 --tcp=2020:8080 --endpoints=31.31.31.1:1,32.32.32.1:1,33.33.33.1:1 --mode=fullnat + +## IPV6 Stuff +$hexec user ip -6 addr add 3ffe::1/64 dev euserr1 +$hexec user ip -6 route add default via 3ffe::10 +$hexec user ethtool --offload euserr1 rx off tx off +$hexec user ethtool -K euserr1 gso off + +$hexec r1 ip -6 addr add 3ffe::10/64 dev er1user +$hexec r1 ethtool --offload er1user rx off tx off +$hexec r1 ethtool -K er1user gso off + +$hexec r1 ip -6 addr add 4ffe::10/64 dev er1llb1 +$hexec r1 ip -6 route add default via 4ffe::1 +$hexec r1 ethtool --offload er1llb1 rx off tx off +$hexec r1 ethtool -K er1llb1 gso off + +$hexec llb1 ip -6 addr add 4ffe::1/64 dev ellb1r1 +$hexec llb1 ethtool --offload ellb1r1 rx off tx off +$hexec llb1 ethtool -K ellb1r1 gso off + +$hexec llb1 ip -6 addr add 5ffe::1/64 dev ellb1r2 +$hexec llb1 ethtool --offload ellb1r2 rx off tx off +$hexec llb1 ethtool -K ellb1r2 gso off + +$hexec r2 ip -6 addr add 5ffe::10/64 dev er2llb1 +$hexec r2 ethtool --offload er2llb1 rx off tx off +$hexec r2 ethtool -K er2llb1 gso off + +#Default route towards r1 +$hexec llb1 ip -6 route add default via 5ffe::10 + +#Default route towards llb1 +$hexec r2 ip -6 route add default via 5ffe::1 + +$hexec r2 ip -6 addr add 6ffa::10/64 dev er2ep1 +$hexec r2 ethtool --offload er2ep1 rx off tx off +$hexec r2 ethtool -K er2ep1 gso off + +$hexec r2 ip -6 addr add 6ffb::10/64 dev er2ep2 +$hexec r2 ethtool --offload er2ep2 rx off tx off +$hexec r2 ethtool -K er2ep2 gso off + +$hexec r2 ip -6 addr add 6ffc::10/64 dev er2ep3 +$hexec r2 ethtool --offload er2ep3 rx off tx off +$hexec r2 ethtool -K er2ep3 gso off + +$hexec ep1 ip -6 addr add 6ffa::1/64 dev eep1r2 +$hexec ep1 ip -6 route add default via 6ffa::10 +$hexec ep1 ethtool --offload eep1r2 rx off tx off +$hexec ep1 ethtool -K eep1r2 gso off + + +$hexec ep2 ip -6 addr add 6ffb::1/64 dev eep2r2 +$hexec ep2 ip -6 route add default via 6ffb::10 +$hexec ep2 ethtool --offload eep2r2 rx off tx off +$hexec ep2 ethtool -K eep2r2 gso off + +$hexec ep3 ip -6 addr add 6ffc::1/64 dev eep3r2 +$hexec ep3 ip -6 route add default via 6ffc::10 +$hexec ep3 ethtool --offload eep3r2 rx off tx off +$hexec ep3 ethtool -K eep3r2 gso off + +$hexec llb1 ip addr add 2001::1/128 dev lo + +#NAT64 service +$dexec llb1 loxicmd create lb 2001::1 --tcp=1064:8080 --endpoints=31.31.31.1:1,32.32.32.1:1,33.33.33.1:1 + +#NAT66 service +$dexec llb1 loxicmd create lb 2001::1 --tcp=1066:8080 --endpoints=6ffa::1:1,6ffb::1:1,6ffc::1:1 + +sleep 2 + +# K3s setup +KUBECONFIG=--kubeconfig=/etc/rancher/k3s/k3s.yaml +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +# If k3s setup exists, skip installation +if [[ -f "/usr/local/bin/k3s-uninstall.sh" ]]; then + echo "K3s exists" + sleep 10 +else + echo "Start K3s installation" + + # Install k3s without external cloud-manager and disabled servicelb + #curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh - + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external --flannel-backend=none --disable-network-policy --cluster-cidr=10.42.0.0/16,4dde::/64 --service-cidr=10.43.0.0/16,5dde::/108 --node-ip=12.12.12.254,8ffe::2" K3S_KUBECONFIG_MODE="644" sh - + + sleep 10 + + # Install Calico + kubectl $KUBECONFIG create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/tigera-operator.yaml + + #kubectl $KUBECONFIG create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/custom-resources.yaml + kubectl $KUBECONFIG create -f custom-resources.yaml + + # Check kubectl works + kubectl $KUBECONFIG get pods -A + + # Remove taints in k3s if any (usually happens if started without cloud-manager) + kubectl $KUBECONFIG taint nodes --all node.cloudprovider.kubernetes.io/uninitialized=false:NoSchedule- + + # Start loxi-ccm as k3s daemonset + kubectl $KUBECONFIG apply -f https://github.com/loxilb-io/loxi-ccm/raw/master/manifests/loxi-ccm-k3s.yaml + + echo "End K3s installation" +fi + +# Install Bird to work with k3s +sudo apt install bird2 --yes + +sleep 5 + +sudo cp -f bird_config/bird.conf /etc/bird/bird.conf +if [ ! -f /var/log/bird.log ]; then + sudo touch /var/log/bird.log +fi +sudo chown bird:bird /var/log/bird.log +sudo systemctl restart bird + +sleep 10 + +# Wait for cluster to be ready +wait_cluster_ready_full + +# Start nginx pods and services for test +kubectl $KUBECONFIG apply -f nginx.yml +kubectl $KUBECONFIG apply -f nginx-svc-lb.yml + +sleep 5 + +# Start nginx pods and services for test(using kube-loxilb) +kubectl $KUBECONFIG apply -f kube-loxilb.yml +sleep 15 +kubectl $KUBECONFIG apply -f nginx-svc-lb1.yml +sleep 10 +kubectl $KUBECONFIG apply -f udp-svc-lb.yml +sleep 10 +kubectl $KUBECONFIG apply -f sctp-svc-lb.yml +sleep 10 +kubectl $KUBECONFIG apply -f nginx-svc-lb1-ipv6.yml +sleep 30 + +# External LB service must be created by now +kubectl $KUBECONFIG get svc + +#Route back to llb1 +sudo ip route add 1.1.1.1/32 via 12.12.12.1 +sudo ip route add 3ffe::1/128 via 8ffe::1 + +# Route back to user +$hexec llb1 ip route add 1.1.1.0/24 via 11.11.11.254 +$hexec llb1 ip -6 route add 3ffe::0/64 via 4ffe::10 + + +# Wait for cluster to be ready +wait_cluster_ready_full diff --git a/cicd/k3s-calico-dual-stack/custom-resources.yaml b/cicd/k3s-calico-dual-stack/custom-resources.yaml new file mode 100644 index 000000000..90b6d482a --- /dev/null +++ b/cicd/k3s-calico-dual-stack/custom-resources.yaml @@ -0,0 +1,32 @@ +# This section includes base Calico installation configuration. +# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.Installation +apiVersion: operator.tigera.io/v1 +kind: Installation +metadata: + name: default +spec: + # Configures Calico networking. + calicoNetwork: + # Note: The ipPools section cannot be modified post-install. + ipPools: + - blockSize: 26 + cidr: 192.168.0.0/16 + encapsulation: VXLANCrossSubnet + natOutgoing: Enabled + nodeSelector: all() + - blockSize: 122 + cidr: 2002::0/64 + encapsulation: None + natOutgoing: Enabled + nodeSelector: all() + +--- + +# This section configures the Calico API server. +# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer +apiVersion: operator.tigera.io/v1 +kind: APIServer +metadata: + name: default +spec: {} + diff --git a/cicd/k3s-calico-dual-stack/custom-resources.yaml.bk b/cicd/k3s-calico-dual-stack/custom-resources.yaml.bk new file mode 100644 index 000000000..90b6d482a --- /dev/null +++ b/cicd/k3s-calico-dual-stack/custom-resources.yaml.bk @@ -0,0 +1,32 @@ +# This section includes base Calico installation configuration. +# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.Installation +apiVersion: operator.tigera.io/v1 +kind: Installation +metadata: + name: default +spec: + # Configures Calico networking. + calicoNetwork: + # Note: The ipPools section cannot be modified post-install. + ipPools: + - blockSize: 26 + cidr: 192.168.0.0/16 + encapsulation: VXLANCrossSubnet + natOutgoing: Enabled + nodeSelector: all() + - blockSize: 122 + cidr: 2002::0/64 + encapsulation: None + natOutgoing: Enabled + nodeSelector: all() + +--- + +# This section configures the Calico API server. +# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer +apiVersion: operator.tigera.io/v1 +kind: APIServer +metadata: + name: default +spec: {} + diff --git a/cicd/k3s-calico-dual-stack/install_k3s.sh b/cicd/k3s-calico-dual-stack/install_k3s.sh new file mode 100755 index 000000000..481b56a8c --- /dev/null +++ b/cicd/k3s-calico-dual-stack/install_k3s.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +KUBECONFIG=--kubeconfig=/etc/rancher/k3s/k3s.yaml +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +# If k3s setup exists, skip installation +if [[ -f "/usr/local/bin/k3s-uninstall.sh" ]]; then + echo "K3s exists" + sleep 10 +else + echo "Start K3s installation" + +# Install k3s without external cloud-manager and disabled servicelb + #curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external --flannel-backend=none --cluster-cidr=10.42.0.0/16" K3S_KUBECONFIG_MODE="644" sh - + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external --flannel-backend=none --disable-network-policy --cluster-cidr=10.42.0.0/16,4dde::/64 --service-cidr=10.43.0.0/16,5dde::/108 --node-ip=12.12.12.254,8ffe::2" K3S_KUBECONFIG_MODE="644" sh - + + sleep 10 + + # Install Calico + kubectl $KUBECONFIG create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/tigera-operator.yaml + + #kubectl $KUBECONFIG create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/custom-resources.yaml + kubectl $KUBECONFIG apply -f custom-resources.yaml + + # Check kubectl works + kubectl $KUBECONFIG get pods -A + + # Remove taints in k3s if any (usually happens if started without cloud-manager) + kubectl $KUBECONFIG taint nodes --all node.cloudprovider.kubernetes.io/uninitialized=false:NoSchedule- + + # Start loxi-ccm as k3s daemonset + kubectl $KUBECONFIG apply -f https://github.com/loxilb-io/loxi-ccm/raw/master/manifests/loxi-ccm-k3s.yaml + + echo "End K3s installation" +fi diff --git a/cicd/k3s-calico-dual-stack/k3s.sh b/cicd/k3s-calico-dual-stack/k3s.sh new file mode 100644 index 000000000..e96f474da --- /dev/null +++ b/cicd/k3s-calico-dual-stack/k3s.sh @@ -0,0 +1,1047 @@ +#!/bin/sh +set -e +set -o noglob + +# Usage: +# curl ... | ENV_VAR=... sh - +# or +# ENV_VAR=... ./install.sh +# +# Example: +# Installing a server without traefik: +# curl ... | INSTALL_K3S_EXEC="--disable=traefik" sh - +# Installing an agent to point at a server: +# curl ... | K3S_TOKEN=xxx K3S_URL=https://server-url:6443 sh - +# +# Environment variables: +# - K3S_* +# Environment variables which begin with K3S_ will be preserved for the +# systemd service to use. Setting K3S_URL without explicitly setting +# a systemd exec command will default the command to "agent", and we +# enforce that K3S_TOKEN is also set. +# +# - INSTALL_K3S_SKIP_DOWNLOAD +# If set to true will not download k3s hash or binary. +# +# - INSTALL_K3S_FORCE_RESTART +# If set to true will always restart the K3s service +# +# - INSTALL_K3S_SYMLINK +# If set to 'skip' will not create symlinks, 'force' will overwrite, +# default will symlink if command does not exist in path. +# +# - INSTALL_K3S_SKIP_ENABLE +# If set to true will not enable or start k3s service. +# +# - INSTALL_K3S_SKIP_START +# If set to true will not start k3s service. +# +# - INSTALL_K3S_VERSION +# Version of k3s to download from github. Will attempt to download from the +# stable channel if not specified. +# +# - INSTALL_K3S_COMMIT +# Commit of k3s to download from temporary cloud storage. +# * (for developer & QA use) +# +# - INSTALL_K3S_BIN_DIR +# Directory to install k3s binary, links, and uninstall script to, or use +# /usr/local/bin as the default +# +# - INSTALL_K3S_BIN_DIR_READ_ONLY +# If set to true will not write files to INSTALL_K3S_BIN_DIR, forces +# setting INSTALL_K3S_SKIP_DOWNLOAD=true +# +# - INSTALL_K3S_SYSTEMD_DIR +# Directory to install systemd service and environment files to, or use +# /etc/systemd/system as the default +# +# - INSTALL_K3S_EXEC or script arguments +# Command with flags to use for launching k3s in the systemd service, if +# the command is not specified will default to "agent" if K3S_URL is set +# or "server" if not. The final systemd command resolves to a combination +# of EXEC and script args ($@). +# +# The following commands result in the same behavior: +# curl ... | INSTALL_K3S_EXEC="--disable=traefik" sh -s - +# curl ... | INSTALL_K3S_EXEC="server --disable=traefik" sh -s - +# curl ... | INSTALL_K3S_EXEC="server" sh -s - --disable=traefik +# curl ... | sh -s - server --disable=traefik +# curl ... | sh -s - --disable=traefik +# +# - INSTALL_K3S_NAME +# Name of systemd service to create, will default from the k3s exec command +# if not specified. If specified the name will be prefixed with 'k3s-'. +# +# - INSTALL_K3S_TYPE +# Type of systemd service to create, will default from the k3s exec command +# if not specified. +# +# - INSTALL_K3S_SELINUX_WARN +# If set to true will continue if k3s-selinux policy is not found. +# +# - INSTALL_K3S_SKIP_SELINUX_RPM +# If set to true will skip automatic installation of the k3s RPM. +# +# - INSTALL_K3S_CHANNEL_URL +# Channel URL for fetching k3s download URL. +# Defaults to 'https://update.k3s.io/v1-release/channels'. +# +# - INSTALL_K3S_CHANNEL +# Channel to use for fetching k3s download URL. +# Defaults to 'stable'. + +GITHUB_URL=https://github.com/k3s-io/k3s/releases +STORAGE_URL=https://k3s-ci-builds.s3.amazonaws.com +DOWNLOADER= + +# --- helper functions for logs --- +info() +{ + echo '[INFO] ' "$@" +} +warn() +{ + echo '[WARN] ' "$@" >&2 +} +fatal() +{ + echo '[ERROR] ' "$@" >&2 + exit 1 +} + +# --- fatal if no systemd or openrc --- +verify_system() { + if [ -x /sbin/openrc-run ]; then + HAS_OPENRC=true + return + fi + if [ -x /bin/systemctl ] || type systemctl > /dev/null 2>&1; then + HAS_SYSTEMD=true + return + fi + fatal 'Can not find systemd or openrc to use as a process supervisor for k3s' +} + +# --- add quotes to command arguments --- +quote() { + for arg in "$@"; do + printf '%s\n' "$arg" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" + done +} + +# --- add indentation and trailing slash to quoted args --- +quote_indent() { + printf ' \\\n' + for arg in "$@"; do + printf '\t%s \\\n' "$(quote "$arg")" + done +} + +# --- escape most punctuation characters, except quotes, forward slash, and space --- +escape() { + printf '%s' "$@" | sed -e 's/\([][!#$%&()*;<=>?\_`{|}]\)/\\\1/g;' +} + +# --- escape double quotes --- +escape_dq() { + printf '%s' "$@" | sed -e 's/"/\\"/g' +} + +# --- ensures $K3S_URL is empty or begins with https://, exiting fatally otherwise --- +verify_k3s_url() { + case "${K3S_URL}" in + "") + ;; + https://*) + ;; + *) + fatal "Only https:// URLs are supported for K3S_URL (have ${K3S_URL})" + ;; + esac +} + +# --- define needed environment variables --- +setup_env() { + # --- use command args if passed or create default --- + case "$1" in + # --- if we only have flags discover if command should be server or agent --- + (-*|"") + if [ -z "${K3S_URL}" ]; then + CMD_K3S=server + else + if [ -z "${K3S_TOKEN}" ] && [ -z "${K3S_TOKEN_FILE}" ]; then + fatal "Defaulted k3s exec command to 'agent' because K3S_URL is defined, but K3S_TOKEN or K3S_TOKEN_FILE is not defined." + fi + CMD_K3S=agent + fi + ;; + # --- command is provided --- + (*) + CMD_K3S=$1 + shift + ;; + esac + + verify_k3s_url + + CMD_K3S_EXEC="${CMD_K3S}$(quote_indent "$@")" + + # --- use systemd name if defined or create default --- + if [ -n "${INSTALL_K3S_NAME}" ]; then + SYSTEM_NAME=k3s-${INSTALL_K3S_NAME} + else + if [ "${CMD_K3S}" = server ]; then + SYSTEM_NAME=k3s + else + SYSTEM_NAME=k3s-${CMD_K3S} + fi + fi + + # --- check for invalid characters in system name --- + valid_chars=$(printf '%s' "${SYSTEM_NAME}" | sed -e 's/[][!#$%&()*;<=>?\_`{|}/[:space:]]/^/g;' ) + if [ "${SYSTEM_NAME}" != "${valid_chars}" ]; then + invalid_chars=$(printf '%s' "${valid_chars}" | sed -e 's/[^^]/ /g') + fatal "Invalid characters for system name: + ${SYSTEM_NAME} + ${invalid_chars}" + fi + + # --- use sudo if we are not already root --- + SUDO=sudo + if [ $(id -u) -eq 0 ]; then + SUDO= + fi + + # --- use systemd type if defined or create default --- + if [ -n "${INSTALL_K3S_TYPE}" ]; then + SYSTEMD_TYPE=${INSTALL_K3S_TYPE} + else + SYSTEMD_TYPE=notify + fi + + # --- use binary install directory if defined or create default --- + if [ -n "${INSTALL_K3S_BIN_DIR}" ]; then + BIN_DIR=${INSTALL_K3S_BIN_DIR} + else + # --- use /usr/local/bin if root can write to it, otherwise use /opt/bin if it exists + BIN_DIR=/usr/local/bin + if ! $SUDO sh -c "touch ${BIN_DIR}/k3s-ro-test && rm -rf ${BIN_DIR}/k3s-ro-test"; then + if [ -d /opt/bin ]; then + BIN_DIR=/opt/bin + fi + fi + fi + + # --- use systemd directory if defined or create default --- + if [ -n "${INSTALL_K3S_SYSTEMD_DIR}" ]; then + SYSTEMD_DIR="${INSTALL_K3S_SYSTEMD_DIR}" + else + SYSTEMD_DIR=/etc/systemd/system + fi + + # --- set related files from system name --- + SERVICE_K3S=${SYSTEM_NAME}.service + UNINSTALL_K3S_SH=${UNINSTALL_K3S_SH:-${BIN_DIR}/${SYSTEM_NAME}-uninstall.sh} + KILLALL_K3S_SH=${KILLALL_K3S_SH:-${BIN_DIR}/k3s-killall.sh} + + # --- use service or environment location depending on systemd/openrc --- + if [ "${HAS_SYSTEMD}" = true ]; then + FILE_K3S_SERVICE=${SYSTEMD_DIR}/${SERVICE_K3S} + FILE_K3S_ENV=${SYSTEMD_DIR}/${SERVICE_K3S}.env + elif [ "${HAS_OPENRC}" = true ]; then + $SUDO mkdir -p /etc/rancher/k3s + FILE_K3S_SERVICE=/etc/init.d/${SYSTEM_NAME} + FILE_K3S_ENV=/etc/rancher/k3s/${SYSTEM_NAME}.env + fi + + # --- get hash of config & exec for currently installed k3s --- + PRE_INSTALL_HASHES=$(get_installed_hashes) + + # --- if bin directory is read only skip download --- + if [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ]; then + INSTALL_K3S_SKIP_DOWNLOAD=true + fi + + # --- setup channel values + INSTALL_K3S_CHANNEL_URL=${INSTALL_K3S_CHANNEL_URL:-'https://update.k3s.io/v1-release/channels'} + INSTALL_K3S_CHANNEL=${INSTALL_K3S_CHANNEL:-'stable'} +} + +# --- check if skip download environment variable set --- +can_skip_download_binary() { + if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ] && [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != binary ]; then + return 1 + fi +} + +can_skip_download_selinux() { + if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ] && [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != selinux ]; then + return 1 + fi +} + +# --- verify an executable k3s binary is installed --- +verify_k3s_is_executable() { + if [ ! -x ${BIN_DIR}/k3s ]; then + fatal "Executable k3s binary not found at ${BIN_DIR}/k3s" + fi +} + +# --- set arch and suffix, fatal if architecture not supported --- +setup_verify_arch() { + if [ -z "$ARCH" ]; then + ARCH=$(uname -m) + fi + case $ARCH in + amd64) + ARCH=amd64 + SUFFIX= + ;; + x86_64) + ARCH=amd64 + SUFFIX= + ;; + arm64) + ARCH=arm64 + SUFFIX=-${ARCH} + ;; + s390x) + ARCH=s390x + SUFFIX=-${ARCH} + ;; + aarch64) + ARCH=arm64 + SUFFIX=-${ARCH} + ;; + arm*) + ARCH=arm + SUFFIX=-${ARCH}hf + ;; + *) + fatal "Unsupported architecture $ARCH" + esac +} + +# --- verify existence of network downloader executable --- +verify_downloader() { + # Return failure if it doesn't exist or is no executable + [ -x "$(command -v $1)" ] || return 1 + + # Set verified executable as our downloader program and return success + DOWNLOADER=$1 + return 0 +} + +# --- create temporary directory and cleanup when done --- +setup_tmp() { + TMP_DIR=$(mktemp -d -t k3s-install.XXXXXXXXXX) + TMP_HASH=${TMP_DIR}/k3s.hash + TMP_BIN=${TMP_DIR}/k3s.bin + cleanup() { + code=$? + set +e + trap - EXIT + rm -rf ${TMP_DIR} + exit $code + } + trap cleanup INT EXIT +} + +# --- use desired k3s version if defined or find version from channel --- +get_release_version() { + if [ -n "${INSTALL_K3S_COMMIT}" ]; then + VERSION_K3S="commit ${INSTALL_K3S_COMMIT}" + elif [ -n "${INSTALL_K3S_VERSION}" ]; then + VERSION_K3S=${INSTALL_K3S_VERSION} + else + info "Finding release for channel ${INSTALL_K3S_CHANNEL}" + version_url="${INSTALL_K3S_CHANNEL_URL}/${INSTALL_K3S_CHANNEL}" + case $DOWNLOADER in + curl) + VERSION_K3S=$(curl -w '%{url_effective}' -L -s -S ${version_url} -o /dev/null | sed -e 's|.*/||') + ;; + wget) + VERSION_K3S=$(wget -SqO /dev/null ${version_url} 2>&1 | grep -i Location | sed -e 's|.*/||') + ;; + *) + fatal "Incorrect downloader executable '$DOWNLOADER'" + ;; + esac + fi + info "Using ${VERSION_K3S} as release" +} + +# --- get k3s-selinux version --- +get_k3s_selinux_version() { + available_version="k3s-selinux-1.2-2.${rpm_target}.noarch.rpm" + info "Finding available k3s-selinux versions" + + # run verify_downloader in case it binary installation was skipped + verify_downloader curl || verify_downloader wget || fatal 'Can not find curl or wget for downloading files' + + case $DOWNLOADER in + curl) + DOWNLOADER_OPTS="-s" + ;; + wget) + DOWNLOADER_OPTS="-q -O -" + ;; + *) + fatal "Incorrect downloader executable '$DOWNLOADER'" + ;; + esac + for i in {1..3}; do + set +e + if [ "${rpm_channel}" = "testing" ]; then + version=$(timeout 5 ${DOWNLOADER} ${DOWNLOADER_OPTS} https://api.github.com/repos/k3s-io/k3s-selinux/releases | grep browser_download_url | awk '{ print $2 }' | grep -oE "[^\/]+${rpm_target}\.noarch\.rpm" | head -n 1) + else + version=$(timeout 5 ${DOWNLOADER} ${DOWNLOADER_OPTS} https://api.github.com/repos/k3s-io/k3s-selinux/releases/latest | grep browser_download_url | awk '{ print $2 }' | grep -oE "[^\/]+${rpm_target}\.noarch\.rpm") + fi + set -e + if [ "${version}" != "" ]; then + break + fi + sleep 1 + done + if [ "${version}" == "" ]; then + warn "Failed to get available versions of k3s-selinux..defaulting to ${available_version}" + return + fi + available_version=${version} +} + +# --- download from github url --- +download() { + [ $# -eq 2 ] || fatal 'download needs exactly 2 arguments' + + case $DOWNLOADER in + curl) + curl -o $1 -sfL $2 + ;; + wget) + wget -qO $1 $2 + ;; + *) + fatal "Incorrect executable '$DOWNLOADER'" + ;; + esac + + # Abort if download command failed + [ $? -eq 0 ] || fatal 'Download failed' +} + +# --- download hash from github url --- +download_hash() { + if [ -n "${INSTALL_K3S_COMMIT}" ]; then + HASH_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT}.sha256sum + else + HASH_URL=${GITHUB_URL}/download/${VERSION_K3S}/sha256sum-${ARCH}.txt + fi + info "Downloading hash ${HASH_URL}" + download ${TMP_HASH} ${HASH_URL} + HASH_EXPECTED=$(grep " k3s${SUFFIX}$" ${TMP_HASH}) + HASH_EXPECTED=${HASH_EXPECTED%%[[:blank:]]*} +} + +# --- check hash against installed version --- +installed_hash_matches() { + if [ -x ${BIN_DIR}/k3s ]; then + HASH_INSTALLED=$(sha256sum ${BIN_DIR}/k3s) + HASH_INSTALLED=${HASH_INSTALLED%%[[:blank:]]*} + if [ "${HASH_EXPECTED}" = "${HASH_INSTALLED}" ]; then + return + fi + fi + return 1 +} + +# --- download binary from github url --- +download_binary() { + if [ -n "${INSTALL_K3S_COMMIT}" ]; then + BIN_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT} + else + BIN_URL=${GITHUB_URL}/download/${VERSION_K3S}/k3s${SUFFIX} + fi + info "Downloading binary ${BIN_URL}" + download ${TMP_BIN} ${BIN_URL} +} + +# --- verify downloaded binary hash --- +verify_binary() { + info "Verifying binary download" + HASH_BIN=$(sha256sum ${TMP_BIN}) + HASH_BIN=${HASH_BIN%%[[:blank:]]*} + if [ "${HASH_EXPECTED}" != "${HASH_BIN}" ]; then + fatal "Download sha256 does not match ${HASH_EXPECTED}, got ${HASH_BIN}" + fi +} + +# --- setup permissions and move binary to system directory --- +setup_binary() { + chmod 755 ${TMP_BIN} + info "Installing k3s to ${BIN_DIR}/k3s" + $SUDO chown root:root ${TMP_BIN} + $SUDO mv -f ${TMP_BIN} ${BIN_DIR}/k3s +} + +# --- setup selinux policy --- +setup_selinux() { + case ${INSTALL_K3S_CHANNEL} in + *testing) + rpm_channel=testing + ;; + *latest) + rpm_channel=latest + ;; + *) + rpm_channel=stable + ;; + esac + + rpm_site="rpm.rancher.io" + if [ "${rpm_channel}" = "testing" ]; then + rpm_site="rpm-testing.rancher.io" + fi + + [ -r /etc/os-release ] && . /etc/os-release + if [ `expr "${ID_LIKE}" : ".*suse.*"` != 0 ]; then + rpm_target=sle + rpm_site_infix=microos + package_installer=zypper + if [ "${ID_LIKE:-}" = suse ] && [ "${VARIANT_ID:-}" = sle-micro ]; then + rpm_target=sle + rpm_site_infix=slemicro + package_installer=zypper + fi + elif [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then + rpm_target=coreos + rpm_site_infix=coreos + package_installer=rpm-ostree + elif [ "${VERSION_ID%%.*}" = "7" ]; then + rpm_target=el7 + rpm_site_infix=centos/7 + package_installer=yum + elif [ "${VERSION_ID%%.*}" = "8" ] || [ "${VERSION_ID%%.*}" -gt "36" ]; then + rpm_target=el8 + rpm_site_infix=centos/8 + package_installer=yum + else + rpm_target=el9 + rpm_site_infix=centos/9 + package_installer=yum + fi + + if [ "${package_installer}" = "rpm-ostree" ] && [ -x /bin/yum ]; then + package_installer=yum + fi + + if [ "${package_installer}" = "yum" ] && [ -x /usr/bin/dnf ]; then + package_installer=dnf + fi + + policy_hint="please install: + ${package_installer} install -y container-selinux + ${package_installer} install -y https://${rpm_site}/k3s/${rpm_channel}/common/${rpm_site_infix}/noarch/${available_version} +" + + if [ "$INSTALL_K3S_SKIP_SELINUX_RPM" = true ] || can_skip_download_selinux || [ ! -d /usr/share/selinux ]; then + info "Skipping installation of SELinux RPM" + else + get_k3s_selinux_version + install_selinux_rpm ${rpm_site} ${rpm_channel} ${rpm_target} ${rpm_site_infix} + fi + + policy_error=fatal + if [ "$INSTALL_K3S_SELINUX_WARN" = true ] || [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then + policy_error=warn + fi + + if ! $SUDO chcon -u system_u -r object_r -t container_runtime_exec_t ${BIN_DIR}/k3s >/dev/null 2>&1; then + if $SUDO grep '^\s*SELINUX=enforcing' /etc/selinux/config >/dev/null 2>&1; then + $policy_error "Failed to apply container_runtime_exec_t to ${BIN_DIR}/k3s, ${policy_hint}" + fi + elif [ ! -f /usr/share/selinux/packages/k3s.pp ]; then + if [ -x /usr/sbin/transactional-update ] || [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then + warn "Please reboot your machine to activate the changes and avoid data loss." + else + $policy_error "Failed to find the k3s-selinux policy, ${policy_hint}" + fi + fi +} + +install_selinux_rpm() { + if [ -r /etc/redhat-release ] || [ -r /etc/centos-release ] || [ -r /etc/oracle-release ] || [ -r /etc/fedora-release ] || [ "${ID_LIKE%%[ ]*}" = "suse" ]; then + repodir=/etc/yum.repos.d + if [ -d /etc/zypp/repos.d ]; then + repodir=/etc/zypp/repos.d + fi + set +o noglob + $SUDO rm -f ${repodir}/rancher-k3s-common*.repo + set -o noglob + if [ -r /etc/redhat-release ] && [ "${3}" = "el7" ]; then + $SUDO yum install -y yum-utils + $SUDO yum-config-manager --enable rhel-7-server-extras-rpms + fi + $SUDO tee ${repodir}/rancher-k3s-common.repo >/dev/null << EOF +[rancher-k3s-common-${2}] +name=Rancher K3s Common (${2}) +baseurl=https://${1}/k3s/${2}/common/${4}/noarch +enabled=1 +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=https://${1}/public.key +EOF + case ${3} in + sle) + rpm_installer="zypper --gpg-auto-import-keys" + if [ "${TRANSACTIONAL_UPDATE=false}" != "true" ] && [ -x /usr/sbin/transactional-update ]; then + transactional_update_run="transactional-update --no-selfupdate -d run" + rpm_installer="transactional-update --no-selfupdate -d run ${rpm_installer}" + : "${INSTALL_K3S_SKIP_START:=true}" + fi + # create the /var/lib/rpm-state in SLE systems to fix the prein selinux macro + ${transactional_update_run} mkdir -p /var/lib/rpm-state + ;; + coreos) + rpm_installer="rpm-ostree" + # rpm_install_extra_args="--apply-live" + : "${INSTALL_K3S_SKIP_START:=true}" + ;; + *) + rpm_installer="yum" + ;; + esac + if [ "${rpm_installer}" = "yum" ] && [ -x /usr/bin/dnf ]; then + rpm_installer=dnf + fi + if rpm -q --quiet k3s-selinux; then + # remove k3s-selinux module before upgrade to allow container-selinux to upgrade safely + if check_available_upgrades container-selinux ${3} && check_available_upgrades k3s-selinux ${3}; then + MODULE_PRIORITY=$($SUDO semodule --list=full | grep k3s | cut -f1 -d" ") + if [ -n "${MODULE_PRIORITY}" ]; then + $SUDO semodule -X $MODULE_PRIORITY -r k3s || true + fi + fi + fi + # shellcheck disable=SC2086 + $SUDO ${rpm_installer} install -y "k3s-selinux" + fi + return +} + +check_available_upgrades() { + set +e + case ${2} in + sle) + available_upgrades=$($SUDO zypper -q -t -s 11 se -s -u --type package $1 | tail -n 1 | grep -v "No matching" | awk '{print $3}') + ;; + coreos) + # currently rpm-ostree does not support search functionality https://github.com/coreos/rpm-ostree/issues/1877 + ;; + *) + available_upgrades=$($SUDO yum -q --refresh list $1 --upgrades | tail -n 1 | awk '{print $2}') + ;; + esac + set -e + if [ -n "${available_upgrades}" ]; then + return 0 + fi + return 1 +} +# --- download and verify k3s --- +download_and_verify() { + if can_skip_download_binary; then + info 'Skipping k3s download and verify' + verify_k3s_is_executable + return + fi + + setup_verify_arch + verify_downloader curl || verify_downloader wget || fatal 'Can not find curl or wget for downloading files' + setup_tmp + get_release_version + download_hash + + if installed_hash_matches; then + info 'Skipping binary downloaded, installed k3s matches hash' + return + fi + + download_binary + verify_binary + setup_binary +} + +# --- add additional utility links --- +create_symlinks() { + [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return + [ "${INSTALL_K3S_SYMLINK}" = skip ] && return + + for cmd in kubectl crictl ctr; do + if [ ! -e ${BIN_DIR}/${cmd} ] || [ "${INSTALL_K3S_SYMLINK}" = force ]; then + which_cmd=$(command -v ${cmd} 2>/dev/null || true) + if [ -z "${which_cmd}" ] || [ "${INSTALL_K3S_SYMLINK}" = force ]; then + info "Creating ${BIN_DIR}/${cmd} symlink to k3s" + $SUDO ln -sf k3s ${BIN_DIR}/${cmd} + else + info "Skipping ${BIN_DIR}/${cmd} symlink to k3s, command exists in PATH at ${which_cmd}" + fi + else + info "Skipping ${BIN_DIR}/${cmd} symlink to k3s, already exists" + fi + done +} + +# --- create killall script --- +create_killall() { + [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return + info "Creating killall script ${KILLALL_K3S_SH}" + $SUDO tee ${KILLALL_K3S_SH} >/dev/null << \EOF +#!/bin/sh +[ $(id -u) -eq 0 ] || exec sudo $0 $@ + +for bin in /var/lib/rancher/k3s/data/**/bin/; do + [ -d $bin ] && export PATH=$PATH:$bin:$bin/aux +done + +set -x + +for service in /etc/systemd/system/k3s*.service; do + [ -s $service ] && systemctl stop $(basename $service) +done + +for service in /etc/init.d/k3s*; do + [ -x $service ] && $service stop +done + +pschildren() { + ps -e -o ppid= -o pid= | \ + sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \ + grep -w "^$1" | \ + cut -f2 +} + +pstree() { + for pid in $@; do + echo $pid + for child in $(pschildren $pid); do + pstree $child + done + done +} + +killtree() { + kill -9 $( + { set +x; } 2>/dev/null; + pstree $@; + set -x; + ) 2>/dev/null +} + +remove_interfaces() { + # Delete network interface(s) that match 'master cni0' + ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do + iface=${iface%%@*} + [ -z "$iface" ] || ip link delete $iface + done + + # Delete cni related interfaces + ip link delete cni0 + ip link delete flannel.1 + ip link delete flannel-v6.1 + ip link delete kube-ipvs0 + ip link delete flannel-wg + ip link delete flannel-wg-v6 + + # Restart tailscale + if [ -n "$(command -v tailscale)" ]; then + tailscale set --advertise-routes= + fi +} + +getshims() { + ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1 +} + +killtree $({ set +x; } 2>/dev/null; getshims; set -x) + +do_unmount_and_remove() { + set +x + while read -r _ path _; do + case "$path" in $1*) echo "$path" ;; esac + done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"' + set -x +} + +do_unmount_and_remove '/run/k3s' +do_unmount_and_remove '/var/lib/rancher/k3s' +do_unmount_and_remove '/var/lib/kubelet/pods' +do_unmount_and_remove '/var/lib/kubelet/plugins' +do_unmount_and_remove '/run/netns/cni-' + +# Remove CNI namespaces +ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete + +remove_interfaces + +rm -rf /var/lib/cni/ +iptables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | iptables-restore +ip6tables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | ip6tables-restore +EOF + $SUDO chmod 755 ${KILLALL_K3S_SH} + $SUDO chown root:root ${KILLALL_K3S_SH} +} + +# --- create uninstall script --- +create_uninstall() { + [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return + info "Creating uninstall script ${UNINSTALL_K3S_SH}" + $SUDO tee ${UNINSTALL_K3S_SH} >/dev/null << EOF +#!/bin/sh +set -x +[ \$(id -u) -eq 0 ] || exec sudo \$0 \$@ + +${KILLALL_K3S_SH} + +if command -v systemctl; then + systemctl disable ${SYSTEM_NAME} + systemctl reset-failed ${SYSTEM_NAME} + systemctl daemon-reload +fi +if command -v rc-update; then + rc-update delete ${SYSTEM_NAME} default +fi + +rm -f ${FILE_K3S_SERVICE} +rm -f ${FILE_K3S_ENV} + +remove_uninstall() { + rm -f ${UNINSTALL_K3S_SH} +} +trap remove_uninstall EXIT + +if (ls ${SYSTEMD_DIR}/k3s*.service || ls /etc/init.d/k3s*) >/dev/null 2>&1; then + set +x; echo 'Additional k3s services installed, skipping uninstall of k3s'; set -x + exit +fi + +for cmd in kubectl crictl ctr; do + if [ -L ${BIN_DIR}/\$cmd ]; then + rm -f ${BIN_DIR}/\$cmd + fi +done + +rm -rf /etc/rancher/k3s +rm -rf /run/k3s +rm -rf /run/flannel +rm -rf /var/lib/rancher/k3s +rm -rf /var/lib/kubelet +rm -f ${BIN_DIR}/k3s +rm -f ${KILLALL_K3S_SH} + +if type yum >/dev/null 2>&1; then + yum remove -y k3s-selinux + rm -f /etc/yum.repos.d/rancher-k3s-common*.repo +elif type rpm-ostree >/dev/null 2>&1; then + rpm-ostree uninstall k3s-selinux + rm -f /etc/yum.repos.d/rancher-k3s-common*.repo +elif type zypper >/dev/null 2>&1; then + uninstall_cmd="zypper remove -y k3s-selinux" + if [ "\${TRANSACTIONAL_UPDATE=false}" != "true" ] && [ -x /usr/sbin/transactional-update ]; then + uninstall_cmd="transactional-update --no-selfupdate -d run \$uninstall_cmd" + fi + \$uninstall_cmd + rm -f /etc/zypp/repos.d/rancher-k3s-common*.repo +fi +EOF + $SUDO chmod 755 ${UNINSTALL_K3S_SH} + $SUDO chown root:root ${UNINSTALL_K3S_SH} +} + +# --- disable current service if loaded -- +systemd_disable() { + $SUDO systemctl disable ${SYSTEM_NAME} >/dev/null 2>&1 || true + $SUDO rm -f /etc/systemd/system/${SERVICE_K3S} || true + $SUDO rm -f /etc/systemd/system/${SERVICE_K3S}.env || true +} + +# --- capture current env and create file containing k3s_ variables --- +create_env_file() { + info "env: Creating environment file ${FILE_K3S_ENV}" + $SUDO touch ${FILE_K3S_ENV} + $SUDO chmod 0600 ${FILE_K3S_ENV} + sh -c export | while read x v; do echo $v; done | grep -E '^(K3S|CONTAINERD)_' | $SUDO tee ${FILE_K3S_ENV} >/dev/null + sh -c export | while read x v; do echo $v; done | grep -Ei '^(NO|HTTP|HTTPS)_PROXY' | $SUDO tee -a ${FILE_K3S_ENV} >/dev/null +} + +# --- write systemd service file --- +create_systemd_service_file() { + info "systemd: Creating service file ${FILE_K3S_SERVICE}" + $SUDO tee ${FILE_K3S_SERVICE} >/dev/null << EOF +[Unit] +Description=Lightweight Kubernetes +Documentation=https://k3s.io +Wants=network-online.target +After=network-online.target + +[Install] +WantedBy=multi-user.target + +[Service] +Type=${SYSTEMD_TYPE} +EnvironmentFile=-/etc/default/%N +EnvironmentFile=-/etc/sysconfig/%N +EnvironmentFile=-${FILE_K3S_ENV} +KillMode=process +Delegate=yes +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=1048576 +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +TimeoutStartSec=0 +Restart=always +RestartSec=5s +ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service' +ExecStartPre=-/sbin/modprobe br_netfilter +ExecStartPre=-/sbin/modprobe overlay +ExecStart=${BIN_DIR}/k3s \\ + ${CMD_K3S_EXEC} + +EOF +} + +# --- write openrc service file --- +create_openrc_service_file() { + LOG_FILE=/var/log/${SYSTEM_NAME}.log + + info "openrc: Creating service file ${FILE_K3S_SERVICE}" + $SUDO tee ${FILE_K3S_SERVICE} >/dev/null << EOF +#!/sbin/openrc-run + +depend() { + after network-online + want cgroups +} + +start_pre() { + rm -f /tmp/k3s.* +} + +supervisor=supervise-daemon +name=${SYSTEM_NAME} +command="${BIN_DIR}/k3s" +command_args="$(escape_dq "${CMD_K3S_EXEC}") + >>${LOG_FILE} 2>&1" + +output_log=${LOG_FILE} +error_log=${LOG_FILE} + +pidfile="/var/run/${SYSTEM_NAME}.pid" +respawn_delay=5 +respawn_max=0 + +set -o allexport +if [ -f /etc/environment ]; then . /etc/environment; fi +if [ -f ${FILE_K3S_ENV} ]; then . ${FILE_K3S_ENV}; fi +set +o allexport +EOF + $SUDO chmod 0755 ${FILE_K3S_SERVICE} + + $SUDO tee /etc/logrotate.d/${SYSTEM_NAME} >/dev/null << EOF +${LOG_FILE} { + missingok + notifempty + copytruncate +} +EOF +} + +# --- write systemd or openrc service file --- +create_service_file() { + [ "${HAS_SYSTEMD}" = true ] && create_systemd_service_file + [ "${HAS_OPENRC}" = true ] && create_openrc_service_file + return 0 +} + +# --- get hashes of the current k3s bin and service files +get_installed_hashes() { + $SUDO sha256sum ${BIN_DIR}/k3s ${FILE_K3S_SERVICE} ${FILE_K3S_ENV} 2>&1 || true +} + +# --- enable and start systemd service --- +systemd_enable() { + info "systemd: Enabling ${SYSTEM_NAME} unit" + $SUDO systemctl enable ${FILE_K3S_SERVICE} >/dev/null + $SUDO systemctl daemon-reload >/dev/null +} + +systemd_start() { + info "systemd: Starting ${SYSTEM_NAME}" + $SUDO systemctl restart ${SYSTEM_NAME} +} + +# --- enable and start openrc service --- +openrc_enable() { + info "openrc: Enabling ${SYSTEM_NAME} service for default runlevel" + $SUDO rc-update add ${SYSTEM_NAME} default >/dev/null +} + +openrc_start() { + info "openrc: Starting ${SYSTEM_NAME}" + $SUDO ${FILE_K3S_SERVICE} restart +} + +# --- startup systemd or openrc service --- +service_enable_and_start() { + if [ -f "/proc/cgroups" ] && [ "$(grep memory /proc/cgroups | while read -r n n n enabled; do echo $enabled; done)" -eq 0 ]; + then + info 'Failed to find memory cgroup, you may need to add "cgroup_memory=1 cgroup_enable=memory" to your linux cmdline (/boot/cmdline.txt on a Raspberry Pi)' + fi + + [ "${INSTALL_K3S_SKIP_ENABLE}" = true ] && return + + [ "${HAS_SYSTEMD}" = true ] && systemd_enable + [ "${HAS_OPENRC}" = true ] && openrc_enable + + [ "${INSTALL_K3S_SKIP_START}" = true ] && return + + POST_INSTALL_HASHES=$(get_installed_hashes) + if [ "${PRE_INSTALL_HASHES}" = "${POST_INSTALL_HASHES}" ] && [ "${INSTALL_K3S_FORCE_RESTART}" != true ]; then + info 'No change detected so skipping service start' + return + fi + + if command -v iptables-save 1> /dev/null && command -v iptables-restore 1> /dev/null + then + $SUDO iptables-save | grep -v KUBE- | grep -iv flannel | $SUDO iptables-restore + fi + if command -v ip6tables-save 1> /dev/null && command -v ip6tables-restore 1> /dev/null + then + $SUDO ip6tables-save | grep -v KUBE- | grep -iv flannel | $SUDO ip6tables-restore + fi + + [ "${HAS_SYSTEMD}" = true ] && systemd_start + [ "${HAS_OPENRC}" = true ] && openrc_start + return 0 +} + +# --- re-evaluate args to include env command --- +eval set -- $(escape "${INSTALL_K3S_EXEC}") $(quote "$@") + +# --- run the install process -- +{ + verify_system + setup_env "$@" + download_and_verify + setup_selinux + create_symlinks + create_killall + create_uninstall + systemd_disable + create_env_file + create_service_file + service_enable_and_start +} diff --git a/cicd/k3s-calico-dual-stack/kube-loxilb.yml b/cicd/k3s-calico-dual-stack/kube-loxilb.yml new file mode 100644 index 000000000..27f522a85 --- /dev/null +++ b/cicd/k3s-calico-dual-stack/kube-loxilb.yml @@ -0,0 +1,130 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-loxilb + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - endpoints + - services + - services/status + verbs: + - get + - watch + - list + - patch + - update + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - watch + - list + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-loxilb +subjects: + - kind: ServiceAccount + name: kube-loxilb + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-loxilb + namespace: kube-system + labels: + app: loxilb +spec: + replicas: 1 + selector: + matchLabels: + app: loxilb + template: + metadata: + labels: + app: loxilb + spec: + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + priorityClassName: system-node-critical + serviceAccountName: kube-loxilb + terminationGracePeriodSeconds: 0 + containers: + - name: kube-loxilb + image: ghcr.io/loxilb-io/kube-loxilb:latest + imagePullPolicy: Always + command: + - /bin/kube-loxilb + args: + - --loxiURL=http://12.12.12.1:11111 + - --externalCIDR=123.123.123.1/24 + - --externalCIDR6=2001::1/128 + #- --monitor + #- --setBGP + #- --setLBMode=1 + #- --config=/opt/loxilb/agent/kube-loxilb.conf + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + capabilities: + add: ["NET_ADMIN", "NET_RAW"] diff --git a/cicd/k3s-calico-dual-stack/nginx-svc-lb.yml b/cicd/k3s-calico-dual-stack/nginx-svc-lb.yml new file mode 100644 index 000000000..6f4f2984b --- /dev/null +++ b/cicd/k3s-calico-dual-stack/nginx-svc-lb.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx-lb +spec: + selector: + app.kubernetes.io/name: proxy + ports: + - protocol: TCP + port: 80 + type: LoadBalancer diff --git a/cicd/k3s-calico-dual-stack/nginx-svc-lb1-ipv6.yml b/cicd/k3s-calico-dual-stack/nginx-svc-lb1-ipv6.yml new file mode 100644 index 000000000..55fd3f4be --- /dev/null +++ b/cicd/k3s-calico-dual-stack/nginx-svc-lb1-ipv6.yml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx-lb1-ipv6 + annotations: + # If there is a need to do liveness check from loxilb + loxilb.io/liveness: "no" + # Specify LB mode - one of default, onearm or fullnat + loxilb.io/lbmode: "default" + # Specify loxilb IPAM mode - one of ipv4, ipv6 or ipv6to4 + loxilb.io/ipam: "ipv6" +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: nginx-test-ipv6 + ports: + - port: 55002 + targetPort: 80 + type: LoadBalancer + ipFamilies: + - IPv6 +--- +apiVersion: v1 +kind: Pod +metadata: + name: nginx-test-ipv6 + labels: + what: nginx-test-ipv6 +spec: + containers: + - name: nginx-test-ipv6 + image: nginx:stable + ports: + - containerPort: 80 diff --git a/cicd/k3s-calico-dual-stack/nginx-svc-lb1.yml b/cicd/k3s-calico-dual-stack/nginx-svc-lb1.yml new file mode 100644 index 000000000..5bcf72fd4 --- /dev/null +++ b/cicd/k3s-calico-dual-stack/nginx-svc-lb1.yml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx-lb1 +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: nginx-test + ports: + - port: 55002 + targetPort: 80 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: nginx-test + labels: + what: nginx-test +spec: + containers: + - name: nginx-test + image: nginx:stable + ports: + - containerPort: 80 diff --git a/cicd/k3s-calico-dual-stack/nginx.yml b/cicd/k3s-calico-dual-stack/nginx.yml new file mode 100644 index 000000000..8f1374908 --- /dev/null +++ b/cicd/k3s-calico-dual-stack/nginx.yml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx + labels: + app.kubernetes.io/name: proxy +spec: + containers: + - name: nginx + image: nginx:stable + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-service +spec: + selector: + app.kubernetes.io/name: proxy + ports: + - name: name-of-service-port + protocol: TCP + port: 8080 + targetPort: 80 diff --git a/cicd/k3s-calico-dual-stack/rmconfig.sh b/cicd/k3s-calico-dual-stack/rmconfig.sh new file mode 100755 index 000000000..0ea0c7c60 --- /dev/null +++ b/cicd/k3s-calico-dual-stack/rmconfig.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +echo "#########################################" +echo "Removing testbed" +echo "#########################################" + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +source ../common.sh + +sudo kubectl $KUBECONFIG delete -f nginx-svc-lb.yml >> /dev/null 2>&1 +sudo kubectl $KUBECONFIG delete -f nginx.yml >> /dev/null 2>&1 +sudo kubectl $KUBECONFIG delete -f nginx-svc-lb1.yml >> /dev/null 2>&1 +sudo kubectl $KUBECONFIG delete -f sctp-svc-lb.yml >> /dev/null 2>&1 +sudo kubectl $KUBECONFIG delete -f udp-svc-lb.yml >> /dev/null 2>&1 +sudo kubectl $KUBECONFIG delete -f kube-loxilb.yml >> /dev/null 2>&1 +#sudo kubectl $KUBECONFIG delete -f https://github.com/loxilb-io/loxi-ccm/raw/master/manifests/loxi-ccm-k3s.yaml >> /dev/null 2>&1 +#sudo kubectl $KUBECONFIG delete -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/custom-resources.yaml >> /dev/null 2>&1 +#sudo kubectl $KUBECONFIG delete -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/tigera-operator.yaml >> /dev/null 2>&1 + +disconnect_docker_hosts user r1 +disconnect_docker_hosts r1 llb1 +disconnect_docker_hosts llb1 r2 +disconnect_docker_hosts r2 ep1 +disconnect_docker_hosts r2 ep2 +disconnect_docker_hosts r2 ep3 + +delete_docker_host llb1 +delete_docker_host user +delete_docker_host r1 +delete_docker_host r2 +delete_docker_host ep1 +delete_docker_host ep2 +delete_docker_host ep3 +sudo ip link del esysllb1 2>/dev/null +sudo ip link del esysllb2 2>/dev/null + +# If k3s setup exists, remove it +if [[ -f "/usr/local/bin/k3s-uninstall.sh" ]]; then + /usr/local/bin/k3s-uninstall.sh +fi + +sudo apt-get remove bird2 --yes + +echo "#########################################" +echo "Removed testbed" +echo "#########################################" diff --git a/cicd/k3s-calico-dual-stack/sctp-svc-lb.yml b/cicd/k3s-calico-dual-stack/sctp-svc-lb.yml new file mode 100644 index 000000000..e596b0dce --- /dev/null +++ b/cicd/k3s-calico-dual-stack/sctp-svc-lb.yml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: sctp-lb1 + annotations: + loxilb.io/liveness: "yes" + loxilb.io/lbmode: "default" +spec: + loadBalancerClass: loxilb.io/loxilb + externalTrafficPolicy: Local + selector: + what: sctp-test + ports: + - port: 55004 + protocol: SCTP + targetPort: 9999 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: sctp-test + labels: + what: sctp-test +spec: + containers: + - name: sctp-test + image: alpine/socat + command: [ "sh", "-c"] + args: + - while true; do + socat -v -T2 sctp-l:9999,reuseaddr,fork,bind=$MY_POD_IP system:"echo 'server1'; cat"; + sleep 20; + done; + ports: + - containerPort: 9999 + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP diff --git a/cicd/k3s-calico-dual-stack/udp-svc-lb.yml b/cicd/k3s-calico-dual-stack/udp-svc-lb.yml new file mode 100644 index 000000000..81df995d2 --- /dev/null +++ b/cicd/k3s-calico-dual-stack/udp-svc-lb.yml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Service +metadata: + name: udp-lb1 + annotations: + loxilb.io/liveness: "yes" + loxilb.io/lbmode: "default" +spec: + loadBalancerClass: loxilb.io/loxilb + externalTrafficPolicy: Local + selector: + what: udp-test + ports: + - port: 55003 + protocol: UDP + targetPort: 33333 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: udp-test + labels: + what: udp-test +spec: + containers: + - name: udp-test + image: loxilbio/udp-echo + ports: + - containerPort: 33333 diff --git a/cicd/k3s-calico-dual-stack/validation.sh b/cicd/k3s-calico-dual-stack/validation.sh new file mode 100755 index 000000000..90db07bf6 --- /dev/null +++ b/cicd/k3s-calico-dual-stack/validation.sh @@ -0,0 +1,99 @@ +#!/bin/bash +source ../common.sh +echo calico-k3s-dual-cluster + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +# Set space as the delimiter +IFS=' ' + +for((i=0; i<120; i++)) +do + extLB=$(sudo kubectl $KUBECONFIG get svc | grep "nginx-lb") + read -a strarr <<< "$extLB" + len=${#strarr[*]} + if [[ $((len)) -lt 6 ]]; then + echo "Can't find nginx-lb service" + sleep 1 + continue + fi + if [[ ${strarr[3]} != *"none"* ]]; then + extIP=${strarr[3]} + break + fi + echo "No external LB allocated" + sleep 1 +done + +## Any routing updates ?? +sleep 30 + +echo "ExternalIP $extIP" + +print_debug_info() { + ## Dump some debug info + echo -e "\n\nDEBUG INFO" + echo "*************************************************************************" + echo -e "\n**** k3s svc info ****" + sudo kubectl $KUBECONFIG get svc + echo -e "\n**** k3s pods info ****" + sudo kubectl $KUBECONFIG get pods -A + + echo -e "\n**** llb1 lb-info ****" + $dexec llb1 loxicmd get lb -o wide + echo -e "\n**** loxilb ep-info ****" + $dexec llb1 loxicmd get ep -o wide + echo -e "\n**** llb1 route-info ****" + $dexec llb1 ip route + + echo -e "\n**** r1 route-info ****" + $dexec r1 ip route + echo "*************************************************************************" +} + +code=0 +print_debug_info + +echo -e "\n\nTEST RESULTS" +echo "*********************************************************************************" +out=$($hexec user curl -s --connect-timeout 10 http://$extIP:80) +if [[ ${out} == *"Welcome to nginx"* ]]; then + echo "calico-k3s-dual-cluster (ccm) [OK]" +else + echo "calico-k3s-dual-cluster (ccm) [FAILED]" + code=1 +fi + +out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) + +if [[ ${out} == *"Welcome to nginx"* ]]; then + echo "calico-k3s-dual-cluster (kube-loxilb) tcp [OK]" +else + echo "calico-k3s-dual-cluster (kube-loxilb) tcp [FAILED]" + code=1 +fi + +out=$($hexec user timeout 30 ../common/udp_client $extIP 55003) +if [[ ${out} == *"Client"* ]]; then + echo "calico-k3s-dual-cluster (kube-loxillb) udp [OK]" +else + echo "calico-k3s-dual-cluster (kube-loxillb) udp [FAILED]" + code=1 +fi + +out=$($hexec user timeout 30 ../common/sctp_client 1.1.1.1 41291 $extIP 55004) +if [[ ${out} == *"server1"* ]]; then + echo "calico-k3s-dual-cluster (kube-loxillb) sctp [OK]" +else + echo "calico-k3s-dual-cluster (kube-loxillb) sctp [FAILED]" + code=1 +fi + +if [[ $code -eq 1 ]]; then + echo "calico-k3s-dual-cluster failed" + exit 1 +fi + +exit From 4df73783d92592c86b36f76d3751c599ee10bd8a Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Thu, 22 Jun 2023 23:41:59 -0400 Subject: [PATCH 2/2] removed unneccessary files --- .../custom-resources.yaml.bk | 32 - cicd/k3s-calico-dual-stack/install_k3s.sh | 37 - cicd/k3s-calico-dual-stack/k3s.sh | 1047 ----------------- 3 files changed, 1116 deletions(-) delete mode 100644 cicd/k3s-calico-dual-stack/custom-resources.yaml.bk delete mode 100755 cicd/k3s-calico-dual-stack/install_k3s.sh delete mode 100644 cicd/k3s-calico-dual-stack/k3s.sh diff --git a/cicd/k3s-calico-dual-stack/custom-resources.yaml.bk b/cicd/k3s-calico-dual-stack/custom-resources.yaml.bk deleted file mode 100644 index 90b6d482a..000000000 --- a/cicd/k3s-calico-dual-stack/custom-resources.yaml.bk +++ /dev/null @@ -1,32 +0,0 @@ -# This section includes base Calico installation configuration. -# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.Installation -apiVersion: operator.tigera.io/v1 -kind: Installation -metadata: - name: default -spec: - # Configures Calico networking. - calicoNetwork: - # Note: The ipPools section cannot be modified post-install. - ipPools: - - blockSize: 26 - cidr: 192.168.0.0/16 - encapsulation: VXLANCrossSubnet - natOutgoing: Enabled - nodeSelector: all() - - blockSize: 122 - cidr: 2002::0/64 - encapsulation: None - natOutgoing: Enabled - nodeSelector: all() - ---- - -# This section configures the Calico API server. -# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer -apiVersion: operator.tigera.io/v1 -kind: APIServer -metadata: - name: default -spec: {} - diff --git a/cicd/k3s-calico-dual-stack/install_k3s.sh b/cicd/k3s-calico-dual-stack/install_k3s.sh deleted file mode 100755 index 481b56a8c..000000000 --- a/cicd/k3s-calico-dual-stack/install_k3s.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -KUBECONFIG=--kubeconfig=/etc/rancher/k3s/k3s.yaml -if [ "$1" ]; then - KUBECONFIG="$1" -fi - -# If k3s setup exists, skip installation -if [[ -f "/usr/local/bin/k3s-uninstall.sh" ]]; then - echo "K3s exists" - sleep 10 -else - echo "Start K3s installation" - -# Install k3s without external cloud-manager and disabled servicelb - #curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external --flannel-backend=none --cluster-cidr=10.42.0.0/16" K3S_KUBECONFIG_MODE="644" sh - - curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external --flannel-backend=none --disable-network-policy --cluster-cidr=10.42.0.0/16,4dde::/64 --service-cidr=10.43.0.0/16,5dde::/108 --node-ip=12.12.12.254,8ffe::2" K3S_KUBECONFIG_MODE="644" sh - - - sleep 10 - - # Install Calico - kubectl $KUBECONFIG create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/tigera-operator.yaml - - #kubectl $KUBECONFIG create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/custom-resources.yaml - kubectl $KUBECONFIG apply -f custom-resources.yaml - - # Check kubectl works - kubectl $KUBECONFIG get pods -A - - # Remove taints in k3s if any (usually happens if started without cloud-manager) - kubectl $KUBECONFIG taint nodes --all node.cloudprovider.kubernetes.io/uninitialized=false:NoSchedule- - - # Start loxi-ccm as k3s daemonset - kubectl $KUBECONFIG apply -f https://github.com/loxilb-io/loxi-ccm/raw/master/manifests/loxi-ccm-k3s.yaml - - echo "End K3s installation" -fi diff --git a/cicd/k3s-calico-dual-stack/k3s.sh b/cicd/k3s-calico-dual-stack/k3s.sh deleted file mode 100644 index e96f474da..000000000 --- a/cicd/k3s-calico-dual-stack/k3s.sh +++ /dev/null @@ -1,1047 +0,0 @@ -#!/bin/sh -set -e -set -o noglob - -# Usage: -# curl ... | ENV_VAR=... sh - -# or -# ENV_VAR=... ./install.sh -# -# Example: -# Installing a server without traefik: -# curl ... | INSTALL_K3S_EXEC="--disable=traefik" sh - -# Installing an agent to point at a server: -# curl ... | K3S_TOKEN=xxx K3S_URL=https://server-url:6443 sh - -# -# Environment variables: -# - K3S_* -# Environment variables which begin with K3S_ will be preserved for the -# systemd service to use. Setting K3S_URL without explicitly setting -# a systemd exec command will default the command to "agent", and we -# enforce that K3S_TOKEN is also set. -# -# - INSTALL_K3S_SKIP_DOWNLOAD -# If set to true will not download k3s hash or binary. -# -# - INSTALL_K3S_FORCE_RESTART -# If set to true will always restart the K3s service -# -# - INSTALL_K3S_SYMLINK -# If set to 'skip' will not create symlinks, 'force' will overwrite, -# default will symlink if command does not exist in path. -# -# - INSTALL_K3S_SKIP_ENABLE -# If set to true will not enable or start k3s service. -# -# - INSTALL_K3S_SKIP_START -# If set to true will not start k3s service. -# -# - INSTALL_K3S_VERSION -# Version of k3s to download from github. Will attempt to download from the -# stable channel if not specified. -# -# - INSTALL_K3S_COMMIT -# Commit of k3s to download from temporary cloud storage. -# * (for developer & QA use) -# -# - INSTALL_K3S_BIN_DIR -# Directory to install k3s binary, links, and uninstall script to, or use -# /usr/local/bin as the default -# -# - INSTALL_K3S_BIN_DIR_READ_ONLY -# If set to true will not write files to INSTALL_K3S_BIN_DIR, forces -# setting INSTALL_K3S_SKIP_DOWNLOAD=true -# -# - INSTALL_K3S_SYSTEMD_DIR -# Directory to install systemd service and environment files to, or use -# /etc/systemd/system as the default -# -# - INSTALL_K3S_EXEC or script arguments -# Command with flags to use for launching k3s in the systemd service, if -# the command is not specified will default to "agent" if K3S_URL is set -# or "server" if not. The final systemd command resolves to a combination -# of EXEC and script args ($@). -# -# The following commands result in the same behavior: -# curl ... | INSTALL_K3S_EXEC="--disable=traefik" sh -s - -# curl ... | INSTALL_K3S_EXEC="server --disable=traefik" sh -s - -# curl ... | INSTALL_K3S_EXEC="server" sh -s - --disable=traefik -# curl ... | sh -s - server --disable=traefik -# curl ... | sh -s - --disable=traefik -# -# - INSTALL_K3S_NAME -# Name of systemd service to create, will default from the k3s exec command -# if not specified. If specified the name will be prefixed with 'k3s-'. -# -# - INSTALL_K3S_TYPE -# Type of systemd service to create, will default from the k3s exec command -# if not specified. -# -# - INSTALL_K3S_SELINUX_WARN -# If set to true will continue if k3s-selinux policy is not found. -# -# - INSTALL_K3S_SKIP_SELINUX_RPM -# If set to true will skip automatic installation of the k3s RPM. -# -# - INSTALL_K3S_CHANNEL_URL -# Channel URL for fetching k3s download URL. -# Defaults to 'https://update.k3s.io/v1-release/channels'. -# -# - INSTALL_K3S_CHANNEL -# Channel to use for fetching k3s download URL. -# Defaults to 'stable'. - -GITHUB_URL=https://github.com/k3s-io/k3s/releases -STORAGE_URL=https://k3s-ci-builds.s3.amazonaws.com -DOWNLOADER= - -# --- helper functions for logs --- -info() -{ - echo '[INFO] ' "$@" -} -warn() -{ - echo '[WARN] ' "$@" >&2 -} -fatal() -{ - echo '[ERROR] ' "$@" >&2 - exit 1 -} - -# --- fatal if no systemd or openrc --- -verify_system() { - if [ -x /sbin/openrc-run ]; then - HAS_OPENRC=true - return - fi - if [ -x /bin/systemctl ] || type systemctl > /dev/null 2>&1; then - HAS_SYSTEMD=true - return - fi - fatal 'Can not find systemd or openrc to use as a process supervisor for k3s' -} - -# --- add quotes to command arguments --- -quote() { - for arg in "$@"; do - printf '%s\n' "$arg" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" - done -} - -# --- add indentation and trailing slash to quoted args --- -quote_indent() { - printf ' \\\n' - for arg in "$@"; do - printf '\t%s \\\n' "$(quote "$arg")" - done -} - -# --- escape most punctuation characters, except quotes, forward slash, and space --- -escape() { - printf '%s' "$@" | sed -e 's/\([][!#$%&()*;<=>?\_`{|}]\)/\\\1/g;' -} - -# --- escape double quotes --- -escape_dq() { - printf '%s' "$@" | sed -e 's/"/\\"/g' -} - -# --- ensures $K3S_URL is empty or begins with https://, exiting fatally otherwise --- -verify_k3s_url() { - case "${K3S_URL}" in - "") - ;; - https://*) - ;; - *) - fatal "Only https:// URLs are supported for K3S_URL (have ${K3S_URL})" - ;; - esac -} - -# --- define needed environment variables --- -setup_env() { - # --- use command args if passed or create default --- - case "$1" in - # --- if we only have flags discover if command should be server or agent --- - (-*|"") - if [ -z "${K3S_URL}" ]; then - CMD_K3S=server - else - if [ -z "${K3S_TOKEN}" ] && [ -z "${K3S_TOKEN_FILE}" ]; then - fatal "Defaulted k3s exec command to 'agent' because K3S_URL is defined, but K3S_TOKEN or K3S_TOKEN_FILE is not defined." - fi - CMD_K3S=agent - fi - ;; - # --- command is provided --- - (*) - CMD_K3S=$1 - shift - ;; - esac - - verify_k3s_url - - CMD_K3S_EXEC="${CMD_K3S}$(quote_indent "$@")" - - # --- use systemd name if defined or create default --- - if [ -n "${INSTALL_K3S_NAME}" ]; then - SYSTEM_NAME=k3s-${INSTALL_K3S_NAME} - else - if [ "${CMD_K3S}" = server ]; then - SYSTEM_NAME=k3s - else - SYSTEM_NAME=k3s-${CMD_K3S} - fi - fi - - # --- check for invalid characters in system name --- - valid_chars=$(printf '%s' "${SYSTEM_NAME}" | sed -e 's/[][!#$%&()*;<=>?\_`{|}/[:space:]]/^/g;' ) - if [ "${SYSTEM_NAME}" != "${valid_chars}" ]; then - invalid_chars=$(printf '%s' "${valid_chars}" | sed -e 's/[^^]/ /g') - fatal "Invalid characters for system name: - ${SYSTEM_NAME} - ${invalid_chars}" - fi - - # --- use sudo if we are not already root --- - SUDO=sudo - if [ $(id -u) -eq 0 ]; then - SUDO= - fi - - # --- use systemd type if defined or create default --- - if [ -n "${INSTALL_K3S_TYPE}" ]; then - SYSTEMD_TYPE=${INSTALL_K3S_TYPE} - else - SYSTEMD_TYPE=notify - fi - - # --- use binary install directory if defined or create default --- - if [ -n "${INSTALL_K3S_BIN_DIR}" ]; then - BIN_DIR=${INSTALL_K3S_BIN_DIR} - else - # --- use /usr/local/bin if root can write to it, otherwise use /opt/bin if it exists - BIN_DIR=/usr/local/bin - if ! $SUDO sh -c "touch ${BIN_DIR}/k3s-ro-test && rm -rf ${BIN_DIR}/k3s-ro-test"; then - if [ -d /opt/bin ]; then - BIN_DIR=/opt/bin - fi - fi - fi - - # --- use systemd directory if defined or create default --- - if [ -n "${INSTALL_K3S_SYSTEMD_DIR}" ]; then - SYSTEMD_DIR="${INSTALL_K3S_SYSTEMD_DIR}" - else - SYSTEMD_DIR=/etc/systemd/system - fi - - # --- set related files from system name --- - SERVICE_K3S=${SYSTEM_NAME}.service - UNINSTALL_K3S_SH=${UNINSTALL_K3S_SH:-${BIN_DIR}/${SYSTEM_NAME}-uninstall.sh} - KILLALL_K3S_SH=${KILLALL_K3S_SH:-${BIN_DIR}/k3s-killall.sh} - - # --- use service or environment location depending on systemd/openrc --- - if [ "${HAS_SYSTEMD}" = true ]; then - FILE_K3S_SERVICE=${SYSTEMD_DIR}/${SERVICE_K3S} - FILE_K3S_ENV=${SYSTEMD_DIR}/${SERVICE_K3S}.env - elif [ "${HAS_OPENRC}" = true ]; then - $SUDO mkdir -p /etc/rancher/k3s - FILE_K3S_SERVICE=/etc/init.d/${SYSTEM_NAME} - FILE_K3S_ENV=/etc/rancher/k3s/${SYSTEM_NAME}.env - fi - - # --- get hash of config & exec for currently installed k3s --- - PRE_INSTALL_HASHES=$(get_installed_hashes) - - # --- if bin directory is read only skip download --- - if [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ]; then - INSTALL_K3S_SKIP_DOWNLOAD=true - fi - - # --- setup channel values - INSTALL_K3S_CHANNEL_URL=${INSTALL_K3S_CHANNEL_URL:-'https://update.k3s.io/v1-release/channels'} - INSTALL_K3S_CHANNEL=${INSTALL_K3S_CHANNEL:-'stable'} -} - -# --- check if skip download environment variable set --- -can_skip_download_binary() { - if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ] && [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != binary ]; then - return 1 - fi -} - -can_skip_download_selinux() { - if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ] && [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != selinux ]; then - return 1 - fi -} - -# --- verify an executable k3s binary is installed --- -verify_k3s_is_executable() { - if [ ! -x ${BIN_DIR}/k3s ]; then - fatal "Executable k3s binary not found at ${BIN_DIR}/k3s" - fi -} - -# --- set arch and suffix, fatal if architecture not supported --- -setup_verify_arch() { - if [ -z "$ARCH" ]; then - ARCH=$(uname -m) - fi - case $ARCH in - amd64) - ARCH=amd64 - SUFFIX= - ;; - x86_64) - ARCH=amd64 - SUFFIX= - ;; - arm64) - ARCH=arm64 - SUFFIX=-${ARCH} - ;; - s390x) - ARCH=s390x - SUFFIX=-${ARCH} - ;; - aarch64) - ARCH=arm64 - SUFFIX=-${ARCH} - ;; - arm*) - ARCH=arm - SUFFIX=-${ARCH}hf - ;; - *) - fatal "Unsupported architecture $ARCH" - esac -} - -# --- verify existence of network downloader executable --- -verify_downloader() { - # Return failure if it doesn't exist or is no executable - [ -x "$(command -v $1)" ] || return 1 - - # Set verified executable as our downloader program and return success - DOWNLOADER=$1 - return 0 -} - -# --- create temporary directory and cleanup when done --- -setup_tmp() { - TMP_DIR=$(mktemp -d -t k3s-install.XXXXXXXXXX) - TMP_HASH=${TMP_DIR}/k3s.hash - TMP_BIN=${TMP_DIR}/k3s.bin - cleanup() { - code=$? - set +e - trap - EXIT - rm -rf ${TMP_DIR} - exit $code - } - trap cleanup INT EXIT -} - -# --- use desired k3s version if defined or find version from channel --- -get_release_version() { - if [ -n "${INSTALL_K3S_COMMIT}" ]; then - VERSION_K3S="commit ${INSTALL_K3S_COMMIT}" - elif [ -n "${INSTALL_K3S_VERSION}" ]; then - VERSION_K3S=${INSTALL_K3S_VERSION} - else - info "Finding release for channel ${INSTALL_K3S_CHANNEL}" - version_url="${INSTALL_K3S_CHANNEL_URL}/${INSTALL_K3S_CHANNEL}" - case $DOWNLOADER in - curl) - VERSION_K3S=$(curl -w '%{url_effective}' -L -s -S ${version_url} -o /dev/null | sed -e 's|.*/||') - ;; - wget) - VERSION_K3S=$(wget -SqO /dev/null ${version_url} 2>&1 | grep -i Location | sed -e 's|.*/||') - ;; - *) - fatal "Incorrect downloader executable '$DOWNLOADER'" - ;; - esac - fi - info "Using ${VERSION_K3S} as release" -} - -# --- get k3s-selinux version --- -get_k3s_selinux_version() { - available_version="k3s-selinux-1.2-2.${rpm_target}.noarch.rpm" - info "Finding available k3s-selinux versions" - - # run verify_downloader in case it binary installation was skipped - verify_downloader curl || verify_downloader wget || fatal 'Can not find curl or wget for downloading files' - - case $DOWNLOADER in - curl) - DOWNLOADER_OPTS="-s" - ;; - wget) - DOWNLOADER_OPTS="-q -O -" - ;; - *) - fatal "Incorrect downloader executable '$DOWNLOADER'" - ;; - esac - for i in {1..3}; do - set +e - if [ "${rpm_channel}" = "testing" ]; then - version=$(timeout 5 ${DOWNLOADER} ${DOWNLOADER_OPTS} https://api.github.com/repos/k3s-io/k3s-selinux/releases | grep browser_download_url | awk '{ print $2 }' | grep -oE "[^\/]+${rpm_target}\.noarch\.rpm" | head -n 1) - else - version=$(timeout 5 ${DOWNLOADER} ${DOWNLOADER_OPTS} https://api.github.com/repos/k3s-io/k3s-selinux/releases/latest | grep browser_download_url | awk '{ print $2 }' | grep -oE "[^\/]+${rpm_target}\.noarch\.rpm") - fi - set -e - if [ "${version}" != "" ]; then - break - fi - sleep 1 - done - if [ "${version}" == "" ]; then - warn "Failed to get available versions of k3s-selinux..defaulting to ${available_version}" - return - fi - available_version=${version} -} - -# --- download from github url --- -download() { - [ $# -eq 2 ] || fatal 'download needs exactly 2 arguments' - - case $DOWNLOADER in - curl) - curl -o $1 -sfL $2 - ;; - wget) - wget -qO $1 $2 - ;; - *) - fatal "Incorrect executable '$DOWNLOADER'" - ;; - esac - - # Abort if download command failed - [ $? -eq 0 ] || fatal 'Download failed' -} - -# --- download hash from github url --- -download_hash() { - if [ -n "${INSTALL_K3S_COMMIT}" ]; then - HASH_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT}.sha256sum - else - HASH_URL=${GITHUB_URL}/download/${VERSION_K3S}/sha256sum-${ARCH}.txt - fi - info "Downloading hash ${HASH_URL}" - download ${TMP_HASH} ${HASH_URL} - HASH_EXPECTED=$(grep " k3s${SUFFIX}$" ${TMP_HASH}) - HASH_EXPECTED=${HASH_EXPECTED%%[[:blank:]]*} -} - -# --- check hash against installed version --- -installed_hash_matches() { - if [ -x ${BIN_DIR}/k3s ]; then - HASH_INSTALLED=$(sha256sum ${BIN_DIR}/k3s) - HASH_INSTALLED=${HASH_INSTALLED%%[[:blank:]]*} - if [ "${HASH_EXPECTED}" = "${HASH_INSTALLED}" ]; then - return - fi - fi - return 1 -} - -# --- download binary from github url --- -download_binary() { - if [ -n "${INSTALL_K3S_COMMIT}" ]; then - BIN_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT} - else - BIN_URL=${GITHUB_URL}/download/${VERSION_K3S}/k3s${SUFFIX} - fi - info "Downloading binary ${BIN_URL}" - download ${TMP_BIN} ${BIN_URL} -} - -# --- verify downloaded binary hash --- -verify_binary() { - info "Verifying binary download" - HASH_BIN=$(sha256sum ${TMP_BIN}) - HASH_BIN=${HASH_BIN%%[[:blank:]]*} - if [ "${HASH_EXPECTED}" != "${HASH_BIN}" ]; then - fatal "Download sha256 does not match ${HASH_EXPECTED}, got ${HASH_BIN}" - fi -} - -# --- setup permissions and move binary to system directory --- -setup_binary() { - chmod 755 ${TMP_BIN} - info "Installing k3s to ${BIN_DIR}/k3s" - $SUDO chown root:root ${TMP_BIN} - $SUDO mv -f ${TMP_BIN} ${BIN_DIR}/k3s -} - -# --- setup selinux policy --- -setup_selinux() { - case ${INSTALL_K3S_CHANNEL} in - *testing) - rpm_channel=testing - ;; - *latest) - rpm_channel=latest - ;; - *) - rpm_channel=stable - ;; - esac - - rpm_site="rpm.rancher.io" - if [ "${rpm_channel}" = "testing" ]; then - rpm_site="rpm-testing.rancher.io" - fi - - [ -r /etc/os-release ] && . /etc/os-release - if [ `expr "${ID_LIKE}" : ".*suse.*"` != 0 ]; then - rpm_target=sle - rpm_site_infix=microos - package_installer=zypper - if [ "${ID_LIKE:-}" = suse ] && [ "${VARIANT_ID:-}" = sle-micro ]; then - rpm_target=sle - rpm_site_infix=slemicro - package_installer=zypper - fi - elif [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then - rpm_target=coreos - rpm_site_infix=coreos - package_installer=rpm-ostree - elif [ "${VERSION_ID%%.*}" = "7" ]; then - rpm_target=el7 - rpm_site_infix=centos/7 - package_installer=yum - elif [ "${VERSION_ID%%.*}" = "8" ] || [ "${VERSION_ID%%.*}" -gt "36" ]; then - rpm_target=el8 - rpm_site_infix=centos/8 - package_installer=yum - else - rpm_target=el9 - rpm_site_infix=centos/9 - package_installer=yum - fi - - if [ "${package_installer}" = "rpm-ostree" ] && [ -x /bin/yum ]; then - package_installer=yum - fi - - if [ "${package_installer}" = "yum" ] && [ -x /usr/bin/dnf ]; then - package_installer=dnf - fi - - policy_hint="please install: - ${package_installer} install -y container-selinux - ${package_installer} install -y https://${rpm_site}/k3s/${rpm_channel}/common/${rpm_site_infix}/noarch/${available_version} -" - - if [ "$INSTALL_K3S_SKIP_SELINUX_RPM" = true ] || can_skip_download_selinux || [ ! -d /usr/share/selinux ]; then - info "Skipping installation of SELinux RPM" - else - get_k3s_selinux_version - install_selinux_rpm ${rpm_site} ${rpm_channel} ${rpm_target} ${rpm_site_infix} - fi - - policy_error=fatal - if [ "$INSTALL_K3S_SELINUX_WARN" = true ] || [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then - policy_error=warn - fi - - if ! $SUDO chcon -u system_u -r object_r -t container_runtime_exec_t ${BIN_DIR}/k3s >/dev/null 2>&1; then - if $SUDO grep '^\s*SELINUX=enforcing' /etc/selinux/config >/dev/null 2>&1; then - $policy_error "Failed to apply container_runtime_exec_t to ${BIN_DIR}/k3s, ${policy_hint}" - fi - elif [ ! -f /usr/share/selinux/packages/k3s.pp ]; then - if [ -x /usr/sbin/transactional-update ] || [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then - warn "Please reboot your machine to activate the changes and avoid data loss." - else - $policy_error "Failed to find the k3s-selinux policy, ${policy_hint}" - fi - fi -} - -install_selinux_rpm() { - if [ -r /etc/redhat-release ] || [ -r /etc/centos-release ] || [ -r /etc/oracle-release ] || [ -r /etc/fedora-release ] || [ "${ID_LIKE%%[ ]*}" = "suse" ]; then - repodir=/etc/yum.repos.d - if [ -d /etc/zypp/repos.d ]; then - repodir=/etc/zypp/repos.d - fi - set +o noglob - $SUDO rm -f ${repodir}/rancher-k3s-common*.repo - set -o noglob - if [ -r /etc/redhat-release ] && [ "${3}" = "el7" ]; then - $SUDO yum install -y yum-utils - $SUDO yum-config-manager --enable rhel-7-server-extras-rpms - fi - $SUDO tee ${repodir}/rancher-k3s-common.repo >/dev/null << EOF -[rancher-k3s-common-${2}] -name=Rancher K3s Common (${2}) -baseurl=https://${1}/k3s/${2}/common/${4}/noarch -enabled=1 -gpgcheck=1 -repo_gpgcheck=0 -gpgkey=https://${1}/public.key -EOF - case ${3} in - sle) - rpm_installer="zypper --gpg-auto-import-keys" - if [ "${TRANSACTIONAL_UPDATE=false}" != "true" ] && [ -x /usr/sbin/transactional-update ]; then - transactional_update_run="transactional-update --no-selfupdate -d run" - rpm_installer="transactional-update --no-selfupdate -d run ${rpm_installer}" - : "${INSTALL_K3S_SKIP_START:=true}" - fi - # create the /var/lib/rpm-state in SLE systems to fix the prein selinux macro - ${transactional_update_run} mkdir -p /var/lib/rpm-state - ;; - coreos) - rpm_installer="rpm-ostree" - # rpm_install_extra_args="--apply-live" - : "${INSTALL_K3S_SKIP_START:=true}" - ;; - *) - rpm_installer="yum" - ;; - esac - if [ "${rpm_installer}" = "yum" ] && [ -x /usr/bin/dnf ]; then - rpm_installer=dnf - fi - if rpm -q --quiet k3s-selinux; then - # remove k3s-selinux module before upgrade to allow container-selinux to upgrade safely - if check_available_upgrades container-selinux ${3} && check_available_upgrades k3s-selinux ${3}; then - MODULE_PRIORITY=$($SUDO semodule --list=full | grep k3s | cut -f1 -d" ") - if [ -n "${MODULE_PRIORITY}" ]; then - $SUDO semodule -X $MODULE_PRIORITY -r k3s || true - fi - fi - fi - # shellcheck disable=SC2086 - $SUDO ${rpm_installer} install -y "k3s-selinux" - fi - return -} - -check_available_upgrades() { - set +e - case ${2} in - sle) - available_upgrades=$($SUDO zypper -q -t -s 11 se -s -u --type package $1 | tail -n 1 | grep -v "No matching" | awk '{print $3}') - ;; - coreos) - # currently rpm-ostree does not support search functionality https://github.com/coreos/rpm-ostree/issues/1877 - ;; - *) - available_upgrades=$($SUDO yum -q --refresh list $1 --upgrades | tail -n 1 | awk '{print $2}') - ;; - esac - set -e - if [ -n "${available_upgrades}" ]; then - return 0 - fi - return 1 -} -# --- download and verify k3s --- -download_and_verify() { - if can_skip_download_binary; then - info 'Skipping k3s download and verify' - verify_k3s_is_executable - return - fi - - setup_verify_arch - verify_downloader curl || verify_downloader wget || fatal 'Can not find curl or wget for downloading files' - setup_tmp - get_release_version - download_hash - - if installed_hash_matches; then - info 'Skipping binary downloaded, installed k3s matches hash' - return - fi - - download_binary - verify_binary - setup_binary -} - -# --- add additional utility links --- -create_symlinks() { - [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return - [ "${INSTALL_K3S_SYMLINK}" = skip ] && return - - for cmd in kubectl crictl ctr; do - if [ ! -e ${BIN_DIR}/${cmd} ] || [ "${INSTALL_K3S_SYMLINK}" = force ]; then - which_cmd=$(command -v ${cmd} 2>/dev/null || true) - if [ -z "${which_cmd}" ] || [ "${INSTALL_K3S_SYMLINK}" = force ]; then - info "Creating ${BIN_DIR}/${cmd} symlink to k3s" - $SUDO ln -sf k3s ${BIN_DIR}/${cmd} - else - info "Skipping ${BIN_DIR}/${cmd} symlink to k3s, command exists in PATH at ${which_cmd}" - fi - else - info "Skipping ${BIN_DIR}/${cmd} symlink to k3s, already exists" - fi - done -} - -# --- create killall script --- -create_killall() { - [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return - info "Creating killall script ${KILLALL_K3S_SH}" - $SUDO tee ${KILLALL_K3S_SH} >/dev/null << \EOF -#!/bin/sh -[ $(id -u) -eq 0 ] || exec sudo $0 $@ - -for bin in /var/lib/rancher/k3s/data/**/bin/; do - [ -d $bin ] && export PATH=$PATH:$bin:$bin/aux -done - -set -x - -for service in /etc/systemd/system/k3s*.service; do - [ -s $service ] && systemctl stop $(basename $service) -done - -for service in /etc/init.d/k3s*; do - [ -x $service ] && $service stop -done - -pschildren() { - ps -e -o ppid= -o pid= | \ - sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \ - grep -w "^$1" | \ - cut -f2 -} - -pstree() { - for pid in $@; do - echo $pid - for child in $(pschildren $pid); do - pstree $child - done - done -} - -killtree() { - kill -9 $( - { set +x; } 2>/dev/null; - pstree $@; - set -x; - ) 2>/dev/null -} - -remove_interfaces() { - # Delete network interface(s) that match 'master cni0' - ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do - iface=${iface%%@*} - [ -z "$iface" ] || ip link delete $iface - done - - # Delete cni related interfaces - ip link delete cni0 - ip link delete flannel.1 - ip link delete flannel-v6.1 - ip link delete kube-ipvs0 - ip link delete flannel-wg - ip link delete flannel-wg-v6 - - # Restart tailscale - if [ -n "$(command -v tailscale)" ]; then - tailscale set --advertise-routes= - fi -} - -getshims() { - ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1 -} - -killtree $({ set +x; } 2>/dev/null; getshims; set -x) - -do_unmount_and_remove() { - set +x - while read -r _ path _; do - case "$path" in $1*) echo "$path" ;; esac - done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"' - set -x -} - -do_unmount_and_remove '/run/k3s' -do_unmount_and_remove '/var/lib/rancher/k3s' -do_unmount_and_remove '/var/lib/kubelet/pods' -do_unmount_and_remove '/var/lib/kubelet/plugins' -do_unmount_and_remove '/run/netns/cni-' - -# Remove CNI namespaces -ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete - -remove_interfaces - -rm -rf /var/lib/cni/ -iptables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | iptables-restore -ip6tables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | ip6tables-restore -EOF - $SUDO chmod 755 ${KILLALL_K3S_SH} - $SUDO chown root:root ${KILLALL_K3S_SH} -} - -# --- create uninstall script --- -create_uninstall() { - [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return - info "Creating uninstall script ${UNINSTALL_K3S_SH}" - $SUDO tee ${UNINSTALL_K3S_SH} >/dev/null << EOF -#!/bin/sh -set -x -[ \$(id -u) -eq 0 ] || exec sudo \$0 \$@ - -${KILLALL_K3S_SH} - -if command -v systemctl; then - systemctl disable ${SYSTEM_NAME} - systemctl reset-failed ${SYSTEM_NAME} - systemctl daemon-reload -fi -if command -v rc-update; then - rc-update delete ${SYSTEM_NAME} default -fi - -rm -f ${FILE_K3S_SERVICE} -rm -f ${FILE_K3S_ENV} - -remove_uninstall() { - rm -f ${UNINSTALL_K3S_SH} -} -trap remove_uninstall EXIT - -if (ls ${SYSTEMD_DIR}/k3s*.service || ls /etc/init.d/k3s*) >/dev/null 2>&1; then - set +x; echo 'Additional k3s services installed, skipping uninstall of k3s'; set -x - exit -fi - -for cmd in kubectl crictl ctr; do - if [ -L ${BIN_DIR}/\$cmd ]; then - rm -f ${BIN_DIR}/\$cmd - fi -done - -rm -rf /etc/rancher/k3s -rm -rf /run/k3s -rm -rf /run/flannel -rm -rf /var/lib/rancher/k3s -rm -rf /var/lib/kubelet -rm -f ${BIN_DIR}/k3s -rm -f ${KILLALL_K3S_SH} - -if type yum >/dev/null 2>&1; then - yum remove -y k3s-selinux - rm -f /etc/yum.repos.d/rancher-k3s-common*.repo -elif type rpm-ostree >/dev/null 2>&1; then - rpm-ostree uninstall k3s-selinux - rm -f /etc/yum.repos.d/rancher-k3s-common*.repo -elif type zypper >/dev/null 2>&1; then - uninstall_cmd="zypper remove -y k3s-selinux" - if [ "\${TRANSACTIONAL_UPDATE=false}" != "true" ] && [ -x /usr/sbin/transactional-update ]; then - uninstall_cmd="transactional-update --no-selfupdate -d run \$uninstall_cmd" - fi - \$uninstall_cmd - rm -f /etc/zypp/repos.d/rancher-k3s-common*.repo -fi -EOF - $SUDO chmod 755 ${UNINSTALL_K3S_SH} - $SUDO chown root:root ${UNINSTALL_K3S_SH} -} - -# --- disable current service if loaded -- -systemd_disable() { - $SUDO systemctl disable ${SYSTEM_NAME} >/dev/null 2>&1 || true - $SUDO rm -f /etc/systemd/system/${SERVICE_K3S} || true - $SUDO rm -f /etc/systemd/system/${SERVICE_K3S}.env || true -} - -# --- capture current env and create file containing k3s_ variables --- -create_env_file() { - info "env: Creating environment file ${FILE_K3S_ENV}" - $SUDO touch ${FILE_K3S_ENV} - $SUDO chmod 0600 ${FILE_K3S_ENV} - sh -c export | while read x v; do echo $v; done | grep -E '^(K3S|CONTAINERD)_' | $SUDO tee ${FILE_K3S_ENV} >/dev/null - sh -c export | while read x v; do echo $v; done | grep -Ei '^(NO|HTTP|HTTPS)_PROXY' | $SUDO tee -a ${FILE_K3S_ENV} >/dev/null -} - -# --- write systemd service file --- -create_systemd_service_file() { - info "systemd: Creating service file ${FILE_K3S_SERVICE}" - $SUDO tee ${FILE_K3S_SERVICE} >/dev/null << EOF -[Unit] -Description=Lightweight Kubernetes -Documentation=https://k3s.io -Wants=network-online.target -After=network-online.target - -[Install] -WantedBy=multi-user.target - -[Service] -Type=${SYSTEMD_TYPE} -EnvironmentFile=-/etc/default/%N -EnvironmentFile=-/etc/sysconfig/%N -EnvironmentFile=-${FILE_K3S_ENV} -KillMode=process -Delegate=yes -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNOFILE=1048576 -LimitNPROC=infinity -LimitCORE=infinity -TasksMax=infinity -TimeoutStartSec=0 -Restart=always -RestartSec=5s -ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service' -ExecStartPre=-/sbin/modprobe br_netfilter -ExecStartPre=-/sbin/modprobe overlay -ExecStart=${BIN_DIR}/k3s \\ - ${CMD_K3S_EXEC} - -EOF -} - -# --- write openrc service file --- -create_openrc_service_file() { - LOG_FILE=/var/log/${SYSTEM_NAME}.log - - info "openrc: Creating service file ${FILE_K3S_SERVICE}" - $SUDO tee ${FILE_K3S_SERVICE} >/dev/null << EOF -#!/sbin/openrc-run - -depend() { - after network-online - want cgroups -} - -start_pre() { - rm -f /tmp/k3s.* -} - -supervisor=supervise-daemon -name=${SYSTEM_NAME} -command="${BIN_DIR}/k3s" -command_args="$(escape_dq "${CMD_K3S_EXEC}") - >>${LOG_FILE} 2>&1" - -output_log=${LOG_FILE} -error_log=${LOG_FILE} - -pidfile="/var/run/${SYSTEM_NAME}.pid" -respawn_delay=5 -respawn_max=0 - -set -o allexport -if [ -f /etc/environment ]; then . /etc/environment; fi -if [ -f ${FILE_K3S_ENV} ]; then . ${FILE_K3S_ENV}; fi -set +o allexport -EOF - $SUDO chmod 0755 ${FILE_K3S_SERVICE} - - $SUDO tee /etc/logrotate.d/${SYSTEM_NAME} >/dev/null << EOF -${LOG_FILE} { - missingok - notifempty - copytruncate -} -EOF -} - -# --- write systemd or openrc service file --- -create_service_file() { - [ "${HAS_SYSTEMD}" = true ] && create_systemd_service_file - [ "${HAS_OPENRC}" = true ] && create_openrc_service_file - return 0 -} - -# --- get hashes of the current k3s bin and service files -get_installed_hashes() { - $SUDO sha256sum ${BIN_DIR}/k3s ${FILE_K3S_SERVICE} ${FILE_K3S_ENV} 2>&1 || true -} - -# --- enable and start systemd service --- -systemd_enable() { - info "systemd: Enabling ${SYSTEM_NAME} unit" - $SUDO systemctl enable ${FILE_K3S_SERVICE} >/dev/null - $SUDO systemctl daemon-reload >/dev/null -} - -systemd_start() { - info "systemd: Starting ${SYSTEM_NAME}" - $SUDO systemctl restart ${SYSTEM_NAME} -} - -# --- enable and start openrc service --- -openrc_enable() { - info "openrc: Enabling ${SYSTEM_NAME} service for default runlevel" - $SUDO rc-update add ${SYSTEM_NAME} default >/dev/null -} - -openrc_start() { - info "openrc: Starting ${SYSTEM_NAME}" - $SUDO ${FILE_K3S_SERVICE} restart -} - -# --- startup systemd or openrc service --- -service_enable_and_start() { - if [ -f "/proc/cgroups" ] && [ "$(grep memory /proc/cgroups | while read -r n n n enabled; do echo $enabled; done)" -eq 0 ]; - then - info 'Failed to find memory cgroup, you may need to add "cgroup_memory=1 cgroup_enable=memory" to your linux cmdline (/boot/cmdline.txt on a Raspberry Pi)' - fi - - [ "${INSTALL_K3S_SKIP_ENABLE}" = true ] && return - - [ "${HAS_SYSTEMD}" = true ] && systemd_enable - [ "${HAS_OPENRC}" = true ] && openrc_enable - - [ "${INSTALL_K3S_SKIP_START}" = true ] && return - - POST_INSTALL_HASHES=$(get_installed_hashes) - if [ "${PRE_INSTALL_HASHES}" = "${POST_INSTALL_HASHES}" ] && [ "${INSTALL_K3S_FORCE_RESTART}" != true ]; then - info 'No change detected so skipping service start' - return - fi - - if command -v iptables-save 1> /dev/null && command -v iptables-restore 1> /dev/null - then - $SUDO iptables-save | grep -v KUBE- | grep -iv flannel | $SUDO iptables-restore - fi - if command -v ip6tables-save 1> /dev/null && command -v ip6tables-restore 1> /dev/null - then - $SUDO ip6tables-save | grep -v KUBE- | grep -iv flannel | $SUDO ip6tables-restore - fi - - [ "${HAS_SYSTEMD}" = true ] && systemd_start - [ "${HAS_OPENRC}" = true ] && openrc_start - return 0 -} - -# --- re-evaluate args to include env command --- -eval set -- $(escape "${INSTALL_K3S_EXEC}") $(quote "$@") - -# --- run the install process -- -{ - verify_system - setup_env "$@" - download_and_verify - setup_selinux - create_symlinks - create_killall - create_uninstall - systemd_disable - create_env_file - create_service_file - service_enable_and_start -}