From 50433930da5889013526fc50691e13502604e31b Mon Sep 17 00:00:00 2001 From: Artem Glazychev Date: Mon, 20 Feb 2023 19:33:20 +0700 Subject: [PATCH 1/2] Update packet setup Signed-off-by: Artem Glazychev --- .github/workflows/ci.yaml | 13 +++++++------ go.mod | 2 +- go.sum | 4 ++-- scripts/cleanup-cluster.sh | 3 --- scripts/defaultCNI/kustomization.yaml | 9 +++++++++ scripts/defaultCNI/patch.yaml | 16 ++++++++++++++++ scripts/setup-cluster.sh | 7 +++---- 7 files changed, 38 insertions(+), 16 deletions(-) create mode 100644 scripts/defaultCNI/kustomization.yaml create mode 100644 scripts/defaultCNI/patch.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index df67cc7f..ffc4df7f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -65,14 +65,14 @@ jobs: - name: Install clusterctl run: | - curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.1/clusterctl-linux-amd64 -o clusterctl + curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.3.3/clusterctl-linux-amd64 -o clusterctl chmod +x ./clusterctl mv ./clusterctl /usr/local/bin/clusterctl clusterctl version - name: Install metal CLI run: | - curl -L https://github.com/equinix/metal-cli/releases/download/v0.9.0/metal-linux-amd64 -o metal + curl -L https://github.com/equinix/metal-cli/releases/download/v0.13.0/metal-linux-amd64 -o metal chmod +x ./metal mv ./metal /usr/local/bin/metal metal -v @@ -96,16 +96,16 @@ jobs: METAL_AUTH_TOKEN: ${{ secrets.PACKET_AUTH_TOKEN }} - name: Setup cluster - uses: nick-fields/retry@v2.8.1 + uses: nick-fields/retry@v2.8.3 with: - timeout_minutes: 30 + timeout_minutes: 50 max_attempts: 3 command: | cd ${{ github.repository }} scripts/setup-cluster.sh "/tmp/sshkey" on_retry_command: | cd ${{ github.repository }} - scripts/cleanup-cluster.sh ${CLUSTER_NAME} + scripts/cleanup-cluster.sh env: PACKET_API_KEY: ${{ secrets.PACKET_AUTH_TOKEN }} METAL_AUTH_TOKEN: ${{ secrets.PACKET_AUTH_TOKEN }} @@ -142,7 +142,8 @@ jobs: working-directory: ${{ github.repository }} run: | export KUBECONFIG=$HOME/.kube/config - scripts/cleanup-cluster.sh ${{ steps.setup_ssh.outputs.ssh_id }} + yes | metal ssh-key delete --id "${{ steps.setup_ssh.outputs.ssh_id }}" + scripts/cleanup-cluster.sh env: PROJECT_ID: 383890d0-f5d1-4de1-881a-4d1ede549d18 METAL_AUTH_TOKEN: ${{ secrets.PACKET_AUTH_TOKEN }} diff --git a/go.mod b/go.mod index e1d35cda..ac0dbec6 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/networkservicemesh/integration-k8s-packet go 1.18 require ( - github.com/networkservicemesh/integration-tests v0.0.0-20230110004352-74d7402da8af + github.com/networkservicemesh/integration-tests v0.0.0-20230227120053-ecc410bc9425 github.com/stretchr/testify v1.7.0 ) diff --git a/go.sum b/go.sum index 1416687f..1daeb2e4 100644 --- a/go.sum +++ b/go.sum @@ -158,8 +158,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/networkservicemesh/gotestmd v0.0.0-20220628095933-eabbdc09e0dc h1:1L/OisEFsOyhwaqeJpYmM1nlJ2dBusUMiszPDBlUip0= github.com/networkservicemesh/gotestmd v0.0.0-20220628095933-eabbdc09e0dc/go.mod h1:8EWnekTRNX+NxBdTFE24WqUoM7SgJHbiafDBrIIdOmQ= -github.com/networkservicemesh/integration-tests v0.0.0-20230110004352-74d7402da8af h1:cp8bgP2pZn8NjIPint+EZgmk4881ZiRPwWXvG5CGfrA= -github.com/networkservicemesh/integration-tests v0.0.0-20230110004352-74d7402da8af/go.mod h1:qkqJgckg5vY5STByxzKhzhDB+TKCAU0qasuwfJ0cG+s= +github.com/networkservicemesh/integration-tests v0.0.0-20230227120053-ecc410bc9425 h1:IFZBH3DcSVdn8Dg+Isb0JRMhXIixXvyveVLe4oQ+ejw= +github.com/networkservicemesh/integration-tests v0.0.0-20230227120053-ecc410bc9425/go.mod h1:qkqJgckg5vY5STByxzKhzhDB+TKCAU0qasuwfJ0cG+s= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/scripts/cleanup-cluster.sh b/scripts/cleanup-cluster.sh index 27baee0f..74e3e47d 100755 --- a/scripts/cleanup-cluster.sh +++ b/scripts/cleanup-cluster.sh @@ -1,10 +1,7 @@ #!/bin/bash -x -ssh_id=$1 - ## Cluster API Packet provides, together with the cluster, always creates an address that must be manually deleted -yes | metal ssh-key delete --id "$ssh_id" eip=$(metal ip get -p "${PROJECT_ID}" -o json | jq -r '.[] | select( .tags != null and any(.tags[]; endswith(env.CLUSTER_NAME))) | .id') metal ip remove -i "$eip" kubectl delete cluster "${CLUSTER_NAME}" diff --git a/scripts/defaultCNI/kustomization.yaml b/scripts/defaultCNI/kustomization.yaml new file mode 100644 index 00000000..4f1c1e40 --- /dev/null +++ b/scripts/defaultCNI/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - https://raw.githubusercontent.com/projectcalico/calico/v3.25.0/manifests/calico.yaml + +patchesStrategicMerge: + - patch.yaml diff --git a/scripts/defaultCNI/patch.yaml b/scripts/defaultCNI/patch.yaml new file mode 100644 index 00000000..81fd075e --- /dev/null +++ b/scripts/defaultCNI/patch.yaml @@ -0,0 +1,16 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node +spec: + template: + spec: + containers: + - name: calico-node + env: + - name: IP_AUTODETECTION_METHOD + value: "interface=bond0" diff --git a/scripts/setup-cluster.sh b/scripts/setup-cluster.sh index 39c83665..cead19c4 100755 --- a/scripts/setup-cluster.sh +++ b/scripts/setup-cluster.sh @@ -25,7 +25,7 @@ source scripts/include/wait-pids.sh source scripts/include/wait-start.sh ## Run clusterctl -clusterctl init --infrastructure packet || exit 1 +clusterctl init --infrastructure packet:v0.6.2 || exit 1 clusterctl generate cluster ${CLUSTER_NAME} \ --kubernetes-version ${KUBERNETES_VERSION} \ --control-plane-machine-count=1 \ @@ -33,7 +33,6 @@ clusterctl generate cluster ${CLUSTER_NAME} \ > packet.yaml || exit 2 # If CNI is Calico, we need to make a few corrections to the generated template -sed -r -i 's/(cloud-provider-equinix-metal\/releases\/download\/v[0-9.]*\/)/cloud-provider-equinix-metal\/releases\/download\/v3.5.0\//g' packet.yaml if [[ "$CNI" == "calico-vpp" ]]; then # calico sed -i "/^ initConfiguration:/a \ localAPIEndpoint:\n advertiseAddress: ${CALICO_MASTER_IP}" packet.yaml sed -i "/^ preKubeadmCommands:/a \ - ifenslave -d bond0 ${CALICO_INTERFACE}\n - ip addr change ${CALICO_MASTER_IP}/${CALICO_CIDR_PREFIX} dev ${CALICO_INTERFACE}\n - ip link set up dev ${CALICO_INTERFACE}" packet.yaml @@ -91,7 +90,7 @@ for i in {1..30}; do echo "node count timeout exceeded. exit" exit 11 fi - sleep 10s + sleep 20s done ## Setup SR-IOV @@ -99,7 +98,7 @@ done ## CNI installation if [[ "$CNI" == "default" ]]; then # use calico CNI in case of default - kubectl --kubeconfig=$KUBECONFIG_PACK apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.24.1/manifests/calico.yaml || exit 13 + kubectl --kubeconfig=$KUBECONFIG_PACK apply -k scripts/defaultCNI || exit 13 elif [[ "$CNI" == "calico-vpp" ]]; then # calico-VPP CNI export KUBECONFIG=$KUBECONFIG_PACK /bin/bash scripts/calico/deploy-calico.sh || exit 14 From 7a6ac9739eda0ac4bad23a20b3e5563e61d6a1dd Mon Sep 17 00:00:00 2001 From: Artem Glazychev Date: Tue, 28 Feb 2023 14:51:03 +0700 Subject: [PATCH 2/2] Update calico setup Signed-off-by: Artem Glazychev --- .github/workflows/ci.yaml | 2 +- scripts/calico/deploy-calico.sh | 6 ++++-- scripts/calico/kustomization.yaml | 2 +- scripts/calico/patch.yaml | 32 ++++++++++++++++++++++++++++++- scripts/sriov/config-SRIOV.sh | 3 +-- 5 files changed, 38 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ffc4df7f..19904f92 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -111,7 +111,7 @@ jobs: METAL_AUTH_TOKEN: ${{ secrets.PACKET_AUTH_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PROJECT_ID: 383890d0-f5d1-4de1-881a-4d1ede549d18 - FACILITY: da11 + FACILITY: ch3 NODE_OS: ubuntu_20_04 KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }} CNI: ${{ matrix.CNI }} diff --git a/scripts/calico/deploy-calico.sh b/scripts/calico/deploy-calico.sh index e0021da1..6d43ae20 100755 --- a/scripts/calico/deploy-calico.sh +++ b/scripts/calico/deploy-calico.sh @@ -6,8 +6,10 @@ function on_error() { } trap 'on_error' ERR -kubectl apply -f https://projectcalico.docs.tigera.io/archive/v3.23/manifests/tigera-operator.yaml -kubectl apply -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/v3.23.0/yaml/calico/installation-default.yaml +kubectl create -f https://projectcalico.docs.tigera.io/archive/v3.24/manifests/tigera-operator.yaml +sleep 5s +kubectl wait --for condition=established --timeout=10s crd/installations.operator.tigera.io +kubectl create -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/82c88a14e5e0e3cc5d7f70c52cdbc01c999d3a42/yaml/calico/installation-default.yaml kubectl apply -k scripts/calico kubectl rollout status -n calico-vpp-dataplane ds/calico-vpp-node --timeout=10m diff --git a/scripts/calico/kustomization.yaml b/scripts/calico/kustomization.yaml index 2a1501fd..1296d8f6 100644 --- a/scripts/calico/kustomization.yaml +++ b/scripts/calico/kustomization.yaml @@ -3,7 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - https://raw.githubusercontent.com/projectcalico/vpp-dataplane/v3.23.0/yaml/generated/calico-vpp-nohuge.yaml + - https://raw.githubusercontent.com/projectcalico/vpp-dataplane/82c88a14e5e0e3cc5d7f70c52cdbc01c999d3a42/yaml/generated/calico-vpp-nohuge.yaml patchesStrategicMerge: - patch.yaml diff --git a/scripts/calico/patch.yaml b/scripts/calico/patch.yaml index a0ff05e1..0210b6dc 100644 --- a/scripts/calico/patch.yaml +++ b/scripts/calico/patch.yaml @@ -5,4 +5,34 @@ metadata: name: calico-vpp-config namespace: calico-vpp-dataplane data: - vpp_dataplane_interface: ens6f1 + CALICOVPP_INTERFACES: |- + { + "maxPodIfSpec": { + "rx": 10, "tx": 10, "rxqsz": 1024, "txqsz": 1024 + }, + "defaultPodIfSpec": { + "rx": 1, "tx":1, "isl3": true + }, + "vppHostTapSpec": { + "rx": 1, "tx":1, "rxqsz": 1024, "txqsz": 1024, "isl3": false + }, + "uplinkInterfaces": [ + { + "interfaceName": "ens6f1" + } + ] + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: calico-vpp-node + namespace: calico-vpp-dataplane +spec: + template: + spec: + containers: + - name: agent + image: artgl/calico-agent:82c88a14 + - name: vpp + image: artgl/calico-vpp:82c88a14 diff --git a/scripts/sriov/config-SRIOV.sh b/scripts/sriov/config-SRIOV.sh index 0a1faa76..31de49f2 100755 --- a/scripts/sriov/config-SRIOV.sh +++ b/scripts/sriov/config-SRIOV.sh @@ -35,8 +35,7 @@ function config_link() { retry=0 while [ ${retry} -lt 3 ]; do echo 1 > "${device}/sriov_numvfs" || return 1 - numfs=$(head -n 1 "${device}/sriov_numvfs") - if [ "$numfs" = "1" ]; then + if lspci | grep -q "Virtual Function" ; then vf_kernel_driver="$(softlink_target "${device}/virtfn0/driver")" break else