Merge pull request #322 from networkservicemesh/fix_setup

Update packet setup

.github/workflows/ci.yaml

@@ -65,14 +65,14 @@ jobs:
 
       - name: Install clusterctl
         run: |
-          curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.1/clusterctl-linux-amd64 -o clusterctl
+          curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.3.3/clusterctl-linux-amd64 -o clusterctl
           chmod +x ./clusterctl
           mv ./clusterctl /usr/local/bin/clusterctl
           clusterctl version
 
       - name: Install metal CLI
         run: |
-          curl -L https://github.com/equinix/metal-cli/releases/download/v0.9.0/metal-linux-amd64 -o metal
+          curl -L https://github.com/equinix/metal-cli/releases/download/v0.13.0/metal-linux-amd64 -o metal
           chmod +x ./metal
           mv ./metal /usr/local/bin/metal
           metal -v
@@ -96,22 +96,22 @@ jobs:
           METAL_AUTH_TOKEN: ${{ secrets.PACKET_AUTH_TOKEN }}
 
       - name: Setup cluster
-        uses: nick-fields/retry@v2.8.1
+        uses: nick-fields/retry@v2.8.3
         with:
-          timeout_minutes: 30
+          timeout_minutes: 50
           max_attempts: 3
           command: |
             cd ${{ github.repository }}
             scripts/setup-cluster.sh "/tmp/sshkey"
           on_retry_command: |
             cd ${{ github.repository }}
-            scripts/cleanup-cluster.sh ${CLUSTER_NAME}
+            scripts/cleanup-cluster.sh
         env:
           PACKET_API_KEY: ${{ secrets.PACKET_AUTH_TOKEN }}
           METAL_AUTH_TOKEN: ${{ secrets.PACKET_AUTH_TOKEN }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           PROJECT_ID: 383890d0-f5d1-4de1-881a-4d1ede549d18
-          FACILITY: da11
+          FACILITY: ch3
           NODE_OS: ubuntu_20_04
           KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }}
           CNI: ${{ matrix.CNI }}
@@ -142,7 +142,8 @@ jobs:
         working-directory: ${{ github.repository }}
         run: |
           export KUBECONFIG=$HOME/.kube/config
-          scripts/cleanup-cluster.sh ${{ steps.setup_ssh.outputs.ssh_id }}
+          yes | metal ssh-key delete --id "${{ steps.setup_ssh.outputs.ssh_id }}"
+          scripts/cleanup-cluster.sh
         env:
           PROJECT_ID: 383890d0-f5d1-4de1-881a-4d1ede549d18
           METAL_AUTH_TOKEN: ${{ secrets.PACKET_AUTH_TOKEN }}

go.mod

@@ -3,7 +3,7 @@ module github.com/networkservicemesh/integration-k8s-packet
 go 1.18
 
 require (
-	github.com/networkservicemesh/integration-tests v0.0.0-20230110004352-74d7402da8af
+	github.com/networkservicemesh/integration-tests v0.0.0-20230227120053-ecc410bc9425
 	github.com/stretchr/testify v1.7.0
 )
 

go.sum

@@ -158,8 +158,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/networkservicemesh/gotestmd v0.0.0-20220628095933-eabbdc09e0dc h1:1L/OisEFsOyhwaqeJpYmM1nlJ2dBusUMiszPDBlUip0=
 github.com/networkservicemesh/gotestmd v0.0.0-20220628095933-eabbdc09e0dc/go.mod h1:8EWnekTRNX+NxBdTFE24WqUoM7SgJHbiafDBrIIdOmQ=
-github.com/networkservicemesh/integration-tests v0.0.0-20230110004352-74d7402da8af h1:cp8bgP2pZn8NjIPint+EZgmk4881ZiRPwWXvG5CGfrA=
-github.com/networkservicemesh/integration-tests v0.0.0-20230110004352-74d7402da8af/go.mod h1:qkqJgckg5vY5STByxzKhzhDB+TKCAU0qasuwfJ0cG+s=
+github.com/networkservicemesh/integration-tests v0.0.0-20230227120053-ecc410bc9425 h1:IFZBH3DcSVdn8Dg+Isb0JRMhXIixXvyveVLe4oQ+ejw=
+github.com/networkservicemesh/integration-tests v0.0.0-20230227120053-ecc410bc9425/go.mod h1:qkqJgckg5vY5STByxzKhzhDB+TKCAU0qasuwfJ0cG+s=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

scripts/calico/deploy-calico.sh

@@ -6,8 +6,10 @@ function on_error() {
 }
 trap 'on_error' ERR
 
-kubectl apply -f https://projectcalico.docs.tigera.io/archive/v3.23/manifests/tigera-operator.yaml
-kubectl apply -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/v3.23.0/yaml/calico/installation-default.yaml
+kubectl create -f https://projectcalico.docs.tigera.io/archive/v3.24/manifests/tigera-operator.yaml
+sleep 5s
+kubectl wait --for condition=established --timeout=10s crd/installations.operator.tigera.io
+kubectl create -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/82c88a14e5e0e3cc5d7f70c52cdbc01c999d3a42/yaml/calico/installation-default.yaml
 kubectl apply -k scripts/calico
 
 kubectl rollout status -n calico-vpp-dataplane ds/calico-vpp-node --timeout=10m

scripts/calico/kustomization.yaml

@@ -3,7 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-  - https://raw.githubusercontent.com/projectcalico/vpp-dataplane/v3.23.0/yaml/generated/calico-vpp-nohuge.yaml
+  - https://raw.githubusercontent.com/projectcalico/vpp-dataplane/82c88a14e5e0e3cc5d7f70c52cdbc01c999d3a42/yaml/generated/calico-vpp-nohuge.yaml
 
 patchesStrategicMerge:
   - patch.yaml

scripts/calico/patch.yaml

@@ -5,4 +5,34 @@ metadata:
   name: calico-vpp-config
   namespace: calico-vpp-dataplane
 data:
-  vpp_dataplane_interface: ens6f1
+  CALICOVPP_INTERFACES: |-
+    {
+      "maxPodIfSpec": {
+        "rx": 10, "tx": 10, "rxqsz": 1024, "txqsz": 1024
+      },
+      "defaultPodIfSpec": {
+        "rx": 1, "tx":1, "isl3": true
+      },
+      "vppHostTapSpec": {
+        "rx": 1, "tx":1, "rxqsz": 1024, "txqsz": 1024, "isl3": false
+      },
+      "uplinkInterfaces": [
+        {
+          "interfaceName": "ens6f1"
+        }
+      ]
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: calico-vpp-node
+  namespace: calico-vpp-dataplane
+spec:
+  template:
+    spec:
+      containers:
+        - name: agent
+          image: artgl/calico-agent:82c88a14
+        - name: vpp
+          image: artgl/calico-vpp:82c88a14

scripts/cleanup-cluster.sh

@@ -1,10 +1,7 @@
 #!/bin/bash -x
 
-ssh_id=$1
-
 ## Cluster API Packet provides, together with the cluster, always creates an address that must be manually deleted
 
-yes | metal ssh-key delete --id "$ssh_id"
 eip=$(metal ip get -p "${PROJECT_ID}" -o json | jq -r '.[] | select( .tags != null and any(.tags[]; endswith(env.CLUSTER_NAME))) | .id')
 metal ip remove -i "$eip"
 kubectl delete cluster "${CLUSTER_NAME}"

scripts/defaultCNI/kustomization.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - https://raw.githubusercontent.com/projectcalico/calico/v3.25.0/manifests/calico.yaml
+
+patchesStrategicMerge:
+  - patch.yaml

scripts/defaultCNI/patch.yaml

@@ -0,0 +1,16 @@
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: calico-node
+  namespace: kube-system
+  labels:
+    k8s-app: calico-node
+spec:
+  template:
+    spec:
+      containers:
+        - name: calico-node
+          env:
+            - name: IP_AUTODETECTION_METHOD
+              value: "interface=bond0"

scripts/setup-cluster.sh

@@ -25,15 +25,14 @@ source scripts/include/wait-pids.sh
 source scripts/include/wait-start.sh
 
 ## Run clusterctl
-clusterctl init --infrastructure packet || exit 1
+clusterctl init --infrastructure packet:v0.6.2 || exit 1
 clusterctl generate cluster ${CLUSTER_NAME}  \
   --kubernetes-version ${KUBERNETES_VERSION} \
   --control-plane-machine-count=1            \
   --worker-machine-count=1                   \
   > packet.yaml || exit 2
 
 # If CNI is Calico, we need to make a few corrections to the generated template
-sed -r -i 's/(cloud-provider-equinix-metal\/releases\/download\/v[0-9.]*\/)/cloud-provider-equinix-metal\/releases\/download\/v3.5.0\//g' packet.yaml
 if [[ "$CNI" == "calico-vpp" ]]; then # calico
   sed -i "/^    initConfiguration:/a \      localAPIEndpoint:\n        advertiseAddress: ${CALICO_MASTER_IP}" packet.yaml
   sed -i "/^    preKubeadmCommands:/a \    - ifenslave -d bond0 ${CALICO_INTERFACE}\n    - ip addr change ${CALICO_MASTER_IP}/${CALICO_CIDR_PREFIX} dev ${CALICO_INTERFACE}\n    - ip link set up dev ${CALICO_INTERFACE}" packet.yaml
@@ -91,15 +90,15 @@ for i in {1..30}; do
     echo "node count timeout exceeded. exit"
     exit 11
   fi
-  sleep 10s
+  sleep 20s
 done
 
 ## Setup SR-IOV
 /bin/bash scripts/sriov/setup-SRIOV.sh "${master_node}" "${master_ip}" "${worker_node}" "${worker_ip}" "${sriov_vlan}" "${SSH_OPTS}" || exit 12
 
 ## CNI installation
 if [[ "$CNI" == "default" ]]; then # use calico CNI in case of default
-  kubectl --kubeconfig=$KUBECONFIG_PACK apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.24.1/manifests/calico.yaml || exit 13
+  kubectl --kubeconfig=$KUBECONFIG_PACK apply -k scripts/defaultCNI || exit 13
 elif [[ "$CNI" == "calico-vpp" ]]; then # calico-VPP CNI
   export KUBECONFIG=$KUBECONFIG_PACK
   /bin/bash scripts/calico/deploy-calico.sh || exit 14

scripts/sriov/config-SRIOV.sh

@@ -35,8 +35,7 @@ function config_link() {
   retry=0
   while [ ${retry} -lt 3 ]; do
     echo 1 > "${device}/sriov_numvfs" || return 1
-    numfs=$(head -n 1 "${device}/sriov_numvfs")
-    if [ "$numfs" = "1" ]; then
+    if lspci | grep -q "Virtual Function" ; then
       vf_kernel_driver="$(softlink_target "${device}/virtfn0/driver")"
       break
     else