conformance-ipsec-e2e.yaml

name: Conformance IPsec E2E (ci-ipsec-e2e)

# Any change in triggers needs to be reflected in the concurrency group.
on:
  workflow_dispatch:
    inputs:
      PR-number:
        description: "Pull request number."
        required: true
      context-ref:
        description: "Context in which the workflow runs. If PR is from a fork, will be the PR target branch (general case). If PR is NOT from a fork, will be the PR branch itself (this allows committers to test changes to workflows directly from PRs)."
        required: true
      SHA:
        description: "SHA under test (head of the PR branch)."
        required: true
      extra-args:
        description: "[JSON object] Arbitrary arguments passed from the trigger comment via regex capture group. Parse with 'fromJson(inputs.extra-args).argName' in workflow."
        required: false
        default: '{}'
  push:
    branches:
      - 'renovate/main-**'
  # Run every 8 hours
  schedule:
    - cron:  '0 5/8 * * *'

# By specifying the access of one of the scopes, all of those that are not
# specified are set to 'none'.
permissions:
  # To read actions state with catchpoint/workflow-telemetry-action
  actions: read
  # To be able to access the repository with actions/checkout
  contents: read
  # To allow retrieving information from the PR API
  pull-requests: read
  # To be able to set commit status
  statuses: write

concurrency:
  # Structure:
  # - Workflow name
  # - Event type
  # - A unique identifier depending on event type:
  #   - schedule: SHA
  #   - workflow_dispatch: PR number
  #
  # This structure ensures a unique concurrency group name is generated for each
  # type of testing, such that re-runs will cancel the previous run.
  group: |
    ${{ github.workflow }}
    ${{ github.event_name }}
    ${{
      (github.event_name == 'push' && github.sha) ||
      (github.event_name == 'schedule' && github.sha) ||
      (github.event_name == 'workflow_dispatch' && github.event.inputs.PR-number)
    }}
  cancel-in-progress: true

env:
  test_concurrency: 5

jobs:
  echo-inputs:
    if: ${{ github.event_name == 'workflow_dispatch' }}
    name: Echo Workflow Dispatch Inputs
    runs-on: ubuntu-24.04
    steps:
      - name: Echo Workflow Dispatch Inputs
        run: |
          echo '${{ tojson(inputs) }}'

  commit-status-start:
    name: Commit Status Start
    runs-on: ubuntu-24.04
    steps:
      - name: Set initial commit status
        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
        with:
          sha: ${{ inputs.SHA || github.sha }}

  generate-matrix:
    name: Generate Matrix
    runs-on: ubuntu-24.04
    outputs:
      matrix: ${{ steps.generate-matrix.outputs.matrix }}
    steps:
      - name: Checkout context ref (trusted)
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ inputs.context-ref || github.sha }}
          persist-credentials: false

      - name: Convert YAML to JSON
        run: |
          work_dir=".github/actions/ipsec"
          destination_directory="/tmp/generated/ipsec"
          mkdir -p "${destination_directory}"

          yq -o=json "${work_dir}/configs.yaml" | jq . > "${destination_directory}/matrix.json"

      - name: Generate Matrix
        id: generate-matrix
        run: |
          cd /tmp/generated/ipsec
          echo "Generated matrix:"
          cat /tmp/generated/ipsec/matrix.json
          echo "matrix=$(jq -c . < /tmp/generated/ipsec/matrix.json)" >> $GITHUB_OUTPUT

  wait-for-images:
    name: Wait for images
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: Checkout context ref (trusted)
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ inputs.context-ref || github.sha }}
          persist-credentials: false
      - name: Wait for images
        uses: ./.github/actions/wait-for-images
        with:
          SHA: ${{ inputs.SHA }}

  setup-and-test:
    needs: [wait-for-images, generate-matrix]
    name: 'Setup & Test'
    runs-on: ${{ vars.GH_RUNNER_EXTRA_POWER_UBUNTU_LATEST || 'ubuntu-24.04' }}
    env:
      job_name: 'Setup & Test'
    strategy:
      fail-fast: false
      max-parallel: 16
      matrix:
        include: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}

    timeout-minutes: 75
    steps:
      - name: Collect Workflow Telemetry
        uses: catchpoint/workflow-telemetry-action@94c3c3d9567a0205de6da68a76c428ce4e769af1 # v2.0.0
        with:
          comment_on_pr: false

      - name: Checkout context ref (trusted)
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ inputs.context-ref || github.sha }}
          persist-credentials: false

      - name: Cleanup Disk space in runner
        if: runner.name == 'ubuntu-24.04'
        uses: ./.github/actions/disk-cleanup

      - name: Set Environment Variables
        uses: ./.github/actions/set-env-variables

      - name: Set up job variables
        id: vars
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            SHA="${{ inputs.SHA }}"
          else
            SHA="${{ github.sha }}"
          fi

          CONNECTIVITY_TEST_DEFAULTS="--include-unsafe-tests \
                                      --collect-sysdump-on-failure \
                                      --sysdump-hubble-flows-count=1000000 \
                                      --sysdump-hubble-flows-timeout=5m \
                                      --log-code-owners --exclude-code-owners=${CILIUM_CLI_EXCLUDE_OWNERS} \
                                      --flush-ct"

          echo sha=${SHA} >> $GITHUB_OUTPUT
          echo connectivity_test_defaults=${CONNECTIVITY_TEST_DEFAULTS} >> $GITHUB_OUTPUT

      - name: Derive Cilium installation config and junit type
        id: cilium-config
        uses: ./.github/actions/cilium-config
        with:
          image-tag: ${{ steps.vars.outputs.sha }}
          chart-dir: './untrusted/install/kubernetes/cilium'
          tunnel: ${{ matrix.tunnel }}
          devices: ${{ matrix.devices }}
          endpoint-routes: ${{ matrix.endpoint-routes }}
          ipv6: ${{ matrix.ipv6 }}
          kpr: ${{ matrix.kpr }}
          lb-mode: ${{ matrix.lb-mode }}
          lb-acceleration: ${{ matrix.lb-acceleration }}
          encryption: 'ipsec'
          encryption-node: ${{ matrix.encryption-node }}
          encryption-overlay: ${{ matrix.encryption-overlay }}
          egress-gateway: ${{ matrix.egress-gateway }}
          host-fw: ${{ matrix.host-fw }}
          ingress-controller: ${{ matrix.ingress-controller }}
          mutual-auth: false
          misc: ${{ matrix.misc }}

      - name: Set Kind params
        id: kind-params
        shell: bash
        run: |
          IP_FAM="dual"
          if [ "${{ matrix.ipv6 }}" == "false" ]; then
            IP_FAM="ipv4"
          fi
          echo params="--xdp --secondary-network \"\" 3 \"\" \"\" ${{ matrix.kube-proxy }} $IP_FAM" >> $GITHUB_OUTPUT

      - name: Provision K8s on LVH VM
        uses: ./.github/actions/lvh-kind
        with:
          test-name: e2e-conformance
          kernel: ${{ matrix.kernel }}
          kind-params: "${{ steps.kind-params.outputs.params }}"
          kind-image: ${{ env.KIND_K8S_IMAGE }}

      - name: Setup bootid file
        uses: cilium/little-vm-helper@e87948476ca97050b1f149ab2aec379d0de19b84 # v0.0.23
        with:
          provision: 'false'
          cmd: |
            set -ex
            for container in \$(docker ps -q); do
              docker exec \$container mkdir -p /var/run/cilium/
              docker exec \$container sh -c 'cat /proc/sys/kernel/random/uuid > /var/run/cilium/boot_id'
            done

      - name: Start Cilium KVStore
        id: kvstore
        if: matrix.kvstore == 'true'
        run: |
          make kind-kvstore-start KVSTORE_POD_NAME=kvstore KVSTORE_POD_PORT=2378

          IP=$(kubectl --namespace kube-system get pod kvstore -o jsonpath='{.status.hostIP}')
          echo "config= \
            --set=etcd.enabled=true \
            --set=identityAllocationMode=kvstore \
            --set=etcd.endpoints[0]=http://${IP}:2378 \
          " >> $GITHUB_OUTPUT

      - name: Install Cilium CLI
        uses: cilium/cilium-cli@1ea75a067eb7c4ba11a9f082a365f97a14c331db # v0.18.0
        with:
          skip-build: ${{ env.CILIUM_CLI_SKIP_BUILD }}
          image-repo: ${{ env.CILIUM_CLI_IMAGE_REPO }}
          image-tag: ${{ steps.vars.outputs.sha }}

      # Warning: since this is a privileged workflow, subsequent workflow job
      # steps must take care not to execute untrusted code.
      - name: Checkout pull request branch (NOT TRUSTED)
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ steps.vars.outputs.sha }}
          persist-credentials: false
          path: untrusted
          sparse-checkout: |
            install/kubernetes/cilium

      - name: Install Cilium
        shell: bash
        run: |
          kubectl patch node kind-worker3 --type=json -p='[{"op":"add","path":"/metadata/labels/cilium.io~1no-schedule","value":"true"}]'

          if [[ "${{ matrix.key-one }}" == "gcm(aes)" ]]; then
            key="rfc4106(gcm(aes)) $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64) 128"
          elif [[ "${{ matrix.key-one }}" == "cbc(aes)" ]]; then
            key="hmac(sha256) $(dd if=/dev/urandom count=32 bs=1 2> /dev/null| xxd -p -c 64) cbc(aes) $(dd if=/dev/urandom count=32 bs=1 2> /dev/null| xxd -p -c 64)"
          else
            echo "Invalid key type"; exit 1
          fi
          kubectl create -n kube-system secret generic cilium-ipsec-keys \
            --from-literal=keys="3+ ${key}"

          cilium install ${{ steps.cilium-config.outputs.config }} ${{ steps.kvstore.outputs.config }} --set extraConfig.boot-id-file=/var/run/cilium/boot_id

          cilium status --wait --interactive=false
          kubectl get pods --all-namespaces -o wide
          kubectl -n kube-system exec daemonset/cilium -c cilium-agent -- cilium-dbg status

      - name: Prepare the bpftrace parameters
        id: bpftrace-params
        run: |
          CILIUM_INTERNAL_IPS=$(kubectl get ciliumnode -o jsonpath='{.items[*].spec.addresses[?(@.type=="CiliumInternalIP")].ip}')
          if [[ "${{ matrix.ipv6 }}" == "false" ]]; then
            CILIUM_INTERNAL_IPS="${CILIUM_INTERNAL_IPS// / ::1 } ::1"
          fi

          echo "params=$CILIUM_INTERNAL_IPS" >> $GITHUB_OUTPUT

      - name: Start unencrypted packets check
        uses: ./.github/actions/bpftrace/start
        with:
          script: ./.github/actions/bpftrace/scripts/check-encryption-leaks.bt
          args: ${{ steps.bpftrace-params.outputs.params }} "true" "ipsec"

      - name: Run sequential tests (${{ join(matrix.*, ', ') }})
        shell: bash
        run: |
          mkdir -p cilium-junits

          cilium connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} \
            --sysdump-output-filename "cilium-sysdump-${{ matrix.name }}-<ts>" \
            --junit-file "cilium-junits/${{ env.job_name }} (${{ join(matrix.*, ', ') }}).xml" \
            --junit-property github_job_step="Run tests (${{ join(matrix.*, ', ') }})" \
            --test 'seq-.*'

      - name: Run concurrent tests (${{ join(matrix.*, ', ') }})
        shell: bash
        run: |
          mkdir -p cilium-junits

          cilium connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} \
            --sysdump-output-filename "cilium-sysdump-${{ matrix.name }}-<ts>" \
            --junit-file "cilium-junits/${{ env.job_name }} (${{ join(matrix.*, ', ') }}).xml" \
            --junit-property github_job_step="Run tests (${{ join(matrix.*, ', ') }})" \
            --test '!seq-.*' \
            --test-concurrency=${{ env.test_concurrency }}

      - name: Features tested
        uses: ./.github/actions/feature-status
        with:
          title: "Summary of all features tested"
          json-filename: "${{ env.job_name }} (${{ join(matrix.*, ', ') }})"

      - name: Assert that no unencrypted packets are leaked
        uses: ./.github/actions/bpftrace/check

      - name: Start unencrypted packets check for key rotation
        uses: ./.github/actions/bpftrace/start
        with:
          script: ./.github/actions/bpftrace/scripts/check-encryption-leaks.bt
          # As we are not testing with proxy connections during key rotation,
          # disable the check for proxy traffic.
          args: ${{ steps.bpftrace-params.outputs.params }} "false" "ipsec"

      - name: Setup conn-disrupt-test before rotating (${{ join(matrix.*, ', ') }})
        uses: ./.github/actions/conn-disrupt-test-setup

      - name: Rotate IPsec Key (${{ join(matrix.*, ', ') }})
        uses: ./.github/actions/ipsec-key-rotate
        with:
          key-algo: ${{ matrix.key-two }}
          tunnel: ${{ matrix.tunnel }}
          nb-nodes: 2
          ipv6: true

      - name: Assert that no unencrypted packets are leaked during key rotation
        uses: ./.github/actions/bpftrace/check

      - name: Check conn-disrupt-test after rotating (${{ join(matrix.*, ', ') }})
        uses: ./.github/actions/conn-disrupt-test-check

      - name: Start unencrypted packets check for tests
        uses: ./.github/actions/bpftrace/start
        with:
          script: ./.github/actions/bpftrace/scripts/check-encryption-leaks.bt
          args: ${{ steps.bpftrace-params.outputs.params }} "true" "ipsec"

      - name: Run sequential tests (${{ join(matrix.*, ', ') }})
        shell: bash
        run: |
          mkdir -p cilium-junits

          cilium connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} \
            --sysdump-output-filename "cilium-sysdump-${{ matrix.name }}-<ts>" \
            --junit-file "cilium-junits/${{ env.job_name }} (${{ join(matrix.*, ', ') }}).xml" \
            --junit-property github_job_step="Run tests (${{ join(matrix.*, ', ') }})" \
            --test 'seq-.*'

      - name: Run concurrent tests (${{ join(matrix.*, ', ') }})
        shell: bash
        run: |
          mkdir -p cilium-junits

          TEST='--test "!seq-.*"'
          if [ "${{ matrix.key-one }}" = "gcm(aes)" ] && [ "${{ matrix.key-two }}" = "cbc(aes)" ]; then
            # Until https://github.com/cilium/cilium/issues/29480 is resolved
            TEST='--test "!(seq-.*|pod-to-pod-no-frag)"'
          fi

          cilium connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} \
            --sysdump-output-filename "cilium-sysdump-${{ matrix.name }}-<ts>" \
            --junit-file "cilium-junits/${{ env.job_name }} (${{ join(matrix.*, ', ') }}).xml" \
            --junit-property github_job_step="Run tests (${{ join(matrix.*, ', ') }})" \
            $TEST \
            --test-concurrency=${{ env.test_concurrency }}

      - name: Features tested
        uses: ./.github/actions/feature-status
        with:
          title: "Summary of all features tested after all other tests after rotating ipsec keys"
          json-filename: "${{ env.job_name }} (${{ join(matrix.*, ', ') }}) - after rotating keys"

      - name: Assert that no unencrypted packets are leaked during tests
        uses: ./.github/actions/bpftrace/check

      - name: Fetch artifacts
        if: ${{ !success() }}
        shell: bash
        run: |
          kubectl get pods --all-namespaces -o wide
          cilium status
          mkdir -p cilium-sysdumps
          cilium sysdump --output-filename cilium-sysdump-${{ matrix.name }}-final

          if [ "${{ matrix.kvstore }}" == "true" ]; then
            echo
            echo "# Retrieving Cilium etcd logs"
            kubectl -n kube-system logs kvstore
          fi

      - name: Upload artifacts
        if: ${{ !success() }}
        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
        with:
          name: cilium-sysdumps-${{ matrix.name }}
          path: cilium-sysdump-*.zip

      - name: Upload JUnits [junit]
        if: ${{ always() }}
        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
        with:
          name: cilium-junits-${{ matrix.name }}
          path: cilium-junits/*.xml

      - name: Upload features tested
        if: ${{ always() }}
        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
        with:
          name: features-tested-${{ matrix.name }}
          path: ${{ env.job_name }}*.json

      - name: Publish Test Results As GitHub Summary
        if: ${{ always() }}
        uses: aanm/junit2md@332ebf0fddd34e91b03a832cfafaa826306558f9 # v0.0.3
        with:
          junit-directory: "cilium-junits"

  merge-upload:
    if: ${{ always() }}
    name: Merge and Upload Artifacts
    runs-on: ubuntu-24.04
    needs: setup-and-test
    steps:
      - name: Checkout context ref (trusted)
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ inputs.context-ref || github.sha }}
          persist-credentials: false
      - name: Merge Sysdumps
        uses: ./.github/actions/merge-artifacts
        with:
          name: cilium-sysdumps
          pattern: cilium-sysdumps-*
          token: ${{ secrets.GITHUB_TOKEN }}
      - name: Merge JUnits
        uses: ./.github/actions/merge-artifacts
        with:
          name: cilium-junits
          pattern: cilium-junits-*
          token: ${{ secrets.GITHUB_TOKEN }}
      - name: Merge Features tested
        uses: ./.github/actions/merge-artifacts
        with:
          name: features-tested
          pattern: features-tested-*
          token: ${{ secrets.GITHUB_TOKEN }}

  commit-status-final:
    if: ${{ always() }}
    name: Commit Status Final
    needs: setup-and-test
    runs-on: ubuntu-24.04
    steps:
      - name: Set final commit status
        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
        with:
          sha: ${{ inputs.SHA || github.sha }}
          status: ${{ needs.setup-and-test.result }}