feat: start k3d in ci container and execute e2e test to k8s cluster

Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com>
vdaas · Jul 31, 2024 · fe08e65 · fe08e65
1 parent cd2bbab
commit fe08e65
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 21 deletions.
diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml
@@ -76,6 +76,7 @@ runs:
       shell: bash
       id: k3d_options
       run: |
+        OPTIONS="${OPTIONS} --api-port host.docker.internal:6550"
         if [ "${AGENTS}" != 0 ]; then
           OPTIONS="${OPTIONS} --agents ${AGENTS}"
         fi

diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml
@@ -22,24 +22,29 @@ on:
       - "*.*.*-*"
       - "v*.*.*-*"
   pull_request:
-    types:
-      - "labeled"
+    # types:
+    #   - "labeled"
 env:
   VALUES: .github/helm/values/values-chaos.yaml
   DATASET: fashion-mnist-784-euclidean.hdf5
   INGRESS_PORT: 8081
 jobs:
   dump-contexts-to-log:
-    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
+  detect-ci-container:
+    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
+    uses: ./.github/workflows/_detect-ci-container.yml
   agent-failure:
     name: "E2E chaos test (Agent failure: to test insert/search works even if one of the agents is failing)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -75,9 +80,12 @@ jobs:
             e2e/insert/search
   random-pod-failure:
     name: "E2E chaos test (random Pod failure: to test redundancy)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -113,9 +121,12 @@ jobs:
             e2e/insert/search
   agent-network-partition:
     name: "E2E chaos test (agent network partition: to test retries)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -151,9 +162,12 @@ jobs:
             e2e/insert/search
   clusterwide-network-bandwidth:
     name: "E2E chaos test (network bandwidth: to test it works properly under bandwidth limitation)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config

diff --git a/.github/workflows/e2e-code-bench-agent.yaml b/.github/workflows/e2e-code-bench-agent.yaml
@@ -58,6 +58,7 @@ jobs:
     needs: [detect-ci-container]
     container:
       image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -85,6 +86,7 @@ jobs:
     needs: [detect-ci-container]
     container:
       image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config

diff --git a/.github/workflows/e2e-max-dim.yml b/.github/workflows/e2e-max-dim.yml
@@ -27,16 +27,21 @@ on:
       - "labeled"
 jobs:
   dump-contexts-to-log:
-    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-max-dim'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
+  detect-ci-container:
+    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
+    uses: ./.github/workflows/_detect-ci-container.yml
   e2e-max-dimension-insert:
     name: "E2E test (Max Dimension Insert: skip strict exist check)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config

diff --git a/.github/workflows/e2e-profiling.yml b/.github/workflows/e2e-profiling.yml
@@ -28,16 +28,21 @@ env:
   DATASET: fashion-mnist-784-euclidean.hdf5
 jobs:
   dump-contexts-to-log:
-    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
+  detect-ci-container:
+    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
+    uses: ./.github/workflows/_detect-ci-container.yml
   e2e-profiling:
     name: "E2E profiling"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -22,8 +22,8 @@ on:
       - "*.*.*-*"
       - "v*.*.*-*"
   pull_request:
-    types:
-      - "labeled"
+    # types:
+    #   - "labeled"
 env:
   DATASET: fashion-mnist-784-euclidean.hdf5
 jobs:
@@ -33,11 +33,17 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
+  detect-ci-container:
+    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
+    uses: ./.github/workflows/_detect-ci-container.yml
   e2e-stream-crud:
     name: "E2E test (Stream CRUD)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -73,9 +79,12 @@ jobs:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
   e2e-stream-crud-for-operator:
     name: "E2E test (Stream CRUD) for operator"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -133,9 +142,12 @@ jobs:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
   e2e-stream-crud-under-index-management-jobs:
     name: "E2E test (Stream CRUD) under index management jobs"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -175,9 +187,12 @@ jobs:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
   e2e-stream-crud-skip-exist-check:
     name: "E2E test (Stream CRUD: skip strict exist check)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -213,9 +228,12 @@ jobs:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
   e2e-multiapis-crud:
     name: "E2E test (Multi-APIs CRUD)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -247,9 +265,12 @@ jobs:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
   e2e-jobs:
     name: "E2E test (Jobs)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -278,9 +299,12 @@ jobs:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
   e2e-stream-crud-with-readreplica:
     name: "E2E test (Stream CRUD) with read replica"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -326,9 +350,12 @@ jobs:
           POD_NAME: ${{ steps.deploy_vald_readreplica.outputs.POD_NAME }}
   e2e-stream-crud-with-mirror:
     name: "E2E test (Stream CRUD) with mirror"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--privileged --add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config