Merge branch 'master' into remove_force_consume

pingcap · Jun 9, 2022 · e4ba9d5 · e4ba9d5
2 parents 912ac1c + d75bda0
commit e4ba9d5
Show file tree

Hide file tree

Showing 195 changed files with 13,247 additions and 9,164 deletions.
diff --git a/.github/workflows/dataflow_engine_chaos.yaml b/.github/workflows/dataflow_engine_chaos.yaml
@@ -39,31 +39,8 @@ jobs:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-dataflow-${{ hashFiles('go.sum') }}
 
-      # Set up Kubernetes with K3s
-      - name: Set up K3s cluster
-        run: |
-          curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.18.9+k3s1 sh -s - \
-            --write-kubeconfig-mode=644 \
-            "${k3s_disable_command:---disable}" metrics-server \
-            "${k3s_disable_command:---disable}" traefik \
-            --flannel-backend=none \
-            --docker
-        shell: bash
-
-      # this may be failed sometimes, and I want to exit the workflow directly if failed,
-      # but GitHub Actions doesnt' support early-exit yet, see https://github.com/actions/runner/issues/662.
-      # so, simply wait for a long time.
-      - name: Wait for coredns
-        run: |
-          kubectl rollout status --watch --timeout 600s deployment/coredns -n kube-system
-        shell: bash
-        env:
-          KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-
-      - name: Export KUBECONFIG environment variable
-        run: |
-          echo 'KUBECONFIG=/etc/rancher/k3s/k3s.yaml' >> $GITHUB_ENV
-        shell: bash
+      - name: Create k8s Kind Cluster
+        uses: helm/kind-action@v1.2.0
 
       - name: Print cluster information
         run: |
@@ -73,6 +50,7 @@ jobs:
           kubectl get pods -n kube-system
           kubectl get sc
           kubectl version
+          helm version
 
       - name: Build dataflow engine binary
         run: make df-master df-executor df-chaos-case
@@ -83,6 +61,10 @@ jobs:
           docker build -f $GITHUB_WORKSPACE/engine/chaos/manifests/Dockerfile -t dataflow:chaos $GITHUB_WORKSPACE/bin
           docker image list
 
+      - name: Load docker image to kind cluster
+        run: |
+          kind load docker-image dataflow:chaos --name chart-testing
+
       # Set up metastore and basic services
       - name: Set up metastore and basic services
         run: |
@@ -208,8 +190,8 @@ jobs:
         if: ${{ always() }}
         run: |
           mkdir ./logs
-          sudo cp -r -L /var/log/containers/. ./logs
-          sudo find /var/ -type f | grep -E '.*/(server-master|executor)-[^/]*.log$' | sudo xargs -i cp {} ./logs || true
+          kubectl get pods --no-headers -o custom-columns=":metadata.name"|grep -E "server-master|executor"|xargs -I{} sudo kubectl cp {}:/log/{}.log ./logs/{}.log || true
+          kind export logs ./logs/kind --name chart-testing
           sudo chown -R runner ./logs
 
       # Upload logs as artifact seems not stable, so we set `continue-on-error: true` here.
@@ -221,8 +203,6 @@ jobs:
           name: chaos-base-logs.${{ matrix.chaos-obj }}
           path: |
             ./logs
-            !./logs/coredns-*
-            !./logs/local-path-provisioner-*
 
       # Send feishu notification if failed.
       - name: Feishu notification

diff --git a/.github/workflows/dm_chaos.yaml b/.github/workflows/dm_chaos.yaml
@@ -67,34 +67,9 @@ jobs:
           path: tools/bin
           key: ${{ runner.os }}-ticdc-tools-${{ hashFiles('tools/check/go.sum') }}
 
-      # Set up Kubernetes IN Docker
-      # - name: Set up kind cluster
-      #   uses: helm/kind-action@v1.0.0
-      #   with:
-      #     cluster_name: dm-chaos
-      # Set up Kubernetes with K3s
-      - name: Set up K3s cluster
-        run: |
-          curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.18.9+k3s1 sh -s - \
-            --write-kubeconfig-mode=644 \
-            "${k3s_disable_command:---disable}" metrics-server \
-            "${k3s_disable_command:---disable}" traefik \
-            --flannel-backend=none \
-            --docker
-        shell: bash
-      # this may be failed sometimes, and I want to exit the workflow directly if failed,
-      # but GitHub Actions doesnt' support early-exit yet, see https://github.com/actions/runner/issues/662.
-      # so, simply wait for a long time.
-      - name: Wait for coredns
-        run: |
-          kubectl rollout status --watch --timeout 600s deployment/coredns -n kube-system
-        shell: bash
-        env:
-          KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-      - name: Export KUBECONFIG environment variable
-        run: |
-          echo 'KUBECONFIG=/etc/rancher/k3s/k3s.yaml' >> $GITHUB_ENV
-        shell: bash
+      - name: Create k8s Kind Cluster
+        uses: helm/kind-action@v1.2.0
+
       - name: Print cluster information
         run: |
           kubectl config view
@@ -103,6 +78,7 @@ jobs:
           kubectl get pods -n kube-system
           kubectl get sc
           kubectl version
+          helm version
 
       # Disable AppArmor for MySQL, see https://github.com/moby/moby/issues/7512#issuecomment-61787845
       - name: Disable AppArmor for MySQL
@@ -120,10 +96,11 @@ jobs:
           cp -r $GITHUB_WORKSPACE/dm/chaos/cases/conf/ $GITHUB_WORKSPACE/bin/
           docker build -f $GITHUB_WORKSPACE/dm/chaos/manifests/Dockerfile -t dm:chaos $GITHUB_WORKSPACE/bin
           docker image list
+
       # Load DM docker image into KIND, see https://kind.sigs.k8s.io/docs/user/quick-start/#loading-an-image-into-your-cluster
-      # - name: Load DM docker image into KIND
-      #   run: |
-      #     kind load docker-image dm:chaos --name dm-chaos
+      - name: Load DM docker image into KIND
+        run: |
+          kind load docker-image dm:chaos --name chart-testing
 
       # Set up upstream instances
       - name: Set up sources
@@ -281,8 +258,8 @@ jobs:
         if: ${{ always() }}
         run: |
           mkdir ./logs
-          sudo cp -r -L /var/log/containers/. ./logs
-          sudo find /var/ -type f -regex '.*/dm-[^/]*.log$' | sudo xargs -i cp {} ./logs || true
+          kubectl get pods --no-headers -o custom-columns=":metadata.name"|grep -E "dm-"|xargs -I{} sudo kubectl cp {}:/log/{}.log ./logs/{}.log || true
+          kind export logs ./logs/kind --name chart-testing
           sudo chown -R runner ./logs
       # Update logs as artifact seems not stable, so we set `continue-on-error: true` here.
       - name: Upload logs
@@ -293,8 +270,6 @@ jobs:
           name: chaos-base-logs.${{ matrix.chaos-obj }}
           path: |
             ./logs
-            !./logs/coredns-*
-            !./logs/local-path-provisioner-*
 
       # send Slack notify if failed.
       # NOTE: With the exception of `GITHUB_TOKEN`, secrets are not passed to the runner when a workflow is triggered from a forked repository.

diff --git a/.github/workflows/ticdc_integration.yaml b/.github/workflows/ticdc_integration.yaml
@@ -145,3 +145,127 @@ jobs:
         if: ${{ always() }}
         run: |
           $GITHUB_WORKSPACE/scripts/avro-local-test.sh down
+
+  Canal-integration:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
+          docker volume prune -f
+          docker image prune -f
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+
+      - name: Cache Vendor
+        id: cache-vendor
+        uses: actions/cache@v2
+        with:
+          path: vendor
+          key: ${{ runner.os }}-cdc-integration-vendor-${{ hashFiles('go.sum') }}
+
+      - name: Update Vendor
+        run: go mod vendor
+
+      - name: Pull images
+        run: docker-compose -f ./deployments/ticdc/docker-compose/docker-compose-canal.yml pull --ignore-pull-failures
+
+      - name: TiDB version
+        run: |
+          docker run pingcap/tidb:nightly -V
+          docker run pingcap/tikv:nightly -V
+          docker run pingcap/pd:nightly -V
+
+      - name: Build Integration Framework
+        run: |
+          cd $GITHUB_WORKSPACE/tests/mq_protocol_tests
+          go build
+
+      - name: Run Integration Framework
+        timeout-minutes: 45
+        run: |
+          cd $GITHUB_WORKSPACE/tests/mq_protocol_tests
+          ./mq_protocol_tests -protocol=canal
+
+      - name: Try to dump container logs
+        if: ${{ failure() }}
+        run: docker-compose -f ./deployments/ticdc/docker-compose/docker-compose-canal.yml logs -t
+
+      - uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: logs
+          path: ${{ github.workspace }}/deployments/ticdc/docker-compose/logs/*
+
+      - name: Clean Up
+        if: ${{ always() }}
+        run: |
+          $GITHUB_WORKSPACE/scripts/canal/canal-local-test.sh down
+
+  Avro-integration:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
+          docker volume prune -f
+          docker image prune -f
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+
+      - name: Cache Vendor
+        id: cache-vendor
+        uses: actions/cache@v2
+        with:
+          path: vendor
+          key: ${{ runner.os }}-cdc-integration-vendor-${{ hashFiles('go.sum') }}
+
+      - name: Update Vendor
+        run: go mod vendor
+
+      - name: Pull images
+        run: docker-compose -f ./deployments/ticdc/docker-compose/docker-compose-avro.yml pull --ignore-pull-failures
+
+      - name: TiDB version
+        run: |
+          docker run pingcap/tidb:nightly -V
+          docker run pingcap/tikv:nightly -V
+          docker run pingcap/pd:nightly -V
+
+      - name: Build Integration Framework
+        run: |
+          cd $GITHUB_WORKSPACE/tests/mq_protocol_tests
+          go build
+
+      - name: Run Integration Framework
+        timeout-minutes: 45
+        run: |
+          cd $GITHUB_WORKSPACE/tests/mq_protocol_tests
+          ./mq_protocol_tests
+
+      - name: Try to dump container logs
+        if: ${{ failure() }}
+        run: docker-compose -f ./deployments/ticdc/docker-compose/docker-compose-avro.yml logs -t
+
+      - uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: logs
+          path: ${{ github.workspace }}/deployments/ticdc/docker-compose/logs/*
+
+      - name: Clean Up
+        if: ${{ always() }}
+        run: |
+          $GITHUB_WORKSPACE/scripts/avro-local-test.sh down