Merge remote-tracking branch 'upstream/master' into tck_with_cluster

.clang-tidy

@@ -67,7 +67,7 @@ Checks: '-*,clang-diagnostic-*,clang-analyzer-*,-misc-unused-parameters,
   google-upgrade-googletest-case,
 
   modernize-avoid-bind,
-  modernize-concat-nested-namespaces,
+  -modernize-concat-nested-namespaces,
   modernize-deprecated-headers,
   modernize-deprecated-ios-base-aliases,
   modernize-loop-convert,

.dockerignore

@@ -1,5 +1,4 @@
 *.md
-LICENSES/
 docs/
 
 *.swp

.github/workflows/nightly.yml

@@ -4,14 +4,18 @@ on:
   schedule:
     - cron: '0 18 * * *'
 
+concurrency:
+  group: nightly
+  cancel-in-progress: true
+
 defaults:
   run:
     shell: bash
 
 jobs:
   package:
     name: build package
-    runs-on: self-hosted
+    runs-on: [self-hosted, nebula]
     strategy:
       fail-fast: false
       matrix:
@@ -50,11 +54,11 @@ jobs:
           endpoint: ${{ secrets.OSS_ENDPOINT }}
           bucket: nebula-graph
           asset-path: pkg-build/cpack_output
-          target-path: package/v2-nightly/${{ steps.vars.outputs.subdir }}
+          target-path: package/nightly/${{ steps.vars.outputs.subdir }}
 
   docker:
     name: build docker image
-    runs-on: self-hosted
+    runs-on: [self-hosted, nebula]
     strategy:
       fail-fast: false
       matrix:
@@ -85,7 +89,7 @@ jobs:
 
   coverage:
     name: coverage
-    runs-on: self-hosted
+    runs-on: [self-hosted, nebula]
     strategy:
       fail-fast: false
       matrix:

.github/workflows/pull_request.yml

@@ -7,6 +7,10 @@ on:
       - master
       - 'v[0-9]+.*'
 
+concurrency:
+  group: ${{ github.head_ref }}
+  cancel-in-progress: true
+
 defaults:
   run:
     shell: bash
@@ -19,27 +23,35 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          fetch-depth: 2
+          ref: ${{ github.event.pull_request.base.sha }}
+      - uses: actions/checkout@v2
+        with:
+          clean: false
+      - name: Check License Header
+        uses: apache/skywalking-eyes@main
+      - name: Ensure clang-format-10 is available
+        run: |
+          command -v clang-format-10 > /dev/null || (apt-get update && apt-get install -y clang-format-10)
       - name: Cpplint
         run: |
           ln -snf $PWD/.linters/cpp/hooks/pre-commit.sh $PWD/.linters/cpp/pre-commit.sh
-          .linters/cpp/pre-commit.sh $(git --no-pager diff --diff-filter=d --name-only HEAD^ HEAD)
+          .linters/cpp/pre-commit.sh $(git --no-pager diff --diff-filter=d --name-only ${{ github.event.pull_request.base.sha }} HEAD)
       - name: Format check
         run: |
-          res=$(git diff -U0 --no-color HEAD^ | /usr/share/clang/clang-format-10/clang-format-diff.py -p1)
-          [[ ! -z "$res" ]] && exit 1 || true
+          git diff -U0 --no-color ${{ github.event.pull_request.base.sha }} HEAD | /usr/share/clang/clang-format-10/clang-format-diff.py -p1 | tee /tmp/.clang-format-diff
+          [ -s /tmp/.clang-format-diff ] && exit 1 || true
       - uses: actions/setup-python@v2
         with:
           python-version: 3.7
       - name: Prepare Gherkin exec environ
         run: make init-all -C tests
       - name: Check Gherkin feature format
-        run: make check -C tests
+        run: make check-and-diff -C tests
 
   build:
     name: build
     needs: lint
-    runs-on: self-hosted
+    runs-on: [self-hosted, nebula]
     strategy:
       fail-fast: false
       matrix:

.github/workflows/rc.yml

@@ -0,0 +1,109 @@
+name: rc
+
+on:
+  push:
+    branches:
+      - 'v[0-9]+.*'
+
+concurrency:
+  group: rc
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  package:
+    name: build package
+    runs-on: [self-hosted, nebula]
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu1604
+          - ubuntu1804
+          - ubuntu2004
+          - centos7
+          - centos8
+    container:
+      image: vesoft/nebula-dev:${{ matrix.os }}
+    env:
+      BUILD_DIR: ./pkg-build
+      CPACK_DIR: ./pkg-build/cpack_output
+      SYMS_DIR: ./pkg-build/symbols
+    steps:
+      - uses: webiny/action-post-run@2.0.1
+        with:
+          run: sh -c "find . -mindepth 1 -delete"
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/tagname-action
+        id: tag
+      - name: package
+        run: ./package/package.sh -b ${{ steps.tag.outputs.tag }} -t RelWithDebInfo -r OFF -p ON -s TRUE
+      - name: output some vars
+        run: |
+          tar zcf ${{ env.CPACK_DIR }}/nebula-${{ steps.tag.outputs.tagnum }}.tar.gz --exclude=${{ env.BUILD_DIR }} ./*
+          find ${{ env.CPACK_DIR }} -type f \( -iname \*.deb -o -iname \*.rpm -o -iname \*.tar.gz \) -exec bash -c "sha256sum {} > {}.sha256sum.txt" \;
+      - uses: ./.github/actions/upload-to-oss-action
+        with:
+          key-id: ${{ secrets.OSS_ID }}
+          key-secret: ${{ secrets.OSS_SECRET }}
+          endpoint: ${{ secrets.OSS_ENDPOINT }}
+          bucket: nebula-graph
+          asset-path: ${{ env.CPACK_DIR }}
+          target-path: rc/${{ steps.tag.outputs.tagnum }}
+      - uses: ./.github/actions/upload-to-oss-action
+        with:
+          key-id: ${{ secrets.OSS_ID }}
+          key-secret: ${{ secrets.OSS_SECRET }}
+          endpoint: ${{ secrets.OSS_ENDPOINT }}
+          bucket: nebula-graph
+          asset-path: ${{ env.SYMS_DIR }}
+          target-path: rc/${{ steps.tag.outputs.tagnum }}/symbols
+
+  docker_build:
+    name: docker-build
+    runs-on: [self-hosted, nebula]
+    strategy:
+      fail-fast: false
+      matrix:
+        service:
+          - graphd
+          - metad
+          - storaged
+          - tools
+    steps:
+      - uses: webiny/action-post-run@2.0.1
+        with:
+          run: sh -c "find . -mindepth 1 -delete"
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/tagname-action
+        id: tagname
+      - id: docker
+        run: |
+          majorver=$(git tag -l --sort=v:refname | tail -n1 | cut -f1 -d'.')
+          tag=""
+          if [[ $majorver == ${{ steps.tagname.outputs.majorver }} ]]; then
+            tag="${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:latest"
+          fi
+          echo "::set-output name=tag::$tag"
+      - uses: docker/setup-qemu-action@v1
+      - uses: docker/setup-buildx-action@v1
+      - uses: docker/login-action@v1
+        with:
+          registry: ${{ secrets.HARBOR_REGISTRY }}
+          username: ${{ secrets.HARBOR_USERNAME }}
+          password: ${{ secrets.HARBOR_PASSWORD }}
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./docker/Dockerfile.${{ matrix.service }}
+          platforms: linux/amd64,linux/arm64
+          tags: |
+            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.tag }}
+            ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.majorver }}
+            ${{ steps.docker.outputs.tag }}
+          push: true
+          build-args: |
+            BRANCH=${{ steps.tagname.outputs.tag }}

.github/workflows/release.yml

@@ -5,62 +5,35 @@ on:
     types:
       - published
 
+concurrency:
+  group: release
+  cancel-in-progress: true
+
 defaults:
   run:
     shell: bash
 
 jobs:
   package:
-    name: build package
-    runs-on: self-hosted
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-          - ubuntu1604
-          - ubuntu1804
-          - ubuntu2004
-          - centos7
-          - centos8
+    name: package
+    runs-on: ubuntu-latest
     container:
-      image: vesoft/nebula-dev:${{ matrix.os }}
-    env:
-      BUILD_DIR: ./pkg-build
-      CPACK_DIR: ./pkg-build/cpack_output
-      SYMS_DIR: ./pkg-build/symbols
+      image: vesoft/nebula-dev:centos7
     steps:
-      - uses: webiny/action-post-run@2.0.1
-        with:
-          run: sh -c "find . -mindepth 1 -delete"
       - uses: actions/checkout@v2
       - uses: ./.github/actions/tagname-action
         id: tag
-      - name: package
-        run: ./package/package.sh -b ${{ steps.tag.outputs.tag }} -t RelWithDebInfo -r OFF -p ON -s TRUE
-      - name: output some vars
-        run: |
-          tar zcf ${{ env.CPACK_DIR }}/nebula-${{ steps.tag.outputs.tagnum }}.tar.gz --exclude=${{ env.BUILD_DIR }} ./*
-          find ${{ env.CPACK_DIR }} -type f \( -iname \*.deb -o -iname \*.rpm -o -iname \*.tar.gz \) -exec bash -c "sha256sum {} > {}.sha256sum.txt" \;
-      - uses: ./.github/actions/upload-to-oss-action
-        with:
-          key-id: ${{ secrets.OSS_ID }}
-          key-secret: ${{ secrets.OSS_SECRET }}
-          endpoint: ${{ secrets.OSS_ENDPOINT }}
-          bucket: nebula-graph
-          asset-path: ${{ env.CPACK_DIR }}
-          target-path: package/${{ steps.tag.outputs.tagnum }}
-      - uses: ./.github/actions/upload-to-oss-action
-        with:
-          key-id: ${{ secrets.OSS_ID }}
-          key-secret: ${{ secrets.OSS_SECRET }}
-          endpoint: ${{ secrets.OSS_ENDPOINT }}
-          bucket: nebula-graph
-          asset-path: ${{ env.SYMS_DIR }}
-          target-path: package/${{ steps.tag.outputs.tagnum }}/symbols
+      - run: |
+          ossutil64 cp -rf \
+            -i ${{ secrets.OSS_ID }} \
+            -k ${{ secrets.OSS_SECRET }} \
+            -e ${{ secrets.OSS_ENDPOINT }} \
+            oss://nebula-graph/rc/${{ steps.tag.outputs.tagnum }} \
+            oss://nebula-graph/package/${{ steps.tag.outputs.tagnum }}
 
   docker_build:
     name: docker-build
-    runs-on: self-hosted
+    runs-on: [self-hosted, nebula]
     strategy:
       fail-fast: false
       matrix:
@@ -81,24 +54,17 @@ jobs:
           majorver=$(git tag -l --sort=v:refname | tail -n1 | cut -f1 -d'.')
           tag=""
           if [[ $majorver == ${{ steps.tagname.outputs.majorver }} ]]; then
-            tag="vesoft/nebula-${{ matrix.service }}:latest"
+            tag="latest"
           fi
           echo "::set-output name=tag::$tag"
-      - uses: docker/setup-qemu-action@v1
-      - uses: docker/setup-buildx-action@v1
-      - uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./docker/Dockerfile.${{ matrix.service }}
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.tag }}
-            vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.majorver }}
-            ${{ steps.docker.outputs.tag }}
-          push: true
-          build-args: |
-            BRANCH=${{ steps.tagname.outputs.tag }}
+      - name: Sync docker images
+        env:
+          FROM_IMAGE: docker://${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}
+          TO_IMAGE: docker://docker.io/vesoft/nebula-${{ matrix.service }}
+          CMD: docker run --rm -ti quay.io/containers/skopeo:v1.4.1 copy -a --src-creds ${{ secrets.HARBOR_USERNAME }}:${{ secrets.HARBOR_PASSWORD }} --dest-creds ${{ secrets.DOCKER_USERNAME }}:${{ secrets.DOCKER_PASSWORD }}
+        run: |
+          ${{ env.CMD }} ${{ env.FROM_IMAGE }}:${{ steps.tagname.outputs.tag }} ${{ env.TO_IMAGE }}:${{ steps.tagname.outputs.tag }}
+          ${{ env.CMD }} ${{ env.FROM_IMAGE }}:${{ steps.tagname.outputs.tag }} ${{ env.TO_IMAGE }}:${{ steps.tagname.outputs.majorver }}
+          if [[ ! -z "${{ steps.docker.outputs.tag }}" ]]; then
+            ${{ env.CMD }} ${{ env.FROM_IMAGE }}:${{ steps.tagname.outputs.tag }} ${{ env.TO_IMAGE }}:${{ steps.docker.outputs.tag }}
+          fi