Fuzzy filter #26
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Define workflow name. | |
| name: MatrixOne Utils CI | |
| on: | |
| pull_request_target: | |
| types: [ opened, synchronize, reopened ] | |
| branches: [ main,'[0-9]+.[0-9]+*' ] | |
| concurrency: | |
| group: ${{ github.event.pull_request.head.repo.full_name}}/${{ github.event.pull_request.head.ref }}/${{ github.workflow }} | |
| cancel-in-progress: true | |
| jobs: | |
| check_organization_user: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| in_org: ${{ steps.check_in_org.outputs.in_org }} | |
| safe_label: ${{ steps.check_safe_label.outputs.safe_label }} | |
| steps: | |
| - id: check_in_org | |
| name: CHECK ORGANIZATION USER | |
| run: | | |
| PAGE=1; | |
| PER_PAGE=100; | |
| USER_IN_ORG="0"; | |
| while true; do | |
| users=$(curl -L \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.TOKEN_ACTION }}" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| "https://api.github.com/orgs/matrixorigin/members?page=$PAGE&per_page=$PER_PAGE"); | |
| if [ $(echo $users | jq ".[].login" | grep -c "${{ github.event.pull_request.user.login }}" ) -eq 1 ]; then | |
| USER_IN_ORG="1"; | |
| break; | |
| fi | |
| if [ $(echo $users | jq ".[].login" | wc -l ) -eq $PER_PAGE ]; then | |
| PAGE=$(($PAGE+1)); | |
| users=""; | |
| else | |
| break; | |
| fi | |
| done | |
| echo "in_org=$USER_IN_ORG" >> $GITHUB_OUTPUT; | |
| - id: check_safe_label | |
| name: CHECK PULL REQUEST LABEL | |
| run: | | |
| labels=$(curl -L \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.TOKEN_ACTION }}" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| '${{ github.event.pull_request.url}}' | jq ".labels"); | |
| if [ $(echo $labels | jq ".[].name" | grep -c "safe-to-test" ) -ge 1 ]; then | |
| echo "safe_label=1" >> $GITHUB_OUTPUT; | |
| else | |
| echo "safe_label=0" >> $GITHUB_OUTPUT; | |
| fi | |
| ut_coverage: | |
| if: ${{ needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1' }} | |
| name: PR Unit Test Coverage | |
| needs: [check_organization_user] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: '3' | |
| path: ./matrixone | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| - name: Set up Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: '1.20' | |
| cache: true | |
| - name: Set Variables | |
| run: | | |
| cd $GITHUB_WORKSPACE/matrixone | |
| echo "test_scope=$(go list ./... | grep -v 'driver\|engine/aoe\|engine/memEngine\|pkg/catalog' | xargs)" >> $GITHUB_ENV | |
| echo "raw_ut_coverage='ut_coverage.raw'" >> $GITHUB_ENV | |
| echo "raw_bvt_coverage='bvt_coverage.raw'" >> $GITHUB_ENV | |
| echo "raw_bvt_coverage_temp='bvt_coverage_temp.raw'" >> $GITHUB_ENV | |
| echo "raw_ut_coverage_temp='ut_coverage_temp.raw'" >> $GITHUB_ENV | |
| echo "all_coverage='all_coverage.raw'" >> $GITHUB_ENV | |
| echo "test_report='test_report.out'" >> $GITHUB_ENV | |
| echo "ut_html_coverage='ut_coverage.html'" >> $GITHUB_ENV | |
| echo "bvt_html_coverage='bvt_coverage.html'" >> $GITHUB_ENV | |
| echo "all_html_coverage='all_coverage.html'" >> $GITHUB_ENV | |
| echo "ut_excluded_pkg='pkg/pb\|pkg/sql/parsers/goyacc\|yaccpar'" >> $GITHUB_ENV | |
| echo "bvt_excluded_pkg='pkg/pb\|yaccpar'" >> $GITHUB_ENV | |
| echo "coverage_line=75" >> $GITHUB_ENV | |
| - name: Set env | |
| run: | | |
| echo "endpoint=${{ secrets.S3ENDPOINT }}" >> $GITHUB_ENV | |
| echo "region=${{ secrets.S3REGION }}" >> $GITHUB_ENV | |
| echo "apikey=${{ secrets.S3APIKEY }}" >> $GITHUB_ENV | |
| echo "apisecret=${{ secrets.S3APISECRET }}" >> $GITHUB_ENV | |
| echo "bucket=${{ secrets.S3BUCKET }}" >> $GITHUB_ENV | |
| - name: Get Changed Paths | |
| uses: guguducken/change-paths-action@v0.0.7 | |
| id: ut-pr | |
| with: | |
| github-token: ${{ secrets.TOKEN_ACTION }} | |
| ignore: "/,etc/,optools/,LICENSES/,cgo/,cmd/,docs/,proto/,test/,.github/,pkg/pb/" | |
| source-path: ${{ env.test_scope }} | |
| - name: Start PR Unit Test | |
| if: ${{ steps.ut-pr.outputs.paths != '' }} | |
| id: test_ut | |
| timeout-minutes: 60 | |
| run: | | |
| cd $GITHUB_WORKSPACE/matrixone | |
| echo "-----------------------ut start-----------------------" | |
| CGO_CFLAGS="-I${GITHUB_WORKSPACE}/matrixone/cgo" | |
| CGO_LDFLAGS="-L${GITHUB_WORKSPACE}/matrixone/cgo -lmo -lm" | |
| make clean && make config && make cgo | |
| go test -v -cover -short -tags matrixone_test -p 6 -covermode=set -coverprofile ${{ env.raw_ut_coverage }} -count=1 ${{ steps.ut-pr.outputs.paths }} | |
| echo "-----------------------ut finish-----------------------" | |
| - name: Clone mo-tester repository | |
| if: ${{ steps.ut-pr.outputs.paths != '' }} | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: matrixorigin/mo-tester | |
| path: ./mo-tester | |
| ref: main | |
| - name: Set up JDK 8 for x64 | |
| if: ${{ steps.ut-pr.outputs.paths != '' }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| architecture: x64 | |
| - name: Start BVT Test Service | |
| if: ${{ steps.ut-pr.outputs.paths != '' }} | |
| id: test_bvt_start | |
| timeout-minutes: 10 | |
| run: | | |
| cd $GITHUB_WORKSPACE/matrixone | |
| if [ -d "./store" ]; then rm -rf ./store; fi | |
| GOLDFLAGS=-ldflags="-X 'main.GoVersion=$(go version)' -X 'main.BranchName=$(git rev-parse --abbrev-ref HEAD)' -X 'main.CommitID=$(git rev-parse HEAD)' -X 'main.BuildTime=$(date)' -X 'main.Version=$(git describe --always --tags $(git rev-list --tags --max-count=1))'" | |
| export GOCOVERDIR=$GITHUB_WORKSPACE/matrixone | |
| go build -cover "$GOLDFLAGS" -o mo-service ./cmd/mo-service/ | |
| ./optools/run_bvt.sh $GITHUB_WORKSPACE/matrixone launch | |
| - name: Start PR BVT Test | |
| if: ${{ steps.ut-pr.outputs.paths != '' }} | |
| id: test_bvt | |
| continue-on-error: true | |
| run: | | |
| cd $GITHUB_WORKSPACE/mo-tester | |
| sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml | |
| cat mo.yml | |
| echo "=============================" | |
| ./run.sh -n -g -o -p $GITHUB_WORKSPACE/matrixone/test/distributed/cases -e optimistic 2>&1 | |
| - name: Stop MO Service | |
| id: test_bvt_stop | |
| if: ${{ steps.ut-pr.outputs.paths != '' }} | |
| run: | | |
| ps -aux | grep 'mo-service' | grep -v 'grep' | awk '{print $2}' | xargs kill -s SIGINT | |
| i=1 && while [ -n "$(ps -aux|grep 'mo-service' | grep -v 'grep' | awk '{print $2}')" ] && [ $i -le 300 ]; do echo "mo-service kill not finished......$i/300"; i=$(($i+1)); sleep 1s; done | |
| if [ $i -gt 300 ]; then echo 'stop mo-service failed...'; exit 1; else echo 'stop mo-service successed...'; fi | |
| - name: Coverage Calculate | |
| if: ${{ steps.ut-pr.outputs.paths != '' }} | |
| id: calculate_result | |
| run: | | |
| cd $GITHUB_WORKSPACE/matrixone | |
| go tool covdata textfmt -i="$GITHUB_WORKSPACE/matrixone" -o ${{ env.raw_bvt_coverage }} | |
| cat ${{ env.raw_bvt_coverage }} | grep "${{ steps.ut-pr.outputs.files }}" | grep -v ${{ env.bvt_excluded_pkg }} | tee ${{ env.raw_bvt_coverage_temp }} > /dev/null | |
| cat ${{ env.raw_ut_coverage }} | grep "${{ steps.ut-pr.outputs.files }}" | grep -v ${{ env.ut_excluded_pkg }} | tee ${{ env.raw_ut_coverage_temp }} > /dev/null | |
| echo "======================= Coverage Calculate Start =======================" | |
| awk -F '[ ]+' 'FNR==NR{arr[$1,$2]=$3;next}{if($3>=1){arr[$1,$2]=1}else{arr[$1,$2]+=$3}}END{for(i in arr){split(i,ind,SUBSEP);printf("%s %s %s\n",ind[1],ind[2],arr[i])};}' ${{ env.raw_ut_coverage_temp }} ${{ env.raw_bvt_coverage_temp }} | tee ${{ env.all_coverage }} > /dev/null | |
| awk -F '[:, ]+' '{all[$1]+=$4;hit[$1]+=0;if($5>0) hit[$1]+=$4;}END{for(i in all)if(all[i]!=0){print(i,all[i],hit[i],hit[i]/all[i]*100)}else{print(i,all[i],hit[i],100)}}' ${{ env.all_coverage }} | tee ${{ env.test_report }} > /dev/null | |
| awk 'BEGIN{flag=1}{if($4<${{ env.coverage_line }}){flag=0;print("\033[31merror:\033[0m the code coverage of \033[31m"$1"\033[0m is \033[31m"$4"%\033[0m which is below ${{ env.coverage_line }}%");} else { print("\033[32minfo:\033[0m the code coverage of "$1" is "$4"% which is up to ${{ env.coverage_line }}%") }}END{if(flag==0) {exit 1;}}' ${{ env.test_report }} | |
| - name: Package Test Result | |
| if: ${{ steps.ut-pr.outputs.paths != '' && failure() && steps.test_ut.conclusion != 'failure' && steps.test_bvt_start.conclusion != 'failure' && steps.test_bvt_stop.conclusion != 'failure' }} | |
| continue-on-error: true | |
| run: | | |
| cd $GITHUB_WORKSPACE/matrixone | |
| echo "======================= Coverage HTML Generate Start =======================" | |
| sed -i '1s/^/mode: set/' ${{ env.raw_ut_coverage_temp }} | |
| go tool cover -o ${{ env.ut_html_coverage }} -html=${{ env.raw_ut_coverage_temp }} | |
| echo "HTML for UT test generated is finished" | |
| sed -i '1s/^/mode: set/' ${{ env.raw_bvt_coverage_temp }} | |
| go tool cover -o ${{ env.bvt_html_coverage }} -html=${{ env.raw_bvt_coverage_temp }} | |
| echo "HTML for BVT test generated is finished" | |
| sed -i '1s/^/mode: set/' ${{ env.all_coverage }} | |
| go tool cover -o ${{ env.all_html_coverage }} -html=${{ env.all_coverage }} | |
| echo "HTML for ALL test generated is finished" | |
| echo "======================= Coverage HTML Generate Finished =======================" | |
| zip -r code_coverage_reports_${{ github.event.pull_request.number}}_${{ github.run_id }}.zip ${{ env.all_html_coverage }} ${{ env.raw_ut_coverage }} ${{ env.raw_ut_coverage_temp }} ${{ env.test_report }} ${{ env.raw_bvt_coverage_temp }} ${{ env.raw_bvt_coverage }} ${{ env.all_coverage }} ${{ env.ut_html_coverage }} ${{ env.bvt_html_coverage }} 'mo-service.log' | |
| rm -rf ${{ env.all_html_coverage }} ${{ env.raw_ut_coverage }} ${{ env.raw_ut_coverage_temp }} ${{ env.test_report }} ${{ env.raw_bvt_coverage_temp }} ${{ env.raw_bvt_coverage }} ${{ env.all_coverage }} ${{ env.ut_html_coverage }} ${{ env.bvt_html_coverage }} 'mo-service.log' | |
| - uses: actions/upload-artifact@v3 | |
| if: ${{ steps.ut-pr.outputs.paths != '' && failure() && steps.test_ut.conclusion != 'failure' && steps.test_bvt_start.conclusion != 'failure' && steps.test_bvt_stop.conclusion != 'failure' }} | |
| continue-on-error: true | |
| with: | |
| name: code-coverage-reports-${{ github.event.pull_request.number}}-${{ github.run_id }}.zip | |
| path: | | |
| ${{ github.workspace }}/matrixone/code_coverage_reports_${{ github.event.pull_request.number}}_${{ github.run_id }}.zip | |
| retention-days: 7 | |
| pr-size-label: | |
| runs-on: ubuntu-latest | |
| needs: [check_organization_user] | |
| steps: | |
| - name: size-label | |
| uses: "guguducken/label-size-action@v0.0.2" | |
| with: | |
| size_token: ${{ secrets.TOKEN_ACTION }} | |
| ignore: ".md,.pb.go" | |
| sizes: > | |
| { | |
| "XS":0, | |
| "S":10, | |
| "M":100, | |
| "L":500, | |
| "XL":1000, | |
| "XXL":2000 | |
| } | |
| docu_issue_check: | |
| runs-on: ubuntu-latest | |
| needs: [check_organization_user] | |
| if: ${{ github.event.action == 'opened' }} | |
| steps: | |
| - uses: guguducken/pull-issue-reviewer@v0.3.0 | |
| with: | |
| token_action: ${{ secrets.TOKEN_ACTION }} | |
| uri_notice: ${{ secrets.DOCU_GROUP_HOOK }} | |
| name_label: "doc influence" | |
| reviewers: "lacrimosaprinz,dengn" | |
| mentions: "Prinz,DengNan" | |
| chatgpt-review: | |
| name: ChatGPT Review | |
| needs: [check_organization_user] | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event.action == 'opened' }} | |
| steps: | |
| - uses: feiskyer/ChatGPT-Reviewer@v0 | |
| name: ChatGPT Review | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.TOKEN_ACTION }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| with: | |
| model: "gpt-3.5-turbo" | |
| temperature: 0.2 | |
| review_per_file: false | |
| comment_per_file: false |