refactor： refractor ci and test/build

.codecov.yml

@@ -26,7 +26,3 @@ codecov:
     # yes: will delay sending notifications until all ci is finished
     # no: will send notifications without checking ci status and wait till "after_n_builds" are uploaded
     require_ci_to_pass: false
-    # number of expected builds to receive before sending notifications
-    # we can set this to prevent notifications of partial results due to CI parallelism
-    # set this with respect to the sum of test parallelism in Circle CI configuration.
-    after_n_builds: 6

.github/workflows/baisc_checks.yml

@@ -1,41 +1,59 @@
-name: basic checks
+name: basic-check
 
-on: ["push", "pull_request"]
+on:
+  push:
+    branches:
+      - master
+      - main
+  pull_request:
+    branches:
+      - '**'
 
 jobs:
+
   check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     steps:
-      - name: dependencies
-        run: sudo apt update & sudo apt upgrade & sudo apt install --reinstall mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y
-
       - uses: actions/checkout@v2
-        with:
-          submodules: 'true'
 
-      - name: setup go
+      - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: '1.16'
+          go-version: 1.17
+
+      - name: install deps
+        run: |
+          sudo apt-get update
+          sudo apt-get -o Acquire::Retries=3 install make ncftp mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y && sudo apt upgrade -y
 
-      - name: deps
-        run: make deps
+      - name: dep
+        env:
+          GOPROXY: "https://proxy.golang.org,direct"
+          GO111MODULE: "on"
+        run: |
+          make deps
 
-      - name: build
-        run: make all
+      - name: Build
+        env:
+          GOPROXY: "https://proxy.golang.org,direct"
+          GO111MODULE: "on"
+        run: |
+          make
 
-      - name: tests-shared
-        run: make test-venus-shared
+      - name: Lint
+        run: |
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.44.2
+          golangci-lint run --timeout 10m
 
-      - name: compatible all
+      - name: Compatible all
         run: |
           make compatible-all
 
-      - name: gen all
+      - name: Gen all
         run: |
           make gen-all
 
-      - name: detect changes
+      - name: Detect changes
         run: |
           git status --porcelain
-          test -z "$(git status --porcelain)"
+          test -z "$(git status --porcelain)"

.github/workflows/tag-workflow.yml

@@ -87,15 +87,11 @@ jobs:
         uses: actions/setup-go@v2
         with:
           go-version: 1.17
-      - name: Install GCC
-        uses: egor-tensin/setup-gcc@v1
-        with:
-          version: latest
-          platform: x64
-      - name: Setup Rust
-        run: (sudo apt-get update && sudo apt-get install -y clang libssl-dev && which cargo && which rustc) || (curl https://sh.rustup.rs -sSf | sh -s -- -y)
       - name: install deps
-        run: sudo apt-get install make ncftp mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y && sudo apt upgrade -y
+        run: |
+          sudo apt-get update
+          sudo apt-get -o Acquire::Retries=3 install make ncftp mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y && sudo apt upgrade -y
+
       - name: Build
         run: |
           go clean --modcache && make deps && make

.github/workflows/test.yml

@@ -0,0 +1,76 @@
+name: Test
+
+on:
+  push:
+    branches:
+      - master
+      - main
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+
+  test:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+
+      - name: install deps
+        run: |
+          sudo apt-get update
+          sudo apt-get -o Acquire::Retries=3 install make ncftp mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y && sudo apt upgrade -y
+
+      - name: install statediff globally
+        run: |
+          ## statediff is optional; we succeed even if compilation fails.
+          mkdir -p /tmp/statediff
+          git clone https://github.com/filecoin-project/statediff.git /tmp/statediff
+          cd /tmp/statediff
+          go install ./cmd/statediff || exit 0
+
+      - name: dep
+        env:
+          GOPROXY: "https://proxy.golang.org,direct"
+          GO111MODULE: "on"
+        run: |
+          make deps
+
+      - name: Build
+        env:
+          GOPROXY: "https://proxy.golang.org,direct"
+          GO111MODULE: "on"
+        run: |
+          make
+
+      - name: Download vk file
+        run: |
+          ./venus fetch --size=0
+
+      - name: GenTool
+        run: |
+          go build -o genesis-file-server ./tools/genesis-file-server
+          go build -o gengen ./tools/gengen
+          ./gengen --keypath ./fixtures/live --out-car ./fixtures/live/genesis.car --out-json  ./fixtures/live/gen.json --config ./fixtures/setup.json
+          ./gengen --keypath ./fixtures/test --out-car ./fixtures/test/genesis.car --out-json  ./fixtures/test/gen.json --config ./fixtures/setup.json
+
+      - name: Unit Test
+        run: go test -coverpkg=./... -coverprofile=coverage_unit.txt -covermode=atomic -timeout=30m -parallel=4 -v ./... -integration=false -unit=true
+
+      - name: Integration Test
+        run: go test -coverpkg=./... -coverprofile=coverage_integration.txt -covermode=atomic -timeout=30m -parallel=4  -v ./... -integration=true -unit=false
+
+      - name: Upload
+        uses: codecov/codecov-action@v2
+        with:
+          token:
+          files: ./coverage_unit.txt,./coverage_integration.txt
+          flags: unittests,integration
+          name: venus
+          fail_ci_if_error: true
+          verbose: true

.gitignore

@@ -42,5 +42,8 @@ debug.test
 
 // HomeBrew
 Brewfile.lock.json
-
+build-dep
+coverage.*
 receipt.json
+genesis-file-server
+gengen

.golangci.yml

@@ -5,7 +5,7 @@ linters:
     - govet
     - misspell
     - goconst
-    - golint
+    - revive
     - errcheck
     - unconvert
     - staticcheck

Makefile

@@ -1,28 +1,46 @@
 export CGO_CFLAGS_ALLOW=-D__BLST_PORTABLE__
 export CGO_CFLAGS=-D__BLST_PORTABLE__
 
-all:
-	go run ./build/*.go build
+all: build
+.PHONY: all
 
-deps:
-	git submodule update --init
-	go run ./build/*.go smartdeps
+## variables
 
-lint:
-	go run ./build/*.go lint
+# git modules that need to be loaded
+MODULES:=
 
-test: test-venus-shared
-	go run ./build/*.go test -timeout=30m
+ldflags=-X=github.com/filecoin-project/venus/pkg/constants.CurrentCommit=+git.$(subst -,.,$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null))
+ifneq ($(strip $(LDFLAGS)),)
+	    ldflags+=-extldflags=$(LDFLAGS)
+	endif
 
-# WARNING THIS BUILDS A GO PLUGIN AND PLUGINS *DO NOT* WORK ON WINDOWS SYSTEMS
-iptb:
-	make -C tools/iptb-plugins all
+GOFLAGS+=-ldflags="$(ldflags)"
 
-clean:
-	rm ./venus
+## FFI
 
-	rm -rf ./extern/filecoin-ffi
-	rm -rf ./extern/test-vectors
+FFI_PATH:=extern/filecoin-ffi/
+FFI_DEPS:=.install-filcrypto
+FFI_DEPS:=$(addprefix $(FFI_PATH),$(FFI_DEPS))
+
+$(FFI_DEPS): build-dep/.filecoin-install ;
+
+build-dep/.filecoin-install: $(FFI_PATH)
+	$(MAKE) -C $(FFI_PATH) $(FFI_DEPS:$(FFI_PATH)%=%)
+	@touch $@
+
+MODULES+=$(FFI_PATH)
+BUILD_DEPS+=build-dep/.filecoin-install
+CLEAN+=build-dep/.filecoin-install
+
+## modules
+build-dep:
+	mkdir $@
+
+$(MODULES): build-dep/.update-modules;
+# dummy file that marks the last time modules were updated
+build-dep/.update-modules: build-dep;
+	git submodule update --init --recursive
+	touch $@
 
 gen-all: cborgen gogen inline-gen api-gen
 
@@ -75,3 +93,23 @@ actor-render:
 
 actor-replica:
 	cd venus-devtool && go run ./compatible/actors/*.go replica --dst ../venus-shared/actors/
+
+test:
+	go build -o genesis-file-server ./tools/genesis-file-server
+	go build -o gengen ./tools/gengen
+	./gengen --keypath ./fixtures/live --out-car ./fixtures/live/genesis.car --out-json  ./fixtures/live/gen.json --config ./fixtures/setup.json
+	./gengen --keypath ./fixtures/test --out-car ./fixtures/test/genesis.car --out-json  ./fixtures/test/gen.json --config ./fixtures/setup.json
+	 go test  -v ./... -integration=true -unit=false
+
+lint: $(BUILD_DEPS)
+	staticcheck ./...
+
+deps: $(BUILD_DEPS)
+
+dist-clean:
+	git clean -xdff
+	git submodule deinit --all -f
+
+build: $(BUILD_DEPS)
+	rm -f venus
+	go build -o ./venus $(GOFLAGS) .

app/node/inspector_api.go

@@ -1,9 +1,8 @@
 package node
 
 import (
-	"fmt"
-	"github.com/filecoin-project/venus/build/flags"
 	"github.com/filecoin-project/venus/pkg/config"
+	"github.com/filecoin-project/venus/pkg/constants"
 	"github.com/filecoin-project/venus/pkg/repo"
 	sysi "github.com/whyrusleeping/go-sysinfo"
 	"os"
@@ -146,5 +145,5 @@ func (g *inspector) Config() *config.Config {
 
 // FilecoinVersion returns the version of venus.
 func (g *inspector) FilecoinVersion() string {
-	return fmt.Sprintf("%s %s", flags.GitTag, flags.GitCommit)
+	return constants.UserVersion()
 }

app/node/test/setup.go

@@ -3,6 +3,7 @@ package test
 import (
 	"context"
 	"encoding/json"
+	th "github.com/filecoin-project/venus/pkg/testhelpers"
 	"os"
 	"testing"
 	"time"
@@ -11,7 +12,6 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/venus/app/node"
-	"github.com/filecoin-project/venus/build/project"
 	"github.com/filecoin-project/venus/pkg/clock"
 	"github.com/filecoin-project/venus/pkg/constants"
 	gengen "github.com/filecoin-project/venus/tools/gengen/util"
@@ -26,7 +26,7 @@ func CreateBootstrapSetup(t *testing.T) (*ChainSeed, *gengen.GenesisCfg, clock.C
 	fakeClock := clock.NewFake(time.Unix(genTime, 0))
 
 	// Load genesis config fixture.
-	genCfgPath := project.Root("fixtures/setup.json")
+	genCfgPath := th.Root("fixtures/setup.json")
 	genCfg := loadGenesisConfig(t, genCfgPath)
 	genCfg.Miners = append(genCfg.Miners, &gengen.CreateStorageMinerConfig{
 		Owner:         5,

app/submodule/discovery/discovery_submodule.go

@@ -118,7 +118,7 @@ func (discovery *DiscoverySubmodule) Start(offline bool) error {
 		// Wait for bootstrap to be sufficient connected
 		discovery.BootstrapReady.Wait()
 	}
-
+	log.Info("discovery module start")
 	return nil
 }
 

app/submodule/paych/paych_api.go

@@ -15,7 +15,7 @@ import (
 	"github.com/filecoin-project/venus/pkg/paychmgr"
 )
 
-type PaychAPI struct {
+type PaychAPI struct { //nolint
 	paychMgr *paychmgr.Manager
 }
 

app/submodule/syncer/syncer_submodule.go

@@ -243,31 +243,6 @@ func (syncer *SyncerSubmodule) handleIncomingBlocks(ctx context.Context, msg pub
 	return nil
 }
 
-// nolint
-func (syncer *SyncerSubmodule) loadLocalFullTipset(ctx context.Context, tsk types.TipSetKey) (*types.FullTipSet, error) {
-	ts, err := syncer.ChainModule.ChainReader.GetTipSet(ctx, tsk)
-	if err != nil {
-		return nil, err
-	}
-
-	fts := &types.FullTipSet{}
-	for _, b := range ts.Blocks() {
-		smsgs, bmsgs, err := syncer.ChainModule.MessageStore.LoadMetaMessages(ctx, b.Messages)
-		if err != nil {
-			return nil, err
-		}
-
-		fb := &types.FullBlock{
-			Header:       b,
-			BLSMessages:  bmsgs,
-			SECPMessages: smsgs,
-		}
-		fts.Blocks = append(fts.Blocks, fb)
-	}
-
-	return fts, nil
-}
-
 // Start starts the syncer submodule for a node.
 func (syncer *SyncerSubmodule) Start(ctx context.Context) error {
 	// setup topic

app/submodule/wallet/remotewallet/parse.go

@@ -7,9 +7,9 @@ import (
 )
 
 var (
-	regJWTToken = regexp.MustCompile("[a-zA-Z0-9\\-_]{5,}\\.[a-zA-Z0-9\\-_]{5,}\\.[a-zA-Z0-9\\-_]{5,}")                                                                                                                            //nolint
-	regUUID     = regexp.MustCompile("[0-9a-fA-F]{8}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{12}")                                                                                                        //nolint
-	regIPv4     = regexp.MustCompile("/ip4/(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/tcp/[0-9]{4,5}/http") //nolint
+	regJWTToken = regexp.MustCompile(`[a-zA-Z0-9\-_]{5,}\.[a-zA-Z0-9\-_]{5,}\.[a-zA-Z0-9\-_]{5,}`)
+	regUUID     = regexp.MustCompile(`[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}`)
+	regIPv4     = regexp.MustCompile(`/ip4/(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/tcp/[0-9]{4,5}/http`)
 )
 
 const (