Test/kgw (#446)

* test kgw
* go 1.22 compatiable `go mod vendor`

.github/workflows/ci.yaml

@@ -10,10 +10,12 @@ on:
       - main
 
 jobs:
-  unit-test:
-    runs-on: ubuntu-latest
+  test:
+    #runs-on: ubuntu-latest
+    runs-on: self-hosted
     if: ${{ !github.event.pull_request.draft }} # only run on non-draft PRs
     steps:
+      # shared setup for all tests
       - name: Checkout
         uses: actions/checkout@v3
         with:
@@ -26,18 +28,18 @@ jobs:
           version: '23.4'
           repo-token: ${{ secrets.KWIL_MACH_SECRET }}
 
-#      - name: Install swagger-codegen
-#        uses: swagger-api/swagger-codegen/.github/actions/generate/action.yml@main
-
       - name: Install Taskfile
         uses: arduino/setup-task@v1
         with:
           repo-token: ${{ secrets.KWIL_MACH_SECRET }}
 
+      #ubuntu-latest has go 1.21 installed https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md#go
+      #self-hosted also has go 1.21 installed
+      #the default behavior here will load pre-installed go version
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.21'
+          go-version: '1.21.x'
           check-latest: true
 
       - name: Install dependencies
@@ -48,6 +50,7 @@ jobs:
           git config --global url."https://${GH_ACCESS_TOKEN}:x-oauth-basic@github.com/kwilteam/".insteadOf "https://github.com/kwilteam/"
           task install:deps
 
+      # checks
       - name: Check tidiness of go.mod and go.sum
         run: |
           ./scripts/mods/check_tidy
@@ -73,46 +76,12 @@ jobs:
           skip-pkg-cache: true
           args: ./... ./core/... ./test/... ./parse/... --timeout=10m --config=.golangci.yml --skip-dirs ./core/rpc/protobuf
 
+      # unit test
       - name: Run unit test
         run: |
           task test:unit
 
-  acceptance-test:
-    runs-on: ubuntu-latest
-    if: ${{ !github.event.pull_request.draft }} # only run on non-draft PRs
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          token: ${{ secrets.KWIL_MACH_SECRET }}
-
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v2
-        with:
-          version: '23.4'
-          repo-token: ${{ secrets.KWIL_MACH_SECRET }}
-
-      - name: Install Taskfile
-        uses: arduino/setup-task@v1
-        with:
-          repo-token: ${{ secrets.KWIL_MACH_SECRET }}
-
-      - name: Setup Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: '1.21'
-          check-latest: true
-
-      - name: Install dependencies
-        env:
-          GH_ACCESS_TOKEN: ${{ secrets.KWIL_MACH_SECRET }}
-        run: |
-          go version
-          git config --global url."https://${GH_ACCESS_TOKEN}:x-oauth-basic@github.com/kwilteam/".insteadOf "https://github.com/kwilteam/"
-          task install:deps
-          go mod download
-
+      # integration test
       - name: Generate go vendor
         #for faster builds and private repos, need to run this after pb:compile:v1
         run: |
@@ -146,10 +115,34 @@ jobs:
           task build:cli
           task build:admin
 
-      - name: Pull Docker image
+      - name: Pull math extension docker image
         run: |
           docker pull kwilbrennan/extensions-math:multi-arch --platform linux/amd64
 
+      - name: Pull kgw repo & create vendor
+        # we only pull the repo, not build the image, because we want to use the cache
+        # provided by the docker/build-push-action
+        # vendor is used to bypass private repo issues
+        run: |
+          rm -rf /tmp/kgw
+          git clone https://github.com/kwilteam/kgw.git /tmp/kgw
+          cd /tmp/kgw
+          GOWORK=off go mod vendor
+          cd -
+
+      - name: Build kgw image
+        id: docker_build_kgw
+        uses: docker/build-push-action@v4
+        with:
+          context: /tmp/kgw
+          load: true
+          builder: ${{ steps.buildx.outputs.name }}
+          file: /tmp/kgw/Dockerfile
+          push: false
+          tags: kgw:latest
+          cache-from: type=local,src=/tmp/.buildx-cache-kgw
+          cache-to: type=local,dest=/tmp/.buildx-cache-kgw-new
+
       - name: Build kwild image
         id: docker_build_kwild
         uses: docker/build-push-action@v4
@@ -167,6 +160,7 @@ jobs:
           cache-from: type=local,src=/tmp/.buildx-cache-kwild
           cache-to: type=local,dest=/tmp/.buildx-cache-kwild-new
 
+      # maybe no need
       - name: Run acceptance test
         run: |
           echo "UID=$(id -u)" >> test/acceptance/.env
@@ -175,7 +169,17 @@ jobs:
           echo "KACT_DOCKER_COMPOSE_OVERRIDE_FILE=docker-compose.override.yml" >> test/acceptance/.env
           task test:act:nb
 
+      - name: Run integration test
+        run: |
+          echo "UID=$(id -u)" >> test/integration/.env
+          echo "GID=$(id -g)" >> test/integration/.env
+          cp test/integration/docker-compose.override.yml.example test/integration/docker-compose.override.yml
+          echo "KIT_DOCKER_COMPOSE_OVERRIDE_FILE=docker-compose.override.yml" >> test/integration/.env
+          task test:it:nb:all
+
       - name: Move cache
         run: |
           rm -rf /tmp/.buildx-cache-kwild
           mv /tmp/.buildx-cache-kwild-new /tmp/.buildx-cache-kwild
+          rm -rf /tmp/.buildx-cache-kgw
+            mv /tmp/.buildx-cache-kgw-new /tmp/.buildx-cache-kgw

Taskfile.yml

@@ -111,7 +111,8 @@ tasks:
     deps:
       - task: vendor:clean
     cmds:
-      - go mod vendor
+      # somehow GOWORK=off has no effect on local dev workflow, it will include local module changes
+      - GOWORK=off go mod vendor
 
   vendor:clean:
     desc: Clean vendor
@@ -176,7 +177,7 @@ tasks:
     desc: Start the dev environment(with testnet) without rebuilding docker image
     dir: test # different module
     cmds:
-      - go test ./integration -timeout 12h -dev -v
+      - go test ./integration -run ^TestLocalDevSetup$ -timeout 12h -dev -v
 
   # ************ test ************
   # test with build:docker task support passing CLI_ARGS to go test, e.g. task test:act -- -debug
@@ -216,7 +217,7 @@ tasks:
       - go test ./... -tags=ext_test -count=1 -race
 
   test:it:
-    desc: Run integration tests
+    desc: Run integration tests ('short' mode)
     deps:
       - task: build:cli
       - task: build:admin
@@ -225,6 +226,12 @@ tasks:
       - task: test:it:nb
 
   test:it:nb:
+    desc: Run integration tests ('short' mode)
+    dir: test # different module
+    cmds:
+      - go test -short -count=1 -timeout 0 ./integration -v {{.CLI_ARGS}}
+
+  test:it:nb:all:
     desc: Run integration tests
     dir: test # different module
     cmds:

core/rpc/client/user/http/client.go

@@ -81,12 +81,16 @@ func (c *Client) Broadcast(ctx context.Context, tx *transactions.Transaction, sy
 		// ErrInsufficientBalance, ErrWrongChain, etc. but how? the response
 		// body had better have retained the response error details!
 		if res != nil {
-			// fmt.Println("broadcast", res.StatusCode, res.Status)
 			if swaggerErr, ok := err.(httpTx.GenericSwaggerError); ok {
 				body := swaggerErr.Body() // fmt.Println(string(body))
-				return nil, parseBroadcastError(body)
+				if ok, _err := parseBroadcastError(body); ok {
+					return nil, _err
+				} else {
+					return nil, err // return the original error(unparsed)
+				}
 			}
 		}
+
 		return nil, err
 	}
 	defer res.Body.Close()
@@ -265,12 +269,14 @@ func (c *Client) Query(ctx context.Context, dbid string, query string) ([]map[st
 	return unmarshalMapResults(decodedResult)
 }
 
-func parseBroadcastError(respTxt []byte) error {
+// parseBroadcastError parses the response body from a broadcast error.
+// It returns true if the error was parsed successfully, false otherwise.
+func parseBroadcastError(respTxt []byte) (bool, error) {
 	var protoStatus status.Status
 	err := protojson.Unmarshal(respTxt, &protoStatus) // jsonpb is deprecated, otherwise we could use the resp.Body directly
 	if err != nil {
 		if err = json.Unmarshal(respTxt, &protoStatus); err != nil {
-			return err
+			return false, err
 		}
 	}
 	stat := grpcStatus.FromProto(&protoStatus)
@@ -308,8 +314,7 @@ func parseBroadcastError(respTxt []byte) error {
 		}
 	}
 
-	return err
-
+	return true, err
 }
 
 func parseErrorResponse(respTxt []byte) error {

internal/abci/abci.go

@@ -211,7 +211,8 @@ func (a *AbciApp) CheckTx(ctx context.Context, incoming *abciTypes.RequestCheckT
 		// Verify the correct chain ID is set, if it is set.
 		if protected := tx.Body.ChainID != ""; protected && tx.Body.ChainID != a.cfg.ChainID {
 			code = codeWrongChain
-			logger.Info("wrong chain ID", zap.String("payloadType", tx.Body.PayloadType.String()))
+			logger.Info("wrong chain ID",
+				zap.String("payloadType", tx.Body.PayloadType.String()))
 			return &abciTypes.ResponseCheckTx{Code: code.Uint32(), Log: "wrong chain ID"}, nil
 		}
 		// Verify Payload type
@@ -263,7 +264,7 @@ func (a *AbciApp) executeTx(ctx context.Context, rawTx []byte, logger *log.Logge
 			Code:    code.Uint32(),
 			GasUsed: gasUsed,
 			Events:  events,
-			Log:     "success",
+			Log:     "",
 			// Data, GasWanted, Info, Codespace
 		}
 

internal/services/grpc/txsvc/v1/call.go

@@ -13,6 +13,7 @@ import (
 func (s *Service) Call(ctx context.Context, req *txpb.CallRequest) (*txpb.CallResponse, error) {
 	body, msg, err := convertActionCall(req)
 	if err != nil {
+		// NOTE: http api needs to be able to get the error message
 		return nil, status.Errorf(codes.InvalidArgument, "failed to convert action call: %s", err.Error())
 	}