4-nodes automated integration test infrastructure (#46)

* feat: added working Dockerfile

* test: multinode docker & docker-compose

* misc: added integration test in github action

* misc: divided unit test & integration test

* fix: added ETH port

* test: added basis lib

* misc: added test trigger

* test: added a test - delegation

* test: added test trigger

* test: added tests: store code

* test: added test - InstantiateContract

* test: added test - contract execution

* docs: added README of integration test

* misc: added go setup before integration test

* test: automated all prepared genesis.json creation process

resolves #48

* test: changed broadcast mode as block and delete sleep

.github/workflows/integration-test.yaml

@@ -0,0 +1,33 @@
+name: Integration test
+
+on:
+  push:
+    branches: [ "main", "cube", "hypercube", "tesseract", "release/*" ]
+  pull_request:
+    branches: [ "*" ]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Setup network
+      run: |
+        mkdir ~/genesis
+        cd integration_test && docker-compose up -d
+        docker ps
+        sleep 10
+
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.18.x
+
+    - name: Integration test
+      run: cd integration_test && go test
+
+    - name: Teardown
+      run: cd integration_test && docker-compose down

Dockerfile

@@ -0,0 +1,54 @@
+#
+# xpla localnet
+#
+# build:
+#   docker build --force-rm -t xpladev/xpla .
+# run:
+#   docker run --rm -it --env-file=path/to/.env --name xpla-localnet xpladev/xpla
+
+### BUILD
+FROM golang:1.18-alpine AS build
+WORKDIR /localnet
+
+# Create appuser.
+RUN adduser -D -g '' valiuser
+# Install required binaries
+RUN apk add --update --no-cache zip git make cmake build-base linux-headers musl-dev libc-dev
+
+# Copy source files
+COPY . /localnet/
+
+ENV LIBWASMVM_VERSION=v1.0.0
+
+RUN git clone --depth 1 https://github.com/microsoft/mimalloc; cd mimalloc; mkdir build; cd build; cmake ..; make -j$(nproc); make install
+ENV MIMALLOC_RESERVE_HUGE_OS_PAGES=4
+
+# See https://github.com/CosmWasm/wasmvm/releases
+ADD https://github.com/CosmWasm/wasmvm/releases/download/${LIBWASMVM_VERSION}/libwasmvm_muslc.aarch64.a /lib/libwasmvm_muslc.aarch64.a
+ADD https://github.com/CosmWasm/wasmvm/releases/download/${LIBWASMVM_VERSION}/libwasmvm_muslc.x86_64.a /lib/libwasmvm_muslc.x86_64.a
+RUN sha256sum /lib/libwasmvm_muslc.aarch64.a | grep 7d2239e9f25e96d0d4daba982ce92367aacf0cbd95d2facb8442268f2b1cc1fc
+RUN sha256sum /lib/libwasmvm_muslc.x86_64.a | grep f6282df732a13dec836cda1f399dd874b1e3163504dbd9607c6af915b2740479
+
+# Copy the library you want to the final location that will be found by the linker flag `-lwasmvm_muslc`
+RUN cp /lib/libwasmvm_muslc.`uname -m`.a /lib/libwasmvm_muslc.a
+
+# Build executable
+RUN LEDGER_ENABLED=false BUILD_TAGS=muslc LDFLAGS='-linkmode=external -extldflags "-L/localnet/mimalloc/build -lmimalloc -Wl,-z,muldefs -static"' make build
+
+# --------------------------------------------------------
+FROM alpine:3.15 AS runtime
+
+WORKDIR /opt
+RUN [ "mkdir", "-p", "/opt/integration_test" ]
+
+COPY --from=build /localnet/build/xplad /usr/bin/xplad
+COPY --from=build /localnet/integration_test /opt/integration_test
+
+# Expose Cosmos ports
+EXPOSE 9090
+EXPOSE 8545
+EXPOSE 26656
+#EXPOSE 26657
+
+# Set entry point
+CMD [ "/usr/bin/xplad", "version" ]

Makefile

@@ -85,6 +85,9 @@ all: install
 install: go.sum
 	go install -mod=readonly $(BUILD_FLAGS) ./cmd/xplad
 
+build: go.sum
+	go build -mod=readonly $(BUILD_FLAGS) -o build/xplad ./cmd/xplad
+
 .PHONY: test
 test: go.sum
 	go test -short ./...

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/golang/protobuf v1.5.2
 	github.com/gorilla/mux v1.8.0
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.2
 	github.com/rakyll/statik v0.1.7
 	github.com/spf13/cast v1.5.0
@@ -120,7 +121,6 @@ require (
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.2 // indirect
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.34.0 // indirect

integration_test/.env

@@ -0,0 +1,2 @@
+# only for integration test purpose!
+XPLAHOME=/opt/.xpla

integration_test/README.md

@@ -0,0 +1,48 @@
+# Integration Test
+
+This module is basically for the automated integration test. But if you want to execute by yourself for testing (test testing) purpose, this tests are also executable on your local. Please follow the step below:
+
+## Prerequisites
+
+- Docker >= 20.10
+- Docker compose >= 2.12
+
+## How to run
+
+```bash
+# 1. From the repo root, move to the integration_test, and execute docker compose
+cd integration_test
+docker-compose up -d
+
+# 2. Wait for building. Once done without error, you may check the nodes running
+docker ps
+
+#CONTAINER ID   IMAGE                    COMMAND                  CREATED          STATUS          PORTS                                                                             NAMES
+#648b7146ce8c   integration_test-node1   "sh -c 'MONIKER=vali…"   44 minutes ago   Up 44 minutes   0.0.0.0:8545->8545/tcp, 0.0.0.0:9090->9090/tcp, 0.0.0.0:26656->26656/tcp          xpla-localnet-validator1
+#97279d567135   integration_test-node2   "sh -c 'MONIKER=vali…"   44 minutes ago   Up 44 minutes   8545/tcp, 9090/tcp, 0.0.0.0:9100->9100/tcp, 26656/tcp, 0.0.0.0:26666->26666/tcp   xpla-localnet-validator2
+#f604f68c3f82   integration_test-node3   "sh -c 'MONIKER=vali…"   44 minutes ago   Up 44 minutes   8545/tcp, 9090/tcp, 0.0.0.0:9110->9110/tcp, 26656/tcp, 0.0.0.0:26676->26676/tcp   xpla-localnet-validator3
+#c3d0d9daefd2   integration_test-node4   "sh -c 'MONIKER=vali…"   44 minutes ago   Up 44 minutes   8545/tcp, 9090/tcp, 0.0.0.0:9120->9120/tcp, 26656/tcp, 0.0.0.0:26686->26686/tcp   xpla-localnet-validator4
+
+# 3. Execute tests
+go test
+
+# Do not execute short mode
+# (X) go test -short
+
+# ...
+# PASS
+# ok      github.com/xpladev/xpla/integration_test        29.365s
+
+# If you see the pass sign, you may down the nodes
+docker-compose down
+```
+
+## Test scenario
+
+### WASM
+
+- Send `delegation` tx
+- Upload the contract binary and get a code ID
+- With the code ID above, try to instantiate the contract
+- Execute the contract
+- Assert from querying the contract in each test step by assertion

integration_test/docker-compose.yml

@@ -0,0 +1,101 @@
+version: "3.7"
+services:
+  node1:
+    container_name: xpla-localnet-validator1
+
+    build:
+      context: ../.
+      target: runtime
+
+    volumes:
+      - ~/genesis:/genesis:rw
+
+    ports:
+      - "8545:8545"
+      - "9090:9090"
+      - "26656:26656"
+
+    entrypoint: sh -c "MONIKER=validator1 XPLAHOME=$XPLAHOME sh /opt/integration_test/entrypoint_master.sh"
+
+    networks:
+      vpcbr:
+        ipv4_address: 192.167.100.1
+
+  node2:
+    container_name: xpla-localnet-validator2
+
+    volumes:
+      - ~/genesis:/genesis:ro
+
+    depends_on:
+      - node1
+
+    build:
+      context: ../.
+      target: runtime
+
+    ports:
+      - "8555:8555"
+      - "9100:9100"
+      - "26666:26666"
+
+    entrypoint: sh -c "MONIKER=validator2 XPLAHOME=$XPLAHOME sh /opt/integration_test/entrypoint_secondary.sh"
+
+    networks:
+      vpcbr:
+        ipv4_address: 192.167.100.2
+
+  node3:
+    container_name: xpla-localnet-validator3
+
+    volumes:
+      - ~/genesis:/genesis:ro
+
+    depends_on:
+      - node1
+
+    build:
+      context: ../.
+      target: runtime
+
+    ports:
+      - "8565:8565"
+      - "9110:9110"
+      - "26676:26676"
+
+    entrypoint: sh -c "MONIKER=validator3 XPLAHOME=$XPLAHOME sh /opt/integration_test/entrypoint_secondary.sh"
+
+    networks:
+      vpcbr:
+        ipv4_address: 192.167.100.3
+
+  node4:
+    container_name: xpla-localnet-validator4
+
+    volumes:
+      - ~/genesis:/genesis:ro
+
+    depends_on:
+      - node1
+
+    build:
+      context: ../.
+      target: runtime
+
+    ports:
+      - "8575:8575"
+      - "9120:9120"
+      - "26686:26686"
+
+    entrypoint: sh -c "MONIKER=validator4 XPLAHOME=$XPLAHOME sh /opt/integration_test/entrypoint_secondary.sh"
+
+    networks:
+      vpcbr:
+        ipv4_address: 192.167.100.4
+
+networks:
+  vpcbr:
+    driver: bridge
+    ipam:
+     config:
+       - subnet: 192.167.0.0/16

integration_test/entrypoint.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+# MONIKER=validator1|validator2|validator3|validator4 sh /opt/integration_test/entrypoint.sh
+
+# 1. chain init
+/usr/bin/xplad init $MONIKER --chain-id localtest_1-1 --home $XPLAHOME
+
+# 2. copy the node setting files to the node home dir
+cp -r /opt/integration_test/$MONIKER/* /opt/.xpla/config
+
+# 3. register my validator & users keyfile
+/usr/bin/xplad keys add validator1 --recover --home $XPLAHOME < /opt/integration_test/test_keys/$MONIKER.mnemonics
+/usr/bin/xplad keys add user1 --recover --home $XPLAHOME < /opt/integration_test/test_keys/user1.mnemonics
+/usr/bin/xplad keys add user2 --recover --home $XPLAHOME < /opt/integration_test/test_keys/user2.mnemonics
+
+# 4. check genesis.json
+/usr/bin/xplad validate-genesis --home $XPLAHOME
+
+# 5. start daemon
+/usr/bin/xplad start --home=$XPLAHOME

integration_test/entrypoint_master.sh

@@ -0,0 +1,89 @@
+#!/bin/sh
+# MONIKER=validator1|validator2|validator3|validator4 sh /opt/integration_test/entrypoint.sh
+
+# File is not atomic and somestimes secondary nodes copies the existing old genesisfile
+# It should be cleared
+rm -f /genesis/*
+
+# 1. chain init
+/usr/bin/xplad init $MONIKER --chain-id localtest_1-1 --home $XPLAHOME
+
+# 2. Register the keys
+# xpla1z2k85n48ydfvzslrugwzl4j2u7vtdyf3xvucmc
+/usr/bin/xplad keys add validator1 --recover --keyring-backend test --home $XPLAHOME < /opt/integration_test/test_keys/validator1.mnemonics
+# xpla16wx7ye3ce060tjvmmpu8lm0ak5xr7gm2dp0kpt
+/usr/bin/xplad keys add validator2 --recover --keyring-backend test --home $XPLAHOME < /opt/integration_test/test_keys/validator2.mnemonics
+# xpla1pe9mc2q72u94sn2gg52ramrt26x5efw6hr5gt4
+/usr/bin/xplad keys add validator3 --recover --keyring-backend test --home $XPLAHOME < /opt/integration_test/test_keys/validator3.mnemonics
+# xpla1luqjvjyns9e92h06tq6zqtw76k8xtegfcerzjr
+/usr/bin/xplad keys add validator4 --recover --keyring-backend test --home $XPLAHOME < /opt/integration_test/test_keys/validator4.mnemonics
+# xpla1y6gnay0pv49asun56la09jcmhg2kc949mpftvt
+/usr/bin/xplad keys add user1 --recover --keyring-backend test --home $XPLAHOME < /opt/integration_test/test_keys/user1.mnemonics
+# xpla1u27snswkjpenlscgvszcfjmz8uy2y5qacx0826
+/usr/bin/xplad keys add user2 --recover --keyring-backend test --home $XPLAHOME < /opt/integration_test/test_keys/user2.mnemonics
+
+# 3. Add the genesis accounts
+/usr/bin/xplad add-genesis-account $(/usr/bin/xplad keys show validator1 -a --keyring-backend test --home $XPLAHOME) 100000000000000000000axpla --keyring-backend test --home $XPLAHOME
+/usr/bin/xplad add-genesis-account $(/usr/bin/xplad keys show validator2 -a --keyring-backend test --home $XPLAHOME) 100000000000000000000axpla --keyring-backend test --home $XPLAHOME
+/usr/bin/xplad add-genesis-account $(/usr/bin/xplad keys show validator3 -a --keyring-backend test --home $XPLAHOME) 100000000000000000000axpla --keyring-backend test --home $XPLAHOME
+/usr/bin/xplad add-genesis-account $(/usr/bin/xplad keys show validator4 -a --keyring-backend test --home $XPLAHOME) 100000000000000000000axpla --keyring-backend test --home $XPLAHOME
+/usr/bin/xplad add-genesis-account $(/usr/bin/xplad keys show user1 -a --keyring-backend test --home $XPLAHOME) 100000000000000000000axpla --keyring-backend test --home $XPLAHOME
+/usr/bin/xplad add-genesis-account $(/usr/bin/xplad keys show user2 -a --keyring-backend test --home $XPLAHOME) 100000000000000000000axpla --keyring-backend test --home $XPLAHOME
+
+# 4. Get the node keys and create gentxs
+for IDX in 1 2 3 4
+do
+    # 1) Copy the credentials
+    cp /opt/integration_test/validator$IDX/node_key.json $XPLAHOME/config
+    cp /opt/integration_test/validator$IDX/priv_validator_key.json $XPLAHOME/config
+
+    # 2) Execute a gentx
+    /usr/bin/xplad gentx validator$IDX 2000000000000000000axpla  \
+        --chain-id="localtest_1-1" \
+        --pubkey=$(xplad tendermint show-validator --home $XPLAHOME) \
+        --min-self-delegation=1 \
+        --moniker=validator$IDX \
+        --commission-rate=0.1 \
+        --commission-max-rate=0.2 \
+        --commission-max-change-rate=0.01 \
+        --ip="192.167.100.$IDX" \
+        --keyring-backend test \
+        --home $XPLAHOME
+
+done
+
+# 5. Do collect gentxs
+/usr/bin/xplad collect-gentxs --home $XPLAHOME
+
+# 6. Replace staking denom
+sed -i 's/"bond_denom": "stake"/"bond_denom": "axpla"/g' $XPLAHOME/config/genesis.json
+sed -i 's/"evm_denom": "aphoton",/"evm_denom": "axpla",/g' $XPLAHOME/config/genesis.json
+sed -i 's/"mint_denom": "stake",/"mint_denom": "axpla",/g' $XPLAHOME/config/genesis.json
+sed -i 's/"denom": "stake",/"denom": "axpla",/g' $XPLAHOME/config/genesis.json
+sed -i 's/"max_gas": "-1",/"max_gas": "5000000",/g' $XPLAHOME/config/genesis.json
+sed -i 's/"no_base_fee": false,/"no_base_fee": true,/g' $XPLAHOME/config/genesis.json
+sed -i 's/"inflation": "0.130000000000000000",/"inflation": "0.000000000000000000",/g' $XPLAHOME/config/genesis.json
+sed -i 's/"inflation_rate_change": "0.130000000000000000",/"inflation_rate_change": "0.000000000000000000",/g' $XPLAHOME/config/genesis.json
+sed -i 's/"inflation_min": "0.070000000000000000",/"inflation_min": "0.000000000000000000",/g' $XPLAHOME/config/genesis.json
+
+/usr/bin/xplad validate-genesis --home $XPLAHOME
+
+# 7. Copy to the shared folder
+cp $XPLAHOME/config/genesis.json /genesis
+
+### ALL DONE FOR GENESIS CONFIG
+### followings are for validator setting
+
+# 1. Copy the node setting files to the node home dir
+cp -r /opt/integration_test/$MONIKER/* $XPLAHOME/config
+
+# 2. Get genesis.json from the shared
+cp /genesis/genesis.json $XPLAHOME/config
+
+# 4. check genesis.json
+/usr/bin/xplad validate-genesis --home $XPLAHOME
+cat $XPLAHOME/config/genesis.json
+
+# 5. start daemon
+/usr/bin/xplad tendermint unsafe-reset-all --home=$XPLAHOME
+/usr/bin/xplad start --home=$XPLAHOME