QBFT: prevent maximum request timeout overflow (#1665) #258
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Build Check | |
on: | |
push: | |
paths-ignore: | |
- 'docs/**' | |
- '**.md' | |
- 'mkdocs.yml' | |
- '.gitignore' | |
branches: | |
- master | |
env: | |
GO_VERSION: 1.19 | |
GOPATH: ${{ github.workspace }}/go | |
WORKING_DIR: ${{ github.workspace }}/go/src/github.com/ethereum/go-ethereum | |
jobs: | |
build: | |
name: 'Run tests and build on ${{ matrix.os }}' | |
strategy: | |
fail-fast: false | |
matrix: | |
# Not enable for macos as there's a consistent failure: | |
# --- FAIL: TestUPNP_DDWRT (2.20s) | |
# ###[error] natupnp_test.go:165: not discovered | |
# must be sommething with Github Actions VM networking setup. | |
# Event Ubuntu requires a workaround | |
os: [ "ubuntu-20.04" ] | |
env: | |
QUORUM_IGNORE_TEST_PACKAGES: github.com/ethereum/go-ethereum/les,github.com/ethereum/go-ethereum/les/flowcontrol,github.com/ethereum/go-ethereum/mobile | |
runs-on: ${{ matrix.os }} | |
steps: | |
- name: 'Setup Go ${{ env.GO_VERSION }}' | |
uses: actions/setup-go@v1 | |
with: | |
go-version: ${{ env.GO_VERSION }} | |
- name: 'Check out project files' | |
uses: actions/checkout@v2 | |
with: | |
submodules: recursive | |
path: ${{ env.WORKING_DIR }} | |
- name: 'Apply workaround to fix networking in Linux' | |
if: runner.os == 'Linux' | |
run: | | |
# https://github.com/actions/virtual-environments/issues/798 | |
sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf | |
- name: 'Prepare environment' | |
run: | | |
echo "$(go env GOPATH)/bin" >> $GITHUB_PATH | |
- name: 'Run tests and build all' | |
working-directory: ${{ env.WORKING_DIR }} | |
run: | | |
make test all | |
docker-build: | |
name: 'Build Docker image' | |
runs-on: ubuntu-20.04 | |
steps: | |
- name: 'Check out project files' | |
uses: actions/checkout@v2 | |
- name: 'Build docker image' | |
id: build | |
run: | | |
output_dir=${{ runner.temp }}/docker | |
mkdir -p $output_dir | |
docker build -t quorumengineering/quorum:pr . | |
docker save quorumengineering/quorum:pr > quorum-pr.tar | |
tar cfvz $output_dir/quorum-pr.tar.gz quorum-pr.tar | |
echo "::set-output name=output_dir::$output_dir" | |
- name: 'Upload workflow artifact - Docker image' | |
uses: actions/upload-artifact@v1 | |
with: | |
name: docker-image | |
path: ${{ steps.build.outputs.output_dir }} | |
acceptance-tests-basic: | |
name: Acceptance tests (${{ matrix.tag }}) | |
needs: | |
- docker-build | |
if: success() | |
strategy: | |
fail-fast: false | |
matrix: | |
# list of tag expression being executed in parallel | |
# for PR, only selective tests are run. | |
# More comprehensive suites are scheduled to run in master | |
tag: | |
- 'basic || basic-raft || (advanced && raft) || networks/typical::raft' | |
- 'basic || basic-istanbul || (advanced && istanbul) || empty-block-period || networks/typical::istanbul' | |
- 'basic || basic-istanbul || (advanced && istanbul) || empty-block-period || block-reward || networks/typical::qbft' | |
- 'gcmode && block-sync && networks/template::raft-3plus1' | |
- 'gcmode && block-sync && networks/template::istanbul-3plus1' | |
- 'gcmode && block-sync && networks/template::qbft-3plus1' | |
- 'learner-peer-management || raftdnsenable && networks/template::raft-3plus1' | |
- 'validator-management && networks/template::qbft-3plus1' | |
- 'validator-management && networks/template::istanbul-3plus1' | |
- 'hybrid-validator-management-manage-besu && networks/typical-hybrid::hybrid-template-q2b1' | |
- 'hybrid-validator-management-manage-quorum && networks/typical-hybrid::hybrid-template-q1b2' | |
- 'qbft-transition-network && networks/template::qbft-4nodes-transition' | |
- 'basic || basic-raft || (advanced && raft) || networks/plugins::raft' | |
- 'basic || basic-istanbul || (advanced && istanbul) || networks/plugins::qbft' | |
- 'basic || basic-istanbul || (advanced && istanbul) || networks/plugins::istanbul' | |
- 'basic || basic-raft || (advanced && raft) || networks/plugins::raft-account-plugin-hashicorp-vault' | |
- 'basic || basic-istanbul || (advanced && istanbul) || networks/plugins::qbft-account-plugin-hashicorp-vault' | |
- 'basic || basic-istanbul || (advanced && istanbul) || networks/plugins::istanbul-account-plugin-hashicorp-vault' | |
- 'basic-rpc-security || networks/plugins::raft-rpc-security' | |
- 'basic-rpc-security || networks/plugins::qbft-rpc-security' | |
- 'basic-rpc-security || networks/plugins::istanbul-rpc-security' | |
- 'migration && networks/template::raft-4nodes' | |
- 'migration && networks/template::istanbul-4nodes' | |
- 'migration && networks/template::raft-4nodes-ancientdb' | |
- 'migration && networks/template::istanbul-4nodes-ancientdb' | |
- 'permissions-v1 && networks/template::raft-3plus1' | |
- 'permissions-v2 && networks/template::raft-3plus1' | |
- 'privacy-enhancements-upgrade || networks/template::raft-4nodes-pe' | |
- 'privacy-enhancements-upgrade || networks/template::istanbul-4nodes-pe' | |
- 'multitenancy && networks/plugins::raft-multitenancy' | |
- 'basic || basic-raft || (advanced && raft) || networks/typical::raft-simple-mps' | |
- 'basic || basic-istanbul || (advanced && istanbul) || networks/typical::qbft-simple-mps' | |
- 'basic || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul-simple-mps' | |
- 'basic || networks/typical::raftmps' | |
- 'basic || networks/typical::qbftmps' | |
- 'basic || networks/typical::istanbulmps' | |
- 'mps-upgrade-txtrace || networks/template::raft-4nodes-mps' | |
- 'mps-upgrade-txtrace || networks/template::istanbul-4nodes-mps' | |
- 'mps-mixed-network-psr-check || networks/template::raft-4nodes-mps-mixed' | |
- 'mps-mixed-network-psr-check || networks/template::istanbul-4nodes-mps-mixed' | |
- '(basic && !nosupport && !mps && !(spam && !raw) && !eth-api-signed && !privacy-enhancements-disabled && !graphql && !async && !extension && !storage-root && !personal-api-signed) || networks/typical-hybrid::hybrid' | |
runs-on: ubuntu-20.04 | |
steps: | |
- name: 'Download workflow artifact - Docker image' | |
uses: actions/download-artifact@v1 | |
with: | |
name: docker-image | |
- name: 'Load Docker image' | |
id: setup | |
run: | | |
tar xfvz docker-image/quorum-pr.tar.gz | |
docker load --input quorum-pr.tar | |
docker_env_file="${{ runner.temp }}/env.list" | |
echo "TF_VAR_quorum_docker_image={ name = \"quorumengineering/quorum:pr\", local = true }" >> $docker_env_file | |
echo "::set-output name=outputDir::${{ runner.temp }}" | |
echo "::set-output name=dockerEnvFile::$docker_env_file" | |
- name: 'Run acceptance tests' | |
run: | | |
cat ${{ steps.setup.outputs.dockerEnvFile }} | |
docker run --rm \ | |
--network host \ | |
-v /var/run/docker.sock:/var/run/docker.sock \ | |
-v ${{ steps.setup.outputs.outputDir }}:${{ steps.setup.outputs.outputDir }} \ | |
--env-file ${{ steps.setup.outputs.dockerEnvFile }} \ | |
quorumengineering/acctests:latest test \ | |
-Pauto \ | |
-Dauto.outputDir=${{ steps.setup.outputs.outputDir }} \ | |
-Dtags="${{ matrix.tag }}" | |
- name: 'Debug' | |
run: | | |
docker images | |
docker ps -a | |
acceptance-tests-extra: | |
name: Acceptance tests (${{ matrix.tag }}) | |
needs: | |
- docker-build | |
if: success() | |
strategy: | |
fail-fast: false | |
matrix: | |
# list of tag expression being executed in parallel | |
include: | |
# privacy enhancements tests | |
- tag: '(basic && !privacy-enhancements-disabled) || privacy-enhancements || mandatory-recipients || basic-raft || (advanced && raft) || networks/typical::raft' | |
privacy-enhancements: true | |
privacy-precompile: false | |
privacy-marker-transactions: false | |
- tag: '(basic && !privacy-enhancements-disabled) || privacy-enhancements || mandatory-recipients || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul' | |
privacy-enhancements: true | |
privacy-precompile: false | |
privacy-marker-transactions: false | |
# privacy precompile/privacy marker transaction tests | |
- tag: 'basic || basic-raft || (advanced && raft) || networks/typical::raft' | |
privacy-enhancements: false | |
privacy-precompile: true | |
privacy-marker-transactions: false | |
- tag: 'basic || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul' | |
privacy-enhancements: false | |
privacy-precompile: true | |
privacy-marker-transactions: false | |
- tag: 'basic || basic-istanbul || (advanced && istanbul) || networks/typical::qbft' | |
privacy-enhancements: false | |
privacy-precompile: true | |
privacy-marker-transactions: false | |
- tag: '(multitenancy || privacy-precompile-enabled) && networks/plugins::raft-multitenancy' | |
privacy-enhancements: false | |
privacy-precompile: true | |
privacy-marker-transactions: true | |
- tag: '(basic && !privacy-precompile-disabled) || basic-raft || (advanced && raft) || networks/typical::raft-simple-mps' | |
privacy-enhancements: false | |
privacy-precompile: true | |
privacy-marker-transactions: true | |
- tag: '(basic && !privacy-precompile-disabled) || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul-simple-mps' | |
privacy-enhancements: false | |
privacy-precompile: true | |
privacy-marker-transactions: true | |
- tag: '(basic && !privacy-precompile-disabled) || basic-istanbul || (advanced && istanbul) || networks/typical::qbft-simple-mps' | |
privacy-enhancements: false | |
privacy-precompile: true | |
privacy-marker-transactions: true | |
- tag: '(basic && !privacy-precompile-disabled) || networks/typical::raftmps' | |
privacy-enhancements: false | |
privacy-precompile: true | |
privacy-marker-transactions: true | |
- tag: '(basic && !privacy-precompile-disabled) || networks/typical::istanbulmps' | |
privacy-enhancements: false | |
privacy-precompile: true | |
privacy-marker-transactions: true | |
- tag: '(basic && !privacy-precompile-disabled) || networks/typical::qbftmps' | |
privacy-enhancements: false | |
privacy-precompile: true | |
privacy-marker-transactions: true | |
# privacy enhancements + privacy precompile/privacy marker transaction tests | |
- tag: '(basic && !privacy-enhancements-disabled && !privacy-precompile-disabled) || privacy-enhancements || mandatory-recipients || privacy-precompile-enabled || basic-raft || (advanced && raft) || networks/typical::raft' | |
privacy-enhancements: true | |
privacy-precompile: true | |
privacy-marker-transactions: true | |
- tag: '(basic && !privacy-enhancements-disabled && !privacy-precompile-disabled) || privacy-enhancements || mandatory-recipients || privacy-precompile-enabled || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul' | |
privacy-enhancements: true | |
privacy-precompile: true | |
privacy-marker-transactions: true | |
- tag: '(basic && !privacy-enhancements-disabled && !privacy-precompile-disabled) || privacy-enhancements || mandatory-recipients || privacy-precompile-enabled || basic-istanbul || (advanced && istanbul) || networks/typical::qbft' | |
privacy-enhancements: true | |
privacy-precompile: true | |
privacy-marker-transactions: true | |
- tag: 'privacy-precompile-compatibility && networks/template::raft-4nodes' | |
privacy-enhancements: false | |
privacy-precompile: true | |
privacy-marker-transactions: false # do not enable pmts as the test will do this on the necessary nodes | |
- tag: 'privacy-precompile-compatibility && networks/template::istanbul-4nodes' | |
privacy-enhancements: false | |
privacy-precompile: true | |
privacy-marker-transactions: false # do not enable pmts as the test will do this on the necessary nodes | |
runs-on: ubuntu-20.04 | |
steps: | |
- name: 'Download workflow artifact - Docker image' | |
uses: actions/download-artifact@v1 | |
with: | |
name: docker-image | |
- name: 'Load Docker image' | |
id: setup | |
run: | | |
tar xfvz docker-image/quorum-pr.tar.gz | |
docker load --input quorum-pr.tar | |
docker_env_file="${{ runner.temp }}/env.list" | |
echo "TF_VAR_quorum_docker_image={ name = \"quorumengineering/quorum:pr\", local = true }" >> $docker_env_file | |
echo "TF_VAR_privacy_enhancements={block=0, enabled=${{ matrix.privacy-enhancements}}}" >> $docker_env_file | |
echo "TF_VAR_privacy_precompile={block=0, enabled=${{ matrix.privacy-precompile}}}" >> $docker_env_file | |
echo "TF_VAR_privacy_marker_transactions=${{ matrix.privacy-marker-transactions}}" >> $docker_env_file | |
echo "::set-output name=outputDir::${{ runner.temp }}" | |
echo "::set-output name=dockerEnvFile::$docker_env_file" | |
- name: 'Run extra acceptance tests' | |
run: | | |
cat ${{ steps.setup.outputs.dockerEnvFile }} | |
docker run --rm \ | |
--network host \ | |
-v /var/run/docker.sock:/var/run/docker.sock \ | |
-v ${{ steps.setup.outputs.outputDir }}:${{ steps.setup.outputs.outputDir }} \ | |
--env-file ${{ steps.setup.outputs.dockerEnvFile }} \ | |
quorumengineering/acctests:latest test \ | |
-Pauto \ | |
-Dauto.outputDir=${{ steps.setup.outputs.outputDir }} \ | |
-Dtags="${{ matrix.tag }}" | |
- name: 'Debug' | |
run: | | |
docker images | |
docker ps -a | |
peeps-tests: | |
name: Run PEEPS tests | |
needs: | |
- docker-build | |
runs-on: ubuntu-20.04 | |
steps: | |
- name: 'Checkout' | |
uses: actions/checkout@v2 | |
- name: 'Download workflow artifact - Docker image' | |
uses: actions/download-artifact@v1 | |
with: | |
name: docker-image | |
- name: 'Load Docker image' | |
id: setup | |
run: | | |
tar xfvz docker-image/quorum-pr.tar.gz | |
docker load --input quorum-pr.tar | |
docker image tag quorumengineering/quorum:pr quorumengineering/quorum:develop | |
docker image ls | |
- name: Set up Java | |
uses: actions/setup-java@v2 | |
with: | |
distribution: 'adopt' | |
java-version: 11 | |
check-latest: true | |
- name: PEEPS | |
run: | | |
cd build | |
./run-peeps.sh | |
- name: PEEPS Test Report | |
uses: mikepenz/action-junit-report@v2 | |
if: always() | |
with: | |
report_paths: '**/build/test-results/*/TEST-*.xml' | |
check_name: PEEPS test report | |
publish-docker: | |
name: Publish Docker Image | |
needs: | |
- build | |
- acceptance-tests-basic | |
- acceptance-tests-extra | |
runs-on: ubuntu-20.04 | |
steps: | |
- name: 'Checkout' | |
uses: actions/checkout@v2 | |
- name: 'Build and publish to Docker Hub' | |
uses: docker/build-push-action@v1 | |
with: | |
username: ${{ secrets.DOCKER_USERNAME }} | |
password: ${{ secrets.DOCKER_ACCESS_TOKEN }} | |
repository: ${{ secrets.DOCKER_REPO }} | |
tags: develop | |
add_git_labels: true | |
notify: | |
if: always() | |
name: Notify | |
needs: | |
- build | |
- publish-docker | |
runs-on: ubuntu-20.04 | |
steps: | |
- name: 'Setup metadata' | |
id: setup | |
run: | | |
gitref_path="${{ github.ref }}" | |
gitref_path=${gitref_path/refs\/heads/tree} # for refs/heads/my-branch | |
gitref_path=${gitref_path/refs\/tags/tree} # for refs/tags/v1.0.0 | |
gitref_path=${gitref_path#refs\/} # for refs/pull/123/merge | |
gitref_path=${gitref_path%/merge} # for refs/pull/123/merge | |
echo "::set-output name=gitref-path::$gitref_path" | |
- name: 'Prepare Slack message with full info' | |
id: status | |
uses: actions/github-script@0.8.0 | |
with: | |
script: | | |
var gitref_path = "${{ steps.setup.outputs.gitref-path }}" | |
//////////////////////////////////// | |
// retrieve workflow run data | |
//////////////////////////////////// | |
console.log("get workflow run") | |
const wf_run = await github.actions.getWorkflowRun({ | |
owner: context.repo.owner, | |
repo: context.repo.repo, | |
run_id: ${{ github.run_id }} | |
}) | |
console.log(wf_run.data) | |
console.log("get jobs for workflow run:", wf_run.data.jobs_url) | |
const jobs_response = await github.request(wf_run.data.jobs_url) | |
//////////////////////////////////// | |
// build slack notification message | |
//////////////////////////////////// | |
// some utility functions | |
var date_diff_func = function(start, end) { | |
var duration = end - start | |
// format the duration | |
var delta = duration / 1000 | |
var days = Math.floor(delta / 86400) | |
delta -= days * 86400 | |
var hours = Math.floor(delta / 3600) % 24 | |
delta -= hours * 3600 | |
var minutes = Math.floor(delta / 60) % 60 | |
delta -= minutes * 60 | |
var seconds = Math.floor(delta % 60) | |
var format_func = function(v, text, check) { | |
if (v <= 0 && check) { | |
return "" | |
} else { | |
return v + text | |
} | |
} | |
return format_func(days, "d", true) + format_func(hours, "h", true) + format_func(minutes, "m", true) + format_func(seconds, "s", false) | |
} | |
var status_icon_func = function(s) { | |
switch (s) { | |
case "w_success": | |
return ":white_check_mark:" | |
case "w_failure": | |
return ":no_entry:" | |
case "w_cancelled": | |
return ":warning:" | |
case "success": | |
return "\u2713" | |
case "failure": | |
return "\u2717" | |
default: | |
return "\u20e0" | |
} | |
} | |
// build the message | |
var job_blocks = [] | |
var is_wf_success = true | |
var is_wf_failure = false | |
for (j of jobs_response.data.jobs) { | |
console.log(j.name, ":", j.status, j.conclusion, j.started_at, j.completed_at) | |
// ignore the current job running this script | |
if (j.status != "completed") { | |
continue | |
} | |
if (j.conclusion != "success") { | |
is_wf_success = false | |
} | |
if (j.conclusion == "failure") { | |
is_wf_failure = true | |
} | |
job_blocks.push({ | |
type: "section", | |
text: { | |
type: "mrkdwn", | |
text: `${status_icon_func(j.conclusion)} <${j.html_url}|${j.name}> took ${date_diff_func(new Date(j.started_at), new Date(j.completed_at))}` | |
} | |
}) | |
} | |
var workflow_status = "w_cancelled" | |
if (is_wf_success) { | |
workflow_status = "w_success" | |
} else if (is_wf_failure) { | |
workflow_status = "w_failure" | |
} | |
var context_elements = [ | |
{ | |
"type": "mrkdwn", | |
"text": "*Repo:* <https://github.com/${{ github.repository }}|${{ github.repository }}>" | |
}, | |
{ | |
"type": "mrkdwn", | |
"text": `*Branch:* <https://github.com/${{ github.repository }}/${gitref_path}|${{ github.ref }}>` | |
}, | |
{ | |
"type": "mrkdwn", | |
"text": `*Event:* ${wf_run.data.event}` | |
}, | |
{ | |
"type": "mrkdwn", | |
"text": `*Commit:* <https://github.com/${{ github.repository }}/commit/${wf_run.data.head_commit.id}|${wf_run.data.head_commit.id.substr(0, 8)}>` | |
}, | |
{ | |
"type": "mrkdwn", | |
"text": `*Author:* ${wf_run.data.head_commit.author.name}` | |
} | |
] | |
var header_blocks = [ | |
{ | |
type: "section", | |
text: { | |
type: "mrkdwn", | |
text: `${status_icon_func(workflow_status)} *${{ github.workflow }}* <${wf_run.data.html_url}|#${{ github.run_number }}> took ${date_diff_func(new Date(wf_run.data.created_at), new Date(wf_run.data.updated_at))}` | |
} | |
}, | |
{ | |
type: "context", | |
elements: context_elements, | |
}, | |
{ | |
type: "divider" | |
} | |
] | |
var slack_msg = { | |
blocks: [].concat(header_blocks, job_blocks) | |
} | |
return slack_msg | |
- name: 'Prepare Slack message with partial info' | |
id: short_status | |
if: failure() | |
uses: actions/github-script@0.8.0 | |
with: | |
script: | | |
//////////////////////////////////// | |
// retrieve workflow run data | |
//////////////////////////////////// | |
const wf_run = await github.actions.getWorkflowRun({ | |
owner: context.repo.owner, | |
repo: context.repo.repo, | |
run_id: ${{ github.run_id }} | |
}) | |
var date_diff_func = function(start, end) { | |
var duration = end - start | |
// format the duration | |
var delta = duration / 1000 | |
var days = Math.floor(delta / 86400) | |
delta -= days * 86400 | |
var hours = Math.floor(delta / 3600) % 24 | |
delta -= hours * 3600 | |
var minutes = Math.floor(delta / 60) % 60 | |
delta -= minutes * 60 | |
var seconds = Math.floor(delta % 60) | |
var format_func = function(v, text, check) { | |
if (v <= 0 && check) { | |
return "" | |
} else { | |
return v + text | |
} | |
} | |
return format_func(days, "d", true) + format_func(hours, "h", true) + format_func(minutes, "m", true) + format_func(seconds, "s", false) | |
} | |
var slack_msg = { | |
blocks: [ | |
{ | |
type: "section", | |
text: { | |
type: "mrkdwn", | |
text: `:skull_and_crossbones: *${{ github.workflow }}* <${wf_run.data.html_url}|#${{ github.run_number }}> (took ${date_diff_func(new Date(wf_run.data.created_at), new Date(wf_run.data.updated_at))})` | |
} | |
} | |
] | |
} | |
return slack_msg | |
- name: 'Send to Slack' | |
if: always() | |
run: | | |
cat <<JSON > long_message.json | |
${{ steps.status.outputs.result }} | |
JSON | |
cat <<JSON > short_message.json | |
${{ steps.short_status.outputs.result }} | |
JSON | |
_post() { | |
curl -X POST ${{ secrets.SLACK_WEBHOOK_URL }} -H "Content-type: application/json" --data "@${1}" | |
} | |
_post "long_message.json" || _post "short_message.json" | |