Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/master' into HEAD
Browse files Browse the repository at this point in the history
  • Loading branch information
robin-near committed Nov 2, 2023
2 parents 8d277bc + 3fcd356 commit 510d501
Show file tree
Hide file tree
Showing 213 changed files with 3,345 additions and 3,644 deletions.
1 change: 0 additions & 1 deletion .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ steps:
python3 scripts/check_nightly.py
python3 scripts/check_pytests.py
python3 scripts/check_fuzzing.py
python3 scripts/fix_nightly_feature_flags.py
Expand Down
169 changes: 169 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
name: CI

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

on:
pull_request:
merge_group:

jobs:
backward_compat:
name: "Backward Compatibility"
runs-on: ubuntu-22.04-16core
defaults:
run:
working-directory: ./pytest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: 3.11
cache: pip
- uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
with:
prefix-key: "0" # change this to invalidate CI cache
save-if: "false" # use the cache from nextest, but don’t double-save
# TODO: the python script should use the quick-release cargo profile, but currently does not
- run: pip3 install --user -r requirements.txt
- run: python3 tests/sanity/backward_compatible.py

cargo_nextest:
name: "Cargo Nextest (${{matrix.name}})"
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- name: Linux
os: ubuntu-22.04-16core
flags: ""
- name: Linux Nightly
os: ubuntu-22.04-16core
flags: "--features nightly,test_features"
- name: MacOS
os: macos-latest-xlarge
# FIXME: some of these tests don't work very well on MacOS at the moment. Should fix
# them at earliest convenience :)
flags: "--exclude integration-tests --exclude node-runtime --exclude runtime-params-estimator --exclude near-network --exclude estimator-warehouse"
timeout-minutes: 90

steps:
# Some of the tests allocate really sparse maps, so heuristic-based overcommit limits are not
# appropriate here.
# FIXME(#9634): remove this once the issue is resolved.
- run: sudo sysctl vm.overcommit_memory=1 || true
- uses: actions/checkout@v4
- uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
with:
crate: cargo-nextest
- uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
with:
crate: cargo-deny
- uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
with:
prefix-key: "0" # change this to invalidate CI cache
- run: cargo nextest run --locked --workspace -p '*' --cargo-profile quick-release --profile ci ${{ matrix.flags }}
env:
RUST_BACKTRACE: short

db_migration:
name: "Database Migration"
runs-on: ubuntu-22.04-16core
defaults:
run:
working-directory: ./pytest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: 3.11
cache: pip
- uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
with:
prefix-key: "0" # change this to invalidate CI cache
save-if: "false" # use the cache from nextest, but don’t double-save
- run: pip3 install --user -r requirements.txt
- run: python3 tests/sanity/db_migration.py

protobuf_backward_compat:
name: "Protobuf Backward Compatibility"
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- uses: bufbuild/buf-setup-action@1158f4fa81bc02e1ff62abcca6d516c9e24c77da
- uses: bufbuild/buf-breaking-action@a074e988ee34efcd4927079e79c611f428354c01
with:
against: "https://github.com/near/nearcore.git#ref=${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}"

sanity_checks:
name: "Sanity Checks"
runs-on: ubuntu-22.04-16core
strategy:
fail-fast: false
timeout-minutes: 90

steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: 3.11
cache: pip
- uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
with:
prefix-key: "0" # change this to invalidate CI cache
save-if: "false" # use the cache from nextest, but don’t double-save
- run: cargo build -p neard --bin neard --features nightly
- name: run spin_up_cluster.py
# Note: We're not running spin_up_cluster.py for non-nightly
# because spinning up non-nightly clusters is already covered
# by other steps in the CI, e.g. upgradable.
run: |
cd pytest
python3 -m pip install --progress-bar off --user -r requirements.txt
python3 tests/sanity/spin_up_cluster.py
- run: cargo build --profile quick-release -p neard --bin neard
- run: python3 scripts/state/update_res.py check

- run: python3 scripts/check_nightly.py
- run: python3 scripts/check_pytests.py

- run: python3 scripts/fix_nightly_feature_flags.py

# TODO: this should probably be a separate job
- run: ./scripts/formatting --check

# TODO: this should probably be a separate job
- name: check rpc_errors_schema.json
# TODO: the command ran and suggested are different. why? unify.
run: |
rm -f target/rpc_errors_schema.json
cargo check -p near-jsonrpc --features dump_errors_schema
if ! git --no-pager diff --no-index chain/jsonrpc/res/rpc_errors_schema.json target/rpc_errors_schema.json; then
set +x
echo 'The RPC errors schema reflects outdated typing structure; please run'
echo ' ./chain/jsonrpc/build_errors_schema.sh'
exit 1
fi >&2
upgradability:
name: "Upgradability"
runs-on: ubuntu-22.04-16core
defaults:
run:
working-directory: ./pytest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: 3.11
cache: pip
- uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
with:
prefix-key: "0" # change this to invalidate CI cache
save-if: "false" # use the cache from nextest, but don’t double-save
- run: pip3 install --user -r requirements.txt
- run: python3 tests/sanity/upgradable.py
29 changes: 11 additions & 18 deletions .github/workflows/master_fuzzer_binaries.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
name: Build fuzz targets from master
name: Fuzz Master
on:
push:
branches:
- master

jobs:
build_fuzzers:
runs-on: "ubuntu-20.04"
name: Build Fuzzers
runs-on: "ubuntu-22.04-16core"

permissions:
contents: "read"
Expand All @@ -24,19 +25,9 @@ jobs:
rustup install nightly
rustup default nightly
- name: Set swap to 10G
shell: bash
run: |
export SWAP_FILE=$(swapon --show=NAME | tail -n 1)
sudo swapoff $SWAP_FILE
sudo rm $SWAP_FILE
sudo fallocate -l 10G $SWAP_FILE
sudo chmod 600 $SWAP_FILE
sudo mkswap $SWAP_FILE
sudo swapon $SWAP_FILE
- name: Install cargo fuzz subcommand crate
run: cargo install cargo-fuzz
- uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
with:
crate: cargo-bolero

- run: rustup target add --toolchain nightly wasm32-unknown-unknown

Expand All @@ -47,6 +38,8 @@ jobs:

- uses: actions/checkout@master

- run: pip install -r scripts/build_fuzzers_requirements.txt

- run: python3 scripts/build_fuzzers.py ${{ github.ref_name }}
- name: "Compile fuzzers and upload to GCS"
run: |
NAME="nearcore-${{ github.ref_name }}-$(env TZ=Etc/UTC date +"%Y%m%d%H%M%S")"
cargo +nightly bolero build-clusterfuzz --all-features --profile fuzz
gsutil cp -Z target/fuzz/clusterfuzz.tar "gs://fuzzer_targets/${{ github.ref_name }}/$NAME.tar.gz"
19 changes: 12 additions & 7 deletions .github/workflows/ondemand_fuzzer_binaries.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,9 @@ jobs:
sudo mkswap $SWAP_FILE
sudo swapon $SWAP_FILE
- name: Install cargo fuzz subcommand crate
run: cargo install cargo-fuzz
- uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
with:
crate: cargo-bolero

- run: rustup target add --toolchain nightly wasm32-unknown-unknown

Expand All @@ -69,16 +70,20 @@ jobs:
with:
ref: ${{ github.event.inputs.branch_ref }}

- run: pip install -r scripts/build_fuzzers_requirements.txt

- name: Build Release branch fuzz targets
if: ${{ github.event.action == 'released'}}
run: python3 scripts/build_fuzzers.py release
run: echo "branch_type=release" >> "$GITHUB_ENV"

- name: Build RC branch fuzz targets
if: ${{ github.event.action == 'prereleased'}}
run: python3 scripts/build_fuzzers.py rc
run: echo "branch_type=rc" >> "$GITHUB_ENV"

- name: Build fuzz targets from ${{ github.event.inputs.branch_ref }}" branch
if: ${{ github.event_name == 'workflow_dispatch'}}
run: python3 scripts/build_fuzzers.py ${{ github.event.inputs.branch_type }}
run: echo "branch_type=${{ github.event.inputs.branch_type }}" >> "$GITHUB_ENV"

- name: "Compile fuzzers and upload to GCS"
- run: |
NAME="nearcore-$branch_type-$(env TZ=Etc/UTC date +"%Y%m%d%H%M%S")"
cargo +nightly bolero build-clusterfuzz --all-features --profile fuzz
gsutil cp -Z target/fuzz/clusterfuzz.tar "gs://fuzzer_targets/$branch_type/$NAME.tar.gz"
45 changes: 0 additions & 45 deletions .github/workflows/rust.yml

This file was deleted.

15 changes: 12 additions & 3 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,22 @@
## [unreleased]

### Protocol Changes
* The support for fixed shards in shard layout was removed. [#9219](https://github.com/near/nearcore/pull/9219)

* Restrict the creation of non-implicit top-level account that are longer than 32 bytes. Only the registrar account can create them. [#9589](https://github.com/near/nearcore/pull/9589)
* Adjust the number of block producers and chunk producers on testnet to facilitate testing of chunk-only producers [#9563](https://github.com/near/nearcore/pull/9563)


### Non-protocol Changes

* Add prometheus metrics for the internal state of the doomslug. [#9458](https://github.com/near/nearcore/pull/9458)
* Fix `EXPERIMENTAL_protocol_config` to apply overrides from `EpochConfig`. [#9692](https://github.com/near/nearcore/pull/9692)

## 1.36.0

### Protocol Changes

* The support for fixed shards in shard layout was removed. [#9219](https://github.com/near/nearcore/pull/9219)

### Non-protocol Changes

* New option `transaction_pool_size_limit` in `config.json` allows to limit the size of the node's transaction pool.
Expand All @@ -17,8 +28,6 @@
* Our more efficient and scalable V2 routing protocol is implemented. It shadows the V1 protocol for now while we verify its performance. [#9187](https://github.com/near/nearcore/pull/9187)
* The default config now enables TIER1 outbound connections by default. [#9349](https://github.com/near/nearcore/pull/9349)
* State Sync from GCS is available for experimental use. [#9398](https://github.com/near/nearcore/pull/9398)
* Add prometheus metrics for the internal state of the doomslug. [#9458](https://github.com/near/nearcore/pull/9458)
* Fix `EXPERIMENTAL_protocol_config` to apply overrides from `EpochConfig`. [#9692](https://github.com/near/nearcore/pull/9692)

## 1.35.0

Expand Down
Loading

0 comments on commit 510d501

Please sign in to comment.