From f7958f70b38436b24d2fca109b58e596eb6f4c78 Mon Sep 17 00:00:00 2001 From: Andrew Plaza Date: Wed, 19 May 2021 16:23:46 -0400 Subject: [PATCH 1/7] need to test release.yml --- .github/workflows/release.yml | 67 +++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..4e3fd883 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,67 @@ +name: Release CI +on: + push: + branches: + - release +jobs: + deploy: + # A strategy is used to define various build environments this job will run. + # + # To say it simple, this will create 3 separate independent jobs which will + # run on ubuntu, mac & windows. + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + # Runs when the commit message contains "[Released]" + # if: "contains(github.event.head_commit.message, '[Released]')" + steps: + - uses: actions/checkout@v2 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + - name: Build + run: cargo build --release + + # Condition to only run this whenever the runner os is Mac + - name: Build Release Mac + if: matrix.os == 'macos-latest' + run: | + strip target/release/aow + mkdir -p release + tar -C ./target/release/ -czvf ./release/aow-mac.tar.gz ./aow + + # Condition to only run this whenever the runner os is Ubuntu + - name: Build Release Linux + if: matrix.os == 'ubuntu-latest' + run: | + cargo install cargo-deb + cargo deb + strip target/release/aow + mkdir -p release + mv target/debian/*.deb ./release/aow-linux.deb + tar -C ./target/release/ -czvf ./release/aow-linux.tar.gz ./aow + + # Condition to only run this whenever the runner os is Windows + - name: Build Release Win + if: matrix.os == 'windows-latest' + run: | + mkdir -p release + tar -C ./target/release/ -czvf ./release/aow-win.tar.gz ./aow.exe + + # This will draft a new release & will attach the binaries produced by the above outputs. + # You still need to publish this release though after job ends. + - name: Release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + draft: true + files: | + ./release/*.tar.gz + ./release/*.zip + ./release/*.deb + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # You don't need to add this in secrets it's by default. From 82a47b1121e54c53ee161a6a09df14979a2dd8c1 Mon Sep 17 00:00:00 2001 From: Andrew Plaza Date: Wed, 19 May 2021 22:57:56 -0400 Subject: [PATCH 2/7] improve release workflow --- .github/workflows/release.yml | 48 +++++++++++++++++------------------ 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4e3fd883..6f2b3b11 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,8 +1,8 @@ name: Release CI on: push: - branches: - - release + branches: + - release jobs: deploy: # A strategy is used to define various build environments this job will run. @@ -11,7 +11,7 @@ jobs: # run on ubuntu, mac & windows. strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} # Runs when the commit message contains "[Released]" # if: "contains(github.event.head_commit.message, '[Released]')" @@ -25,43 +25,41 @@ jobs: - name: Build run: cargo build --release - # Condition to only run this whenever the runner os is Mac - - name: Build Release Mac - if: matrix.os == 'macos-latest' - run: | - strip target/release/aow - mkdir -p release - tar -C ./target/release/ -czvf ./release/aow-mac.tar.gz ./aow - - # Condition to only run this whenever the runner os is Ubuntu - - name: Build Release Linux + - name: Package for Debian if: matrix.os == 'ubuntu-latest' run: | cargo install cargo-deb cargo deb - strip target/release/aow + strip target/release/test_gh_actions mkdir -p release - mv target/debian/*.deb ./release/aow-linux.deb - tar -C ./target/release/ -czvf ./release/aow-linux.tar.gz ./aow - - # Condition to only run this whenever the runner os is Windows - - name: Build Release Win - if: matrix.os == 'windows-latest' + mv target/debian/*.deb ./release/test_gh_actions-linux.deb + tar -C ./target/release/ -czvf ./release/test_gh_actions-linux.tar.gz ./test_gh_actions + + - name: Package pure binary + if: matrix.os == 'ubuntu-latest' run: | + strip target/release/test_gh_actions mkdir -p release - tar -C ./target/release/ -czvf ./release/aow-win.tar.gz ./aow.exe + mv target/release/test_gh_actions ./release/test_gh_actions + - name: Get changelog + id: changelog_reader + uses: mindsers/changelog-reader-action@v2 + with: + validation_depth: 2 + path: ./CHANGELOG.md # This will draft a new release & will attach the binaries produced by the above outputs. # You still need to publish this release though after job ends. - name: Release uses: softprops/action-gh-release@v1 with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} + tag_name: ${{ steps.changelog_reader.outputs.version }} + release_name: Release ${{ steps.changelog_reader.outputs.version }} + body: ${{ steps.changelog_reader.outputs.changes }} draft: true files: | ./release/*.tar.gz - ./release/*.zip ./release/*.deb + ./release/test_gh_actions env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # You don't need to add this in secrets it's by default. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From bd5f351bf2ceccb3d761d24891e665a06fbf87d7 Mon Sep 17 00:00:00 2001 From: Andrew Plaza Date: Wed, 19 May 2021 23:24:12 -0400 Subject: [PATCH 3/7] Refactor Changelog - make changelog _actually_ follow https://keepachangelog.com/en/1.0.0/ to a T - create release workflow which automates creating binaries --- .github/workflows/release.yml | 2 +- CHANGELOG.md | 146 +++++++++++++++--------------- bin/polkadot-archive/CHANGELOG.md | 19 ++++ 3 files changed, 93 insertions(+), 74 deletions(-) create mode 100644 bin/polkadot-archive/CHANGELOG.md diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6f2b3b11..de201a18 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,7 +34,7 @@ jobs: mkdir -p release mv target/debian/*.deb ./release/test_gh_actions-linux.deb tar -C ./target/release/ -czvf ./release/test_gh_actions-linux.tar.gz ./test_gh_actions - + - name: Package pure binary if: matrix.os == 'ubuntu-latest' run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index bf0289b4..6f47e9dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,83 +1,83 @@ -# Changelog for `substrate-archive` - -## **[Unreleased]** - -## **[v0.5.1]** - -- [Added] Release checklist -- [Changed] Pinned to `substrate`/`polkadot` release v0.8.30 -- [Changed] Keeping the `substrate-archive` and `substrate-archive-backend` crates's versions aligned. - -## **[v0.5.0]** - -### Substrate Archive - -- [Changed] Use SQLX for migrations - - this will create a new table in the database, `sqlx_migrations` or similiar. - - the older table, `__refinery_migrations` can be safely dropped - -#### Api Changes - -- [Added] New `ArchiveBuilder` struct for constructing the indexer. - - [Changed] returns a trait rather than a struct. - - [Changed] the returned trait,`Archive` manages the underlying actor runtime - - [Changed] no longer need to instantiate a client and manually pass it to the Archive - - [Changed] rename `run_with` to `drive` - - [Changed] Archive now accepts just a postgres URL instead of a postgres URL split into its parts. This should - make configuring the archive more straightforward. Takes from environment variable `DATABASE_URL` if not passed to the - archive directly -- [Added] Archive now reads the `CHAIN_DATA_DB` environment variable if the path to the backend chain database is not passed directly. -- [Removed] Archive no longer needs an RPC url to function -- [Changed] Print out the Postgres URL at startup -- [Added] `max_block_load` to configure the maximum number of blocks loaded at once -- [Added] Two new options to the configuration relative to state tracing: - - `targets` for specifying runtime targets to trace in WASM - - `folder` where WASM blobs with tracing enabled are kept. - - More on state-tracing [here](https://github.com/paritytech/substrate-archive/wiki/6.\)-State-Tracing-&-Balance-Reconciliation) -- [Changed] Archive config is now separated into sections for readability. Migration is manual but looking at the new `archive.conf` in `polkadot-archive` or `node-template-archive` folders should help. - -#### Internal Changes - -- [Changed] Postgres SQL queries are now type checked -- [QoL] Refactor file layout to `substrate-archive` and `substrate-archive-backend`. -- [perf] Decouple Database actors -- [QoL] upgrade to SQLx 0.5.0 -- [perf] Overhaul of block indexing. Now uses a Iterator to only collect batches of blocks from the database, -taking advantage of the better reading performance of sequential data access. Gathering blocks by RPC is no longer done. -- [perf] a new module `runtime_version_cache` is introduced in order to cache and run a binary search on runtime version & blocks. -- [perf] better queries for the set difference between the storage and blocks table - makes querying for missing storage more efficient -- [err] Better handling of SQL errors -- [perf] switch to a leaner, 'lower-level' actor framework (xtra) -- [perf] switch to a background-task-queue for executing blocks. This uses significantly less memory and - persists blocks that need to be executed on-disk. -- [QoL] remove the last frame dependency, `frame-system`. Archive now relies only on generic traits defined in substrate-core. -- new method of batch inserts avoids starving the Postgres Pool of connections -- [perf] Notification stream from Postgres indexes storage changes in the background. -- [Changed] Switch substrate-archive `LocalCallExecutor` with Substrate's `LocalCallExecutor` - -### Polkadot Archive - -- [Changed] Config file is now optional. Can configure polkadot archive entirely through environment variables. - - the environment variables that need to be set are `CHAIN_DATA_DB` and `DATABASE_URL`. -- [Changed] Polkadot archive will archive `polkadot` by default if the `--chain` CLI option is not passed. -- [Changed] remove `rpc_url` from the polkadot-archive TOML configuration file -- [Changed] All options in config file apart from `db_url`. - -## **[v0.4.0]** +# Changelog + +All notable changes for substrate-archive will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [v0.5.1] - 2021-05-06 +### Added +- Release checklist +### Changed +- Pinned to `substrate`/`polkadot` release v0.8.30 +- Keeping the `substrate-archive` and `substrate-archive-backend` crates's versions aligned. + +## [v0.5.0] - 2021-03-29 + +### Added +- `ArchiveBuilder` struct for constructing the indexer. + - returns a trait, `Archive` that manages the underlying actor runtime +- Archive now reads the `CHAIN_DATA_DB` environment variable if the path to the backend chain database is not passed directly. +- `max_block_load` to configure the maximum number of blocks loaded at once +- Two new options to the configuration relative to state tracing: + - `targets` for specifying runtime targets to trace in WASM + - `folder` where WASM blobs with tracing enabled are kept. + - More on state-tracing [here](https://github.com/paritytech/substrate-archive/wiki/6.\)-State-Tracing-&-Balance-Reconciliation) +- Runtime versions between blocks are cached, speeding up block indexing. +- Notification stream from Postgres indexes storage changes in the background. + +### Changed +- no longer need to instantiate a client and manually pass it to the Archive +- rename `run_with` to `drive` +- Archive now accepts a postgres URL instead of a postgres URL split into its parts. +- Archive will take Postgres URL from environment variable `DATABASE_URL` as a fallback. +- Use SQLX for migrations + - SQLX will create a new table in the database, `sqlx_migrations` or similiar. + - the older table, `__refinery_migrations` can be safely dropped +- Print out the Postgres URL at startup +- config is now separated into sections for readability. Migration is manual but looking at the new `archive.conf` in `polkadot-archive` or `node-template-archive` folders should help. +- Postgres SQL queries are now type checked +- Refactor file layout to `substrate-archive` and `substrate-archive-backend`. +- Decouple Database actors +- upgrade to SQLx 0.5.0 +- Overhaul of block indexing. Now uses a Iterator to only collect batches of blocks from the database, + taking advantage of the better reading performance of sequential data access. Gathering blocks by RPC is no longer done. +- speed up query for difference between the storage and blocks table - +- switch to a leaner actor framework, [xtra](https://github.com/Restioson/xtra) +- switch to a persistant background-task-queue for executing blocks, [coil](https://github.com/insipx/coil). +- batch inserts no longer starve postgres pool of connections +- Switch substrate-archive `LocalCallExecutor` with Substrate's `LocalCallExecutor` + +### Removed +- the last frame dependency, `frame-system`. Archive now relies only on generic traits defined in substrate-core. +- RPC URL. An RPC URL is no longer required to function. + +## [v0.4.0] - 2021-01-24 +### Added +- integrate database migrations into substrate-archive library +### Changed - Speed up Storage Indexing by re-executing blocks -- [internal] Remove some depdendencies for service and backend -- fixes for storage inserts - refactored error types -- integrate database migrations into substrate-archive library -## [v0.3.1] +### Removed +- dependencies for service and backend -- add node-template-archive +### Fixed +- storage inserts -## **[v0.3.0]** +## [v0.3.1] +### Added +- node-template-archive -- Use a rocksdb-backed substrate client instead of RPC for indexing +## [v0.3.0] +### Added - Create a CLI for indexing kusama - New PostgreSQL Schema - Actors to model dataflow + +### Changed +- Use a rocksdb-backed substrate client instead of RPC for indexing + diff --git a/bin/polkadot-archive/CHANGELOG.md b/bin/polkadot-archive/CHANGELOG.md new file mode 100644 index 00000000..698da21e --- /dev/null +++ b/bin/polkadot-archive/CHANGELOG.md @@ -0,0 +1,19 @@ +# Changelog + +All notable changes for polkadot-archive will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.2.3] +### Changed +- Config file is now optional. Can configure polkadot archive entirely through environment variables. + - the environment variables that need to be set are `CHAIN_DATA_DB` and `DATABASE_URL`. +- Polkadot archive will archive `polkadot` by default if the `--chain` CLI option is not passed. + +### Removed +- `rpc_url` from the polkadot-archive TOML configuration file + + From 2337b9a00ccc5c9d45a9b6f5361174a948333740 Mon Sep 17 00:00:00 2001 From: Andrew Plaza Date: Wed, 19 May 2021 23:58:10 -0400 Subject: [PATCH 4/7] Change workflow file paths & update RELEASE-CHECKLIST --- .github/workflows/release.yml | 37 +++++++++++++++-------------------- RELEASE-CHECKLIST.md | 33 ++++++++++++------------------- 2 files changed, 28 insertions(+), 42 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index de201a18..9373cf37 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,16 +5,10 @@ on: - release jobs: deploy: - # A strategy is used to define various build environments this job will run. - # - # To say it simple, this will create 3 separate independent jobs which will - # run on ubuntu, mac & windows. strategy: matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} - # Runs when the commit message contains "[Released]" - # if: "contains(github.event.head_commit.message, '[Released]')" steps: - uses: actions/checkout@v2 - name: Install Rust @@ -22,25 +16,27 @@ jobs: with: toolchain: stable profile: minimal - - name: Build - run: cargo build --release + + - name: Build Polkadot Archive + run: cargo build --release --manifest-path bin/polkadot-archive/Cargo.toml + + - name: Prepare binary + run: | + mkdir -p release + strip bin/polkadot-archive/target/release/polkadot-archive - name: Package for Debian if: matrix.os == 'ubuntu-latest' run: | cargo install cargo-deb - cargo deb - strip target/release/test_gh_actions - mkdir -p release - mv target/debian/*.deb ./release/test_gh_actions-linux.deb - tar -C ./target/release/ -czvf ./release/test_gh_actions-linux.tar.gz ./test_gh_actions + cargo deb --manifest-path ./bin/polkadot-archive/Cargo.toml + mv ./bin/polkadot-archive/target/debian/*.deb ./release/polkadot-archive-linux.deb + tar -C ./bin/polkadot-archive/target/release/ -czvf ./release/polkadot-archive-linux.tar.gz ./polkadot-archive - - name: Package pure binary + - name: Package just binary if: matrix.os == 'ubuntu-latest' run: | - strip target/release/test_gh_actions - mkdir -p release - mv target/release/test_gh_actions ./release/test_gh_actions + mv bin/polkadot-archive/target/release/polkadot-archive ./release/polkadot-archive - name: Get changelog id: changelog_reader @@ -48,18 +44,17 @@ jobs: with: validation_depth: 2 path: ./CHANGELOG.md - # This will draft a new release & will attach the binaries produced by the above outputs. - # You still need to publish this release though after job ends. + - name: Release uses: softprops/action-gh-release@v1 with: tag_name: ${{ steps.changelog_reader.outputs.version }} - release_name: Release ${{ steps.changelog_reader.outputs.version }} + name: Release ${{ steps.changelog_reader.outputs.version }} body: ${{ steps.changelog_reader.outputs.changes }} draft: true files: | ./release/*.tar.gz ./release/*.deb - ./release/test_gh_actions + ./release/polkadot_archive env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/RELEASE-CHECKLIST.md b/RELEASE-CHECKLIST.md index be71e748..d5c9ce4b 100644 --- a/RELEASE-CHECKLIST.md +++ b/RELEASE-CHECKLIST.md @@ -5,24 +5,15 @@ Here's how to make a new release of `substrate-archive`. ## Checklist 1. Make a new branch, name it whatever you want, but `vx.y.z-prep` is a good template -1. Update `Cargo.toml` in `substrate-archive` and `substrate-archive-backend` with the new version number. -1. Update all references to `substrate` crates and `polkadot` to their respective latest releases. -1. Update the `CHANGELOG` as specified [here](https://keepachangelog.com/en/1.0.0/). -1. Run all tests: `TEST_DATABASE_URL="postgres://localhost:5432/archive cargo test --all`. -1. Push the PR against the `release` branch. -1. Once reviewed, merge it to `release`. -1. Tag the `release` branch` with `git tag vx.y.z` and push the tags with `git push --tags` -1. Build a binary for debian, compatible with the current glibc version (`v2.31`) - 1. `docker run --rm -it debian:jessie` (requires at least 8Gb RAM) - 1. install required dependencies `apt-get update && apt-get -y install git curl gcc clang` - 1. install rust from rustup.rs: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y` - 1. reload the shell: `source $HOME/.cargo/env` - 1. build archive: `git clone https://github.com/paritytech/substrate-archive.git --branch release --single-branch && cd substrate-archive/bin/polkadot-archive && SKIP_WASM_BUILD=1 cargo build --release` - 1. keeping the container running, in a another terminal find the id of the docker container with `docker ps -a`, copy the binary to host with `docker cp $YOUR_CONTAINER_ID:/substrate-archive/bin/polkadot-archive/target/release/polkadot-archive .` -1. Prepare a new draft release in the github UI - 1. Upload the new binary - 1. Tag the new release against the **release** branch - 1. Write the release notes - 1. Get a review of the draft release from the team - 1. Publish the release -1. Signal to devops that there is a new release available +2. Update `Cargo.toml` in `substrate-archive` and `substrate-archive-backend` with the new version number. +3. Update all references to `substrate` crates and `polkadot` to their respective latest releases. +4. Update the `CHANGELOG` as specified [here](https://keepachangelog.com/en/1.0.0/). +5. Run all tests: `TEST_DATABASE_URL="postgres://localhost:5432/archive cargo test --all`. +6. Push the PR against the `release` branch. +7. Once reviewed, merge it to `release`. Upon merging, Github Actions will create a draft release and upload +binaries. +8. Tag the `release` branch` with `git tag vx.y.z` and push the tags with `git push --tags`. +9. Review the draft release in the github UI. +10. Get a review of the draft release from the team. +11. Publish the release from github UI. +12. Signal to devops that there is a new release available. From 5eef921d7317244bf05fa6f1c0ad6674e334da63 Mon Sep 17 00:00:00 2001 From: Andrew Plaza Date: Thu, 20 May 2021 00:05:48 -0400 Subject: [PATCH 5/7] fix spacing --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f47e9dc..0b9eb816 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [v0.5.1] - 2021-05-06 ### Added - Release checklist + ### Changed - Pinned to `substrate`/`polkadot` release v0.8.30 - Keeping the `substrate-archive` and `substrate-archive-backend` crates's versions aligned. ## [v0.5.0] - 2021-03-29 - ### Added - `ArchiveBuilder` struct for constructing the indexer. - returns a trait, `Archive` that manages the underlying actor runtime From 972a7af929bd1ab9bc19a89c200ebe55db45f261 Mon Sep 17 00:00:00 2001 From: Andrew Plaza Date: Thu, 20 May 2021 00:06:43 -0400 Subject: [PATCH 6/7] remove macos from os matrix --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9373cf37..a6c74774 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ jobs: deploy: strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 From 9466deddefa0ce380471907b5ce92f301b59b5e8 Mon Sep 17 00:00:00 2001 From: Andrew Plaza Date: Thu, 20 May 2021 11:09:28 -0400 Subject: [PATCH 7/7] Update RELEASE-CHECKLIST.md Co-authored-by: Tarik Gul <47201679+TarikGul@users.noreply.github.com> --- RELEASE-CHECKLIST.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE-CHECKLIST.md b/RELEASE-CHECKLIST.md index d5c9ce4b..0c58b56d 100644 --- a/RELEASE-CHECKLIST.md +++ b/RELEASE-CHECKLIST.md @@ -12,7 +12,7 @@ Here's how to make a new release of `substrate-archive`. 6. Push the PR against the `release` branch. 7. Once reviewed, merge it to `release`. Upon merging, Github Actions will create a draft release and upload binaries. -8. Tag the `release` branch` with `git tag vx.y.z` and push the tags with `git push --tags`. +8. Tag the `release` branch with `git tag vx.y.z` and push the tags with `git push --tags`. 9. Review the draft release in the github UI. 10. Get a review of the draft release from the team. 11. Publish the release from github UI.