diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..44d8173 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,7 @@ +version: 2.1 +orbs: + hello: circleci/hello-build@0.0.5 +workflows: + "Hello Workflow": + jobs: + - hello/hello-build diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5874b8c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.git +target diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..cb67232 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,7 @@ +**Do you want to request a *feature* or report a *bug*?** + +**What is the current behavior?** + +**If the current behavior is a bug, please provide the steps to reproduce and if possible a minimal demo of the problem.** + +**What is the expected behavior?** diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..e69de29 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..44043c3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,114 @@ +version: 2 +updates: +- package-ecosystem: cargo + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 10 + ignore: + - dependency-name: hyper + versions: + - ">= 0.14.a, < 0.15" + - dependency-name: jsonrpc-core + versions: + - ">= 12.a, < 13" + - dependency-name: postgres + versions: + - ">= 0.17.a, < 0.18" + - dependency-name: rand + versions: + - ">= 0.7.a, < 0.8" + - dependency-name: reqwest + versions: + - ">= 0.11.a, < 0.12" + - dependency-name: futures + versions: + - 0.3.14 + - dependency-name: wasmparser + versions: + - 0.77.0 + - dependency-name: serde + versions: + - 1.0.123 + - 1.0.124 + - 1.0.125 + - dependency-name: priority-queue + versions: + - 1.0.5 + - 1.1.1 + - dependency-name: rand + versions: + - 0.8.3 + - dependency-name: wasmtime + versions: + - 0.24.0 + - 0.25.0 + - dependency-name: syn + versions: + - 1.0.48 + - 1.0.62 + - dependency-name: num-bigint + versions: + - 0.4.0 + - dependency-name: postgres + versions: + - 0.19.0 + - dependency-name: ipfs-api + versions: + - 0.10.0 + - 0.11.0 + - dependency-name: backtrace + versions: + - 0.3.56 + - dependency-name: lru_time_cache + versions: + - 0.11.6 + - 0.11.7 + - 0.11.8 + - dependency-name: mockall + versions: + - 0.9.1 + - dependency-name: serde_json + versions: + - 1.0.62 + - 1.0.64 + - dependency-name: shellexpand + versions: + - 2.1.0 + - dependency-name: async-trait + versions: + - 0.1.42 + - dependency-name: toml + versions: + - 0.5.8 + - dependency-name: regex + versions: + - 1.4.3 + - dependency-name: uuid + versions: + - 0.8.2 + - dependency-name: thiserror + versions: + - 1.0.23 + - 1.0.24 + - dependency-name: http + versions: + - 0.2.3 + - dependency-name: env_logger + versions: + - 0.8.3 + - dependency-name: slog-term + versions: + - 2.8.0 + - dependency-name: jsonrpc-http-server + versions: + - 17.0.0 + - dependency-name: bytes + versions: + - 1.0.1 + - dependency-name: slog-async + versions: + - 2.6.0 + - dependency-name: jsonrpc-core + versions: + - 17.0.0 diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 0000000..f4fdaad --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,17 @@ +# See https://github.com/actions-rs/audit-check +name: Security audit +on: + # push: + # paths: + # - '**/Cargo.toml' + # - '**/Cargo.lock' + schedule: + - cron: '0 0 */7 * *' +jobs: + security_audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/audit-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..984ca99 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,301 @@ +name: Continuous Integration + +on: + push: + branches: [master] + pull_request: + types: [opened, synchronize, reopened] + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: full + THEGRAPH_STORE_POSTGRES_DIESEL_URL: "postgresql://postgres:postgres@localhost:5432/graph_node_test" + +jobs: + unit-tests: + name: Run unit tests + strategy: + fail-fast: false + matrix: + rust: ["stable"] + runs-on: ubuntu-latest + services: + ipfs: + image: ipfs/go-ipfs:v0.10.0 + ports: + - 5001:5001 + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: graph_node_test + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - name: Checkout sources + uses: actions/checkout@v2 + + - name: Cache cargo registry + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: test-cargo-${{ hashFiles('**/Cargo.toml') }} + + - name: Install rust toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + override: true + + - name: Install lld + run: sudo apt-get install -y lld + + - name: Run unit tests + uses: actions-rs/cargo@v1 + env: + RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" + with: + command: test + args: --verbose --workspace --exclude graph-tests -- --nocapture + + runner-tests: + name: Subgraph Runner integration tests + strategy: + fail-fast: false + matrix: + rust: ["stable"] + runs-on: ubuntu-latest + services: + ipfs: + image: ipfs/go-ipfs:v0.10.0 + ports: + - 5001:5001 + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: graph_node_test + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - name: Checkout sources + uses: actions/checkout@v2 + + - name: Cache cargo registry + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: test-cargo-${{ hashFiles('**/Cargo.toml') }} + + - name: Install rust toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + override: true + + - name: Install lld + run: sudo apt-get install -y lld + + - name: Run runner tests + id: runner-tests-1 + uses: actions-rs/cargo@v1 + env: + RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" + TESTS_GANACHE_HARD_WAIT_SECONDS: "30" + with: + command: test + args: --verbose --package graph-tests -- --skip parallel_integration_tests + + integration-tests: + name: Run integration tests + strategy: + fail-fast: false + matrix: + rust: ["stable"] + runs-on: ubuntu-latest + + steps: + - name: Checkout sources + uses: actions/checkout@v2 + + - name: Cache cargo registry + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: test-cargo-${{ hashFiles('**/Cargo.toml') }} + + - name: Install Node 14 + uses: actions/setup-node@v2 + with: + node-version: "14" + + - name: Install rust toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + override: true + + - name: Install lld and jq + run: sudo apt-get install -y lld jq + + - name: Build graph-node + env: + RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" + uses: actions-rs/cargo@v1 + with: + command: build + + # Integration tests are a bit flaky, running them twice increases the + # chances of one run succeeding + - name: Run integration tests (round 1) + id: integration-tests-1 + uses: actions-rs/cargo@v1 + env: + RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" + N_CONCURRENT_TESTS: "1" + TESTS_GANACHE_HARD_WAIT_SECONDS: "30" + with: + command: test + args: --verbose --package graph-tests parallel_integration_tests -- --nocapture + continue-on-error: true + - name: Run integration tests (round 2) + id: integration-tests-2 + uses: actions-rs/cargo@v1 + if: ${{ steps.integration-tests-1.outcome == 'failure' }} + env: + RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" + N_CONCURRENT_TESTS: "1" + TESTS_GANACHE_HARD_WAIT_SECONDS: "30" + with: + command: test + args: --verbose --package graph-tests parallel_integration_tests -- --nocapture + + rustfmt: + name: Check rustfmt style + strategy: + matrix: + rust: ["stable"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + components: rustfmt + override: true + + - name: Check formatting + uses: actions-rs/cargo@v1 + env: + RUSTFLAGS: "-D warnings" + with: + command: fmt + args: --all -- --check + + clippy: + name: Report Clippy warnings + strategy: + matrix: + rust: ["stable"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + components: clippy + override: true + # Unlike rustfmt, Clippy actually compiles stuff so it benefits from + # caching. + - name: Cache cargo registry + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: check-cargo-${{ hashFiles('**/Cargo.toml') }} + + - name: Run Clippy + uses: actions-rs/cargo@v1 + # We do *not* block builds if Clippy complains. It's just here to let us + # keep an eye out on the warnings it produces. + continue-on-error: true + with: + command: clippy + + release-check: + name: Build in release mode + strategy: + matrix: + rust: ["stable"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + override: true + - name: Cache cargo registry + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: check-cargo-${{ hashFiles('**/Cargo.toml') }} + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get -y install libpq-dev + + - name: Cargo check (debug) + uses: actions-rs/cargo@v1 + env: + RUSTFLAGS: "-D warnings" + with: + command: check + args: --tests + + - name: Cargo check (release) + env: + RUSTFLAGS: "-D warnings" + uses: actions-rs/cargo@v1 + with: + command: check + args: --release + + version-check: + name: Check that all graph-node crates have the same version + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Checks through all Cargo.toml files, making sure their version is unique + run: | + source 'scripts/toml-utils.sh' + + ALL_TOML_FILE_NAMES=$(get_all_toml_files) + ALL_TOML_VERSIONS=$(get_all_toml_versions $ALL_TOML_FILE_NAMES) + + ./scripts/lines-unique.sh $ALL_TOML_VERSIONS diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..874a814 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +# Generated by Cargo +# will have compiled files and executables +/target/ + +# These are backup files generated by rustfmt +**/*.rs.bk +**/*.idea +**/*.DS_STORE + +# IDE files +.vscode/ + +# Ignore data files created by running the Docker Compose setup locally +/docker/data/ +/docker/parity/chains/ +/docker/parity/network/ + +/tests/integration-tests/**/build +/tests/integration-tests/**/generated +/tests/integration-tests/**/node_modules +/tests/integration-tests/**/yarn.lock +/tests/integration-tests/**/yarn-error.log + +# Built solidity contracts. +/tests/integration-tests/**/bin +/tests/integration-tests/**/truffle_output diff --git a/.ignore b/.ignore new file mode 100644 index 0000000..c3517de --- /dev/null +++ b/.ignore @@ -0,0 +1,4 @@ +# Make `cargo watch` ignore changes in integration tests +tests/integration-tests/**/build +tests/integration-tests/**/generated +tests/integration-tests/**/node_modules diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..dcaabb1 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,87 @@ +dist: bionic +language: rust +# This line would cache cargo-audit once installed, +# but instead will fail from the 10 minute timeout after +# printing the line "creating directory /home/travis/.cache/sccache" +#cache: cargo +rust: + - stable + - beta + +# Select pre-installed services +addons: + postgresql: "10" + apt: + packages: + - postgresql-10 + - postgresql-client-10 +services: + - postgresql + - docker + +before_install: + # Install Node.js 11.x + - nvm install 11 && nvm use 11 + # Install IPFS + - wget "https://dist.ipfs.io/go-ipfs/v0.10.0/go-ipfs_v0.10.0_linux-amd64.tar.gz" -O /tmp/ipfs.tar.gz + - pushd . && cd $HOME/bin && tar -xzvf /tmp/ipfs.tar.gz && popd + - export PATH="$HOME/bin/go-ipfs:$PATH" + - ipfs init + +matrix: + fast_finish: true + include: + # Some env var is always necessary to differentiate included builds + # Check coding style + - env: CHECK_FORMATTING=true + rust: stable + script: + - rustup component add rustfmt + - cargo fmt --all -- --check + + # Make sure release builds compile + - env: CHECK_RELEASE=true + rust: stable + script: + - cargo check --release + + # Check for warnings + - env: RUSTFLAGS="-D warnings" + rust: stable + script: + - cargo check --tests + + # Build tagged commits in release mode + - env: RELEASE=true + if: tag IS present + script: + - cargo build -p graph-node --release + - mv target/release/graph-node target/release/graph-node-$TRAVIS_OS_NAME + +env: + global: + - PGPORT=5432 + - THEGRAPH_STORE_POSTGRES_DIESEL_URL=postgresql://travis:travis@localhost:5432/graph_node_test + # Added because https://nodejs.org/dist/ had issues + - NVM_NODEJS_ORG_MIRROR=https://cnpmjs.org/mirrors/node/ + +# Test pipeline +before_script: + - psql -c "ALTER USER travis WITH PASSWORD 'travis';" + - psql -c 'create database graph_node_test;' -U travis + +script: + # Run tests + - ipfs daemon &> /dev/null & + - RUST_BACKTRACE=1 cargo test --verbose --all -- --nocapture + - killall ipfs + +deploy: + provider: releases + api_key: + secure: ygpZedRG+/Qg/lPhifyNQ+4rExjZ4nGyJjB4DYT1fuePMyKXfiCPGicaWRGR3ZnZGNRjdKaIkF97vBsZ0aHwW+AykwOxlXrkAFvCKA0Tb82vaYqCLrBs/Y5AEhuCWLFDz5cXDPMkptf+uLX/s3JCF0Mxo5EBN2JfBQ8vS6ScKEwqn2TiLLBQKTQ4658TFM4H5KiXktpyVVdlRvpoS3pRIPMqNU/QpGPQigaiKyYD5+azCrAXeaKT9bBS1njVbxI69Go4nraWZn7wIhZCrwJ+MxGNTOxwasypsWm/u1umhRVLM1rL2i7RRqkIvzwn22YMaU7FZKCx8huXcj0cB8NtHZSw7GhJDDDv3e7puZxl3m/c/7ks76UF95syLzoM/9FWEFew8Ti+5MApzKQj5YWHOCIEzBWPeqAcA8Y+Az7w2h1ZgNbjDgSvjGAFSpE8m+SM0A2TOOZ1g/t/yfbEl8CWO6Y8v2x1EONkp7X0CqJgASMp+h8kzKCbuYyRnghlToY+5wYuh4M9Qg9UeJCt9dOblRBVJwW5CFr62kgE/gso8F9tXXHkRTv3hfk5madZR1Vn5A7KadEO8epfV4IQNsd+VHfoxoJSprx5f77Q2bLMBD1GT/qMqECgSznoTkU5ajkKJRqUw4AwLTohrYir76j61eQfxOhXExY/EM8xvlxpd1w= + file: target/release/graph-node-$TRAVIS_OS_NAME + repo: graphprotocol/graph-node + on: + tags: true + skip_cleanup: true diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..1fe0dd2 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,46 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at info@thegraph.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..aa67ac5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,96 @@ + +# Contributing to graph-node + +Welcome to the Graph Protocol! Thanks a ton for your interest in contributing. + +If you run into any problems feel free to create an issue. PRs are much appreciated for simple things. Here's [a list of good first issues](https://github.com/graphprotocol/graph-node/labels/good%20first%20issue). If it's something more complex we'd appreciate having a quick chat in GitHub Issues or Discord. + +Join the conversation on our [Discord](https://discord.gg/9a5VCua). + +Please follow the [Code of Conduct](https://github.com/graphprotocol/graph-node/blob/master/CODE_OF_CONDUCT.md) for all the communications and at events. Thank you! + +## Development flow + +Install development helpers: + +```sh +cargo install cargo-watch +rustup component add rustfmt-preview +``` + +Set environment variables: + +```sh +# Only required when testing the Diesel/Postgres store +export THEGRAPH_STORE_POSTGRES_DIESEL_URL= +``` + +- **Note** You can follow Docker Compose instructions in [store/test-store/README.md](./store/test-store/README.md#docker-compose) to easily run a Postgres instance and use `postgresql://graph:graph@127.0.0.1:5432/graph-test` as the Postgres database URL value. + +While developing, a useful command to run in the background is this: + +```sh +cargo watch \ + -x "fmt --all" \ + -x check \ + -x "test -- --test-threads=1" \ + -x "doc --no-deps" +``` + +This will watch your source directory and continuously do the following on changes: + +1. Build all packages in the workspace `target/`. +2. Generate docs for all packages in the workspace in `target/doc/`. +3. Automatically format all your source files. + +### Integrations Tests + +The tests can (and should) be run against a sharded store. See [store/test-store/README.md](./store/test-store/README.md) for +detailed instructions about how to run the sharded integrations tests. + +## Commit messages and pull requests + +We use the following format for commit messages: +`{crate-name}: {Brief description of changes}`, for example: `store: Support 'Or' filters`. + +If multiple crates are being changed list them all like this: `core, +graphql: Add event source to store` If all (or most) crates are affected +by the commit, start the message with `all: `. + +The body of the message can be terse, with just enough information to +explain what the commit does overall. In a lot of cases, more extensive +explanations of _how_ the commit achieves its goal are better as comments +in the code. + +Commits in a pull request should be structured in such a way that each +commit consists of a small logical step towards the overall goal of the +pull request. Your pull request should make it as easy as possible for the +reviewer to follow each change you are making. For example, it is a good +idea to separate simple mechanical changes like renaming a method that +touches many files from logic changes. Your pull request should not be +structured into commits according to how you implemented your feature, +often indicated by commit messages like 'Fix problem' or 'Cleanup'. Flex a +bit, and make the world think that you implemented your feature perfectly, +in small logical steps, in one sitting without ever having to touch up +something you did earlier in the pull request. (In reality, that means +you'll use `git rebase -i` a lot) + +Please do not merge master into your branch as you develop your pull +request; instead, rebase your branch on top of the latest master if your +pull request branch is long-lived. + +We try to keep the hostory of the `master` branch linear, and avoid merge +commits. Once your pull request is approved, merge it following these +steps: +``` +git checkout master +git pull master +git rebase master my/branch +git push -f +git checkout master +git merge my/branch +git push +``` + +Allegedly, clicking on the `Rebase and merge` button in the Github UI has +the same effect. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..3a10677 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,5353 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + +[[package]] +name = "addr2line" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a2e47a1fbe209ee101dd6d61285226744c6c8d3c21c8dc878ba6cb9f467f3a" +dependencies = [ + "gimli 0.24.0", +] + +[[package]] +name = "addr2line" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61f2b7f93d2c7d2b08263acaa4a363b3e276806c68af6134c44f523bf1aacd" +dependencies = [ + "gimli 0.25.0", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" + +[[package]] +name = "arc-swap" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e906254e445520903e7fc9da4f709886c84ae4bc4ddaf0e093188d66df4dc820" + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + +[[package]] +name = "ascii" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" + +[[package]] +name = "async-recursion" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cda8f4bcc10624c4e85bc66b3f452cca98cfa5ca002dc83a16aad2367641bea" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic_refcell" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b5e5f48b927f04e952dedc932f31995a65a0bf65ec971c74436e51bf6e970d" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "axum" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4af7447fc1214c1f3a1ace861d0216a6c8bb13965b64bbad9650f375b67689a" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa 1.0.1", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde", + "sync_wrapper", + "tokio", + "tower 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-http", + "tower-layer 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "axum-core" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bdc19781b16e32f8a7200368a336fa4509d4b72ef15dd4e41df5290855ee1e6" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", +] + +[[package]] +name = "backtrace" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a905d892734eea339e896738c14b9afce22b5318f64b951e70bf3844419b01" +dependencies = [ + "addr2line 0.16.0", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide 0.4.4", + "object 0.26.0", + "rustc-demangle", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "base64-url" +version = "1.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a99c239d0c7e77c85dddfa9cebce48704b3c49550fcd3b84dd637e4484899f" +dependencies = [ + "base64", +] + +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" +dependencies = [ + "serde", +] + +[[package]] +name = "bigdecimal" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1374191e2dd25f9ae02e3aa95041ed5d747fc77b3c102b49fe2dd9a8117a6244" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da1976d75adbe5fbc88130ecd119529cf1cc6a93ae1546d8696ee66f0d21af1" + +[[package]] +name = "bitvec" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1489fcb93a5bb47da0462ca93ad252ad6af2145cce58d10d46a83931ba9f016b" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "blake2b_simd" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72936ee4afc7f8f736d1c38383b56480b5497b4617b4a77bdbf1d2ababc76127" +dependencies = [ + "arrayref", + "arrayvec 0.7.2", + "constant_time_eq", +] + +[[package]] +name = "blake2s_simd" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db539cc2b5f6003621f1cd9ef92d7ded8ea5232c7de0f9faa2de251cd98730d4" +dependencies = [ + "arrayref", + "arrayvec 0.7.2", + "constant_time_eq", +] + +[[package]] +name = "blake3" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b64485778c4f16a6a5a9d335e80d449ac6c70cdd6a06d2af18a6f6f775a125b3" +dependencies = [ + "arrayref", + "arrayvec 0.5.2", + "cc", + "cfg-if 0.1.10", + "constant_time_eq", + "crypto-mac 0.8.0", + "digest 0.9.0", +] + +[[package]] +name = "blake3" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f" +dependencies = [ + "arrayref", + "arrayvec 0.7.2", + "cc", + "cfg-if 1.0.0", + "constant_time_eq", + "digest 0.10.3", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bollard" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "699194c00f3a2effd3358d47f880646818e3d483190b17ebcdf598c654fb77e9" +dependencies = [ + "base64", + "bollard-stubs", + "bytes", + "chrono", + "ct-logs", + "dirs-next", + "futures-core", + "futures-util", + "hex", + "http", + "hyper", + "hyper-unix-connector", + "log", + "pin-project", + "serde", + "serde_derive", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util 0.6.7", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2f2e73fffe9455141e170fb9c1feb0ac521ec7e7dcd47a7cab72a658490fb8" +dependencies = [ + "chrono", + "serde", + "serde_with", +] + +[[package]] +name = "bs58" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" + +[[package]] +name = "bstr" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90682c8d613ad3373e66de8c6411e0ae2ab2571e879d2efbf73558cc66f21279" +dependencies = [ + "memchr", +] + +[[package]] +name = "bumpalo" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" + +[[package]] +name = "byte-slice-cast" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c751592b77c499e7bce34d99d67c2c11bdc0574e9a488ddade14150a4698" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + +[[package]] +name = "cc" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-integer", + "num-traits", + "serde", + "time", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "cid" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ed9c8b2d17acb8110c46f1da5bf4a696d745e1474a16db0cd2b49cd0249bf2" +dependencies = [ + "core2", + "multibase", + "multihash", + "serde", + "unsigned-varint", +] + +[[package]] +name = "clap" +version = "3.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "indexmap", + "once_cell", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_derive" +version = "3.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" +dependencies = [ + "heck 0.4.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5538cd660450ebeb4234cfecf8f2284b844ffc4c50531e66d584ad5b91293613" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "cmake" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7b858541263efe664aead4a5209a4ae5c5d2811167d4ed4ee0944503f8d2089" +dependencies = [ + "cc", +] + +[[package]] +name = "combine" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" +dependencies = [ + "ascii", + "byteorder", + "either", + "memchr", + "unreachable", +] + +[[package]] +name = "console" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a50aab2529019abfabfa93f1e6c41ef392f91fbf179b347a7e96abb524884a08" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "regex", + "terminal_size", + "unicode-width", + "winapi", + "winapi-util", +] + +[[package]] +name = "const_fn_assert" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27d614f23f34f7b5165a77dc1591f497e2518f9cec4b4f4b92bfc4dc6cf7a190" + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + +[[package]] +name = "cpp_demangle" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea47428dc9d2237f3c6bc134472edfd63ebba0af932e783506dcfd66f10d18a" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "cpufeatures" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +dependencies = [ + "libc", +] + +[[package]] +name = "cranelift-bforest" +version = "0.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ca3560686e7c9c7ed7e0fe77469f2410ba5d7781b1acaa9adc8d8deea28e3e" +dependencies = [ + "cranelift-entity", +] + +[[package]] +name = "cranelift-codegen" +version = "0.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf9bf1ffffb6ce3d2e5ebc83549bd2436426c99b31cc550d521364cbe35d276" +dependencies = [ + "cranelift-bforest", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-entity", + "gimli 0.24.0", + "log", + "regalloc", + "serde", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cc21936a5a6d07e23849ffe83e5c1f6f50305c074f4b2970ca50c13bf55b821" +dependencies = [ + "cranelift-codegen-shared", + "cranelift-entity", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca5b6ffaa87560bebe69a5446449da18090b126037920b0c1c6d5945f72faf6b" +dependencies = [ + "serde", +] + +[[package]] +name = "cranelift-entity" +version = "0.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d6b4a8bef04f82e4296782646f733c641d09497df2fabf791323fefaa44c64c" +dependencies = [ + "serde", +] + +[[package]] +name = "cranelift-frontend" +version = "0.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b783b351f966fce33e3c03498cb116d16d97a8f9978164a60920bd0d3a99c" +dependencies = [ + "cranelift-codegen", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-native" +version = "0.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77c88d3dd48021ff1e37e978a00098524abd3513444ae252c08d37b310b3d2a" +dependencies = [ + "cranelift-codegen", + "target-lexicon", +] + +[[package]] +name = "cranelift-wasm" +version = "0.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edb6d408e2da77cdbbd65466298d44c86ae71c1785d2ab0d8657753cdb4d9d89" +dependencies = [ + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "itertools", + "log", + "serde", + "smallvec", + "thiserror", + "wasmparser", +] + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crossbeam" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c02a4d71819009c192cf4872265391563fd6a84c81ff2c0f2a7026ca4c1d85c" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b10ddc024425c88c2ad148c1b0fd53f4c6d38db9697c9f1588381212fa657c9" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +dependencies = [ + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "crypto-mac" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "ct-logs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +dependencies = [ + "sct 0.6.1", +] + +[[package]] +name = "ctor" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757c0ded2af11d8e739c4daea1ac623dd1624b06c844cf3f5a39f1bdbd99bb12" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c34d8efb62d0c2d7f60ece80f75e5c63c1588ba68032740494b0b9a996466e3" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade7bff147130fe5e6d39f089c6bd49ec0250f35d70b2eebf72afdfc919f15cc" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + +[[package]] +name = "data-encoding-macro" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86927b7cd2fe88fa698b87404b287ab98d1a0063a34071d92e575b72d3029aca" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5bbed42daaa95e780b60a50546aa345b8413a1e46f9a40a12907d3598f038db" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "defer" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "647605a6345d5e89c3950a36a638c56478af9b414c55c6f2477c73b115f9acde" + +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn", +] + +[[package]] +name = "diesel" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d" +dependencies = [ + "bigdecimal", + "bitflags", + "byteorder", + "chrono", + "diesel_derives", + "num-bigint", + "num-integer", + "num-traits", + "pq-sys", + "r2d2", + "serde_json", +] + +[[package]] +name = "diesel-derive-enum" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8910921b014e2af16298f006de12aa08af894b71f0f49a486ab6d74b17bbed" +dependencies = [ + "heck 0.4.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "diesel-dynamic-schema" +version = "1.0.0" +source = "git+https://github.com/diesel-rs/diesel-dynamic-schema?rev=a8ec4fb1#a8ec4fb11de6242488ba3698d74406f4b5073dc4" +dependencies = [ + "diesel", +] + +[[package]] +name = "diesel_derives" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "diesel_migrations" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c" +dependencies = [ + "migrations_internals", + "migrations_macros", +] + +[[package]] +name = "diff" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "digest" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +dependencies = [ + "block-buffer 0.10.2", + "crypto-common", + "subtle", +] + +[[package]] +name = "directories-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "either" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + +[[package]] +name = "encoding_rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime 1.3.0", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "env_logger" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" +dependencies = [ + "atty", + "humantime 2.1.0", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "envconfig" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea81cc7e21f55a9d9b1efb6816904978d0bfbe31a50347cb24b2e75564bcac9b" +dependencies = [ + "envconfig_derive", +] + +[[package]] +name = "envconfig_derive" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfca278e5f84b45519acaaff758ebfa01f18e96998bc24b8f1b722dd804b9bf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "errno" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68f2fb9cae9d37c9b2b3584aba698a2e97f72d7aef7b9f7aa71d8b54ce46fe" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14ca354e36190500e1e1fb267c647932382b54053c50b14970856c0b00a35067" +dependencies = [ + "gcc", + "libc", +] + +[[package]] +name = "ethabi" +version = "17.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b" +dependencies = [ + "ethereum-types", + "hex", + "once_cell", + "regex", + "serde", + "serde_json", + "sha3", + "thiserror", + "uint", +] + +[[package]] +name = "ethbloom" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef" +dependencies = [ + "crunchy", + "fixed-hash", + "impl-rlp", + "impl-serde", + "tiny-keccak 2.0.2", +] + +[[package]] +name = "ethereum-types" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6" +dependencies = [ + "ethbloom", + "fixed-hash", + "impl-rlp", + "impl-serde", + "primitive-types", + "uint", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "file-per-thread-logger" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fdbe0d94371f9ce939b555dd342d0686cc4c0cadbcd4b61d70af5ff97eb4126" +dependencies = [ + "env_logger 0.7.1", + "log", +] + +[[package]] +name = "firestorm" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31586bda1b136406162e381a3185a506cdfc1631708dd40cba2f6628d8634499" + +[[package]] +name = "firestorm" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3d6188b8804df28032815ea256b6955c9625c24da7525f387a7af02fbb8f01" + +[[package]] +name = "fixed-hash" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +dependencies = [ + "byteorder", + "rand", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "fixedbitset" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "398ea4fabe40b9b0d885340a2a991a44c8a645624075ad966d21f88688e2b69e" + +[[package]] +name = "flate2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +dependencies = [ + "crc32fast", + "miniz_oxide 0.5.3", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + +[[package]] +name = "futures" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1adc00f486adfc9ce99f77d717836f0c5aa84965eb0b4f051f4e83f7cab53f8b" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74ed2411805f6e4e3d9bc904c95d5d423b89b3b25dc0250aa74729de20629ff9" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af51b1b4a7fdff033703db39de8802c673eb91855f2e0d47dcf3bf2c0ef01f99" + +[[package]] +name = "futures-executor" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d0d535a57b87e1ae31437b892713aee90cd2d7b0ee48727cd11fc72ef54761c" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b0e06c393068f3a6ef246c75cdca793d6a46347e75286933e5e75fd2fd11582" + +[[package]] +name = "futures-macro" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c54913bae956fb8df7f4dc6fc90362aa72e69148e3f39041fbe8742d21e0ac57" +dependencies = [ + "autocfg", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f30aaa67363d119812743aa5f33c201a7a66329f97d1a887022971feea4b53" + +[[package]] +name = "futures-task" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe54a98670017f3be909561f6ad13e810d9a51f3f061b902062ca3da80799f2" + +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + +[[package]] +name = "futures-util" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eb846bfd58e44a8481a00049e82c43e0ccb5d61f8dc071057cb19249dd4d78" +dependencies = [ + "autocfg", + "futures 0.1.31", + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "gcc" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189" +dependencies = [ + "fallible-iterator", + "indexmap", + "stable_deref_trait", +] + +[[package]] +name = "gimli" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7" + +[[package]] +name = "git-testament" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "096cb9c8aa6f1924d079bf417f1d1685286958ff362fa58ae4d65a53ffec6c02" +dependencies = [ + "git-testament-derive", + "no-std-compat", +] + +[[package]] +name = "git-testament-derive" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45ceded7b01141664c3fc4a50199c408a6ed247e6c8415dc005e895f1d233374" +dependencies = [ + "chrono", + "log", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "globset" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10463d9ff00a2a068db14231982f5132edebad0d7660cd956a1c30292dbcbfbd" +dependencies = [ + "aho-corasick", + "bstr", + "fnv", + "log", + "regex", +] + +[[package]] +name = "graph" +version = "0.27.0" +dependencies = [ + "Inflector", + "anyhow", + "async-stream", + "async-trait", + "atomic_refcell", + "bigdecimal", + "bytes", + "chrono", + "cid", + "clap", + "diesel", + "diesel_derives", + "envconfig", + "ethabi", + "futures 0.1.31", + "futures 0.3.16", + "graphql-parser", + "hex", + "http", + "isatty", + "itertools", + "lazy_static", + "maplit", + "num-bigint", + "num-traits", + "num_cpus", + "parking_lot 0.12.1", + "petgraph", + "priority-queue", + "prometheus", + "prost", + "prost-types", + "rand", + "reqwest", + "semver", + "serde", + "serde_derive", + "serde_json", + "serde_plain", + "serde_yaml", + "slog", + "slog-async", + "slog-envlogger", + "slog-term", + "stable-hash 0.3.3", + "stable-hash 0.4.2", + "strum", + "strum_macros", + "test-store", + "thiserror", + "tiny-keccak 1.5.0", + "tokio", + "tokio-retry", + "tokio-stream", + "tonic", + "tonic-build", + "url", + "wasmparser", + "web3", +] + +[[package]] +name = "graph-chain-arweave" +version = "0.27.0" +dependencies = [ + "base64-url", + "diesel", + "graph", + "graph-runtime-derive", + "graph-runtime-wasm", + "prost", + "prost-types", + "serde", + "sha2 0.10.5", + "tonic-build", +] + +[[package]] +name = "graph-chain-common" +version = "0.27.0" +dependencies = [ + "anyhow", + "heck 0.4.0", + "protobuf 3.1.0", + "protobuf-parse", +] + +[[package]] +name = "graph-chain-cosmos" +version = "0.27.0" +dependencies = [ + "anyhow", + "graph", + "graph-chain-common", + "graph-runtime-derive", + "graph-runtime-wasm", + "prost", + "prost-types", + "semver", + "serde", + "tonic-build", +] + +[[package]] +name = "graph-chain-ethereum" +version = "0.27.0" +dependencies = [ + "anyhow", + "base64", + "dirs-next", + "envconfig", + "futures 0.1.31", + "graph", + "graph-runtime-derive", + "graph-runtime-wasm", + "hex", + "http", + "itertools", + "jsonrpc-core", + "lazy_static", + "prost", + "prost-types", + "semver", + "serde", + "test-store", + "tiny-keccak 1.5.0", + "tonic-build", +] + +[[package]] +name = "graph-chain-near" +version = "0.27.0" +dependencies = [ + "base64", + "diesel", + "graph", + "graph-runtime-derive", + "graph-runtime-wasm", + "prost", + "prost-types", + "serde", + "tonic-build", +] + +[[package]] +name = "graph-chain-substreams" +version = "0.27.0" +dependencies = [ + "anyhow", + "async-stream", + "dirs-next", + "envconfig", + "futures 0.1.31", + "graph", + "graph-core", + "graph-runtime-wasm", + "hex", + "http", + "itertools", + "jsonrpc-core", + "lazy_static", + "prost", + "prost-types", + "semver", + "serde", + "tiny-keccak 1.5.0", + "tokio", + "tonic-build", +] + +[[package]] +name = "graph-core" +version = "0.27.0" +dependencies = [ + "anyhow", + "async-stream", + "async-trait", + "atomic_refcell", + "bytes", + "cid", + "futures 0.1.31", + "futures 0.3.16", + "graph", + "graph-chain-arweave", + "graph-chain-cosmos", + "graph-chain-ethereum", + "graph-chain-near", + "graph-chain-substreams", + "graph-mock", + "graph-runtime-wasm", + "graphql-parser", + "hex", + "lazy_static", + "lru_time_cache", + "pretty_assertions", + "semver", + "serde", + "serde_json", + "serde_yaml", + "test-store", + "tower 0.4.12 (git+https://github.com/tower-rs/tower.git)", + "tower-test", +] + +[[package]] +name = "graph-graphql" +version = "0.27.0" +dependencies = [ + "Inflector", + "anyhow", + "async-recursion", + "crossbeam", + "defer", + "graph", + "graph-chain-ethereum", + "graphql-parser", + "graphql-tools", + "indexmap", + "lazy_static", + "parking_lot 0.12.1", + "pretty_assertions", + "stable-hash 0.3.3", + "stable-hash 0.4.2", + "test-store", +] + +[[package]] +name = "graph-mock" +version = "0.27.0" +dependencies = [ + "graph", +] + +[[package]] +name = "graph-node" +version = "0.27.0" +dependencies = [ + "clap", + "crossbeam-channel", + "diesel", + "env_logger 0.9.0", + "futures 0.3.16", + "git-testament", + "graph", + "graph-chain-arweave", + "graph-chain-cosmos", + "graph-chain-ethereum", + "graph-chain-near", + "graph-chain-substreams", + "graph-core", + "graph-graphql", + "graph-runtime-wasm", + "graph-server-http", + "graph-server-index-node", + "graph-server-json-rpc", + "graph-server-metrics", + "graph-server-websocket", + "graph-store-postgres", + "graphql-parser", + "http", + "json-structural-diff", + "lazy_static", + "prometheus", + "regex", + "serde", + "serde_regex", + "shellexpand", + "toml", + "url", +] + +[[package]] +name = "graph-runtime-derive" +version = "0.27.0" +dependencies = [ + "heck 0.4.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "graph-runtime-test" +version = "0.27.0" +dependencies = [ + "graph", + "graph-chain-ethereum", + "graph-core", + "graph-mock", + "graph-runtime-derive", + "graph-runtime-wasm", + "rand", + "semver", + "test-store", + "wasmtime", +] + +[[package]] +name = "graph-runtime-wasm" +version = "0.27.0" +dependencies = [ + "anyhow", + "async-trait", + "atomic_refcell", + "bs58", + "bytes", + "defer", + "ethabi", + "futures 0.1.31", + "graph", + "graph-runtime-derive", + "hex", + "lazy_static", + "never", + "parity-wasm", + "semver", + "strum", + "strum_macros", + "uuid 1.1.2", + "wasm-instrument", + "wasmtime", +] + +[[package]] +name = "graph-server-http" +version = "0.27.0" +dependencies = [ + "futures 0.1.31", + "graph", + "graph-graphql", + "graph-mock", + "graphql-parser", + "http", + "hyper", + "serde", +] + +[[package]] +name = "graph-server-index-node" +version = "0.27.0" +dependencies = [ + "blake3 1.3.1", + "either", + "futures 0.3.16", + "graph", + "graph-chain-arweave", + "graph-chain-cosmos", + "graph-chain-ethereum", + "graph-chain-near", + "graph-graphql", + "graphql-parser", + "http", + "hyper", + "lazy_static", + "serde", +] + +[[package]] +name = "graph-server-json-rpc" +version = "0.27.0" +dependencies = [ + "graph", + "jsonrpsee", + "serde", +] + +[[package]] +name = "graph-server-metrics" +version = "0.27.0" +dependencies = [ + "graph", + "http", + "hyper", + "lazy_static", + "serde", +] + +[[package]] +name = "graph-server-websocket" +version = "0.27.0" +dependencies = [ + "anyhow", + "futures 0.1.31", + "graph", + "graphql-parser", + "http", + "lazy_static", + "serde", + "serde_derive", + "tokio-tungstenite", + "uuid 0.8.2", +] + +[[package]] +name = "graph-store-postgres" +version = "0.27.0" +dependencies = [ + "Inflector", + "anyhow", + "async-trait", + "blake3 1.3.1", + "clap", + "derive_more", + "diesel", + "diesel-derive-enum", + "diesel-dynamic-schema", + "diesel_derives", + "diesel_migrations", + "fallible-iterator", + "futures 0.3.16", + "git-testament", + "graph", + "graph-chain-ethereum", + "graph-graphql", + "graph-mock", + "graphql-parser", + "hex", + "hex-literal", + "itertools", + "lazy_static", + "lru_time_cache", + "maybe-owned", + "openssl", + "pin-utils", + "postgres", + "postgres-openssl", + "rand", + "serde", + "stable-hash 0.3.3", + "test-store", + "uuid 1.1.2", +] + +[[package]] +name = "graph-tests" +version = "0.27.0" +dependencies = [ + "anyhow", + "async-stream", + "bollard", + "cid", + "futures 0.3.16", + "graph", + "graph-chain-ethereum", + "graph-chain-near", + "graph-core", + "graph-graphql", + "graph-mock", + "graph-node", + "graph-store-postgres", + "graphql-parser", + "hex", + "lazy_static", + "port_check", + "serde_yaml", + "slog", + "tokio", + "tokio-stream", +] + +[[package]] +name = "graphql-parser" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ebc8013b4426d5b81a4364c419a95ed0b404af2b82e2457de52d9348f0e474" +dependencies = [ + "combine", + "thiserror", +] + +[[package]] +name = "graphql-tools" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a71c3ac880d8383537914ea7b45d99c6946e15976faa33a42b6c339ef4a2fb8" +dependencies = [ + "graphql-parser", + "lazy_static", + "serde", + "serde_json", +] + +[[package]] +name = "h2" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util 0.7.1", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3" + +[[package]] +name = "headers" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c4eb0471fcb85846d8b0690695ef354f9afb11cb03cac2e1d7c9253351afb0" +dependencies = [ + "base64", + "bitflags", + "bytes", + "headers-core", + "http", + "httpdate", + "mime", + "sha-1", +] + +[[package]] +name = "headers-core" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" +dependencies = [ + "http", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hex-literal" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" + +[[package]] +name = "hmac" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" +dependencies = [ + "crypto-mac 0.10.1", + "digest 0.9.0", +] + +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +dependencies = [ + "bytes", + "fnv", + "itoa 1.0.1", +] + +[[package]] +name = "http-body" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + +[[package]] +name = "httparse" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" + +[[package]] +name = "httpdate" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa 1.0.1", + "pin-project-lite", + "socket2", + "tokio", + "tower-service 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "hyper-unix-connector" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24ef1fd95d34b4ff007d3f0590727b5cf33572cace09b42032fc817dc8b16557" +dependencies = [ + "anyhow", + "hex", + "hyper", + "pin-project", + "tokio", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c495f162af0bf17656d0014a0eded5f3cd2f365fdd204548c2869db89359dc7" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "js-sys", + "once_cell", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "ibig" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c5022ee7f7a2feb0bd2fdc4b8ec882cd14903cebf33e7c1847e3f3a282f8b7" +dependencies = [ + "cfg-if 1.0.0", + "const_fn_assert", + "num-traits", + "rand", + "static_assertions", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "impl-codec" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-rlp" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28220f89297a075ddc7245cd538076ee98b01f2a9c23a53a4f1105d5a322808" +dependencies = [ + "rlp", +] + +[[package]] +name = "impl-serde" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4551f042f3438e64dbd6226b20527fc84a6e1fe65688b58746a2f53623f25f5c" +dependencies = [ + "serde", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5dacb10c5b3bb92d46ba347505a9041e676bb20ad220101326bffb0c93031ee" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "indexmap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +dependencies = [ + "autocfg", + "hashbrown", + "serde", +] + +[[package]] +name = "input_buffer" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f97967975f448f1a7ddb12b0bc41069d09ed6a1c161a92687e057325db35d413" +dependencies = [ + "bytes", +] + +[[package]] +name = "instant" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "ipnet" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" + +[[package]] +name = "isatty" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31a8281fc93ec9693494da65fbf28c0c2aa60a2eaec25dc58e2f31952e95edc" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "redox_syscall 0.1.57", + "winapi", +] + +[[package]] +name = "itertools" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + +[[package]] +name = "jobserver" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5ca711fd837261e14ec9e674f092cbb931d3fa1482b017ae59328ddc6f3212b" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json-structural-diff" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25c7940d3c84d2079306c176c7b2b37622b6bc5e43fbd1541b1e4a4e1fd02045" +dependencies = [ + "console", + "difflib", + "regex", + "serde_json", +] + +[[package]] +name = "jsonrpc-core" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" +dependencies = [ + "futures 0.3.16", + "futures-executor", + "futures-util", + "log", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "jsonrpsee" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bd0d559d5e679b1ab2f869b486a11182923863b1b3ee8b421763cdd707b783a" +dependencies = [ + "jsonrpsee-core", + "jsonrpsee-http-server", + "jsonrpsee-types", +] + +[[package]] +name = "jsonrpsee-core" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3dc3e9cf2ba50b7b1d7d76a667619f82846caa39e8e8daa8a4962d74acaddca" +dependencies = [ + "anyhow", + "arrayvec 0.7.2", + "async-trait", + "beef", + "futures-channel", + "futures-util", + "globset", + "http", + "hyper", + "jsonrpsee-types", + "lazy_static", + "parking_lot 0.12.1", + "rand", + "rustc-hash", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", + "unicase", +] + +[[package]] +name = "jsonrpsee-http-server" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03802f0373a38c2420c70b5144742d800b509e2937edc4afb116434f07120117" +dependencies = [ + "futures-channel", + "futures-util", + "hyper", + "jsonrpsee-core", + "jsonrpsee-types", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-futures", +] + +[[package]] +name = "jsonrpsee-types" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e290bba767401b646812f608c099b922d8142603c9e73a50fb192d3ac86f4a0d" +dependencies = [ + "anyhow", + "beef", + "serde", + "serde_json", + "thiserror", + "tracing", +] + +[[package]] +name = "keccak" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "leb128" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" + +[[package]] +name = "libc" +version = "0.2.131" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c3b4822ccebfa39c02fc03d1534441b22ead323fa0f48bb7ddd8e6ba076a40" + +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" + +[[package]] +name = "lock_api" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "lru_time_cache" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd" + +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "matchit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" + +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + +[[package]] +name = "md-5" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +dependencies = [ + "block-buffer 0.9.0", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memoffset" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +dependencies = [ + "autocfg", +] + +[[package]] +name = "migrations_internals" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860" +dependencies = [ + "diesel", +] + +[[package]] +name = "migrations_macros" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" +dependencies = [ + "migrations_internals", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "miniz_oxide" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + +[[package]] +name = "more-asserts" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0debeb9fcf88823ea64d64e4a815ab1643f33127d995978e099942ce38f25238" + +[[package]] +name = "multibase" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" +dependencies = [ + "base-x", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3db354f401db558759dfc1e568d010a5d4146f4d3f637be1275ec4a3cf09689" +dependencies = [ + "blake2b_simd", + "blake2s_simd", + "blake3 1.3.1", + "core2", + "digest 0.10.3", + "multihash-derive", + "sha2 0.10.5", + "sha3", + "unsigned-varint", +] + +[[package]] +name = "multihash-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc076939022111618a5026d3be019fd8b366e76314538ff9a1b59ffbcbf98bcd" +dependencies = [ + "proc-macro-crate", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "native-tls" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "never" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91" + +[[package]] +name = "no-std-compat" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5b3dd1c072ee7963717671d1ca129f1048fda25edea6b752bfc71ac8854170" +dependencies = [ + "crc32fast", + "indexmap", +] + +[[package]] +name = "object" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c55827317fb4c08822499848a14237d2874d6f139828893017237e7ab93eb386" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl" +version = "0.10.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" + +[[package]] +name = "openssl-sys" +version = "0.9.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "os_str_bytes" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" + +[[package]] +name = "output_vt100" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9" +dependencies = [ + "winapi", +] + +[[package]] +name = "parity-scale-codec" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a7f3fcf5e45fc28b84dcdab6b983e77f197ec01f325a33f404ba6855afd1070" +dependencies = [ + "arrayvec 0.7.2", + "bitvec", + "byte-slice-cast", + "impl-trait-for-tuples", + "parity-scale-codec-derive", + "serde", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6e626dc84025ff56bf1476ed0e30d10c84d7f89a475ef46ebabee1095a8fba" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "parity-wasm" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1ad0aff30c1da14b1254fcb2af73e1fa9a28670e584a626f53a369d0e157304" + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.5", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.1", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.2.10", + "smallvec", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall 0.2.10", + "smallvec", + "windows-sys", +] + +[[package]] +name = "paste" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "petgraph" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "phf" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "port_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6519412c9e0d4be579b9f0618364d19cb434b324fc6ddb1b27b1e682c7105ed" + +[[package]] +name = "postgres" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7871ee579860d8183f542e387b176a25f2656b9fb5211e045397f745a68d1c2" +dependencies = [ + "bytes", + "fallible-iterator", + "futures 0.3.16", + "log", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "postgres-openssl" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1de0ea6504e07ca78355a6fb88ad0f36cafe9e696cbc6717f16a207f3a60be72" +dependencies = [ + "futures 0.3.16", + "openssl", + "tokio", + "tokio-openssl", + "tokio-postgres", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff3e0f70d32e20923cabf2df02913be7c1842d4c772db8065c00fcfdd1d1bff3" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand", + "sha2 0.9.5", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "430f4131e1b7657b0cd9a2b0c3408d77c9a43a042d300b8c77f981dffcc43a2f" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "pq-sys" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac25eee5a0582f45a67e837e350d784e7003bd29a5f460796772061ca49ffda" +dependencies = [ + "vcpkg", +] + +[[package]] +name = "pretty_assertions" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a25e9bcb20aa780fd0bb16b72403a9064d6b3f22f026946029acb941a50af755" +dependencies = [ + "ctor", + "diff", + "output_vt100", + "yansi", +] + +[[package]] +name = "prettyplease" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9e07e3a46d0771a8a06b5f4441527802830b43e679ba12f44960f48dd4c6803" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primitive-types" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" +dependencies = [ + "fixed-hash", + "impl-codec", + "impl-rlp", + "impl-serde", + "uint", +] + +[[package]] +name = "priority-queue" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a03dfae8e64d4aa651415e2a4321f9f09f2e388a2f8bec36bed03bc22c0b687" +dependencies = [ + "indexmap", + "take_mut", +] + +[[package]] +name = "proc-macro-crate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebace6889caf889b4d3f76becee12e90353f2b8c7d875534a71e5742f8f6f83" +dependencies = [ + "thiserror", + "toml", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" + +[[package]] +name = "proc-macro2" +version = "1.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prometheus" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45c8babc29389186697fe5a2a4859d697825496b83db5d0b65271cdc0488e88c" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "libc", + "memchr", + "parking_lot 0.12.1", + "protobuf 2.25.0", + "reqwest", + "thiserror", +] + +[[package]] +name = "prost" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71adf41db68aa0daaefc69bb30bcd68ded9b9abaad5d1fbb6304c4fb390e083e" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120fbe7988713f39d780a58cf1a7ef0d7ef66c6d87e5aa3438940c05357929f4" +dependencies = [ + "bytes", + "cfg-if 1.0.0", + "cmake", + "heck 0.4.0", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "regex", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68" +dependencies = [ + "bytes", + "prost", +] + +[[package]] +name = "protobuf" +version = "2.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020f86b07722c5c4291f7c723eac4676b3892d47d9a7708dc2779696407f039b" + +[[package]] +name = "protobuf" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee4a7d8b91800c8f167a6268d1a1026607368e1adc84e98fe044aeb905302f7" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror", +] + +[[package]] +name = "protobuf-parse" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b1447dd751c434cc1b415579837ebd0411ed7d67d465f38010da5d7cd33af4d" +dependencies = [ + "anyhow", + "indexmap", + "log", + "protobuf 3.1.0", + "protobuf-support", + "tempfile", + "thiserror", + "which", +] + +[[package]] +name = "protobuf-support" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ca157fe12fc7ee2e315f2f735e27df41b3d97cdd70ea112824dac1ffb08ee1c" +dependencies = [ + "thiserror", +] + +[[package]] +name = "psm" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14ce37fa8c0428a37307d163292add09b3aedc003472e6b3622486878404191d" +dependencies = [ + "cc", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r2d2" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +dependencies = [ + "log", + "parking_lot 0.11.2", + "scheduled-thread-pool", +] + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_syscall" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +dependencies = [ + "getrandom", + "redox_syscall 0.2.10", +] + +[[package]] +name = "regalloc" +version = "0.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "571f7f397d61c4755285cd37853fe8e03271c243424a907415909379659381c5" +dependencies = [ + "log", + "rustc-hash", + "serde", + "smallvec", +] + +[[package]] +name = "regex" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "region" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877e54ea2adcd70d80e9179344c97f93ef0dffd6b03e1f4529e6e83ab2fa9ae0" +dependencies = [ + "bitflags", + "libc", + "mach", + "winapi", +] + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "mime_guess", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rlp" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "999508abb0ae792aabed2460c45b89106d97fe4adac593bdaef433c2605847b5" +dependencies = [ + "bytes", + "rustc-hex", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dead70b0b5e03e9c814bcb6b01e03e68f7c57a80aa48c72ec92152ab3e818d49" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hex" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fbfeb8d0ddb84706bc597a5574ab8912817c52a397f819e5b614e2265206921" +dependencies = [ + "log", + "ring", + "sct 0.7.0", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.0", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360" +dependencies = [ + "base64", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7522c9de787ff061458fe9a829dc790a3f5b22dc571694fc5883f448b94d9a9" +dependencies = [ + "base64", +] + +[[package]] +name = "rustversion" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61b3909d758bb75c79f23d4736fac9433868679d3ad2ea7a61e3c25cfda9a088" + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + +[[package]] +name = "scheduled-thread-pool" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +dependencies = [ + "parking_lot 0.11.2", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "scroll" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda28d4b4830b807a8b43f7b0e6b5df875311b3e7621d84577188c175b6ec1ec" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaaae8f38bb311444cfb7f1979af0bc9240d95795f75f9ceddf6a59b79ceffa0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "secp256k1" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c42e6f1735c5f00f51e43e28d6634141f2bcad10931b2609ddd74a86d751260" +dependencies = [ + "secp256k1-sys", +] + +[[package]] +name = "secp256k1-sys" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957da2573cde917463ece3570eab4a0b3f19de6f1646cde62e6fd3868f566036" +dependencies = [ + "cc", +] + +[[package]] +name = "security-framework" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1" +dependencies = [ + "serde", +] + +[[package]] +name = "serde" +version = "1.0.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f03b9878abf6d14e6779d3f24f07b2cfa90352cfec4acc5aab8f1ac7f146fae8" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a024926d3432516606328597e0f224a51355a493b49fdd67e9209187cbe55ecc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "336b10da19a12ad094b59d870ebde26a45402e5b470add4b5fd03c5048a32127" +dependencies = [ + "itoa 0.4.7", + "ryu", + "serde", +] + +[[package]] +name = "serde_plain" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95455e7e29fada2052e72170af226fbe368a4ca33dee847875325d9fdb133858" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_regex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8136f1a4ea815d7eac4101cfd0b16dc0cb5e1fe1b8609dfd728058656b7badf" +dependencies = [ + "regex", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa 0.4.7", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad9fdbb69badc8916db738c25efd04f0a65297d26c2f8de4b62e57b8c12bc72" +dependencies = [ + "rustversion", + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1569374bd54623ec8bd592cf22ba6e03c0f177ff55fbc8c29a49e296e7adecf" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_yaml" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +dependencies = [ + "indexmap", + "ryu", + "serde", + "yaml-rust", +] + +[[package]] +name = "sha-1" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a0c8611594e2ab4ebbf06ec7cbbf0a99450b8570e96cbf5188b5d5f6ef18d81" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpufeatures 0.1.5", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "sha2" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpufeatures 0.1.5", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "sha2" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9db03534dff993187064c4e0c05a5708d2a9728ace9a8959b77bedf415dac5" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures 0.2.2", + "digest 0.10.3", +] + +[[package]] +name = "sha3" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "881bf8156c87b6301fc5ca6b27f11eeb2761224c7081e69b409d5a1951a70c86" +dependencies = [ + "digest 0.10.3", + "keccak", +] + +[[package]] +name = "shellexpand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83bdb7831b2d85ddf4a7b148aa19d0587eddbe8671a436b7bd1182eaad0f2829" +dependencies = [ + "dirs-next", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729a25c17d72b06c68cb47955d44fda88ad2d3e7d77e025663fdd69b93dd71a1" + +[[package]] +name = "slab" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" + +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "slog-async" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "766c59b252e62a34651412870ff55d8c4e6d04df19b43eecb2703e417b097ffe" +dependencies = [ + "crossbeam-channel", + "slog", + "take_mut", + "thread_local", +] + +[[package]] +name = "slog-envlogger" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "906a1a0bc43fed692df4b82a5e2fbfc3733db8dad8bb514ab27a4f23ad04f5c0" +dependencies = [ + "log", + "regex", + "slog", + "slog-async", + "slog-scope", + "slog-stdlog", + "slog-term", +] + +[[package]] +name = "slog-scope" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786" +dependencies = [ + "arc-swap", + "lazy_static", + "slog", +] + +[[package]] +name = "slog-stdlog" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6706b2ace5bbae7291d3f8d2473e2bfab073ccd7d03670946197aec98471fa3e" +dependencies = [ + "log", + "slog", + "slog-scope", +] + +[[package]] +name = "slog-term" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95c1e7e5aab61ced6006149ea772770b84a0d16ce0f7885def313e4829946d76" +dependencies = [ + "atty", + "chrono", + "slog", + "term", + "thread_local", +] + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "socket2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765f090f0e423d2b55843402a07915add955e7d60657db13707a159727326cad" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "soketto" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" +dependencies = [ + "base64", + "bytes", + "futures 0.3.16", + "httparse", + "log", + "rand", + "sha-1", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "stable-hash" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10196e68950ed99c0d2db7a30ffaf4dfe0bbf2f9af2ae0457ee8ad396e0a2dd7" +dependencies = [ + "blake3 0.3.8", + "firestorm 0.4.6", + "ibig", + "lazy_static", + "leb128", + "num-traits", +] + +[[package]] +name = "stable-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af75bd21beb162eab69de76abbb803d4111735ead00d5086dcc6f4ddb3b53cc9" +dependencies = [ + "blake3 0.3.8", + "firestorm 0.5.0", + "ibig", + "lazy_static", + "leb128", + "num-traits", + "xxhash-rust", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "stringprep" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strum" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf86bbcfd1fa9670b7a129f64fc0c9fcbbfe4f1bc4210e9e98fe71ffc12cde2" + +[[package]] +name = "strum_macros" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d06aaeeee809dbc59eb4556183dd927df67db1540de5be8d3ec0b6636358a5ec" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] +name = "syn" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" + +[[package]] +name = "synstructure" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "474aaa926faa1603c40b7885a9eaea29b444d1cb2850cb7c0e37bb1a4182f4fa" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "target-lexicon" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0652da4c4121005e9ed22b79f6c5f2d9e2752906b53a33e9490489ba421a6fb" + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "rand", + "redox_syscall 0.2.10", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "test-store" +version = "0.27.0" +dependencies = [ + "diesel", + "graph", + "graph-graphql", + "graph-mock", + "graph-node", + "graph-store-postgres", + "graphql-parser", + "hex-literal", + "lazy_static", + "serde", +] + +[[package]] +name = "textwrap" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" + +[[package]] +name = "thiserror" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi", + "winapi", +] + +[[package]] +name = "tiny-keccak" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d8a021c69bb74a44ccedb824a046447e2c84a01df9e5c20779750acb38e11b2" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinyvec" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "848a1e1181b9f6753b5e96a092749e29b11d19ede67dfbbd6c7dc7e0f49b5338" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c27a64b625de6d309e8c57716ba93021dccf1b3b5c97edd6d3dd2d2135afc0a" +dependencies = [ + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "parking_lot 0.11.2", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90c49f106be240de154571dd31fbe48acb10ba6c6dd6f6517ad603abffa42de9" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-openssl" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a" +dependencies = [ + "futures-util", + "openssl", + "openssl-sys", + "tokio", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2b1383c7e4fb9a09e292c7c6afb7da54418d53b045f1c1fac7a911411a2b8b" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures 0.3.16", + "log", + "parking_lot 0.11.2", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "socket2", + "tokio", + "tokio-util 0.6.7", +] + +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4151fda0cf2798550ad0b34bcfc9b9dcc2a9d2471c895c68f3a8818e54f2389e" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-stream" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util 0.7.1", +] + +[[package]] +name = "tokio-test" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53474327ae5e166530d17f2d956afcb4f8a004de581b3cae10f12006bc8163e3" +dependencies = [ + "async-stream", + "bytes", + "futures-core", + "tokio", + "tokio-stream", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e96bb520beab540ab664bd5a9cfeaa1fcd846fa68c830b42e2c8963071251d2" +dependencies = [ + "futures-util", + "log", + "pin-project", + "tokio", + "tungstenite", +] + +[[package]] +name = "tokio-util" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + +[[package]] +name = "tonic" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30fb54bf1e446f44d870d260d99957e7d11fb9d0a0f5bd1a662ad1411cc103f9" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "flate2", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "prost-derive", + "rustls-native-certs", + "rustls-pemfile 0.3.0", + "tokio", + "tokio-rustls", + "tokio-stream", + "tokio-util 0.7.1", + "tower 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-layer 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9263bf4c9bfaae7317c1c2faf7f18491d2fe476f70c414b73bf5d445b00ffa1" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util 0.7.1", + "tower-layer 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.12" +source = "git+https://github.com/tower-rs/tower.git#ee826286fd1f994eabf14229e1e579ae29237386" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "tokio", + "tokio-util 0.7.1", + "tower-layer 0.3.1 (git+https://github.com/tower-rs/tower.git)", + "tower-service 0.3.1 (git+https://github.com/tower-rs/tower.git)", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e980386f06883cf4d0578d6c9178c81f68b45d77d00f2c2c1bc034b3439c2c56" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-layer 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tower-layer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" + +[[package]] +name = "tower-layer" +version = "0.3.1" +source = "git+https://github.com/tower-rs/tower.git#ee826286fd1f994eabf14229e1e579ae29237386" + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "git+https://github.com/tower-rs/tower.git#ee826286fd1f994eabf14229e1e579ae29237386" + +[[package]] +name = "tower-test" +version = "0.4.0" +source = "git+https://github.com/tower-rs/tower.git#ee826286fd1f994eabf14229e1e579ae29237386" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", + "tokio-test", + "tower-layer 0.3.1 (git+https://github.com/tower-rs/tower.git)", + "tower-service 0.3.1 (git+https://github.com/tower-rs/tower.git)", +] + +[[package]] +name = "tracing" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +dependencies = [ + "cfg-if 1.0.0", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "tungstenite" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fe8dada8c1a3aeca77d6b51a4f1314e0f4b8e438b7b1b71e3ddaca8080e4093" +dependencies = [ + "base64", + "byteorder", + "bytes", + "http", + "httparse", + "input_buffer", + "log", + "rand", + "sha-1", + "thiserror", + "url", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "uint" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6470ab50f482bde894a037a57064480a246dbfdd5960bd65a44824693f08da5f" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-ident" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" + +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +dependencies = [ + "void", +] + +[[package]] +name = "unsigned-varint" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d86a8dc7f45e4c1b0d30e43038c38f274e77af056aa5f74b93c2cf9eb3c1c836" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "uuid" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f" +dependencies = [ + "getrandom", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" +dependencies = [ + "cfg-if 1.0.0", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16646b21c3add8e13fdb8f20172f8a28c3dbf62f45406bcff0233188226cfe0c" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" + +[[package]] +name = "wasm-instrument" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bca81f5279342b38b17d9acbf007a46ddeb73144e2bd5f0a21bfa9fc5d4ab3e" +dependencies = [ + "parity-wasm", +] + +[[package]] +name = "wasmparser" +version = "0.78.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52144d4c78e5cf8b055ceab8e5fa22814ce4315d6002ad32cfd914f37c12fd65" + +[[package]] +name = "wasmtime" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b310b9d20fcf59385761d1ade7a3ef06aecc380e3d3172035b919eaf7465d9f7" +dependencies = [ + "anyhow", + "backtrace", + "bincode", + "cfg-if 1.0.0", + "cpp_demangle", + "indexmap", + "lazy_static", + "libc", + "log", + "paste", + "psm", + "region", + "rustc-demangle", + "serde", + "smallvec", + "target-lexicon", + "wasmparser", + "wasmtime-cache", + "wasmtime-environ", + "wasmtime-fiber", + "wasmtime-jit", + "wasmtime-profiling", + "wasmtime-runtime", + "wat", + "winapi", +] + +[[package]] +name = "wasmtime-cache" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d14d500d5c3dc5f5c097158feee123d64b3097f0d836a2a27dff9c761c73c843" +dependencies = [ + "anyhow", + "base64", + "bincode", + "directories-next", + "errno", + "file-per-thread-logger", + "libc", + "log", + "serde", + "sha2 0.9.5", + "toml", + "winapi", + "zstd", +] + +[[package]] +name = "wasmtime-cranelift" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c525b39f062eada7db3c1298287b96dcb6e472b9f6b22501300b28d9fa7582f6" +dependencies = [ + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "cranelift-wasm", + "target-lexicon", + "wasmparser", + "wasmtime-environ", +] + +[[package]] +name = "wasmtime-debug" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5d2a763e7a6fc734218e0e463196762a4f409c483063d81e0e85f96343b2e0a" +dependencies = [ + "anyhow", + "gimli 0.24.0", + "more-asserts", + "object 0.24.0", + "target-lexicon", + "thiserror", + "wasmparser", + "wasmtime-environ", +] + +[[package]] +name = "wasmtime-environ" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f64d0c2d881c31b0d65c1f2695e022d71eb60b9fbdd336aacca28208b58eac90" +dependencies = [ + "cfg-if 1.0.0", + "cranelift-codegen", + "cranelift-entity", + "cranelift-wasm", + "gimli 0.24.0", + "indexmap", + "log", + "more-asserts", + "serde", + "thiserror", + "wasmparser", +] + +[[package]] +name = "wasmtime-fiber" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a089d44cd7e2465d41a53b840a5b4fca1bf6d1ecfebc970eac9592b34ea5f0b3" +dependencies = [ + "cc", + "libc", + "winapi", +] + +[[package]] +name = "wasmtime-jit" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4539ea734422b7c868107e2187d7746d8affbcaa71916d72639f53757ad707" +dependencies = [ + "addr2line 0.15.2", + "anyhow", + "cfg-if 1.0.0", + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "cranelift-native", + "cranelift-wasm", + "gimli 0.24.0", + "log", + "more-asserts", + "object 0.24.0", + "rayon", + "region", + "serde", + "target-lexicon", + "thiserror", + "wasmparser", + "wasmtime-cranelift", + "wasmtime-debug", + "wasmtime-environ", + "wasmtime-obj", + "wasmtime-profiling", + "wasmtime-runtime", + "winapi", +] + +[[package]] +name = "wasmtime-obj" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1a8ff85246d091828e2225af521a6208ed28c997bb5c39eb697366dc2e2f2b" +dependencies = [ + "anyhow", + "more-asserts", + "object 0.24.0", + "target-lexicon", + "wasmtime-debug", + "wasmtime-environ", +] + +[[package]] +name = "wasmtime-profiling" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e24364d522dcd67c897c8fffc42e5bdfc57207bbb6d7eeade0da9d4a7d70105b" +dependencies = [ + "anyhow", + "cfg-if 1.0.0", + "gimli 0.24.0", + "lazy_static", + "libc", + "object 0.24.0", + "scroll", + "serde", + "target-lexicon", + "wasmtime-environ", + "wasmtime-runtime", +] + +[[package]] +name = "wasmtime-runtime" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51e57976e8a19a18a18e002c6eb12e5769554204238e47ff155fda1809ef0f7" +dependencies = [ + "anyhow", + "backtrace", + "cc", + "cfg-if 1.0.0", + "indexmap", + "lazy_static", + "libc", + "log", + "mach", + "memoffset", + "more-asserts", + "rand", + "region", + "thiserror", + "wasmtime-environ", + "wasmtime-fiber", + "winapi", +] + +[[package]] +name = "wast" +version = "37.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bc7b9a76845047ded00e031754ff410afee0d50fbdf62b55bdeecd245063d68" +dependencies = [ + "leb128", +] + +[[package]] +name = "wat" +version = "1.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2cc8d9a69d1ab28a41d9149bb06bb927aba8fc9d56625f8b597a564c83f50" +dependencies = [ + "wast", +] + +[[package]] +name = "web-sys" +version = "0.3.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c70a82d842c9979078c772d4a1344685045f1a5628f677c2b2eab4dd7d2696" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web3" +version = "0.19.0-graph" +source = "git+https://github.com/graphprotocol/rust-web3?branch=graph-patches-onto-0.18#7f8eb6dfcc13a4186f9b42f91de950646bc4a833" +dependencies = [ + "arrayvec 0.7.2", + "base64", + "bytes", + "derive_more", + "ethabi", + "ethereum-types", + "futures 0.3.16", + "futures-timer", + "headers", + "hex", + "idna", + "jsonrpc-core", + "log", + "once_cell", + "parking_lot 0.12.1", + "pin-project", + "reqwest", + "rlp", + "secp256k1", + "serde", + "serde_json", + "soketto", + "tiny-keccak 2.0.2", + "tokio", + "tokio-stream", + "tokio-util 0.6.7", + "url", + "web3-async-native-tls", +] + +[[package]] +name = "web3-async-native-tls" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f6d8d1636b2627fe63518d5a9b38a569405d9c9bc665c43c9c341de57227ebb" +dependencies = [ + "native-tls", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "which" +version = "4.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea187a8ef279bc014ec368c27a920da2024d2a711109bfbe3440585d5cf27ad9" +dependencies = [ + "either", + "lazy_static", + "libc", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" + +[[package]] +name = "windows_i686_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" + +[[package]] +name = "windows_i686_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi", +] + +[[package]] +name = "wyz" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b31594f29d27036c383b53b59ed3476874d518f0efb151b27a4c275141390e" +dependencies = [ + "tap", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "074914ea4eec286eb8d1fd745768504f420a1f7b7919185682a4a267bed7d2e7" + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + +[[package]] +name = "zstd" +version = "0.6.1+zstd.1.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de55e77f798f205d8561b8fe2ef57abfb6e0ff2abe7fd3c089e119cdb5631a3" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "3.0.1+zstd.1.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1387cabcd938127b30ce78c4bf00b30387dddf704e3f0881dbc4ff62b5566f8c" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "1.4.20+zstd.1.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd5b733d7cf2d9447e2c3e76a5589b4f5e5ae065c22a2bc0b023cbc331b6c8e" +dependencies = [ + "cc", + "libc", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..3bb9c26 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,26 @@ +[workspace] +members = [ + "core", + "chain/*", + "graphql", + "mock", + "node", + "runtime/wasm", + "runtime/derive", + "runtime/test", + "server/http", + "server/json-rpc", + "server/index-node", + "server/metrics", + "store/postgres", + "store/test-store", + "graph", + "tests", +] + +# Incremental compilation on Rust 1.58 causes an ICE on build. As soon as graph node builds again, these can be removed. +[profile.test] +incremental = false + +[profile.dev] +incremental = false diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 0000000..63d51c2 --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +Copyright (c) 2018 Graph Protocol, Inc. and contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..6ef9ca9 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Graph Protocol, Inc. and contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 0000000..497c56d --- /dev/null +++ b/NEWS.md @@ -0,0 +1,871 @@ +# NEWS + +## Unreleased + +### New DB table for dynamic data sources + +For new subgraph deployments, dynamic data sources will be recorded under the `sgd*.data_sources$` +table, rather than `subgraphs.dynamic_ethereum_contract_data_source`. As a consequence +new deployments will not work correctly on earlier graph node versions, so +_downgrading to an earlier graph node version is not supported_. +See issue #3405 for other details. + +## 0.27.0 + +- Store writes are now carried out in parallel to the rest of the subgraph process, improving indexing performance for subgraphs with significant store interaction. Metrics & monitoring was updated for this new pipelined process; +- This adds support for apiVersion 0.0.7, which makes receipts accessible in Ethereum event handlers. [Documentation link](https://thegraph.com/docs/en/developing/creating-a-subgraph/#transaction-receipts-in-event-handlers); +- This introduces some improvements to the subgraph GraphQL API, which now supports filtering on the basis of, and filtering for entities which changed from a certain block; +- Support was added for Arweave indexing. Tendermint was renamed to Cosmos in Graph Node. These integrations are still in "beta"; +- Callhandler block filtering for contract calls now works as intended (this was a longstanding bug); +- Gas costing for mappings is still set at a very high default, as we continue to benchmark and refine this metric; +- A new `graphman fix block` command was added to easily refresh a block in the block cache, or clear the cache for a given network; +- IPFS file fetching now uses `files/stat`, as `object` was deprecated; +- Subgraphs indexing via a Firehose can now take advantage of Firehose-side filtering; +- NEAR subgraphs can now match accounts for receipt filtering via prefixes or suffixes. + +## Upgrade notes + +- In the case of you having custom SQL, there's a [new SQL migration](https://github.com/graphprotocol/graph-node/blob/master/store/postgres/migrations/2022-04-26-125552_alter_deployment_schemas_version/up.sql); +- On the pipelining of the store writes, there's now a new environment variable `GRAPH_STORE_WRITE_QUEUE` (default value is `5`), that if set to `0`, the old synchronous behaviour will come in instead. The value stands for the amount of write/revert parallel operations [#3177](https://github.com/graphprotocol/graph-node/pull/3177); +- There's now support for TLS connections in the PostgreSQL `notification_listener` [#3503](https://github.com/graphprotocol/graph-node/pull/3503); +- GraphQL HTTP and WebSocket ports can now be set via environment variables [#2832](https://github.com/graphprotocol/graph-node/pull/2832); +- The genesis block can be set via the `GRAPH_ETHEREUM_GENESIS_BLOCK_NUMBER` env var [#3650](https://github.com/graphprotocol/graph-node/pull/3650); +- There's a new experimental feature to limit the number of subgraphs for a specific web3 provider. [Link for documentation](https://github.com/graphprotocol/graph-node/blob/master/docs/config.md#controlling-the-number-of-subgraphs-using-a-provider); +- Two new GraphQL validation environment variables were included: `ENABLE_GRAPHQL_VALIDATIONS` and `SILENT_GRAPHQL_VALIDATIONS`, which are documented [here](https://github.com/graphprotocol/graph-node/blob/master/docs/environment-variables.md#graphql); +- A bug fix for `graphman index` was landed, which fixed the behavior where if one deployment was used by multiple names would result in the command not working [#3416](https://github.com/graphprotocol/graph-node/pull/3416); +- Another fix landed for `graphman`, the bug would allow the `unassign`/`reassign` commands to make two or more nodes index the same subgraph by mistake [#3478](https://github.com/graphprotocol/graph-node/pull/3478); +- Error messages of eth RPC providers should be clearer during `graph-node` start up [#3422](https://github.com/graphprotocol/graph-node/pull/3422); +- Env var `GRAPH_STORE_CONNECTION_MIN_IDLE` will no longer panic, instead it will log a warning if it exceeds the `pool_size` [#3489](https://github.com/graphprotocol/graph-node/pull/3489); +- Failed GraphQL queries now have proper timing information in the service metrics [#3508](https://github.com/graphprotocol/graph-node/pull/3508); +- Non-primary shards now can be disabled through setting the `pool_size` to `0` [#3513](https://github.com/graphprotocol/graph-node/pull/3513); +- Queries with large results now have a `query_id` [#3514](https://github.com/graphprotocol/graph-node/pull/3514); +- It's now possible to disable the LFU Cache by setting `GRAPH_QUERY_LFU_CACHE_SHARDS` to `0` [#3522](https://github.com/graphprotocol/graph-node/pull/3522); +- `GRAPH_ACCOUNT_TABLES` env var is not supported anymore [#3525](https://github.com/graphprotocol/graph-node/pull/3525); +- [New documentation](https://github.com/graphprotocol/graph-node/blob/master/docs/implementation/metadata.md) landed on the metadata tables; +- `GRAPH_GRAPHQL_MAX_OPERATIONS_PER_CONNECTION` for GraphQL subscriptions now has a default of `1000` [#3735](https://github.com/graphprotocol/graph-node/pull/3735) + +## 0.26.0 + +### Features + +- Gas metering #2414 +- Adds support for Solidity Custom Errors #2577 +- Debug fork tool #2995 #3292 +- Automatically remove unused deployments #3023 +- Fix fulltextsearch space handling #3048 +- Allow placing new deployments onto one of several shards #3049 +- Make NEAR subgraphs update their sync status #3108 +- GraphQL validations #3164 +- Add special treatment for immutable entities #3201 +- Tendermint integration #3212 +- Skip block updates when triggers are empty #3223 #3268 +- Use new GraphiQL version #3252 +- GraphQL prefetching #3256 +- Allow using Bytes as well as String/ID for the id of entities #3271 +- GraphQL route for dumping entity changes in subgraph and block #3275 +- Firehose filters #3323 +- NEAR filters #3372 + +### Robustness + +- Improve our `CacheWeight` estimates #2935 +- Refactor GraphQL execution #3005 +- Setup databases in parallel #3019 +- Block ingestor now fetches receipts in parallel #3030 +- Prevent subscriptions from back-pressuring the notification queue #3053 +- Avoid parsing X triggers if the filter is empty #3083 +- Pipeline `BlockStream` #3085 +- More robust `proofOfIndexing` GraphQL route #3348 + +### `graphman` + +- Add `run` command, for running a subgraph up to a block #3079 +- Add `analyze` command, for analyzing a PostgreSQL table, which can improve performance #3170 +- Add `index create` command, for adding an index to certain attributes #3175 +- Add `index list` command, for listing indexes #3198 +- Add `index drop` command, for dropping indexes #3198 + +### Dependency Updates + +These are the main ones: + +- Updated protobuf to latest version for NEAR #2947 +- Update `web3` crate #2916 #3120 #3338 +- Update `graphql-parser` to `v0.4.0` #3020 +- Bump `itertools` from `0.10.1` to `0.10.3` #3037 +- Bump `clap` from `2.33.3` to `2.34.0` #3039 +- Bump `serde_yaml` from `0.8.21` to `0.8.23` #3065 +- Bump `tokio` from `1.14.0` to `1.15.0` #3092 +- Bump `indexmap` from `1.7.0` to `1.8.0` #3143 +- Update `ethabi` to its latest version #3144 +- Bump `structopt` from `0.3.25` to `0.3.26` #3180 +- Bump `anyhow` from `1.0.45` to `1.0.53` #3182 +- Bump `quote` from `1.0.9` to `1.0.16` #3112 #3183 #3384 +- Bump `tokio` from `1.15.0` to `1.16.1` #3208 +- Bump `semver` from `1.0.4` to `1.0.5` #3229 +- Bump `async-stream` from `0.3.2` to `0.3.3` #3361 +- Update `jsonrpc-server` #3313 + +### Misc + +- More context when logging RPC calls #3128 +- Increase default reorg threshold to 250 for Ethereum #3308 +- Improve traces error logs #3353 +- Add warning and continue on parse input failures for Ethereum #3326 + +### Upgrade Notes + +When upgrading to this version, we recommend taking a brief look into these changes: + +- Gas metering #2414 + - Now there's a gas limit for subgraph mappings, if the limit is reached the subgraph will fail with a non-deterministic error, you can make them recover via the environment variable `GRAPH_MAX_GAS_PER_HANDLER` +- Improve our `CacheWeight` estimates #2935 + - This is relevant because a couple of releases back we've added a limit for the memory size of a query result. That limit is based of the `CacheWeight`. + +These are some of the features that will probably be helpful for indexers 😊 + +- Allow placing new deployments onto one of several shards #3049 +- GraphQL route for dumping entity changes in subgraph and block #3275 +- Unused deployments are automatically removed now #3023 + - The interval can be set via `GRAPH_REMOVE_UNUSED_INTERVAL` +- Setup databases in parallel #3019 +- Block ingestor now fetches receipts in parallel #3030 + - `GRAPH_ETHEREUM_FETCH_TXN_RECEIPTS_IN_BATCHES` can be set to `true` for the old fetching behavior +- More robust `proofOfIndexing` GraphQL route #3348 + - A token can be set via `GRAPH_POI_ACCESS_TOKEN` to limit access to the POI route +- The new `graphman` commands 🙂 + + +### Api Version 0.0.7 and Spec Version 0.0.5 +This release brings API Version 0.0.7 in mappings, which allows Ethereum event handlers to require transaction receipts to be present in the `Event` object. +Refer to [PR #3373](https://github.com/graphprotocol/graph-node/pull/3373) for instructions on how to enable that. + + +## 0.25.2 + +This release includes two changes: + +- Bug fix of blocks being skipped from processing when: a deterministic error happens **and** the `index-node` gets restarted. Issue [#3236](https://github.com/graphprotocol/graph-node/issues/3236), Pull Request: [#3316](https://github.com/graphprotocol/graph-node/pull/3316). +- Automatic retries for non-deterministic errors. Issue [#2945](https://github.com/graphprotocol/graph-node/issues/2945), Pull Request: [#2988](https://github.com/graphprotocol/graph-node/pull/2988). + +This is the last patch on the `0.25` minor version, soon `0.26.0` will be released. While that we recommend updating to this version to avoid determinism issues that could be caused on `graph-node` restarts. + +## 0.25.1 + +This release only adds two fixes: + +- The first is to address an issue with decoding the input of some calls [#3194](https://github.com/graphprotocol/graph-node/issues/3194) where subgraphs that would try to index contracts related to those would fail. Now they can advance normally. +- The second one is to fix a non-determinism issue with the retry mechanism for errors. Whenever a non-deterministic error happened, we would keep retrying to process the block, however we should've clear the `EntityCache` on each run so that the error entity changes don't get transacted/saved in the database in the next run. This could make the POI generation non-deterministic for subgraphs that failed and retried for non-deterministic reasons, adding a new entry to the database for the POI. + +We strongly recommend updating to this version as quickly as possible. + +## 0.25.0 + +### Api Version 0.0.6 +This release ships support for API version 0.0.6 in mappings: +- Added `nonce` field for `Transaction` objects. +- Added `baseFeePerGas` field for `Block` objects ([EIP-1559](https://eips.ethereum.org/EIPS/eip-1559)). + +#### Block Cache Invalidation and Reset + +All cached block data must be refetched to account for the new `Block` and `Trasaction` +struct versions, so this release includes a `graph-node` startup check that will: +1. Truncate all block cache tables. +2. Bump the `db_version` value from `2` to `3`. + +_(Table truncation is a fast operation and no downtime will occur because of that.)_ + + +### Ethereum + +- 'Out of gas' errors on contract calls are now considered deterministic errors, + so they can be handled by `try_` calls. The gas limit is 50 million. + +### Environment Variables + +- The `GRAPH_ETH_CALL_GAS` environment is removed to prevent misuse, its value + is now hardcoded to 50 million. + +### Multiblockchain +- Initial support for NEAR subgraphs. +- Added `FirehoseBlockStream` implementation of `BlockStream` (#2716) + +### Misc +- Rust docker image is now based on Debian Buster. +- Optimizations to the PostgreSQL notification queue. +- Improve PostgreSQL robustness in multi-sharded setups. (#2815) +- Added 'networks' to the 'subgraphFeatures' endpoint. (#2826) +- Check and limit the size of GraphQL query results. (#2845) +- Allow `_in` and `_not_in` GraphQL filters. (#2841) +- Add PoI for failed subgraphs. (#2748) +- Make `graphman rewind` safer to use. (#2879) +- Add `subgraphErrors` for all GraphQL schemas. (#2894) +- Add `Graph-Attestable` response header. (#2946) +- Add support for minimum block constraint in GraphQL queries (`number_gte`) (#2868). +- Handle revert cases from Hardhat and Ganache (#2984) +- Fix bug on experimental prefetching optimization feature (#2899) + + +## 0.24.2 + +This release only adds a fix for an issue where certain GraphQL queries +could lead to `graph-node` running out of memory even on very large +systems. This release adds code that checks the size of GraphQL responses +as they are assembled, and can warn about large responses in the logs +resp. abort query execution based on the values of the two new environment +variables `GRAPH_GRAPHQL_WARN_RESULT_SIZE` and +`GRAPH_GRAPHQL_ERROR_RESULT_SIZE`. It also adds Prometheus metrics +`query_result_size` and `query_result_max` to track the memory consumption +of successful GraphQL queries. The unit for the two environment variables +is bytes, based on an estimate of the memory used by the result; it is best +to set them after observing the Prometheus metrics for a while to establish +what constitutes a reasonable limit for them. + +We strongly recommend updating to this version as quickly as possible. + +## 0.24.1 + +### Feature Management + +This release supports the upcoming Spec Version 0.0.4 that enables subgraph features to be declared in the manifest and +validated during subgraph deployment +[#2682](https://github.com/graphprotocol/graph-node/pull/2682) +[#2746](https://github.com/graphprotocol/graph-node/pull/2746). + +> Subgraphs using previous versions are still supported and won't be affected by this change. + +#### New Indexer GraphQL query: `subgraphFetaures` + +It is now possible to query for the features a subgraph uses given its Qm-hash ID. + +For instance, the following query... + +```graphql +{ + subgraphFeatures(subgraphId: "QmW9ajg2oTyPfdWKyUkxc7cTJejwdyCbRrSivfryTfFe5D") { + features + errors + } +} +``` + +... would produce this result: + +```json +{ + "data": { + "subgraphFeatures": { + "errors": [], + "features": [ + "nonFatalErrors", + "ipfsOnEthereumContracts" + ] + } + } +} +``` + +Subraphs with any Spec Version can be queried that way. + +### Api Version 0.0.5 + +- Added better error message for null pointers in the runtime [#2780](https://github.com/graphprotocol/graph-node/pull/2780). + +### Environment Variables + +- When `GETH_ETH_CALL_ERRORS_ENV` is unset, it doesn't make `eth_call` errors to be considered determinsistic anymore [#2784](https://github.com/graphprotocol/graph-node/pull/2784) + +### Robustness + +- Tolerate a non-primary shard being down during startup [#2727](https://github.com/graphprotocol/graph-node/pull/2727). +- Check that at least one replica for each shard has a non-zero weight [#2749](https://github.com/graphprotocol/graph-node/pull/2749). +- Reduce locking for the chain head listener [#2763](https://github.com/graphprotocol/graph-node/pull/2763). + +### Logs + +- Improve block ingestor error reporting for missing receipts [#2743](https://github.com/graphprotocol/graph-node/pull/2743). + +## 0.24.0 + +### Api Version 0.0.5 + +This release ships support for API version 0.0.5 in mappings. hIt contains a fix for call handlers +and the long awaited AssemblyScript version upgrade! + +- AssemblyScript upgrade: The mapping runtime is updated to support up-to-date versions of the + AssemblyScript compiler. The graph-cli/-ts releases to support this are in alpha, soon they will + be released along with a migration guide for subgraphs. +- Call handlers fix: Call handlers will never be triggered on transactions with a failed status, + resolving issue [#2409](https://github.com/graphprotocol/graph-node/issues/2409). Done in [#2511](https://github.com/graphprotocol/graph-node/pull/2511). + +### Logs +- The log `"Skipping handler because the event parameters do not match the event signature."` was downgraded from info to trace level. +- Some block ingestor error logs were upgrded from debug to info level [#2666](https://github.com/graphprotocol/graph-node/pull/2666). + +### Metrics +- `query_semaphore_wait_ms` is now by shard, and has the `pool` and `shard` labels. +- `deployment_failed` metric added, it is `1` if the subgraph has failed and `0` otherwise. + +### Other +- Upgrade to tokio 1.0 and futures 0.3 [#2679](https://github.com/graphprotocol/graph-node/pull/2679), the first major contribution by StreamingFast! +- Support Celo block reward events [#2670](https://github.com/graphprotocol/graph-node/pull/2670). +- Reduce the maximum WASM stack size and make it configurable [#2719](https://github.com/graphprotocol/graph-node/pull/2719). +- For robustness, ensure periodic updates to the chain head listener [#2725](https://github.com/graphprotocol/graph-node/pull/2725). + +## 0.23.1 + +- Fix ipfs timeout detection [#2584](https://github.com/graphprotocol/graph-node/pull/2584). +- Fix discrepancy between a database table and its Diesel model [#2586](https://github.com/graphprotocol/graph-node/pull/2586). + +## 0.23.0 + +The Graph Node internals are being heavily refactored to prepare it for the multichain future. +In the meantime, here are the changes for this release: + +- The `GRAPH_ETH_CALL_BY_NUMBER` environment variable has been removed. Graph Node requires an + Ethereum client that supports EIP-1898, which all major clients support. +- Added support for IPFS versions larger than 0.4. Several changes to make + `graph-node` more tolerant of slow/flaky IPFS nodes. +- Added Ethereum ABI encoding and decoding functionality [#2348](https://github.com/graphprotocol/graph-node/pull/2348). +- Experimental support for configuration files, see the documentation [here](https://github.com/graphprotocol/graph-node/blob/master/docs/config.md). +- Better PoI performance [#2329](https://github.com/graphprotocol/graph-node/pull/2329). +- Improve grafting performance and robustness by copying in batches [#2293](https://github.com/graphprotocol/graph-node/pull/2293). +- Subgraph metadata storage has been simplified and reorganized. External + tools (e.g., Grafana dashboards) that access the database directly will need to be updated. +- Ordering in GraphQL queries is now truly reversible + [#2214](https://github.com/graphprotocol/graph-node/pull/2214/commits/bc559b8df09a7c24f0d718b76fa670313911a6b1) +- The `GRAPH_SQL_STATEMENT_TIMEOUT` environment variable can be used to + enforce a timeout for individual SQL queries that are run in the course of + processing a GraphQL query + [#2285](https://github.com/graphprotocol/graph-node/pull/2285) +- Using `ethereum.call` in mappings in globals is deprecated + +### Graphman +Graphman is a CLI tool to manage your subgraphs. It is now included in the Docker container +[#2289](https://github.com/graphprotocol/graph-node/pull/2289). And new commands have been added: +- `graphman copy` can copy subgraphs across DB shards [#2313](https://github.com/graphprotocol/graph-node/pull/2313). +- `graphman rewind` to rewind a deployment to a given block [#2373](https://github.com/graphprotocol/graph-node/pull/2373). +- `graphman query` to log info about a GraphQL query [#2206](https://github.com/graphprotocol/graph-node/pull/2206). +- `graphman create` to create a subgraph name [#2419](https://github.com/graphprotocol/graph-node/pull/2419). + +### Metrics +- The `deployment_blocks_behind` metric has been removed, and a + `deployment_head` metric has been added. To see how far a deployment is + behind, use the difference between `ethereum_chain_head_number` and + `deployment_head`. +- The `trigger_type` label was removed from the metric `deployment_trigger_processing_duration`. + +## 0.22.0 + +### Feature: Block store sharding +This release makes it possible to [shard the block and call cache](./docs/config.md) for chain +data across multiple independent Postgres databases. **This feature is considered experimental. We +encourage users to try this out in a test environment, but do not recommend it yet for production +use.** In particular, the details of how sharding is configured may change in backwards-incompatible +ways in the future. + +### Feature: Non-fatal errors update +Non-fatal errors (see release 0.20 for details) is documented and can now be enabled on graph-cli. +Various related bug fixes have been made #2121 #2136 #2149 #2160. + +### Improvements +- Add bitwise operations and string constructor to BigInt #2151. +- docker: Allow custom ethereum poll interval #2139. +- Deterministic error work in preparation for gas #2112 + +### Bug fixes +- Fix not contains filter #2146. +- Resolve __typename in _meta field #2118 +- Add CORS for all HTTP responses #2196 + +## 0.21.1 + +- Fix subgraphs failing with a `fatalError` when deployed while already running + (#2104). +- Fix missing `scalar Int` declaration in index node GraphQL API, causing + indexer-service queries to fail (#2104). + +## 0.21.0 + +### Feature: Database sharding + +This release makes it possible to [shard subgraph +storage](./docs/config.md) and spread subgraph deployments, and the load +coming from indexing and querying them across multiple independent Postgres +databases. + +**This feature is considered experimenatal. We encourage users to try this +out in a test environment, but do not recommend it yet for production use** +In particular, the details of how sharding is configured may change in +backwards-incompatible ways in the future. + +### Breaking change: Require a block number in `proofOfIndexing` queries + +This changes the `proofOfIndexing` GraphQL API from + +```graphql +type Query { + proofOfIndexing(subgraph: String!, blockHash: Bytes!, indexer: Bytes): Bytes +} +``` + +to + +```graphql +type Query { + proofOfIndexing( + subgraph: String! + blockNumber: Int! + blockHash: Bytes! + indexer: Bytes + ): Bytes +} +``` + +This allows the indexer agent to provide a block number and hash to be able +to obtain a POI even if this block is not cached in the Ethereum blocks +cache. Prior to this, the POI would be `null` if this wasn't the case, even +if the subgraph deployment in question was up to date, leading to the indexer +missing out on indexing rewards. + +### Misc + +- Fix non-determinism caused by not (always) correctly reverting dynamic + sources when handling reorgs. +- Integrate the query cache into subscriptions to improve their performance. +- Add `graphman` crate for managing Graph Node infrastructure. +- Improve query cache logging. +- Expose indexing status port (`8030`) from Docker image. +- Remove support for unnecessary data sources `templates` inside subgraph + data sources. They are only supported at the top level. +- Avoid sending empty store events through the database. +- Fix database connection deadlocks. +- Rework the codebase to use `anyhow` instead of `failure`. +- Log stack trace in case of database connection timeouts, to help with root-causing. +- Fix stack overflows in GraphQL parsing. +- Disable fulltext search by default (it is nondeterministic and therefore + not currently supported in the network). + +## 0.20.0 + +**NOTE: JSONB storage is no longer supported. Do not upgrade to this +release if you still have subgraphs that were deployed with a version +before 0.16. They need to be redeployed before updating to this version.** + +You can check if you have JSONB subgraphs by running the query `select count(*) from deployment_schemas where version='split'` in `psql`. If that +query returns `0`, you do not have JSONB subgraphs and it is safe to upgrde +to this version. + +### Feature: `_meta` field + +Subgraphs sometimes fall behind, be it due to failing or the Graph Node may be having issues. The +`_meta` field can now be added to any query so that it is possible to determine against which block +the query was effectively executed. Applications can use this to warn users if the data becomes +stale. It is as simple as adding this to your query: + +```graphql +_meta { + block { + number + hash + } +} +``` + +### Feature: Non-fatal errors + +Indexing errors on already synced subgraphs no longer need to cause the entire subgraph to grind to +a halt. Subgraphs can now be configured to continue syncing in the presence of errors, by simply +skipping the problematic handler. This gives subgraph authors time to correct their subgraphs while the nodes can continue to serve up-to-date the data. This requires setting a flag on the subgraph manifest: + +```yaml +features: + - nonFatalErrors +``` + +And the query must also opt-in to querying data with potential inconsistencies: + +```graphql +foos(first: 100, subgraphError: allow) { + id +} +``` + +If the subgraph encounters and error the query will return both the data and a graphql error with +the message `indexing_error`. + +Note that some errors are still fatal, to be non-fatal the error must be known to be deterministic. The `_meta` field can be used to check if the subgraph has skipped over errors: + +```graphql +_meta { + hasIndexingErrors +} +``` + +The `features` section of the manifest requires depending on the graph-cli master branch until the next version (after `0.19.0`) is released. + +### Ethereum + +- Support for `tuple[]` (#1973). +- Support multiple Ethereum endpoints per network with different capabilities (#1810). + +### Performance + +- Avoid cloning results assembled from partial results (#1907). + +### Security + +- Add `cargo-audit` to the build process, update dependencies (#1998). + +## 0.19.2 + +- Add `GRAPH_ETH_CALL_BY_NUMBER` environment variable for disabling + EIP-1898 (#1957). +- Disable `ipfs.cat` by default, as it is non-deterministic (#1958). + +## 0.19.1 + +- Detect reorgs during query execution (#1801). +- Annotate SQL queries with the GraphQL query ID that caused them (#1946). +- Fix potential deadlock caused by reentering the load manager semaphore (#1948). +- Fix fulltext query issue with optional and unset fields (#1937 via #1938). +- Fix build warnings with --release (#1949 via #1953). +- Dependency updates: async-trait, chrono, wasmparser. + +## 0.19.0 + +- Skip `trace_filter` on empty blocks (#1923). +- Ensure runtime hosts are unique to avoid double-counting, improve logging + (#1904). +- Add administrative Postgres views (#1889). +- Limit the GraphQL `skip` argument in the same way as we limit `first` (#1912). +- Fix GraphQL fragment bugs (#1825). +- Don't crash node and show better error when multiple graph nodes are indexing + the same subgraph (#1903). +- Add a query semaphore to allow to control the number of concurrent queries and + subscription queries being executed (#1802). +- Call Ethereum contracts by block hash (#1905). +- Fix fetching the correct function ABI from the contract ABI (#1886). +- Add LFU cache for historical queries (#1878, #1879, #1891). +- Log GraphQL queries only once (#1873). +- Gracefully fail on a null block hash and encoding failures in the Ethereum + adapter (#1872). +- Improve metrics by using labels more (#1868, ...) +- Log when decoding a contract call result fails to decode (#1842). +- Fix Ethereum node requirements parsing based on the manifest (#1834). +- Speed up queries that involve checking for inclusion in an array (#1820). +- Add better error message when blocking a query due to load management (#1822). +- Support multiple Ethereum nodes/endpoints per network, with different + capabilities (#1810). +- Change how we index foreign keys (#1811). +- Add an experimental Ethereum node config file (#1819). +- Allow using GraphQL variables in block constraints (#1803). +- Add Solidity struct array / Ethereum tuple array support (#1815). +- Resolve subgraph names in a blocking task (#1797). +- Add environmen variable options for sensitive arguments (#1784). +- USe blocking task for store events (#1789). +- Refactor servers, log GraphQL panics (#1783). +- Remove excessive logging in the store (#1772). +- Add dynamic load management for GraphQL queries (#1762, #1773, #1774). +- Add ability to block certain queries (#1749, #1771). +- Log the complexity of each query executed (#1752). +- Add support for running against read-only Postgres replicas (#1746, #1748, + #1753, #1750, #1754, #1860). +- Catch invalid opcode reverts on Geth (#1744). +- Optimize queries for single-object lookups (#1734). +- Increase the maximum number of blocking threads (#1742). +- Increase default JSON-RPC timeout (#1732). +- Ignore flaky network indexers tests (#1724). +- Change default max block range size to 1000 (#1727). +- Fixed aliased scalar fields (#1726). +- Fix issue inserting fulltext fields when all included field values are null (#1710). +- Remove frequent "GraphQL query served" log message (#1719). +- Fix `bigDecimal.devidedBy` (#1715). +- Optimize GraphQL execution, remove non-prefetch code (#1712, #1730, #1733, + #1743, #1775). +- Add a query cache (#1708, #1709, #1747, #1751, #1777). +- Support the new Geth revert format (#1713). +- Switch WASM runtime from wasmi to wasmtime and cranelift (#1700). +- Avoid adding `order by` clauses for single-object lookups (#1703). +- Refactor chain head and store event listeners (#1693). +- Properly escape single quotes in strings for SQL queries (#1695). +- Revamp how Graph Node Docker image is built (#1644). +- Add BRIN indexes to speed up revert handling (#1683). +- Don't store chain head block in `SubgraphDeployment` entity (#1673). +- Allow varying block constraints across different GraphQL query fields (#1685). +- Handle database tables that have `text` columns where they should have enums (#1681). +- Make contract call cache collision-free (#1680). +- Fix a SQL query in `cleanup_cached_blocks` (#1672). +- Exit process when panicking in the notification listener (#1671). +- Rebase ethabi and web3 forks on top of upstream (#1662). +- Remove parity-wasm dependency (#1663). +- Normalize `BigDecimal` values, limit `BigDecimal` exponent (#1640). +- Strip nulls from strings (#1656). +- Fetch genesis block by number `0` instead of `"earliest"` (#1658). +- Speed up GraphQL query execution (#1648). +- Fetch event logs in parallel (#1646). +- Cheaper block polling (#1646). +- Improve indexing status API (#1609, #1655, #1659, #1718). +- Log Postgres contention again (#1643). +- Allow `User-Agent` in CORS headers (#1635). +- Docker: Increase startup wait timeouts (Postgres, IPFS) to 120s (#1634). +- Allow using `Bytes` for `id` fields (#1607). +- Increase Postgres connection pool size (#1620). +- Fix entities updated after being removed in the same block (#1632). +- Pass `log_index` to mappings in place of `transaction_log_index` (required for + Geth). +- Don't return `__typename` to mappings (#1629). +- Log warnings after 10 successive failed `eth_call` requests. This makes + it more visible when graph-node is not operating against an Ethereum + archive node (#1606). +- Improve use of async/await across the codebase. +- Add Proof Of Indexing (POI). +- Add first implementation of subgraph grafting. +- Add integration test for handling Ganache reverts (#1590). +- Log all GraphQL and SQL queries performed by a node, controlled through + the `GRAPH_LOG_QUERY_TIMING` [environment + variable](docs/environment-variables.md) (#1595). +- Fix loading more than 200 dynamic data sources (#1596). +- Fix fulltext schema validation (`includes` fields). +- Dependency updates: anyhow, async-trait, bs58, blake3, bytes, chrono, clap, + crossbeam-channel derive_more, diesel-derive-enum, duct, ethabi, + git-testament, hex-literal, hyper, indexmap, jsonrpc-core, mockall, once_cell, + petgraph, reqwest, semver, serde, serde_json, slog-term, tokio, wasmparser. + +## 0.18.0 + +**NOTE: JSONB storage is deprecated and will be removed in the next release. +This only affects subgraphs that were deployed with a graph-node version +before 0.16. Starting with this version, graph-node will print a warning for +any subgraph that uses JSONB storage when that subgraph starts syncing. Please +check your logs for this warning. You can remove the warning by redeploying +the subgraph.** + +### Feature: Fulltext Search (#1521) + +A frequently requested feature has been support for more advanced text-based +search, e.g. to power search fields in dApps. This release introduces a +`@fulltext` directive on a new, reserved `_Schema_` type to define fulltext +search APIs that can then be used in queries. The example below shows how +such an API can be defined in the subgraph schema: + +```graphql +type _Schema_ + @fulltext( + name: "artistSearch" + language: en + algorithm: rank + include: [ + { + entity: "Artist" + fields: [ + { name: "name" } + { name: "bio" } + { name: "genre" } + { name: "promoCopy" } + ] + } + ] + ) +``` + +This will add a special database column for `Artist` entities that can be +used for fulltext search queries across all included entity fields, based on +the `tsvector` and `tsquery` features provided by Postgres. + +The `@fulltext` directive will also add an `artistSearch` field on the root +query object to the generated subgraph GraphQL API, which can be used as +follows: + +```graphql +{ + artistSearch(text: "breaks & electro & detroit") { + id + name + bio + } +} +``` + +For more information about the supported operators (like the `&` in the above +query), please refer to the [Postgres +documentation](https://www.postgresql.org/docs/10/textsearch.html). + +### Feature: 3Box Profiles (#1574) + +[3Box](https://3box.io) has become a popular solution for integrating user +profiles into dApps. Starting with this release, it is possible to fetch profile +data for Ethereum addresses and DIDs. Example usage: + +```ts +import { box } from '@graphprotocol/graph-ts' + +let profile = box.profile("0xc8d807011058fcc0FB717dcd549b9ced09b53404") +if (profile !== null) { + let name = profile.get("name") + ... +} + +let profileFromDid = box.profile( + "id:3:bafyreia7db37k7epoc4qaifound6hk7swpwfkhudvdug4bgccjw6dh77ue" +) +... +``` + +### Feature: Arweave Transaction Data (#1574) + +This release enables accessing [Arweave](https://arweave.org) transaction data +using Arweave transaction IDs: + +```ts +import { arweave, json } from '@graphprotocol/graph-ts' + +let data = arweave.transactionData( + "W2czhcswOAe4TgL4Q8kHHqoZ1jbFBntUCrtamYX_rOU" +) + +if (data !== null) { + let data = json.fromBytes(data) + ... +} + +``` + +### Feature: Data Source Context (#1404 via #1537) + +Data source contexts allow passing extra configuration when creating a data +source from a template. As an example, let's say a subgraph tracks exchanges +that are associated with a particular trading pair, which is included in the +`NewExchange` event. That information can be passed into the dynamically +created data source, like so: + +```ts +import { DataSourceContext } from "@graphprotocol/graph-ts"; +import { Exchange } from "../generated/templates"; + +export function handleNewExchange(event: NewExchange): void { + let context = new DataSourceContext(); + context.setString("tradingPair", event.params.tradingPair); + Exchange.createWithContext(event.params.exchange, context); +} +``` + +Inside a mapping of the Exchange template, the context can then be accessed +as follows: + +```ts +import { dataSource } from '@graphprotocol/graph-ts' + +... + +let context = dataSource.context() +let tradingPair = context.getString('tradingPair') +``` + +There are setters and getters like `setString` and `getString` for all value +types to make working with data source contexts convenient. + +### Feature: Error Handling for JSON Parsing (#1588 via #1578) + +With contracts anchoring JSON data on IPFS on chain, there is no guarantee +that this data is actually valid JSON. Until now, failure to parse JSON in +subgraph mappings would fail the subgraph. This release adds a new +`json.try_fromBytes` host export that allows subgraph to gracefully handle +JSON parsing errors. + +```ts +import { json } from '@graphprotocol/graph-ts' + +export function handleSomeEvent(event: SomeEvent): void { + // JSON data as bytes, e.g. retrieved from IPFS + let data = ... + + // This returns a `Result`, meaning that the error type is + // just a boolean (true if there was an error, false if parsing succeeded). + // The actual error message is logged automatically. + let result = json.try_fromBytes(data) + + if (result.isOk) { // or !result.isError + // Do something with the JSON value + let value = result.value + ... + } else { + // Handle the error + let error = result.error + ... + } +} +``` + +### Ethereum + +- Add support for calling overloaded contract functions (#48 via #1440). +- Add integration test for calling overloaded contract functions (#1441). +- Avoid `eth_getLogs` requests with block ranges too large for Ethereum nodes + to handle (#1536). +- Simplify `eth_getLogs` fetching logic to reduce the risk of being rate + limited by Ethereum nodes and the risk of overloading them (#1540). +- Retry JSON-RPC responses with a `-32000` error (Alchemy uses this for + timeouts) (#1539). +- Reduce block range size for `trace_filter` requests to prevent request + timeouts out (#1547). +- Fix loading dynamically created data sources with `topic0` event handlers + from the database (#1580). +- Fix handling contract call reverts in newer versions of Ganache (#1591). + +### IPFS + +- Add support for checking multiple IPFS nodes when fetching files (#1498). + +### GraphQL + +- Use correct network when resolving block numbers in time travel queries + (#1508). +- Fix enum field validation in subgraph schemas (#1495). +- Prevent WebSocket connections from hogging the blocking thread pool and + freezing the node (#1522). + +### Database + +- Switch subgraph metadata from JSONB to relational storage (#1394 via #1454, + #1457, #1459). +- Clean up large notifications less frequently (#1505). +- Add metric for Postgres connection errors (#1484). +- Log SQL queries executed as part of the GraphQL API (#1465, #1466, #1468). +- Log entities returned by SQL queries (#1503). +- Fix several GraphQL prefetch / SQL query execution issues (#1523, #1524, + #1526). +- Print deprecation warnings for JSONB subgraphs (#1527). +- Make sure reorg handling does not affect metadata of other subgraphs (#1538). + +### Performance + +- Maintain an in-memory entity cache across blocks to speed up `store.get` + (#1381 via #1416). +- Speed up revert handling by making use of cached blocks (#1449). +- Speed up simple queries by delaying building JSON objects for results (#1476). +- Resolve block numbers to hashes using cached blocks when possible (#1477). +- Improve GraphQL prefetching performance by using lateral joins (#1450 via + #1483). +- Vastly reduce memory consumption when indexing data sources created from + templates (#1494). + +### Misc + +- Default to IPFS 0.4.23 in the Docker Compose setup (#1592). +- Support Elasticsearch endpoints without HTTP basic auth (#1576). +- Fix `--version` not reporting the current version (#967 via #1567). +- Convert more code to async/await and simplify async logic (#1558, #1560, + #1571). +- Use lossy, more tolerant UTF-8 conversion when converting strings to bytes + (#1541). +- Detect when a node is unresponsive and kill it (#1507). +- Dump core when exiting because of a fatal error (#1512). +- Update to futures 0.3 and tokio 0.2, enabling `async`/`await` (#1448). +- Log block and full transaction hash when handlers fail (#1496). +- Speed up network indexer tests (#1453). +- Fix Travis to always install Node.js 11.x. (#1588). +- Dependency updates: bytes, chrono, crossbeam-channel, ethabi, failure, + futures, hex, hyper, indexmap, jsonrpc-http-server, num-bigint, + priority-queue, reqwest, rust-web3, serde, serde_json, slog-async, slog-term, + tokio, tokio-tungstenite, walkdir, url. diff --git a/README.md b/README.md new file mode 100644 index 0000000..76cb9f6 --- /dev/null +++ b/README.md @@ -0,0 +1,199 @@ +# Graph Node + +[![Build Status](https://github.com/graphprotocol/graph-node/actions/workflows/ci.yml/badge.svg)](https://github.com/graphprotocol/graph-node/actions/workflows/ci.yml?query=branch%3Amaster) +[![Getting Started Docs](https://img.shields.io/badge/docs-getting--started-brightgreen.svg)](docs/getting-started.md) + +[The Graph](https://thegraph.com/) is a protocol for building decentralized applications (dApps) quickly on Ethereum and IPFS using GraphQL. + +Graph Node is an open source Rust implementation that event sources the Ethereum blockchain to deterministically update a data store that can be queried via the GraphQL endpoint. + +For detailed instructions and more context, check out the [Getting Started Guide](docs/getting-started.md). + +## Quick Start + +### Prerequisites + +To build and run this project you need to have the following installed on your system: + +- Rust (latest stable) – [How to install Rust](https://www.rust-lang.org/en-US/install.html) + - Note that `rustfmt`, which is part of the default Rust installation, is a build-time requirement. +- PostgreSQL – [PostgreSQL Downloads](https://www.postgresql.org/download/) +- IPFS – [Installing IPFS](https://docs.ipfs.io/install/) + +For Ethereum network data, you can either run your own Ethereum node or use an Ethereum node provider of your choice. + +**Minimum Hardware Requirements:** + +- To build graph-node with `cargo`, 8GB RAM are required. + +### Running a Local Graph Node + +This is a quick example to show a working Graph Node. It is a [subgraph for Gravatars](https://github.com/graphprotocol/example-subgraph). + +1. Install IPFS and run `ipfs init` followed by `ipfs daemon`. +2. Install PostgreSQL and run `initdb -D .postgres` followed by `pg_ctl -D .postgres -l logfile start` and `createdb graph-node`. +3. If using Ubuntu, you may need to install additional packages: + - `sudo apt-get install -y clang libpq-dev libssl-dev pkg-config` +4. In the terminal, clone https://github.com/graphprotocol/example-subgraph, and install dependencies and generate types for contract ABIs: + +``` +yarn +yarn codegen +``` + +5. In the terminal, clone https://github.com/graphprotocol/graph-node, and run `cargo build`. + +Once you have all the dependencies set up, you can run the following: + +``` +cargo run -p graph-node --release -- \ + --postgres-url postgresql://USERNAME[:PASSWORD]@localhost:5432/graph-node \ + --ethereum-rpc NETWORK_NAME:[CAPABILITIES]:URL \ + --ipfs 127.0.0.1:5001 +``` + +Try your OS username as `USERNAME` and `PASSWORD`. For details on setting +the connection string, check the [Postgres +documentation](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING). +`graph-node` uses a few Postgres extensions. If the Postgres user with which +you run `graph-node` is a superuser, `graph-node` will enable these +extensions when it initalizes the database. If the Postgres user is not a +superuser, you will need to create the extensions manually since only +superusers are allowed to do that. To create them you need to connect as a +superuser, which in many installations is the `postgres` user: + +```bash + psql -q -X -U graph-node <; +EOF + +``` + +This will also spin up a GraphiQL interface at `http://127.0.0.1:8000/`. + +6. With this Gravatar example, to get the subgraph working locally run: + +``` +yarn create-local +``` + +Then you can deploy the subgraph: + +``` +yarn deploy-local +``` + +This will build and deploy the subgraph to the Graph Node. It should start indexing the subgraph immediately. + +### Command-Line Interface + +``` +USAGE: + graph-node [FLAGS] [OPTIONS] --ethereum-ipc --ethereum-rpc --ethereum-ws --ipfs --postgres-url + +FLAGS: + --debug Enable debug logging + -h, --help Prints help information + -V, --version Prints version information + +OPTIONS: + --admin-port Port for the JSON-RPC admin server [default: 8020] + --elasticsearch-password + Password to use for Elasticsearch logging [env: ELASTICSEARCH_PASSWORD] + + --elasticsearch-url + Elasticsearch service to write subgraph logs to [env: ELASTICSEARCH_URL=] + + --elasticsearch-user User to use for Elasticsearch logging [env: ELASTICSEARCH_USER=] + --ethereum-ipc + Ethereum network name (e.g. 'mainnet'), optional comma-seperated capabilities (eg full,archive), and an Ethereum IPC pipe, separated by a ':' + + --ethereum-polling-interval + How often to poll the Ethereum node for new blocks [env: ETHEREUM_POLLING_INTERVAL=] [default: 500] + + --ethereum-rpc + Ethereum network name (e.g. 'mainnet'), optional comma-seperated capabilities (eg 'full,archive'), and an Ethereum RPC URL, separated by a ':' + + --ethereum-ws + Ethereum network name (e.g. 'mainnet'), optional comma-seperated capabilities (eg `full,archive), and an Ethereum WebSocket URL, separated by a ':' + + --node-id + A unique identifier for this node instance. Should have the same value between consecutive node restarts [default: default] + + --http-port Port for the GraphQL HTTP server [default: 8000] + --ipfs HTTP address of an IPFS node + --postgres-url Location of the Postgres database used for storing entities + --subgraph <[NAME:]IPFS_HASH> Name and IPFS hash of the subgraph manifest + --ws-port Port for the GraphQL WebSocket server [default: 8001] +``` + +### Advanced Configuration + +The command line arguments generally are all that is needed to run a +`graph-node` instance. For advanced uses, various aspects of `graph-node` +can further be configured through [environment +variables](https://github.com/graphprotocol/graph-node/blob/master/docs/environment-variables.md). Very +large `graph-node` instances can also split the work of querying and +indexing across [multiple databases](./docs/config.md). + +## Project Layout + +- `node` — A local Graph Node. +- `graph` — A library providing traits for system components and types for + common data. +- `core` — A library providing implementations for core components, used by all + nodes. +- `chain/ethereum` — A library with components for obtaining data from + Ethereum. +- `graphql` — A GraphQL implementation with API schema generation, + introspection, and more. +- `mock` — A library providing mock implementations for all system components. +- `runtime/wasm` — A library for running WASM data-extraction scripts. +- `server/http` — A library providing a GraphQL server over HTTP. +- `store/postgres` — A Postgres store with a GraphQL-friendly interface + and audit logs. + +## Roadmap + +🔨 = In Progress + +🛠 = Feature complete. Additional testing required. + +✅ = Feature complete + + +| Feature | Status | +| ------- | :------: | +| **Ethereum** | | +| Indexing smart contract events | ✅ | +| Handle chain reorganizations | ✅ | +| **Mappings** | | +| WASM-based mappings| ✅ | +| TypeScript-to-WASM toolchain | ✅ | +| Autogenerated TypeScript types | ✅ | +| **GraphQL** | | +| Query entities by ID | ✅ | +| Query entity collections | ✅ | +| Pagination | ✅ | +| Filtering | ✅ | +| Block-based Filtering | ✅ | +| Entity relationships | ✅ | +| Subscriptions | ✅ | + + +## Contributing + +Please check [CONTRIBUTING.md](CONTRIBUTING.md) for development flow and conventions we use. +Here's [a list of good first issues](https://github.com/graphprotocol/graph-node/labels/good%20first%20issue). + +## License + +Copyright © 2018-2019 Graph Protocol, Inc. and contributors. + +The Graph is dual-licensed under the [MIT license](LICENSE-MIT) and the [Apache License, Version 2.0](LICENSE-APACHE). + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/chain/arweave/.gitignore b/chain/arweave/.gitignore new file mode 100644 index 0000000..97442b5 --- /dev/null +++ b/chain/arweave/.gitignore @@ -0,0 +1 @@ +google.protobuf.rs \ No newline at end of file diff --git a/chain/arweave/Cargo.toml b/chain/arweave/Cargo.toml new file mode 100644 index 0000000..5d94dd9 --- /dev/null +++ b/chain/arweave/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "graph-chain-arweave" +version = "0.27.0" +edition = "2021" + +[build-dependencies] +tonic-build = { version = "0.7.1", features = ["prost"]} + +[dependencies] +base64-url = "1.4.13" +graph = { path = "../../graph" } +prost = "0.10.1" +prost-types = "0.10.1" +serde = "1.0" +sha2 = "0.10.5" + +graph-runtime-wasm = { path = "../../runtime/wasm" } +graph-runtime-derive = { path = "../../runtime/derive" } + +[dev-dependencies] +diesel = { version = "1.4.7", features = ["postgres", "serde_json", "numeric", "r2d2"] } diff --git a/chain/arweave/build.rs b/chain/arweave/build.rs new file mode 100644 index 0000000..e2ede2a --- /dev/null +++ b/chain/arweave/build.rs @@ -0,0 +1,7 @@ +fn main() { + println!("cargo:rerun-if-changed=proto"); + tonic_build::configure() + .out_dir("src/protobuf") + .compile(&["proto/type.proto"], &["proto"]) + .expect("Failed to compile Firehose Arweave proto(s)"); +} diff --git a/chain/arweave/proto/type.proto b/chain/arweave/proto/type.proto new file mode 100644 index 0000000..b3a41a4 --- /dev/null +++ b/chain/arweave/proto/type.proto @@ -0,0 +1,108 @@ +syntax = "proto3"; + +package sf.arweave.type.v1; + +option go_package = "github.com/ChainSafe/firehose-arweave/pb/sf/arweave/type/v1;pbcodec"; + +message BigInt { + bytes bytes = 1; +} + +message Block { + // Firehose block version (unrelated to Arweave block version) + uint32 ver = 1; + // The block identifier + bytes indep_hash = 2; + // The nonce chosen to solve the mining problem + bytes nonce = 3; + // `indep_hash` of the previous block in the weave + bytes previous_block = 4; + // POSIX time of block discovery + uint64 timestamp = 5; + // POSIX time of the last difficulty retarget + uint64 last_retarget = 6; + // Mining difficulty; the number `hash` must be greater than. + BigInt diff = 7; + // How many blocks have passed since the genesis block + uint64 height = 8; + // Mining solution hash of the block; must satisfy the mining difficulty + bytes hash = 9; + // Merkle root of the tree of Merkle roots of block's transactions' data. + bytes tx_root = 10; + // Transactions contained within this block + repeated Transaction txs = 11; + // The root hash of the Merkle Patricia Tree containing + // all wallet (account) balances and the identifiers + // of the last transactions posted by them; if any. + bytes wallet_list = 12; + // (string or) Address of the account to receive the block rewards. Can also be unclaimed which is encoded as a null byte + bytes reward_addr = 13; + // Tags that a block producer can add to a block + repeated Tag tags = 14; + // Size of reward pool + BigInt reward_pool = 15; + // Size of the weave in bytes + BigInt weave_size = 16; + // Size of this block in bytes + BigInt block_size = 17; + // Required after the version 1.8 fork. Zero otherwise. + // The sum of the average number of hashes computed + // by the network to produce the past blocks including this one. + BigInt cumulative_diff = 18; + // Required after the version 1.8 fork. Null byte otherwise. + // The Merkle root of the block index - the list of {`indep_hash`; `weave_size`; `tx_root`} triplets + bytes hash_list_merkle = 20; + // The proof of access; Used after v2.4 only; set as defaults otherwise + ProofOfAccess poa = 21; +} + +// A succinct proof of access to a recall byte found in a TX +message ProofOfAccess { + // The recall byte option chosen; global offset of index byte + string option = 1; + // The path through the Merkle tree of transactions' `data_root`s; + // from the `data_root` being proven to the corresponding `tx_root` + bytes tx_path = 2; + // The path through the Merkle tree of identifiers of chunks of the + // corresponding transaction; from the chunk being proven to the + // corresponding `data_root`. + bytes data_path = 3; + // The data chunk. + bytes chunk = 4; +} + +message Transaction { + // 1 or 2 for v1 or v2 transactions. More allowable in the future + uint32 format = 1; + // The transaction identifier. + bytes id = 2; + // Either the identifier of the previous transaction from the same + // wallet or the identifier of one of the last ?MAX_TX_ANCHOR_DEPTH blocks. + bytes last_tx = 3; + // The public key the transaction is signed with. + bytes owner = 4; + // A list of arbitrary key-value pairs + repeated Tag tags = 5; + // The address of the recipient; if any. The SHA2-256 hash of the public key. + bytes target = 6; + // The amount of Winstons to send to the recipient; if any. + BigInt quantity = 7; + // The data to upload; if any. For v2 transactions; the field is optional + // - a fee is charged based on the `data_size` field; + // data may be uploaded any time later in chunks. + bytes data = 8; + // Size in bytes of the transaction data. + BigInt data_size = 9; + // The Merkle root of the Merkle tree of data chunks. + bytes data_root = 10; + // The signature. + bytes signature = 11; + // The fee in Winstons. + BigInt reward = 12; +} + + +message Tag { + bytes name = 1; + bytes value = 2; +} diff --git a/chain/arweave/src/adapter.rs b/chain/arweave/src/adapter.rs new file mode 100644 index 0000000..70e5084 --- /dev/null +++ b/chain/arweave/src/adapter.rs @@ -0,0 +1,261 @@ +use crate::capabilities::NodeCapabilities; +use crate::{data_source::DataSource, Chain}; +use graph::blockchain as bc; +use graph::prelude::*; +use sha2::{Digest, Sha256}; +use std::collections::HashSet; + +const MATCH_ALL_WILDCARD: &str = ""; +// Size of sha256(pubkey) +const SHA256_LEN: usize = 32; + +#[derive(Clone, Debug, Default)] +pub struct TriggerFilter { + pub(crate) block_filter: ArweaveBlockFilter, + pub(crate) transaction_filter: ArweaveTransactionFilter, +} + +impl bc::TriggerFilter for TriggerFilter { + fn extend<'a>(&mut self, data_sources: impl Iterator + Clone) { + let TriggerFilter { + block_filter, + transaction_filter, + } = self; + + block_filter.extend(ArweaveBlockFilter::from_data_sources(data_sources.clone())); + transaction_filter.extend(ArweaveTransactionFilter::from_data_sources(data_sources)); + } + + fn node_capabilities(&self) -> NodeCapabilities { + NodeCapabilities {} + } + + fn extend_with_template( + &mut self, + _data_source: impl Iterator::DataSourceTemplate>, + ) { + } + + fn to_firehose_filter(self) -> Vec { + vec![] + } +} + +/// ArweaveBlockFilter will match every block regardless of source being set. +/// see docs: https://thegraph.com/docs/en/supported-networks/arweave/ +#[derive(Clone, Debug, Default)] +pub(crate) struct ArweaveTransactionFilter { + owners_pubkey: HashSet>, + owners_sha: HashSet>, + match_all: bool, +} + +impl ArweaveTransactionFilter { + pub fn matches(&self, owner: &[u8]) -> bool { + if self.match_all { + return true; + } + + if owner.len() == SHA256_LEN { + return self.owners_sha.contains(owner); + } + + self.owners_pubkey.contains(owner) || self.owners_sha.contains(&sha256(owner)) + } + + pub fn from_data_sources<'a>(iter: impl IntoIterator) -> Self { + let owners: Vec> = iter + .into_iter() + .filter(|data_source| { + data_source.source.owner.is_some() + && !data_source.mapping.transaction_handlers.is_empty() + }) + .map(|ds| match &ds.source.owner { + Some(str) if MATCH_ALL_WILDCARD.eq(str) => MATCH_ALL_WILDCARD.as_bytes().to_owned(), + owner @ _ => { + base64_url::decode(&owner.clone().unwrap_or_default()).unwrap_or_default() + } + }) + .collect(); + + let (owners_sha, long) = owners + .into_iter() + .partition::>, _>(|owner| owner.len() == SHA256_LEN); + + let (owners_pubkey, wildcard) = long + .into_iter() + .partition::>, _>(|long| long.len() != MATCH_ALL_WILDCARD.len()); + + let match_all = wildcard.len() != 0; + + let owners_sha: Vec> = owners_sha + .into_iter() + .chain::>>(owners_pubkey.iter().map(|long| sha256(&long)).collect()) + .collect(); + + Self { + match_all, + owners_pubkey: HashSet::from_iter(owners_pubkey), + owners_sha: HashSet::from_iter(owners_sha), + } + } + + pub fn extend(&mut self, other: ArweaveTransactionFilter) { + let ArweaveTransactionFilter { + owners_pubkey, + owners_sha, + match_all, + } = self; + + owners_pubkey.extend(other.owners_pubkey); + owners_sha.extend(other.owners_sha); + *match_all = *match_all || other.match_all; + } +} + +/// ArweaveBlockFilter will match every block regardless of source being set. +/// see docs: https://thegraph.com/docs/en/supported-networks/arweave/ +#[derive(Clone, Debug, Default)] +pub(crate) struct ArweaveBlockFilter { + pub trigger_every_block: bool, +} + +impl ArweaveBlockFilter { + pub fn from_data_sources<'a>(iter: impl IntoIterator) -> Self { + Self { + trigger_every_block: iter + .into_iter() + .any(|data_source| !data_source.mapping.block_handlers.is_empty()), + } + } + + pub fn extend(&mut self, other: ArweaveBlockFilter) { + self.trigger_every_block = self.trigger_every_block || other.trigger_every_block; + } +} + +fn sha256(bs: &[u8]) -> Vec { + let mut hasher = Sha256::new(); + hasher.update(bs); + let res = hasher.finalize(); + res.to_vec() +} + +#[cfg(test)] +mod test { + use std::sync::Arc; + + use graph::{prelude::Link, semver::Version}; + + use crate::data_source::{DataSource, Mapping, Source, TransactionHandler}; + + use super::{ArweaveTransactionFilter, MATCH_ALL_WILDCARD}; + + const ARWEAVE_PUBKEY_EXAMPLE: &str = "x-62w7g2yKACOgP_d04bhG8IX-AWgPrxHl2JgZBDdNLfAsidiiAaoIZPeM8K5gGvl7-8QVk79YV4OC878Ey0gXi7Atj5BouRyXnFMjJcPVXVyBoYCBuG7rJDDmh4_Ilon6vVOuHVIZ47Vb0tcgsxgxdvVFC2mn9N_SBl23pbeICNJZYOH57kf36gicuV_IwYSdqlQ0HQ_psjmg8EFqO7xzvAMP5HKW3rqTrYZxbCew2FkM734ysWckT39TpDBPx3HrFOl6obUdQWkHNOeKyzcsKFDywNgVWZOb89CYU7JFYlwX20io39ZZv0UJUOEFNjtVHkT_s0_A2O9PltsrZLLlQXZUuYASdbAPD2g_qXfhmPBZ0SXPWCDY-UVwVN1ncwYmk1F_i35IA8kAKsajaltD2wWDQn9g5mgJAWWn2xhLqkbwGbdwQMRD0-0eeuy1uzCooJQCC_bPJksoqkYwB9SGOjkayf4r4oZ2QDY4FicCsswz4Od_gud30ZWyHjWgqGzSFYFzawDBS1Gr_nu_q5otFrv20ZGTxYqGsLHWq4VHs6KjsQvzgBjfyb0etqHQEPJJmbQmY3LSogR4bxdReUHhj2EK9xIB-RKzDvDdL7fT5K0V9MjbnC2uktA0VjLlvwJ64_RhbQhxdp_zR39r-zyCXT-brPEYW1-V7Ey9K3XUE"; + const ARWEAVE_SHA_EXAMPLE: &str = "ahLxjCMCHr1ZE72VDDoaK4IKiLUUpeuo8t-M6y23DXw"; + + #[test] + fn transaction_filter_wildcard_matches_all() { + let dss = vec![ + new_datasource(None, 10), + new_datasource(Some(base64_url::encode(MATCH_ALL_WILDCARD.into())), 10), + new_datasource(Some(base64_url::encode("owner").into()), 10), + new_datasource(Some(ARWEAVE_PUBKEY_EXAMPLE.into()), 10), + ]; + + let dss: Vec<&DataSource> = dss.iter().collect(); + + let filter = ArweaveTransactionFilter::from_data_sources(dss); + assert_eq!(true, filter.matches("asdas".as_bytes())) + } + + #[test] + fn transaction_filter_match() { + let dss = vec![ + new_datasource(None, 10), + new_datasource(Some(ARWEAVE_PUBKEY_EXAMPLE.into()), 10), + ]; + + let dss: Vec<&DataSource> = dss.iter().collect(); + + let filter = ArweaveTransactionFilter::from_data_sources(dss); + assert_eq!(false, filter.matches("asdas".as_bytes())); + assert_eq!( + true, + filter.matches( + &base64_url::decode(ARWEAVE_SHA_EXAMPLE).expect("failed to parse sha example") + ) + ); + assert_eq!( + true, + filter.matches( + &base64_url::decode(ARWEAVE_PUBKEY_EXAMPLE).expect("failed to parse PK example") + ) + ) + } + + #[test] + fn transaction_filter_extend_match() { + let dss = vec![ + new_datasource(None, 10), + new_datasource(Some(ARWEAVE_SHA_EXAMPLE.into()), 10), + ]; + + let dss: Vec<&DataSource> = dss.iter().collect(); + + let filter = ArweaveTransactionFilter::from_data_sources(dss); + assert_eq!(false, filter.matches("asdas".as_bytes())); + assert_eq!( + true, + filter.matches( + &base64_url::decode(ARWEAVE_SHA_EXAMPLE).expect("failed to parse sha example") + ) + ); + assert_eq!( + true, + filter.matches( + &base64_url::decode(ARWEAVE_PUBKEY_EXAMPLE).expect("failed to parse PK example") + ) + ) + } + + #[test] + fn transaction_filter_extend_wildcard_matches_all() { + let dss = vec![ + new_datasource(None, 10), + new_datasource(Some(MATCH_ALL_WILDCARD.into()), 10), + new_datasource(Some("owner".into()), 10), + ]; + + let dss: Vec<&DataSource> = dss.iter().collect(); + + let mut filter = ArweaveTransactionFilter::default(); + + filter.extend(ArweaveTransactionFilter::from_data_sources(dss)); + assert_eq!(true, filter.matches("asdas".as_bytes())); + assert_eq!(true, filter.matches(ARWEAVE_PUBKEY_EXAMPLE.as_bytes())); + assert_eq!(true, filter.matches(ARWEAVE_SHA_EXAMPLE.as_bytes())) + } + + fn new_datasource(owner: Option, start_block: i32) -> DataSource { + DataSource { + kind: "".into(), + network: None, + name: "".into(), + source: Source { owner, start_block }, + mapping: Mapping { + api_version: Version::new(1, 2, 3), + language: "".into(), + entities: vec![], + block_handlers: vec![], + transaction_handlers: vec![TransactionHandler { + handler: "my_handler".into(), + }], + runtime: Arc::new(vec![]), + link: Link { link: "".into() }, + }, + context: Arc::new(None), + creation_block: None, + } + } +} diff --git a/chain/arweave/src/capabilities.rs b/chain/arweave/src/capabilities.rs new file mode 100644 index 0000000..27c7622 --- /dev/null +++ b/chain/arweave/src/capabilities.rs @@ -0,0 +1,37 @@ +use graph::{anyhow::Error, impl_slog_value}; +use std::cmp::{Ordering, PartialOrd}; +use std::fmt; +use std::str::FromStr; + +use crate::data_source::DataSource; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NodeCapabilities {} + +impl PartialOrd for NodeCapabilities { + fn partial_cmp(&self, _other: &Self) -> Option { + None + } +} + +impl FromStr for NodeCapabilities { + type Err = Error; + + fn from_str(_s: &str) -> Result { + Ok(NodeCapabilities {}) + } +} + +impl fmt::Display for NodeCapabilities { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("arweave") + } +} + +impl_slog_value!(NodeCapabilities, "{}"); + +impl graph::blockchain::NodeCapabilities for NodeCapabilities { + fn from_data_sources(_data_sources: &[DataSource]) -> Self { + NodeCapabilities {} + } +} diff --git a/chain/arweave/src/chain.rs b/chain/arweave/src/chain.rs new file mode 100644 index 0000000..14e52c1 --- /dev/null +++ b/chain/arweave/src/chain.rs @@ -0,0 +1,334 @@ +use graph::blockchain::{Block, BlockchainKind}; +use graph::cheap_clone::CheapClone; +use graph::data::subgraph::UnifiedMappingApiVersion; +use graph::firehose::{FirehoseEndpoint, FirehoseEndpoints}; +use graph::prelude::{MetricsRegistry, TryFutureExt}; +use graph::{ + anyhow, + blockchain::{ + block_stream::{ + BlockStreamEvent, BlockWithTriggers, FirehoseError, + FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, + }, + firehose_block_stream::FirehoseBlockStream, + BlockHash, BlockPtr, Blockchain, IngestorError, RuntimeAdapter as RuntimeAdapterTrait, + }, + components::store::DeploymentLocator, + firehose::{self as firehose, ForkStep}, + prelude::{async_trait, o, BlockNumber, ChainStore, Error, Logger, LoggerFactory}, +}; +use prost::Message; +use std::sync::Arc; + +use crate::adapter::TriggerFilter; +use crate::capabilities::NodeCapabilities; +use crate::data_source::{DataSourceTemplate, UnresolvedDataSourceTemplate}; +use crate::runtime::RuntimeAdapter; +use crate::trigger::{self, ArweaveTrigger}; +use crate::{ + codec, + data_source::{DataSource, UnresolvedDataSource}, +}; +use graph::blockchain::block_stream::{BlockStream, FirehoseCursor}; + +pub struct Chain { + logger_factory: LoggerFactory, + name: String, + firehose_endpoints: Arc, + chain_store: Arc, + metrics_registry: Arc, +} + +impl std::fmt::Debug for Chain { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "chain: arweave") + } +} + +impl Chain { + pub fn new( + logger_factory: LoggerFactory, + name: String, + chain_store: Arc, + firehose_endpoints: FirehoseEndpoints, + metrics_registry: Arc, + ) -> Self { + Chain { + logger_factory, + name, + firehose_endpoints: Arc::new(firehose_endpoints), + chain_store, + metrics_registry, + } + } +} + +#[async_trait] +impl Blockchain for Chain { + const KIND: BlockchainKind = BlockchainKind::Arweave; + + type Block = codec::Block; + + type DataSource = DataSource; + + type UnresolvedDataSource = UnresolvedDataSource; + + type DataSourceTemplate = DataSourceTemplate; + + type UnresolvedDataSourceTemplate = UnresolvedDataSourceTemplate; + + type TriggerData = crate::trigger::ArweaveTrigger; + + type MappingTrigger = crate::trigger::ArweaveTrigger; + + type TriggerFilter = crate::adapter::TriggerFilter; + + type NodeCapabilities = crate::capabilities::NodeCapabilities; + + fn triggers_adapter( + &self, + _loc: &DeploymentLocator, + _capabilities: &Self::NodeCapabilities, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + let adapter = TriggersAdapter {}; + Ok(Arc::new(adapter)) + } + + async fn new_firehose_block_stream( + &self, + deployment: DeploymentLocator, + block_cursor: FirehoseCursor, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + let adapter = self + .triggers_adapter(&deployment, &NodeCapabilities {}, unified_api_version) + .expect(&format!("no adapter for network {}", self.name,)); + + let firehose_endpoint = match self.firehose_endpoints.random() { + Some(e) => e.clone(), + None => return Err(anyhow::format_err!("no firehose endpoint available")), + }; + + let logger = self + .logger_factory + .subgraph_logger(&deployment) + .new(o!("component" => "FirehoseBlockStream")); + + let firehose_mapper = Arc::new(FirehoseMapper { + endpoint: firehose_endpoint.cheap_clone(), + }); + + Ok(Box::new(FirehoseBlockStream::new( + deployment.hash, + firehose_endpoint, + subgraph_current_block, + block_cursor, + firehose_mapper, + adapter, + filter, + start_blocks, + logger, + self.metrics_registry.clone(), + ))) + } + + async fn new_polling_block_stream( + &self, + _deployment: DeploymentLocator, + _start_blocks: Vec, + _subgraph_current_block: Option, + _filter: Arc, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + panic!("Arweave does not support polling block stream") + } + + fn chain_store(&self) -> Arc { + self.chain_store.clone() + } + + async fn block_pointer_from_number( + &self, + logger: &Logger, + number: BlockNumber, + ) -> Result { + let firehose_endpoint = match self.firehose_endpoints.random() { + Some(e) => e.clone(), + None => return Err(anyhow::format_err!("no firehose endpoint available").into()), + }; + + firehose_endpoint + .block_ptr_for_number::(logger, number) + .map_err(Into::into) + .await + } + + fn runtime_adapter(&self) -> Arc> { + Arc::new(RuntimeAdapter {}) + } + + fn is_firehose_supported(&self) -> bool { + true + } +} + +pub struct TriggersAdapter {} + +#[async_trait] +impl TriggersAdapterTrait for TriggersAdapter { + async fn scan_triggers( + &self, + _from: BlockNumber, + _to: BlockNumber, + _filter: &TriggerFilter, + ) -> Result>, Error> { + panic!("Should never be called since not used by FirehoseBlockStream") + } + + async fn triggers_in_block( + &self, + _logger: &Logger, + block: codec::Block, + filter: &TriggerFilter, + ) -> Result, Error> { + // TODO: Find the best place to introduce an `Arc` and avoid this clone. + let shared_block = Arc::new(block.clone()); + + let TriggerFilter { + block_filter, + transaction_filter, + } = filter; + + let txs = block + .clone() + .txs + .into_iter() + .filter(|tx| transaction_filter.matches(&tx.owner)) + .map(|tx| trigger::TransactionWithBlockPtr { + tx: Arc::new(tx.clone()), + block: shared_block.clone(), + }) + .collect::>(); + + let mut trigger_data: Vec<_> = txs + .into_iter() + .map(|tx| ArweaveTrigger::Transaction(Arc::new(tx))) + .collect(); + + if block_filter.trigger_every_block { + trigger_data.push(ArweaveTrigger::Block(shared_block.cheap_clone())); + } + + Ok(BlockWithTriggers::new(block, trigger_data)) + } + + async fn is_on_main_chain(&self, _ptr: BlockPtr) -> Result { + panic!("Should never be called since not used by FirehoseBlockStream") + } + + async fn ancestor_block( + &self, + _ptr: BlockPtr, + _offset: BlockNumber, + ) -> Result, Error> { + panic!("Should never be called since FirehoseBlockStream cannot resolve it") + } + + /// Panics if `block` is genesis. + /// But that's ok since this is only called when reverting `block`. + async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { + // FIXME (Arweave): Might not be necessary for Arweave support for now + Ok(Some(BlockPtr { + hash: BlockHash::from(vec![0xff; 48]), + number: block.number.saturating_sub(1), + })) + } +} + +pub struct FirehoseMapper { + endpoint: Arc, +} + +#[async_trait] +impl FirehoseMapperTrait for FirehoseMapper { + async fn to_block_stream_event( + &self, + logger: &Logger, + response: &firehose::Response, + adapter: &Arc>, + filter: &TriggerFilter, + ) -> Result, FirehoseError> { + let step = ForkStep::from_i32(response.step).unwrap_or_else(|| { + panic!( + "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", + response.step + ) + }); + + let any_block = response + .block + .as_ref() + .expect("block payload information should always be present"); + + // Right now, this is done in all cases but in reality, with how the BlockStreamEvent::Revert + // is defined right now, only block hash and block number is necessary. However, this information + // is not part of the actual bstream::BlockResponseV2 payload. As such, we need to decode the full + // block which is useless. + // + // Check about adding basic information about the block in the bstream::BlockResponseV2 or maybe + // define a slimmed down stuct that would decode only a few fields and ignore all the rest. + let block = codec::Block::decode(any_block.value.as_ref())?; + + use ForkStep::*; + match step { + StepNew => Ok(BlockStreamEvent::ProcessBlock( + adapter.triggers_in_block(logger, block, filter).await?, + FirehoseCursor::from(response.cursor.clone()), + )), + + StepUndo => { + let parent_ptr = block + .parent_ptr() + .expect("Genesis block should never be reverted"); + + Ok(BlockStreamEvent::Revert( + parent_ptr, + FirehoseCursor::from(response.cursor.clone()), + )) + } + + StepIrreversible => { + panic!("irreversible step is not handled and should not be requested in the Firehose request") + } + + StepUnknown => { + panic!("unknown step should not happen in the Firehose response") + } + } + } + + async fn block_ptr_for_number( + &self, + logger: &Logger, + number: BlockNumber, + ) -> Result { + self.endpoint + .block_ptr_for_number::(logger, number) + .await + } + + // # FIXME + // + // the final block of arweave is itself in the current implementation + async fn final_block_ptr_for( + &self, + _logger: &Logger, + block: &codec::Block, + ) -> Result { + Ok(block.ptr()) + } +} diff --git a/chain/arweave/src/codec.rs b/chain/arweave/src/codec.rs new file mode 100644 index 0000000..09da7fe --- /dev/null +++ b/chain/arweave/src/codec.rs @@ -0,0 +1,37 @@ +#[rustfmt::skip] +#[path = "protobuf/sf.arweave.r#type.v1.rs"] +mod pbcodec; + +use graph::{blockchain::Block as BlockchainBlock, blockchain::BlockPtr, prelude::BlockNumber}; + +pub use pbcodec::*; + +impl BlockchainBlock for Block { + fn number(&self) -> i32 { + BlockNumber::try_from(self.height).unwrap() + } + + fn ptr(&self) -> BlockPtr { + BlockPtr { + hash: self.indep_hash.clone().into(), + number: self.number(), + } + } + + fn parent_ptr(&self) -> Option { + if self.height == 0 { + return None; + } + + Some(BlockPtr { + hash: self.previous_block.clone().into(), + number: self.number().saturating_sub(1), + }) + } +} + +impl AsRef<[u8]> for BigInt { + fn as_ref(&self) -> &[u8] { + self.bytes.as_ref() + } +} diff --git a/chain/arweave/src/data_source.rs b/chain/arweave/src/data_source.rs new file mode 100644 index 0000000..d378174 --- /dev/null +++ b/chain/arweave/src/data_source.rs @@ -0,0 +1,376 @@ +use graph::blockchain::{Block, TriggerWithHandler}; +use graph::components::store::StoredDynamicDataSource; +use graph::data::subgraph::DataSourceContext; +use graph::prelude::SubgraphManifestValidationError; +use graph::{ + anyhow::{anyhow, Error}, + blockchain::{self, Blockchain}, + prelude::{ + async_trait, info, BlockNumber, CheapClone, DataSourceTemplateInfo, Deserialize, Link, + LinkResolver, Logger, + }, + semver, +}; +use std::{convert::TryFrom, sync::Arc}; + +use crate::chain::Chain; +use crate::trigger::ArweaveTrigger; + +pub const ARWEAVE_KIND: &str = "arweave"; + +/// Runtime representation of a data source. +#[derive(Clone, Debug)] +pub struct DataSource { + pub kind: String, + pub network: Option, + pub name: String, + pub(crate) source: Source, + pub mapping: Mapping, + pub context: Arc>, + pub creation_block: Option, +} + +impl blockchain::DataSource for DataSource { + // FIXME + // + // need to decode the base64url encoding? + fn address(&self) -> Option<&[u8]> { + self.source.owner.as_ref().map(String::as_bytes) + } + + fn start_block(&self) -> BlockNumber { + self.source.start_block + } + + fn match_and_decode( + &self, + trigger: &::TriggerData, + block: &Arc<::Block>, + _logger: &Logger, + ) -> Result>, Error> { + if self.source.start_block > block.number() { + return Ok(None); + } + + let handler = match trigger { + // A block trigger matches if a block handler is present. + ArweaveTrigger::Block(_) => match self.handler_for_block() { + Some(handler) => &handler.handler, + None => return Ok(None), + }, + // A transaction trigger matches if a transaction handler is present. + ArweaveTrigger::Transaction(_) => match self.handler_for_transaction() { + Some(handler) => &handler.handler, + None => return Ok(None), + }, + }; + + Ok(Some(TriggerWithHandler::::new( + trigger.cheap_clone(), + handler.to_owned(), + block.ptr(), + ))) + } + + fn name(&self) -> &str { + &self.name + } + + fn kind(&self) -> &str { + &self.kind + } + + fn network(&self) -> Option<&str> { + self.network.as_ref().map(|s| s.as_str()) + } + + fn context(&self) -> Arc> { + self.context.cheap_clone() + } + + fn creation_block(&self) -> Option { + self.creation_block + } + + fn is_duplicate_of(&self, other: &Self) -> bool { + let DataSource { + kind, + network, + name, + source, + mapping, + context, + + // The creation block is ignored for detection duplicate data sources. + // Contract ABI equality is implicit in `source` and `mapping.abis` equality. + creation_block: _, + } = self; + + // mapping_request_sender, host_metrics, and (most of) host_exports are operational structs + // used at runtime but not needed to define uniqueness; each runtime host should be for a + // unique data source. + kind == &other.kind + && network == &other.network + && name == &other.name + && source == &other.source + && mapping.block_handlers == other.mapping.block_handlers + && context == &other.context + } + + fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { + // FIXME (Arweave): Implement me! + todo!() + } + + fn from_stored_dynamic_data_source( + _template: &DataSourceTemplate, + _stored: StoredDynamicDataSource, + ) -> Result { + // FIXME (Arweave): Implement me correctly + todo!() + } + + fn validate(&self) -> Vec { + let mut errors = Vec::new(); + + if self.kind != ARWEAVE_KIND { + errors.push(anyhow!( + "data source has invalid `kind`, expected {} but found {}", + ARWEAVE_KIND, + self.kind + )) + } + + // Validate that there is a `source` address if there are transaction handlers + let no_source_address = self.address().is_none(); + let has_transaction_handlers = !self.mapping.transaction_handlers.is_empty(); + if no_source_address && has_transaction_handlers { + errors.push(SubgraphManifestValidationError::SourceAddressRequired.into()); + }; + + // Validate that there are no more than one of both block handlers and transaction handlers + if self.mapping.block_handlers.len() > 1 { + errors.push(anyhow!("data source has duplicated block handlers")); + } + if self.mapping.transaction_handlers.len() > 1 { + errors.push(anyhow!("data source has duplicated transaction handlers")); + } + + errors + } + + fn api_version(&self) -> semver::Version { + self.mapping.api_version.clone() + } + + fn runtime(&self) -> Option>> { + Some(self.mapping.runtime.cheap_clone()) + } +} + +impl DataSource { + fn from_manifest( + kind: String, + network: Option, + name: String, + source: Source, + mapping: Mapping, + context: Option, + ) -> Result { + // Data sources in the manifest are created "before genesis" so they have no creation block. + let creation_block = None; + + Ok(DataSource { + kind, + network, + name, + source, + mapping, + context: Arc::new(context), + creation_block, + }) + } + + fn handler_for_block(&self) -> Option<&MappingBlockHandler> { + self.mapping.block_handlers.first() + } + + fn handler_for_transaction(&self) -> Option<&TransactionHandler> { + self.mapping.transaction_handlers.first() + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub struct UnresolvedDataSource { + pub kind: String, + pub network: Option, + pub name: String, + pub(crate) source: Source, + pub mapping: UnresolvedMapping, + pub context: Option, +} + +#[async_trait] +impl blockchain::UnresolvedDataSource for UnresolvedDataSource { + async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + _manifest_idx: u32, + ) -> Result { + let UnresolvedDataSource { + kind, + network, + name, + source, + mapping, + context, + } = self; + + info!(logger, "Resolve data source"; "name" => &name, "source_address" => format_args!("{:?}", base64_url::encode(&source.owner.clone().unwrap_or_default())), "source_start_block" => source.start_block); + + let mapping = mapping.resolve(resolver, logger).await?; + + DataSource::from_manifest(kind, network, name, source, mapping, context) + } +} + +/// # TODO +/// +/// add templates for arweave subgraphs +impl TryFrom> for DataSource { + type Error = Error; + + fn try_from(_info: DataSourceTemplateInfo) -> Result { + Err(anyhow!("Arweave subgraphs do not support templates")) + } +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +pub struct BaseDataSourceTemplate { + pub kind: String, + pub network: Option, + pub name: String, + pub mapping: M, +} + +pub type UnresolvedDataSourceTemplate = BaseDataSourceTemplate; +pub type DataSourceTemplate = BaseDataSourceTemplate; + +#[async_trait] +impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { + async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + _manifest_idx: u32, + ) -> Result { + let UnresolvedDataSourceTemplate { + kind, + network, + name, + mapping, + } = self; + + info!(logger, "Resolve data source template"; "name" => &name); + + Ok(DataSourceTemplate { + kind, + network, + name, + mapping: mapping.resolve(resolver, logger).await?, + }) + } +} + +impl blockchain::DataSourceTemplate for DataSourceTemplate { + fn name(&self) -> &str { + &self.name + } + + fn api_version(&self) -> semver::Version { + self.mapping.api_version.clone() + } + + fn runtime(&self) -> Option>> { + Some(self.mapping.runtime.cheap_clone()) + } + + fn manifest_idx(&self) -> u32 { + unreachable!("arweave does not support dynamic data sources") + } +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UnresolvedMapping { + pub api_version: String, + pub language: String, + pub entities: Vec, + #[serde(default)] + pub block_handlers: Vec, + #[serde(default)] + pub transaction_handlers: Vec, + pub file: Link, +} + +impl UnresolvedMapping { + pub async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + ) -> Result { + let UnresolvedMapping { + api_version, + language, + entities, + block_handlers, + transaction_handlers, + file: link, + } = self; + + let api_version = semver::Version::parse(&api_version)?; + + info!(logger, "Resolve mapping"; "link" => &link.link); + let module_bytes = resolver.cat(logger, &link).await?; + + Ok(Mapping { + api_version, + language, + entities, + block_handlers, + transaction_handlers, + runtime: Arc::new(module_bytes), + link, + }) + } +} + +#[derive(Clone, Debug)] +pub struct Mapping { + pub api_version: semver::Version, + pub language: String, + pub entities: Vec, + pub block_handlers: Vec, + pub transaction_handlers: Vec, + pub runtime: Arc>, + pub link: Link, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct MappingBlockHandler { + pub handler: String, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct TransactionHandler { + pub handler: String, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub(crate) struct Source { + // A data source that does not have an owner can only have block handlers. + pub(crate) owner: Option, + #[serde(rename = "startBlock", default)] + pub(crate) start_block: BlockNumber, +} diff --git a/chain/arweave/src/lib.rs b/chain/arweave/src/lib.rs new file mode 100644 index 0000000..a497e77 --- /dev/null +++ b/chain/arweave/src/lib.rs @@ -0,0 +1,10 @@ +mod adapter; +mod capabilities; +mod chain; +mod codec; +mod data_source; +mod runtime; +mod trigger; + +pub use crate::chain::Chain; +pub use codec::Block; diff --git a/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs b/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs new file mode 100644 index 0000000..98a1359 --- /dev/null +++ b/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs @@ -0,0 +1,141 @@ +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BigInt { + #[prost(bytes="vec", tag="1")] + pub bytes: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Block { + /// Firehose block version (unrelated to Arweave block version) + #[prost(uint32, tag="1")] + pub ver: u32, + /// The block identifier + #[prost(bytes="vec", tag="2")] + pub indep_hash: ::prost::alloc::vec::Vec, + /// The nonce chosen to solve the mining problem + #[prost(bytes="vec", tag="3")] + pub nonce: ::prost::alloc::vec::Vec, + /// `indep_hash` of the previous block in the weave + #[prost(bytes="vec", tag="4")] + pub previous_block: ::prost::alloc::vec::Vec, + /// POSIX time of block discovery + #[prost(uint64, tag="5")] + pub timestamp: u64, + /// POSIX time of the last difficulty retarget + #[prost(uint64, tag="6")] + pub last_retarget: u64, + /// Mining difficulty; the number `hash` must be greater than. + #[prost(message, optional, tag="7")] + pub diff: ::core::option::Option, + /// How many blocks have passed since the genesis block + #[prost(uint64, tag="8")] + pub height: u64, + /// Mining solution hash of the block; must satisfy the mining difficulty + #[prost(bytes="vec", tag="9")] + pub hash: ::prost::alloc::vec::Vec, + /// Merkle root of the tree of Merkle roots of block's transactions' data. + #[prost(bytes="vec", tag="10")] + pub tx_root: ::prost::alloc::vec::Vec, + /// Transactions contained within this block + #[prost(message, repeated, tag="11")] + pub txs: ::prost::alloc::vec::Vec, + /// The root hash of the Merkle Patricia Tree containing + /// all wallet (account) balances and the identifiers + /// of the last transactions posted by them; if any. + #[prost(bytes="vec", tag="12")] + pub wallet_list: ::prost::alloc::vec::Vec, + /// (string or) Address of the account to receive the block rewards. Can also be unclaimed which is encoded as a null byte + #[prost(bytes="vec", tag="13")] + pub reward_addr: ::prost::alloc::vec::Vec, + /// Tags that a block producer can add to a block + #[prost(message, repeated, tag="14")] + pub tags: ::prost::alloc::vec::Vec, + /// Size of reward pool + #[prost(message, optional, tag="15")] + pub reward_pool: ::core::option::Option, + /// Size of the weave in bytes + #[prost(message, optional, tag="16")] + pub weave_size: ::core::option::Option, + /// Size of this block in bytes + #[prost(message, optional, tag="17")] + pub block_size: ::core::option::Option, + /// Required after the version 1.8 fork. Zero otherwise. + /// The sum of the average number of hashes computed + /// by the network to produce the past blocks including this one. + #[prost(message, optional, tag="18")] + pub cumulative_diff: ::core::option::Option, + /// Required after the version 1.8 fork. Null byte otherwise. + /// The Merkle root of the block index - the list of {`indep_hash`; `weave_size`; `tx_root`} triplets + #[prost(bytes="vec", tag="20")] + pub hash_list_merkle: ::prost::alloc::vec::Vec, + /// The proof of access; Used after v2.4 only; set as defaults otherwise + #[prost(message, optional, tag="21")] + pub poa: ::core::option::Option, +} +/// A succinct proof of access to a recall byte found in a TX +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProofOfAccess { + /// The recall byte option chosen; global offset of index byte + #[prost(string, tag="1")] + pub option: ::prost::alloc::string::String, + /// The path through the Merkle tree of transactions' `data_root`s; + /// from the `data_root` being proven to the corresponding `tx_root` + #[prost(bytes="vec", tag="2")] + pub tx_path: ::prost::alloc::vec::Vec, + /// The path through the Merkle tree of identifiers of chunks of the + /// corresponding transaction; from the chunk being proven to the + /// corresponding `data_root`. + #[prost(bytes="vec", tag="3")] + pub data_path: ::prost::alloc::vec::Vec, + /// The data chunk. + #[prost(bytes="vec", tag="4")] + pub chunk: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Transaction { + /// 1 or 2 for v1 or v2 transactions. More allowable in the future + #[prost(uint32, tag="1")] + pub format: u32, + /// The transaction identifier. + #[prost(bytes="vec", tag="2")] + pub id: ::prost::alloc::vec::Vec, + /// Either the identifier of the previous transaction from the same + /// wallet or the identifier of one of the last ?MAX_TX_ANCHOR_DEPTH blocks. + #[prost(bytes="vec", tag="3")] + pub last_tx: ::prost::alloc::vec::Vec, + /// The public key the transaction is signed with. + #[prost(bytes="vec", tag="4")] + pub owner: ::prost::alloc::vec::Vec, + /// A list of arbitrary key-value pairs + #[prost(message, repeated, tag="5")] + pub tags: ::prost::alloc::vec::Vec, + /// The address of the recipient; if any. The SHA2-256 hash of the public key. + #[prost(bytes="vec", tag="6")] + pub target: ::prost::alloc::vec::Vec, + /// The amount of Winstons to send to the recipient; if any. + #[prost(message, optional, tag="7")] + pub quantity: ::core::option::Option, + /// The data to upload; if any. For v2 transactions; the field is optional + /// - a fee is charged based on the `data_size` field; + /// data may be uploaded any time later in chunks. + #[prost(bytes="vec", tag="8")] + pub data: ::prost::alloc::vec::Vec, + /// Size in bytes of the transaction data. + #[prost(message, optional, tag="9")] + pub data_size: ::core::option::Option, + /// The Merkle root of the Merkle tree of data chunks. + #[prost(bytes="vec", tag="10")] + pub data_root: ::prost::alloc::vec::Vec, + /// The signature. + #[prost(bytes="vec", tag="11")] + pub signature: ::prost::alloc::vec::Vec, + /// The fee in Winstons. + #[prost(message, optional, tag="12")] + pub reward: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Tag { + #[prost(bytes="vec", tag="1")] + pub name: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="2")] + pub value: ::prost::alloc::vec::Vec, +} diff --git a/chain/arweave/src/runtime/abi.rs b/chain/arweave/src/runtime/abi.rs new file mode 100644 index 0000000..4f3d400 --- /dev/null +++ b/chain/arweave/src/runtime/abi.rs @@ -0,0 +1,191 @@ +use crate::codec; +use crate::trigger::TransactionWithBlockPtr; +use graph::runtime::gas::GasCounter; +use graph::runtime::{asc_new, AscHeap, AscPtr, DeterministicHostError, ToAscObj}; +use graph_runtime_wasm::asc_abi::class::{Array, Uint8Array}; + +pub(crate) use super::generated::*; + +impl ToAscObj for codec::Tag { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscTag { + name: asc_new(heap, self.name.as_slice(), gas)?, + value: asc_new(heap, self.value.as_slice(), gas)?, + }) + } +} + +impl ToAscObj for Vec> { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let content = self + .into_iter() + .map(|x| asc_new(heap, x.as_slice(), gas)) + .collect::>, _>>()?; + Ok(AscTransactionArray(Array::new(&*content, heap, gas)?)) + } +} + +impl ToAscObj for Vec { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let content = self + .iter() + .map(|x| asc_new(heap, x, gas)) + .collect::, _>>()?; + Ok(AscTagArray(Array::new(&*content, heap, gas)?)) + } +} + +impl ToAscObj for codec::ProofOfAccess { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscProofOfAccess { + option: asc_new(heap, &self.option, gas)?, + tx_path: asc_new(heap, self.tx_path.as_slice(), gas)?, + data_path: asc_new(heap, self.data_path.as_slice(), gas)?, + chunk: asc_new(heap, self.chunk.as_slice(), gas)?, + }) + } +} + +impl ToAscObj for codec::Transaction { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscTransaction { + format: self.format, + id: asc_new(heap, self.id.as_slice(), gas)?, + last_tx: asc_new(heap, self.last_tx.as_slice(), gas)?, + owner: asc_new(heap, self.owner.as_slice(), gas)?, + tags: asc_new(heap, &self.tags, gas)?, + target: asc_new(heap, self.target.as_slice(), gas)?, + quantity: asc_new( + heap, + self.quantity + .as_ref() + .map(|b| b.as_ref()) + .unwrap_or_default(), + gas, + )?, + data: asc_new(heap, self.data.as_slice(), gas)?, + data_size: asc_new( + heap, + self.data_size + .as_ref() + .map(|b| b.as_ref()) + .unwrap_or_default(), + gas, + )?, + data_root: asc_new(heap, self.data_root.as_slice(), gas)?, + signature: asc_new(heap, self.signature.as_slice(), gas)?, + reward: asc_new( + heap, + self.reward.as_ref().map(|b| b.as_ref()).unwrap_or_default(), + gas, + )?, + }) + } +} + +impl ToAscObj for codec::Block { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscBlock { + indep_hash: asc_new(heap, self.indep_hash.as_slice(), gas)?, + nonce: asc_new(heap, self.nonce.as_slice(), gas)?, + previous_block: asc_new(heap, self.previous_block.as_slice(), gas)?, + timestamp: self.timestamp, + last_retarget: self.last_retarget, + diff: asc_new( + heap, + self.diff.as_ref().map(|b| b.as_ref()).unwrap_or_default(), + gas, + )?, + height: self.height, + hash: asc_new(heap, self.hash.as_slice(), gas)?, + tx_root: asc_new(heap, self.tx_root.as_slice(), gas)?, + txs: asc_new( + heap, + &self + .txs + .iter() + .map(|tx| tx.id.clone().into()) + .collect::>>(), + gas, + )?, + wallet_list: asc_new(heap, self.wallet_list.as_slice(), gas)?, + reward_addr: asc_new(heap, self.reward_addr.as_slice(), gas)?, + tags: asc_new(heap, &self.tags, gas)?, + reward_pool: asc_new( + heap, + self.reward_pool + .as_ref() + .map(|b| b.as_ref()) + .unwrap_or_default(), + gas, + )?, + weave_size: asc_new( + heap, + self.weave_size + .as_ref() + .map(|b| b.as_ref()) + .unwrap_or_default(), + gas, + )?, + block_size: asc_new( + heap, + self.block_size + .as_ref() + .map(|b| b.as_ref()) + .unwrap_or_default(), + gas, + )?, + cumulative_diff: asc_new( + heap, + self.cumulative_diff + .as_ref() + .map(|b| b.as_ref()) + .unwrap_or_default(), + gas, + )?, + hash_list_merkle: asc_new(heap, self.hash_list_merkle.as_slice(), gas)?, + poa: self + .poa + .as_ref() + .map(|poa| asc_new(heap, poa, gas)) + .unwrap_or(Ok(AscPtr::null()))?, + }) + } +} + +impl ToAscObj for TransactionWithBlockPtr { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscTransactionWithBlockPtr { + tx: asc_new(heap, &self.tx.as_ref(), gas)?, + block: asc_new(heap, self.block.as_ref(), gas)?, + }) + } +} diff --git a/chain/arweave/src/runtime/generated.rs b/chain/arweave/src/runtime/generated.rs new file mode 100644 index 0000000..e8a10fd --- /dev/null +++ b/chain/arweave/src/runtime/generated.rs @@ -0,0 +1,128 @@ +use graph::runtime::{AscIndexId, AscPtr, AscType, DeterministicHostError, IndexForAscTypeId}; +use graph::semver::Version; +use graph_runtime_derive::AscType; +use graph_runtime_wasm::asc_abi::class::{Array, AscString, Uint8Array}; + +#[repr(C)] +#[derive(AscType, Default)] +pub struct AscBlock { + pub timestamp: u64, + pub last_retarget: u64, + pub height: u64, + pub indep_hash: AscPtr, + pub nonce: AscPtr, + pub previous_block: AscPtr, + pub diff: AscPtr, + pub hash: AscPtr, + pub tx_root: AscPtr, + pub txs: AscPtr, + pub wallet_list: AscPtr, + pub reward_addr: AscPtr, + pub tags: AscPtr, + pub reward_pool: AscPtr, + pub weave_size: AscPtr, + pub block_size: AscPtr, + pub cumulative_diff: AscPtr, + pub hash_list_merkle: AscPtr, + pub poa: AscPtr, +} + +impl AscIndexId for AscBlock { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveBlock; +} + +#[repr(C)] +#[derive(AscType)] +pub struct AscProofOfAccess { + pub option: AscPtr, + pub tx_path: AscPtr, + pub data_path: AscPtr, + pub chunk: AscPtr, +} + +impl AscIndexId for AscProofOfAccess { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveProofOfAccess; +} + +#[repr(C)] +#[derive(AscType)] +pub struct AscTransaction { + pub format: u32, + pub id: AscPtr, + pub last_tx: AscPtr, + pub owner: AscPtr, + pub tags: AscPtr, + pub target: AscPtr, + pub quantity: AscPtr, + pub data: AscPtr, + pub data_size: AscPtr, + pub data_root: AscPtr, + pub signature: AscPtr, + pub reward: AscPtr, +} + +impl AscIndexId for AscTransaction { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTransaction; +} + +#[repr(C)] +#[derive(AscType)] +pub struct AscTag { + pub name: AscPtr, + pub value: AscPtr, +} + +impl AscIndexId for AscTag { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTag; +} + +#[repr(C)] +pub struct AscTransactionArray(pub(crate) Array>); + +impl AscType for AscTransactionArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl AscIndexId for AscTransactionArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTransactionArray; +} + +#[repr(C)] +pub struct AscTagArray(pub(crate) Array>); + +impl AscType for AscTagArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl AscIndexId for AscTagArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTagArray; +} + +#[repr(C)] +#[derive(AscType)] +pub struct AscTransactionWithBlockPtr { + pub tx: AscPtr, + pub block: AscPtr, +} + +impl AscIndexId for AscTransactionWithBlockPtr { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTransactionWithBlockPtr; +} diff --git a/chain/arweave/src/runtime/mod.rs b/chain/arweave/src/runtime/mod.rs new file mode 100644 index 0000000..f44391c --- /dev/null +++ b/chain/arweave/src/runtime/mod.rs @@ -0,0 +1,6 @@ +pub use runtime_adapter::RuntimeAdapter; + +pub mod abi; +pub mod runtime_adapter; + +mod generated; diff --git a/chain/arweave/src/runtime/runtime_adapter.rs b/chain/arweave/src/runtime/runtime_adapter.rs new file mode 100644 index 0000000..c5fa9e1 --- /dev/null +++ b/chain/arweave/src/runtime/runtime_adapter.rs @@ -0,0 +1,11 @@ +use crate::{data_source::DataSource, Chain}; +use blockchain::HostFn; +use graph::{anyhow::Error, blockchain}; + +pub struct RuntimeAdapter {} + +impl blockchain::RuntimeAdapter for RuntimeAdapter { + fn host_fns(&self, _ds: &DataSource) -> Result, Error> { + Ok(vec![]) + } +} diff --git a/chain/arweave/src/trigger.rs b/chain/arweave/src/trigger.rs new file mode 100644 index 0000000..9d2f7ad --- /dev/null +++ b/chain/arweave/src/trigger.rs @@ -0,0 +1,137 @@ +use graph::blockchain::Block; +use graph::blockchain::TriggerData; +use graph::cheap_clone::CheapClone; +use graph::prelude::web3::types::H256; +use graph::prelude::BlockNumber; +use graph::runtime::asc_new; +use graph::runtime::gas::GasCounter; +use graph::runtime::AscHeap; +use graph::runtime::AscPtr; +use graph::runtime::DeterministicHostError; +use graph_runtime_wasm::module::ToAscPtr; +use std::{cmp::Ordering, sync::Arc}; + +use crate::codec; + +// Logging the block is too verbose, so this strips the block from the trigger for Debug. +impl std::fmt::Debug for ArweaveTrigger { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + #[derive(Debug)] + pub enum MappingTriggerWithoutBlock { + Block, + Transaction(Arc), + } + + let trigger_without_block = match self { + ArweaveTrigger::Block(_) => MappingTriggerWithoutBlock::Block, + ArweaveTrigger::Transaction(tx) => { + MappingTriggerWithoutBlock::Transaction(tx.tx.clone()) + } + }; + + write!(f, "{:?}", trigger_without_block) + } +} + +impl ToAscPtr for ArweaveTrigger { + fn to_asc_ptr( + self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + Ok(match self { + ArweaveTrigger::Block(block) => asc_new(heap, block.as_ref(), gas)?.erase(), + ArweaveTrigger::Transaction(tx) => asc_new(heap, tx.as_ref(), gas)?.erase(), + }) + } +} + +#[derive(Clone)] +pub enum ArweaveTrigger { + Block(Arc), + Transaction(Arc), +} + +impl CheapClone for ArweaveTrigger { + fn cheap_clone(&self) -> ArweaveTrigger { + match self { + ArweaveTrigger::Block(block) => ArweaveTrigger::Block(block.cheap_clone()), + ArweaveTrigger::Transaction(tx) => ArweaveTrigger::Transaction(tx.cheap_clone()), + } + } +} + +impl PartialEq for ArweaveTrigger { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Block(a_ptr), Self::Block(b_ptr)) => a_ptr == b_ptr, + (Self::Transaction(a_tx), Self::Transaction(b_tx)) => a_tx.tx.id == b_tx.tx.id, + _ => false, + } + } +} + +impl Eq for ArweaveTrigger {} + +impl ArweaveTrigger { + pub fn block_number(&self) -> BlockNumber { + match self { + ArweaveTrigger::Block(block) => block.number(), + ArweaveTrigger::Transaction(tx) => tx.block.number(), + } + } + + pub fn block_hash(&self) -> H256 { + match self { + ArweaveTrigger::Block(block) => block.ptr().hash_as_h256(), + ArweaveTrigger::Transaction(tx) => tx.block.ptr().hash_as_h256(), + } + } +} + +impl Ord for ArweaveTrigger { + fn cmp(&self, other: &Self) -> Ordering { + match (self, other) { + // Keep the order when comparing two block triggers + (Self::Block(..), Self::Block(..)) => Ordering::Equal, + + // Block triggers always come last + (Self::Block(..), _) => Ordering::Greater, + (_, Self::Block(..)) => Ordering::Less, + + // Execution outcomes have no intrinsic ordering information so we keep the order in + // which they are included in the `txs` field of `Block`. + (Self::Transaction(..), Self::Transaction(..)) => Ordering::Equal, + } + } +} + +impl PartialOrd for ArweaveTrigger { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl TriggerData for ArweaveTrigger { + fn error_context(&self) -> std::string::String { + match self { + ArweaveTrigger::Block(..) => { + format!("Block #{} ({})", self.block_number(), self.block_hash()) + } + ArweaveTrigger::Transaction(tx) => { + format!( + "Tx #{}, block #{}({})", + base64_url::encode(&tx.tx.id), + self.block_number(), + self.block_hash() + ) + } + } + } +} + +pub struct TransactionWithBlockPtr { + // REVIEW: Do we want to actually also have those two below behind an `Arc` wrapper? + pub tx: Arc, + pub block: Arc, +} diff --git a/chain/common/Cargo.toml b/chain/common/Cargo.toml new file mode 100644 index 0000000..364e8e3 --- /dev/null +++ b/chain/common/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "graph-chain-common" +version = "0.27.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +protobuf = "3.0.2" +protobuf-parse = "3.0.2" +anyhow = "1" +heck = "0.4" diff --git a/chain/common/src/lib.rs b/chain/common/src/lib.rs new file mode 100644 index 0000000..f1e5aeb --- /dev/null +++ b/chain/common/src/lib.rs @@ -0,0 +1,218 @@ +use std::collections::HashMap; +use std::fmt::Debug; + +use anyhow::Error; +use protobuf::descriptor::field_descriptor_proto::Label; +use protobuf::descriptor::field_descriptor_proto::Type; +use protobuf::descriptor::DescriptorProto; +use protobuf::descriptor::FieldDescriptorProto; +use protobuf::descriptor::OneofDescriptorProto; +use protobuf::Message; +use protobuf::UnknownValueRef; +use std::convert::From; +use std::path::Path; + +const REQUIRED_ID: u32 = 66001; + +#[derive(Debug, Clone)] +pub struct Field { + pub name: String, + pub type_name: String, + pub required: bool, + pub is_enum: bool, + pub is_array: bool, + pub fields: Vec, +} + +#[derive(Debug, Clone)] +pub struct PType { + pub name: String, + pub fields: Vec, + pub descriptor: DescriptorProto, +} + +impl PType { + pub fn fields(&self) -> Option { + let mut v = Vec::new(); + if let Some(vv) = self.req_fields_as_string() { + v.push(vv); + } + if let Some(vv) = self.enum_fields_as_string() { + v.push(vv); + } + + if v.len() < 1 { + None + } else { + Some(v.join(",")) + } + } + + pub fn has_req_fields(&self) -> bool { + self.fields.iter().any(|f| f.required) + } + + pub fn req_fields_as_string(&self) -> Option { + if self.has_req_fields() { + Some(format!( + "__required__{{{}}}", + self.fields + .iter() + .filter(|f| f.required) + .map(|f| format!("{}: {}", f.name, f.type_name)) + .collect::>() + .join(",") + )) + } else { + None + } + } + + pub fn has_enum(&self) -> bool { + self.fields.iter().any(|f| f.is_enum) + } + + pub fn enum_fields_as_string(&self) -> Option { + if !self.has_enum() { + return None; + } + + Some( + self.fields + .iter() + .filter(|f| f.is_enum) + .map(|f| { + let pairs = f + .fields + .iter() + .map(|f| format!("{}: {}", f.name, f.type_name)) + .collect::>() + .join(","); + + format!("{}{{{}}}", f.name, pairs) + }) + .collect::>() + .join(","), + ) + } +} + +impl From<&FieldDescriptorProto> for Field { + fn from(fd: &FieldDescriptorProto) -> Self { + let options = fd.options.unknown_fields(); + + let type_name = if let Some(type_name) = fd.type_name.as_ref() { + type_name.to_owned() + } else { + if let Type::TYPE_BYTES = fd.type_() { + "Vec".to_owned() + } else { + use heck::ToUpperCamelCase; + fd.name().to_string().to_upper_camel_case() + } + }; + + Field { + name: fd.name().to_owned(), + type_name: type_name.rsplit(".").next().unwrap().to_owned(), + required: options + .iter() + //(firehose.required) = true, UnknownValueRef::Varint(0) => false, UnknownValueRef::Varint(1) => true + .find(|f| f.0 == REQUIRED_ID && UnknownValueRef::Varint(1) == f.1) + .is_some(), + is_enum: false, + is_array: Label::LABEL_REPEATED == fd.label(), + fields: vec![], + } + } +} + +impl From<&OneofDescriptorProto> for Field { + fn from(fd: &OneofDescriptorProto) -> Self { + Field { + name: fd.name().to_owned(), + type_name: "".to_owned(), + required: false, + is_enum: true, + is_array: false, + fields: vec![], + } + } +} + +impl From<&DescriptorProto> for PType { + fn from(dp: &DescriptorProto) -> Self { + let mut fields = dp + .oneof_decl + .iter() + .enumerate() + .map(|(index, fd)| { + let mut fld = Field::from(fd); + + fld.fields = dp + .field + .iter() + .filter(|fd| fd.oneof_index.is_some()) + .filter(|fd| *fd.oneof_index.as_ref().unwrap() as usize == index) + .map(|fd| Field::from(fd)) + .collect::>(); + + fld + }) + .collect::>(); + + fields.extend( + dp.field + .iter() + .filter(|fd| fd.oneof_index.is_none()) + .map(|fd| Field::from(fd)) + .collect::>(), + ); + + PType { + name: dp.name().to_owned(), + fields, + descriptor: dp.clone(), + } + } +} + +pub fn parse_proto_file<'a, P>(file_path: P) -> Result, Error> +where + P: 'a + AsRef + Debug, +{ + let dir = if let Some(p) = file_path.as_ref().parent() { + p + } else { + return Err(anyhow::anyhow!( + "Unable to derive parent path for {:?}", + file_path + )); + }; + + let fd = protobuf_parse::Parser::new() + .include(dir) + .input(&file_path) + .file_descriptor_set()?; + + assert!(fd.file.len() == 1); + assert!(fd.file[0].has_name()); + + let file_name = file_path + .as_ref() + .clone() + .file_name() + .unwrap() + .to_str() + .unwrap(); + assert!(fd.file[0].name() == file_name); + + let ret_val = fd + .file + .iter() //should be just 1 file + .flat_map(|f| f.message_type.iter()) + .map(|dp| (dp.name().to_owned(), PType::from(dp))) + .collect::>(); + + Ok(ret_val) +} diff --git a/chain/common/tests/resources/acme.proto b/chain/common/tests/resources/acme.proto new file mode 100644 index 0000000..16a0bef --- /dev/null +++ b/chain/common/tests/resources/acme.proto @@ -0,0 +1,56 @@ +syntax = "proto3"; + +package sf.acme.type.v1; + +option go_package = "github.com/streamingfast/firehose-acme/pb/sf/acme/type/v1;pbacme"; + +import "firehose/annotations.proto"; + +message Block { + uint64 height = 1; + string hash = 2; + string prevHash = 3; + uint64 timestamp = 4; + repeated Transaction transactions = 5; +} + +message Transaction { + string type = 1 [(firehose.required) = true]; + string hash = 2; + string sender = 3; + string receiver = 4; + BigInt amount = 5; + BigInt fee = 6; + bool success = 7; + repeated Event events = 8; +} + +message Event { + string type = 1; + repeated Attribute attributes = 2; +} + +message Attribute { + string key = 1; + string value = 2; +} + +message BigInt { + bytes bytes = 1; +} + +message EnumTest { + oneof sum { + Attribute attribute = 1; + BigInt big_int = 2; + } +} + +message MixedEnumTest { + string key = 1; + + oneof sum { + Attribute attribute = 2; + BigInt big_int = 3; + } +} diff --git a/chain/common/tests/resources/firehose/annotations.proto b/chain/common/tests/resources/firehose/annotations.proto new file mode 100644 index 0000000..2541e7f --- /dev/null +++ b/chain/common/tests/resources/firehose/annotations.proto @@ -0,0 +1,9 @@ +package firehose; + +import "google/protobuf/descriptor.proto"; + +// 66K range is arbitrary picked number, might be conflict + +extend google.protobuf.FieldOptions { + optional bool required = 66001; +} diff --git a/chain/common/tests/test-acme.rs b/chain/common/tests/test-acme.rs new file mode 100644 index 0000000..554e4ec --- /dev/null +++ b/chain/common/tests/test-acme.rs @@ -0,0 +1,89 @@ +const PROTO_FILE: &str = "tests/resources/acme.proto"; + +use graph_chain_common::*; + +#[test] +fn check_repeated_type_ok() { + let types = parse_proto_file(PROTO_FILE).expect("Unable to read proto file!"); + + let array_types = types + .iter() + .flat_map(|(_, t)| t.fields.iter()) + .filter(|t| t.is_array) + .map(|t| t.type_name.clone()) + .collect::>(); + + let mut array_types = array_types.into_iter().collect::>(); + array_types.sort(); + + assert_eq!(3, array_types.len()); + assert_eq!(array_types, vec!["Attribute", "Event", "Transaction"]); +} + +#[test] +fn check_type_count_ok() { + let types = parse_proto_file(PROTO_FILE).expect("Unable to read proto file!"); + assert_eq!(7, types.len()); +} + +#[test] +fn required_ok() { + let types = parse_proto_file(PROTO_FILE).expect("Unable to read proto file!"); + let msg = types.get("Transaction"); + assert!(msg.is_some(), "\"Transaction\" type is not available!"); + + let ptype = msg.unwrap(); + assert_eq!(8, ptype.fields.len()); + + ptype.fields.iter().for_each(|f| { + match f.name.as_ref() { + "type" => assert!(f.required, "Transaction.type field should be required!"), + "hash" => assert!( + !f.required, + "Transaction.hash field should NOT be required!" + ), + "sender" => assert!( + !f.required, + "Transaction.sender field should NOT be required!" + ), + "receiver" => assert!( + !f.required, + "Transaction.receiver field should NOT be required!" + ), + "amount" => assert!( + !f.required, + "Transaction.amount field should NOT be required!" + ), + "fee" => assert!(!f.required, "Transaction.fee field should NOT be required!"), + "success" => assert!( + !f.required, + "Transaction.success field should NOT be required!" + ), + "events" => assert!( + !f.required, + "Transaction.events field should NOT be required!" + ), + _ => assert!(false, "Unexpected message field [{}]!", f.name), + }; + }); +} + +#[test] +fn enum_ok() { + let types = parse_proto_file(PROTO_FILE).expect("Unable to read proto file!"); + let msg = types.get("EnumTest"); + assert!(msg.is_some(), "\"EnumTest\" type is not available!"); + + let ptype = msg.unwrap(); + assert_eq!(1, ptype.fields.len()); +} + +#[test] +fn enum_mixed_ok() { + let types = parse_proto_file(PROTO_FILE).expect("Unable to read proto file!"); + let msg = types.get("MixedEnumTest"); + assert!(msg.is_some(), "\"MixedEnumTest\" type is not available!"); + + let ptype = msg.unwrap(); + assert_eq!(2, ptype.fields.len()); +} diff --git a/chain/cosmos/Cargo.toml b/chain/cosmos/Cargo.toml new file mode 100644 index 0000000..de085f5 --- /dev/null +++ b/chain/cosmos/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "graph-chain-cosmos" +version = "0.27.0" +edition = "2018" + +[build-dependencies] +tonic-build = { version = "0.7.1", features = ["prost"] } +graph-chain-common = { path = "../common" } + +[dependencies] +graph = { path = "../../graph" } +prost = "0.10.1" +prost-types = "0.10.1" +serde = "1.0" +anyhow = "1.0" +semver = "1.0.3" + +graph-runtime-wasm = { path = "../../runtime/wasm" } +graph-runtime-derive = { path = "../../runtime/derive" } diff --git a/chain/cosmos/build.rs b/chain/cosmos/build.rs new file mode 100644 index 0000000..fc07b49 --- /dev/null +++ b/chain/cosmos/build.rs @@ -0,0 +1,54 @@ +const PROTO_FILE: &str = "proto/type.proto"; + +fn main() { + println!("cargo:rerun-if-changed=proto"); + + let types = + graph_chain_common::parse_proto_file(PROTO_FILE).expect("Unable to parse proto file!"); + + let array_types = types + .iter() + .flat_map(|(_, t)| t.fields.iter()) + .filter(|t| t.is_array) + .map(|t| t.type_name.clone()) + .collect::>(); + + let mut builder = tonic_build::configure().out_dir("src/protobuf"); + + for (name, ptype) in types { + //generate Asc + builder = builder.type_attribute( + name.clone(), + format!( + "#[graph_runtime_derive::generate_asc_type({})]", + ptype.fields().unwrap_or_default() + ), + ); + + //generate data index id + builder = builder.type_attribute( + name.clone(), + "#[graph_runtime_derive::generate_network_type_id(Cosmos)]", + ); + + //generate conversion from rust type to asc + builder = builder.type_attribute( + name.clone(), + format!( + "#[graph_runtime_derive::generate_from_rust_type({})]", + ptype.fields().unwrap_or_default() + ), + ); + + if array_types.contains(&ptype.name) { + builder = builder.type_attribute( + name.clone(), + "#[graph_runtime_derive::generate_array_type(Cosmos)]", + ); + } + } + + builder + .compile(&["proto/type.proto"], &["proto"]) + .expect("Failed to compile Firehose Cosmos proto(s)"); +} diff --git a/chain/cosmos/proto/cosmos_proto/cosmos.proto b/chain/cosmos/proto/cosmos_proto/cosmos.proto new file mode 100644 index 0000000..5c63b86 --- /dev/null +++ b/chain/cosmos/proto/cosmos_proto/cosmos.proto @@ -0,0 +1,97 @@ +syntax = "proto3"; +package cosmos_proto; + +import "google/protobuf/descriptor.proto"; + +option go_package = "github.com/cosmos/cosmos-proto;cosmos_proto"; + +extend google.protobuf.MessageOptions { + + // implements_interface is used to indicate the type name of the interface + // that a message implements so that it can be used in google.protobuf.Any + // fields that accept that interface. A message can implement multiple + // interfaces. Interfaces should be declared using a declare_interface + // file option. + repeated string implements_interface = 93001; +} + +extend google.protobuf.FieldOptions { + + // accepts_interface is used to annotate that a google.protobuf.Any + // field accepts messages that implement the specified interface. + // Interfaces should be declared using a declare_interface file option. + string accepts_interface = 93001; + + // scalar is used to indicate that this field follows the formatting defined + // by the named scalar which should be declared with declare_scalar. Code + // generators may choose to use this information to map this field to a + // language-specific type representing the scalar. + string scalar = 93002; +} + +extend google.protobuf.FileOptions { + + // declare_interface declares an interface type to be used with + // accepts_interface and implements_interface. Interface names are + // expected to follow the following convention such that their declaration + // can be discovered by tools: for a given interface type a.b.C, it is + // expected that the declaration will be found in a protobuf file named + // a/b/interfaces.proto in the file descriptor set. + repeated InterfaceDescriptor declare_interface = 793021; + + // declare_scalar declares a scalar type to be used with + // the scalar field option. Scalar names are + // expected to follow the following convention such that their declaration + // can be discovered by tools: for a given scalar type a.b.C, it is + // expected that the declaration will be found in a protobuf file named + // a/b/scalars.proto in the file descriptor set. + repeated ScalarDescriptor declare_scalar = 793022; +} + +// InterfaceDescriptor describes an interface type to be used with +// accepts_interface and implements_interface and declared by declare_interface. +message InterfaceDescriptor { + + // name is the name of the interface. It should be a short-name (without + // a period) such that the fully qualified name of the interface will be + // package.name, ex. for the package a.b and interface named C, the + // fully-qualified name will be a.b.C. + string name = 1; + + // description is a human-readable description of the interface and its + // purpose. + string description = 2; +} + +// ScalarDescriptor describes an scalar type to be used with +// the scalar field option and declared by declare_scalar. +// Scalars extend simple protobuf built-in types with additional +// syntax and semantics, for instance to represent big integers. +// Scalars should ideally define an encoding such that there is only one +// valid syntactical representation for a given semantic meaning, +// i.e. the encoding should be deterministic. +message ScalarDescriptor { + + // name is the name of the scalar. It should be a short-name (without + // a period) such that the fully qualified name of the scalar will be + // package.name, ex. for the package a.b and scalar named C, the + // fully-qualified name will be a.b.C. + string name = 1; + + // description is a human-readable description of the scalar and its + // encoding format. For instance a big integer or decimal scalar should + // specify precisely the expected encoding format. + string description = 2; + + // field_type is the type of field with which this scalar can be used. + // Scalars can be used with one and only one type of field so that + // encoding standards and simple and clear. Currently only string and + // bytes fields are supported for scalars. + repeated ScalarType field_type = 3; +} + +enum ScalarType { + SCALAR_TYPE_UNSPECIFIED = 0; + SCALAR_TYPE_STRING = 1; + SCALAR_TYPE_BYTES = 2; +} diff --git a/chain/cosmos/proto/firehose/annotations.proto b/chain/cosmos/proto/firehose/annotations.proto new file mode 100644 index 0000000..2541e7f --- /dev/null +++ b/chain/cosmos/proto/firehose/annotations.proto @@ -0,0 +1,9 @@ +package firehose; + +import "google/protobuf/descriptor.proto"; + +// 66K range is arbitrary picked number, might be conflict + +extend google.protobuf.FieldOptions { + optional bool required = 66001; +} diff --git a/chain/cosmos/proto/gogoproto/gogo.proto b/chain/cosmos/proto/gogoproto/gogo.proto new file mode 100644 index 0000000..49e78f9 --- /dev/null +++ b/chain/cosmos/proto/gogoproto/gogo.proto @@ -0,0 +1,145 @@ +// Protocol Buffers for Go with Gadgets +// +// Copyright (c) 2013, The GoGo Authors. All rights reserved. +// http://github.com/gogo/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; +package gogoproto; + +import "google/protobuf/descriptor.proto"; + +option java_package = "com.google.protobuf"; +option java_outer_classname = "GoGoProtos"; +option go_package = "github.com/gogo/protobuf/gogoproto"; + +extend google.protobuf.EnumOptions { + optional bool goproto_enum_prefix = 62001; + optional bool goproto_enum_stringer = 62021; + optional bool enum_stringer = 62022; + optional string enum_customname = 62023; + optional bool enumdecl = 62024; +} + +extend google.protobuf.EnumValueOptions { + optional string enumvalue_customname = 66001; +} + +extend google.protobuf.FileOptions { + optional bool goproto_getters_all = 63001; + optional bool goproto_enum_prefix_all = 63002; + optional bool goproto_stringer_all = 63003; + optional bool verbose_equal_all = 63004; + optional bool face_all = 63005; + optional bool gostring_all = 63006; + optional bool populate_all = 63007; + optional bool stringer_all = 63008; + optional bool onlyone_all = 63009; + + optional bool equal_all = 63013; + optional bool description_all = 63014; + optional bool testgen_all = 63015; + optional bool benchgen_all = 63016; + optional bool marshaler_all = 63017; + optional bool unmarshaler_all = 63018; + optional bool stable_marshaler_all = 63019; + + optional bool sizer_all = 63020; + + optional bool goproto_enum_stringer_all = 63021; + optional bool enum_stringer_all = 63022; + + optional bool unsafe_marshaler_all = 63023; + optional bool unsafe_unmarshaler_all = 63024; + + optional bool goproto_extensions_map_all = 63025; + optional bool goproto_unrecognized_all = 63026; + optional bool gogoproto_import = 63027; + optional bool protosizer_all = 63028; + optional bool compare_all = 63029; + optional bool typedecl_all = 63030; + optional bool enumdecl_all = 63031; + + optional bool goproto_registration = 63032; + optional bool messagename_all = 63033; + + optional bool goproto_sizecache_all = 63034; + optional bool goproto_unkeyed_all = 63035; +} + +extend google.protobuf.MessageOptions { + optional bool goproto_getters = 64001; + optional bool goproto_stringer = 64003; + optional bool verbose_equal = 64004; + optional bool face = 64005; + optional bool gostring = 64006; + optional bool populate = 64007; + optional bool stringer = 67008; + optional bool onlyone = 64009; + + optional bool equal = 64013; + optional bool description = 64014; + optional bool testgen = 64015; + optional bool benchgen = 64016; + optional bool marshaler = 64017; + optional bool unmarshaler = 64018; + optional bool stable_marshaler = 64019; + + optional bool sizer = 64020; + + optional bool unsafe_marshaler = 64023; + optional bool unsafe_unmarshaler = 64024; + + optional bool goproto_extensions_map = 64025; + optional bool goproto_unrecognized = 64026; + + optional bool protosizer = 64028; + optional bool compare = 64029; + + optional bool typedecl = 64030; + + optional bool messagename = 64033; + + optional bool goproto_sizecache = 64034; + optional bool goproto_unkeyed = 64035; +} + +extend google.protobuf.FieldOptions { + optional bool nullable = 65001; + optional bool embed = 65002; + optional string customtype = 65003; + optional string customname = 65004; + optional string jsontag = 65005; + optional string moretags = 65006; + optional string casttype = 65007; + optional string castkey = 65008; + optional string castvalue = 65009; + + optional bool stdtime = 65010; + optional bool stdduration = 65011; + optional bool wktpointer = 65012; + + optional string castrepeated = 65013; +} diff --git a/chain/cosmos/proto/type.proto b/chain/cosmos/proto/type.proto new file mode 100644 index 0000000..19db384 --- /dev/null +++ b/chain/cosmos/proto/type.proto @@ -0,0 +1,353 @@ +syntax = "proto3"; + +package sf.cosmos.type.v1; + +option go_package = "github.com/figment-networks/proto-cosmos/pb/sf/cosmos/type/v1;pbcosmos"; + +import "google/protobuf/descriptor.proto"; +import "google/protobuf/any.proto"; +import "gogoproto/gogo.proto"; +import "cosmos_proto/cosmos.proto"; +import "firehose/annotations.proto"; + +message Block { + Header header = 1 [(firehose.required) = true, (gogoproto.nullable) = false]; + EvidenceList evidence = 2 [(gogoproto.nullable) = false]; + Commit last_commit = 3; + ResponseBeginBlock result_begin_block = 4 [(firehose.required) = true]; + ResponseEndBlock result_end_block = 5 [(firehose.required) = true]; + repeated TxResult transactions = 7; + repeated Validator validator_updates = 8; +} + +// HeaderOnlyBlock is a standard [Block] structure where all other fields are +// removed so that hydrating that object from a [Block] bytes payload will +// drastically reduce allocated memory required to hold the full block. +// +// This can be used to unpack a [Block] when only the [Header] information +// is required and greatly reduce required memory. +message HeaderOnlyBlock { + Header header = 1 [(firehose.required) = true, (gogoproto.nullable) = false]; +} + +message EventData { + Event event = 1 [(firehose.required) = true]; + HeaderOnlyBlock block = 2 [(firehose.required) = true]; +} + +message TransactionData { + TxResult tx = 1 [(firehose.required) = true]; + HeaderOnlyBlock block = 2 [(firehose.required) = true]; +} + +message Header { + Consensus version = 1 [(gogoproto.nullable) = false]; + string chain_id = 2 [(gogoproto.customname) = "ChainID"]; + uint64 height = 3; + Timestamp time = 4 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; + BlockID last_block_id = 5 [(firehose.required) = true, (gogoproto.nullable) = false]; + bytes last_commit_hash = 6; + bytes data_hash = 7; + bytes validators_hash = 8; + bytes next_validators_hash = 9; + bytes consensus_hash = 10; + bytes app_hash = 11; + bytes last_results_hash = 12; + bytes evidence_hash = 13; + bytes proposer_address = 14; + bytes hash = 15; +} + +message Consensus { + option (gogoproto.equal) = true; + + uint64 block = 1; + uint64 app = 2; +} + +message Timestamp { + int64 seconds = 1; + int32 nanos = 2; +} + +message BlockID { + bytes hash = 1; + PartSetHeader part_set_header = 2 [(gogoproto.nullable) = false]; +} + +message PartSetHeader { + uint32 total = 1; + bytes hash = 2; +} + +message EvidenceList { + repeated Evidence evidence = 1 [(gogoproto.nullable) = false]; +} + +message Evidence { + oneof sum { + DuplicateVoteEvidence duplicate_vote_evidence = 1; + LightClientAttackEvidence light_client_attack_evidence = 2; + } +} + +message DuplicateVoteEvidence { + EventVote vote_a = 1; + EventVote vote_b = 2; + int64 total_voting_power = 3; + int64 validator_power = 4; + Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; +} + +message EventVote { + SignedMsgType event_vote_type = 1 [json_name = "type"]; + uint64 height = 2; + int32 round = 3; + BlockID block_id = 4 [(gogoproto.nullable) = false, (gogoproto.customname) = "BlockID"]; + Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; + bytes validator_address = 6; + int32 validator_index = 7; + bytes signature = 8; +} + +enum SignedMsgType { + option (gogoproto.goproto_enum_stringer) = true; + option (gogoproto.goproto_enum_prefix) = false; + + SIGNED_MSG_TYPE_UNKNOWN = 0 [(gogoproto.enumvalue_customname) = "UnknownType"]; + SIGNED_MSG_TYPE_PREVOTE = 1 [(gogoproto.enumvalue_customname) = "PrevoteType"]; + SIGNED_MSG_TYPE_PRECOMMIT = 2 [(gogoproto.enumvalue_customname) = "PrecommitType"]; + SIGNED_MSG_TYPE_PROPOSAL = 32 [(gogoproto.enumvalue_customname) = "ProposalType"]; +} + +message LightClientAttackEvidence { + LightBlock conflicting_block = 1; + int64 common_height = 2; + repeated Validator byzantine_validators = 3; + int64 total_voting_power = 4; + Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; +} + +message LightBlock { + SignedHeader signed_header = 1; + ValidatorSet validator_set = 2; +} + +message SignedHeader { + Header header = 1; + Commit commit = 2; +} + +message Commit { + int64 height = 1; + int32 round = 2; + BlockID block_id = 3 [(gogoproto.nullable) = false, (gogoproto.customname) = "BlockID"]; + repeated CommitSig signatures = 4 [(gogoproto.nullable) = false]; +} + +message CommitSig { + BlockIDFlag block_id_flag = 1; + bytes validator_address = 2; + Timestamp timestamp = 3 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; + bytes signature = 4; +} + +enum BlockIDFlag { + option (gogoproto.goproto_enum_stringer) = true; + option (gogoproto.goproto_enum_prefix) = false; + + BLOCK_ID_FLAG_UNKNOWN = 0 [(gogoproto.enumvalue_customname) = "BlockIDFlagUnknown"]; + BLOCK_ID_FLAG_ABSENT = 1 [(gogoproto.enumvalue_customname) = "BlockIDFlagAbsent"]; + BLOCK_ID_FLAG_COMMIT = 2 [(gogoproto.enumvalue_customname) = "BlockIDFlagCommit"]; + BLOCK_ID_FLAG_NIL = 3 [(gogoproto.enumvalue_customname) = "BlockIDFlagNil"]; +} + +message ValidatorSet { + repeated Validator validators = 1; + Validator proposer = 2; + int64 total_voting_power = 3; +} + +message Validator { + bytes address = 1; + PublicKey pub_key = 2 [(gogoproto.nullable) = false]; + int64 voting_power = 3; + int64 proposer_priority = 4; +} + +message PublicKey { + option (gogoproto.compare) = true; + option (gogoproto.equal) = true; + + oneof sum { + bytes ed25519 = 1; + bytes secp256k1 = 2; + } +} + +message ResponseBeginBlock { + repeated Event events = 1 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "events,omitempty"]; +} + +message Event { + string event_type = 1 [json_name = "type"]; + repeated EventAttribute attributes = 2 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "attributes,omitempty"]; +} + +message EventAttribute { + string key = 1; + string value = 2; + bool index = 3; +} + +message ResponseEndBlock { + repeated ValidatorUpdate validator_updates = 1; + ConsensusParams consensus_param_updates = 2; + repeated Event events = 3; +} + +message ValidatorUpdate { + bytes address = 1; + PublicKey pub_key = 2 [(gogoproto.nullable) = false]; + int64 power = 3; +} + +message ConsensusParams { + BlockParams block = 1 [(gogoproto.nullable) = false]; + EvidenceParams evidence = 2 [(gogoproto.nullable) = false]; + ValidatorParams validator = 3 [(gogoproto.nullable) = false]; + VersionParams version = 4 [(gogoproto.nullable) = false]; +} + +message BlockParams { + int64 max_bytes = 1; + int64 max_gas = 2; +} + +message EvidenceParams { + int64 max_age_num_blocks = 1; + Duration max_age_duration = 2 [(gogoproto.nullable) = false, (gogoproto.stdduration) = true]; + int64 max_bytes = 3; +} + +message Duration { + int64 seconds = 1; + int32 nanos = 2; +} + +message ValidatorParams { + option (gogoproto.populate) = true; + option (gogoproto.equal) = true; + + repeated string pub_key_types = 1; +} + +message VersionParams { + option (gogoproto.populate) = true; + option (gogoproto.equal) = true; + + uint64 app_version = 1; +} + +message TxResult { + uint64 height = 1; + uint32 index = 2; + Tx tx = 3 [(firehose.required) = true]; + ResponseDeliverTx result = 4 [(firehose.required) = true]; + bytes hash = 5; +} + +message Tx { + TxBody body = 1 [(firehose.required) = true]; + AuthInfo auth_info = 2; + repeated bytes signatures = 3; +} + +message TxBody { + repeated google.protobuf.Any messages = 1; + string memo = 2; + uint64 timeout_height = 3; + repeated google.protobuf.Any extension_options = 1023; + repeated google.protobuf.Any non_critical_extension_options = 2047; +} + +message Any { + string type_url = 1; + bytes value = 2; +} + +message AuthInfo { + repeated SignerInfo signer_infos = 1; + Fee fee = 2; + Tip tip = 3; +} + +message SignerInfo { + google.protobuf.Any public_key = 1; + ModeInfo mode_info = 2; + uint64 sequence = 3; +} + +message ModeInfo { + oneof sum { + ModeInfoSingle single = 1; + ModeInfoMulti multi = 2; + } +} + +message ModeInfoSingle { + SignMode mode = 1; +} + +enum SignMode { + SIGN_MODE_UNSPECIFIED = 0; + SIGN_MODE_DIRECT = 1; + SIGN_MODE_TEXTUAL = 2; + SIGN_MODE_LEGACY_AMINO_JSON = 127; +} + +message ModeInfoMulti { + CompactBitArray bitarray = 1; + repeated ModeInfo mode_infos = 2; +} + +message CompactBitArray { + option (gogoproto.goproto_stringer) = false; + + uint32 extra_bits_stored = 1; + bytes elems = 2; +} + +message Fee { + repeated Coin amount = 1 [(gogoproto.nullable) = false, (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins"]; + uint64 gas_limit = 2; + string payer = 3 [(cosmos_proto.scalar) = "cosmos.AddressString"]; + string granter = 4 [(cosmos_proto.scalar) = "cosmos.AddressString"]; +} + +message Coin { + option (gogoproto.equal) = true; + + string denom = 1; + string amount = 2 [(gogoproto.customtype) = "Int", (gogoproto.nullable) = false]; +} + +message Tip { + repeated Coin amount = 1 [(gogoproto.nullable) = false, (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins"]; + string tipper = 2 [(cosmos_proto.scalar) = "cosmos.AddressString"]; +} + +message ResponseDeliverTx { + uint32 code = 1; + bytes data = 2; + string log = 3; + string info = 4; + int64 gas_wanted = 5; + int64 gas_used = 6; + repeated Event events = 7 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "events,omitempty"]; + string codespace = 8; +} + +message ValidatorSetUpdates { + repeated Validator validator_updates = 1; +} diff --git a/chain/cosmos/src/adapter.rs b/chain/cosmos/src/adapter.rs new file mode 100644 index 0000000..d73b8b0 --- /dev/null +++ b/chain/cosmos/src/adapter.rs @@ -0,0 +1,160 @@ +use std::collections::HashSet; + +use prost::Message; +use prost_types::Any; + +use crate::capabilities::NodeCapabilities; +use crate::{data_source::DataSource, Chain}; +use graph::blockchain as bc; +use graph::firehose::EventTypeFilter; +use graph::prelude::*; + +const EVENT_TYPE_FILTER_TYPE_URL: &str = + "type.googleapis.com/sf.cosmos.transform.v1.EventTypeFilter"; + +#[derive(Clone, Debug, Default)] +pub struct TriggerFilter { + pub(crate) event_type_filter: CosmosEventTypeFilter, + pub(crate) block_filter: CosmosBlockFilter, +} + +impl bc::TriggerFilter for TriggerFilter { + fn extend<'a>(&mut self, data_sources: impl Iterator + Clone) { + self.event_type_filter + .extend_from_data_sources(data_sources.clone()); + self.block_filter.extend_from_data_sources(data_sources); + } + + fn node_capabilities(&self) -> NodeCapabilities { + NodeCapabilities {} + } + + fn extend_with_template( + &mut self, + _data_source: impl Iterator::DataSourceTemplate>, + ) { + } + + fn to_firehose_filter(self) -> Vec { + if self.block_filter.trigger_every_block { + return vec![]; + } + + if self.event_type_filter.event_types.is_empty() { + return vec![]; + } + + let filter = EventTypeFilter { + event_types: Vec::from_iter(self.event_type_filter.event_types), + }; + + vec![Any { + type_url: EVENT_TYPE_FILTER_TYPE_URL.to_string(), + value: filter.encode_to_vec(), + }] + } +} + +pub type EventType = String; + +#[derive(Clone, Debug, Default)] +pub(crate) struct CosmosEventTypeFilter { + pub event_types: HashSet, +} + +impl CosmosEventTypeFilter { + pub(crate) fn matches(&self, event_type: &EventType) -> bool { + self.event_types.contains(event_type) + } + + fn extend_from_data_sources<'a>(&mut self, data_sources: impl Iterator) { + self.event_types.extend( + data_sources.flat_map(|data_source| data_source.events().map(ToString::to_string)), + ); + } +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct CosmosBlockFilter { + pub trigger_every_block: bool, +} + +impl CosmosBlockFilter { + fn extend_from_data_sources<'a>( + &mut self, + mut data_sources: impl Iterator, + ) { + if !self.trigger_every_block { + self.trigger_every_block = data_sources.any(DataSource::has_block_handler); + } + } +} + +#[cfg(test)] +mod test { + use graph::blockchain::TriggerFilter as _; + + use super::*; + + #[test] + fn test_trigger_filters() { + let cases = [ + (TriggerFilter::test_new(false, &[]), None), + (TriggerFilter::test_new(true, &[]), None), + (TriggerFilter::test_new(true, &["event_1", "event_2"]), None), + ( + TriggerFilter::test_new(false, &["event_1", "event_2", "event_3"]), + Some(event_type_filter_with(&["event_1", "event_2", "event_3"])), + ), + ]; + + for (trigger_filter, expected_filter) in cases { + let firehose_filter = trigger_filter.to_firehose_filter(); + let decoded_filter = decode_filter(firehose_filter); + + assert_eq!(decoded_filter.is_some(), expected_filter.is_some()); + + if let (Some(mut expected_filter), Some(mut decoded_filter)) = + (expected_filter, decoded_filter) + { + // event types may be in different order + expected_filter.event_types.sort(); + decoded_filter.event_types.sort(); + + assert_eq!(decoded_filter, expected_filter); + } + } + } + + impl TriggerFilter { + pub(crate) fn test_new(trigger_every_block: bool, event_types: &[&str]) -> TriggerFilter { + TriggerFilter { + event_type_filter: CosmosEventTypeFilter { + event_types: event_types.iter().map(ToString::to_string).collect(), + }, + block_filter: CosmosBlockFilter { + trigger_every_block, + }, + } + } + } + + fn event_type_filter_with(event_types: &[&str]) -> EventTypeFilter { + EventTypeFilter { + event_types: event_types.iter().map(ToString::to_string).collect(), + } + } + + fn decode_filter(proto_filters: Vec) -> Option { + assert!(proto_filters.len() <= 1); + + let proto_filter = proto_filters.get(0)?; + + assert_eq!(proto_filter.type_url, EVENT_TYPE_FILTER_TYPE_URL); + + let firehose_filter = EventTypeFilter::decode(&*proto_filter.value) + .expect("Could not decode EventTypeFilter from protobuf Any"); + + Some(firehose_filter) + } +} diff --git a/chain/cosmos/src/capabilities.rs b/chain/cosmos/src/capabilities.rs new file mode 100644 index 0000000..8905812 --- /dev/null +++ b/chain/cosmos/src/capabilities.rs @@ -0,0 +1,33 @@ +use std::cmp::PartialOrd; +use std::fmt; +use std::str::FromStr; + +use anyhow::Error; +use graph::impl_slog_value; + +use crate::DataSource; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd)] +pub struct NodeCapabilities {} + +impl FromStr for NodeCapabilities { + type Err = Error; + + fn from_str(_s: &str) -> Result { + Ok(NodeCapabilities {}) + } +} + +impl fmt::Display for NodeCapabilities { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("cosmos") + } +} + +impl_slog_value!(NodeCapabilities, "{}"); + +impl graph::blockchain::NodeCapabilities for NodeCapabilities { + fn from_data_sources(_data_sources: &[DataSource]) -> Self { + NodeCapabilities {} + } +} diff --git a/chain/cosmos/src/chain.rs b/chain/cosmos/src/chain.rs new file mode 100644 index 0000000..9e62546 --- /dev/null +++ b/chain/cosmos/src/chain.rs @@ -0,0 +1,581 @@ +use std::sync::Arc; + +use graph::blockchain::block_stream::FirehoseCursor; +use graph::cheap_clone::CheapClone; +use graph::data::subgraph::UnifiedMappingApiVersion; +use graph::prelude::MetricsRegistry; +use graph::{ + anyhow::anyhow, + blockchain::{ + block_stream::{ + BlockStream, BlockStreamEvent, BlockWithTriggers, FirehoseError, + FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, + }, + firehose_block_stream::FirehoseBlockStream, + Block as _, BlockHash, BlockPtr, Blockchain, BlockchainKind, IngestorError, + RuntimeAdapter as RuntimeAdapterTrait, + }, + components::store::DeploymentLocator, + firehose::{self, FirehoseEndpoint, FirehoseEndpoints, ForkStep}, + prelude::{async_trait, o, BlockNumber, ChainStore, Error, Logger, LoggerFactory}, +}; +use prost::Message; + +use crate::capabilities::NodeCapabilities; +use crate::data_source::{ + DataSource, DataSourceTemplate, EventOrigin, UnresolvedDataSource, UnresolvedDataSourceTemplate, +}; +use crate::trigger::CosmosTrigger; +use crate::RuntimeAdapter; +use crate::{codec, TriggerFilter}; + +pub struct Chain { + logger_factory: LoggerFactory, + name: String, + firehose_endpoints: Arc, + chain_store: Arc, + metrics_registry: Arc, +} + +impl std::fmt::Debug for Chain { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "chain: cosmos") + } +} + +impl Chain { + pub fn new( + logger_factory: LoggerFactory, + name: String, + chain_store: Arc, + firehose_endpoints: FirehoseEndpoints, + metrics_registry: Arc, + ) -> Self { + Chain { + logger_factory, + name, + firehose_endpoints: Arc::new(firehose_endpoints), + chain_store, + metrics_registry, + } + } +} + +#[async_trait] +impl Blockchain for Chain { + const KIND: BlockchainKind = BlockchainKind::Cosmos; + + type Block = codec::Block; + + type DataSource = DataSource; + + type UnresolvedDataSource = UnresolvedDataSource; + + type DataSourceTemplate = DataSourceTemplate; + + type UnresolvedDataSourceTemplate = UnresolvedDataSourceTemplate; + + type TriggerData = CosmosTrigger; + + type MappingTrigger = CosmosTrigger; + + type TriggerFilter = TriggerFilter; + + type NodeCapabilities = NodeCapabilities; + + fn triggers_adapter( + &self, + _loc: &DeploymentLocator, + _capabilities: &Self::NodeCapabilities, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + let adapter = TriggersAdapter {}; + Ok(Arc::new(adapter)) + } + + async fn new_firehose_block_stream( + &self, + deployment: DeploymentLocator, + block_cursor: FirehoseCursor, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + let adapter = self + .triggers_adapter(&deployment, &NodeCapabilities {}, unified_api_version) + .unwrap_or_else(|_| panic!("no adapter for network {}", self.name)); + + let firehose_endpoint = match self.firehose_endpoints.random() { + Some(e) => e.clone(), + None => return Err(anyhow!("no firehose endpoint available",)), + }; + + let logger = self + .logger_factory + .subgraph_logger(&deployment) + .new(o!("component" => "FirehoseBlockStream")); + + let firehose_mapper = Arc::new(FirehoseMapper { + endpoint: firehose_endpoint.cheap_clone(), + }); + + Ok(Box::new(FirehoseBlockStream::new( + deployment.hash, + firehose_endpoint, + subgraph_current_block, + block_cursor, + firehose_mapper, + adapter, + filter, + start_blocks, + logger, + self.metrics_registry.clone(), + ))) + } + + async fn new_polling_block_stream( + &self, + _deployment: DeploymentLocator, + _start_blocks: Vec, + _subgraph_start_block: Option, + _filter: Arc, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + panic!("Cosmos does not support polling block stream") + } + + fn chain_store(&self) -> Arc { + self.chain_store.cheap_clone() + } + + async fn block_pointer_from_number( + &self, + logger: &Logger, + number: BlockNumber, + ) -> Result { + let firehose_endpoint = match self.firehose_endpoints.random() { + Some(e) => e.clone(), + None => return Err(anyhow!("no firehose endpoint available").into()), + }; + + firehose_endpoint + .block_ptr_for_number::(logger, number) + .await + .map_err(Into::into) + } + + fn runtime_adapter(&self) -> Arc> { + Arc::new(RuntimeAdapter {}) + } + + fn is_firehose_supported(&self) -> bool { + true + } +} + +pub struct TriggersAdapter {} + +#[async_trait] +impl TriggersAdapterTrait for TriggersAdapter { + async fn scan_triggers( + &self, + _from: BlockNumber, + _to: BlockNumber, + _filter: &TriggerFilter, + ) -> Result>, Error> { + panic!("Should never be called since not used by FirehoseBlockStream") + } + + async fn triggers_in_block( + &self, + _logger: &Logger, + block: codec::Block, + filter: &TriggerFilter, + ) -> Result, Error> { + let shared_block = Arc::new(block.clone()); + + let header_only_block = codec::HeaderOnlyBlock::from(&block); + + let mut triggers: Vec<_> = shared_block + .begin_block_events()? + .cloned() + // FIXME (Cosmos): Optimize. Should use an Arc instead of cloning the + // block. This is not currently possible because EventData is automatically + // generated. + .filter_map(|event| { + filter_event_trigger(filter, event, &header_only_block, EventOrigin::BeginBlock) + }) + .chain(shared_block.tx_events()?.cloned().filter_map(|event| { + filter_event_trigger(filter, event, &header_only_block, EventOrigin::DeliverTx) + })) + .chain( + shared_block + .end_block_events()? + .cloned() + .filter_map(|event| { + filter_event_trigger( + filter, + event, + &header_only_block, + EventOrigin::EndBlock, + ) + }), + ) + .collect(); + + triggers.extend( + shared_block + .transactions() + .cloned() + .map(|tx| CosmosTrigger::with_transaction(tx, header_only_block.clone())), + ); + + if filter.block_filter.trigger_every_block { + triggers.push(CosmosTrigger::Block(shared_block.cheap_clone())); + } + + Ok(BlockWithTriggers::new(block, triggers)) + } + + async fn is_on_main_chain(&self, _ptr: BlockPtr) -> Result { + panic!("Should never be called since not used by FirehoseBlockStream") + } + + async fn ancestor_block( + &self, + _ptr: BlockPtr, + _offset: BlockNumber, + ) -> Result, Error> { + panic!("Should never be called since not used by FirehoseBlockStream") + } + + /// Panics if `block` is genesis. + /// But that's ok since this is only called when reverting `block`. + async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { + Ok(Some(BlockPtr { + hash: BlockHash::from(vec![0xff; 32]), + number: block.number.saturating_sub(1), + })) + } +} + +/// Returns a new event trigger only if the given event matches the event filter. +fn filter_event_trigger( + filter: &TriggerFilter, + event: codec::Event, + block: &codec::HeaderOnlyBlock, + origin: EventOrigin, +) -> Option { + if filter.event_type_filter.matches(&event.event_type) { + Some(CosmosTrigger::with_event(event, block.clone(), origin)) + } else { + None + } +} + +pub struct FirehoseMapper { + endpoint: Arc, +} + +#[async_trait] +impl FirehoseMapperTrait for FirehoseMapper { + async fn to_block_stream_event( + &self, + logger: &Logger, + response: &firehose::Response, + adapter: &Arc>, + filter: &TriggerFilter, + ) -> Result, FirehoseError> { + let step = ForkStep::from_i32(response.step).unwrap_or_else(|| { + panic!( + "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", + response.step + ) + }); + + let any_block = response + .block + .as_ref() + .expect("block payload information should always be present"); + + // Right now, this is done in all cases but in reality, with how the BlockStreamEvent::Revert + // is defined right now, only block hash and block number is necessary. However, this information + // is not part of the actual bstream::BlockResponseV2 payload. As such, we need to decode the full + // block which is useless. + // + // Check about adding basic information about the block in the bstream::BlockResponseV2 or maybe + // define a slimmed down struct that would decode only a few fields and ignore all the rest. + let sp = codec::Block::decode(any_block.value.as_ref())?; + + match step { + ForkStep::StepNew => Ok(BlockStreamEvent::ProcessBlock( + adapter.triggers_in_block(logger, sp, filter).await?, + FirehoseCursor::from(response.cursor.clone()), + )), + + ForkStep::StepUndo => { + let parent_ptr = sp + .parent_ptr() + .map_err(FirehoseError::from)? + .expect("Genesis block should never be reverted"); + + Ok(BlockStreamEvent::Revert( + parent_ptr, + FirehoseCursor::from(response.cursor.clone()), + )) + } + + ForkStep::StepIrreversible => { + panic!("irreversible step is not handled and should not be requested in the Firehose request") + } + + ForkStep::StepUnknown => { + panic!("unknown step should not happen in the Firehose response") + } + } + } + + async fn block_ptr_for_number( + &self, + logger: &Logger, + number: BlockNumber, + ) -> Result { + self.endpoint + .block_ptr_for_number::(logger, number) + .await + } + + async fn final_block_ptr_for( + &self, + logger: &Logger, + block: &codec::Block, + ) -> Result { + // Cosmos provides instant block finality. + self.endpoint + .block_ptr_for_number::(logger, block.number()) + .await + } +} + +#[cfg(test)] +mod test { + use graph::prelude::{ + slog::{o, Discard, Logger}, + tokio, + }; + + use super::*; + + use codec::{ + Block, Event, Header, HeaderOnlyBlock, ResponseBeginBlock, ResponseDeliverTx, + ResponseEndBlock, TxResult, + }; + + #[tokio::test] + async fn test_trigger_filters() { + let adapter = TriggersAdapter {}; + let logger = Logger::root(Discard, o!()); + + let block_with_events = Block::test_with_event_types( + vec!["begin_event_1", "begin_event_2", "begin_event_3"], + vec!["tx_event_1", "tx_event_2", "tx_event_3"], + vec!["end_event_1", "end_event_2", "end_event_3"], + ); + + let header_only_block = HeaderOnlyBlock::from(&block_with_events); + + let cases = [ + ( + Block::test_new(), + TriggerFilter::test_new(false, &[]), + vec![], + ), + ( + Block::test_new(), + TriggerFilter::test_new(true, &[]), + vec![CosmosTrigger::Block(Arc::new(Block::test_new()))], + ), + ( + Block::test_new(), + TriggerFilter::test_new(false, &["event_1", "event_2", "event_3"]), + vec![], + ), + ( + block_with_events.clone(), + TriggerFilter::test_new(false, &["begin_event_3", "tx_event_3", "end_event_3"]), + vec![ + CosmosTrigger::with_event( + Event::test_with_type("begin_event_3"), + header_only_block.clone(), + EventOrigin::BeginBlock, + ), + CosmosTrigger::with_event( + Event::test_with_type("tx_event_3"), + header_only_block.clone(), + EventOrigin::DeliverTx, + ), + CosmosTrigger::with_event( + Event::test_with_type("end_event_3"), + header_only_block.clone(), + EventOrigin::EndBlock, + ), + CosmosTrigger::with_transaction( + TxResult::test_with_event_type("tx_event_1"), + header_only_block.clone(), + ), + CosmosTrigger::with_transaction( + TxResult::test_with_event_type("tx_event_2"), + header_only_block.clone(), + ), + CosmosTrigger::with_transaction( + TxResult::test_with_event_type("tx_event_3"), + header_only_block.clone(), + ), + ], + ), + ( + block_with_events.clone(), + TriggerFilter::test_new(true, &["begin_event_3", "tx_event_2", "end_event_1"]), + vec![ + CosmosTrigger::Block(Arc::new(block_with_events.clone())), + CosmosTrigger::with_event( + Event::test_with_type("begin_event_3"), + header_only_block.clone(), + EventOrigin::BeginBlock, + ), + CosmosTrigger::with_event( + Event::test_with_type("tx_event_2"), + header_only_block.clone(), + EventOrigin::DeliverTx, + ), + CosmosTrigger::with_event( + Event::test_with_type("end_event_1"), + header_only_block.clone(), + EventOrigin::EndBlock, + ), + CosmosTrigger::with_transaction( + TxResult::test_with_event_type("tx_event_1"), + header_only_block.clone(), + ), + CosmosTrigger::with_transaction( + TxResult::test_with_event_type("tx_event_2"), + header_only_block.clone(), + ), + CosmosTrigger::with_transaction( + TxResult::test_with_event_type("tx_event_3"), + header_only_block.clone(), + ), + ], + ), + ]; + + for (block, trigger_filter, expected_triggers) in cases { + let triggers = adapter + .triggers_in_block(&logger, block, &trigger_filter) + .await + .expect("failed to get triggers in block"); + + assert_eq!( + triggers.trigger_data.len(), + expected_triggers.len(), + "Expected trigger list to contain exactly {:?}, but it didn't: {:?}", + expected_triggers, + triggers.trigger_data + ); + + // they may not be in the same order + for trigger in expected_triggers { + assert!( + triggers.trigger_data.contains(&trigger), + "Expected trigger list to contain {:?}, but it only contains: {:?}", + trigger, + triggers.trigger_data + ); + } + } + } + + impl Block { + fn test_new() -> Block { + Block::test_with_event_types(vec![], vec![], vec![]) + } + + fn test_with_event_types( + begin_event_types: Vec<&str>, + tx_event_types: Vec<&str>, + end_event_types: Vec<&str>, + ) -> Block { + Block { + header: Some(Header { + version: None, + chain_id: "test".to_string(), + height: 1, + time: None, + last_block_id: None, + last_commit_hash: vec![], + data_hash: vec![], + validators_hash: vec![], + next_validators_hash: vec![], + consensus_hash: vec![], + app_hash: vec![], + last_results_hash: vec![], + evidence_hash: vec![], + proposer_address: vec![], + hash: vec![], + }), + evidence: None, + last_commit: None, + result_begin_block: Some(ResponseBeginBlock { + events: begin_event_types + .into_iter() + .map(Event::test_with_type) + .collect(), + }), + result_end_block: Some(ResponseEndBlock { + validator_updates: vec![], + consensus_param_updates: None, + events: end_event_types + .into_iter() + .map(Event::test_with_type) + .collect(), + }), + transactions: tx_event_types + .into_iter() + .map(TxResult::test_with_event_type) + .collect(), + validator_updates: vec![], + } + } + } + + impl Event { + fn test_with_type(event_type: &str) -> Event { + Event { + event_type: event_type.to_string(), + attributes: vec![], + } + } + } + + impl TxResult { + fn test_with_event_type(event_type: &str) -> TxResult { + TxResult { + height: 1, + index: 1, + tx: None, + result: Some(ResponseDeliverTx { + code: 1, + data: vec![], + log: "".to_string(), + info: "".to_string(), + gas_wanted: 1, + gas_used: 1, + codespace: "".to_string(), + events: vec![Event::test_with_type(event_type)], + }), + hash: vec![], + } + } + } +} diff --git a/chain/cosmos/src/codec.rs b/chain/cosmos/src/codec.rs new file mode 100644 index 0000000..7b2a299 --- /dev/null +++ b/chain/cosmos/src/codec.rs @@ -0,0 +1,199 @@ +pub(crate) use crate::protobuf::pbcodec::*; + +use graph::blockchain::Block as BlockchainBlock; +use graph::{ + blockchain::BlockPtr, + prelude::{anyhow::anyhow, BlockNumber, Error}, +}; + +use std::convert::TryFrom; + +impl Block { + pub fn header(&self) -> Result<&Header, Error> { + self.header + .as_ref() + .ok_or_else(|| anyhow!("block data missing header field")) + } + + pub fn events(&self) -> Result, Error> { + let events = self + .begin_block_events()? + .chain(self.tx_events()?) + .chain(self.end_block_events()?); + + Ok(events) + } + + pub fn begin_block_events(&self) -> Result, Error> { + let events = self + .result_begin_block + .as_ref() + .ok_or_else(|| anyhow!("block data missing result_begin_block field"))? + .events + .iter(); + + Ok(events) + } + + pub fn tx_events(&self) -> Result, Error> { + if self.transactions.iter().any(|tx| tx.result.is_none()) { + return Err(anyhow!("block data transaction missing result field")); + } + + let events = self.transactions.iter().flat_map(|tx| { + tx.result + .as_ref() + .map(|b| b.events.iter()) + .into_iter() + .flatten() + }); + + Ok(events) + } + + pub fn end_block_events(&self) -> Result, Error> { + let events = self + .result_end_block + .as_ref() + .ok_or_else(|| anyhow!("block data missing result_end_block field"))? + .events + .iter(); + + Ok(events) + } + + pub fn transactions(&self) -> impl Iterator { + self.transactions.iter() + } + + pub fn parent_ptr(&self) -> Result, Error> { + let header = self.header()?; + + Ok(header + .last_block_id + .as_ref() + .map(|last_block_id| BlockPtr::from((last_block_id.hash.clone(), header.height - 1)))) + } +} + +impl TryFrom for BlockPtr { + type Error = Error; + + fn try_from(b: Block) -> Result { + BlockPtr::try_from(&b) + } +} + +impl<'a> TryFrom<&'a Block> for BlockPtr { + type Error = Error; + + fn try_from(b: &'a Block) -> Result { + let header = b.header()?; + Ok(BlockPtr::from((header.hash.clone(), header.height))) + } +} + +impl BlockchainBlock for Block { + fn number(&self) -> i32 { + BlockNumber::try_from(self.header().unwrap().height).unwrap() + } + + fn ptr(&self) -> BlockPtr { + BlockPtr::try_from(self).unwrap() + } + + fn parent_ptr(&self) -> Option { + self.parent_ptr().unwrap() + } +} + +impl HeaderOnlyBlock { + pub fn header(&self) -> Result<&Header, Error> { + self.header + .as_ref() + .ok_or_else(|| anyhow!("block data missing header field")) + } + + pub fn parent_ptr(&self) -> Result, Error> { + let header = self.header()?; + + Ok(header + .last_block_id + .as_ref() + .map(|last_block_id| BlockPtr::from((last_block_id.hash.clone(), header.height - 1)))) + } +} + +impl From<&Block> for HeaderOnlyBlock { + fn from(b: &Block) -> HeaderOnlyBlock { + HeaderOnlyBlock { + header: b.header.clone(), + } + } +} + +impl TryFrom for BlockPtr { + type Error = Error; + + fn try_from(b: HeaderOnlyBlock) -> Result { + BlockPtr::try_from(&b) + } +} + +impl<'a> TryFrom<&'a HeaderOnlyBlock> for BlockPtr { + type Error = Error; + + fn try_from(b: &'a HeaderOnlyBlock) -> Result { + let header = b.header()?; + + Ok(BlockPtr::from((header.hash.clone(), header.height))) + } +} + +impl BlockchainBlock for HeaderOnlyBlock { + fn number(&self) -> i32 { + BlockNumber::try_from(self.header().unwrap().height).unwrap() + } + + fn ptr(&self) -> BlockPtr { + BlockPtr::try_from(self).unwrap() + } + + fn parent_ptr(&self) -> Option { + self.parent_ptr().unwrap() + } +} + +impl EventData { + pub fn event(&self) -> Result<&Event, Error> { + self.event + .as_ref() + .ok_or_else(|| anyhow!("event data missing event field")) + } + pub fn block(&self) -> Result<&HeaderOnlyBlock, Error> { + self.block + .as_ref() + .ok_or_else(|| anyhow!("event data missing block field")) + } +} + +impl TransactionData { + pub fn tx_result(&self) -> Result<&TxResult, Error> { + self.tx + .as_ref() + .ok_or_else(|| anyhow!("transaction data missing tx field")) + } + + pub fn response_deliver_tx(&self) -> Result<&ResponseDeliverTx, Error> { + self.tx_result()? + .result + .as_ref() + .ok_or_else(|| anyhow!("transaction data missing result field")) + } + + pub fn block(&self) -> Result<&HeaderOnlyBlock, Error> { + self.block + .as_ref() + .ok_or_else(|| anyhow!("transaction data missing block field")) + } +} diff --git a/chain/cosmos/src/data_source.rs b/chain/cosmos/src/data_source.rs new file mode 100644 index 0000000..7535a50 --- /dev/null +++ b/chain/cosmos/src/data_source.rs @@ -0,0 +1,559 @@ +use std::collections::{HashMap, HashSet}; +use std::{convert::TryFrom, sync::Arc}; + +use anyhow::{Error, Result}; + +use graph::{ + blockchain::{self, Block, Blockchain, TriggerWithHandler}, + components::store::StoredDynamicDataSource, + data::subgraph::DataSourceContext, + prelude::{ + anyhow, async_trait, info, BlockNumber, CheapClone, DataSourceTemplateInfo, Deserialize, + Link, LinkResolver, Logger, + }, +}; + +use crate::chain::Chain; +use crate::codec; +use crate::trigger::CosmosTrigger; + +pub const COSMOS_KIND: &str = "cosmos"; + +const DYNAMIC_DATA_SOURCE_ERROR: &str = "Cosmos subgraphs do not support dynamic data sources"; +const TEMPLATE_ERROR: &str = "Cosmos subgraphs do not support templates"; + +/// Runtime representation of a data source. +// Note: Not great for memory usage that this needs to be `Clone`, considering how there may be tens +// of thousands of data sources in memory at once. +#[derive(Clone, Debug)] +pub struct DataSource { + pub kind: String, + pub network: Option, + pub name: String, + pub source: Source, + pub mapping: Mapping, + pub context: Arc>, + pub creation_block: Option, +} + +impl blockchain::DataSource for DataSource { + fn address(&self) -> Option<&[u8]> { + None + } + + fn start_block(&self) -> BlockNumber { + self.source.start_block + } + + fn match_and_decode( + &self, + trigger: &::TriggerData, + block: &Arc<::Block>, + _logger: &Logger, + ) -> Result>> { + if self.source.start_block > block.number() { + return Ok(None); + } + + let handler = match trigger { + CosmosTrigger::Block(_) => match self.handler_for_block() { + Some(handler) => handler.handler, + None => return Ok(None), + }, + + CosmosTrigger::Event { event_data, origin } => { + match self.handler_for_event(event_data.event()?, *origin) { + Some(handler) => handler.handler, + None => return Ok(None), + } + } + + CosmosTrigger::Transaction(_) => match self.handler_for_transaction() { + Some(handler) => handler.handler, + None => return Ok(None), + }, + }; + + Ok(Some(TriggerWithHandler::::new( + trigger.cheap_clone(), + handler, + block.ptr(), + ))) + } + + fn name(&self) -> &str { + &self.name + } + + fn kind(&self) -> &str { + &self.kind + } + + fn network(&self) -> Option<&str> { + self.network.as_deref() + } + + fn context(&self) -> Arc> { + self.context.cheap_clone() + } + + fn creation_block(&self) -> Option { + self.creation_block + } + + fn is_duplicate_of(&self, other: &Self) -> bool { + let DataSource { + kind, + network, + name, + source, + mapping, + context, + + // The creation block is ignored for detection duplicate data sources. + // Contract ABI equality is implicit in `source` and `mapping.abis` equality. + creation_block: _, + } = self; + + // mapping_request_sender, host_metrics, and (most of) host_exports are operational structs + // used at runtime but not needed to define uniqueness; each runtime host should be for a + // unique data source. + kind == &other.kind + && network == &other.network + && name == &other.name + && source == &other.source + && mapping.block_handlers == other.mapping.block_handlers + && mapping.event_handlers == other.mapping.event_handlers + && mapping.transaction_handlers == other.mapping.transaction_handlers + && context == &other.context + } + + fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { + unimplemented!("{}", DYNAMIC_DATA_SOURCE_ERROR); + } + + fn from_stored_dynamic_data_source( + _template: &DataSourceTemplate, + _stored: StoredDynamicDataSource, + ) -> Result { + Err(anyhow!(DYNAMIC_DATA_SOURCE_ERROR)) + } + + fn validate(&self) -> Vec { + let mut errors = Vec::new(); + + if self.kind != COSMOS_KIND { + errors.push(anyhow!( + "data source has invalid `kind`, expected {} but found {}", + COSMOS_KIND, + self.kind + )) + } + + // Ensure there is only one block handler + if self.mapping.block_handlers.len() > 1 { + errors.push(anyhow!("data source has duplicated block handlers")); + } + + // Ensure there is only one transaction handler + if self.mapping.transaction_handlers.len() > 1 { + errors.push(anyhow!("data source has duplicated transaction handlers")); + } + + // Ensure that each event type + origin filter combination has only one handler + + // group handler origin filters by event type + let mut event_types = HashMap::with_capacity(self.mapping.event_handlers.len()); + for event_handler in self.mapping.event_handlers.iter() { + let origins = event_types + .entry(&event_handler.event) + // 3 is the maximum number of valid handlers for an event type (1 for each origin) + .or_insert(HashSet::with_capacity(3)); + + // insert returns false if value was already in the set + if !origins.insert(event_handler.origin) { + errors.push(multiple_origin_err( + &event_handler.event, + event_handler.origin, + )) + } + } + + // Ensure each event type either has: + // 1 handler with no origin filter + // OR + // 1 or more handlers with origin filter + for (event_type, origins) in event_types.iter() { + if origins.len() > 1 { + if !origins.iter().all(Option::is_some) { + errors.push(combined_origins_err(event_type)) + } + } + } + + errors + } + + fn api_version(&self) -> semver::Version { + self.mapping.api_version.clone() + } + + fn runtime(&self) -> Option>> { + Some(self.mapping.runtime.cheap_clone()) + } +} + +impl DataSource { + fn from_manifest( + kind: String, + network: Option, + name: String, + source: Source, + mapping: Mapping, + context: Option, + ) -> Result { + // Data sources in the manifest are created "before genesis" so they have no creation block. + let creation_block = None; + + Ok(DataSource { + kind, + network, + name, + source, + mapping, + context: Arc::new(context), + creation_block, + }) + } + + fn handler_for_block(&self) -> Option { + self.mapping.block_handlers.first().cloned() + } + + fn handler_for_transaction(&self) -> Option { + self.mapping.transaction_handlers.first().cloned() + } + + fn handler_for_event( + &self, + event: &codec::Event, + event_origin: EventOrigin, + ) -> Option { + self.mapping + .event_handlers + .iter() + .find(|handler| { + let event_type_matches = event.event_type == handler.event; + + if let Some(handler_origin) = handler.origin { + event_type_matches && event_origin == handler_origin + } else { + event_type_matches + } + }) + .cloned() + } + + pub(crate) fn has_block_handler(&self) -> bool { + !self.mapping.block_handlers.is_empty() + } + + /// Return an iterator over all event types from event handlers. + pub(crate) fn events(&self) -> impl Iterator { + self.mapping + .event_handlers + .iter() + .map(|handler| handler.event.as_str()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub struct UnresolvedDataSource { + pub kind: String, + pub network: Option, + pub name: String, + pub source: Source, + pub mapping: UnresolvedMapping, + pub context: Option, +} + +#[async_trait] +impl blockchain::UnresolvedDataSource for UnresolvedDataSource { + async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + _manifest_idx: u32, + ) -> Result { + let UnresolvedDataSource { + kind, + network, + name, + source, + mapping, + context, + } = self; + + info!(logger, "Resolve data source"; "name" => &name, "source" => &source.start_block); + + let mapping = mapping.resolve(resolver, logger).await?; + + DataSource::from_manifest(kind, network, name, source, mapping, context) + } +} + +impl TryFrom> for DataSource { + type Error = Error; + + fn try_from(_info: DataSourceTemplateInfo) -> Result { + Err(anyhow!(TEMPLATE_ERROR)) + } +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +pub struct BaseDataSourceTemplate { + pub kind: String, + pub network: Option, + pub name: String, + pub mapping: M, +} + +pub type UnresolvedDataSourceTemplate = BaseDataSourceTemplate; +pub type DataSourceTemplate = BaseDataSourceTemplate; + +#[async_trait] +impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { + async fn resolve( + self, + _resolver: &Arc, + _logger: &Logger, + _manifest_idx: u32, + ) -> Result { + Err(anyhow!(TEMPLATE_ERROR)) + } +} + +impl blockchain::DataSourceTemplate for DataSourceTemplate { + fn name(&self) -> &str { + unimplemented!("{}", TEMPLATE_ERROR); + } + + fn api_version(&self) -> semver::Version { + unimplemented!("{}", TEMPLATE_ERROR); + } + + fn runtime(&self) -> Option>> { + unimplemented!("{}", TEMPLATE_ERROR); + } + + fn manifest_idx(&self) -> u32 { + unimplemented!("{}", TEMPLATE_ERROR); + } +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UnresolvedMapping { + pub api_version: String, + pub language: String, + pub entities: Vec, + #[serde(default)] + pub block_handlers: Vec, + #[serde(default)] + pub event_handlers: Vec, + #[serde(default)] + pub transaction_handlers: Vec, + pub file: Link, +} + +impl UnresolvedMapping { + pub async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + ) -> Result { + let UnresolvedMapping { + api_version, + language, + entities, + block_handlers, + event_handlers, + transaction_handlers, + file: link, + } = self; + + let api_version = semver::Version::parse(&api_version)?; + + info!(logger, "Resolve mapping"; "link" => &link.link); + let module_bytes = resolver.cat(logger, &link).await?; + + Ok(Mapping { + api_version, + language, + entities, + block_handlers: block_handlers.clone(), + event_handlers: event_handlers.clone(), + transaction_handlers: transaction_handlers.clone(), + runtime: Arc::new(module_bytes), + link, + }) + } +} + +#[derive(Clone, Debug)] +pub struct Mapping { + pub api_version: semver::Version, + pub language: String, + pub entities: Vec, + pub block_handlers: Vec, + pub event_handlers: Vec, + pub transaction_handlers: Vec, + pub runtime: Arc>, + pub link: Link, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct MappingBlockHandler { + pub handler: String, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct MappingEventHandler { + pub event: String, + pub origin: Option, + pub handler: String, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct MappingTransactionHandler { + pub handler: String, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct Source { + #[serde(rename = "startBlock", default)] + pub start_block: BlockNumber, +} + +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Deserialize)] +pub enum EventOrigin { + BeginBlock, + DeliverTx, + EndBlock, +} + +fn multiple_origin_err(event_type: &str, origin: Option) -> Error { + let origin_err_name = match origin { + Some(origin) => format!("{:?}", origin), + None => "no".to_string(), + }; + + anyhow!( + "data source has multiple {} event handlers with {} origin", + event_type, + origin_err_name, + ) +} + +fn combined_origins_err(event_type: &str) -> Error { + anyhow!( + "data source has combined origin and no-origin {} event handlers", + event_type + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + use graph::blockchain::DataSource as _; + + #[test] + fn test_event_handlers_origin_validation() { + let cases = [ + ( + DataSource::with_event_handlers(vec![ + MappingEventHandler::with_origin("event_1", None), + MappingEventHandler::with_origin("event_2", None), + MappingEventHandler::with_origin("event_3", None), + ]), + vec![], + ), + ( + DataSource::with_event_handlers(vec![ + MappingEventHandler::with_origin("event_1", Some(EventOrigin::BeginBlock)), + MappingEventHandler::with_origin("event_2", Some(EventOrigin::BeginBlock)), + MappingEventHandler::with_origin("event_1", Some(EventOrigin::DeliverTx)), + MappingEventHandler::with_origin("event_1", Some(EventOrigin::EndBlock)), + MappingEventHandler::with_origin("event_2", Some(EventOrigin::DeliverTx)), + MappingEventHandler::with_origin("event_2", Some(EventOrigin::EndBlock)), + ]), + vec![], + ), + ( + DataSource::with_event_handlers(vec![ + MappingEventHandler::with_origin("event_1", None), + MappingEventHandler::with_origin("event_1", None), + MappingEventHandler::with_origin("event_2", None), + MappingEventHandler::with_origin("event_2", Some(EventOrigin::BeginBlock)), + MappingEventHandler::with_origin("event_3", Some(EventOrigin::EndBlock)), + MappingEventHandler::with_origin("event_3", Some(EventOrigin::EndBlock)), + ]), + vec![ + multiple_origin_err("event_1", None), + combined_origins_err("event_2"), + multiple_origin_err("event_3", Some(EventOrigin::EndBlock)), + ], + ), + ]; + + for (data_source, errors) in &cases { + let validation_errors = data_source.validate(); + + assert_eq!(errors.len(), validation_errors.len()); + + for error in errors.iter() { + assert!( + validation_errors + .iter() + .any(|validation_error| validation_error.to_string() == error.to_string()), + r#"expected "{}" to be in validation errors, but it wasn't"#, + error + ); + } + } + } + + impl DataSource { + fn with_event_handlers(event_handlers: Vec) -> DataSource { + DataSource { + kind: "cosmos".to_string(), + network: None, + name: "Test".to_string(), + source: Source { start_block: 1 }, + mapping: Mapping { + api_version: semver::Version::new(0, 0, 0), + language: "".to_string(), + entities: vec![], + block_handlers: vec![], + event_handlers, + transaction_handlers: vec![], + runtime: Arc::new(vec![]), + link: "test".to_string().into(), + }, + context: Arc::new(None), + creation_block: None, + } + } + } + + impl MappingEventHandler { + fn with_origin(event_type: &str, origin: Option) -> MappingEventHandler { + MappingEventHandler { + event: event_type.to_string(), + origin, + handler: "handler".to_string(), + } + } + } +} diff --git a/chain/cosmos/src/lib.rs b/chain/cosmos/src/lib.rs new file mode 100644 index 0000000..634cbe0 --- /dev/null +++ b/chain/cosmos/src/lib.rs @@ -0,0 +1,19 @@ +mod adapter; +mod capabilities; +pub mod chain; +pub mod codec; +mod data_source; +mod protobuf; +pub mod runtime; +mod trigger; + +pub use self::runtime::RuntimeAdapter; + +// ETHDEP: These concrete types should probably not be exposed. +pub use data_source::{DataSource, DataSourceTemplate}; + +pub use crate::adapter::TriggerFilter; +pub use crate::chain::Chain; + +pub use protobuf::pbcodec; +pub use protobuf::pbcodec::Block; diff --git a/chain/cosmos/src/protobuf/.gitignore b/chain/cosmos/src/protobuf/.gitignore new file mode 100644 index 0000000..9678694 --- /dev/null +++ b/chain/cosmos/src/protobuf/.gitignore @@ -0,0 +1,4 @@ +/google.protobuf.rs +/gogoproto.rs +/cosmos_proto.rs +/firehose.rs diff --git a/chain/cosmos/src/protobuf/mod.rs b/chain/cosmos/src/protobuf/mod.rs new file mode 100644 index 0000000..c3292e6 --- /dev/null +++ b/chain/cosmos/src/protobuf/mod.rs @@ -0,0 +1,8 @@ +#[rustfmt::skip] +#[path = "sf.cosmos.r#type.v1.rs"] +pub mod pbcodec; + +pub use graph_runtime_wasm::asc_abi::class::{Array, AscEnum, AscString, Uint8Array}; + +pub use crate::runtime::abi::*; +pub use pbcodec::*; diff --git a/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs b/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs new file mode 100644 index 0000000..d720b39 --- /dev/null +++ b/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs @@ -0,0 +1,641 @@ +#[graph_runtime_derive::generate_asc_type(__required__{header: Header,result_begin_block: ResponseBeginBlock,result_end_block: ResponseEndBlock})] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type(__required__{header: Header,result_begin_block: ResponseBeginBlock,result_end_block: ResponseEndBlock})] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Block { + #[prost(message, optional, tag="1")] + pub header: ::core::option::Option
, + #[prost(message, optional, tag="2")] + pub evidence: ::core::option::Option, + #[prost(message, optional, tag="3")] + pub last_commit: ::core::option::Option, + #[prost(message, optional, tag="4")] + pub result_begin_block: ::core::option::Option, + #[prost(message, optional, tag="5")] + pub result_end_block: ::core::option::Option, + #[prost(message, repeated, tag="7")] + pub transactions: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="8")] + pub validator_updates: ::prost::alloc::vec::Vec, +} +/// HeaderOnlyBlock is a standard \[Block\] structure where all other fields are +/// removed so that hydrating that object from a \[Block\] bytes payload will +/// drastically reduce allocated memory required to hold the full block. +/// +/// This can be used to unpack a \[Block\] when only the \[Header\] information +/// is required and greatly reduce required memory. +#[graph_runtime_derive::generate_asc_type(__required__{header: Header})] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type(__required__{header: Header})] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct HeaderOnlyBlock { + #[prost(message, optional, tag="1")] + pub header: ::core::option::Option
, +} +#[graph_runtime_derive::generate_asc_type(__required__{event: Event,block: HeaderOnlyBlock})] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type(__required__{event: Event,block: HeaderOnlyBlock})] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EventData { + #[prost(message, optional, tag="1")] + pub event: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub block: ::core::option::Option, +} +#[graph_runtime_derive::generate_asc_type(__required__{tx: TxResult,block: HeaderOnlyBlock})] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type(__required__{tx: TxResult,block: HeaderOnlyBlock})] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionData { + #[prost(message, optional, tag="1")] + pub tx: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub block: ::core::option::Option, +} +#[graph_runtime_derive::generate_asc_type(__required__{last_block_id: BlockID})] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type(__required__{last_block_id: BlockID})] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Header { + #[prost(message, optional, tag="1")] + pub version: ::core::option::Option, + #[prost(string, tag="2")] + pub chain_id: ::prost::alloc::string::String, + #[prost(uint64, tag="3")] + pub height: u64, + #[prost(message, optional, tag="4")] + pub time: ::core::option::Option, + #[prost(message, optional, tag="5")] + pub last_block_id: ::core::option::Option, + #[prost(bytes="vec", tag="6")] + pub last_commit_hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="7")] + pub data_hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="8")] + pub validators_hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="9")] + pub next_validators_hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="10")] + pub consensus_hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="11")] + pub app_hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="12")] + pub last_results_hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="13")] + pub evidence_hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="14")] + pub proposer_address: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="15")] + pub hash: ::prost::alloc::vec::Vec, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Consensus { + #[prost(uint64, tag="1")] + pub block: u64, + #[prost(uint64, tag="2")] + pub app: u64, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Timestamp { + #[prost(int64, tag="1")] + pub seconds: i64, + #[prost(int32, tag="2")] + pub nanos: i32, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockId { + #[prost(bytes="vec", tag="1")] + pub hash: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="2")] + pub part_set_header: ::core::option::Option, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PartSetHeader { + #[prost(uint32, tag="1")] + pub total: u32, + #[prost(bytes="vec", tag="2")] + pub hash: ::prost::alloc::vec::Vec, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EvidenceList { + #[prost(message, repeated, tag="1")] + pub evidence: ::prost::alloc::vec::Vec, +} +#[graph_runtime_derive::generate_asc_type(sum{duplicate_vote_evidence: DuplicateVoteEvidence,light_client_attack_evidence: LightClientAttackEvidence})] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type(sum{duplicate_vote_evidence: DuplicateVoteEvidence,light_client_attack_evidence: LightClientAttackEvidence})] +#[graph_runtime_derive::generate_array_type(Cosmos)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Evidence { + #[prost(oneof="evidence::Sum", tags="1, 2")] + pub sum: ::core::option::Option, +} +/// Nested message and enum types in `Evidence`. +pub mod evidence { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Sum { + #[prost(message, tag="1")] + DuplicateVoteEvidence(super::DuplicateVoteEvidence), + #[prost(message, tag="2")] + LightClientAttackEvidence(super::LightClientAttackEvidence), + } +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DuplicateVoteEvidence { + #[prost(message, optional, tag="1")] + pub vote_a: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub vote_b: ::core::option::Option, + #[prost(int64, tag="3")] + pub total_voting_power: i64, + #[prost(int64, tag="4")] + pub validator_power: i64, + #[prost(message, optional, tag="5")] + pub timestamp: ::core::option::Option, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EventVote { + #[prost(enumeration="SignedMsgType", tag="1")] + pub event_vote_type: i32, + #[prost(uint64, tag="2")] + pub height: u64, + #[prost(int32, tag="3")] + pub round: i32, + #[prost(message, optional, tag="4")] + pub block_id: ::core::option::Option, + #[prost(message, optional, tag="5")] + pub timestamp: ::core::option::Option, + #[prost(bytes="vec", tag="6")] + pub validator_address: ::prost::alloc::vec::Vec, + #[prost(int32, tag="7")] + pub validator_index: i32, + #[prost(bytes="vec", tag="8")] + pub signature: ::prost::alloc::vec::Vec, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LightClientAttackEvidence { + #[prost(message, optional, tag="1")] + pub conflicting_block: ::core::option::Option, + #[prost(int64, tag="2")] + pub common_height: i64, + #[prost(message, repeated, tag="3")] + pub byzantine_validators: ::prost::alloc::vec::Vec, + #[prost(int64, tag="4")] + pub total_voting_power: i64, + #[prost(message, optional, tag="5")] + pub timestamp: ::core::option::Option, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LightBlock { + #[prost(message, optional, tag="1")] + pub signed_header: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub validator_set: ::core::option::Option, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SignedHeader { + #[prost(message, optional, tag="1")] + pub header: ::core::option::Option
, + #[prost(message, optional, tag="2")] + pub commit: ::core::option::Option, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Commit { + #[prost(int64, tag="1")] + pub height: i64, + #[prost(int32, tag="2")] + pub round: i32, + #[prost(message, optional, tag="3")] + pub block_id: ::core::option::Option, + #[prost(message, repeated, tag="4")] + pub signatures: ::prost::alloc::vec::Vec, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[graph_runtime_derive::generate_array_type(Cosmos)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CommitSig { + #[prost(enumeration="BlockIdFlag", tag="1")] + pub block_id_flag: i32, + #[prost(bytes="vec", tag="2")] + pub validator_address: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="3")] + pub timestamp: ::core::option::Option, + #[prost(bytes="vec", tag="4")] + pub signature: ::prost::alloc::vec::Vec, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ValidatorSet { + #[prost(message, repeated, tag="1")] + pub validators: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="2")] + pub proposer: ::core::option::Option, + #[prost(int64, tag="3")] + pub total_voting_power: i64, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[graph_runtime_derive::generate_array_type(Cosmos)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Validator { + #[prost(bytes="vec", tag="1")] + pub address: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="2")] + pub pub_key: ::core::option::Option, + #[prost(int64, tag="3")] + pub voting_power: i64, + #[prost(int64, tag="4")] + pub proposer_priority: i64, +} +#[graph_runtime_derive::generate_asc_type(sum{ed25519: Vec,secp256k1: Vec})] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type(sum{ed25519: Vec,secp256k1: Vec})] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PublicKey { + #[prost(oneof="public_key::Sum", tags="1, 2")] + pub sum: ::core::option::Option, +} +/// Nested message and enum types in `PublicKey`. +pub mod public_key { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Sum { + #[prost(bytes, tag="1")] + Ed25519(::prost::alloc::vec::Vec), + #[prost(bytes, tag="2")] + Secp256k1(::prost::alloc::vec::Vec), + } +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ResponseBeginBlock { + #[prost(message, repeated, tag="1")] + pub events: ::prost::alloc::vec::Vec, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[graph_runtime_derive::generate_array_type(Cosmos)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Event { + #[prost(string, tag="1")] + pub event_type: ::prost::alloc::string::String, + #[prost(message, repeated, tag="2")] + pub attributes: ::prost::alloc::vec::Vec, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[graph_runtime_derive::generate_array_type(Cosmos)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EventAttribute { + #[prost(string, tag="1")] + pub key: ::prost::alloc::string::String, + #[prost(string, tag="2")] + pub value: ::prost::alloc::string::String, + #[prost(bool, tag="3")] + pub index: bool, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ResponseEndBlock { + #[prost(message, repeated, tag="1")] + pub validator_updates: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="2")] + pub consensus_param_updates: ::core::option::Option, + #[prost(message, repeated, tag="3")] + pub events: ::prost::alloc::vec::Vec, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[graph_runtime_derive::generate_array_type(Cosmos)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ValidatorUpdate { + #[prost(bytes="vec", tag="1")] + pub address: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="2")] + pub pub_key: ::core::option::Option, + #[prost(int64, tag="3")] + pub power: i64, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ConsensusParams { + #[prost(message, optional, tag="1")] + pub block: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub evidence: ::core::option::Option, + #[prost(message, optional, tag="3")] + pub validator: ::core::option::Option, + #[prost(message, optional, tag="4")] + pub version: ::core::option::Option, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockParams { + #[prost(int64, tag="1")] + pub max_bytes: i64, + #[prost(int64, tag="2")] + pub max_gas: i64, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EvidenceParams { + #[prost(int64, tag="1")] + pub max_age_num_blocks: i64, + #[prost(message, optional, tag="2")] + pub max_age_duration: ::core::option::Option, + #[prost(int64, tag="3")] + pub max_bytes: i64, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Duration { + #[prost(int64, tag="1")] + pub seconds: i64, + #[prost(int32, tag="2")] + pub nanos: i32, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ValidatorParams { + #[prost(string, repeated, tag="1")] + pub pub_key_types: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct VersionParams { + #[prost(uint64, tag="1")] + pub app_version: u64, +} +#[graph_runtime_derive::generate_asc_type(__required__{tx: Tx,result: ResponseDeliverTx})] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type(__required__{tx: Tx,result: ResponseDeliverTx})] +#[graph_runtime_derive::generate_array_type(Cosmos)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TxResult { + #[prost(uint64, tag="1")] + pub height: u64, + #[prost(uint32, tag="2")] + pub index: u32, + #[prost(message, optional, tag="3")] + pub tx: ::core::option::Option, + #[prost(message, optional, tag="4")] + pub result: ::core::option::Option, + #[prost(bytes="vec", tag="5")] + pub hash: ::prost::alloc::vec::Vec, +} +#[graph_runtime_derive::generate_asc_type(__required__{body: TxBody})] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type(__required__{body: TxBody})] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Tx { + #[prost(message, optional, tag="1")] + pub body: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub auth_info: ::core::option::Option, + #[prost(bytes="vec", repeated, tag="3")] + pub signatures: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TxBody { + #[prost(message, repeated, tag="1")] + pub messages: ::prost::alloc::vec::Vec<::prost_types::Any>, + #[prost(string, tag="2")] + pub memo: ::prost::alloc::string::String, + #[prost(uint64, tag="3")] + pub timeout_height: u64, + #[prost(message, repeated, tag="1023")] + pub extension_options: ::prost::alloc::vec::Vec<::prost_types::Any>, + #[prost(message, repeated, tag="2047")] + pub non_critical_extension_options: ::prost::alloc::vec::Vec<::prost_types::Any>, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[graph_runtime_derive::generate_array_type(Cosmos)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Any { + #[prost(string, tag="1")] + pub type_url: ::prost::alloc::string::String, + #[prost(bytes="vec", tag="2")] + pub value: ::prost::alloc::vec::Vec, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AuthInfo { + #[prost(message, repeated, tag="1")] + pub signer_infos: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="2")] + pub fee: ::core::option::Option, + #[prost(message, optional, tag="3")] + pub tip: ::core::option::Option, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[graph_runtime_derive::generate_array_type(Cosmos)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SignerInfo { + #[prost(message, optional, tag="1")] + pub public_key: ::core::option::Option<::prost_types::Any>, + #[prost(message, optional, tag="2")] + pub mode_info: ::core::option::Option, + #[prost(uint64, tag="3")] + pub sequence: u64, +} +#[graph_runtime_derive::generate_asc_type(sum{single: ModeInfoSingle,multi: ModeInfoMulti})] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type(sum{single: ModeInfoSingle,multi: ModeInfoMulti})] +#[graph_runtime_derive::generate_array_type(Cosmos)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ModeInfo { + #[prost(oneof="mode_info::Sum", tags="1, 2")] + pub sum: ::core::option::Option, +} +/// Nested message and enum types in `ModeInfo`. +pub mod mode_info { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Sum { + #[prost(message, tag="1")] + Single(super::ModeInfoSingle), + #[prost(message, tag="2")] + Multi(super::ModeInfoMulti), + } +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ModeInfoSingle { + #[prost(enumeration="SignMode", tag="1")] + pub mode: i32, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ModeInfoMulti { + #[prost(message, optional, tag="1")] + pub bitarray: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub mode_infos: ::prost::alloc::vec::Vec, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CompactBitArray { + #[prost(uint32, tag="1")] + pub extra_bits_stored: u32, + #[prost(bytes="vec", tag="2")] + pub elems: ::prost::alloc::vec::Vec, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Fee { + #[prost(message, repeated, tag="1")] + pub amount: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="2")] + pub gas_limit: u64, + #[prost(string, tag="3")] + pub payer: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub granter: ::prost::alloc::string::String, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[graph_runtime_derive::generate_array_type(Cosmos)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Coin { + #[prost(string, tag="1")] + pub denom: ::prost::alloc::string::String, + #[prost(string, tag="2")] + pub amount: ::prost::alloc::string::String, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Tip { + #[prost(message, repeated, tag="1")] + pub amount: ::prost::alloc::vec::Vec, + #[prost(string, tag="2")] + pub tipper: ::prost::alloc::string::String, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ResponseDeliverTx { + #[prost(uint32, tag="1")] + pub code: u32, + #[prost(bytes="vec", tag="2")] + pub data: ::prost::alloc::vec::Vec, + #[prost(string, tag="3")] + pub log: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub info: ::prost::alloc::string::String, + #[prost(int64, tag="5")] + pub gas_wanted: i64, + #[prost(int64, tag="6")] + pub gas_used: i64, + #[prost(message, repeated, tag="7")] + pub events: ::prost::alloc::vec::Vec, + #[prost(string, tag="8")] + pub codespace: ::prost::alloc::string::String, +} +#[graph_runtime_derive::generate_asc_type()] +#[graph_runtime_derive::generate_network_type_id(Cosmos)] +#[graph_runtime_derive::generate_from_rust_type()] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ValidatorSetUpdates { + #[prost(message, repeated, tag="1")] + pub validator_updates: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum SignedMsgType { + Unknown = 0, + Prevote = 1, + Precommit = 2, + Proposal = 32, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum BlockIdFlag { + Unknown = 0, + Absent = 1, + Commit = 2, + Nil = 3, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum SignMode { + Unspecified = 0, + Direct = 1, + Textual = 2, + LegacyAminoJson = 127, +} diff --git a/chain/cosmos/src/runtime/abi.rs b/chain/cosmos/src/runtime/abi.rs new file mode 100644 index 0000000..3c5f0dd --- /dev/null +++ b/chain/cosmos/src/runtime/abi.rs @@ -0,0 +1,79 @@ +use crate::protobuf::*; +pub use graph::semver::Version; + +pub use graph::runtime::{ + asc_new, gas::GasCounter, AscHeap, AscIndexId, AscPtr, AscType, AscValue, + DeterministicHostError, IndexForAscTypeId, ToAscObj, +}; +/* +TODO: AscBytesArray seem to be generic to all chains, but AscIndexId pins it to Cosmos +****************** this can be moved to runtime graph/runtime/src/asc_heap.rs, but IndexForAscTypeId::CosmosBytesArray ****** +*/ +pub struct AscBytesArray(pub Array>); + +impl ToAscObj for Vec> { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let content: Result, _> = self + .iter() + .map(|x| asc_new(heap, &graph_runtime_wasm::asc_abi::class::Bytes(x), gas)) + .collect(); + + Ok(AscBytesArray(Array::new(&content?, heap, gas)?)) + } +} + +//this can be moved to runtime +impl AscType for AscBytesArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) + } +} + +//we will have to keep this chain specific (Inner/Outer) +impl AscIndexId for AscBytesArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::CosmosBytesArray; +} + +/************************************************************************** */ +// this can be moved to runtime - prost_types::Any +impl ToAscObj for prost_types::Any { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscAny { + type_url: asc_new(heap, &self.type_url, gas)?, + value: asc_new( + heap, + &graph_runtime_wasm::asc_abi::class::Bytes(&self.value), + gas, + )?, + ..Default::default() + }) + } +} + +//this can be moved to runtime - prost_types::Any +impl ToAscObj for Vec { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); + + Ok(AscAnyArray(Array::new(&content?, heap, gas)?)) + } +} diff --git a/chain/cosmos/src/runtime/mod.rs b/chain/cosmos/src/runtime/mod.rs new file mode 100644 index 0000000..17a8ec8 --- /dev/null +++ b/chain/cosmos/src/runtime/mod.rs @@ -0,0 +1,336 @@ +pub use runtime_adapter::RuntimeAdapter; + +pub mod abi; +pub mod runtime_adapter; + +#[cfg(test)] +mod test { + use crate::protobuf::*; + + use graph::semver::Version; + + /// A macro that takes an ASC struct value definition and calls AscBytes methods to check that + /// memory layout is padded properly. + macro_rules! assert_asc_bytes { + ($struct_name:ident { + $($field:ident : $field_value:expr),+ + $(,)? // trailing + }) => { + let value = $struct_name { + $($field: $field_value),+ + }; + + // just call the function. it will panic on misalignments + let asc_bytes = value.to_asc_bytes().unwrap(); + + let value_004 = $struct_name::from_asc_bytes(&asc_bytes, &Version::new(0, 0, 4)).unwrap(); + let value_005 = $struct_name::from_asc_bytes(&asc_bytes, &Version::new(0, 0, 5)).unwrap(); + + // turn the values into bytes again to verify that they are the same as the original + // because these types usually don't implement PartialEq + assert_eq!( + asc_bytes, + value_004.to_asc_bytes().unwrap(), + "Expected {} v0.0.4 asc bytes to be the same", + stringify!($struct_name) + ); + assert_eq!( + asc_bytes, + value_005.to_asc_bytes().unwrap(), + "Expected {} v0.0.5 asc bytes to be the same", + stringify!($struct_name) + ); + }; + } + + #[test] + fn test_asc_type_alignment() { + // TODO: automatically generate these tests for each struct in derive(AscType) macro + + assert_asc_bytes!(AscBlock { + header: new_asc_ptr(), + evidence: new_asc_ptr(), + last_commit: new_asc_ptr(), + result_begin_block: new_asc_ptr(), + result_end_block: new_asc_ptr(), + transactions: new_asc_ptr(), + validator_updates: new_asc_ptr(), + }); + + assert_asc_bytes!(AscHeaderOnlyBlock { + header: new_asc_ptr(), + }); + + assert_asc_bytes!(AscEventData { + event: new_asc_ptr(), + block: new_asc_ptr(), + }); + + assert_asc_bytes!(AscTransactionData { + tx: new_asc_ptr(), + block: new_asc_ptr(), + }); + + assert_asc_bytes!(AscHeader { + version: new_asc_ptr(), + chain_id: new_asc_ptr(), + height: 20, + time: new_asc_ptr(), + last_block_id: new_asc_ptr(), + last_commit_hash: new_asc_ptr(), + data_hash: new_asc_ptr(), + validators_hash: new_asc_ptr(), + next_validators_hash: new_asc_ptr(), + consensus_hash: new_asc_ptr(), + app_hash: new_asc_ptr(), + last_results_hash: new_asc_ptr(), + evidence_hash: new_asc_ptr(), + proposer_address: new_asc_ptr(), + hash: new_asc_ptr(), + }); + + assert_asc_bytes!(AscConsensus { block: 0, app: 0 }); + + assert_asc_bytes!(AscTimestamp { + seconds: 20, + nanos: 20, + }); + + assert_asc_bytes!(AscBlockId { + hash: new_asc_ptr(), + part_set_header: new_asc_ptr(), + }); + + assert_asc_bytes!(AscPartSetHeader { + total: 20, + hash: new_asc_ptr(), + }); + + assert_asc_bytes!(AscEvidenceList { + evidence: new_asc_ptr(), + }); + + assert_asc_bytes!(AscEvidence { + duplicate_vote_evidence: new_asc_ptr(), + light_client_attack_evidence: new_asc_ptr(), + }); + + assert_asc_bytes!(AscDuplicateVoteEvidence { + vote_a: new_asc_ptr(), + vote_b: new_asc_ptr(), + total_voting_power: 20, + validator_power: 20, + timestamp: new_asc_ptr(), + }); + + assert_asc_bytes!(AscEventVote { + event_vote_type: 20, + height: 20, + round: 20, + block_id: new_asc_ptr(), + timestamp: new_asc_ptr(), + validator_address: new_asc_ptr(), + validator_index: 20, + signature: new_asc_ptr(), + }); + + assert_asc_bytes!(AscLightClientAttackEvidence { + conflicting_block: new_asc_ptr(), + common_height: 20, + total_voting_power: 20, + byzantine_validators: new_asc_ptr(), + timestamp: new_asc_ptr(), + }); + + assert_asc_bytes!(AscLightBlock { + signed_header: new_asc_ptr(), + validator_set: new_asc_ptr(), + }); + + assert_asc_bytes!(AscSignedHeader { + header: new_asc_ptr(), + commit: new_asc_ptr(), + }); + + assert_asc_bytes!(AscCommit { + height: 20, + round: 20, + block_id: new_asc_ptr(), + signatures: new_asc_ptr(), + }); + + assert_asc_bytes!(AscCommitSig { + block_id_flag: 20, + validator_address: new_asc_ptr(), + timestamp: new_asc_ptr(), + signature: new_asc_ptr(), + }); + + assert_asc_bytes!(AscValidatorSet { + validators: new_asc_ptr(), + proposer: new_asc_ptr(), + total_voting_power: 20, + }); + + assert_asc_bytes!(AscValidator { + address: new_asc_ptr(), + pub_key: new_asc_ptr(), + voting_power: 20, + proposer_priority: 20, + }); + + assert_asc_bytes!(AscPublicKey { + ed25519: new_asc_ptr(), + secp256k1: new_asc_ptr(), + }); + + assert_asc_bytes!(AscResponseBeginBlock { + events: new_asc_ptr(), + }); + + assert_asc_bytes!(AscEvent { + event_type: new_asc_ptr(), + attributes: new_asc_ptr(), + }); + + assert_asc_bytes!(AscEventAttribute { + key: new_asc_ptr(), + value: new_asc_ptr(), + index: true, + }); + + assert_asc_bytes!(AscResponseEndBlock { + validator_updates: new_asc_ptr(), + consensus_param_updates: new_asc_ptr(), + events: new_asc_ptr(), + }); + + assert_asc_bytes!(AscValidatorUpdate { + address: new_asc_ptr(), + pub_key: new_asc_ptr(), + power: 20, + }); + + assert_asc_bytes!(AscConsensusParams { + block: new_asc_ptr(), + evidence: new_asc_ptr(), + validator: new_asc_ptr(), + version: new_asc_ptr(), + }); + + assert_asc_bytes!(AscBlockParams { + max_bytes: 20, + max_gas: 20, + }); + + assert_asc_bytes!(AscEvidenceParams { + max_age_num_blocks: 20, + max_age_duration: new_asc_ptr(), + max_bytes: 20, + }); + + assert_asc_bytes!(AscDuration { + seconds: 20, + nanos: 20, + }); + + assert_asc_bytes!(AscValidatorParams { + pub_key_types: new_asc_ptr(), + }); + + assert_asc_bytes!(AscVersionParams { app_version: 20 }); + + assert_asc_bytes!(AscTxResult { + height: 20, + index: 20, + tx: new_asc_ptr(), + result: new_asc_ptr(), + hash: new_asc_ptr(), + }); + + assert_asc_bytes!(AscTx { + body: new_asc_ptr(), + auth_info: new_asc_ptr(), + signatures: new_asc_ptr(), + }); + + assert_asc_bytes!(AscTxBody { + messages: new_asc_ptr(), + memo: new_asc_ptr(), + timeout_height: 20, + extension_options: new_asc_ptr(), + non_critical_extension_options: new_asc_ptr(), + }); + + assert_asc_bytes!(AscAny { + type_url: new_asc_ptr(), + value: new_asc_ptr(), + }); + + assert_asc_bytes!(AscAuthInfo { + signer_infos: new_asc_ptr(), + fee: new_asc_ptr(), + tip: new_asc_ptr(), + }); + + assert_asc_bytes!(AscSignerInfo { + public_key: new_asc_ptr(), + mode_info: new_asc_ptr(), + sequence: 20, + }); + + assert_asc_bytes!(AscModeInfo { + single: new_asc_ptr(), + multi: new_asc_ptr(), + }); + + assert_asc_bytes!(AscModeInfoSingle { mode: 20 }); + + assert_asc_bytes!(AscModeInfoMulti { + bitarray: new_asc_ptr(), + mode_infos: new_asc_ptr(), + }); + + assert_asc_bytes!(AscCompactBitArray { + extra_bits_stored: 20, + elems: new_asc_ptr(), + }); + + assert_asc_bytes!(AscFee { + amount: new_asc_ptr(), + gas_limit: 20, + payer: new_asc_ptr(), + granter: new_asc_ptr(), + }); + + assert_asc_bytes!(AscCoin { + denom: new_asc_ptr(), + amount: new_asc_ptr(), + }); + + assert_asc_bytes!(AscTip { + amount: new_asc_ptr(), + tipper: new_asc_ptr(), + }); + + assert_asc_bytes!(AscResponseDeliverTx { + code: 20, + data: new_asc_ptr(), + log: new_asc_ptr(), + info: new_asc_ptr(), + gas_wanted: 20, + gas_used: 20, + events: new_asc_ptr(), + codespace: new_asc_ptr(), + }); + + assert_asc_bytes!(AscValidatorSetUpdates { + validator_updates: new_asc_ptr(), + }); + } + + // non-null AscPtr + fn new_asc_ptr() -> AscPtr { + AscPtr::new(12) + } +} diff --git a/chain/cosmos/src/runtime/runtime_adapter.rs b/chain/cosmos/src/runtime/runtime_adapter.rs new file mode 100644 index 0000000..4bced40 --- /dev/null +++ b/chain/cosmos/src/runtime/runtime_adapter.rs @@ -0,0 +1,12 @@ +use crate::{Chain, DataSource}; +use anyhow::Result; +use blockchain::HostFn; +use graph::blockchain; + +pub struct RuntimeAdapter {} + +impl blockchain::RuntimeAdapter for RuntimeAdapter { + fn host_fns(&self, _ds: &DataSource) -> Result> { + Ok(vec![]) + } +} diff --git a/chain/cosmos/src/trigger.rs b/chain/cosmos/src/trigger.rs new file mode 100644 index 0000000..35880c5 --- /dev/null +++ b/chain/cosmos/src/trigger.rs @@ -0,0 +1,230 @@ +use std::{cmp::Ordering, sync::Arc}; + +use graph::blockchain::{Block, BlockHash, TriggerData}; +use graph::cheap_clone::CheapClone; +use graph::prelude::{BlockNumber, Error}; +use graph::runtime::{asc_new, gas::GasCounter, AscHeap, AscPtr, DeterministicHostError}; +use graph_runtime_wasm::module::ToAscPtr; + +use crate::codec; +use crate::data_source::EventOrigin; + +// Logging the block is too verbose, so this strips the block from the trigger for Debug. +impl std::fmt::Debug for CosmosTrigger { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + #[derive(Debug)] + pub enum MappingTriggerWithoutBlock<'e> { + Block, + Event { + event_type: &'e str, + origin: EventOrigin, + }, + Transaction, + } + + let trigger_without_block = match self { + CosmosTrigger::Block(_) => MappingTriggerWithoutBlock::Block, + CosmosTrigger::Event { event_data, origin } => MappingTriggerWithoutBlock::Event { + event_type: &event_data.event().map_err(|_| std::fmt::Error)?.event_type, + origin: *origin, + }, + CosmosTrigger::Transaction(_) => MappingTriggerWithoutBlock::Transaction, + }; + + write!(f, "{:?}", trigger_without_block) + } +} + +impl ToAscPtr for CosmosTrigger { + fn to_asc_ptr( + self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + Ok(match self { + CosmosTrigger::Block(block) => asc_new(heap, block.as_ref(), gas)?.erase(), + CosmosTrigger::Event { event_data, .. } => { + asc_new(heap, event_data.as_ref(), gas)?.erase() + } + CosmosTrigger::Transaction(transaction_data) => { + asc_new(heap, transaction_data.as_ref(), gas)?.erase() + } + }) + } +} + +#[derive(Clone)] +pub enum CosmosTrigger { + Block(Arc), + Event { + event_data: Arc, + origin: EventOrigin, + }, + Transaction(Arc), +} + +impl CheapClone for CosmosTrigger { + fn cheap_clone(&self) -> CosmosTrigger { + match self { + CosmosTrigger::Block(block) => CosmosTrigger::Block(block.cheap_clone()), + CosmosTrigger::Event { event_data, origin } => CosmosTrigger::Event { + event_data: event_data.cheap_clone(), + origin: *origin, + }, + CosmosTrigger::Transaction(transaction_data) => { + CosmosTrigger::Transaction(transaction_data.cheap_clone()) + } + } + } +} + +impl PartialEq for CosmosTrigger { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Block(a_ptr), Self::Block(b_ptr)) => a_ptr == b_ptr, + ( + Self::Event { + event_data: a_event_data, + origin: a_origin, + }, + Self::Event { + event_data: b_event_data, + origin: b_origin, + }, + ) => { + if let (Ok(a_event), Ok(b_event)) = (a_event_data.event(), b_event_data.event()) { + a_event.event_type == b_event.event_type && a_origin == b_origin + } else { + false + } + } + (Self::Transaction(a_ptr), Self::Transaction(b_ptr)) => a_ptr == b_ptr, + _ => false, + } + } +} + +impl Eq for CosmosTrigger {} + +impl CosmosTrigger { + pub(crate) fn with_event( + event: codec::Event, + block: codec::HeaderOnlyBlock, + origin: EventOrigin, + ) -> CosmosTrigger { + CosmosTrigger::Event { + event_data: Arc::new(codec::EventData { + event: Some(event), + block: Some(block), + }), + origin, + } + } + + pub(crate) fn with_transaction( + tx_result: codec::TxResult, + block: codec::HeaderOnlyBlock, + ) -> CosmosTrigger { + CosmosTrigger::Transaction(Arc::new(codec::TransactionData { + tx: Some(tx_result), + block: Some(block), + })) + } + + pub fn block_number(&self) -> Result { + match self { + CosmosTrigger::Block(block) => Ok(block.number()), + CosmosTrigger::Event { event_data, .. } => event_data.block().map(|b| b.number()), + CosmosTrigger::Transaction(transaction_data) => { + transaction_data.block().map(|b| b.number()) + } + } + } + + pub fn block_hash(&self) -> Result { + match self { + CosmosTrigger::Block(block) => Ok(block.hash()), + CosmosTrigger::Event { event_data, .. } => event_data.block().map(|b| b.hash()), + CosmosTrigger::Transaction(transaction_data) => { + transaction_data.block().map(|b| b.hash()) + } + } + } +} + +impl Ord for CosmosTrigger { + fn cmp(&self, other: &Self) -> Ordering { + match (self, other) { + // Keep the order when comparing two block triggers + (Self::Block(..), Self::Block(..)) => Ordering::Equal, + + // Block triggers always come last + (Self::Block(..), _) => Ordering::Greater, + (_, Self::Block(..)) => Ordering::Less, + + // Events have no intrinsic ordering information, so we keep the order in + // which they are included in the `events` field + (Self::Event { .. }, Self::Event { .. }) => Ordering::Equal, + + // Transactions are ordered by their index inside the block + (Self::Transaction(a), Self::Transaction(b)) => { + if let (Ok(a_tx_result), Ok(b_tx_result)) = (a.tx_result(), b.tx_result()) { + a_tx_result.index.cmp(&b_tx_result.index) + } else { + Ordering::Equal + } + } + + // When comparing events and transactions, transactions go first + (Self::Transaction(..), Self::Event { .. }) => Ordering::Less, + (Self::Event { .. }, Self::Transaction(..)) => Ordering::Greater, + } + } +} + +impl PartialOrd for CosmosTrigger { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl TriggerData for CosmosTrigger { + fn error_context(&self) -> std::string::String { + match self { + CosmosTrigger::Block(..) => { + if let (Ok(block_number), Ok(block_hash)) = (self.block_number(), self.block_hash()) + { + format!("block #{block_number}, hash {block_hash}") + } else { + "block".to_string() + } + } + CosmosTrigger::Event { event_data, origin } => { + if let (Ok(event), Ok(block_number), Ok(block_hash)) = + (event_data.event(), self.block_number(), self.block_hash()) + { + format!( + "event type {}, origin: {:?}, block #{block_number}, hash {block_hash}", + event.event_type, origin, + ) + } else { + "event in block".to_string() + } + } + CosmosTrigger::Transaction(transaction_data) => { + if let (Ok(block_number), Ok(block_hash), Ok(response_deliver_tx)) = ( + self.block_number(), + self.block_hash(), + transaction_data.response_deliver_tx(), + ) { + format!( + "block #{block_number}, hash {block_hash}, transaction log: {}", + response_deliver_tx.log + ) + } else { + "transaction block".to_string() + } + } + } + } +} diff --git a/chain/ethereum/Cargo.toml b/chain/ethereum/Cargo.toml new file mode 100644 index 0000000..b3143cd --- /dev/null +++ b/chain/ethereum/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "graph-chain-ethereum" +version = "0.27.0" +edition = "2021" + +[dependencies] +envconfig = "0.10.0" +futures = "0.1.21" +http = "0.2.4" +jsonrpc-core = "18.0.0" +graph = { path = "../../graph" } +lazy_static = "1.2.0" +serde = "1.0" +prost = "0.10.4" +prost-types = "0.10.1" +dirs-next = "2.0" +anyhow = "1.0" +tiny-keccak = "1.5.0" +hex = "0.4.3" +semver = "1.0.12" + +itertools = "0.10.3" + +graph-runtime-wasm = { path = "../../runtime/wasm" } +graph-runtime-derive = { path = "../../runtime/derive" } + +[dev-dependencies] +test-store = { path = "../../store/test-store" } +base64 = "0.13.0" + +[build-dependencies] +tonic-build = { version = "0.7.2", features = ["prost"] } diff --git a/chain/ethereum/build.rs b/chain/ethereum/build.rs new file mode 100644 index 0000000..0efb360 --- /dev/null +++ b/chain/ethereum/build.rs @@ -0,0 +1,8 @@ +fn main() { + println!("cargo:rerun-if-changed=proto"); + + tonic_build::configure() + .out_dir("src/protobuf") + .compile(&["proto/codec.proto"], &["proto"]) + .expect("Failed to compile Firehose Ethereum proto(s)"); +} diff --git a/chain/ethereum/examples/firehose.rs b/chain/ethereum/examples/firehose.rs new file mode 100644 index 0000000..d2088b2 --- /dev/null +++ b/chain/ethereum/examples/firehose.rs @@ -0,0 +1,102 @@ +use anyhow::Error; +use graph::{ + env::env_var, + prelude::{prost, tokio, tonic}, + {firehose, firehose::FirehoseEndpoint, firehose::ForkStep}, +}; +use graph_chain_ethereum::codec; +use hex::ToHex; +use prost::Message; +use std::sync::Arc; +use tonic::Streaming; + +#[tokio::main] +async fn main() -> Result<(), Error> { + let mut cursor: Option = None; + let token_env = env_var("SF_API_TOKEN", "".to_string()); + let mut token: Option = None; + if token_env.len() > 0 { + token = Some(token_env); + } + + let firehose = Arc::new(FirehoseEndpoint::new( + "firehose", + "https://api.streamingfast.io:443", + token, + false, + false, + 1, + )); + + loop { + println!("Connecting to the stream!"); + let mut stream: Streaming = match firehose + .clone() + .stream_blocks(firehose::Request { + start_block_num: 12369739, + stop_block_num: 12369739, + start_cursor: match &cursor { + Some(c) => c.clone(), + None => String::from(""), + }, + fork_steps: vec![ForkStep::StepNew as i32, ForkStep::StepUndo as i32], + ..Default::default() + }) + .await + { + Ok(s) => s, + Err(e) => { + println!("Could not connect to stream! {}", e); + continue; + } + }; + + loop { + let resp = match stream.message().await { + Ok(Some(t)) => t, + Ok(None) => { + println!("Stream completed"); + return Ok(()); + } + Err(e) => { + println!("Error getting message {}", e); + break; + } + }; + + let b = codec::Block::decode(resp.block.unwrap().value.as_ref()); + match b { + Ok(b) => { + println!( + "Block #{} ({}) ({})", + b.number, + hex::encode(b.hash), + resp.step + ); + b.transaction_traces.iter().for_each(|trx| { + let mut logs: Vec = vec![]; + trx.calls.iter().for_each(|call| { + call.logs.iter().for_each(|log| { + logs.push(format!( + "Log {} Topics, Address {}, Trx Index {}, Block Index {}", + log.topics.len(), + log.address.encode_hex::(), + log.index, + log.block_index + )); + }) + }); + + if logs.len() > 0 { + println!("Transaction {}", trx.hash.encode_hex::()); + logs.iter().for_each(|log| println!("{}", log)); + } + }); + + cursor = Some(resp.cursor) + } + Err(e) => panic!("Unable to decode {:?}", e), + } + } + } +} diff --git a/chain/ethereum/proto/codec.proto b/chain/ethereum/proto/codec.proto new file mode 100644 index 0000000..3c9f737 --- /dev/null +++ b/chain/ethereum/proto/codec.proto @@ -0,0 +1,508 @@ +syntax = "proto3"; + +package sf.ethereum.type.v2; + +option go_package = "github.com/streamingfast/sf-ethereum/types/pb/sf/ethereum/type/v2;pbeth"; + +import "google/protobuf/timestamp.proto"; + +message Block { + int32 ver = 1; + bytes hash = 2; + uint64 number = 3; + uint64 size = 4; + BlockHeader header = 5; + + // Uncles represents block produced with a valid solution but were not actually choosen + // as the canonical block for the given height so they are mostly "forked" blocks. + // + // If the Block has been produced using the Proof of Stake consensus algorithm, this + // field will actually be always empty. + repeated BlockHeader uncles = 6; + + repeated TransactionTrace transaction_traces = 10; + repeated BalanceChange balance_changes = 11; + repeated CodeChange code_changes = 20; + + reserved 40; // bool filtering_applied = 40 [deprecated = true]; + reserved 41; // string filtering_include_filter_expr = 41 [deprecated = true]; + reserved 42; // string filtering_exclude_filter_expr = 42 [deprecated = true]; +} + +// HeaderOnlyBlock is used to optimally unpack the [Block] structure (note the +// corresponding message number for the `header` field) while consuming less +// memory, when only the `header` is desired. +// +// WARN: this is a client-side optimization pattern and should be moved in the +// consuming code. +message HeaderOnlyBlock { + BlockHeader header = 5; +} + +// BlockWithRefs is a lightweight block, with traces and transactions +// purged from the `block` within, and only. It is used in transports +// to pass block data around. +message BlockWithRefs { + string id = 1; + Block block = 2; + TransactionRefs transaction_trace_refs = 3; + bool irreversible = 4; +} + +message TransactionRefs { + repeated bytes hashes = 1; +} + +message UnclesHeaders { + repeated BlockHeader uncles = 1; +} + +message BlockRef { + bytes hash = 1; + uint64 number = 2; +} + +message BlockHeader { + bytes parent_hash = 1; + + // Uncle hash of the block, some reference it as `sha3Uncles`, but `sha3`` is badly worded, so we prefer `uncle_hash`, also + // referred as `ommers` in EIP specification. + // + // If the Block containing this `BlockHeader` has been produced using the Proof of Stake + // consensus algorithm, this field will actually be constant and set to `0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347`. + bytes uncle_hash = 2; + + bytes coinbase = 3; + bytes state_root = 4; + bytes transactions_root = 5; + bytes receipt_root = 6; + bytes logs_bloom = 7; + + // Difficulty is the difficulty of the Proof of Work algorithm that was required to compute a solution. + // + // If the Block containing this `BlockHeader` has been produced using the Proof of Stake + // consensus algorithm, this field will actually be constant and set to `0x00`. + BigInt difficulty = 8; + + // TotalDifficulty is the sum of all previous blocks difficulty including this block difficulty. + // + // If the Block containing this `BlockHeader` has been produced using the Proof of Stake + // consensus algorithm, this field will actually be constant and set to the terminal total difficulty + // that was required to transition to Proof of Stake algorithm, which varies per network. It is set to + // 58 750 000 000 000 000 000 000 on Ethereum Mainnet and to 10 790 000 on Ethereum Testnet Goerli. + BigInt total_difficulty = 17; + + uint64 number = 9; + uint64 gas_limit = 10; + uint64 gas_used = 11; + google.protobuf.Timestamp timestamp = 12; + + // ExtraData is free-form bytes included in the block by the "miner". While on Yellow paper of + // Ethereum this value is maxed to 32 bytes, other consensus algorithm like Clique and some other + // forks are using bigger values to carry special consensus data. + // + // If the Block containing this `BlockHeader` has been produced using the Proof of Stake + // consensus algorithm, this field is strictly enforced to be <= 32 bytes. + bytes extra_data = 13; + + // MixHash is used to prove, when combined with the `nonce` that sufficient amount of computation has been + // achieved and that the solution found is valid. + bytes mix_hash = 14; + + // Nonce is used to prove, when combined with the `mix_hash` that sufficient amount of computation has been + // achieved and that the solution found is valid. + // + // If the Block containing this `BlockHeader` has been produced using the Proof of Stake + // consensus algorithm, this field will actually be constant and set to `0`. + uint64 nonce = 15; + + // Hash is the hash of the block which is actually the computation: + // + // Keccak256(rlp([ + // parent_hash, + // uncle_hash, + // coinbase, + // state_root, + // transactions_root, + // receipt_root, + // logs_bloom, + // difficulty, + // number, + // gas_limit, + // gas_used, + // timestamp, + // extra_data, + // mix_hash, + // nonce, + // base_fee_per_gas + // ])) + // + bytes hash = 16; + + // Base fee per gas according to EIP-1559 (e.g. London Fork) rules, only set if London is present/active on the chain. + BigInt base_fee_per_gas = 18; +} + +message BigInt { + bytes bytes = 1; +} + +message TransactionTrace { + // consensus + bytes to = 1; + uint64 nonce = 2; + // GasPrice represents the effective price that has been paid for each gas unit of this transaction. Over time, the + // Ethereum rules changes regarding GasPrice field here. Before London fork, the GasPrice was always set to the + // fixed gas price. After London fork, this value has different meaning depending on the transaction type (see `Type` field). + // + // In cases where `TransactionTrace.Type == TRX_TYPE_LEGACY || TRX_TYPE_ACCESS_LIST`, then GasPrice has the same meaning + // as before the London fork. + // + // In cases where `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE`, then GasPrice is the effective gas price paid + // for the transaction which is equals to `BlockHeader.BaseFeePerGas + TransactionTrace.` + BigInt gas_price = 3; + + // GasLimit is the maximum of gas unit the sender of the transaction is willing to consume when perform the EVM + // execution of the whole transaction + uint64 gas_limit = 4; + + // Value is the amount of Ether transferred as part of this transaction. + BigInt value = 5; + + // Input data the transaction will receive for execution of EVM. + bytes input = 6; + + // V is the recovery ID value for the signature Y point. + bytes v = 7; + + // R is the signature's X point on the elliptic curve (32 bytes). + bytes r = 8; + + // S is the signature's Y point on the elliptic curve (32 bytes). + bytes s = 9; + + // GasUsed is the total amount of gas unit used for the whole execution of the transaction. + uint64 gas_used = 10; + + // Type represents the Ethereum transaction type, available only since EIP-2718 & EIP-2930 activation which happened on Berlin fork. + // The value is always set even for transaction before Berlin fork because those before the fork are still legacy transactions. + Type type = 12; + + enum Type { + // All transactions that ever existed prior Berlin fork before EIP-2718 was implemented. + TRX_TYPE_LEGACY = 0; + + // Field that specifies an access list of contract/storage_keys that is going to be used + // in this transaction. + // + // Added in Berlin fork (EIP-2930). + TRX_TYPE_ACCESS_LIST = 1; + + // Transaction that specifies an access list just like TRX_TYPE_ACCESS_LIST but in addition defines the + // max base gas gee and max priority gas fee to pay for this transaction. Transaction's of those type are + // executed against EIP-1559 rules which dictates a dynamic gas cost based on the congestion of the network. + TRX_TYPE_DYNAMIC_FEE = 2; + } + + // AcccessList represents the storage access this transaction has agreed to do in which case those storage + // access cost less gas unit per access. + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_ACCESS_LIST || TRX_TYPE_DYNAMIC_FEE` which + // is possible only if Berlin (TRX_TYPE_ACCESS_LIST) nor London (TRX_TYPE_DYNAMIC_FEE) fork are active on the chain. + repeated AccessTuple access_list = 14; + + // MaxFeePerGas is the maximum fee per gas the user is willing to pay for the transaction gas used. + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only + // if London fork is active on the chain. + BigInt max_fee_per_gas = 11; + + // MaxPriorityFeePerGas is priority fee per gas the user to pay in extra to the miner on top of the block's + // base fee. + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only + // if London fork is active on the chain. + BigInt max_priority_fee_per_gas = 13; + + // meta + uint32 index = 20; + bytes hash = 21; + bytes from = 22; + bytes return_data = 23; + bytes public_key = 24; + uint64 begin_ordinal = 25; + uint64 end_ordinal = 26; + + TransactionTraceStatus status = 30; + TransactionReceipt receipt = 31; + repeated Call calls = 32; +} + + +// AccessTuple represents a list of storage keys for a given contract's address and is used +// for AccessList construction. +message AccessTuple { + bytes address = 1; + repeated bytes storage_keys = 2; +} + +// TransactionTraceWithBlockRef +message TransactionTraceWithBlockRef { + TransactionTrace trace = 1; + BlockRef block_ref = 2; +} + +enum TransactionTraceStatus { + UNKNOWN = 0; + SUCCEEDED = 1; + FAILED = 2; + REVERTED = 3; +} + +message TransactionReceipt { + // State root is an intermediate state_root hash, computed in-between transactions to make + // **sure** you could build a proof and point to state in the middle of a block. Geth client + // uses `PostState + root + PostStateOrStatus`` while Parity used `status_code, root...`` this piles + // hardforks, see (read the EIPs first): + // - https://github.com/eoscanada/go-ethereum-private/blob/deep-mind/core/types/receipt.go#L147 + // - https://github.com/eoscanada/go-ethereum-private/blob/deep-mind/core/types/receipt.go#L50-L86 + // - https://github.com/ethereum/EIPs/blob/master/EIPS/eip-658.md + // + // Moreover, the notion of `Outcome`` in parity, which segregates the two concepts, which are + // stored in the same field `status_code`` can be computed based on such a hack of the `state_root` + // field, following `EIP-658`. + // + // Before Byzantinium hard fork, this field is always empty. + bytes state_root = 1; + uint64 cumulative_gas_used = 2; + bytes logs_bloom = 3; + repeated Log logs = 4; +} + +message Log { + bytes address = 1; + repeated bytes topics = 2; + bytes data = 3; + + // Index is the index of the log relative to the transaction. This index + // is always populated regardless of the state revertion of the the call + // that emitted this log. + uint32 index = 4; + + // BlockIndex represents the index of the log relative to the Block. + // + // An **important** notice is that this field will be 0 when the call + // that emitted the log has been reverted by the chain. + // + // Currently, there is two locations where a Log can be obtained: + // - block.transaction_traces[].receipt.logs[] + // - block.transaction_traces[].calls[].logs[] + // + // In the `receipt` case, the logs will be populated only when the call + // that emitted them has not been reverted by the chain and when in this + // position, the `blockIndex` is always populated correctly. + // + // In the case of `calls` case, for `call` where `stateReverted == true`, + // the `blockIndex` value will always be 0. + uint32 blockIndex = 6; + + uint64 ordinal = 7; +} + +message Call { + uint32 index = 1; + uint32 parent_index = 2; + uint32 depth = 3; + CallType call_type = 4; + bytes caller = 5; + bytes address = 6; + BigInt value = 7; + uint64 gas_limit = 8; + uint64 gas_consumed = 9; + bytes return_data = 13; + bytes input = 14; + bool executed_code = 15; + bool suicide = 16; + + /* hex representation of the hash -> preimage */ + map keccak_preimages = 20; + repeated StorageChange storage_changes = 21; + repeated BalanceChange balance_changes = 22; + repeated NonceChange nonce_changes = 24; + repeated Log logs = 25; + repeated CodeChange code_changes = 26; + + // Deprecated: repeated bytes created_accounts + reserved 27; + + repeated GasChange gas_changes = 28; + + // Deprecated: repeated GasEvent gas_events + reserved 29; + + // In Ethereum, a call can be either: + // - Successfull, execution passes without any problem encountered + // - Failed, execution failed, and remaining gas should be consumed + // - Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded + // + // When a call is either `failed` or `reverted`, the `status_failed` field + // below is set to `true`. If the status is `reverted`, then both `status_failed` + // and `status_reverted` are going to be set to `true`. + bool status_failed = 10; + bool status_reverted = 12; + + // Populated when a call either failed or reverted, so when `status_failed == true`, + // see above for details about those flags. + string failure_reason = 11; + + // This field represents wheter or not the state changes performed + // by this call were correctly recorded by the blockchain. + // + // On Ethereum, a transaction can record state changes even if some + // of its inner nested calls failed. This is problematic however since + // a call will invalidate all its state changes as well as all state + // changes performed by its child call. This means that even if a call + // has a status of `SUCCESS`, the chain might have reverted all the state + // changes it performed. + // + // ```text + // Trx 1 + // Call #1 + // Call #2 + // Call #3 + // |--- Failure here + // Call #4 + // ``` + // + // In the transaction above, while Call #2 and Call #3 would have the + // status `EXECUTED` + bool state_reverted = 30; + + uint64 begin_ordinal = 31; + uint64 end_ordinal = 32; + + repeated AccountCreation account_creations = 33; + + reserved 50; // repeated ERC20BalanceChange erc20_balance_changes = 50 [deprecated = true]; + reserved 51; // repeated ERC20TransferEvent erc20_transfer_events = 51 [deprecated = true]; + reserved 60; // bool filtering_matched = 60 [deprecated = true]; +} + +enum CallType { + UNSPECIFIED = 0; + CALL = 1; // direct? what's the name for `Call` alone? + CALLCODE = 2; + DELEGATE = 3; + STATIC = 4; + CREATE = 5; // create2 ? any other form of calls? +} + +message StorageChange { + bytes address = 1; + bytes key = 2; + bytes old_value = 3; + bytes new_value = 4; + + uint64 ordinal = 5; +} + +message BalanceChange { + bytes address = 1; + BigInt old_value = 2; + BigInt new_value = 3; + Reason reason = 4; + + // Obtain all balanche change reasons under deep mind repository: + // + // ```shell + // ack -ho 'BalanceChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq + // ``` + enum Reason { + REASON_UNKNOWN = 0; + REASON_REWARD_MINE_UNCLE = 1; + REASON_REWARD_MINE_BLOCK = 2; + REASON_DAO_REFUND_CONTRACT = 3; + REASON_DAO_ADJUST_BALANCE = 4; + REASON_TRANSFER = 5; + REASON_GENESIS_BALANCE = 6; + REASON_GAS_BUY = 7; + REASON_REWARD_TRANSACTION_FEE = 8; + REASON_REWARD_FEE_RESET = 14; + REASON_GAS_REFUND = 9; + REASON_TOUCH_ACCOUNT = 10; + REASON_SUICIDE_REFUND = 11; + REASON_SUICIDE_WITHDRAW = 13; + REASON_CALL_BALANCE_OVERRIDE = 12; + // Used on chain(s) where some Ether burning happens + REASON_BURN = 15; + } + + uint64 ordinal = 5; +} + +message NonceChange { + bytes address = 1; + uint64 old_value = 2; + uint64 new_value = 3; + uint64 ordinal = 4; +} + +message AccountCreation { + bytes account = 1; + uint64 ordinal = 2; +} + +message CodeChange { + bytes address = 1; + bytes old_hash = 2; + bytes old_code = 3; + bytes new_hash = 4; + bytes new_code = 5; + + uint64 ordinal = 6; +} + +// The gas change model represents the reason why some gas cost has occurred. +// The gas is computed per actual op codes. Doing them completely might prove +// overwhelming in most cases. +// +// Hence, we only index some of them, those that are costy like all the calls +// one, log events, return data, etc. +message GasChange { + uint64 old_value = 1; + uint64 new_value = 2; + Reason reason = 3; + + // Obtain all gas change reasons under deep mind repository: + // + // ```shell + // ack -ho 'GasChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq + // ``` + enum Reason { + REASON_UNKNOWN = 0; + REASON_CALL = 1; + REASON_CALL_CODE = 2; + REASON_CALL_DATA_COPY = 3; + REASON_CODE_COPY = 4; + REASON_CODE_STORAGE = 5; + REASON_CONTRACT_CREATION = 6; + REASON_CONTRACT_CREATION2 = 7; + REASON_DELEGATE_CALL = 8; + REASON_EVENT_LOG = 9; + REASON_EXT_CODE_COPY = 10; + REASON_FAILED_EXECUTION = 11; + REASON_INTRINSIC_GAS = 12; + REASON_PRECOMPILED_CONTRACT = 13; + REASON_REFUND_AFTER_EXECUTION = 14; + REASON_RETURN = 15; + REASON_RETURN_DATA_COPY = 16; + REASON_REVERT = 17; + REASON_SELF_DESTRUCT = 18; + REASON_STATIC_CALL = 19; + + // Added in Berlin fork (Geth 1.10+) + REASON_STATE_COLD_ACCESS = 20; + } + + uint64 ordinal = 4; +} \ No newline at end of file diff --git a/chain/ethereum/src/adapter.rs b/chain/ethereum/src/adapter.rs new file mode 100644 index 0000000..18d5854 --- /dev/null +++ b/chain/ethereum/src/adapter.rs @@ -0,0 +1,1639 @@ +use anyhow::Error; +use ethabi::{Error as ABIError, Function, ParamType, Token}; +use futures::Future; +use graph::blockchain::ChainIdentifier; +use graph::firehose::CallToFilter; +use graph::firehose::CombinedFilter; +use graph::firehose::LogFilter; +use itertools::Itertools; +use prost::Message; +use prost_types::Any; +use std::cmp; +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::marker::Unpin; +use thiserror::Error; +use tiny_keccak::keccak256; +use web3::types::{Address, Log, H256}; + +use graph::prelude::*; +use graph::{ + blockchain as bc, + components::metrics::{CounterVec, GaugeVec, HistogramVec}, + petgraph::{self, graphmap::GraphMap}, +}; + +const COMBINED_FILTER_TYPE_URL: &str = + "type.googleapis.com/sf.ethereum.transform.v1.CombinedFilter"; + +use crate::capabilities::NodeCapabilities; +use crate::data_source::{BlockHandlerFilter, DataSource}; +use crate::{Chain, Mapping, ENV_VARS}; + +pub type EventSignature = H256; +pub type FunctionSelector = [u8; 4]; + +#[derive(Clone, Debug)] +pub struct EthereumContractCall { + pub address: Address, + pub block_ptr: BlockPtr, + pub function: Function, + pub args: Vec, +} + +#[derive(Error, Debug)] +pub enum EthereumContractCallError { + #[error("ABI error: {0}")] + ABIError(#[from] ABIError), + /// `Token` is not of expected `ParamType` + #[error("type mismatch, token {0:?} is not of kind {1:?}")] + TypeError(Token, ParamType), + #[error("error encoding input call data: {0}")] + EncodingError(ethabi::Error), + #[error("call error: {0}")] + Web3Error(web3::Error), + #[error("call reverted: {0}")] + Revert(String), + #[error("ethereum node took too long to perform call")] + Timeout, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Ord, PartialOrd, Hash)] +enum LogFilterNode { + Contract(Address), + Event(EventSignature), +} + +/// Corresponds to an `eth_getLogs` call. +#[derive(Clone, Debug)] +pub struct EthGetLogsFilter { + pub contracts: Vec
, + pub event_signatures: Vec, +} + +impl EthGetLogsFilter { + fn from_contract(address: Address) -> Self { + EthGetLogsFilter { + contracts: vec![address], + event_signatures: vec![], + } + } + + fn from_event(event: EventSignature) -> Self { + EthGetLogsFilter { + contracts: vec![], + event_signatures: vec![event], + } + } +} + +impl fmt::Display for EthGetLogsFilter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.contracts.len() == 1 { + write!( + f, + "contract {:?}, {} events", + self.contracts[0], + self.event_signatures.len() + ) + } else if self.event_signatures.len() == 1 { + write!( + f, + "event {:?}, {} contracts", + self.event_signatures[0], + self.contracts.len() + ) + } else { + write!(f, "unreachable") + } + } +} + +#[derive(Clone, Debug, Default)] +pub struct TriggerFilter { + pub(crate) log: EthereumLogFilter, + pub(crate) call: EthereumCallFilter, + pub(crate) block: EthereumBlockFilter, +} + +impl TriggerFilter { + pub(crate) fn requires_traces(&self) -> bool { + !self.call.is_empty() || self.block.requires_traces() + } +} + +impl bc::TriggerFilter for TriggerFilter { + fn extend<'a>(&mut self, data_sources: impl Iterator + Clone) { + self.log + .extend(EthereumLogFilter::from_data_sources(data_sources.clone())); + self.call + .extend(EthereumCallFilter::from_data_sources(data_sources.clone())); + self.block + .extend(EthereumBlockFilter::from_data_sources(data_sources)); + } + + fn node_capabilities(&self) -> NodeCapabilities { + NodeCapabilities { + archive: false, + traces: self.requires_traces(), + } + } + + fn extend_with_template( + &mut self, + data_sources: impl Iterator::DataSourceTemplate>, + ) { + for data_source in data_sources.into_iter() { + self.log + .extend(EthereumLogFilter::from_mapping(&data_source.mapping)); + + self.call + .extend(EthereumCallFilter::from_mapping(&data_source.mapping)); + + self.block + .extend(EthereumBlockFilter::from_mapping(&data_source.mapping)); + } + } + + fn to_firehose_filter(self) -> Vec { + let EthereumBlockFilter { + contract_addresses: _contract_addresses, + trigger_every_block, + } = self.block.clone(); + + let log_filters: Vec = self.log.into(); + let mut call_filters: Vec = self.call.into(); + call_filters.extend(Into::>::into(self.block)); + + if call_filters.is_empty() && log_filters.is_empty() && !trigger_every_block { + return Vec::new(); + } + + let combined_filter = CombinedFilter { + log_filters, + call_filters, + send_all_block_headers: trigger_every_block, + }; + + vec![Any { + type_url: COMBINED_FILTER_TYPE_URL.into(), + value: combined_filter.encode_to_vec(), + }] + } +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct EthereumLogFilter { + /// Log filters can be represented as a bipartite graph between contracts and events. An edge + /// exists between a contract and an event if a data source for the contract has a trigger for + /// the event. + /// Edges are of `bool` type and indicates when a trigger requires a transaction receipt. + contracts_and_events_graph: GraphMap, + + /// Event sigs with no associated address, matching on all addresses. + /// Maps to a boolean representing if a trigger requires a transaction receipt. + wildcard_events: HashMap, +} + +impl Into> for EthereumLogFilter { + fn into(self) -> Vec { + self.eth_get_logs_filters() + .map( + |EthGetLogsFilter { + contracts, + event_signatures, + }| LogFilter { + addresses: contracts + .iter() + .map(|addr| addr.to_fixed_bytes().to_vec()) + .collect_vec(), + event_signatures: event_signatures + .iter() + .map(|sig| sig.to_fixed_bytes().to_vec()) + .collect_vec(), + }, + ) + .collect_vec() + } +} + +impl EthereumLogFilter { + /// Check if this filter matches the specified `Log`. + pub fn matches(&self, log: &Log) -> bool { + // First topic should be event sig + match log.topics.first() { + None => false, + + Some(sig) => { + // The `Log` matches the filter either if the filter contains + // a (contract address, event signature) pair that matches the + // `Log`, or if the filter contains wildcard event that matches. + let contract = LogFilterNode::Contract(log.address); + let event = LogFilterNode::Event(*sig); + self.contracts_and_events_graph + .all_edges() + .any(|(s, t, _)| (s == contract && t == event) || (t == contract && s == event)) + || self.wildcard_events.contains_key(sig) + } + } + } + + /// Similar to [`matches`], checks if a transaction receipt is required for this log filter. + pub fn requires_transaction_receipt( + &self, + event_signature: &H256, + contract_address: Option<&Address>, + ) -> bool { + if let Some(true) = self.wildcard_events.get(event_signature) { + true + } else if let Some(address) = contract_address { + let contract = LogFilterNode::Contract(*address); + let event = LogFilterNode::Event(*event_signature); + self.contracts_and_events_graph + .all_edges() + .any(|(s, t, r)| { + *r && (s == contract && t == event) || (t == contract && s == event) + }) + } else { + false + } + } + + pub fn from_data_sources<'a>(iter: impl IntoIterator) -> Self { + let mut this = EthereumLogFilter::default(); + for ds in iter { + for event_handler in ds.mapping.event_handlers.iter() { + let event_sig = event_handler.topic0(); + match ds.address { + Some(contract) => { + this.contracts_and_events_graph.add_edge( + LogFilterNode::Contract(contract), + LogFilterNode::Event(event_sig), + event_handler.receipt, + ); + } + None => { + this.wildcard_events + .insert(event_sig, event_handler.receipt); + } + } + } + } + this + } + + pub fn from_mapping(mapping: &Mapping) -> Self { + let mut this = EthereumLogFilter::default(); + for event_handler in &mapping.event_handlers { + let signature = event_handler.topic0(); + this.wildcard_events + .insert(signature, event_handler.receipt); + } + this + } + + /// Extends this log filter with another one. + pub fn extend(&mut self, other: EthereumLogFilter) { + if other.is_empty() { + return; + }; + + // Destructure to make sure we're checking all fields. + let EthereumLogFilter { + contracts_and_events_graph, + wildcard_events, + } = other; + for (s, t, e) in contracts_and_events_graph.all_edges() { + self.contracts_and_events_graph.add_edge(s, t, *e); + } + self.wildcard_events.extend(wildcard_events); + } + + /// An empty filter is one that never matches. + pub fn is_empty(&self) -> bool { + // Destructure to make sure we're checking all fields. + let EthereumLogFilter { + contracts_and_events_graph, + wildcard_events, + } = self; + contracts_and_events_graph.edge_count() == 0 && wildcard_events.is_empty() + } + + /// Filters for `eth_getLogs` calls. The filters will not return false positives. This attempts + /// to balance between having granular filters but too many calls and having few calls but too + /// broad filters causing the Ethereum endpoint to timeout. + pub fn eth_get_logs_filters(self) -> impl Iterator { + // Start with the wildcard event filters. + let mut filters = self + .wildcard_events + .into_iter() + .map(|(event, _)| EthGetLogsFilter::from_event(event)) + .collect_vec(); + + // The current algorithm is to repeatedly find the maximum cardinality vertex and turn all + // of its edges into a filter. This is nice because it is neutral between filtering by + // contract or by events, if there are many events that appear on only one data source + // we'll filter by many events on a single contract, but if there is an event that appears + // on a lot of data sources we'll filter by many contracts with a single event. + // + // From a theoretical standpoint we're finding a vertex cover, and this is not the optimal + // algorithm to find a minimum vertex cover, but should be fine as an approximation. + // + // One optimization we're not doing is to merge nodes that have the same neighbors into a + // single node. For example if a subgraph has two data sources, each with the same two + // events, we could cover that with a single filter and no false positives. However that + // might cause the filter to become too broad, so at the moment it seems excessive. + let mut g = self.contracts_and_events_graph; + while g.edge_count() > 0 { + let mut push_filter = |filter: EthGetLogsFilter| { + // Sanity checks: + // - The filter is not a wildcard because all nodes have neighbors. + // - The graph is bipartite. + assert!(filter.contracts.len() > 0 && filter.event_signatures.len() > 0); + assert!(filter.contracts.len() == 1 || filter.event_signatures.len() == 1); + filters.push(filter); + }; + + // If there are edges, there are vertexes. + let max_vertex = g.nodes().max_by_key(|&n| g.neighbors(n).count()).unwrap(); + let mut filter = match max_vertex { + LogFilterNode::Contract(address) => EthGetLogsFilter::from_contract(address), + LogFilterNode::Event(event_sig) => EthGetLogsFilter::from_event(event_sig), + }; + for neighbor in g.neighbors(max_vertex) { + match neighbor { + LogFilterNode::Contract(address) => { + if filter.contracts.len() == ENV_VARS.get_logs_max_contracts { + // The batch size was reached, register the filter and start a new one. + let event = filter.event_signatures[0]; + push_filter(filter); + filter = EthGetLogsFilter::from_event(event); + } + filter.contracts.push(address); + } + LogFilterNode::Event(event_sig) => filter.event_signatures.push(event_sig), + } + } + + push_filter(filter); + g.remove_node(max_vertex); + } + filters.into_iter() + } +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct EthereumCallFilter { + // Each call filter has a map of filters keyed by address, each containing a tuple with + // start_block and the set of function signatures + pub contract_addresses_function_signatures: + HashMap)>, + + pub wildcard_signatures: HashSet, +} + +impl Into> for EthereumCallFilter { + fn into(self) -> Vec { + if self.is_empty() { + return Vec::new(); + } + + let EthereumCallFilter { + contract_addresses_function_signatures, + wildcard_signatures, + } = self; + + let mut filters: Vec = contract_addresses_function_signatures + .into_iter() + .map(|(addr, (_, sigs))| CallToFilter { + addresses: vec![addr.to_fixed_bytes().to_vec()], + signatures: sigs.into_iter().map(|x| x.to_vec()).collect_vec(), + }) + .collect(); + + if !wildcard_signatures.is_empty() { + filters.push(CallToFilter { + addresses: vec![], + signatures: wildcard_signatures + .into_iter() + .map(|x| x.to_vec()) + .collect_vec(), + }); + } + + filters + } +} + +impl EthereumCallFilter { + pub fn matches(&self, call: &EthereumCall) -> bool { + // Calls returned by Firehose actually contains pure transfers and smart + // contract calls. If the input is less than 4 bytes, we assume it's a pure transfer + // and discards those. + if call.input.0.len() < 4 { + return false; + } + + // The `call.input.len()` is validated in the + // DataSource::match_and_decode function. + // Those calls are logged as warning and skipped. + // + // See 280b0108-a96e-4738-bb37-60ce11eeb5bf + let call_signature = &call.input.0[..4]; + + // Ensure the call is to a contract the filter expressed an interest in + match self.contract_addresses_function_signatures.get(&call.to) { + // If the call is to a contract with no specified functions, keep the call + // + // Allows the ability to genericly match on all calls to a contract. + // Caveat is this catch all clause limits you from matching with a specific call + // on the same address + Some(v) if v.1.is_empty() => true, + // There are some relevant signatures to test + // this avoids having to call extend for every match call, checks the contract specific funtions, then falls + // back on wildcards + Some(v) => { + let sig = &v.1; + sig.contains(call_signature) || self.wildcard_signatures.contains(call_signature) + } + // no contract specific functions, check wildcards + None => self.wildcard_signatures.contains(call_signature), + } + } + + pub fn from_mapping(mapping: &Mapping) -> Self { + let functions = mapping + .call_handlers + .iter() + .map(move |call_handler| { + let sig = keccak256(call_handler.function.as_bytes()); + [sig[0], sig[1], sig[2], sig[3]] + }) + .collect(); + + Self { + wildcard_signatures: functions, + contract_addresses_function_signatures: HashMap::new(), + } + } + + pub fn from_data_sources<'a>(iter: impl IntoIterator) -> Self { + iter.into_iter() + .filter_map(|data_source| data_source.address.map(|addr| (addr, data_source))) + .map(|(contract_addr, data_source)| { + let start_block = data_source.start_block; + data_source + .mapping + .call_handlers + .iter() + .map(move |call_handler| { + let sig = keccak256(call_handler.function.as_bytes()); + (start_block, contract_addr, [sig[0], sig[1], sig[2], sig[3]]) + }) + }) + .flatten() + .collect() + } + + /// Extends this call filter with another one. + pub fn extend(&mut self, other: EthereumCallFilter) { + if other.is_empty() { + return; + }; + + let EthereumCallFilter { + contract_addresses_function_signatures, + wildcard_signatures, + } = other; + + // Extend existing address / function signature key pairs + // Add new address / function signature key pairs from the provided EthereumCallFilter + for (address, (proposed_start_block, new_sigs)) in + contract_addresses_function_signatures.into_iter() + { + match self + .contract_addresses_function_signatures + .get_mut(&address) + { + Some((existing_start_block, existing_sigs)) => { + *existing_start_block = cmp::min(proposed_start_block, *existing_start_block); + existing_sigs.extend(new_sigs); + } + None => { + self.contract_addresses_function_signatures + .insert(address, (proposed_start_block, new_sigs)); + } + } + } + + self.wildcard_signatures.extend(wildcard_signatures); + } + + /// An empty filter is one that never matches. + pub fn is_empty(&self) -> bool { + // Destructure to make sure we're checking all fields. + let EthereumCallFilter { + contract_addresses_function_signatures, + wildcard_signatures: wildcard_matches, + } = self; + contract_addresses_function_signatures.is_empty() && wildcard_matches.is_empty() + } +} + +impl FromIterator<(BlockNumber, Address, FunctionSelector)> for EthereumCallFilter { + fn from_iter(iter: I) -> Self + where + I: IntoIterator, + { + let mut lookup: HashMap)> = HashMap::new(); + iter.into_iter() + .for_each(|(start_block, address, function_signature)| { + if !lookup.contains_key(&address) { + lookup.insert(address, (start_block, HashSet::default())); + } + lookup.get_mut(&address).map(|set| { + if set.0 > start_block { + set.0 = start_block + } + set.1.insert(function_signature); + set + }); + }); + EthereumCallFilter { + contract_addresses_function_signatures: lookup, + wildcard_signatures: HashSet::new(), + } + } +} + +impl From<&EthereumBlockFilter> for EthereumCallFilter { + fn from(ethereum_block_filter: &EthereumBlockFilter) -> Self { + Self { + contract_addresses_function_signatures: ethereum_block_filter + .contract_addresses + .iter() + .map(|(start_block_opt, address)| { + (address.clone(), (*start_block_opt, HashSet::default())) + }) + .collect::)>>(), + wildcard_signatures: HashSet::new(), + } + } +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct EthereumBlockFilter { + pub contract_addresses: HashSet<(BlockNumber, Address)>, + pub trigger_every_block: bool, +} + +impl Into> for EthereumBlockFilter { + fn into(self) -> Vec { + self.contract_addresses + .into_iter() + .map(|(_, addr)| addr) + .sorted() + .dedup_by(|x, y| x == y) + .map(|addr| CallToFilter { + addresses: vec![addr.to_fixed_bytes().to_vec()], + signatures: vec![], + }) + .collect_vec() + } +} + +impl EthereumBlockFilter { + /// from_mapping ignores contract addresses in this use case because templates can't provide Address or BlockNumber + /// ahead of time. This means the filters applied to the block_stream need to be broad, in this case, + /// specifically, will match all blocks. The blocks are then further filtered by the subgraph instance manager + /// which keeps track of deployed contracts and relevant addresses. + pub fn from_mapping(mapping: &Mapping) -> Self { + Self { + contract_addresses: HashSet::new(), + trigger_every_block: mapping.block_handlers.len() != 0, + } + } + + pub fn from_data_sources<'a>(iter: impl IntoIterator) -> Self { + iter.into_iter() + .filter(|data_source| data_source.address.is_some()) + .fold(Self::default(), |mut filter_opt, data_source| { + let has_block_handler_with_call_filter = data_source + .mapping + .block_handlers + .clone() + .into_iter() + .any(|block_handler| match block_handler.filter { + Some(ref filter) if *filter == BlockHandlerFilter::Call => true, + _ => false, + }); + + let has_block_handler_without_filter = data_source + .mapping + .block_handlers + .clone() + .into_iter() + .any(|block_handler| block_handler.filter.is_none()); + + filter_opt.extend(Self { + trigger_every_block: has_block_handler_without_filter, + contract_addresses: if has_block_handler_with_call_filter { + vec![( + data_source.start_block, + data_source.address.unwrap().to_owned(), + )] + .into_iter() + .collect() + } else { + HashSet::default() + }, + }); + filter_opt + }) + } + + pub fn extend(&mut self, other: EthereumBlockFilter) { + if other.is_empty() { + return; + }; + + let EthereumBlockFilter { + contract_addresses, + trigger_every_block, + } = other; + + self.trigger_every_block = self.trigger_every_block || trigger_every_block; + + for other in contract_addresses { + let (other_start_block, other_address) = other; + + match self.find_contract_address(&other.1) { + Some((current_start_block, current_address)) => { + if other_start_block < current_start_block { + self.contract_addresses + .remove(&(current_start_block, current_address)); + self.contract_addresses + .insert((other_start_block, other_address)); + } + } + None => { + self.contract_addresses + .insert((other_start_block, other_address)); + } + } + } + } + + fn requires_traces(&self) -> bool { + !self.contract_addresses.is_empty() + } + + /// An empty filter is one that never matches. + pub fn is_empty(&self) -> bool { + // If we are triggering every block, we are of course not empty + if self.trigger_every_block { + return false; + } + + self.contract_addresses.is_empty() + } + + fn find_contract_address(&self, candidate: &Address) -> Option<(i32, Address)> { + self.contract_addresses + .iter() + .find(|(_, current_address)| candidate == current_address) + .cloned() + } +} + +pub enum ProviderStatus { + Working, + VersionFail, + GenesisFail, + VersionTimeout, + GenesisTimeout, +} + +impl From for f64 { + fn from(state: ProviderStatus) -> Self { + match state { + ProviderStatus::Working => 0.0, + ProviderStatus::VersionFail => 1.0, + ProviderStatus::GenesisFail => 2.0, + ProviderStatus::VersionTimeout => 3.0, + ProviderStatus::GenesisTimeout => 4.0, + } + } +} + +const STATUS_HELP: &str = "0 = ok, 1 = net_version failed, 2 = get genesis failed, 3 = net_version timeout, 4 = get genesis timeout"; +#[derive(Clone)] +pub struct ProviderEthRpcMetrics { + request_duration: Box, + errors: Box, + status: Box, +} + +impl ProviderEthRpcMetrics { + pub fn new(registry: Arc) -> Self { + let request_duration = registry + .new_histogram_vec( + "eth_rpc_request_duration", + "Measures eth rpc request duration", + vec![String::from("method"), String::from("provider")], + vec![0.05, 0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6], + ) + .unwrap(); + let errors = registry + .new_counter_vec( + "eth_rpc_errors", + "Counts eth rpc request errors", + vec![String::from("method"), String::from("provider")], + ) + .unwrap(); + let status_help = format!("Whether the provider has failed ({STATUS_HELP})"); + let status = registry + .new_gauge_vec( + "eth_rpc_status", + &status_help, + vec![String::from("provider")], + ) + .unwrap(); + Self { + request_duration, + errors, + status, + } + } + + pub fn observe_request(&self, duration: f64, method: &str, provider: &str) { + self.request_duration + .with_label_values(&[method, provider]) + .observe(duration); + } + + pub fn add_error(&self, method: &str, provider: &str) { + self.errors.with_label_values(&[method, provider]).inc(); + } + + pub fn set_status(&self, status: ProviderStatus, provider: &str) { + self.status + .with_label_values(&[provider]) + .set(status.into()); + } +} + +#[derive(Clone)] +pub struct SubgraphEthRpcMetrics { + request_duration: GaugeVec, + errors: CounterVec, + deployment: String, +} + +impl SubgraphEthRpcMetrics { + pub fn new(registry: Arc, subgraph_hash: &str) -> Self { + let request_duration = registry + .global_gauge_vec( + "deployment_eth_rpc_request_duration", + "Measures eth rpc request duration for a subgraph deployment", + vec!["deployment", "method", "provider"].as_slice(), + ) + .unwrap(); + let errors = registry + .global_counter_vec( + "deployment_eth_rpc_errors", + "Counts eth rpc request errors for a subgraph deployment", + vec!["deployment", "method", "provider"].as_slice(), + ) + .unwrap(); + Self { + request_duration, + errors, + deployment: subgraph_hash.into(), + } + } + + pub fn observe_request(&self, duration: f64, method: &str, provider: &str) { + self.request_duration + .with_label_values(&[&self.deployment, method, provider]) + .set(duration); + } + + pub fn add_error(&self, method: &str, provider: &str) { + self.errors + .with_label_values(&[&self.deployment, method, provider]) + .inc(); + } +} + +/// Common trait for components that watch and manage access to Ethereum. +/// +/// Implementations may be implemented against an in-process Ethereum node +/// or a remote node over RPC. +#[async_trait] +pub trait EthereumAdapter: Send + Sync + 'static { + fn url_hostname(&self) -> &str; + + /// The `provider.label` from the adapter's configuration + fn provider(&self) -> &str; + + /// Ask the Ethereum node for some identifying information about the Ethereum network it is + /// connected to. + async fn net_identifiers(&self) -> Result; + + /// Get the latest block, including full transactions. + fn latest_block( + &self, + logger: &Logger, + ) -> Box + Send + Unpin>; + + /// Get the latest block, with only the header and transaction hashes. + fn latest_block_header( + &self, + logger: &Logger, + ) -> Box, Error = bc::IngestorError> + Send>; + + fn load_block( + &self, + logger: &Logger, + block_hash: H256, + ) -> Box + Send>; + + /// Load Ethereum blocks in bulk, returning results as they come back as a Stream. + /// May use the `chain_store` as a cache. + fn load_blocks( + &self, + logger: Logger, + chain_store: Arc, + block_hashes: HashSet, + ) -> Box, Error = Error> + Send>; + + /// Find a block by its hash. + fn block_by_hash( + &self, + logger: &Logger, + block_hash: H256, + ) -> Box, Error = Error> + Send>; + + fn block_by_number( + &self, + logger: &Logger, + block_number: BlockNumber, + ) -> Box, Error = Error> + Send>; + + /// Load full information for the specified `block` (in particular, transaction receipts). + fn load_full_block( + &self, + logger: &Logger, + block: LightEthereumBlock, + ) -> Pin> + Send>>; + + /// Load block pointer for the specified `block number`. + fn block_pointer_from_number( + &self, + logger: &Logger, + block_number: BlockNumber, + ) -> Box + Send>; + + /// Find a block by its number, according to the Ethereum node. + /// + /// Careful: don't use this function without considering race conditions. + /// Chain reorgs could happen at any time, and could affect the answer received. + /// Generally, it is only safe to use this function with blocks that have received enough + /// confirmations to guarantee no further reorgs, **and** where the Ethereum node is aware of + /// those confirmations. + /// If the Ethereum node is far behind in processing blocks, even old blocks can be subject to + /// reorgs. + fn block_hash_by_block_number( + &self, + logger: &Logger, + block_number: BlockNumber, + ) -> Box, Error = Error> + Send>; + + /// Call the function of a smart contract. + fn contract_call( + &self, + logger: &Logger, + call: EthereumContractCall, + cache: Arc, + ) -> Box, Error = EthereumContractCallError> + Send>; +} + +#[cfg(test)] +mod tests { + use crate::adapter::{FunctionSelector, COMBINED_FILTER_TYPE_URL}; + + use super::{EthereumBlockFilter, LogFilterNode}; + use super::{EthereumCallFilter, EthereumLogFilter, TriggerFilter}; + + use graph::blockchain::TriggerFilter as _; + use graph::firehose::{CallToFilter, CombinedFilter, LogFilter, MultiLogFilter}; + use graph::petgraph::graphmap::GraphMap; + use graph::prelude::ethabi::ethereum_types::H256; + use graph::prelude::web3::types::Address; + use graph::prelude::web3::types::Bytes; + use graph::prelude::EthereumCall; + use hex::ToHex; + use itertools::Itertools; + use prost::Message; + use prost_types::Any; + + use std::collections::{HashMap, HashSet}; + use std::iter::FromIterator; + use std::str::FromStr; + + #[test] + fn ethereum_log_filter_codec() { + let hex_addr = "0x4c7b8591c50f4ad308d07d6294f2945e074420f5"; + let address = Address::from_str(hex_addr).expect("unable to parse addr"); + assert_eq!(hex_addr, format!("0x{}", address.encode_hex::())); + + let event_sigs = vec![ + "0xafb42f194014ece77df0f9e4bc3ced9757555dc1fe7dc803161a2de3b7c4839a", + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + ]; + + let hex_sigs = event_sigs + .iter() + .map(|addr| { + format!( + "0x{}", + H256::from_str(addr) + .expect("unable to parse addr") + .encode_hex::() + ) + }) + .collect_vec(); + + assert_eq!(event_sigs, hex_sigs); + + let sigs = event_sigs + .iter() + .map(|addr| { + H256::from_str(addr) + .expect("unable to parse addr") + .to_fixed_bytes() + .to_vec() + }) + .collect_vec(); + + let filter = LogFilter { + addresses: vec![address.to_fixed_bytes().to_vec()], + event_signatures: sigs, + }; + // This base64 was provided by Streamingfast as a binding example of the expected encoded for the + // addresses and signatures above. + let expected_base64 = "CloKFEx7hZHFD0rTCNB9YpTylF4HRCD1EiCvtC8ZQBTs533w+eS8PO2XV1Vdwf59yAMWGi3jt8SDmhIg3fJSrRviyJtpwrBo/DeNqpUrp/FjxKEWKPVaTfUjs+8="; + + let filter = MultiLogFilter { + log_filters: vec![filter], + }; + + let output = base64::encode(filter.encode_to_vec()); + assert_eq!(expected_base64, output); + } + + #[test] + fn ethereum_call_filter_codec() { + let hex_addr = "0xeed2b7756e295a9300e53dd049aeb0751899bae3"; + let sig = "a9059cbb"; + let mut fs: FunctionSelector = [0u8; 4]; + let hex_sig = hex::decode(sig).expect("failed to parse sig"); + fs.copy_from_slice(&hex_sig[..]); + + let actual_sig = hex::encode(fs); + assert_eq!(sig, actual_sig); + + let filter = LogFilter { + addresses: vec![Address::from_str(hex_addr) + .expect("failed to parse address") + .to_fixed_bytes() + .to_vec()], + event_signatures: vec![fs.to_vec()], + }; + + // This base64 was provided by Streamingfast as a binding example of the expected encoded for the + // addresses and signatures above. + let expected_base64 = "ChTu0rd1bilakwDlPdBJrrB1GJm64xIEqQWcuw=="; + + let output = base64::encode(filter.encode_to_vec()); + assert_eq!(expected_base64, output); + } + + #[test] + fn ethereum_trigger_filter_to_firehose() { + let address = Address::from_low_u64_be; + let sig = H256::from_low_u64_le; + let mut filter = TriggerFilter { + log: EthereumLogFilter { + contracts_and_events_graph: GraphMap::new(), + wildcard_events: HashMap::new(), + }, + call: EthereumCallFilter { + contract_addresses_function_signatures: HashMap::from_iter(vec![ + (address(0), (0, HashSet::from_iter(vec![[0u8; 4]]))), + (address(1), (1, HashSet::from_iter(vec![[1u8; 4]]))), + (address(2), (2, HashSet::new())), + ]), + wildcard_signatures: HashSet::new(), + }, + block: EthereumBlockFilter { + contract_addresses: HashSet::from_iter([ + (100, address(1000)), + (200, address(2000)), + (300, address(3000)), + (400, address(1000)), + (500, address(1000)), + ]), + trigger_every_block: false, + }, + }; + + let expected_call_filters = vec![ + CallToFilter { + addresses: vec![address(0).to_fixed_bytes().to_vec()], + signatures: vec![[0u8; 4].to_vec()], + }, + CallToFilter { + addresses: vec![address(1).to_fixed_bytes().to_vec()], + signatures: vec![[1u8; 4].to_vec()], + }, + CallToFilter { + addresses: vec![address(2).to_fixed_bytes().to_vec()], + signatures: vec![], + }, + CallToFilter { + addresses: vec![address(1000).to_fixed_bytes().to_vec()], + signatures: vec![], + }, + CallToFilter { + addresses: vec![address(2000).to_fixed_bytes().to_vec()], + signatures: vec![], + }, + CallToFilter { + addresses: vec![address(3000).to_fixed_bytes().to_vec()], + signatures: vec![], + }, + ]; + + filter.log.contracts_and_events_graph.add_edge( + LogFilterNode::Contract(address(10)), + LogFilterNode::Event(sig(100)), + false, + ); + filter.log.contracts_and_events_graph.add_edge( + LogFilterNode::Contract(address(10)), + LogFilterNode::Event(sig(101)), + false, + ); + filter.log.contracts_and_events_graph.add_edge( + LogFilterNode::Contract(address(20)), + LogFilterNode::Event(sig(100)), + false, + ); + + let expected_log_filters = vec![ + LogFilter { + addresses: vec![address(10).to_fixed_bytes().to_vec()], + event_signatures: vec![sig(101).to_fixed_bytes().to_vec()], + }, + LogFilter { + addresses: vec![ + address(10).to_fixed_bytes().to_vec(), + address(20).to_fixed_bytes().to_vec(), + ], + event_signatures: vec![sig(100).to_fixed_bytes().to_vec()], + }, + ]; + + let firehose_filter = filter.clone().to_firehose_filter(); + assert_eq!(1, firehose_filter.len()); + + let firehose_filter: HashMap<_, _> = HashMap::from_iter::>( + firehose_filter + .into_iter() + .map(|any| (any.type_url.clone(), any)) + .collect_vec(), + ); + + let mut combined_filter = &firehose_filter + .get(COMBINED_FILTER_TYPE_URL.into()) + .expect("a CombinedFilter") + .value[..]; + + let combined_filter = + CombinedFilter::decode(&mut combined_filter).expect("combined filter to decode"); + + let CombinedFilter { + log_filters: mut actual_log_filters, + call_filters: mut actual_call_filters, + send_all_block_headers: actual_send_all_block_headers, + } = combined_filter; + + actual_call_filters.sort_by(|a, b| a.addresses.cmp(&b.addresses)); + for filter in actual_call_filters.iter_mut() { + filter.signatures.sort(); + } + assert_eq!(expected_call_filters, actual_call_filters); + + actual_log_filters.sort_by(|a, b| a.addresses.cmp(&b.addresses)); + for filter in actual_log_filters.iter_mut() { + filter.event_signatures.sort(); + } + assert_eq!(expected_log_filters, actual_log_filters); + assert_eq!(false, actual_send_all_block_headers); + } + + #[test] + fn ethereum_trigger_filter_to_firehose_every_block_plus_logfilter() { + let address = Address::from_low_u64_be; + let sig = H256::from_low_u64_le; + let mut filter = TriggerFilter { + log: EthereumLogFilter { + contracts_and_events_graph: GraphMap::new(), + wildcard_events: HashMap::new(), + }, + call: EthereumCallFilter { + contract_addresses_function_signatures: HashMap::new(), + wildcard_signatures: HashSet::new(), + }, + block: EthereumBlockFilter { + contract_addresses: HashSet::new(), + trigger_every_block: true, + }, + }; + + filter.log.contracts_and_events_graph.add_edge( + LogFilterNode::Contract(address(10)), + LogFilterNode::Event(sig(101)), + false, + ); + + let expected_log_filters = vec![LogFilter { + addresses: vec![address(10).to_fixed_bytes().to_vec()], + event_signatures: vec![sig(101).to_fixed_bytes().to_vec()], + }]; + + let firehose_filter = filter.clone().to_firehose_filter(); + assert_eq!(1, firehose_filter.len()); + + let firehose_filter: HashMap<_, _> = HashMap::from_iter::>( + firehose_filter + .into_iter() + .map(|any| (any.type_url.clone(), any)) + .collect_vec(), + ); + + let mut combined_filter = &firehose_filter + .get(COMBINED_FILTER_TYPE_URL.into()) + .expect("a CombinedFilter") + .value[..]; + + let combined_filter = + CombinedFilter::decode(&mut combined_filter).expect("combined filter to decode"); + + let CombinedFilter { + log_filters: mut actual_log_filters, + call_filters: actual_call_filters, + send_all_block_headers: actual_send_all_block_headers, + } = combined_filter; + + assert_eq!(0, actual_call_filters.len()); + + actual_log_filters.sort_by(|a, b| a.addresses.cmp(&b.addresses)); + for filter in actual_log_filters.iter_mut() { + filter.event_signatures.sort(); + } + assert_eq!(expected_log_filters, actual_log_filters); + + assert_eq!(true, actual_send_all_block_headers); + } + + #[test] + fn matching_ethereum_call_filter() { + let call = |to: Address, input: Vec| EthereumCall { + to, + input: bytes(input), + ..Default::default() + }; + + let mut filter = EthereumCallFilter { + contract_addresses_function_signatures: HashMap::from_iter(vec![ + (address(0), (0, HashSet::from_iter(vec![[0u8; 4]]))), + (address(1), (1, HashSet::from_iter(vec![[1u8; 4]]))), + (address(2), (2, HashSet::new())), + ]), + wildcard_signatures: HashSet::new(), + }; + let filter2 = EthereumCallFilter { + contract_addresses_function_signatures: HashMap::from_iter(vec![( + address(0), + (0, HashSet::from_iter(vec![[10u8; 4]])), + )]), + wildcard_signatures: HashSet::from_iter(vec![[11u8; 4]]), + }; + + assert_eq!( + false, + filter.matches(&call(address(2), vec![])), + "call with empty bytes are always ignore, whatever the condition" + ); + + assert_eq!( + false, + filter.matches(&call(address(4), vec![1; 36])), + "call with incorrect address should be ignored" + ); + + assert_eq!( + true, + filter.matches(&call(address(1), vec![1; 36])), + "call with correct address & signature should match" + ); + + assert_eq!( + true, + filter.matches(&call(address(1), vec![1; 32])), + "call with correct address & signature, but with incorrect input size should match" + ); + + assert_eq!( + false, + filter.matches(&call(address(1), vec![4u8; 36])), + "call with correct address but incorrect signature for a specific contract filter (i.e. matches some signatures) should be ignored" + ); + + assert_eq!( + false, + filter.matches(&call(address(0), vec![11u8; 36])), + "this signature should not match filter1, this avoid false passes if someone changes the code" + ); + assert_eq!( + false, + filter2.matches(&call(address(1), vec![10u8; 36])), + "this signature should not match filter2 because the address is not the expected one" + ); + assert_eq!( + true, + filter2.matches(&call(address(0), vec![10u8; 36])), + "this signature should match filter2 on the non wildcard clause" + ); + assert_eq!( + true, + filter2.matches(&call(address(0), vec![11u8; 36])), + "this signature should match filter2 on the wildcard clause" + ); + + // extend filter1 and test the filter 2 stuff again + filter.extend(filter2); + assert_eq!( + true, + filter.matches(&call(address(0), vec![11u8; 36])), + "this signature should not match filter1, this avoid false passes if someone changes the code" + ); + assert_eq!( + false, + filter.matches(&call(address(1), vec![10u8; 36])), + "this signature should not match filter2 because the address is not the expected one" + ); + assert_eq!( + true, + filter.matches(&call(address(0), vec![10u8; 36])), + "this signature should match filter2 on the non wildcard clause" + ); + assert_eq!( + true, + filter.matches(&call(address(0), vec![11u8; 36])), + "this signature should match filter2 on the wildcard clause" + ); + } + + #[test] + fn extending_ethereum_block_filter_no_found() { + let mut base = EthereumBlockFilter { + contract_addresses: HashSet::new(), + trigger_every_block: false, + }; + + let extension = EthereumBlockFilter { + contract_addresses: HashSet::from_iter(vec![(10, address(1))]), + trigger_every_block: false, + }; + + base.extend(extension); + + assert_eq!( + HashSet::from_iter(vec![(10, address(1))]), + base.contract_addresses, + ); + } + + #[test] + fn extending_ethereum_block_filter_conflict_picks_lowest_block_from_ext() { + let mut base = EthereumBlockFilter { + contract_addresses: HashSet::from_iter(vec![(10, address(1))]), + trigger_every_block: false, + }; + + let extension = EthereumBlockFilter { + contract_addresses: HashSet::from_iter(vec![(2, address(1))]), + trigger_every_block: false, + }; + + base.extend(extension); + + assert_eq!( + HashSet::from_iter(vec![(2, address(1))]), + base.contract_addresses, + ); + } + + #[test] + fn extending_ethereum_block_filter_conflict_picks_lowest_block_from_base() { + let mut base = EthereumBlockFilter { + contract_addresses: HashSet::from_iter(vec![(2, address(1))]), + trigger_every_block: false, + }; + + let extension = EthereumBlockFilter { + contract_addresses: HashSet::from_iter(vec![(10, address(1))]), + trigger_every_block: false, + }; + + base.extend(extension); + + assert_eq!( + HashSet::from_iter(vec![(2, address(1))]), + base.contract_addresses, + ); + } + + #[test] + fn extending_ethereum_block_filter_every_block_in_ext() { + let mut base = EthereumBlockFilter { + contract_addresses: HashSet::default(), + trigger_every_block: false, + }; + + let extension = EthereumBlockFilter { + contract_addresses: HashSet::default(), + trigger_every_block: true, + }; + + base.extend(extension); + + assert_eq!(true, base.trigger_every_block); + } + + #[test] + fn extending_ethereum_block_filter_every_block_in_base_and_merge_contract_addresses() { + let mut base = EthereumBlockFilter { + contract_addresses: HashSet::from_iter(vec![(10, address(2))]), + trigger_every_block: true, + }; + + let extension = EthereumBlockFilter { + contract_addresses: HashSet::from_iter(vec![]), + trigger_every_block: false, + }; + + base.extend(extension); + + assert_eq!(true, base.trigger_every_block); + assert_eq!( + HashSet::from_iter(vec![(10, address(2))]), + base.contract_addresses, + ); + } + + #[test] + fn extending_ethereum_block_filter_every_block_in_ext_and_merge_contract_addresses() { + let mut base = EthereumBlockFilter { + contract_addresses: HashSet::from_iter(vec![(10, address(2))]), + trigger_every_block: false, + }; + + let extension = EthereumBlockFilter { + contract_addresses: HashSet::from_iter(vec![(10, address(1))]), + trigger_every_block: true, + }; + + base.extend(extension); + + assert_eq!(true, base.trigger_every_block); + assert_eq!( + HashSet::from_iter(vec![(10, address(2)), (10, address(1))]), + base.contract_addresses, + ); + } + + #[test] + fn extending_ethereum_call_filter() { + let mut base = EthereumCallFilter { + contract_addresses_function_signatures: HashMap::from_iter(vec![ + ( + Address::from_low_u64_be(0), + (0, HashSet::from_iter(vec![[0u8; 4]])), + ), + ( + Address::from_low_u64_be(1), + (1, HashSet::from_iter(vec![[1u8; 4]])), + ), + ]), + wildcard_signatures: HashSet::new(), + }; + let extension = EthereumCallFilter { + contract_addresses_function_signatures: HashMap::from_iter(vec![ + ( + Address::from_low_u64_be(0), + (2, HashSet::from_iter(vec![[2u8; 4]])), + ), + ( + Address::from_low_u64_be(3), + (3, HashSet::from_iter(vec![[3u8; 4]])), + ), + ]), + wildcard_signatures: HashSet::new(), + }; + base.extend(extension); + + assert_eq!( + base.contract_addresses_function_signatures + .get(&Address::from_low_u64_be(0)), + Some(&(0, HashSet::from_iter(vec![[0u8; 4], [2u8; 4]]))) + ); + assert_eq!( + base.contract_addresses_function_signatures + .get(&Address::from_low_u64_be(3)), + Some(&(3, HashSet::from_iter(vec![[3u8; 4]]))) + ); + assert_eq!( + base.contract_addresses_function_signatures + .get(&Address::from_low_u64_be(1)), + Some(&(1, HashSet::from_iter(vec![[1u8; 4]]))) + ); + } + + fn address(id: u64) -> Address { + Address::from_low_u64_be(id) + } + + fn bytes(value: Vec) -> Bytes { + Bytes::from(value) + } +} + +// Tests `eth_get_logs_filters` in instances where all events are filtered on by all contracts. +// This represents, for example, the relationship between dynamic data sources and their events. +#[test] +fn complete_log_filter() { + use std::collections::BTreeSet; + + // Test a few combinations of complete graphs. + for i in [1, 2] { + let events: BTreeSet<_> = (0..i).map(H256::from_low_u64_le).collect(); + + for j in [1, 1000, 2000, 3000] { + let contracts: BTreeSet<_> = (0..j).map(Address::from_low_u64_le).collect(); + + // Construct the complete bipartite graph with i events and j contracts. + let mut contracts_and_events_graph = GraphMap::new(); + for &contract in &contracts { + for &event in &events { + contracts_and_events_graph.add_edge( + LogFilterNode::Contract(contract), + LogFilterNode::Event(event), + false, + ); + } + } + + // Run `eth_get_logs_filters`, which is what we want to test. + let logs_filters: Vec<_> = EthereumLogFilter { + contracts_and_events_graph, + wildcard_events: HashMap::new(), + } + .eth_get_logs_filters() + .collect(); + + // Assert that a contract or event is filtered on iff it was present in the graph. + assert_eq!( + logs_filters + .iter() + .map(|l| l.contracts.iter()) + .flatten() + .copied() + .collect::>(), + contracts + ); + assert_eq!( + logs_filters + .iter() + .map(|l| l.event_signatures.iter()) + .flatten() + .copied() + .collect::>(), + events + ); + + // Assert that chunking works. + for filter in logs_filters { + assert!(filter.contracts.len() <= ENV_VARS.get_logs_max_contracts); + } + } + } +} + +#[test] +fn log_filter_require_transacion_receipt_method() { + // test data + let event_signature_a = H256::zero(); + let event_signature_b = H256::from_low_u64_be(1); + let event_signature_c = H256::from_low_u64_be(2); + let contract_a = Address::from_low_u64_be(3); + let contract_b = Address::from_low_u64_be(4); + let contract_c = Address::from_low_u64_be(5); + + let wildcard_event_with_receipt = H256::from_low_u64_be(6); + let wildcard_event_without_receipt = H256::from_low_u64_be(7); + let wildcard_events = [ + (wildcard_event_with_receipt, true), + (wildcard_event_without_receipt, false), + ] + .into_iter() + .collect(); + + let alien_event_signature = H256::from_low_u64_be(8); // those will not be inserted in the graph + let alien_contract_address = Address::from_low_u64_be(9); + + // test graph nodes + let event_a_node = LogFilterNode::Event(event_signature_a); + let event_b_node = LogFilterNode::Event(event_signature_b); + let event_c_node = LogFilterNode::Event(event_signature_c); + let contract_a_node = LogFilterNode::Contract(contract_a); + let contract_b_node = LogFilterNode::Contract(contract_b); + let contract_c_node = LogFilterNode::Contract(contract_c); + + // build test graph with the following layout: + // + // ```dot + // graph bipartite { + // + // // conected and require a receipt + // event_a -- contract_a [ receipt=true ] + // event_b -- contract_b [ receipt=true ] + // event_c -- contract_c [ receipt=true ] + // + // // connected but don't require a receipt + // event_a -- contract_b [ receipt=false ] + // event_b -- contract_a [ receipt=false ] + // } + // ``` + let mut contracts_and_events_graph = GraphMap::new(); + + let event_a_id = contracts_and_events_graph.add_node(event_a_node); + let event_b_id = contracts_and_events_graph.add_node(event_b_node); + let event_c_id = contracts_and_events_graph.add_node(event_c_node); + let contract_a_id = contracts_and_events_graph.add_node(contract_a_node); + let contract_b_id = contracts_and_events_graph.add_node(contract_b_node); + let contract_c_id = contracts_and_events_graph.add_node(contract_c_node); + contracts_and_events_graph.add_edge(event_a_id, contract_a_id, true); + contracts_and_events_graph.add_edge(event_b_id, contract_b_id, true); + contracts_and_events_graph.add_edge(event_a_id, contract_b_id, false); + contracts_and_events_graph.add_edge(event_b_id, contract_a_id, false); + contracts_and_events_graph.add_edge(event_c_id, contract_c_id, true); + + let filter = EthereumLogFilter { + contracts_and_events_graph, + wildcard_events, + }; + + // connected contracts and events graph + assert!(filter.requires_transaction_receipt(&event_signature_a, Some(&contract_a))); + assert!(filter.requires_transaction_receipt(&event_signature_b, Some(&contract_b))); + assert!(filter.requires_transaction_receipt(&event_signature_c, Some(&contract_c))); + assert!(!filter.requires_transaction_receipt(&event_signature_a, Some(&contract_b))); + assert!(!filter.requires_transaction_receipt(&event_signature_b, Some(&contract_a))); + + // Event C and Contract C are not connected to the other events and contracts + assert!(!filter.requires_transaction_receipt(&event_signature_a, Some(&contract_c))); + assert!(!filter.requires_transaction_receipt(&event_signature_b, Some(&contract_c))); + assert!(!filter.requires_transaction_receipt(&event_signature_c, Some(&contract_a))); + assert!(!filter.requires_transaction_receipt(&event_signature_c, Some(&contract_b))); + + // Wildcard events + assert!(filter.requires_transaction_receipt(&wildcard_event_with_receipt, None)); + assert!(!filter.requires_transaction_receipt(&wildcard_event_without_receipt, None)); + + // Alien events and contracts always return false + assert!( + !filter.requires_transaction_receipt(&alien_event_signature, Some(&alien_contract_address)) + ); + assert!(!filter.requires_transaction_receipt(&alien_event_signature, None)); + assert!(!filter.requires_transaction_receipt(&alien_event_signature, Some(&contract_a))); + assert!(!filter.requires_transaction_receipt(&alien_event_signature, Some(&contract_b))); + assert!(!filter.requires_transaction_receipt(&alien_event_signature, Some(&contract_c))); + assert!(!filter.requires_transaction_receipt(&event_signature_a, Some(&alien_contract_address))); + assert!(!filter.requires_transaction_receipt(&event_signature_b, Some(&alien_contract_address))); + assert!(!filter.requires_transaction_receipt(&event_signature_c, Some(&alien_contract_address))); +} diff --git a/chain/ethereum/src/capabilities.rs b/chain/ethereum/src/capabilities.rs new file mode 100644 index 0000000..15ce850 --- /dev/null +++ b/chain/ethereum/src/capabilities.rs @@ -0,0 +1,86 @@ +use anyhow::Error; +use graph::impl_slog_value; +use std::fmt; +use std::str::FromStr; +use std::{ + cmp::{Ord, Ordering, PartialOrd}, + collections::BTreeSet, +}; + +use crate::DataSource; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NodeCapabilities { + pub archive: bool, + pub traces: bool, +} + +// Take all NodeCapabilities fields into account when ordering +// A NodeCapabilities instance is considered equal or greater than another +// if all of its fields are equal or greater than the other +impl Ord for NodeCapabilities { + fn cmp(&self, other: &Self) -> Ordering { + match ( + self.archive.cmp(&other.archive), + self.traces.cmp(&other.traces), + ) { + (Ordering::Greater, Ordering::Greater) => Ordering::Greater, + (Ordering::Greater, Ordering::Equal) => Ordering::Greater, + (Ordering::Equal, Ordering::Greater) => Ordering::Greater, + (Ordering::Equal, Ordering::Equal) => Ordering::Equal, + (Ordering::Less, _) => Ordering::Less, + (_, Ordering::Less) => Ordering::Less, + } + } +} + +impl PartialOrd for NodeCapabilities { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl FromStr for NodeCapabilities { + type Err = Error; + + fn from_str(s: &str) -> Result { + let capabilities: BTreeSet<&str> = s.split(',').collect(); + Ok(NodeCapabilities { + archive: capabilities.contains("archive"), + traces: capabilities.contains("traces"), + }) + } +} + +impl fmt::Display for NodeCapabilities { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let NodeCapabilities { archive, traces } = self; + + let mut capabilities = vec![]; + if *archive { + capabilities.push("archive"); + } + if *traces { + capabilities.push("traces"); + } + + f.write_str(&capabilities.join(", ")) + } +} + +impl_slog_value!(NodeCapabilities, "{}"); + +impl graph::blockchain::NodeCapabilities for NodeCapabilities { + fn from_data_sources(data_sources: &[DataSource]) -> Self { + NodeCapabilities { + archive: data_sources.iter().any(|ds| { + ds.mapping + .requires_archive() + .expect("failed to parse mappings") + }), + traces: data_sources.into_iter().any(|ds| { + ds.mapping.has_call_handler() || ds.mapping.has_block_handler_with_call_filter() + }), + } + } +} diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs new file mode 100644 index 0000000..c793cd4 --- /dev/null +++ b/chain/ethereum/src/chain.rs @@ -0,0 +1,687 @@ +use anyhow::Result; +use anyhow::{Context, Error}; +use graph::blockchain::{BlockchainKind, TriggersAdapterSelector}; +use graph::data::subgraph::UnifiedMappingApiVersion; +use graph::firehose::{FirehoseEndpoint, FirehoseEndpoints, ForkStep}; +use graph::prelude::{EthereumBlock, EthereumCallCache, LightEthereumBlock, LightEthereumBlockExt}; +use graph::slog::debug; +use graph::{ + blockchain::{ + block_stream::{ + BlockStreamEvent, BlockWithTriggers, FirehoseError, + FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, + }, + firehose_block_stream::FirehoseBlockStream, + polling_block_stream::PollingBlockStream, + Block, BlockPtr, Blockchain, ChainHeadUpdateListener, IngestorError, + RuntimeAdapter as RuntimeAdapterTrait, TriggerFilter as _, + }, + cheap_clone::CheapClone, + components::store::DeploymentLocator, + firehose, + prelude::{ + async_trait, o, serde_json as json, BlockNumber, ChainStore, EthereumBlockWithCalls, + Future01CompatExt, Logger, LoggerFactory, MetricsRegistry, NodeId, + }, +}; +use prost::Message; +use std::collections::HashSet; +use std::iter::FromIterator; +use std::sync::Arc; + +use crate::data_source::DataSourceTemplate; +use crate::data_source::UnresolvedDataSourceTemplate; +use crate::{ + adapter::EthereumAdapter as _, + codec, + data_source::{DataSource, UnresolvedDataSource}, + ethereum_adapter::{ + blocks_with_triggers, get_calls, parse_block_triggers, parse_call_triggers, + parse_log_triggers, + }, + SubgraphEthRpcMetrics, TriggerFilter, ENV_VARS, +}; +use crate::{network::EthereumNetworkAdapters, EthereumAdapter}; +use graph::blockchain::block_stream::{BlockStream, BlockStreamBuilder, FirehoseCursor}; + +/// Celo Mainnet: 42220, Testnet Alfajores: 44787, Testnet Baklava: 62320 +const CELO_CHAIN_IDS: [u64; 3] = [42220, 44787, 62320]; + +pub struct EthereumStreamBuilder {} + +#[async_trait] +impl BlockStreamBuilder for EthereumStreamBuilder { + async fn build_firehose( + &self, + chain: &Chain, + deployment: DeploymentLocator, + block_cursor: FirehoseCursor, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc<::TriggerFilter>, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>> { + let requirements = filter.node_capabilities(); + let adapter = chain + .triggers_adapter(&deployment, &requirements, unified_api_version) + .expect(&format!( + "no adapter for network {} with capabilities {}", + chain.name, requirements + )); + + let firehose_endpoint = match chain.firehose_endpoints.random() { + Some(e) => e.clone(), + None => return Err(anyhow::format_err!("no firehose endpoint available",)), + }; + + let logger = chain + .logger_factory + .subgraph_logger(&deployment) + .new(o!("component" => "FirehoseBlockStream")); + + let firehose_mapper = Arc::new(FirehoseMapper { + endpoint: firehose_endpoint.cheap_clone(), + }); + + Ok(Box::new(FirehoseBlockStream::new( + deployment.hash, + firehose_endpoint, + subgraph_current_block, + block_cursor, + firehose_mapper, + adapter, + filter, + start_blocks, + logger, + chain.registry.clone(), + ))) + } + + async fn build_polling( + &self, + _chain: Arc, + _deployment: DeploymentLocator, + _start_blocks: Vec, + _subgraph_current_block: Option, + _filter: Arc<::TriggerFilter>, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>> { + todo!() + } +} + +pub struct EthereumAdapterSelector { + logger_factory: LoggerFactory, + adapters: Arc, + firehose_endpoints: Arc, + registry: Arc, + chain_store: Arc, +} + +impl EthereumAdapterSelector { + pub fn new( + logger_factory: LoggerFactory, + adapters: Arc, + firehose_endpoints: Arc, + registry: Arc, + chain_store: Arc, + ) -> Self { + Self { + logger_factory, + adapters, + firehose_endpoints, + registry, + chain_store, + } + } +} + +impl TriggersAdapterSelector for EthereumAdapterSelector { + fn triggers_adapter( + &self, + loc: &DeploymentLocator, + capabilities: &::NodeCapabilities, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + let logger = self + .logger_factory + .subgraph_logger(&loc) + .new(o!("component" => "BlockStream")); + + let eth_adapter = if capabilities.traces && self.firehose_endpoints.len() > 0 { + debug!(logger, "Removing 'traces' capability requirement for adapter as FirehoseBlockStream will provide the traces"); + let adjusted_capabilities = crate::capabilities::NodeCapabilities { + archive: capabilities.archive, + traces: false, + }; + + self.adapters.cheapest_with(&adjusted_capabilities)?.clone() + } else { + self.adapters.cheapest_with(capabilities)?.clone() + }; + + let ethrpc_metrics = Arc::new(SubgraphEthRpcMetrics::new(self.registry.clone(), &loc.hash)); + + let adapter = TriggersAdapter { + logger: logger.clone(), + ethrpc_metrics, + eth_adapter, + chain_store: self.chain_store.cheap_clone(), + unified_api_version, + }; + Ok(Arc::new(adapter)) + } +} + +pub struct Chain { + logger_factory: LoggerFactory, + name: String, + node_id: NodeId, + registry: Arc, + firehose_endpoints: Arc, + eth_adapters: Arc, + chain_store: Arc, + call_cache: Arc, + chain_head_update_listener: Arc, + reorg_threshold: BlockNumber, + pub is_ingestible: bool, + block_stream_builder: Arc>, + adapter_selector: Arc>, + runtime_adapter: Arc>, +} + +impl std::fmt::Debug for Chain { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "chain: ethereum") + } +} + +impl Chain { + /// Creates a new Ethereum [`Chain`]. + pub fn new( + logger_factory: LoggerFactory, + name: String, + node_id: NodeId, + registry: Arc, + chain_store: Arc, + call_cache: Arc, + firehose_endpoints: FirehoseEndpoints, + eth_adapters: EthereumNetworkAdapters, + chain_head_update_listener: Arc, + block_stream_builder: Arc>, + adapter_selector: Arc>, + runtime_adapter: Arc>, + reorg_threshold: BlockNumber, + is_ingestible: bool, + ) -> Self { + Chain { + logger_factory, + name, + node_id, + registry, + firehose_endpoints: Arc::new(firehose_endpoints), + eth_adapters: Arc::new(eth_adapters), + chain_store, + call_cache, + chain_head_update_listener, + block_stream_builder, + adapter_selector, + runtime_adapter, + reorg_threshold, + is_ingestible, + } + } + + /// Returns a handler to this chain's [`EthereumCallCache`]. + pub fn call_cache(&self) -> Arc { + self.call_cache.clone() + } + + pub fn cheapest_adapter(&self) -> Arc { + self.eth_adapters.cheapest().unwrap().clone() + } +} + +#[async_trait] +impl Blockchain for Chain { + const KIND: BlockchainKind = BlockchainKind::Ethereum; + const ALIASES: &'static [&'static str] = &["ethereum/contract"]; + + type Block = BlockFinality; + + type DataSource = DataSource; + + type UnresolvedDataSource = UnresolvedDataSource; + + type DataSourceTemplate = DataSourceTemplate; + + type UnresolvedDataSourceTemplate = UnresolvedDataSourceTemplate; + + type TriggerData = crate::trigger::EthereumTrigger; + + type MappingTrigger = crate::trigger::MappingTrigger; + + type TriggerFilter = crate::adapter::TriggerFilter; + + type NodeCapabilities = crate::capabilities::NodeCapabilities; + + fn triggers_adapter( + &self, + loc: &DeploymentLocator, + capabilities: &Self::NodeCapabilities, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + self.adapter_selector + .triggers_adapter(loc, capabilities, unified_api_version) + } + + async fn new_firehose_block_stream( + &self, + deployment: DeploymentLocator, + block_cursor: FirehoseCursor, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + self.block_stream_builder + .build_firehose( + self, + deployment, + block_cursor, + start_blocks, + subgraph_current_block, + filter, + unified_api_version, + ) + .await + } + + async fn new_polling_block_stream( + &self, + deployment: DeploymentLocator, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + let requirements = filter.node_capabilities(); + let adapter = self + .triggers_adapter(&deployment, &requirements, unified_api_version.clone()) + .expect(&format!( + "no adapter for network {} with capabilities {}", + self.name, requirements + )); + + let logger = self + .logger_factory + .subgraph_logger(&deployment) + .new(o!("component" => "BlockStream")); + let chain_store = self.chain_store().clone(); + let chain_head_update_stream = self + .chain_head_update_listener + .subscribe(self.name.clone(), logger.clone()); + + // Special case: Detect Celo and set the threshold to 0, so that eth_getLogs is always used. + // This is ok because Celo blocks are always final. And we _need_ to do this because + // some events appear only in eth_getLogs but not in transaction receipts. + // See also ca0edc58-0ec5-4c89-a7dd-2241797f5e50. + let chain_id = self.eth_adapters.cheapest().unwrap().chain_id().await?; + let reorg_threshold = match CELO_CHAIN_IDS.contains(&chain_id) { + false => self.reorg_threshold, + true => 0, + }; + + Ok(Box::new(PollingBlockStream::new( + chain_store, + chain_head_update_stream, + adapter, + self.node_id.clone(), + deployment.hash, + filter, + start_blocks, + reorg_threshold, + logger, + ENV_VARS.max_block_range_size, + ENV_VARS.target_triggers_per_block_range, + unified_api_version, + subgraph_current_block, + ))) + } + + fn chain_store(&self) -> Arc { + self.chain_store.clone() + } + + async fn block_pointer_from_number( + &self, + logger: &Logger, + number: BlockNumber, + ) -> Result { + let eth_adapter = self + .eth_adapters + .cheapest() + .with_context(|| format!("no adapter for chain {}", self.name))? + .clone(); + eth_adapter + .block_pointer_from_number(logger, number) + .compat() + .await + } + + fn runtime_adapter(&self) -> Arc> { + self.runtime_adapter.clone() + } + + fn is_firehose_supported(&self) -> bool { + ENV_VARS.is_firehose_preferred && self.firehose_endpoints.len() > 0 + } +} + +/// This is used in `EthereumAdapter::triggers_in_block`, called when re-processing a block for +/// newly created data sources. This allows the re-processing to be reorg safe without having to +/// always fetch the full block data. +#[derive(Clone, Debug)] +pub enum BlockFinality { + /// If a block is final, we only need the header and the triggers. + Final(Arc), + + // If a block may still be reorged, we need to work with more local data. + NonFinal(EthereumBlockWithCalls), +} + +impl Default for BlockFinality { + fn default() -> Self { + Self::Final(Arc::default()) + } +} + +impl BlockFinality { + pub(crate) fn light_block(&self) -> &Arc { + match self { + BlockFinality::Final(block) => block, + BlockFinality::NonFinal(block) => &block.ethereum_block.block, + } + } +} + +impl<'a> From<&'a BlockFinality> for BlockPtr { + fn from(block: &'a BlockFinality) -> BlockPtr { + match block { + BlockFinality::Final(b) => BlockPtr::from(&**b), + BlockFinality::NonFinal(b) => BlockPtr::from(&b.ethereum_block), + } + } +} + +impl Block for BlockFinality { + fn ptr(&self) -> BlockPtr { + match self { + BlockFinality::Final(block) => block.block_ptr(), + BlockFinality::NonFinal(block) => block.ethereum_block.block.block_ptr(), + } + } + + fn parent_ptr(&self) -> Option { + match self { + BlockFinality::Final(block) => block.parent_ptr(), + BlockFinality::NonFinal(block) => block.ethereum_block.block.parent_ptr(), + } + } + + fn data(&self) -> Result { + // The serialization here very delicately depends on how the + // `ChainStore`'s `blocks` and `ancestor_block` return the data we + // store here. This should be fixed in a better way to ensure we + // serialize/deserialize appropriately. + // + // Commit #d62e9846 inadvertently introduced a variation in how + // chain stores store ethereum blocks in that they now sometimes + // store an `EthereumBlock` that has a `block` field with a + // `LightEthereumBlock`, and sometimes they just store the + // `LightEthereumBlock` directly. That causes issues because the + // code reading from the chain store always expects the JSON data to + // have the form of an `EthereumBlock`. + // + // Even though this bug is fixed now and we always use the + // serialization of an `EthereumBlock`, there are still chain stores + // in existence that used the old serialization form, and we need to + // deal with that when deserializing + // + // see also 7736e440-4c6b-11ec-8c4d-b42e99f52061 + match self { + BlockFinality::Final(block) => { + let eth_block = EthereumBlock { + block: block.clone(), + transaction_receipts: vec![], + }; + json::to_value(eth_block) + } + BlockFinality::NonFinal(block) => json::to_value(&block.ethereum_block), + } + } +} + +pub struct DummyDataSourceTemplate; + +pub struct TriggersAdapter { + logger: Logger, + ethrpc_metrics: Arc, + chain_store: Arc, + eth_adapter: Arc, + unified_api_version: UnifiedMappingApiVersion, +} + +#[async_trait] +impl TriggersAdapterTrait for TriggersAdapter { + async fn scan_triggers( + &self, + from: BlockNumber, + to: BlockNumber, + filter: &TriggerFilter, + ) -> Result>, Error> { + blocks_with_triggers( + self.eth_adapter.clone(), + self.logger.clone(), + self.chain_store.clone(), + self.ethrpc_metrics.clone(), + from, + to, + filter, + self.unified_api_version.clone(), + ) + .await + } + + async fn triggers_in_block( + &self, + logger: &Logger, + block: BlockFinality, + filter: &TriggerFilter, + ) -> Result, Error> { + let block = get_calls( + self.eth_adapter.as_ref(), + logger.clone(), + self.ethrpc_metrics.clone(), + filter.requires_traces(), + block, + ) + .await?; + + match &block { + BlockFinality::Final(_) => { + let block_number = block.number() as BlockNumber; + let blocks = blocks_with_triggers( + self.eth_adapter.clone(), + logger.clone(), + self.chain_store.clone(), + self.ethrpc_metrics.clone(), + block_number, + block_number, + filter, + self.unified_api_version.clone(), + ) + .await?; + assert!(blocks.len() == 1); + Ok(blocks.into_iter().next().unwrap()) + } + BlockFinality::NonFinal(full_block) => { + let mut triggers = Vec::new(); + triggers.append(&mut parse_log_triggers( + &filter.log, + &full_block.ethereum_block, + )); + triggers.append(&mut parse_call_triggers(&filter.call, &full_block)?); + triggers.append(&mut parse_block_triggers(&filter.block, &full_block)); + Ok(BlockWithTriggers::new(block, triggers)) + } + } + } + + async fn is_on_main_chain(&self, ptr: BlockPtr) -> Result { + self.eth_adapter + .is_on_main_chain(&self.logger, ptr.clone()) + .await + } + + async fn ancestor_block( + &self, + ptr: BlockPtr, + offset: BlockNumber, + ) -> Result, Error> { + let block: Option = self + .chain_store + .cheap_clone() + .ancestor_block(ptr, offset) + .await? + .map(json::from_value) + .transpose()?; + Ok(block.map(|block| { + BlockFinality::NonFinal(EthereumBlockWithCalls { + ethereum_block: block, + calls: None, + }) + })) + } + + async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { + use futures::stream::Stream; + use graph::prelude::LightEthereumBlockExt; + + let blocks = self + .eth_adapter + .load_blocks( + self.logger.cheap_clone(), + self.chain_store.cheap_clone(), + HashSet::from_iter(Some(block.hash_as_h256())), + ) + .collect() + .compat() + .await?; + assert_eq!(blocks.len(), 1); + + Ok(blocks[0].parent_ptr()) + } +} + +pub struct FirehoseMapper { + endpoint: Arc, +} + +#[async_trait] +impl FirehoseMapperTrait for FirehoseMapper { + async fn to_block_stream_event( + &self, + logger: &Logger, + response: &firehose::Response, + adapter: &Arc>, + filter: &TriggerFilter, + ) -> Result, FirehoseError> { + let step = ForkStep::from_i32(response.step).unwrap_or_else(|| { + panic!( + "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", + response.step + ) + }); + let any_block = response + .block + .as_ref() + .expect("block payload information should always be present"); + + // Right now, this is done in all cases but in reality, with how the BlockStreamEvent::Revert + // is defined right now, only block hash and block number is necessary. However, this information + // is not part of the actual firehose::Response payload. As such, we need to decode the full + // block which is useless. + // + // Check about adding basic information about the block in the firehose::Response or maybe + // define a slimmed down stuct that would decode only a few fields and ignore all the rest. + let block = codec::Block::decode(any_block.value.as_ref())?; + + use firehose::ForkStep::*; + match step { + StepNew => { + // See comment(437a9f17-67cc-478f-80a3-804fe554b227) ethereum_block.calls is always Some even if calls + // is empty + let ethereum_block: EthereumBlockWithCalls = (&block).try_into()?; + + // triggers in block never actually calls the ethereum traces api. + // TODO: Split the trigger parsing from call retrieving. + let block_with_triggers = adapter + .triggers_in_block(logger, BlockFinality::NonFinal(ethereum_block), filter) + .await?; + + Ok(BlockStreamEvent::ProcessBlock( + block_with_triggers, + FirehoseCursor::from(response.cursor.clone()), + )) + } + + StepUndo => { + let parent_ptr = block + .parent_ptr() + .expect("Genesis block should never be reverted"); + + Ok(BlockStreamEvent::Revert( + parent_ptr, + FirehoseCursor::from(response.cursor.clone()), + )) + } + + StepIrreversible => { + unreachable!("irreversible step is not handled and should not be requested in the Firehose request") + } + + StepUnknown => { + unreachable!("unknown step should not happen in the Firehose response") + } + } + } + + async fn block_ptr_for_number( + &self, + logger: &Logger, + number: BlockNumber, + ) -> Result { + self.endpoint + .block_ptr_for_number::(logger, number) + .await + } + + async fn final_block_ptr_for( + &self, + logger: &Logger, + block: &BlockFinality, + ) -> Result { + // Firehose for Ethereum has an hard-coded confirmations for finality sets to 200 block + // behind the current block. The magic value 200 here comes from this hard-coded Firehose + // value. + let final_block_number = match block.number() { + x if x >= 200 => x - 200, + _ => 0, + }; + + self.endpoint + .block_ptr_for_number::(logger, final_block_number) + .await + } +} diff --git a/chain/ethereum/src/codec.rs b/chain/ethereum/src/codec.rs new file mode 100644 index 0000000..b1ab800 --- /dev/null +++ b/chain/ethereum/src/codec.rs @@ -0,0 +1,455 @@ +#[rustfmt::skip] +#[path = "protobuf/sf.ethereum.r#type.v2.rs"] +mod pbcodec; + +use anyhow::format_err; +use graph::{ + blockchain::{Block as BlockchainBlock, BlockPtr}, + prelude::{ + web3, + web3::types::{Bytes, H160, H2048, H256, H64, U256, U64}, + BlockNumber, Error, EthereumBlock, EthereumBlockWithCalls, EthereumCall, + LightEthereumBlock, + }, +}; +use std::sync::Arc; +use std::{convert::TryFrom, fmt::Debug}; + +use crate::chain::BlockFinality; + +pub use pbcodec::*; + +trait TryDecodeProto: Sized +where + U: TryFrom, + >::Error: Debug, + V: From, +{ + fn try_decode_proto(self, label: &'static str) -> Result { + let u = U::try_from(self).map_err(|e| format_err!("invalid {}: {:?}", label, e))?; + let v = V::from(u); + Ok(v) + } +} + +impl TryDecodeProto<[u8; 256], H2048> for &[u8] {} +impl TryDecodeProto<[u8; 32], H256> for &[u8] {} +impl TryDecodeProto<[u8; 20], H160> for &[u8] {} + +impl Into for &BigInt { + fn into(self) -> web3::types::U256 { + web3::types::U256::from_big_endian(&self.bytes) + } +} + +pub struct CallAt<'a> { + call: &'a Call, + block: &'a Block, + trace: &'a TransactionTrace, +} + +impl<'a> CallAt<'a> { + pub fn new(call: &'a Call, block: &'a Block, trace: &'a TransactionTrace) -> Self { + Self { call, block, trace } + } +} + +impl<'a> TryInto for CallAt<'a> { + type Error = Error; + + fn try_into(self) -> Result { + Ok(EthereumCall { + from: self.call.caller.try_decode_proto("call from address")?, + to: self.call.address.try_decode_proto("call to address")?, + value: self + .call + .value + .as_ref() + .map_or_else(|| U256::from(0), |v| v.into()), + gas_used: U256::from(self.call.gas_consumed), + input: Bytes(self.call.input.clone()), + output: Bytes(self.call.return_data.clone()), + block_hash: self.block.hash.try_decode_proto("call block hash")?, + block_number: self.block.number as i32, + transaction_hash: Some(self.trace.hash.try_decode_proto("call transaction hash")?), + transaction_index: self.trace.index as u64, + }) + } +} + +impl TryInto for Call { + type Error = Error; + + fn try_into(self) -> Result { + Ok(web3::types::Call { + from: self.caller.try_decode_proto("call from address")?, + to: self.address.try_decode_proto("call to address")?, + value: self + .value + .as_ref() + .map_or_else(|| U256::from(0), |v| v.into()), + gas: U256::from(self.gas_limit), + input: Bytes::from(self.input.clone()), + call_type: CallType::from_i32(self.call_type) + .ok_or_else(|| format_err!("invalid call type: {}", self.call_type,))? + .into(), + }) + } +} + +impl Into for CallType { + fn into(self) -> web3::types::CallType { + match self { + CallType::Unspecified => web3::types::CallType::None, + CallType::Call => web3::types::CallType::Call, + CallType::Callcode => web3::types::CallType::CallCode, + CallType::Delegate => web3::types::CallType::DelegateCall, + CallType::Static => web3::types::CallType::StaticCall, + + // FIXME (SF): Really not sure what this should map to, we are using None for now, need to revisit + CallType::Create => web3::types::CallType::None, + } + } +} + +pub struct LogAt<'a> { + log: &'a Log, + block: &'a Block, + trace: &'a TransactionTrace, +} + +impl<'a> LogAt<'a> { + pub fn new(log: &'a Log, block: &'a Block, trace: &'a TransactionTrace) -> Self { + Self { log, block, trace } + } +} + +impl<'a> TryInto for LogAt<'a> { + type Error = Error; + + fn try_into(self) -> Result { + Ok(web3::types::Log { + address: self.log.address.try_decode_proto("log address")?, + topics: self + .log + .topics + .iter() + .map(|t| t.try_decode_proto("topic")) + .collect::, Error>>()?, + data: Bytes::from(self.log.data.clone()), + block_hash: Some(self.block.hash.try_decode_proto("log block hash")?), + block_number: Some(U64::from(self.block.number)), + transaction_hash: Some(self.trace.hash.try_decode_proto("log transaction hash")?), + transaction_index: Some(U64::from(self.trace.index as u64)), + log_index: Some(U256::from(self.log.block_index)), + transaction_log_index: Some(U256::from(self.log.index)), + log_type: None, + removed: None, + }) + } +} + +impl Into for TransactionTraceStatus { + fn into(self) -> web3::types::U64 { + let status: Option = self.into(); + status.unwrap_or_else(|| web3::types::U64::from(0)) + } +} + +impl Into> for TransactionTraceStatus { + fn into(self) -> Option { + match self { + Self::Unknown => { + panic!("Got a transaction trace with status UNKNOWN, datasource is broken") + } + Self::Succeeded => Some(web3::types::U64::from(1)), + Self::Failed => Some(web3::types::U64::from(0)), + Self::Reverted => Some(web3::types::U64::from(0)), + } + } +} + +pub struct TransactionTraceAt<'a> { + trace: &'a TransactionTrace, + block: &'a Block, +} + +impl<'a> TransactionTraceAt<'a> { + pub fn new(trace: &'a TransactionTrace, block: &'a Block) -> Self { + Self { trace, block } + } +} + +impl<'a> TryInto for TransactionTraceAt<'a> { + type Error = Error; + + fn try_into(self) -> Result { + Ok(web3::types::Transaction { + hash: self.trace.hash.try_decode_proto("transaction hash")?, + nonce: U256::from(self.trace.nonce), + block_hash: Some(self.block.hash.try_decode_proto("transaction block hash")?), + block_number: Some(U64::from(self.block.number)), + transaction_index: Some(U64::from(self.trace.index as u64)), + from: Some( + self.trace + .from + .try_decode_proto("transaction from address")?, + ), + to: Some(self.trace.to.try_decode_proto("transaction to address")?), + value: self.trace.value.as_ref().map_or(U256::zero(), |x| x.into()), + gas_price: self.trace.gas_price.as_ref().map(|x| x.into()), + gas: U256::from(self.trace.gas_limit), + input: Bytes::from(self.trace.input.clone()), + v: None, + r: None, + s: None, + raw: None, + access_list: None, + max_fee_per_gas: None, + max_priority_fee_per_gas: None, + transaction_type: None, + }) + } +} + +impl TryInto for &Block { + type Error = Error; + + fn try_into(self) -> Result { + Ok(BlockFinality::NonFinal(self.try_into()?)) + } +} + +impl TryInto for &Block { + type Error = Error; + + fn try_into(self) -> Result { + let header = self + .header + .as_ref() + .expect("block header should always be present from gRPC Firehose"); + + let block = EthereumBlockWithCalls { + ethereum_block: EthereumBlock { + block: Arc::new(LightEthereumBlock { + hash: Some(self.hash.try_decode_proto("block hash")?), + number: Some(U64::from(self.number)), + author: header.coinbase.try_decode_proto("author / coinbase")?, + parent_hash: header.parent_hash.try_decode_proto("parent hash")?, + uncles_hash: header.uncle_hash.try_decode_proto("uncle hash")?, + state_root: header.state_root.try_decode_proto("state root")?, + transactions_root: header + .transactions_root + .try_decode_proto("transactions root")?, + receipts_root: header.receipt_root.try_decode_proto("receipt root")?, + gas_used: U256::from(header.gas_used), + gas_limit: U256::from(header.gas_limit), + base_fee_per_gas: None, + extra_data: Bytes::from(header.extra_data.clone()), + logs_bloom: match &header.logs_bloom.len() { + 0 => None, + _ => Some(header.logs_bloom.try_decode_proto("logs bloom")?), + }, + timestamp: U256::from( + header + .timestamp + .as_ref() + .map_or_else(|| U256::default(), |v| U256::from(v.seconds)), + ), + difficulty: header + .difficulty + .as_ref() + .map_or_else(|| U256::default(), |v| v.into()), + total_difficulty: Some( + header + .total_difficulty + .as_ref() + .map_or_else(|| U256::default(), |v| v.into()), + ), + // FIXME (SF): Firehose does not have seal fields, are they really used? Might be required for POA chains only also, I've seen that stuff on xDai (is this important?) + seal_fields: vec![], + uncles: self + .uncles + .iter() + .map(|u| u.hash.try_decode_proto("uncle hash")) + .collect::, _>>()?, + transactions: self + .transaction_traces + .iter() + .map(|t| TransactionTraceAt::new(t, &self).try_into()) + .collect::, Error>>()?, + size: Some(U256::from(self.size)), + mix_hash: Some(header.mix_hash.try_decode_proto("mix hash")?), + nonce: Some(H64::from_low_u64_be(header.nonce)), + }), + transaction_receipts: self + .transaction_traces + .iter() + .filter_map(|t| { + t.receipt.as_ref().map(|r| { + Ok(web3::types::TransactionReceipt { + transaction_hash: t.hash.try_decode_proto("transaction hash")?, + transaction_index: U64::from(t.index), + block_hash: Some( + self.hash.try_decode_proto("transaction block hash")?, + ), + block_number: Some(U64::from(self.number)), + cumulative_gas_used: U256::from(r.cumulative_gas_used), + // FIXME (SF): What is the rule here about gas_used being None, when it's 0? + gas_used: Some(U256::from(t.gas_used)), + contract_address: { + match t.calls.len() { + 0 => None, + _ => { + match CallType::from_i32(t.calls[0].call_type) + .ok_or_else(|| { + format_err!( + "invalid call type: {}", + t.calls[0].call_type, + ) + })? { + CallType::Create => { + Some(t.calls[0].address.try_decode_proto( + "transaction contract address", + )?) + } + _ => None, + } + } + } + }, + logs: r + .logs + .iter() + .map(|l| LogAt::new(l, &self, t).try_into()) + .collect::, Error>>()?, + status: TransactionTraceStatus::from_i32(t.status) + .ok_or_else(|| { + format_err!( + "invalid transaction trace status: {}", + t.status + ) + })? + .into(), + root: match r.state_root.len() { + 0 => None, // FIXME (SF): should this instead map to [0;32]? + // FIXME (SF): if len < 32, what do we do? + _ => Some( + r.state_root.try_decode_proto("transaction state root")?, + ), + }, + logs_bloom: r + .logs_bloom + .try_decode_proto("transaction logs bloom")?, + from: t.from.try_decode_proto("transaction from")?, + to: Some(t.to.try_decode_proto("transaction to")?), + transaction_type: None, + effective_gas_price: None, + }) + }) + }) + .collect::, Error>>()? + .into_iter() + // Transaction receipts will be shared along the code, so we put them into an + // Arc here to avoid excessive cloning. + .map(Arc::new) + .collect(), + }, + // Comment (437a9f17-67cc-478f-80a3-804fe554b227): This Some() will avoid calls in the triggers_in_block + // TODO: Refactor in a way that this is no longer needed. + calls: Some( + self.transaction_traces + .iter() + .flat_map(|trace| { + trace + .calls + .iter() + .filter(|call| !call.status_reverted && !call.status_failed) + .map(|call| CallAt::new(call, self, trace).try_into()) + .collect::>>() + }) + .collect::>()?, + ), + }; + + Ok(block) + } +} + +impl BlockHeader { + pub fn parent_ptr(&self) -> Option { + match self.parent_hash.len() { + 0 => None, + _ => Some(BlockPtr::from(( + H256::from_slice(self.parent_hash.as_ref()), + self.number - 1, + ))), + } + } +} + +impl<'a> From<&'a BlockHeader> for BlockPtr { + fn from(b: &'a BlockHeader) -> BlockPtr { + BlockPtr::from((H256::from_slice(b.hash.as_ref()), b.number)) + } +} + +impl<'a> From<&'a Block> for BlockPtr { + fn from(b: &'a Block) -> BlockPtr { + BlockPtr::from((H256::from_slice(b.hash.as_ref()), b.number)) + } +} + +impl Block { + pub fn header(&self) -> &BlockHeader { + self.header.as_ref().unwrap() + } + + pub fn ptr(&self) -> BlockPtr { + BlockPtr::from(self.header()) + } + + pub fn parent_ptr(&self) -> Option { + self.header().parent_ptr() + } +} + +impl BlockchainBlock for Block { + fn number(&self) -> i32 { + BlockNumber::try_from(self.header().number).unwrap() + } + + fn ptr(&self) -> BlockPtr { + self.into() + } + + fn parent_ptr(&self) -> Option { + self.parent_ptr() + } +} + +impl HeaderOnlyBlock { + pub fn header(&self) -> &BlockHeader { + self.header.as_ref().unwrap() + } +} + +impl<'a> From<&'a HeaderOnlyBlock> for BlockPtr { + fn from(b: &'a HeaderOnlyBlock) -> BlockPtr { + BlockPtr::from(b.header()) + } +} + +impl BlockchainBlock for HeaderOnlyBlock { + fn number(&self) -> i32 { + BlockNumber::try_from(self.header().number).unwrap() + } + + fn ptr(&self) -> BlockPtr { + self.into() + } + + fn parent_ptr(&self) -> Option { + self.header().parent_ptr() + } +} diff --git a/chain/ethereum/src/data_source.rs b/chain/ethereum/src/data_source.rs new file mode 100644 index 0000000..9bf8949 --- /dev/null +++ b/chain/ethereum/src/data_source.rs @@ -0,0 +1,1075 @@ +use anyhow::{anyhow, Error}; +use anyhow::{ensure, Context}; +use graph::blockchain::TriggerWithHandler; +use graph::components::store::StoredDynamicDataSource; +use graph::prelude::ethabi::ethereum_types::H160; +use graph::prelude::ethabi::StateMutability; +use graph::prelude::futures03::future::try_join; +use graph::prelude::futures03::stream::FuturesOrdered; +use graph::prelude::{Link, SubgraphManifestValidationError}; +use graph::slog::{o, trace}; +use std::str::FromStr; +use std::{convert::TryFrom, sync::Arc}; +use tiny_keccak::{keccak256, Keccak}; + +use graph::{ + blockchain::{self, Blockchain}, + prelude::{ + async_trait, + ethabi::{Address, Contract, Event, Function, LogParam, ParamType, RawLog}, + info, serde_json, warn, + web3::types::{Log, Transaction, H256}, + BlockNumber, CheapClone, DataSourceTemplateInfo, Deserialize, EthereumCall, + LightEthereumBlock, LightEthereumBlockExt, LinkResolver, Logger, TryStreamExt, + }, +}; + +use graph::data::subgraph::{calls_host_fn, DataSourceContext, Source}; + +use crate::chain::Chain; +use crate::trigger::{EthereumBlockTriggerType, EthereumTrigger, MappingTrigger}; + +// The recommended kind is `ethereum`, `ethereum/contract` is accepted for backwards compatibility. +const ETHEREUM_KINDS: &[&str] = &["ethereum/contract", "ethereum"]; + +/// Runtime representation of a data source. +// Note: Not great for memory usage that this needs to be `Clone`, considering how there may be tens +// of thousands of data sources in memory at once. +#[derive(Clone, Debug)] +pub struct DataSource { + pub kind: String, + pub network: Option, + pub name: String, + pub manifest_idx: u32, + pub address: Option
, + pub start_block: BlockNumber, + pub mapping: Mapping, + pub context: Arc>, + pub creation_block: Option, + pub contract_abi: Arc, +} + +impl blockchain::DataSource for DataSource { + fn address(&self) -> Option<&[u8]> { + self.address.as_ref().map(|x| x.as_bytes()) + } + + fn start_block(&self) -> BlockNumber { + self.start_block + } + + fn match_and_decode( + &self, + trigger: &::TriggerData, + block: &Arc<::Block>, + logger: &Logger, + ) -> Result>, Error> { + let block = block.light_block(); + self.match_and_decode(trigger, block, logger) + } + + fn name(&self) -> &str { + &self.name + } + + fn kind(&self) -> &str { + &self.kind + } + + fn network(&self) -> Option<&str> { + self.network.as_ref().map(|s| s.as_str()) + } + + fn context(&self) -> Arc> { + self.context.cheap_clone() + } + + fn creation_block(&self) -> Option { + self.creation_block + } + + fn is_duplicate_of(&self, other: &Self) -> bool { + let DataSource { + kind, + network, + name, + manifest_idx, + address, + mapping, + context, + + // The creation block is ignored for detection duplicate data sources. + // Contract ABI equality is implicit in `mapping.abis` equality. + creation_block: _, + contract_abi: _, + start_block: _, + } = self; + + // mapping_request_sender, host_metrics, and (most of) host_exports are operational structs + // used at runtime but not needed to define uniqueness; each runtime host should be for a + // unique data source. + kind == &other.kind + && network == &other.network + && name == &other.name + && manifest_idx == &other.manifest_idx + && address == &other.address + && mapping.abis == other.mapping.abis + && mapping.event_handlers == other.mapping.event_handlers + && mapping.call_handlers == other.mapping.call_handlers + && mapping.block_handlers == other.mapping.block_handlers + && context == &other.context + } + + fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { + let param = self.address.map(|addr| addr.0.into()); + StoredDynamicDataSource { + manifest_idx: self.manifest_idx, + param, + context: self + .context + .as_ref() + .as_ref() + .map(|ctx| serde_json::to_value(&ctx).unwrap()), + creation_block: self.creation_block, + is_offchain: false, + } + } + + fn from_stored_dynamic_data_source( + template: &DataSourceTemplate, + stored: StoredDynamicDataSource, + ) -> Result { + let StoredDynamicDataSource { + manifest_idx, + param, + context, + creation_block, + is_offchain, + } = stored; + + ensure!( + !is_offchain, + "attempted to convert offchain data source to ethereum data source" + ); + + let context = context.map(serde_json::from_value).transpose()?; + + let contract_abi = template.mapping.find_abi(&template.source.abi)?; + + let address = param.map(|x| H160::from_slice(&x)); + Ok(DataSource { + kind: template.kind.to_string(), + network: template.network.as_ref().map(|s| s.to_string()), + name: template.name.clone(), + manifest_idx, + address, + start_block: 0, + mapping: template.mapping.clone(), + context: Arc::new(context), + creation_block, + contract_abi, + }) + } + + fn validate(&self) -> Vec { + let mut errors = vec![]; + + if !ETHEREUM_KINDS.contains(&self.kind.as_str()) { + errors.push(anyhow!( + "data source has invalid `kind`, expected `ethereum` but found {}", + self.kind + )) + } + + // Validate that there is a `source` address if there are call or block handlers + let no_source_address = self.address().is_none(); + let has_call_handlers = !self.mapping.call_handlers.is_empty(); + let has_block_handlers = !self.mapping.block_handlers.is_empty(); + if no_source_address && (has_call_handlers || has_block_handlers) { + errors.push(SubgraphManifestValidationError::SourceAddressRequired.into()); + }; + + // Validate that there are no more than one of each type of block_handler + let has_too_many_block_handlers = { + let mut non_filtered_block_handler_count = 0; + let mut call_filtered_block_handler_count = 0; + self.mapping + .block_handlers + .iter() + .for_each(|block_handler| { + if block_handler.filter.is_none() { + non_filtered_block_handler_count += 1 + } else { + call_filtered_block_handler_count += 1 + } + }); + non_filtered_block_handler_count > 1 || call_filtered_block_handler_count > 1 + }; + if has_too_many_block_handlers { + errors.push(anyhow!("data source has duplicated block handlers")); + } + + // Validate that event handlers don't require receipts for API versions lower than 0.0.7 + let api_version = self.api_version(); + if api_version < semver::Version::new(0, 0, 7) { + for event_handler in &self.mapping.event_handlers { + if event_handler.receipt { + errors.push(anyhow!( + "data source has event handlers that require transaction receipts, but this \ + is only supported for apiVersion >= 0.0.7" + )); + break; + } + } + } + + errors + } + + fn api_version(&self) -> semver::Version { + self.mapping.api_version.clone() + } + + fn runtime(&self) -> Option>> { + Some(self.mapping.runtime.cheap_clone()) + } +} + +impl DataSource { + fn from_manifest( + kind: String, + network: Option, + name: String, + source: Source, + mapping: Mapping, + context: Option, + manifest_idx: u32, + ) -> Result { + // Data sources in the manifest are created "before genesis" so they have no creation block. + let creation_block = None; + let contract_abi = mapping + .find_abi(&source.abi) + .with_context(|| format!("data source `{}`", name))?; + + Ok(DataSource { + kind, + network, + name, + manifest_idx, + address: source.address, + start_block: source.start_block, + mapping, + context: Arc::new(context), + creation_block, + contract_abi, + }) + } + + fn handlers_for_log(&self, log: &Log) -> Result, Error> { + // Get signature from the log + let topic0 = log.topics.get(0).context("Ethereum event has no topics")?; + + let handlers = self + .mapping + .event_handlers + .iter() + .filter(|handler| *topic0 == handler.topic0()) + .cloned() + .collect::>(); + + Ok(handlers) + } + + fn handler_for_call(&self, call: &EthereumCall) -> Result, Error> { + // First four bytes of the input for the call are the first four + // bytes of hash of the function signature + ensure!( + call.input.0.len() >= 4, + "Ethereum call has input with less than 4 bytes" + ); + + let target_method_id = &call.input.0[..4]; + + Ok(self + .mapping + .call_handlers + .iter() + .find(move |handler| { + let fhash = keccak256(handler.function.as_bytes()); + let actual_method_id = [fhash[0], fhash[1], fhash[2], fhash[3]]; + target_method_id == actual_method_id + }) + .cloned()) + } + + fn handler_for_block( + &self, + trigger_type: &EthereumBlockTriggerType, + ) -> Option { + match trigger_type { + EthereumBlockTriggerType::Every => self + .mapping + .block_handlers + .iter() + .find(move |handler| handler.filter == None) + .cloned(), + EthereumBlockTriggerType::WithCallTo(_address) => self + .mapping + .block_handlers + .iter() + .find(move |handler| handler.filter == Some(BlockHandlerFilter::Call)) + .cloned(), + } + } + + /// Returns the contract event with the given signature, if it exists. A an event from the ABI + /// will be matched if: + /// 1. An event signature is equal to `signature`. + /// 2. There are no equal matches, but there is exactly one event that equals `signature` if all + /// `indexed` modifiers are removed from the parameters. + fn contract_event_with_signature(&self, signature: &str) -> Option<&Event> { + // Returns an `Event(uint256,address)` signature for an event, without `indexed` hints. + fn ambiguous_event_signature(event: &Event) -> String { + format!( + "{}({})", + event.name, + event + .inputs + .iter() + .map(|input| format!("{}", event_param_type_signature(&input.kind))) + .collect::>() + .join(",") + ) + } + + // Returns an `Event(indexed uint256,address)` type signature for an event. + fn event_signature(event: &Event) -> String { + format!( + "{}({})", + event.name, + event + .inputs + .iter() + .map(|input| format!( + "{}{}", + if input.indexed { "indexed " } else { "" }, + event_param_type_signature(&input.kind) + )) + .collect::>() + .join(",") + ) + } + + // Returns the signature of an event parameter type (e.g. `uint256`). + fn event_param_type_signature(kind: &ParamType) -> String { + use ParamType::*; + + match kind { + Address => "address".into(), + Bytes => "bytes".into(), + Int(size) => format!("int{}", size), + Uint(size) => format!("uint{}", size), + Bool => "bool".into(), + String => "string".into(), + Array(inner) => format!("{}[]", event_param_type_signature(&*inner)), + FixedBytes(size) => format!("bytes{}", size), + FixedArray(inner, size) => { + format!("{}[{}]", event_param_type_signature(&*inner), size) + } + Tuple(components) => format!( + "({})", + components + .iter() + .map(|component| event_param_type_signature(&component)) + .collect::>() + .join(",") + ), + } + } + + self.contract_abi + .contract + .events() + .find(|event| event_signature(event) == signature) + .or_else(|| { + // Fallback for subgraphs that don't use `indexed` in event signatures yet: + // + // If there is only one event variant with this name and if its signature + // without `indexed` matches the event signature from the manifest, we + // can safely assume that the event is a match, we don't need to force + // the subgraph to add `indexed`. + + // Extract the event name; if there is no '(' in the signature, + // `event_name` will be empty and not match any events, so that's ok + let parens = signature.find('(').unwrap_or(0); + let event_name = &signature[0..parens]; + + let matching_events = self + .contract_abi + .contract + .events() + .filter(|event| event.name == event_name) + .collect::>(); + + // Only match the event signature without `indexed` if there is + // only a single event variant + if matching_events.len() == 1 + && ambiguous_event_signature(matching_events[0]) == signature + { + Some(matching_events[0]) + } else { + // More than one event variant or the signature + // still doesn't match, even if we ignore `indexed` hints + None + } + }) + } + + fn contract_function_with_signature(&self, target_signature: &str) -> Option<&Function> { + self.contract_abi + .contract + .functions() + .filter(|function| match function.state_mutability { + StateMutability::Payable | StateMutability::NonPayable => true, + StateMutability::Pure | StateMutability::View => false, + }) + .find(|function| { + // Construct the argument function signature: + // `address,uint256,bool` + let mut arguments = function + .inputs + .iter() + .map(|input| format!("{}", input.kind)) + .collect::>() + .join(","); + // `address,uint256,bool) + arguments.push_str(")"); + // `operation(address,uint256,bool)` + let actual_signature = vec![function.name.clone(), arguments].join("("); + target_signature == actual_signature + }) + } + + fn matches_trigger_address(&self, trigger: &EthereumTrigger) -> bool { + let ds_address = match self.address { + Some(addr) => addr, + + // 'wildcard' data sources match any trigger address. + None => return true, + }; + + let trigger_address = match trigger { + EthereumTrigger::Block(_, EthereumBlockTriggerType::WithCallTo(address)) => address, + EthereumTrigger::Call(call) => &call.to, + EthereumTrigger::Log(log, _) => &log.address, + + // Unfiltered block triggers match any data source address. + EthereumTrigger::Block(_, EthereumBlockTriggerType::Every) => return true, + }; + + ds_address == *trigger_address + } + + /// Checks if `trigger` matches this data source, and if so decodes it into a `MappingTrigger`. + /// A return of `Ok(None)` mean the trigger does not match. + fn match_and_decode( + &self, + trigger: &EthereumTrigger, + block: &Arc, + logger: &Logger, + ) -> Result>, Error> { + if !self.matches_trigger_address(&trigger) { + return Ok(None); + } + + if self.start_block > block.number() { + return Ok(None); + } + + match trigger { + EthereumTrigger::Block(_, trigger_type) => { + let handler = match self.handler_for_block(trigger_type) { + Some(handler) => handler, + None => return Ok(None), + }; + Ok(Some(TriggerWithHandler::::new( + MappingTrigger::Block { + block: block.cheap_clone(), + }, + handler.handler, + block.block_ptr(), + ))) + } + EthereumTrigger::Log(log, receipt) => { + let potential_handlers = self.handlers_for_log(log)?; + + // Map event handlers to (event handler, event ABI) pairs; fail if there are + // handlers that don't exist in the contract ABI + let valid_handlers = potential_handlers + .into_iter() + .map(|event_handler| { + // Identify the event ABI in the contract + let event_abi = self + .contract_event_with_signature(event_handler.event.as_str()) + .with_context(|| { + anyhow!( + "Event with the signature \"{}\" not found in \ + contract \"{}\" of data source \"{}\"", + event_handler.event, + self.contract_abi.name, + self.name, + ) + })?; + Ok((event_handler, event_abi)) + }) + .collect::, anyhow::Error>>()?; + + // Filter out handlers whose corresponding event ABIs cannot decode the + // params (this is common for overloaded events that have the same topic0 + // but have indexed vs. non-indexed params that are encoded differently). + // + // Map (handler, event ABI) pairs to (handler, decoded params) pairs. + let mut matching_handlers = valid_handlers + .into_iter() + .filter_map(|(event_handler, event_abi)| { + event_abi + .parse_log(RawLog { + topics: log.topics.clone(), + data: log.data.clone().0, + }) + .map(|log| log.params) + .map_err(|e| { + trace!( + logger, + "Skipping handler because the event parameters do not \ + match the event signature. This is typically the case \ + when parameters are indexed in the event but not in the \ + signature or the other way around"; + "handler" => &event_handler.handler, + "event" => &event_handler.event, + "error" => format!("{}", e), + ); + }) + .ok() + .map(|params| (event_handler, params)) + }) + .collect::>(); + + if matching_handlers.is_empty() { + return Ok(None); + } + + // Process the event with the matching handler + let (event_handler, params) = matching_handlers.pop().unwrap(); + + ensure!( + matching_handlers.is_empty(), + format!( + "Multiple handlers defined for event `{}`, only one is supported", + &event_handler.event + ) + ); + + // Special case: In Celo, there are Epoch Rewards events, which do not have an + // associated transaction and instead have `transaction_hash == block.hash`, + // in which case we pass a dummy transaction to the mappings. + // See also ca0edc58-0ec5-4c89-a7dd-2241797f5e50. + let transaction = if log.transaction_hash != block.hash { + block + .transaction_for_log(&log) + .context("Found no transaction for event")? + } else { + // Infer some fields from the log and fill the rest with zeros. + Transaction { + hash: log.transaction_hash.unwrap(), + block_hash: block.hash, + block_number: block.number, + transaction_index: log.transaction_index, + from: Some(H160::zero()), + ..Transaction::default() + } + }; + + let logging_extras = Arc::new(o! { + "signature" => event_handler.event.to_string(), + "address" => format!("{}", &log.address), + "transaction" => format!("{}", &transaction.hash), + }); + Ok(Some(TriggerWithHandler::::new_with_logging_extras( + MappingTrigger::Log { + block: block.cheap_clone(), + transaction: Arc::new(transaction), + log: log.cheap_clone(), + params, + receipt: receipt.clone(), + }, + event_handler.handler, + block.block_ptr(), + logging_extras, + ))) + } + EthereumTrigger::Call(call) => { + // Identify the call handler for this call + let handler = match self.handler_for_call(&call)? { + Some(handler) => handler, + None => return Ok(None), + }; + + // Identify the function ABI in the contract + let function_abi = self + .contract_function_with_signature(handler.function.as_str()) + .with_context(|| { + anyhow!( + "Function with the signature \"{}\" not found in \ + contract \"{}\" of data source \"{}\"", + handler.function, + self.contract_abi.name, + self.name + ) + })?; + + // Parse the inputs + // + // Take the input for the call, chop off the first 4 bytes, then call + // `function.decode_input` to get a vector of `Token`s. Match the `Token`s + // with the `Param`s in `function.inputs` to create a `Vec`. + let tokens = match function_abi.decode_input(&call.input.0[4..]).with_context( + || { + format!( + "Generating function inputs for the call {:?} failed, raw input: {}", + &function_abi, + hex::encode(&call.input.0) + ) + }, + ) { + Ok(val) => val, + // See also 280b0108-a96e-4738-bb37-60ce11eeb5bf + Err(err) => { + warn!(logger, "Failed parsing inputs, skipping"; "error" => &err.to_string()); + return Ok(None); + } + }; + + ensure!( + tokens.len() == function_abi.inputs.len(), + "Number of arguments in call does not match \ + number of inputs in function signature." + ); + + let inputs = tokens + .into_iter() + .enumerate() + .map(|(i, token)| LogParam { + name: function_abi.inputs[i].name.clone(), + value: token, + }) + .collect::>(); + + // Parse the outputs + // + // Take the output for the call, then call `function.decode_output` to + // get a vector of `Token`s. Match the `Token`s with the `Param`s in + // `function.outputs` to create a `Vec`. + let tokens = function_abi + .decode_output(&call.output.0) + .with_context(|| { + format!( + "Decoding function outputs for the call {:?} failed, raw output: {}", + &function_abi, + hex::encode(&call.output.0) + ) + })?; + + ensure!( + tokens.len() == function_abi.outputs.len(), + "Number of parameters in the call output does not match \ + number of outputs in the function signature." + ); + + let outputs = tokens + .into_iter() + .enumerate() + .map(|(i, token)| LogParam { + name: function_abi.outputs[i].name.clone(), + value: token, + }) + .collect::>(); + + let transaction = Arc::new( + block + .transaction_for_call(&call) + .context("Found no transaction for call")?, + ); + let logging_extras = Arc::new(o! { + "function" => handler.function.to_string(), + "to" => format!("{}", &call.to), + "transaction" => format!("{}", &transaction.hash), + }); + Ok(Some(TriggerWithHandler::::new_with_logging_extras( + MappingTrigger::Call { + block: block.cheap_clone(), + transaction, + call: call.cheap_clone(), + inputs, + outputs, + }, + handler.handler, + block.block_ptr(), + logging_extras, + ))) + } + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub struct UnresolvedDataSource { + pub kind: String, + pub network: Option, + pub name: String, + pub source: Source, + pub mapping: UnresolvedMapping, + pub context: Option, +} + +#[async_trait] +impl blockchain::UnresolvedDataSource for UnresolvedDataSource { + async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + manifest_idx: u32, + ) -> Result { + let UnresolvedDataSource { + kind, + network, + name, + source, + mapping, + context, + } = self; + + info!(logger, "Resolve data source"; "name" => &name, "source_address" => format_args!("{:?}", source.address), "source_start_block" => source.start_block); + + let mapping = mapping.resolve(&*resolver, logger).await?; + + DataSource::from_manifest(kind, network, name, source, mapping, context, manifest_idx) + } +} + +impl TryFrom> for DataSource { + type Error = anyhow::Error; + + fn try_from(info: DataSourceTemplateInfo) -> Result { + let DataSourceTemplateInfo { + template, + params, + context, + creation_block, + } = info; + let template = template.into_onchain().ok_or(anyhow!( + "Cannot create onchain data source from offchain template" + ))?; + + // Obtain the address from the parameters + let string = params + .get(0) + .with_context(|| { + format!( + "Failed to create data source from template `{}`: address parameter is missing", + template.name + ) + })? + .trim_start_matches("0x"); + + let address = Address::from_str(string).with_context(|| { + format!( + "Failed to create data source from template `{}`, invalid address provided", + template.name + ) + })?; + + let contract_abi = template + .mapping + .find_abi(&template.source.abi) + .with_context(|| format!("template `{}`", template.name))?; + + Ok(DataSource { + kind: template.kind, + network: template.network, + name: template.name, + manifest_idx: template.manifest_idx, + address: Some(address), + start_block: 0, + mapping: template.mapping, + context: Arc::new(context), + creation_block: Some(creation_block), + contract_abi, + }) + } +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +pub struct UnresolvedDataSourceTemplate { + pub kind: String, + pub network: Option, + pub name: String, + pub source: TemplateSource, + pub mapping: UnresolvedMapping, +} + +#[derive(Clone, Debug)] +pub struct DataSourceTemplate { + pub kind: String, + pub network: Option, + pub name: String, + pub manifest_idx: u32, + pub source: TemplateSource, + pub mapping: Mapping, +} + +#[async_trait] +impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { + async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + manifest_idx: u32, + ) -> Result { + let UnresolvedDataSourceTemplate { + kind, + network, + name, + source, + mapping, + } = self; + + info!(logger, "Resolve data source template"; "name" => &name); + + Ok(DataSourceTemplate { + kind, + network, + name, + manifest_idx, + source, + mapping: mapping.resolve(resolver, logger).await?, + }) + } +} + +impl blockchain::DataSourceTemplate for DataSourceTemplate { + fn name(&self) -> &str { + &self.name + } + + fn api_version(&self) -> semver::Version { + self.mapping.api_version.clone() + } + + fn runtime(&self) -> Option>> { + Some(self.mapping.runtime.cheap_clone()) + } + + fn manifest_idx(&self) -> u32 { + self.manifest_idx + } +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UnresolvedMapping { + pub kind: String, + pub api_version: String, + pub language: String, + pub entities: Vec, + pub abis: Vec, + #[serde(default)] + pub block_handlers: Vec, + #[serde(default)] + pub call_handlers: Vec, + #[serde(default)] + pub event_handlers: Vec, + pub file: Link, +} + +#[derive(Clone, Debug)] +pub struct Mapping { + pub kind: String, + pub api_version: semver::Version, + pub language: String, + pub entities: Vec, + pub abis: Vec>, + pub block_handlers: Vec, + pub call_handlers: Vec, + pub event_handlers: Vec, + pub runtime: Arc>, + pub link: Link, +} + +impl Mapping { + pub fn requires_archive(&self) -> anyhow::Result { + calls_host_fn(&self.runtime, "ethereum.call") + } + + pub fn has_call_handler(&self) -> bool { + !self.call_handlers.is_empty() + } + + pub fn has_block_handler_with_call_filter(&self) -> bool { + self.block_handlers + .iter() + .any(|handler| matches!(handler.filter, Some(BlockHandlerFilter::Call))) + } + + pub fn find_abi(&self, abi_name: &str) -> Result, Error> { + Ok(self + .abis + .iter() + .find(|abi| abi.name == abi_name) + .ok_or_else(|| anyhow!("No ABI entry with name `{}` found", abi_name))? + .cheap_clone()) + } +} + +impl UnresolvedMapping { + pub async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + ) -> Result { + let UnresolvedMapping { + kind, + api_version, + language, + entities, + abis, + block_handlers, + call_handlers, + event_handlers, + file: link, + } = self; + + info!(logger, "Resolve mapping"; "link" => &link.link); + + let api_version = semver::Version::parse(&api_version)?; + + let (abis, runtime) = try_join( + // resolve each abi + abis.into_iter() + .map(|unresolved_abi| async { + Result::<_, Error>::Ok(Arc::new( + unresolved_abi.resolve(resolver, logger).await?, + )) + }) + .collect::>() + .try_collect::>(), + async { + let module_bytes = resolver.cat(logger, &link).await?; + Ok(Arc::new(module_bytes)) + }, + ) + .await?; + + Ok(Mapping { + kind, + api_version, + language, + entities, + abis, + block_handlers: block_handlers.clone(), + call_handlers: call_handlers.clone(), + event_handlers: event_handlers.clone(), + runtime, + link, + }) + } +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct UnresolvedMappingABI { + pub name: String, + pub file: Link, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct MappingABI { + pub name: String, + pub contract: Contract, +} + +impl UnresolvedMappingABI { + pub async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + ) -> Result { + info!( + logger, + "Resolve ABI"; + "name" => &self.name, + "link" => &self.file.link + ); + + let contract_bytes = resolver.cat(&logger, &self.file).await?; + let contract = Contract::load(&*contract_bytes)?; + Ok(MappingABI { + name: self.name, + contract, + }) + } +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct MappingBlockHandler { + pub handler: String, + pub filter: Option, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum BlockHandlerFilter { + // Call filter will trigger on all blocks where the data source contract + // address has been called + Call, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct MappingCallHandler { + pub function: String, + pub handler: String, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct MappingEventHandler { + pub event: String, + pub topic0: Option, + pub handler: String, + #[serde(default)] + pub receipt: bool, +} + +impl MappingEventHandler { + pub fn topic0(&self) -> H256 { + self.topic0 + .unwrap_or_else(|| string_to_h256(&self.event.replace("indexed ", ""))) + } +} + +/// Hashes a string to a H256 hash. +fn string_to_h256(s: &str) -> H256 { + let mut result = [0u8; 32]; + let data = s.replace(" ", "").into_bytes(); + let mut sponge = Keccak::new_keccak256(); + sponge.update(&data); + sponge.finalize(&mut result); + + // This was deprecated but the replacement seems to not be available in the + // version web3 uses. + #[allow(deprecated)] + H256::from_slice(&result) +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +pub struct TemplateSource { + pub abi: String, +} diff --git a/chain/ethereum/src/env.rs b/chain/ethereum/src/env.rs new file mode 100644 index 0000000..b6c111b --- /dev/null +++ b/chain/ethereum/src/env.rs @@ -0,0 +1,184 @@ +use envconfig::Envconfig; +use graph::env::EnvVarBoolean; +use graph::prelude::{envconfig, lazy_static, BlockNumber}; +use std::fmt; +use std::time::Duration; + +lazy_static! { + pub static ref ENV_VARS: EnvVars = EnvVars::from_env().unwrap(); +} + +#[derive(Clone)] +#[non_exhaustive] +pub struct EnvVars { + /// Controls if firehose should be preferred over RPC if Firehose endpoints + /// are present, if not set, the default behavior is is kept which is to + /// automatically favor Firehose. + /// + /// Set by the flag `GRAPH_ETHEREUM_IS_FIREHOSE_PREFERRED`. On by default. + pub is_firehose_preferred: bool, + /// Additional deterministic errors that have not yet been hardcoded. + /// + /// Set by the environment variable `GRAPH_GETH_ETH_CALL_ERRORS`, separated + /// by `;`. + pub geth_eth_call_errors: Vec, + /// Set by the environment variable `GRAPH_ETH_GET_LOGS_MAX_CONTRACTS`. The + /// default value is 2000. + pub get_logs_max_contracts: usize, + + /// Set by the environment variable `ETHEREUM_REORG_THRESHOLD`. The default + /// value is 250 blocks. + pub reorg_threshold: BlockNumber, + /// Set by the environment variable `ETHEREUM_TRACE_STREAM_STEP_SIZE`. The + /// default value is 50 blocks. + pub trace_stream_step_size: BlockNumber, + /// Maximum range size for `eth.getLogs` requests that don't filter on + /// contract address, only event signature, and are therefore expensive. + /// + /// Set by the environment variable `GRAPH_ETHEREUM_MAX_EVENT_ONLY_RANGE`. The + /// default value is 500 blocks, which is reasonable according to Ethereum + /// node operators. + pub max_event_only_range: BlockNumber, + /// Set by the environment variable `ETHEREUM_BLOCK_BATCH_SIZE`. The + /// default value is 10 blocks. + pub block_batch_size: usize, + /// Maximum number of blocks to request in each chunk. + /// + /// Set by the environment variable `GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE`. + /// The default value is 2000 blocks. + pub max_block_range_size: BlockNumber, + /// This should not be too large that it causes requests to timeout without + /// us catching it, nor too small that it causes us to timeout requests that + /// would've succeeded. We've seen successful `eth_getLogs` requests take + /// over 120 seconds. + /// + /// Set by the environment variable `GRAPH_ETHEREUM_JSON_RPC_TIMEOUT` + /// (expressed in seconds). The default value is 180s. + pub json_rpc_timeout: Duration, + /// This is used for requests that will not fail the subgraph if the limit + /// is reached, but will simply restart the syncing step, so it can be low. + /// This limit guards against scenarios such as requesting a block hash that + /// has been reorged. + /// + /// Set by the environment variable `GRAPH_ETHEREUM_REQUEST_RETRIES`. The + /// default value is 10. + pub request_retries: usize, + /// Set by the environment variable + /// `GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS`. + /// The default value is 1000. + pub block_ingestor_max_concurrent_json_rpc_calls: usize, + /// Set by the flag `GRAPH_ETHEREUM_FETCH_TXN_RECEIPTS_IN_BATCHES`. Enabled + /// by default on macOS (to avoid DNS issues) and disabled by default on all + /// other systems. + pub fetch_receipts_in_batches: bool, + /// `graph_node::config` disallows setting this in a store with multiple + /// shards. See 8b6ad0c64e244023ac20ced7897fe666 for the reason. + /// + /// Set by the flag `GRAPH_ETHEREUM_CLEANUP_BLOCKS`. Off by default. + pub cleanup_blocks: bool, + /// Ideal number of triggers in a range. The range size will adapt to try to + /// meet this. + /// + /// Set by the environment variable + /// `GRAPH_ETHEREUM_TARGET_TRIGGERS_PER_BLOCK_RANGE`. The default value is + /// 100. + pub target_triggers_per_block_range: u64, + /// These are some chains, the genesis block is start from 1 not 0. If this + /// flag is not set, the default value will be 0. + /// + /// Set by the flag `GRAPH_ETHEREUM_GENESIS_BLOCK_NUMBER`. The default value + /// is 0. + pub genesis_block_number: u64, +} + +// This does not print any values avoid accidentally leaking any sensitive env vars +impl fmt::Debug for EnvVars { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "env vars") + } +} + +impl EnvVars { + pub fn from_env() -> Result { + Ok(Inner::init_from_env()?.into()) + } +} + +impl From for EnvVars { + fn from(x: Inner) -> Self { + Self { + is_firehose_preferred: x.is_firehose_preferred.0, + get_logs_max_contracts: x.get_logs_max_contracts, + geth_eth_call_errors: x + .geth_eth_call_errors + .split(';') + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect(), + reorg_threshold: x.reorg_threshold, + trace_stream_step_size: x.trace_stream_step_size, + max_event_only_range: x.max_event_only_range, + block_batch_size: x.block_batch_size, + max_block_range_size: x.max_block_range_size, + json_rpc_timeout: Duration::from_secs(x.json_rpc_timeout_in_secs), + request_retries: x.request_retries, + block_ingestor_max_concurrent_json_rpc_calls: x + .block_ingestor_max_concurrent_json_rpc_calls, + fetch_receipts_in_batches: x + .fetch_receipts_in_batches + .map(|b| b.0) + .unwrap_or(cfg!(target_os = "macos")), + cleanup_blocks: x.cleanup_blocks.0, + target_triggers_per_block_range: x.target_triggers_per_block_range, + genesis_block_number: x.genesis_block_number, + } + } +} + +impl Default for EnvVars { + fn default() -> Self { + ENV_VARS.clone() + } +} + +#[derive(Clone, Debug, Envconfig)] +struct Inner { + #[envconfig(from = "GRAPH_ETHEREUM_IS_FIREHOSE_PREFERRED", default = "true")] + is_firehose_preferred: EnvVarBoolean, + #[envconfig(from = "GRAPH_GETH_ETH_CALL_ERRORS", default = "")] + geth_eth_call_errors: String, + #[envconfig(from = "GRAPH_ETH_GET_LOGS_MAX_CONTRACTS", default = "2000")] + get_logs_max_contracts: usize, + + // JSON-RPC specific. + #[envconfig(from = "ETHEREUM_REORG_THRESHOLD", default = "250")] + reorg_threshold: BlockNumber, + #[envconfig(from = "ETHEREUM_TRACE_STREAM_STEP_SIZE", default = "50")] + trace_stream_step_size: BlockNumber, + #[envconfig(from = "GRAPH_ETHEREUM_MAX_EVENT_ONLY_RANGE", default = "500")] + max_event_only_range: BlockNumber, + #[envconfig(from = "ETHEREUM_BLOCK_BATCH_SIZE", default = "10")] + block_batch_size: usize, + #[envconfig(from = "GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE", default = "2000")] + max_block_range_size: BlockNumber, + #[envconfig(from = "GRAPH_ETHEREUM_JSON_RPC_TIMEOUT", default = "180")] + json_rpc_timeout_in_secs: u64, + #[envconfig(from = "GRAPH_ETHEREUM_REQUEST_RETRIES", default = "10")] + request_retries: usize, + #[envconfig( + from = "GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS", + default = "1000" + )] + block_ingestor_max_concurrent_json_rpc_calls: usize, + #[envconfig(from = "GRAPH_ETHEREUM_FETCH_TXN_RECEIPTS_IN_BATCHES")] + fetch_receipts_in_batches: Option, + #[envconfig(from = "GRAPH_ETHEREUM_CLEANUP_BLOCKS", default = "false")] + cleanup_blocks: EnvVarBoolean, + #[envconfig( + from = "GRAPH_ETHEREUM_TARGET_TRIGGERS_PER_BLOCK_RANGE", + default = "100" + )] + target_triggers_per_block_range: u64, + #[envconfig(from = "GRAPH_ETHEREUM_GENESIS_BLOCK_NUMBER", default = "0")] + genesis_block_number: u64, +} diff --git a/chain/ethereum/src/ethereum_adapter.rs b/chain/ethereum/src/ethereum_adapter.rs new file mode 100644 index 0000000..e31d2b6 --- /dev/null +++ b/chain/ethereum/src/ethereum_adapter.rs @@ -0,0 +1,2162 @@ +use futures::future; +use futures::prelude::*; +use futures03::{future::BoxFuture, stream::FuturesUnordered}; +use graph::blockchain::BlockHash; +use graph::blockchain::ChainIdentifier; +use graph::components::transaction_receipt::LightTransactionReceipt; +use graph::data::subgraph::UnifiedMappingApiVersion; +use graph::data::subgraph::API_VERSION_0_0_5; +use graph::data::subgraph::API_VERSION_0_0_7; +use graph::prelude::ethabi::ParamType; +use graph::prelude::ethabi::Token; +use graph::prelude::tokio::try_join; +use graph::{ + blockchain::{block_stream::BlockWithTriggers, BlockPtr, IngestorError}, + prelude::{ + anyhow::{self, anyhow, bail, ensure, Context}, + async_trait, debug, error, ethabi, + futures03::{self, compat::Future01CompatExt, FutureExt, StreamExt, TryStreamExt}, + hex, info, retry, serde_json as json, stream, tiny_keccak, trace, warn, + web3::{ + self, + types::{ + Address, BlockId, BlockNumber as Web3BlockNumber, Bytes, CallRequest, Filter, + FilterBuilder, Log, Transaction, TransactionReceipt, H256, + }, + }, + BlockNumber, ChainStore, CheapClone, DynTryFuture, Error, EthereumCallCache, Logger, + TimeoutError, TryFutureExt, + }, +}; +use graph::{ + components::ethereum::*, + prelude::web3::api::Web3, + prelude::web3::transports::Batch, + prelude::web3::types::{Trace, TraceFilter, TraceFilterBuilder, H160}, +}; +use itertools::Itertools; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::convert::TryFrom; +use std::iter::FromIterator; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Instant; + +use crate::adapter::ProviderStatus; +use crate::chain::BlockFinality; +use crate::{ + adapter::{ + EthGetLogsFilter, EthereumAdapter as EthereumAdapterTrait, EthereumBlockFilter, + EthereumCallFilter, EthereumContractCall, EthereumContractCallError, EthereumLogFilter, + ProviderEthRpcMetrics, SubgraphEthRpcMetrics, + }, + transport::Transport, + trigger::{EthereumBlockTriggerType, EthereumTrigger}, + TriggerFilter, ENV_VARS, +}; + +#[derive(Clone)] +pub struct EthereumAdapter { + logger: Logger, + url_hostname: Arc, + /// The label for the provider from the configuration + provider: String, + web3: Arc>, + metrics: Arc, + supports_eip_1898: bool, +} + +/// Gas limit for `eth_call`. The value of 50_000_000 is a protocol-wide parameter so this +/// should be changed only for debugging purposes and never on an indexer in the network. This +/// value was chosen because it is the Geth default +/// https://github.com/ethereum/go-ethereum/blob/e4b687cf462870538743b3218906940ae590e7fd/eth/ethconfig/config.go#L91. +/// It is not safe to set something higher because Geth will silently override the gas limit +/// with the default. This means that we do not support indexing against a Geth node with +/// `RPCGasCap` set below 50 million. +// See also f0af4ab0-6b7c-4b68-9141-5b79346a5f61. +const ETH_CALL_GAS: u32 = 50_000_000; + +impl CheapClone for EthereumAdapter { + fn cheap_clone(&self) -> Self { + Self { + logger: self.logger.clone(), + provider: self.provider.clone(), + url_hostname: self.url_hostname.cheap_clone(), + web3: self.web3.cheap_clone(), + metrics: self.metrics.cheap_clone(), + supports_eip_1898: self.supports_eip_1898, + } + } +} + +impl EthereumAdapter { + pub async fn new( + logger: Logger, + provider: String, + url: &str, + transport: Transport, + provider_metrics: Arc, + supports_eip_1898: bool, + ) -> Self { + // Unwrap: The transport was constructed with this url, so it is valid and has a host. + let hostname = graph::url::Url::parse(url) + .unwrap() + .host_str() + .unwrap() + .to_string(); + + let web3 = Arc::new(Web3::new(transport)); + + // Use the client version to check if it is ganache. For compatibility with unit tests, be + // are lenient with errors, defaulting to false. + let is_ganache = web3 + .web3() + .client_version() + .await + .map(|s| s.contains("TestRPC")) + .unwrap_or(false); + + EthereumAdapter { + logger, + provider, + url_hostname: Arc::new(hostname), + web3, + metrics: provider_metrics, + supports_eip_1898: supports_eip_1898 && !is_ganache, + } + } + + async fn traces( + self, + logger: Logger, + subgraph_metrics: Arc, + from: BlockNumber, + to: BlockNumber, + addresses: Vec, + ) -> Result, Error> { + let eth = self.clone(); + let retry_log_message = + format!("trace_filter RPC call for block range: [{}..{}]", from, to); + retry(retry_log_message, &logger) + .limit(ENV_VARS.request_retries) + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || { + let trace_filter: TraceFilter = match addresses.len() { + 0 => TraceFilterBuilder::default() + .from_block(from.into()) + .to_block(to.into()) + .build(), + _ => TraceFilterBuilder::default() + .from_block(from.into()) + .to_block(to.into()) + .to_address(addresses.clone()) + .build(), + }; + + let eth = eth.cheap_clone(); + let logger_for_triggers = logger.clone(); + let logger_for_error = logger.clone(); + let start = Instant::now(); + let subgraph_metrics = subgraph_metrics.clone(); + let provider_metrics = eth.metrics.clone(); + let provider = self.provider.clone(); + + async move { + let result = eth + .web3 + .trace() + .filter(trace_filter) + .await + .map(move |traces| { + if traces.len() > 0 { + if to == from { + debug!( + logger_for_triggers, + "Received {} traces for block {}", + traces.len(), + to + ); + } else { + debug!( + logger_for_triggers, + "Received {} traces for blocks [{}, {}]", + traces.len(), + from, + to + ); + } + } + traces + }) + .map_err(Error::from); + + let elapsed = start.elapsed().as_secs_f64(); + provider_metrics.observe_request(elapsed, "trace_filter", &provider); + subgraph_metrics.observe_request(elapsed, "trace_filter", &provider); + if let Err(e) = &result { + provider_metrics.add_error("trace_filter", &provider); + subgraph_metrics.add_error("trace_filter", &provider); + debug!( + logger_for_error, + "Error querying traces error = {:#} from = {} to = {}", e, from, to + ); + } + result + } + }) + .map_err(move |e| { + e.into_inner().unwrap_or_else(move || { + anyhow::anyhow!( + "Ethereum node took too long to respond to trace_filter \ + (from block {}, to block {})", + from, + to + ) + }) + }) + .await + } + + async fn logs_with_sigs( + &self, + logger: Logger, + subgraph_metrics: Arc, + from: BlockNumber, + to: BlockNumber, + filter: Arc, + too_many_logs_fingerprints: &'static [&'static str], + ) -> Result, TimeoutError> { + let eth_adapter = self.clone(); + let retry_log_message = format!("eth_getLogs RPC call for block range: [{}..{}]", from, to); + retry(retry_log_message, &logger) + .when(move |res: &Result<_, web3::error::Error>| match res { + Ok(_) => false, + Err(e) => !too_many_logs_fingerprints + .iter() + .any(|f| e.to_string().contains(f)), + }) + .limit(ENV_VARS.request_retries) + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || { + let eth_adapter = eth_adapter.cheap_clone(); + let subgraph_metrics = subgraph_metrics.clone(); + let provider_metrics = eth_adapter.metrics.clone(); + let filter = filter.clone(); + let provider = eth_adapter.provider.clone(); + + async move { + let start = Instant::now(); + + // Create a log filter + let log_filter: Filter = FilterBuilder::default() + .from_block(from.into()) + .to_block(to.into()) + .address(filter.contracts.clone()) + .topics(Some(filter.event_signatures.clone()), None, None, None) + .build(); + + // Request logs from client + let result = eth_adapter.web3.eth().logs(log_filter).boxed().await; + let elapsed = start.elapsed().as_secs_f64(); + provider_metrics.observe_request(elapsed, "eth_getLogs", &provider); + subgraph_metrics.observe_request(elapsed, "eth_getLogs", &provider); + if result.is_err() { + provider_metrics.add_error("eth_getLogs", &provider); + subgraph_metrics.add_error("eth_getLogs", &provider); + } + result + } + }) + .await + } + + fn trace_stream( + self, + logger: &Logger, + subgraph_metrics: Arc, + from: BlockNumber, + to: BlockNumber, + addresses: Vec, + ) -> impl Stream + Send { + if from > to { + panic!( + "Can not produce a call stream on a backwards block range: from = {}, to = {}", + from, to, + ); + } + + // Go one block at a time if requesting all traces, to not overload the RPC. + let step_size = match addresses.is_empty() { + false => ENV_VARS.trace_stream_step_size, + true => 1, + }; + + let eth = self.clone(); + let logger = logger.to_owned(); + stream::unfold(from, move |start| { + if start > to { + return None; + } + let end = (start + step_size - 1).min(to); + let new_start = end + 1; + if start == end { + debug!(logger, "Requesting traces for block {}", start); + } else { + debug!(logger, "Requesting traces for blocks [{}, {}]", start, end); + } + Some(futures::future::ok(( + eth.clone() + .traces( + logger.cheap_clone(), + subgraph_metrics.clone(), + start, + end, + addresses.clone(), + ) + .boxed() + .compat(), + new_start, + ))) + }) + .buffered(ENV_VARS.block_batch_size) + .map(stream::iter_ok) + .flatten() + } + + fn log_stream( + &self, + logger: Logger, + subgraph_metrics: Arc, + from: BlockNumber, + to: BlockNumber, + filter: EthGetLogsFilter, + ) -> DynTryFuture<'static, Vec, Error> { + // Codes returned by Ethereum node providers if an eth_getLogs request is too heavy. + // The first one is for Infura when it hits the log limit, the rest for Alchemy timeouts. + const TOO_MANY_LOGS_FINGERPRINTS: &[&str] = &[ + "ServerError(-32005)", + "503 Service Unavailable", + "ServerError(-32000)", + ]; + + if from > to { + panic!( + "cannot produce a log stream on a backwards block range (from={}, to={})", + from, to + ); + } + + // Collect all event sigs + let eth = self.cheap_clone(); + let filter = Arc::new(filter); + + let step = match filter.contracts.is_empty() { + // `to - from + 1` blocks will be scanned. + false => to - from, + true => (to - from).min(ENV_VARS.max_event_only_range - 1), + }; + + // Typically this will loop only once and fetch the entire range in one request. But if the + // node returns an error that signifies the request is to heavy to process, the range will + // be broken down to smaller steps. + futures03::stream::try_unfold((from, step), move |(start, step)| { + let logger = logger.cheap_clone(); + let filter = filter.cheap_clone(); + let eth = eth.cheap_clone(); + let subgraph_metrics = subgraph_metrics.cheap_clone(); + + async move { + if start > to { + return Ok(None); + } + + let end = (start + step).min(to); + debug!( + logger, + "Requesting logs for blocks [{}, {}], {}", start, end, filter + ); + let res = eth + .logs_with_sigs( + logger.cheap_clone(), + subgraph_metrics.cheap_clone(), + start, + end, + filter.cheap_clone(), + TOO_MANY_LOGS_FINGERPRINTS, + ) + .await; + + match res { + Err(e) => { + let string_err = e.to_string(); + + // If the step is already 0, the request is too heavy even for a single + // block. We hope this never happens, but if it does, make sure to error. + if TOO_MANY_LOGS_FINGERPRINTS + .iter() + .any(|f| string_err.contains(f)) + && step > 0 + { + // The range size for a request is `step + 1`. So it's ok if the step + // goes down to 0, in that case we'll request one block at a time. + let new_step = step / 10; + debug!(logger, "Reducing block range size to scan for events"; + "new_size" => new_step + 1); + Ok(Some((vec![], (start, new_step)))) + } else { + warn!(logger, "Unexpected RPC error"; "error" => &string_err); + Err(anyhow!("{}", string_err)) + } + } + Ok(logs) => Ok(Some((logs, (end + 1, step)))), + } + } + }) + .try_concat() + .boxed() + } + + fn call( + &self, + logger: Logger, + contract_address: Address, + call_data: Bytes, + block_ptr: BlockPtr, + ) -> impl Future + Send { + let web3 = self.web3.clone(); + + // Ganache does not support calls by block hash. + // See https://github.com/trufflesuite/ganache-cli/issues/973 + let block_id = if !self.supports_eip_1898 { + BlockId::Number(block_ptr.number.into()) + } else { + BlockId::Hash(block_ptr.hash_as_h256()) + }; + let retry_log_message = format!("eth_call RPC call for block {}", block_ptr); + retry(retry_log_message, &logger) + .when(|result| match result { + Ok(_) | Err(EthereumContractCallError::Revert(_)) => false, + Err(_) => true, + }) + .limit(ENV_VARS.request_retries) + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || { + let call_data = call_data.clone(); + let web3 = web3.cheap_clone(); + + async move { + let req = CallRequest { + to: Some(contract_address), + gas: Some(web3::types::U256::from(ETH_CALL_GAS)), + data: Some(call_data.clone()), + from: None, + gas_price: None, + value: None, + access_list: None, + max_fee_per_gas: None, + max_priority_fee_per_gas: None, + transaction_type: None, + }; + let result = web3.eth().call(req, Some(block_id)).boxed().await; + + // Try to check if the call was reverted. The JSON-RPC response for reverts is + // not standardized, so we have ad-hoc checks for each Ethereum client. + + // 0xfe is the "designated bad instruction" of the EVM, and Solidity uses it for + // asserts. + const PARITY_BAD_INSTRUCTION_FE: &str = "Bad instruction fe"; + + // 0xfd is REVERT, but on some contracts, and only on older blocks, + // this happens. Makes sense to consider it a revert as well. + const PARITY_BAD_INSTRUCTION_FD: &str = "Bad instruction fd"; + + const PARITY_BAD_JUMP_PREFIX: &str = "Bad jump"; + const PARITY_STACK_LIMIT_PREFIX: &str = "Out of stack"; + + // See f0af4ab0-6b7c-4b68-9141-5b79346a5f61. + const PARITY_OUT_OF_GAS: &str = "Out of gas"; + + const PARITY_VM_EXECUTION_ERROR: i64 = -32015; + const PARITY_REVERT_PREFIX: &str = "Reverted 0x"; + const XDAI_REVERT: &str = "revert"; + + // Deterministic Geth execution errors. We might need to expand this as + // subgraphs come across other errors. See + // https://github.com/ethereum/go-ethereum/blob/cd57d5cd38ef692de8fbedaa56598b4e9fbfbabc/core/vm/errors.go + const GETH_EXECUTION_ERRORS: &[&str] = &[ + // The "revert" substring covers a few known error messages, including: + // Hardhat: "error: transaction reverted", + // Ganache and Moonbeam: "vm exception while processing transaction: revert", + // Geth: "execution reverted" + // And others. + "revert", + "invalid jump destination", + "invalid opcode", + // Ethereum says 1024 is the stack sizes limit, so this is deterministic. + "stack limit reached 1024", + // See f0af4ab0-6b7c-4b68-9141-5b79346a5f61 for why the gas limit is considered deterministic. + "out of gas", + ]; + + let env_geth_call_errors = ENV_VARS.geth_eth_call_errors.iter(); + let mut geth_execution_errors = GETH_EXECUTION_ERRORS + .iter() + .map(|s| *s) + .chain(env_geth_call_errors.map(|s| s.as_str())); + + let as_solidity_revert_with_reason = |bytes: &[u8]| { + let solidity_revert_function_selector = + &tiny_keccak::keccak256(b"Error(string)")[..4]; + + match bytes.len() >= 4 && &bytes[..4] == solidity_revert_function_selector { + false => None, + true => ethabi::decode(&[ParamType::String], &bytes[4..]) + .ok() + .and_then(|tokens| tokens[0].clone().into_string()), + } + }; + + match result { + // A successful response. + Ok(bytes) => Ok(bytes), + + // Check for Geth revert. + Err(web3::Error::Rpc(rpc_error)) + if geth_execution_errors + .any(|e| rpc_error.message.to_lowercase().contains(e)) => + { + Err(EthereumContractCallError::Revert(rpc_error.message)) + } + + // Check for Parity revert. + Err(web3::Error::Rpc(ref rpc_error)) + if rpc_error.code.code() == PARITY_VM_EXECUTION_ERROR => + { + match rpc_error.data.as_ref().and_then(|d| d.as_str()) { + Some(data) + if data.starts_with(PARITY_REVERT_PREFIX) + || data.starts_with(PARITY_BAD_JUMP_PREFIX) + || data.starts_with(PARITY_STACK_LIMIT_PREFIX) + || data == PARITY_BAD_INSTRUCTION_FE + || data == PARITY_BAD_INSTRUCTION_FD + || data == PARITY_OUT_OF_GAS + || data == XDAI_REVERT => + { + let reason = if data == PARITY_BAD_INSTRUCTION_FE { + PARITY_BAD_INSTRUCTION_FE.to_owned() + } else { + let payload = data.trim_start_matches(PARITY_REVERT_PREFIX); + hex::decode(payload) + .ok() + .and_then(|payload| { + as_solidity_revert_with_reason(&payload) + }) + .unwrap_or("no reason".to_owned()) + }; + Err(EthereumContractCallError::Revert(reason)) + } + + // The VM execution error was not identified as a revert. + _ => Err(EthereumContractCallError::Web3Error(web3::Error::Rpc( + rpc_error.clone(), + ))), + } + } + + // The error was not identified as a revert. + Err(err) => Err(EthereumContractCallError::Web3Error(err)), + } + } + }) + .map_err(|e| e.into_inner().unwrap_or(EthereumContractCallError::Timeout)) + .boxed() + .compat() + } + + /// Request blocks by hash through JSON-RPC. + fn load_blocks_rpc( + &self, + logger: Logger, + ids: Vec, + ) -> impl Stream, Error = Error> + Send { + let web3 = self.web3.clone(); + + stream::iter_ok::<_, Error>(ids.into_iter().map(move |hash| { + let web3 = web3.clone(); + retry(format!("load block {}", hash), &logger) + .limit(ENV_VARS.request_retries) + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || { + Box::pin(web3.eth().block_with_txs(BlockId::Hash(hash))) + .compat() + .from_err::() + .and_then(move |block| { + block.map(|block| Arc::new(block)).ok_or_else(|| { + anyhow::anyhow!("Ethereum node did not find block {:?}", hash) + }) + }) + .compat() + }) + .boxed() + .compat() + .from_err() + })) + .buffered(ENV_VARS.block_batch_size) + } + + /// Request blocks ptrs for numbers through JSON-RPC. + /// + /// Reorg safety: If ids are numbers, they must be a final blocks. + fn load_block_ptrs_rpc( + &self, + logger: Logger, + block_nums: Vec, + ) -> impl Stream + Send { + let web3 = self.web3.clone(); + + stream::iter_ok::<_, Error>(block_nums.into_iter().map(move |block_num| { + let web3 = web3.clone(); + retry(format!("load block ptr {}", block_num), &logger) + .no_limit() + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || { + let web3 = web3.clone(); + async move { + let block = web3 + .eth() + .block(BlockId::Number(Web3BlockNumber::Number(block_num.into()))) + .boxed() + .await?; + + block.ok_or_else(|| { + anyhow!("Ethereum node did not find block {:?}", block_num) + }) + } + }) + .boxed() + .compat() + .from_err() + })) + .buffered(ENV_VARS.block_batch_size) + .map(|b| b.into()) + } + + /// Check if `block_ptr` refers to a block that is on the main chain, according to the Ethereum + /// node. + /// + /// Careful: don't use this function without considering race conditions. + /// Chain reorgs could happen at any time, and could affect the answer received. + /// Generally, it is only safe to use this function with blocks that have received enough + /// confirmations to guarantee no further reorgs, **and** where the Ethereum node is aware of + /// those confirmations. + /// If the Ethereum node is far behind in processing blocks, even old blocks can be subject to + /// reorgs. + pub(crate) async fn is_on_main_chain( + &self, + logger: &Logger, + block_ptr: BlockPtr, + ) -> Result { + let block_hash = self + .block_hash_by_block_number(&logger, block_ptr.number) + .compat() + .await?; + block_hash + .ok_or_else(|| anyhow!("Ethereum node is missing block #{}", block_ptr.number)) + .map(|block_hash| block_hash == block_ptr.hash_as_h256()) + } + + pub(crate) fn logs_in_block_range( + &self, + logger: &Logger, + subgraph_metrics: Arc, + from: BlockNumber, + to: BlockNumber, + log_filter: EthereumLogFilter, + ) -> DynTryFuture<'static, Vec, Error> { + let eth: Self = self.cheap_clone(); + let logger = logger.clone(); + + futures03::stream::iter(log_filter.eth_get_logs_filters().map(move |filter| { + eth.cheap_clone().log_stream( + logger.cheap_clone(), + subgraph_metrics.cheap_clone(), + from, + to, + filter, + ) + })) + // Real limits on the number of parallel requests are imposed within the adapter. + .buffered(ENV_VARS.block_ingestor_max_concurrent_json_rpc_calls) + .try_concat() + .boxed() + } + + pub(crate) fn calls_in_block_range<'a>( + &self, + logger: &Logger, + subgraph_metrics: Arc, + from: BlockNumber, + to: BlockNumber, + call_filter: &'a EthereumCallFilter, + ) -> Box + Send + 'a> { + let eth = self.clone(); + + let mut addresses: Vec = call_filter + .contract_addresses_function_signatures + .iter() + .filter(|(_addr, (start_block, _fsigs))| start_block <= &to) + .map(|(addr, (_start_block, _fsigs))| *addr) + .collect::>() + .into_iter() + .collect::>(); + + if addresses.is_empty() { + // The filter has no started data sources in the requested range, nothing to do. + // This prevents an expensive call to `trace_filter` with empty `addresses`. + return Box::new(stream::empty()); + } + + if addresses.len() > 100 { + // If the address list is large, request all traces, this avoids generating huge + // requests and potentially getting 413 errors. + addresses = vec![]; + } + + Box::new( + eth.trace_stream(&logger, subgraph_metrics, from, to, addresses) + .filter_map(|trace| EthereumCall::try_from_trace(&trace)) + .filter(move |call| { + // `trace_filter` can only filter by calls `to` an address and + // a block range. Since subgraphs are subscribing to calls + // for a specific contract function an additional filter needs + // to be applied + call_filter.matches(&call) + }), + ) + } + + pub(crate) async fn calls_in_block( + &self, + logger: &Logger, + subgraph_metrics: Arc, + block_number: BlockNumber, + block_hash: H256, + ) -> Result, Error> { + let eth = self.clone(); + let addresses = Vec::new(); + let traces = eth + .trace_stream( + &logger, + subgraph_metrics.clone(), + block_number, + block_number, + addresses, + ) + .collect() + .compat() + .await?; + + // `trace_stream` returns all of the traces for the block, and this + // includes a trace for the block reward which every block should have. + // If there are no traces something has gone wrong. + if traces.is_empty() { + return Err(anyhow!( + "Trace stream returned no traces for block: number = `{}`, hash = `{}`", + block_number, + block_hash, + )); + } + + // Since we can only pull traces by block number and we have + // all the traces for the block, we need to ensure that the + // block hash for the traces is equal to the desired block hash. + // Assume all traces are for the same block. + if traces.iter().nth(0).unwrap().block_hash != block_hash { + return Err(anyhow!( + "Trace stream returned traces for an unexpected block: \ + number = `{}`, hash = `{}`", + block_number, + block_hash, + )); + } + + Ok(traces + .iter() + .filter_map(EthereumCall::try_from_trace) + .collect()) + } + + /// Reorg safety: `to` must be a final block. + pub(crate) fn block_range_to_ptrs( + &self, + logger: Logger, + from: BlockNumber, + to: BlockNumber, + ) -> Box, Error = Error> + Send> { + // Currently we can't go to the DB for this because there might be duplicate entries for + // the same block number. + debug!(&logger, "Requesting hashes for blocks [{}, {}]", from, to); + Box::new( + self.load_block_ptrs_rpc(logger, (from..=to).collect()) + .collect(), + ) + } + + pub async fn chain_id(&self) -> Result { + let logger = self.logger.clone(); + let web3 = self.web3.clone(); + u64::try_from( + retry("chain_id RPC call", &logger) + .no_limit() + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || { + let web3 = web3.cheap_clone(); + async move { web3.eth().chain_id().await } + }) + .await?, + ) + .map_err(Error::msg) + } +} + +#[async_trait] +impl EthereumAdapterTrait for EthereumAdapter { + fn url_hostname(&self) -> &str { + &self.url_hostname + } + + fn provider(&self) -> &str { + &self.provider + } + + async fn net_identifiers(&self) -> Result { + let logger = self.logger.clone(); + + let web3 = self.web3.clone(); + let metrics = self.metrics.clone(); + let provider = self.provider().to_string(); + let net_version_future = retry("net_version RPC call", &logger) + .no_limit() + .timeout_secs(20) + .run(move || { + let web3 = web3.cheap_clone(); + let metrics = metrics.cheap_clone(); + let provider = provider.clone(); + async move { + web3.net().version().await.map_err(|e| { + metrics.set_status(ProviderStatus::VersionFail, &provider); + e.into() + }) + } + }) + .map_err(|e| { + self.metrics + .set_status(ProviderStatus::VersionTimeout, self.provider()); + e + }) + .boxed(); + + let web3 = self.web3.clone(); + let metrics = self.metrics.clone(); + let provider = self.provider().to_string(); + let retry_log_message = format!( + "eth_getBlockByNumber({}, false) RPC call", + ENV_VARS.genesis_block_number + ); + let gen_block_hash_future = retry(retry_log_message, &logger) + .no_limit() + .timeout_secs(30) + .run(move || { + let web3 = web3.cheap_clone(); + let metrics = metrics.cheap_clone(); + let provider = provider.clone(); + async move { + web3.eth() + .block(BlockId::Number(Web3BlockNumber::Number( + ENV_VARS.genesis_block_number.into(), + ))) + .await + .map_err(|e| { + metrics.set_status(ProviderStatus::GenesisFail, &provider); + e + })? + .map(|gen_block| gen_block.hash.map(BlockHash::from)) + .flatten() + .ok_or_else(|| anyhow!("Ethereum node could not find genesis block")) + } + }) + .map_err(|e| { + self.metrics + .set_status(ProviderStatus::GenesisTimeout, self.provider()); + e + }); + + let (net_version, genesis_block_hash) = + try_join!(net_version_future, gen_block_hash_future).map_err(|e| { + anyhow!( + "Ethereum node took too long to read network identifiers: {}", + e + ) + })?; + + let ident = ChainIdentifier { + net_version, + genesis_block_hash, + }; + + self.metrics + .set_status(ProviderStatus::Working, self.provider()); + Ok(ident) + } + + fn latest_block_header( + &self, + logger: &Logger, + ) -> Box, Error = IngestorError> + Send> { + let web3 = self.web3.clone(); + Box::new( + retry("eth_getBlockByNumber(latest) no txs RPC call", logger) + .no_limit() + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || { + let web3 = web3.cheap_clone(); + async move { + let block_opt = web3 + .eth() + .block(Web3BlockNumber::Latest.into()) + .await + .map_err(|e| { + anyhow!("could not get latest block from Ethereum: {}", e) + })?; + + block_opt + .ok_or_else(|| anyhow!("no latest block returned from Ethereum").into()) + } + }) + .map_err(move |e| { + e.into_inner().unwrap_or_else(move || { + anyhow!("Ethereum node took too long to return latest block").into() + }) + }) + .boxed() + .compat(), + ) + } + + fn latest_block( + &self, + logger: &Logger, + ) -> Box + Send + Unpin> { + let web3 = self.web3.clone(); + Box::new( + retry("eth_getBlockByNumber(latest) with txs RPC call", logger) + .no_limit() + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || { + let web3 = web3.cheap_clone(); + async move { + let block_opt = web3 + .eth() + .block_with_txs(Web3BlockNumber::Latest.into()) + .await + .map_err(|e| { + anyhow!("could not get latest block from Ethereum: {}", e) + })?; + block_opt + .ok_or_else(|| anyhow!("no latest block returned from Ethereum").into()) + } + }) + .map_err(move |e| { + e.into_inner().unwrap_or_else(move || { + anyhow!("Ethereum node took too long to return latest block").into() + }) + }) + .boxed() + .compat(), + ) + } + + fn load_block( + &self, + logger: &Logger, + block_hash: H256, + ) -> Box + Send> { + Box::new( + self.block_by_hash(&logger, block_hash) + .and_then(move |block_opt| { + block_opt.ok_or_else(move || { + anyhow!( + "Ethereum node could not find block with hash {}", + block_hash + ) + }) + }), + ) + } + + fn block_by_hash( + &self, + logger: &Logger, + block_hash: H256, + ) -> Box, Error = Error> + Send> { + let web3 = self.web3.clone(); + let logger = logger.clone(); + let retry_log_message = format!( + "eth_getBlockByHash RPC call for block hash {:?}", + block_hash + ); + Box::new( + retry(retry_log_message, &logger) + .limit(ENV_VARS.request_retries) + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || { + Box::pin(web3.eth().block_with_txs(BlockId::Hash(block_hash))) + .compat() + .from_err() + .compat() + }) + .map_err(move |e| { + e.into_inner().unwrap_or_else(move || { + anyhow!("Ethereum node took too long to return block {}", block_hash) + }) + }) + .boxed() + .compat(), + ) + } + + fn block_by_number( + &self, + logger: &Logger, + block_number: BlockNumber, + ) -> Box, Error = Error> + Send> { + let web3 = self.web3.clone(); + let logger = logger.clone(); + let retry_log_message = format!( + "eth_getBlockByNumber RPC call for block number {}", + block_number + ); + Box::new( + retry(retry_log_message, &logger) + .no_limit() + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || { + let web3 = web3.cheap_clone(); + async move { + web3.eth() + .block_with_txs(BlockId::Number(block_number.into())) + .await + .map_err(Error::from) + } + }) + .map_err(move |e| { + e.into_inner().unwrap_or_else(move || { + anyhow!( + "Ethereum node took too long to return block {}", + block_number + ) + }) + }) + .boxed() + .compat(), + ) + } + + fn load_full_block( + &self, + logger: &Logger, + block: LightEthereumBlock, + ) -> Pin> + Send>> + { + let web3 = Arc::clone(&self.web3); + let logger = logger.clone(); + let block_hash = block.hash.expect("block is missing block hash"); + + // The early return is necessary for correctness, otherwise we'll + // request an empty batch which is not valid in JSON-RPC. + if block.transactions.is_empty() { + trace!(logger, "Block {} contains no transactions", block_hash); + return Box::pin(std::future::ready(Ok(EthereumBlock { + block: Arc::new(block), + transaction_receipts: Vec::new(), + }))); + } + let hashes: Vec<_> = block.transactions.iter().map(|txn| txn.hash).collect(); + let receipts_future = if ENV_VARS.fetch_receipts_in_batches { + // Deprecated batching retrieval of transaction receipts. + fetch_transaction_receipts_in_batch_with_retry(web3, hashes, block_hash, logger).boxed() + } else { + let hash_stream = graph::tokio_stream::iter(hashes); + let receipt_stream = graph::tokio_stream::StreamExt::map(hash_stream, move |tx_hash| { + fetch_transaction_receipt_with_retry( + web3.cheap_clone(), + tx_hash, + block_hash, + logger.cheap_clone(), + ) + }) + .buffered(ENV_VARS.block_ingestor_max_concurrent_json_rpc_calls); + graph::tokio_stream::StreamExt::collect::< + Result>, IngestorError>, + >(receipt_stream) + .boxed() + }; + + let block_future = + futures03::TryFutureExt::map_ok(receipts_future, move |transaction_receipts| { + EthereumBlock { + block: Arc::new(block), + transaction_receipts, + } + }); + + Box::pin(block_future) + } + + fn block_pointer_from_number( + &self, + logger: &Logger, + block_number: BlockNumber, + ) -> Box + Send> { + Box::new( + self.block_hash_by_block_number(logger, block_number) + .and_then(move |block_hash_opt| { + block_hash_opt.ok_or_else(|| { + anyhow!( + "Ethereum node could not find start block hash by block number {}", + &block_number + ) + }) + }) + .from_err() + .map(move |block_hash| BlockPtr::from((block_hash, block_number))), + ) + } + + fn block_hash_by_block_number( + &self, + logger: &Logger, + block_number: BlockNumber, + ) -> Box, Error = Error> + Send> { + let web3 = self.web3.clone(); + let retry_log_message = format!( + "eth_getBlockByNumber RPC call for block number {}", + block_number + ); + Box::new( + retry(retry_log_message, &logger) + .no_limit() + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || { + let web3 = web3.cheap_clone(); + async move { + web3.eth() + .block(BlockId::Number(block_number.into())) + .await + .map(|block_opt| block_opt.map(|block| block.hash).flatten()) + .map_err(Error::from) + } + }) + .boxed() + .compat() + .map_err(move |e| { + e.into_inner().unwrap_or_else(move || { + anyhow!( + "Ethereum node took too long to return data for block #{}", + block_number + ) + }) + }), + ) + } + + fn contract_call( + &self, + logger: &Logger, + call: EthereumContractCall, + cache: Arc, + ) -> Box, Error = EthereumContractCallError> + Send> { + // Emit custom error for type mismatches. + for (token, kind) in call + .args + .iter() + .zip(call.function.inputs.iter().map(|p| &p.kind)) + { + if !token.type_check(kind) { + return Box::new(future::err(EthereumContractCallError::TypeError( + token.clone(), + kind.clone(), + ))); + } + } + + // Encode the call parameters according to the ABI + let call_data = match call.function.encode_input(&call.args) { + Ok(data) => data, + Err(e) => return Box::new(future::err(EthereumContractCallError::EncodingError(e))), + }; + + debug!(logger, "eth_call"; + "address" => hex::encode(&call.address), + "data" => hex::encode(&call_data) + ); + + // Check if we have it cached, if not do the call and cache. + Box::new( + match cache + .get_call(call.address, &call_data, call.block_ptr.clone()) + .map_err(|e| error!(logger, "call cache get error"; "error" => e.to_string())) + .ok() + .flatten() + { + Some(result) => { + Box::new(future::ok(result)) as Box + Send> + } + None => { + let cache = cache.clone(); + let call = call.clone(); + let logger = logger.clone(); + Box::new( + self.call( + logger.clone(), + call.address, + Bytes(call_data.clone()), + call.block_ptr.clone(), + ) + .map(move |result| { + // Don't block handler execution on writing to the cache. + let for_cache = result.0.clone(); + let _ = graph::spawn_blocking_allow_panic(move || { + cache + .set_call(call.address, &call_data, call.block_ptr, &for_cache) + .map_err(|e| { + error!(logger, "call cache set error"; + "error" => e.to_string()) + }) + }); + result.0 + }), + ) + } + } + // Decode the return values according to the ABI + .and_then(move |output| { + if output.is_empty() { + // We got a `0x` response. For old Geth, this can mean a revert. It can also be + // that the contract actually returned an empty response. A view call is meant + // to return something, so we treat empty responses the same as reverts. + Err(EthereumContractCallError::Revert("empty response".into())) + } else { + // Decode failures are reverts. The reasoning is that if Solidity fails to + // decode an argument, that's a revert, so the same goes for the output. + call.function.decode_output(&output).map_err(|e| { + EthereumContractCallError::Revert(format!("failed to decode output: {}", e)) + }) + } + }), + ) + } + + /// Load Ethereum blocks in bulk, returning results as they come back as a Stream. + fn load_blocks( + &self, + logger: Logger, + chain_store: Arc, + block_hashes: HashSet, + ) -> Box, Error = Error> + Send> { + let block_hashes: Vec<_> = block_hashes.iter().cloned().collect(); + // Search for the block in the store first then use json-rpc as a backup. + let mut blocks: Vec> = chain_store + .blocks(&block_hashes.iter().map(|&b| b.into()).collect::>()) + .map_err(|e| error!(&logger, "Error accessing block cache {}", e)) + .unwrap_or_default() + .into_iter() + .filter_map(|value| json::from_value(value).ok()) + .map(Arc::new) + .collect(); + + let missing_blocks = Vec::from_iter( + block_hashes + .into_iter() + .filter(|hash| !blocks.iter().any(|b| b.hash == Some(*hash))), + ); + + // Return a stream that lazily loads batches of blocks. + debug!(logger, "Requesting {} block(s)", missing_blocks.len()); + Box::new( + self.load_blocks_rpc(logger.clone(), missing_blocks) + .collect() + .map(move |new_blocks| { + let upsert_blocks: Vec<_> = new_blocks + .iter() + .map(|block| BlockFinality::Final(block.clone())) + .collect(); + let block_refs: Vec<_> = upsert_blocks + .iter() + .map(|block| block as &dyn graph::blockchain::Block) + .collect(); + if let Err(e) = chain_store.upsert_light_blocks(block_refs.as_slice()) { + error!(logger, "Error writing to block cache {}", e); + } + blocks.extend(new_blocks); + blocks.sort_by_key(|block| block.number); + stream::iter_ok(blocks) + }) + .flatten_stream(), + ) + } +} + +/// Returns blocks with triggers, corresponding to the specified range and filters. +/// If a block contains no triggers, there may be no corresponding item in the stream. +/// However the `to` block will always be present, even if triggers are empty. +/// +/// Careful: don't use this function without considering race conditions. +/// Chain reorgs could happen at any time, and could affect the answer received. +/// Generally, it is only safe to use this function with blocks that have received enough +/// confirmations to guarantee no further reorgs, **and** where the Ethereum node is aware of +/// those confirmations. +/// If the Ethereum node is far behind in processing blocks, even old blocks can be subject to +/// reorgs. +/// It is recommended that `to` be far behind the block number of latest block the Ethereum +/// node is aware of. +pub(crate) async fn blocks_with_triggers( + adapter: Arc, + logger: Logger, + chain_store: Arc, + subgraph_metrics: Arc, + from: BlockNumber, + to: BlockNumber, + filter: &TriggerFilter, + unified_api_version: UnifiedMappingApiVersion, +) -> Result>, Error> { + // Each trigger filter needs to be queried for the same block range + // and the blocks yielded need to be deduped. If any error occurs + // while searching for a trigger type, the entire operation fails. + let eth = adapter.clone(); + let call_filter = EthereumCallFilter::from(&filter.block); + + // Scan the block range to find relevant triggers + let trigger_futs: FuturesUnordered, anyhow::Error>>> = + FuturesUnordered::new(); + + // Scan for Logs + if !filter.log.is_empty() { + let logs_future = get_logs_and_transactions( + eth.clone(), + &logger, + subgraph_metrics.clone(), + from, + to, + filter.log.clone(), + &unified_api_version, + ) + .boxed(); + trigger_futs.push(logs_future) + } + // Scan for Calls + if !filter.call.is_empty() { + let calls_future = eth + .calls_in_block_range(&logger, subgraph_metrics.clone(), from, to, &filter.call) + .map(Arc::new) + .map(EthereumTrigger::Call) + .collect() + .compat() + .boxed(); + trigger_futs.push(calls_future) + } + + // Scan for Blocks + if filter.block.trigger_every_block { + let block_future = adapter + .block_range_to_ptrs(logger.clone(), from, to) + .map(move |ptrs| { + ptrs.into_iter() + .map(|ptr| EthereumTrigger::Block(ptr, EthereumBlockTriggerType::Every)) + .collect() + }) + .compat() + .boxed(); + trigger_futs.push(block_future) + } else if !filter.block.contract_addresses.is_empty() { + // To determine which blocks include a call to addresses + // in the block filter, transform the `block_filter` into + // a `call_filter` and run `blocks_with_calls` + let block_future = eth + .calls_in_block_range(&logger, subgraph_metrics.clone(), from, to, &call_filter) + .map(|call| { + EthereumTrigger::Block( + BlockPtr::from(&call), + EthereumBlockTriggerType::WithCallTo(call.to), + ) + }) + .collect() + .compat() + .boxed(); + trigger_futs.push(block_future) + } + + // Get hash for "to" block + let to_hash_fut = adapter + .block_hash_by_block_number(&logger, to) + .and_then(|hash| match hash { + Some(hash) => Ok(hash), + None => { + warn!(logger, + "Ethereum endpoint is behind"; + "url" => eth.url_hostname() + ); + bail!("Block {} not found in the chain", to) + } + }) + .compat(); + + // Join on triggers and block hash resolution + let (triggers, to_hash) = futures03::join!(trigger_futs.try_concat(), to_hash_fut); + + // Unpack and handle possible errors in the previously joined futures + let triggers = + triggers.with_context(|| format!("Failed to obtain triggers for block {}", to))?; + let to_hash = to_hash.with_context(|| format!("Failed to infer hash for block {}", to))?; + + let mut block_hashes: HashSet = + triggers.iter().map(EthereumTrigger::block_hash).collect(); + let mut triggers_by_block: HashMap> = + triggers.into_iter().fold(HashMap::new(), |mut map, t| { + map.entry(t.block_number()).or_default().push(t); + map + }); + + debug!(logger, "Found {} relevant block(s)", block_hashes.len()); + + // Make sure `to` is included, even if empty. + block_hashes.insert(to_hash); + triggers_by_block.entry(to).or_insert(Vec::new()); + + let blocks = adapter + .load_blocks(logger.cheap_clone(), chain_store.clone(), block_hashes) + .and_then( + move |block| match triggers_by_block.remove(&(block.number() as BlockNumber)) { + Some(triggers) => Ok(BlockWithTriggers::new( + BlockFinality::Final(block), + triggers, + )), + None => Err(anyhow!( + "block {} not found in `triggers_by_block`", + block.block_ptr() + )), + }, + ) + .collect() + .compat() + .await?; + + // Filter out call triggers that come from unsuccessful transactions + let mut blocks = if unified_api_version.equal_or_greater_than(&API_VERSION_0_0_5) { + let futures = blocks.into_iter().map(|block| { + filter_call_triggers_from_unsuccessful_transactions(block, ð, &chain_store, &logger) + }); + futures03::future::try_join_all(futures).await? + } else { + blocks + }; + + blocks.sort_by_key(|block| block.ptr().number); + + // Sanity check that the returned blocks are in the correct range. + // Unwrap: `blocks` always includes at least `to`. + let first = blocks.first().unwrap().ptr().number; + let last = blocks.last().unwrap().ptr().number; + if first < from { + return Err(anyhow!( + "block {} returned by the Ethereum node is before {}, the first block of the requested range", + first, + from, + )); + } + if last > to { + return Err(anyhow!( + "block {} returned by the Ethereum node is after {}, the last block of the requested range", + last, + to, + )); + } + + Ok(blocks) +} + +pub(crate) async fn get_calls( + adapter: &EthereumAdapter, + logger: Logger, + subgraph_metrics: Arc, + requires_traces: bool, + block: BlockFinality, +) -> Result { + // For final blocks, or nonfinal blocks where we already checked + // (`calls.is_some()`), do nothing; if we haven't checked for calls, do + // that now + match block { + BlockFinality::Final(_) + | BlockFinality::NonFinal(EthereumBlockWithCalls { + ethereum_block: _, + calls: Some(_), + }) => Ok(block), + BlockFinality::NonFinal(EthereumBlockWithCalls { + ethereum_block, + calls: None, + }) => { + let calls = if !requires_traces || ethereum_block.transaction_receipts.is_empty() { + vec![] + } else { + adapter + .calls_in_block( + &logger, + subgraph_metrics.clone(), + BlockNumber::try_from(ethereum_block.block.number.unwrap().as_u64()) + .unwrap(), + ethereum_block.block.hash.unwrap(), + ) + .await? + }; + Ok(BlockFinality::NonFinal(EthereumBlockWithCalls { + ethereum_block, + calls: Some(calls), + })) + } + } +} + +pub(crate) fn parse_log_triggers( + log_filter: &EthereumLogFilter, + block: &EthereumBlock, +) -> Vec { + if log_filter.is_empty() { + return vec![]; + } + + block + .transaction_receipts + .iter() + .flat_map(move |receipt| { + receipt + .logs + .iter() + .filter(move |log| log_filter.matches(log)) + .map(move |log| { + EthereumTrigger::Log(Arc::new(log.clone()), Some(receipt.cheap_clone())) + }) + }) + .collect() +} + +pub(crate) fn parse_call_triggers( + call_filter: &EthereumCallFilter, + block: &EthereumBlockWithCalls, +) -> anyhow::Result> { + if call_filter.is_empty() { + return Ok(vec![]); + } + + match &block.calls { + Some(calls) => calls + .iter() + .filter(move |call| call_filter.matches(call)) + .map( + move |call| match block.transaction_for_call_succeeded(call) { + Ok(true) => Ok(Some(EthereumTrigger::Call(Arc::new(call.clone())))), + Ok(false) => Ok(None), + Err(e) => Err(e), + }, + ) + .filter_map_ok(|some_trigger| some_trigger) + .collect(), + None => Ok(vec![]), + } +} + +pub(crate) fn parse_block_triggers( + block_filter: &EthereumBlockFilter, + block: &EthereumBlockWithCalls, +) -> Vec { + if block_filter.is_empty() { + return vec![]; + } + + let block_ptr = BlockPtr::from(&block.ethereum_block); + let trigger_every_block = block_filter.trigger_every_block; + let call_filter = EthereumCallFilter::from(block_filter); + let block_ptr2 = block_ptr.cheap_clone(); + let mut triggers = match &block.calls { + Some(calls) => calls + .iter() + .filter(move |call| call_filter.matches(call)) + .map(move |call| { + EthereumTrigger::Block( + block_ptr2.clone(), + EthereumBlockTriggerType::WithCallTo(call.to), + ) + }) + .collect::>(), + None => vec![], + }; + if trigger_every_block { + triggers.push(EthereumTrigger::Block( + block_ptr, + EthereumBlockTriggerType::Every, + )); + } + triggers +} + +async fn fetch_receipt_from_ethereum_client( + eth: &EthereumAdapter, + transaction_hash: &H256, +) -> anyhow::Result { + match eth.web3.eth().transaction_receipt(*transaction_hash).await { + Ok(Some(receipt)) => Ok(receipt), + Ok(None) => bail!("Could not find transaction receipt"), + Err(error) => bail!("Failed to fetch transaction receipt: {}", error), + } +} + +async fn filter_call_triggers_from_unsuccessful_transactions( + mut block: BlockWithTriggers, + eth: &EthereumAdapter, + chain_store: &Arc, + logger: &Logger, +) -> anyhow::Result> { + // Return early if there is no trigger data + if block.trigger_data.is_empty() { + return Ok(block); + } + + let initial_number_of_triggers = block.trigger_data.len(); + + // Get the transaction hash from each call trigger + let transaction_hashes: BTreeSet = block + .trigger_data + .iter() + .filter_map(|trigger| match trigger { + EthereumTrigger::Call(call_trigger) => Some(call_trigger.transaction_hash), + _ => None, + }) + .collect::>>() + .ok_or(anyhow!( + "failed to obtain transaction hash from call triggers" + ))?; + + // Return early if there are no transaction hashes + if transaction_hashes.is_empty() { + return Ok(block); + } + + // And obtain all Transaction values for the calls in this block. + let transactions: Vec<&Transaction> = { + match &block.block { + BlockFinality::Final(ref block) => block + .transactions + .iter() + .filter(|transaction| transaction_hashes.contains(&transaction.hash)) + .collect(), + BlockFinality::NonFinal(_block_with_calls) => { + unreachable!( + "this function should not be called when dealing with non-final blocks" + ) + } + } + }; + + // Confidence check: Did we collect all transactions for the current call triggers? + if transactions.len() != transaction_hashes.len() { + bail!("failed to find transactions in block for the given call triggers") + } + + // We'll also need the receipts for those transactions. In this step we collect all receipts + // we have in store for the current block. + let mut receipts = chain_store + .transaction_receipts_in_block(&block.ptr().hash_as_h256()) + .await? + .into_iter() + .map(|receipt| (receipt.transaction_hash, receipt)) + .collect::>(); + + // Do we have a receipt for each transaction under analysis? + let mut receipts_and_transactions: Vec<(&Transaction, LightTransactionReceipt)> = Vec::new(); + let mut transactions_without_receipt: Vec<&Transaction> = Vec::new(); + for transaction in transactions.iter() { + if let Some(receipt) = receipts.remove(&transaction.hash) { + receipts_and_transactions.push((transaction, receipt)); + } else { + transactions_without_receipt.push(transaction); + } + } + + // When some receipts are missing, we then try to fetch them from our client. + let futures = transactions_without_receipt + .iter() + .map(|transaction| async move { + fetch_receipt_from_ethereum_client(ð, &transaction.hash) + .await + .map(|receipt| (transaction, receipt)) + }); + futures03::future::try_join_all(futures) + .await? + .into_iter() + .for_each(|(transaction, receipt)| { + receipts_and_transactions.push((transaction, receipt.into())) + }); + + // TODO: We should persist those fresh transaction receipts into the store, so we don't incur + // additional Ethereum API calls for future scans on this block. + + // With all transactions and receipts in hand, we can evaluate the success of each transaction + let mut transaction_success: BTreeMap<&H256, bool> = BTreeMap::new(); + for (transaction, receipt) in receipts_and_transactions.into_iter() { + transaction_success.insert( + &transaction.hash, + evaluate_transaction_status(receipt.status), + ); + } + + // Confidence check: Did we inspect the status of all transactions? + if !transaction_hashes + .iter() + .all(|tx| transaction_success.contains_key(tx)) + { + bail!("Not all transactions status were inspected") + } + + // Filter call triggers from unsuccessful transactions + block.trigger_data.retain(|trigger| { + if let EthereumTrigger::Call(call_trigger) = trigger { + // Unwrap: We already checked that those values exist + transaction_success[&call_trigger.transaction_hash.unwrap()] + } else { + // We are not filtering other types of triggers + true + } + }); + + // Log if any call trigger was filtered out + let final_number_of_triggers = block.trigger_data.len(); + let number_of_filtered_triggers = initial_number_of_triggers - final_number_of_triggers; + if number_of_filtered_triggers != 0 { + let noun = { + if number_of_filtered_triggers == 1 { + "call trigger" + } else { + "call triggers" + } + }; + info!(&logger, + "Filtered {} {} from failed transactions", number_of_filtered_triggers, noun ; + "block_number" => block.ptr().block_number()); + } + Ok(block) +} + +/// Deprecated. Wraps the [`fetch_transaction_receipts_in_batch`] in a retry loop. +async fn fetch_transaction_receipts_in_batch_with_retry( + web3: Arc>, + hashes: Vec, + block_hash: H256, + logger: Logger, +) -> Result>, IngestorError> { + let retry_log_message = format!( + "batch eth_getTransactionReceipt RPC call for block {:?}", + block_hash + ); + retry(retry_log_message, &logger) + .limit(ENV_VARS.request_retries) + .no_logging() + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || { + let web3 = web3.cheap_clone(); + let hashes = hashes.clone(); + let logger = logger.cheap_clone(); + fetch_transaction_receipts_in_batch(web3, hashes, block_hash, logger).boxed() + }) + .await + .map_err(|_timeout| anyhow!(block_hash).into()) +} + +/// Deprecated. Attempts to fetch multiple transaction receipts in a batching contex. +async fn fetch_transaction_receipts_in_batch( + web3: Arc>, + hashes: Vec, + block_hash: H256, + logger: Logger, +) -> Result>, IngestorError> { + let batching_web3 = Web3::new(Batch::new(web3.transport().clone())); + let eth = batching_web3.eth(); + let receipt_futures = hashes + .into_iter() + .map(move |hash| { + let logger = logger.cheap_clone(); + eth.transaction_receipt(hash) + .map_err(|web3_error| IngestorError::from(web3_error)) + .and_then(move |some_receipt| async move { + resolve_transaction_receipt(some_receipt, hash, block_hash, logger) + }) + }) + .collect::>(); + + batching_web3.transport().submit_batch().await?; + + let mut collected = vec![]; + for receipt in receipt_futures.into_iter() { + collected.push(Arc::new(receipt.await?)) + } + Ok(collected) +} + +/// Retries fetching a single transaction receipt. +async fn fetch_transaction_receipt_with_retry( + web3: Arc>, + transaction_hash: H256, + block_hash: H256, + logger: Logger, +) -> Result, IngestorError> { + let logger = logger.cheap_clone(); + let retry_log_message = format!( + "eth_getTransactionReceipt RPC call for transaction {:?}", + transaction_hash + ); + retry(retry_log_message, &logger) + .limit(ENV_VARS.request_retries) + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || web3.eth().transaction_receipt(transaction_hash).boxed()) + .await + .map_err(|_timeout| anyhow!(block_hash).into()) + .and_then(move |some_receipt| { + resolve_transaction_receipt(some_receipt, transaction_hash, block_hash, logger) + }) + .map(Arc::new) +} + +fn resolve_transaction_receipt( + transaction_receipt: Option, + transaction_hash: H256, + block_hash: H256, + logger: Logger, +) -> Result { + match transaction_receipt { + // A receipt might be missing because the block was uncled, and the transaction never + // made it back into the main chain. + Some(receipt) => { + // Check if the receipt has a block hash and is for the right block. Parity nodes seem + // to return receipts with no block hash when a transaction is no longer in the main + // chain, so treat that case the same as a receipt being absent entirely. + if receipt.block_hash != Some(block_hash) { + info!( + logger, "receipt block mismatch"; + "receipt_block_hash" => + receipt.block_hash.unwrap_or_default().to_string(), + "block_hash" => + block_hash.to_string(), + "tx_hash" => transaction_hash.to_string(), + ); + + // If the receipt came from a different block, then the Ethereum node no longer + // considers this block to be in the main chain. Nothing we can do from here except + // give up trying to ingest this block. There is no way to get the transaction + // receipt from this block. + Err(IngestorError::BlockUnavailable(block_hash.clone())) + } else { + Ok(receipt) + } + } + None => { + // No receipt was returned. + // + // This can be because the Ethereum node no longer considers this block to be part of + // the main chain, and so the transaction is no longer in the main chain. Nothing we can + // do from here except give up trying to ingest this block. + // + // This could also be because the receipt is simply not available yet. For that case, we + // should retry until it becomes available. + Err(IngestorError::ReceiptUnavailable( + block_hash, + transaction_hash, + )) + } + } +} + +/// Retrieves logs and the associated transaction receipts, if required by the [`EthereumLogFilter`]. +async fn get_logs_and_transactions( + adapter: Arc, + logger: &Logger, + subgraph_metrics: Arc, + from: BlockNumber, + to: BlockNumber, + log_filter: EthereumLogFilter, + unified_api_version: &UnifiedMappingApiVersion, +) -> Result, anyhow::Error> { + // Obtain logs externally + let logs = adapter + .logs_in_block_range( + logger, + subgraph_metrics.cheap_clone(), + from, + to, + log_filter.clone(), + ) + .await?; + + // Not all logs have associated transaction hashes, nor do all triggers require them. + // We also restrict receipts retrieval for some api versions. + let transaction_hashes_by_block: HashMap> = logs + .iter() + .filter(|_| unified_api_version.equal_or_greater_than(&API_VERSION_0_0_7)) + .filter(|log| { + if let Some(signature) = log.topics.first() { + log_filter.requires_transaction_receipt(signature, Some(&log.address)) + } else { + false + } + }) + .filter_map(|log| { + if let (Some(block), Some(txn)) = (log.block_hash, log.transaction_hash) { + Some((block, txn)) + } else { + // Absent block and transaction data might happen for pending transactions, which we + // don't handle. + None + } + }) + .fold( + HashMap::>::new(), + |mut acc, (block_hash, txn_hash)| { + acc.entry(block_hash).or_default().insert(txn_hash); + acc + }, + ); + + // Obtain receipts externally + let transaction_receipts_by_hash = get_transaction_receipts_for_transaction_hashes( + &adapter, + &transaction_hashes_by_block, + subgraph_metrics, + logger.cheap_clone(), + ) + .await?; + + // Associate each log with its receipt, when possible + let mut log_triggers = Vec::new(); + for log in logs.into_iter() { + let optional_receipt = log + .transaction_hash + .and_then(|txn| transaction_receipts_by_hash.get(&txn).cloned()); + let value = EthereumTrigger::Log(Arc::new(log), optional_receipt); + log_triggers.push(value); + } + + Ok(log_triggers) +} + +/// Tries to retrive all transaction receipts for a set of transaction hashes. +async fn get_transaction_receipts_for_transaction_hashes( + adapter: &EthereumAdapter, + transaction_hashes_by_block: &HashMap>, + subgraph_metrics: Arc, + logger: Logger, +) -> Result>, anyhow::Error> { + use std::collections::hash_map::Entry::Vacant; + + let mut receipts_by_hash: HashMap> = HashMap::new(); + + // Return early if input set is empty + if transaction_hashes_by_block.is_empty() { + return Ok(receipts_by_hash); + } + + // Keep a record of all unique transaction hashes for which we'll request receipts. We will + // later use this to check if we have collected the receipts from all required transactions. + let mut unique_transaction_hashes: HashSet<&H256> = HashSet::new(); + + // Request transaction receipts concurrently + let receipt_futures = FuturesUnordered::new(); + + let web3 = Arc::clone(&adapter.web3); + for (block_hash, transaction_hashes) in transaction_hashes_by_block { + for transaction_hash in transaction_hashes { + unique_transaction_hashes.insert(transaction_hash); + let receipt_future = fetch_transaction_receipt_with_retry( + web3.cheap_clone(), + *transaction_hash, + *block_hash, + logger.cheap_clone(), + ); + receipt_futures.push(receipt_future) + } + } + + // Execute futures while monitoring elapsed time + let start = Instant::now(); + let receipts: Vec<_> = match receipt_futures.try_collect().await { + Ok(receipts) => { + let elapsed = start.elapsed().as_secs_f64(); + subgraph_metrics.observe_request( + elapsed, + "eth_getTransactionReceipt", + &adapter.provider, + ); + receipts + } + Err(ingestor_error) => { + subgraph_metrics.add_error("eth_getTransactionReceipt", &adapter.provider); + debug!( + logger, + "Error querying transaction receipts: {}", ingestor_error + ); + return Err(ingestor_error.into()); + } + }; + + // Build a map between transaction hashes and their receipts + for receipt in receipts.into_iter() { + if !unique_transaction_hashes.remove(&receipt.transaction_hash) { + bail!("Received a receipt for a different transaction hash") + } + if let Vacant(entry) = receipts_by_hash.entry(receipt.transaction_hash.clone()) { + entry.insert(receipt); + } else { + bail!("Received a duplicate transaction receipt") + } + } + + // Confidence check: all unique hashes should have been used + ensure!( + unique_transaction_hashes.is_empty(), + "Didn't receive all necessary transaction receipts" + ); + + Ok(receipts_by_hash) +} + +#[cfg(test)] +mod tests { + + use crate::trigger::{EthereumBlockTriggerType, EthereumTrigger}; + + use super::{parse_block_triggers, EthereumBlock, EthereumBlockFilter, EthereumBlockWithCalls}; + use graph::blockchain::BlockPtr; + use graph::prelude::ethabi::ethereum_types::U64; + use graph::prelude::web3::types::{Address, Block, Bytes, H256}; + use graph::prelude::EthereumCall; + use std::collections::HashSet; + use std::iter::FromIterator; + use std::sync::Arc; + + #[test] + fn parse_block_triggers_every_block() { + let block = EthereumBlockWithCalls { + ethereum_block: EthereumBlock { + block: Arc::new(Block { + hash: Some(hash(2)), + number: Some(U64::from(2)), + ..Default::default() + }), + ..Default::default() + }, + calls: Some(vec![EthereumCall { + to: address(4), + input: bytes(vec![1; 36]), + ..Default::default() + }]), + }; + + assert_eq!( + vec![EthereumTrigger::Block( + BlockPtr::from((hash(2), 2)), + EthereumBlockTriggerType::Every + )], + parse_block_triggers( + &EthereumBlockFilter { + contract_addresses: HashSet::from_iter(vec![(10, address(1))]), + trigger_every_block: true, + }, + &block + ), + "every block should generate a trigger even when address don't match" + ); + } + + #[test] + fn parse_block_triggers_specific_call_not_found() { + let block = EthereumBlockWithCalls { + ethereum_block: EthereumBlock { + block: Arc::new(Block { + hash: Some(hash(2)), + number: Some(U64::from(2)), + ..Default::default() + }), + ..Default::default() + }, + calls: Some(vec![EthereumCall { + to: address(4), + input: bytes(vec![1; 36]), + ..Default::default() + }]), + }; + + assert_eq!( + Vec::::new(), + parse_block_triggers( + &EthereumBlockFilter { + contract_addresses: HashSet::from_iter(vec![(1, address(1))]), + trigger_every_block: false, + }, + &block + ), + "block filter specifies address 1 but block does not contain any call to it" + ); + } + + #[test] + fn parse_block_triggers_specific_call_found() { + let block = EthereumBlockWithCalls { + ethereum_block: EthereumBlock { + block: Arc::new(Block { + hash: Some(hash(2)), + number: Some(U64::from(2)), + ..Default::default() + }), + ..Default::default() + }, + calls: Some(vec![EthereumCall { + to: address(4), + input: bytes(vec![1; 36]), + ..Default::default() + }]), + }; + + assert_eq!( + vec![EthereumTrigger::Block( + BlockPtr::from((hash(2), 2)), + EthereumBlockTriggerType::WithCallTo(address(4)) + )], + parse_block_triggers( + &EthereumBlockFilter { + contract_addresses: HashSet::from_iter(vec![(1, address(4))]), + trigger_every_block: false, + }, + &block + ), + "block filter specifies address 4 and block has call to it" + ); + } + + fn address(id: u64) -> Address { + Address::from_low_u64_be(id) + } + + fn hash(id: u8) -> H256 { + H256::from([id; 32]) + } + + fn bytes(value: Vec) -> Bytes { + Bytes::from(value) + } +} diff --git a/chain/ethereum/src/ingestor.rs b/chain/ethereum/src/ingestor.rs new file mode 100644 index 0000000..ac4f906 --- /dev/null +++ b/chain/ethereum/src/ingestor.rs @@ -0,0 +1,219 @@ +use crate::{chain::BlockFinality, EthereumAdapter, EthereumAdapterTrait, ENV_VARS}; +use graph::{ + blockchain::{BlockHash, BlockPtr, IngestorError}, + cheap_clone::CheapClone, + prelude::{ + error, ethabi::ethereum_types::H256, info, tokio, trace, warn, ChainStore, Error, + EthereumBlockWithCalls, Future01CompatExt, LogCode, Logger, + }, +}; +use std::{sync::Arc, time::Duration}; + +pub struct BlockIngestor { + logger: Logger, + ancestor_count: i32, + eth_adapter: Arc, + chain_store: Arc, + polling_interval: Duration, +} + +impl BlockIngestor { + pub fn new( + logger: Logger, + ancestor_count: i32, + eth_adapter: Arc, + chain_store: Arc, + polling_interval: Duration, + ) -> Result { + Ok(BlockIngestor { + logger, + ancestor_count, + eth_adapter, + chain_store, + polling_interval, + }) + } + + pub async fn into_polling_stream(self) { + loop { + match self.do_poll().await { + // Some polls will fail due to transient issues + Err(err @ IngestorError::BlockUnavailable(_)) => { + info!( + self.logger, + "Trying again after block polling failed: {}", err + ); + } + Err(err @ IngestorError::ReceiptUnavailable(_, _)) => { + info!( + self.logger, + "Trying again after block polling failed: {}", err + ); + } + Err(IngestorError::Unknown(inner_err)) => { + warn!( + self.logger, + "Trying again after block polling failed: {}", inner_err + ); + } + Ok(()) => (), + } + + if ENV_VARS.cleanup_blocks { + self.cleanup_cached_blocks() + } + + tokio::time::sleep(self.polling_interval).await; + } + } + + fn cleanup_cached_blocks(&self) { + match self.chain_store.cleanup_cached_blocks(self.ancestor_count) { + Ok(Some((min_block, count))) => { + if count > 0 { + info!( + self.logger, + "Cleaned {} blocks from the block cache. \ + Only blocks with number greater than {} remain", + count, + min_block + ); + } + } + Ok(None) => { /* nothing was cleaned, ignore */ } + Err(e) => warn!( + self.logger, + "Failed to clean blocks from block cache: {}", e + ), + } + } + + async fn do_poll(&self) -> Result<(), IngestorError> { + trace!(self.logger, "BlockIngestor::do_poll"); + + // Get chain head ptr from store + let head_block_ptr_opt = self.chain_store.cheap_clone().chain_head_ptr().await?; + + // To check if there is a new block or not, fetch only the block header since that's cheaper + // than the full block. This is worthwhile because most of the time there won't be a new + // block, as we expect the poll interval to be much shorter than the block time. + let latest_block = self.latest_block().await?; + + // If latest block matches head block in store, nothing needs to be done + if Some(&latest_block) == head_block_ptr_opt.as_ref() { + return Ok(()); + } + + // Compare latest block with head ptr, alert user if far behind + match head_block_ptr_opt { + None => { + info!( + self.logger, + "Downloading latest blocks from Ethereum, this may take a few minutes..." + ); + } + Some(head_block_ptr) => { + let latest_number = latest_block.number; + let head_number = head_block_ptr.number; + let distance = latest_number - head_number; + let blocks_needed = (distance).min(self.ancestor_count); + let code = if distance >= 15 { + LogCode::BlockIngestionLagging + } else { + LogCode::BlockIngestionStatus + }; + if distance > 0 { + info!( + self.logger, + "Syncing {} blocks from Ethereum", + blocks_needed; + "current_block_head" => head_number, + "latest_block_head" => latest_number, + "blocks_behind" => distance, + "blocks_needed" => blocks_needed, + "code" => code, + ); + } + } + } + + // Store latest block in block store. + // Might be a no-op if latest block is one that we have seen. + // ingest_blocks will return a (potentially incomplete) list of blocks that are + // missing. + let mut missing_block_hash = self.ingest_block(&latest_block.hash).await?; + + // Repeatedly fetch missing parent blocks, and ingest them. + // ingest_blocks will continue to tell us about more missing parent + // blocks until we have filled in all missing pieces of the + // blockchain in the block number range we care about. + // + // Loop will terminate because: + // - The number of blocks in the ChainStore in the block number + // range [latest - ancestor_count, latest] is finite. + // - The missing parents in the first iteration have at most block + // number latest-1. + // - Each iteration loads parents of all blocks in the range whose + // parent blocks are not already in the ChainStore, so blocks + // with missing parents in one iteration will not have missing + // parents in the next. + // - Therefore, if the missing parents in one iteration have at + // most block number N, then the missing parents in the next + // iteration will have at most block number N-1. + // - Therefore, the loop will iterate at most ancestor_count times. + while let Some(hash) = missing_block_hash { + missing_block_hash = self.ingest_block(&hash).await?; + } + Ok(()) + } + + async fn ingest_block( + &self, + block_hash: &BlockHash, + ) -> Result, IngestorError> { + // TODO: H256::from_slice can panic + let block_hash = H256::from_slice(block_hash.as_slice()); + + // Get the fully populated block + let block = self + .eth_adapter + .block_by_hash(&self.logger, block_hash) + .compat() + .await? + .ok_or_else(|| IngestorError::BlockUnavailable(block_hash))?; + let ethereum_block = self + .eth_adapter + .load_full_block(&self.logger, block) + .await?; + + // We need something that implements `Block` to store the block; the + // store does not care whether the block is final or not + let ethereum_block = BlockFinality::NonFinal(EthereumBlockWithCalls { + ethereum_block, + calls: None, + }); + + // Store it in the database and try to advance the chain head pointer + self.chain_store + .upsert_block(Arc::new(ethereum_block)) + .await?; + + self.chain_store + .cheap_clone() + .attempt_chain_head_update(self.ancestor_count) + .await + .map(|missing| missing.map(|h256| h256.into())) + .map_err(|e| { + error!(self.logger, "failed to update chain head"); + IngestorError::Unknown(e) + }) + } + + async fn latest_block(&self) -> Result { + self.eth_adapter + .latest_block_header(&self.logger) + .compat() + .await + .map(|block| block.into()) + } +} diff --git a/chain/ethereum/src/lib.rs b/chain/ethereum/src/lib.rs new file mode 100644 index 0000000..eeb207b --- /dev/null +++ b/chain/ethereum/src/lib.rs @@ -0,0 +1,34 @@ +mod adapter; +mod capabilities; +pub mod codec; +mod data_source; +mod env; +mod ethereum_adapter; +mod ingestor; +pub mod runtime; +mod transport; + +pub use self::capabilities::NodeCapabilities; +pub use self::ethereum_adapter::EthereumAdapter; +pub use self::runtime::RuntimeAdapter; +pub use self::transport::Transport; +pub use env::ENV_VARS; + +// ETHDEP: These concrete types should probably not be exposed. +pub use data_source::{DataSource, DataSourceTemplate, Mapping, MappingABI, TemplateSource}; + +pub mod chain; + +pub mod network; +pub mod trigger; + +pub use crate::adapter::{ + EthereumAdapter as EthereumAdapterTrait, EthereumContractCall, EthereumContractCallError, + ProviderEthRpcMetrics, SubgraphEthRpcMetrics, TriggerFilter, +}; +pub use crate::chain::Chain; +pub use crate::network::EthereumNetworks; +pub use ingestor::BlockIngestor; + +#[cfg(test)] +mod tests; diff --git a/chain/ethereum/src/network.rs b/chain/ethereum/src/network.rs new file mode 100644 index 0000000..0981f59 --- /dev/null +++ b/chain/ethereum/src/network.rs @@ -0,0 +1,214 @@ +use anyhow::{anyhow, Context}; +use graph::cheap_clone::CheapClone; +use graph::prelude::rand::{self, seq::IteratorRandom}; +use std::collections::HashMap; +use std::sync::Arc; + +pub use graph::impl_slog_value; +use graph::prelude::Error; + +use crate::adapter::EthereumAdapter as _; +use crate::capabilities::NodeCapabilities; +use crate::EthereumAdapter; + +#[derive(Clone)] +pub struct EthereumNetworkAdapter { + pub capabilities: NodeCapabilities, + adapter: Arc, + /// The maximum number of times this adapter can be used. We use the + /// strong_count on `adapter` to determine whether the adapter is above + /// that limit. That's a somewhat imprecise but convenient way to + /// determine the number of connections + limit: usize, +} + +#[derive(Clone)] +pub struct EthereumNetworkAdapters { + pub adapters: Vec, +} + +impl EthereumNetworkAdapters { + pub fn all_cheapest_with( + &self, + required_capabilities: &NodeCapabilities, + ) -> impl Iterator> + '_ { + let cheapest_sufficient_capability = self + .adapters + .iter() + .find(|adapter| &adapter.capabilities >= required_capabilities) + .map(|adapter| &adapter.capabilities); + + self.adapters + .iter() + .filter(move |adapter| Some(&adapter.capabilities) == cheapest_sufficient_capability) + .filter(|adapter| Arc::strong_count(&adapter.adapter) < adapter.limit) + .map(|adapter| adapter.adapter.cheap_clone()) + } + + pub fn cheapest_with( + &self, + required_capabilities: &NodeCapabilities, + ) -> Result, Error> { + // Select randomly from the cheapest adapters that have sufficent capabilities. + self.all_cheapest_with(required_capabilities) + .choose(&mut rand::thread_rng()) + .with_context(|| { + anyhow!( + "A matching Ethereum network with {:?} was not found.", + required_capabilities + ) + }) + } + + pub fn cheapest(&self) -> Option> { + // EthereumAdapters are sorted by their NodeCapabilities when the EthereumNetworks + // struct is instantiated so they do not need to be sorted here + self.adapters + .iter() + .next() + .map(|ethereum_network_adapter| ethereum_network_adapter.adapter.clone()) + } + + pub fn remove(&mut self, provider: &str) { + self.adapters + .retain(|adapter| adapter.adapter.provider() != provider); + } +} + +#[derive(Clone)] +pub struct EthereumNetworks { + pub networks: HashMap, +} + +impl EthereumNetworks { + pub fn new() -> EthereumNetworks { + EthereumNetworks { + networks: HashMap::new(), + } + } + + pub fn insert( + &mut self, + name: String, + capabilities: NodeCapabilities, + adapter: Arc, + limit: usize, + ) { + let network_adapters = self + .networks + .entry(name) + .or_insert(EthereumNetworkAdapters { adapters: vec![] }); + network_adapters.adapters.push(EthereumNetworkAdapter { + capabilities, + adapter: adapter.clone(), + limit, + }); + } + + pub fn remove(&mut self, name: &str, provider: &str) { + if let Some(adapters) = self.networks.get_mut(name) { + adapters.remove(provider); + } + } + + pub fn extend(&mut self, other_networks: EthereumNetworks) { + self.networks.extend(other_networks.networks); + } + + pub fn flatten(&self) -> Vec<(String, NodeCapabilities, Arc)> { + self.networks + .iter() + .flat_map(|(network_name, network_adapters)| { + network_adapters + .adapters + .iter() + .map(move |network_adapter| { + ( + network_name.clone(), + network_adapter.capabilities, + network_adapter.adapter.clone(), + ) + }) + }) + .collect() + } + + pub fn sort(&mut self) { + for adapters in self.networks.values_mut() { + adapters + .adapters + .sort_by_key(|adapter| adapter.capabilities) + } + } + + pub fn adapter_with_capabilities( + &self, + network_name: String, + requirements: &NodeCapabilities, + ) -> Result, Error> { + self.networks + .get(&network_name) + .ok_or(anyhow!("network not supported: {}", &network_name)) + .and_then(|adapters| adapters.cheapest_with(requirements)) + } +} + +#[cfg(test)] +mod tests { + use super::NodeCapabilities; + + #[test] + fn ethereum_capabilities_comparison() { + let archive = NodeCapabilities { + archive: true, + traces: false, + }; + let traces = NodeCapabilities { + archive: false, + traces: true, + }; + let archive_traces = NodeCapabilities { + archive: true, + traces: true, + }; + let full = NodeCapabilities { + archive: false, + traces: false, + }; + let full_traces = NodeCapabilities { + archive: false, + traces: true, + }; + + // Test all real combinations of capability comparisons + assert_eq!(false, &full >= &archive); + assert_eq!(false, &full >= &traces); + assert_eq!(false, &full >= &archive_traces); + assert_eq!(true, &full >= &full); + assert_eq!(false, &full >= &full_traces); + + assert_eq!(true, &archive >= &archive); + assert_eq!(false, &archive >= &traces); + assert_eq!(false, &archive >= &archive_traces); + assert_eq!(true, &archive >= &full); + assert_eq!(false, &archive >= &full_traces); + + assert_eq!(false, &traces >= &archive); + assert_eq!(true, &traces >= &traces); + assert_eq!(false, &traces >= &archive_traces); + assert_eq!(true, &traces >= &full); + assert_eq!(true, &traces >= &full_traces); + + assert_eq!(true, &archive_traces >= &archive); + assert_eq!(true, &archive_traces >= &traces); + assert_eq!(true, &archive_traces >= &archive_traces); + assert_eq!(true, &archive_traces >= &full); + assert_eq!(true, &archive_traces >= &full_traces); + + assert_eq!(false, &full_traces >= &archive); + assert_eq!(true, &full_traces >= &traces); + assert_eq!(false, &full_traces >= &archive_traces); + assert_eq!(true, &full_traces >= &full); + assert_eq!(true, &full_traces >= &full_traces); + } +} diff --git a/chain/ethereum/src/protobuf/.gitignore b/chain/ethereum/src/protobuf/.gitignore new file mode 100644 index 0000000..0a1cbe2 --- /dev/null +++ b/chain/ethereum/src/protobuf/.gitignore @@ -0,0 +1,3 @@ +# For an unknown reason, the build script generates this file but it should not. +# See https://github.com/hyperium/tonic/issues/757 +google.protobuf.rs \ No newline at end of file diff --git a/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs b/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs new file mode 100644 index 0000000..a4daf1e --- /dev/null +++ b/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs @@ -0,0 +1,588 @@ +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Block { + #[prost(int32, tag="1")] + pub ver: i32, + #[prost(bytes="vec", tag="2")] + pub hash: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="3")] + pub number: u64, + #[prost(uint64, tag="4")] + pub size: u64, + #[prost(message, optional, tag="5")] + pub header: ::core::option::Option, + /// Uncles represents block produced with a valid solution but were not actually choosen + /// as the canonical block for the given height so they are mostly "forked" blocks. + /// + /// If the Block has been produced using the Proof of Stake consensus algorithm, this + /// field will actually be always empty. + #[prost(message, repeated, tag="6")] + pub uncles: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="10")] + pub transaction_traces: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="11")] + pub balance_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="20")] + pub code_changes: ::prost::alloc::vec::Vec, +} +/// HeaderOnlyBlock is used to optimally unpack the \[Block\] structure (note the +/// corresponding message number for the `header` field) while consuming less +/// memory, when only the `header` is desired. +/// +/// WARN: this is a client-side optimization pattern and should be moved in the +/// consuming code. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct HeaderOnlyBlock { + #[prost(message, optional, tag="5")] + pub header: ::core::option::Option, +} +/// BlockWithRefs is a lightweight block, with traces and transactions +/// purged from the `block` within, and only. It is used in transports +/// to pass block data around. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockWithRefs { + #[prost(string, tag="1")] + pub id: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub block: ::core::option::Option, + #[prost(message, optional, tag="3")] + pub transaction_trace_refs: ::core::option::Option, + #[prost(bool, tag="4")] + pub irreversible: bool, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionRefs { + #[prost(bytes="vec", repeated, tag="1")] + pub hashes: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct UnclesHeaders { + #[prost(message, repeated, tag="1")] + pub uncles: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockRef { + #[prost(bytes="vec", tag="1")] + pub hash: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="2")] + pub number: u64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockHeader { + #[prost(bytes="vec", tag="1")] + pub parent_hash: ::prost::alloc::vec::Vec, + /// Uncle hash of the block, some reference it as `sha3Uncles`, but `sha3`` is badly worded, so we prefer `uncle_hash`, also + /// referred as `ommers` in EIP specification. + /// + /// If the Block containing this `BlockHeader` has been produced using the Proof of Stake + /// consensus algorithm, this field will actually be constant and set to `0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347`. + #[prost(bytes="vec", tag="2")] + pub uncle_hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="3")] + pub coinbase: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="4")] + pub state_root: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="5")] + pub transactions_root: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="6")] + pub receipt_root: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="7")] + pub logs_bloom: ::prost::alloc::vec::Vec, + /// Difficulty is the difficulty of the Proof of Work algorithm that was required to compute a solution. + /// + /// If the Block containing this `BlockHeader` has been produced using the Proof of Stake + /// consensus algorithm, this field will actually be constant and set to `0x00`. + #[prost(message, optional, tag="8")] + pub difficulty: ::core::option::Option, + /// TotalDifficulty is the sum of all previous blocks difficulty including this block difficulty. + /// + /// If the Block containing this `BlockHeader` has been produced using the Proof of Stake + /// consensus algorithm, this field will actually be constant and set to the terminal total difficulty + /// that was required to transition to Proof of Stake algorithm, which varies per network. It is set to + /// 58 750 000 000 000 000 000 000 on Ethereum Mainnet and to 10 790 000 on Ethereum Testnet Goerli. + #[prost(message, optional, tag="17")] + pub total_difficulty: ::core::option::Option, + #[prost(uint64, tag="9")] + pub number: u64, + #[prost(uint64, tag="10")] + pub gas_limit: u64, + #[prost(uint64, tag="11")] + pub gas_used: u64, + #[prost(message, optional, tag="12")] + pub timestamp: ::core::option::Option<::prost_types::Timestamp>, + /// ExtraData is free-form bytes included in the block by the "miner". While on Yellow paper of + /// Ethereum this value is maxed to 32 bytes, other consensus algorithm like Clique and some other + /// forks are using bigger values to carry special consensus data. + /// + /// If the Block containing this `BlockHeader` has been produced using the Proof of Stake + /// consensus algorithm, this field is strictly enforced to be <= 32 bytes. + #[prost(bytes="vec", tag="13")] + pub extra_data: ::prost::alloc::vec::Vec, + /// MixHash is used to prove, when combined with the `nonce` that sufficient amount of computation has been + /// achieved and that the solution found is valid. + #[prost(bytes="vec", tag="14")] + pub mix_hash: ::prost::alloc::vec::Vec, + /// Nonce is used to prove, when combined with the `mix_hash` that sufficient amount of computation has been + /// achieved and that the solution found is valid. + /// + /// If the Block containing this `BlockHeader` has been produced using the Proof of Stake + /// consensus algorithm, this field will actually be constant and set to `0`. + #[prost(uint64, tag="15")] + pub nonce: u64, + /// Hash is the hash of the block which is actually the computation: + /// + /// Keccak256(rlp([ + /// parent_hash, + /// uncle_hash, + /// coinbase, + /// state_root, + /// transactions_root, + /// receipt_root, + /// logs_bloom, + /// difficulty, + /// number, + /// gas_limit, + /// gas_used, + /// timestamp, + /// extra_data, + /// mix_hash, + /// nonce, + /// base_fee_per_gas + /// ])) + /// + #[prost(bytes="vec", tag="16")] + pub hash: ::prost::alloc::vec::Vec, + /// Base fee per gas according to EIP-1559 (e.g. London Fork) rules, only set if London is present/active on the chain. + #[prost(message, optional, tag="18")] + pub base_fee_per_gas: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BigInt { + #[prost(bytes="vec", tag="1")] + pub bytes: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionTrace { + /// consensus + #[prost(bytes="vec", tag="1")] + pub to: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="2")] + pub nonce: u64, + /// GasPrice represents the effective price that has been paid for each gas unit of this transaction. Over time, the + /// Ethereum rules changes regarding GasPrice field here. Before London fork, the GasPrice was always set to the + /// fixed gas price. After London fork, this value has different meaning depending on the transaction type (see `Type` field). + /// + /// In cases where `TransactionTrace.Type == TRX_TYPE_LEGACY || TRX_TYPE_ACCESS_LIST`, then GasPrice has the same meaning + /// as before the London fork. + /// + /// In cases where `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE`, then GasPrice is the effective gas price paid + /// for the transaction which is equals to `BlockHeader.BaseFeePerGas + TransactionTrace.` + #[prost(message, optional, tag="3")] + pub gas_price: ::core::option::Option, + /// GasLimit is the maximum of gas unit the sender of the transaction is willing to consume when perform the EVM + /// execution of the whole transaction + #[prost(uint64, tag="4")] + pub gas_limit: u64, + /// Value is the amount of Ether transferred as part of this transaction. + #[prost(message, optional, tag="5")] + pub value: ::core::option::Option, + /// Input data the transaction will receive for execution of EVM. + #[prost(bytes="vec", tag="6")] + pub input: ::prost::alloc::vec::Vec, + /// V is the recovery ID value for the signature Y point. + #[prost(bytes="vec", tag="7")] + pub v: ::prost::alloc::vec::Vec, + /// R is the signature's X point on the elliptic curve (32 bytes). + #[prost(bytes="vec", tag="8")] + pub r: ::prost::alloc::vec::Vec, + /// S is the signature's Y point on the elliptic curve (32 bytes). + #[prost(bytes="vec", tag="9")] + pub s: ::prost::alloc::vec::Vec, + /// GasUsed is the total amount of gas unit used for the whole execution of the transaction. + #[prost(uint64, tag="10")] + pub gas_used: u64, + /// Type represents the Ethereum transaction type, available only since EIP-2718 & EIP-2930 activation which happened on Berlin fork. + /// The value is always set even for transaction before Berlin fork because those before the fork are still legacy transactions. + #[prost(enumeration="transaction_trace::Type", tag="12")] + pub r#type: i32, + /// AcccessList represents the storage access this transaction has agreed to do in which case those storage + /// access cost less gas unit per access. + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_ACCESS_LIST || TRX_TYPE_DYNAMIC_FEE` which + /// is possible only if Berlin (TRX_TYPE_ACCESS_LIST) nor London (TRX_TYPE_DYNAMIC_FEE) fork are active on the chain. + #[prost(message, repeated, tag="14")] + pub access_list: ::prost::alloc::vec::Vec, + /// MaxFeePerGas is the maximum fee per gas the user is willing to pay for the transaction gas used. + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only + /// if London fork is active on the chain. + #[prost(message, optional, tag="11")] + pub max_fee_per_gas: ::core::option::Option, + /// MaxPriorityFeePerGas is priority fee per gas the user to pay in extra to the miner on top of the block's + /// base fee. + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only + /// if London fork is active on the chain. + #[prost(message, optional, tag="13")] + pub max_priority_fee_per_gas: ::core::option::Option, + /// meta + #[prost(uint32, tag="20")] + pub index: u32, + #[prost(bytes="vec", tag="21")] + pub hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="22")] + pub from: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="23")] + pub return_data: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="24")] + pub public_key: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="25")] + pub begin_ordinal: u64, + #[prost(uint64, tag="26")] + pub end_ordinal: u64, + #[prost(enumeration="TransactionTraceStatus", tag="30")] + pub status: i32, + #[prost(message, optional, tag="31")] + pub receipt: ::core::option::Option, + #[prost(message, repeated, tag="32")] + pub calls: ::prost::alloc::vec::Vec, +} +/// Nested message and enum types in `TransactionTrace`. +pub mod transaction_trace { + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] + #[repr(i32)] + pub enum Type { + /// All transactions that ever existed prior Berlin fork before EIP-2718 was implemented. + TrxTypeLegacy = 0, + /// Field that specifies an access list of contract/storage_keys that is going to be used + /// in this transaction. + /// + /// Added in Berlin fork (EIP-2930). + TrxTypeAccessList = 1, + /// Transaction that specifies an access list just like TRX_TYPE_ACCESS_LIST but in addition defines the + /// max base gas gee and max priority gas fee to pay for this transaction. Transaction's of those type are + /// executed against EIP-1559 rules which dictates a dynamic gas cost based on the congestion of the network. + TrxTypeDynamicFee = 2, + } +} +/// AccessTuple represents a list of storage keys for a given contract's address and is used +/// for AccessList construction. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AccessTuple { + #[prost(bytes="vec", tag="1")] + pub address: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", repeated, tag="2")] + pub storage_keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, +} +/// TransactionTraceWithBlockRef +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionTraceWithBlockRef { + #[prost(message, optional, tag="1")] + pub trace: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub block_ref: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionReceipt { + /// State root is an intermediate state_root hash, computed in-between transactions to make + /// **sure** you could build a proof and point to state in the middle of a block. Geth client + /// uses `PostState + root + PostStateOrStatus`` while Parity used `status_code, root...`` this piles + /// hardforks, see (read the EIPs first): + /// - + /// - + /// - + /// + /// Moreover, the notion of `Outcome`` in parity, which segregates the two concepts, which are + /// stored in the same field `status_code`` can be computed based on such a hack of the `state_root` + /// field, following `EIP-658`. + /// + /// Before Byzantinium hard fork, this field is always empty. + #[prost(bytes="vec", tag="1")] + pub state_root: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="2")] + pub cumulative_gas_used: u64, + #[prost(bytes="vec", tag="3")] + pub logs_bloom: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="4")] + pub logs: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Log { + #[prost(bytes="vec", tag="1")] + pub address: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", repeated, tag="2")] + pub topics: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + #[prost(bytes="vec", tag="3")] + pub data: ::prost::alloc::vec::Vec, + /// Index is the index of the log relative to the transaction. This index + /// is always populated regardless of the state revertion of the the call + /// that emitted this log. + #[prost(uint32, tag="4")] + pub index: u32, + /// BlockIndex represents the index of the log relative to the Block. + /// + /// An **important** notice is that this field will be 0 when the call + /// that emitted the log has been reverted by the chain. + /// + /// Currently, there is two locations where a Log can be obtained: + /// - block.transaction_traces\[].receipt.logs[\] + /// - block.transaction_traces\[].calls[].logs[\] + /// + /// In the `receipt` case, the logs will be populated only when the call + /// that emitted them has not been reverted by the chain and when in this + /// position, the `blockIndex` is always populated correctly. + /// + /// In the case of `calls` case, for `call` where `stateReverted == true`, + /// the `blockIndex` value will always be 0. + #[prost(uint32, tag="6")] + pub block_index: u32, + #[prost(uint64, tag="7")] + pub ordinal: u64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Call { + #[prost(uint32, tag="1")] + pub index: u32, + #[prost(uint32, tag="2")] + pub parent_index: u32, + #[prost(uint32, tag="3")] + pub depth: u32, + #[prost(enumeration="CallType", tag="4")] + pub call_type: i32, + #[prost(bytes="vec", tag="5")] + pub caller: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="6")] + pub address: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="7")] + pub value: ::core::option::Option, + #[prost(uint64, tag="8")] + pub gas_limit: u64, + #[prost(uint64, tag="9")] + pub gas_consumed: u64, + #[prost(bytes="vec", tag="13")] + pub return_data: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="14")] + pub input: ::prost::alloc::vec::Vec, + #[prost(bool, tag="15")] + pub executed_code: bool, + #[prost(bool, tag="16")] + pub suicide: bool, + /// hex representation of the hash -> preimage + #[prost(map="string, string", tag="20")] + pub keccak_preimages: ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, + #[prost(message, repeated, tag="21")] + pub storage_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="22")] + pub balance_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="24")] + pub nonce_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="25")] + pub logs: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="26")] + pub code_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="28")] + pub gas_changes: ::prost::alloc::vec::Vec, + /// In Ethereum, a call can be either: + /// - Successfull, execution passes without any problem encountered + /// - Failed, execution failed, and remaining gas should be consumed + /// - Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded + /// + /// When a call is either `failed` or `reverted`, the `status_failed` field + /// below is set to `true`. If the status is `reverted`, then both `status_failed` + /// and `status_reverted` are going to be set to `true`. + #[prost(bool, tag="10")] + pub status_failed: bool, + #[prost(bool, tag="12")] + pub status_reverted: bool, + /// Populated when a call either failed or reverted, so when `status_failed == true`, + /// see above for details about those flags. + #[prost(string, tag="11")] + pub failure_reason: ::prost::alloc::string::String, + /// This field represents wheter or not the state changes performed + /// by this call were correctly recorded by the blockchain. + /// + /// On Ethereum, a transaction can record state changes even if some + /// of its inner nested calls failed. This is problematic however since + /// a call will invalidate all its state changes as well as all state + /// changes performed by its child call. This means that even if a call + /// has a status of `SUCCESS`, the chain might have reverted all the state + /// changes it performed. + /// + /// ```text + /// Trx 1 + /// Call #1 + /// Call #2 + /// Call #3 + /// |--- Failure here + /// Call #4 + /// ``` + /// + /// In the transaction above, while Call #2 and Call #3 would have the + /// status `EXECUTED` + #[prost(bool, tag="30")] + pub state_reverted: bool, + #[prost(uint64, tag="31")] + pub begin_ordinal: u64, + #[prost(uint64, tag="32")] + pub end_ordinal: u64, + #[prost(message, repeated, tag="33")] + pub account_creations: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StorageChange { + #[prost(bytes="vec", tag="1")] + pub address: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="2")] + pub key: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="3")] + pub old_value: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="4")] + pub new_value: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="5")] + pub ordinal: u64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BalanceChange { + #[prost(bytes="vec", tag="1")] + pub address: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="2")] + pub old_value: ::core::option::Option, + #[prost(message, optional, tag="3")] + pub new_value: ::core::option::Option, + #[prost(enumeration="balance_change::Reason", tag="4")] + pub reason: i32, + #[prost(uint64, tag="5")] + pub ordinal: u64, +} +/// Nested message and enum types in `BalanceChange`. +pub mod balance_change { + /// Obtain all balanche change reasons under deep mind repository: + /// + /// ```shell + /// ack -ho 'BalanceChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq + /// ``` + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] + #[repr(i32)] + pub enum Reason { + Unknown = 0, + RewardMineUncle = 1, + RewardMineBlock = 2, + DaoRefundContract = 3, + DaoAdjustBalance = 4, + Transfer = 5, + GenesisBalance = 6, + GasBuy = 7, + RewardTransactionFee = 8, + RewardFeeReset = 14, + GasRefund = 9, + TouchAccount = 10, + SuicideRefund = 11, + SuicideWithdraw = 13, + CallBalanceOverride = 12, + /// Used on chain(s) where some Ether burning happens + Burn = 15, + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NonceChange { + #[prost(bytes="vec", tag="1")] + pub address: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="2")] + pub old_value: u64, + #[prost(uint64, tag="3")] + pub new_value: u64, + #[prost(uint64, tag="4")] + pub ordinal: u64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AccountCreation { + #[prost(bytes="vec", tag="1")] + pub account: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="2")] + pub ordinal: u64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CodeChange { + #[prost(bytes="vec", tag="1")] + pub address: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="2")] + pub old_hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="3")] + pub old_code: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="4")] + pub new_hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="5")] + pub new_code: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="6")] + pub ordinal: u64, +} +/// The gas change model represents the reason why some gas cost has occurred. +/// The gas is computed per actual op codes. Doing them completely might prove +/// overwhelming in most cases. +/// +/// Hence, we only index some of them, those that are costy like all the calls +/// one, log events, return data, etc. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GasChange { + #[prost(uint64, tag="1")] + pub old_value: u64, + #[prost(uint64, tag="2")] + pub new_value: u64, + #[prost(enumeration="gas_change::Reason", tag="3")] + pub reason: i32, + #[prost(uint64, tag="4")] + pub ordinal: u64, +} +/// Nested message and enum types in `GasChange`. +pub mod gas_change { + /// Obtain all gas change reasons under deep mind repository: + /// + /// ```shell + /// ack -ho 'GasChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq + /// ``` + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] + #[repr(i32)] + pub enum Reason { + Unknown = 0, + Call = 1, + CallCode = 2, + CallDataCopy = 3, + CodeCopy = 4, + CodeStorage = 5, + ContractCreation = 6, + ContractCreation2 = 7, + DelegateCall = 8, + EventLog = 9, + ExtCodeCopy = 10, + FailedExecution = 11, + IntrinsicGas = 12, + PrecompiledContract = 13, + RefundAfterExecution = 14, + Return = 15, + ReturnDataCopy = 16, + Revert = 17, + SelfDestruct = 18, + StaticCall = 19, + /// Added in Berlin fork (Geth 1.10+) + StateColdAccess = 20, + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum TransactionTraceStatus { + Unknown = 0, + Succeeded = 1, + Failed = 2, + Reverted = 3, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum CallType { + Unspecified = 0, + /// direct? what's the name for `Call` alone? + Call = 1, + Callcode = 2, + Delegate = 3, + Static = 4, + /// create2 ? any other form of calls? + Create = 5, +} diff --git a/chain/ethereum/src/runtime/abi.rs b/chain/ethereum/src/runtime/abi.rs new file mode 100644 index 0000000..572dfb7 --- /dev/null +++ b/chain/ethereum/src/runtime/abi.rs @@ -0,0 +1,783 @@ +use super::runtime_adapter::UnresolvedContractCall; +use crate::trigger::{ + EthereumBlockData, EthereumCallData, EthereumEventData, EthereumTransactionData, +}; +use graph::{ + prelude::{ + ethabi, + web3::types::{Log, TransactionReceipt, H256}, + BigInt, + }, + runtime::{ + asc_get, asc_new, gas::GasCounter, AscHeap, AscIndexId, AscPtr, AscType, + DeterministicHostError, FromAscObj, IndexForAscTypeId, ToAscObj, + }, +}; +use graph_runtime_derive::AscType; +use graph_runtime_wasm::asc_abi::class::{ + Array, AscAddress, AscBigInt, AscEnum, AscH160, AscString, AscWrapped, EthereumValueKind, + Uint8Array, +}; +use semver::Version; + +type AscH256 = Uint8Array; +type AscH2048 = Uint8Array; + +pub struct AscLogParamArray(Array>); + +impl AscType for AscLogParamArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl ToAscObj for Vec { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); + let content = content?; + Ok(AscLogParamArray(Array::new(&*content, heap, gas)?)) + } +} + +impl AscIndexId for AscLogParamArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayEventParam; +} + +pub struct AscTopicArray(Array>); + +impl AscType for AscTopicArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl ToAscObj for Vec { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let topics = self + .iter() + .map(|topic| asc_new(heap, topic, gas)) + .collect::, _>>()?; + Ok(AscTopicArray(Array::new(&topics, heap, gas)?)) + } +} + +impl AscIndexId for AscTopicArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayH256; +} + +pub struct AscLogArray(Array>); + +impl AscType for AscLogArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl ToAscObj for Vec { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let logs = self + .iter() + .map(|log| asc_new(heap, &log, gas)) + .collect::, _>>()?; + Ok(AscLogArray(Array::new(&logs, heap, gas)?)) + } +} + +impl AscIndexId for AscLogArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayLog; +} + +#[repr(C)] +#[derive(AscType)] +pub struct AscUnresolvedContractCall_0_0_4 { + pub contract_name: AscPtr, + pub contract_address: AscPtr, + pub function_name: AscPtr, + pub function_signature: AscPtr, + pub function_args: AscPtr>>>, +} + +impl AscIndexId for AscUnresolvedContractCall_0_0_4 { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::SmartContractCall; +} + +impl FromAscObj for UnresolvedContractCall { + fn from_asc_obj( + asc_call: AscUnresolvedContractCall_0_0_4, + heap: &H, + gas: &GasCounter, + ) -> Result { + Ok(UnresolvedContractCall { + contract_name: asc_get(heap, asc_call.contract_name, gas)?, + contract_address: asc_get(heap, asc_call.contract_address, gas)?, + function_name: asc_get(heap, asc_call.function_name, gas)?, + function_signature: Some(asc_get(heap, asc_call.function_signature, gas)?), + function_args: asc_get(heap, asc_call.function_args, gas)?, + }) + } +} + +#[repr(C)] +#[derive(AscType)] +pub struct AscUnresolvedContractCall { + pub contract_name: AscPtr, + pub contract_address: AscPtr, + pub function_name: AscPtr, + pub function_args: AscPtr>>>, +} + +impl FromAscObj for UnresolvedContractCall { + fn from_asc_obj( + asc_call: AscUnresolvedContractCall, + heap: &H, + gas: &GasCounter, + ) -> Result { + Ok(UnresolvedContractCall { + contract_name: asc_get(heap, asc_call.contract_name, gas)?, + contract_address: asc_get(heap, asc_call.contract_address, gas)?, + function_name: asc_get(heap, asc_call.function_name, gas)?, + function_signature: None, + function_args: asc_get(heap, asc_call.function_args, gas)?, + }) + } +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscEthereumBlock { + pub hash: AscPtr, + pub parent_hash: AscPtr, + pub uncles_hash: AscPtr, + pub author: AscPtr, + pub state_root: AscPtr, + pub transactions_root: AscPtr, + pub receipts_root: AscPtr, + pub number: AscPtr, + pub gas_used: AscPtr, + pub gas_limit: AscPtr, + pub timestamp: AscPtr, + pub difficulty: AscPtr, + pub total_difficulty: AscPtr, + pub size: AscPtr, +} + +impl AscIndexId for AscEthereumBlock { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumBlock; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscEthereumBlock_0_0_6 { + pub hash: AscPtr, + pub parent_hash: AscPtr, + pub uncles_hash: AscPtr, + pub author: AscPtr, + pub state_root: AscPtr, + pub transactions_root: AscPtr, + pub receipts_root: AscPtr, + pub number: AscPtr, + pub gas_used: AscPtr, + pub gas_limit: AscPtr, + pub timestamp: AscPtr, + pub difficulty: AscPtr, + pub total_difficulty: AscPtr, + pub size: AscPtr, + pub base_fee_per_block: AscPtr, +} + +impl AscIndexId for AscEthereumBlock_0_0_6 { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumBlock; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscEthereumTransaction_0_0_1 { + pub hash: AscPtr, + pub index: AscPtr, + pub from: AscPtr, + pub to: AscPtr, + pub value: AscPtr, + pub gas_limit: AscPtr, + pub gas_price: AscPtr, +} + +impl AscIndexId for AscEthereumTransaction_0_0_1 { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumTransaction; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscEthereumTransaction_0_0_2 { + pub hash: AscPtr, + pub index: AscPtr, + pub from: AscPtr, + pub to: AscPtr, + pub value: AscPtr, + pub gas_limit: AscPtr, + pub gas_price: AscPtr, + pub input: AscPtr, +} + +impl AscIndexId for AscEthereumTransaction_0_0_2 { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumTransaction; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscEthereumTransaction_0_0_6 { + pub hash: AscPtr, + pub index: AscPtr, + pub from: AscPtr, + pub to: AscPtr, + pub value: AscPtr, + pub gas_limit: AscPtr, + pub gas_price: AscPtr, + pub input: AscPtr, + pub nonce: AscPtr, +} + +impl AscIndexId for AscEthereumTransaction_0_0_6 { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumTransaction; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscEthereumEvent +where + T: AscType, + B: AscType, +{ + pub address: AscPtr, + pub log_index: AscPtr, + pub transaction_log_index: AscPtr, + pub log_type: AscPtr, + pub block: AscPtr, + pub transaction: AscPtr, + pub params: AscPtr, +} + +impl AscIndexId for AscEthereumEvent { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumEvent; +} + +impl AscIndexId for AscEthereumEvent { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumEvent; +} + +impl AscIndexId for AscEthereumEvent { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumEvent; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscEthereumLog { + pub address: AscPtr, + pub topics: AscPtr, + pub data: AscPtr, + pub block_hash: AscPtr, + pub block_number: AscPtr, + pub transaction_hash: AscPtr, + pub transaction_index: AscPtr, + pub log_index: AscPtr, + pub transaction_log_index: AscPtr, + pub log_type: AscPtr, + pub removed: AscPtr>, +} + +impl AscIndexId for AscEthereumLog { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::Log; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscEthereumTransactionReceipt { + pub transaction_hash: AscPtr, + pub transaction_index: AscPtr, + pub block_hash: AscPtr, + pub block_number: AscPtr, + pub cumulative_gas_used: AscPtr, + pub gas_used: AscPtr, + pub contract_address: AscPtr, + pub logs: AscPtr, + pub status: AscPtr, + pub root: AscPtr, + pub logs_bloom: AscPtr, +} + +impl AscIndexId for AscEthereumTransactionReceipt { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::TransactionReceipt; +} + +/// Introduced in API Version 0.0.7, this is the same as [`AscEthereumEvent`] with an added +/// `receipt` field. +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscEthereumEvent_0_0_7 +where + T: AscType, + B: AscType, +{ + pub address: AscPtr, + pub log_index: AscPtr, + pub transaction_log_index: AscPtr, + pub log_type: AscPtr, + pub block: AscPtr, + pub transaction: AscPtr, + pub params: AscPtr, + pub receipt: AscPtr, +} + +impl AscIndexId for AscEthereumEvent_0_0_7 { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumEvent; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscLogParam { + pub name: AscPtr, + pub value: AscPtr>, +} + +impl AscIndexId for AscLogParam { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EventParam; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscEthereumCall { + pub address: AscPtr, + pub block: AscPtr, + pub transaction: AscPtr, + pub inputs: AscPtr, + pub outputs: AscPtr, +} + +impl AscIndexId for AscEthereumCall { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumCall; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscEthereumCall_0_0_3 +where + T: AscType, + B: AscType, +{ + pub to: AscPtr, + pub from: AscPtr, + pub block: AscPtr, + pub transaction: AscPtr, + pub inputs: AscPtr, + pub outputs: AscPtr, +} + +impl AscIndexId for AscEthereumCall_0_0_3 +where + T: AscType, + B: AscType, +{ + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumCall; +} + +impl ToAscObj for EthereumBlockData { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscEthereumBlock { + hash: asc_new(heap, &self.hash, gas)?, + parent_hash: asc_new(heap, &self.parent_hash, gas)?, + uncles_hash: asc_new(heap, &self.uncles_hash, gas)?, + author: asc_new(heap, &self.author, gas)?, + state_root: asc_new(heap, &self.state_root, gas)?, + transactions_root: asc_new(heap, &self.transactions_root, gas)?, + receipts_root: asc_new(heap, &self.receipts_root, gas)?, + number: asc_new(heap, &BigInt::from(self.number), gas)?, + gas_used: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_used), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, + timestamp: asc_new(heap, &BigInt::from_unsigned_u256(&self.timestamp), gas)?, + difficulty: asc_new(heap, &BigInt::from_unsigned_u256(&self.difficulty), gas)?, + total_difficulty: asc_new( + heap, + &BigInt::from_unsigned_u256(&self.total_difficulty), + gas, + )?, + size: self + .size + .map(|size| asc_new(heap, &BigInt::from_unsigned_u256(&size), gas)) + .unwrap_or(Ok(AscPtr::null()))?, + }) + } +} + +impl ToAscObj for EthereumBlockData { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscEthereumBlock_0_0_6 { + hash: asc_new(heap, &self.hash, gas)?, + parent_hash: asc_new(heap, &self.parent_hash, gas)?, + uncles_hash: asc_new(heap, &self.uncles_hash, gas)?, + author: asc_new(heap, &self.author, gas)?, + state_root: asc_new(heap, &self.state_root, gas)?, + transactions_root: asc_new(heap, &self.transactions_root, gas)?, + receipts_root: asc_new(heap, &self.receipts_root, gas)?, + number: asc_new(heap, &BigInt::from(self.number), gas)?, + gas_used: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_used), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, + timestamp: asc_new(heap, &BigInt::from_unsigned_u256(&self.timestamp), gas)?, + difficulty: asc_new(heap, &BigInt::from_unsigned_u256(&self.difficulty), gas)?, + total_difficulty: asc_new( + heap, + &BigInt::from_unsigned_u256(&self.total_difficulty), + gas, + )?, + size: self + .size + .map(|size| asc_new(heap, &BigInt::from_unsigned_u256(&size), gas)) + .unwrap_or(Ok(AscPtr::null()))?, + base_fee_per_block: self + .base_fee_per_gas + .map(|base_fee| asc_new(heap, &BigInt::from_unsigned_u256(&base_fee), gas)) + .unwrap_or(Ok(AscPtr::null()))?, + }) + } +} + +impl ToAscObj for EthereumTransactionData { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscEthereumTransaction_0_0_1 { + hash: asc_new(heap, &self.hash, gas)?, + index: asc_new(heap, &BigInt::from(self.index), gas)?, + from: asc_new(heap, &self.from, gas)?, + to: self + .to + .map(|to| asc_new(heap, &to, gas)) + .unwrap_or(Ok(AscPtr::null()))?, + value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, + gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, + }) + } +} + +impl ToAscObj for EthereumTransactionData { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscEthereumTransaction_0_0_2 { + hash: asc_new(heap, &self.hash, gas)?, + index: asc_new(heap, &BigInt::from(self.index), gas)?, + from: asc_new(heap, &self.from, gas)?, + to: self + .to + .map(|to| asc_new(heap, &to, gas)) + .unwrap_or(Ok(AscPtr::null()))?, + value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, + gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, + input: asc_new(heap, &*self.input, gas)?, + }) + } +} + +impl ToAscObj for EthereumTransactionData { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscEthereumTransaction_0_0_6 { + hash: asc_new(heap, &self.hash, gas)?, + index: asc_new(heap, &BigInt::from(self.index), gas)?, + from: asc_new(heap, &self.from, gas)?, + to: self + .to + .map(|to| asc_new(heap, &to, gas)) + .unwrap_or(Ok(AscPtr::null()))?, + value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, + gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, + input: asc_new(heap, &*self.input, gas)?, + nonce: asc_new(heap, &BigInt::from_unsigned_u256(&self.nonce), gas)?, + }) + } +} + +impl ToAscObj> for EthereumEventData +where + T: AscType + AscIndexId, + B: AscType + AscIndexId, + EthereumTransactionData: ToAscObj, + EthereumBlockData: ToAscObj, +{ + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + Ok(AscEthereumEvent { + address: asc_new(heap, &self.address, gas)?, + log_index: asc_new(heap, &BigInt::from_unsigned_u256(&self.log_index), gas)?, + transaction_log_index: asc_new( + heap, + &BigInt::from_unsigned_u256(&self.transaction_log_index), + gas, + )?, + log_type: self + .log_type + .clone() + .map(|log_type| asc_new(heap, &log_type, gas)) + .unwrap_or(Ok(AscPtr::null()))?, + block: asc_new::(heap, &self.block, gas)?, + transaction: asc_new::(heap, &self.transaction, gas)?, + params: asc_new(heap, &self.params, gas)?, + }) + } +} + +impl ToAscObj> + for (EthereumEventData, Option<&TransactionReceipt>) +where + T: AscType + AscIndexId, + B: AscType + AscIndexId, + EthereumTransactionData: ToAscObj, + EthereumBlockData: ToAscObj, +{ + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + let (event_data, optional_receipt) = self; + let AscEthereumEvent { + address, + log_index, + transaction_log_index, + log_type, + block, + transaction, + params, + } = event_data.to_asc_obj(heap, gas)?; + let receipt = if let Some(receipt_data) = optional_receipt { + asc_new(heap, receipt_data, gas)? + } else { + AscPtr::null() + }; + Ok(AscEthereumEvent_0_0_7 { + address, + log_index, + transaction_log_index, + log_type, + block, + transaction, + params, + receipt, + }) + } +} + +impl ToAscObj for Log { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscEthereumLog { + address: asc_new(heap, &self.address, gas)?, + topics: asc_new(heap, &self.topics, gas)?, + data: asc_new(heap, self.data.0.as_slice(), gas)?, + block_hash: self + .block_hash + .map(|block_hash| asc_new(heap, &block_hash, gas)) + .unwrap_or(Ok(AscPtr::null()))?, + block_number: self + .block_number + .map(|block_number| asc_new(heap, &BigInt::from(block_number), gas)) + .unwrap_or(Ok(AscPtr::null()))?, + transaction_hash: self + .transaction_hash + .map(|txn_hash| asc_new(heap, &txn_hash, gas)) + .unwrap_or(Ok(AscPtr::null()))?, + transaction_index: self + .transaction_index + .map(|txn_index| asc_new(heap, &BigInt::from(txn_index), gas)) + .unwrap_or(Ok(AscPtr::null()))?, + log_index: self + .log_index + .map(|log_index| asc_new(heap, &BigInt::from_unsigned_u256(&log_index), gas)) + .unwrap_or(Ok(AscPtr::null()))?, + transaction_log_index: self + .transaction_log_index + .map(|index| asc_new(heap, &BigInt::from_unsigned_u256(&index), gas)) + .unwrap_or(Ok(AscPtr::null()))?, + log_type: self + .log_type + .as_ref() + .map(|log_type| asc_new(heap, &log_type, gas)) + .unwrap_or(Ok(AscPtr::null()))?, + removed: self + .removed + .map(|removed| asc_new(heap, &AscWrapped { inner: removed }, gas)) + .unwrap_or(Ok(AscPtr::null()))?, + }) + } +} + +impl ToAscObj for &TransactionReceipt { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscEthereumTransactionReceipt { + transaction_hash: asc_new(heap, &self.transaction_hash, gas)?, + transaction_index: asc_new(heap, &BigInt::from(self.transaction_index), gas)?, + block_hash: self + .block_hash + .map(|block_hash| asc_new(heap, &block_hash, gas)) + .unwrap_or(Ok(AscPtr::null()))?, + block_number: self + .block_number + .map(|block_number| asc_new(heap, &BigInt::from(block_number), gas)) + .unwrap_or(Ok(AscPtr::null()))?, + cumulative_gas_used: asc_new( + heap, + &BigInt::from_unsigned_u256(&self.cumulative_gas_used), + gas, + )?, + gas_used: self + .gas_used + .map(|gas_used| asc_new(heap, &BigInt::from_unsigned_u256(&gas_used), gas)) + .unwrap_or(Ok(AscPtr::null()))?, + contract_address: self + .contract_address + .map(|contract_address| asc_new(heap, &contract_address, gas)) + .unwrap_or(Ok(AscPtr::null()))?, + logs: asc_new(heap, &self.logs, gas)?, + status: self + .status + .map(|status| asc_new(heap, &BigInt::from(status), gas)) + .unwrap_or(Ok(AscPtr::null()))?, + root: self + .root + .map(|root| asc_new(heap, &root, gas)) + .unwrap_or(Ok(AscPtr::null()))?, + logs_bloom: asc_new(heap, self.logs_bloom.as_bytes(), gas)?, + }) + } +} + +impl ToAscObj for EthereumCallData { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscEthereumCall { + address: asc_new(heap, &self.to, gas)?, + block: asc_new(heap, &self.block, gas)?, + transaction: asc_new(heap, &self.transaction, gas)?, + inputs: asc_new(heap, &self.inputs, gas)?, + outputs: asc_new(heap, &self.outputs, gas)?, + }) + } +} + +impl ToAscObj> + for EthereumCallData +{ + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result< + AscEthereumCall_0_0_3, + DeterministicHostError, + > { + Ok(AscEthereumCall_0_0_3 { + to: asc_new(heap, &self.to, gas)?, + from: asc_new(heap, &self.from, gas)?, + block: asc_new(heap, &self.block, gas)?, + transaction: asc_new(heap, &self.transaction, gas)?, + inputs: asc_new(heap, &self.inputs, gas)?, + outputs: asc_new(heap, &self.outputs, gas)?, + }) + } +} + +impl ToAscObj> + for EthereumCallData +{ + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result< + AscEthereumCall_0_0_3, + DeterministicHostError, + > { + Ok(AscEthereumCall_0_0_3 { + to: asc_new(heap, &self.to, gas)?, + from: asc_new(heap, &self.from, gas)?, + block: asc_new(heap, &self.block, gas)?, + transaction: asc_new(heap, &self.transaction, gas)?, + inputs: asc_new(heap, &self.inputs, gas)?, + outputs: asc_new(heap, &self.outputs, gas)?, + }) + } +} + +impl ToAscObj for ethabi::LogParam { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscLogParam { + name: asc_new(heap, self.name.as_str(), gas)?, + value: asc_new(heap, &self.value, gas)?, + }) + } +} diff --git a/chain/ethereum/src/runtime/mod.rs b/chain/ethereum/src/runtime/mod.rs new file mode 100644 index 0000000..42f0a9d --- /dev/null +++ b/chain/ethereum/src/runtime/mod.rs @@ -0,0 +1,4 @@ +pub use runtime_adapter::RuntimeAdapter; + +pub mod abi; +pub mod runtime_adapter; diff --git a/chain/ethereum/src/runtime/runtime_adapter.rs b/chain/ethereum/src/runtime/runtime_adapter.rs new file mode 100644 index 0000000..e3f4a17 --- /dev/null +++ b/chain/ethereum/src/runtime/runtime_adapter.rs @@ -0,0 +1,226 @@ +use std::{sync::Arc, time::Instant}; + +use crate::data_source::MappingABI; +use crate::{ + capabilities::NodeCapabilities, network::EthereumNetworkAdapters, Chain, DataSource, + EthereumAdapter, EthereumAdapterTrait, EthereumContractCall, EthereumContractCallError, +}; +use anyhow::{Context, Error}; +use blockchain::HostFn; +use graph::runtime::gas::Gas; +use graph::runtime::{AscIndexId, IndexForAscTypeId}; +use graph::{ + blockchain::{self, BlockPtr, HostFnCtx}, + cheap_clone::CheapClone, + prelude::{ + ethabi::{self, Address, Token}, + EthereumCallCache, Future01CompatExt, + }, + runtime::{asc_get, asc_new, AscPtr, HostExportError}, + semver::Version, + slog::{info, trace, Logger}, +}; +use graph_runtime_wasm::asc_abi::class::{AscEnumArray, EthereumValueKind}; + +use super::abi::{AscUnresolvedContractCall, AscUnresolvedContractCall_0_0_4}; + +// When making an ethereum call, the maximum ethereum gas is ETH_CALL_GAS which is 50 million. One +// unit of Ethereum gas is at least 100ns according to these benchmarks [1], so 1000 of our gas. In +// the worst case an Ethereum call could therefore consume 50 billion of our gas. However the +// averarge call a subgraph makes is much cheaper or even cached in the call cache. So this cost is +// set to 5 billion gas as a compromise. This allows for 2000 calls per handler with the current +// limits. +// +// [1] - https://www.sciencedirect.com/science/article/abs/pii/S0166531620300900 +pub const ETHEREUM_CALL: Gas = Gas::new(5_000_000_000); + +pub struct RuntimeAdapter { + pub eth_adapters: Arc, + pub call_cache: Arc, +} + +impl blockchain::RuntimeAdapter for RuntimeAdapter { + fn host_fns(&self, ds: &DataSource) -> Result, Error> { + let abis = ds.mapping.abis.clone(); + let call_cache = self.call_cache.cheap_clone(); + let eth_adapter = self + .eth_adapters + .cheapest_with(&NodeCapabilities { + archive: ds.mapping.requires_archive()?, + traces: false, + })? + .cheap_clone(); + + let ethereum_call = HostFn { + name: "ethereum.call", + func: Arc::new(move |ctx, wasm_ptr| { + ethereum_call(ð_adapter, call_cache.cheap_clone(), ctx, wasm_ptr, &abis) + .map(|ptr| ptr.wasm_ptr()) + }), + }; + + Ok(vec![ethereum_call]) + } +} + +/// function ethereum.call(call: SmartContractCall): Array | null +fn ethereum_call( + eth_adapter: &EthereumAdapter, + call_cache: Arc, + ctx: HostFnCtx<'_>, + wasm_ptr: u32, + abis: &[Arc], +) -> Result, HostExportError> { + ctx.gas.consume_host_fn(ETHEREUM_CALL)?; + + // For apiVersion >= 0.0.4 the call passed from the mapping includes the + // function signature; subgraphs using an apiVersion < 0.0.4 don't pass + // the signature along with the call. + let call: UnresolvedContractCall = if ctx.heap.api_version() >= Version::new(0, 0, 4) { + asc_get::<_, AscUnresolvedContractCall_0_0_4, _>(ctx.heap, wasm_ptr.into(), &ctx.gas)? + } else { + asc_get::<_, AscUnresolvedContractCall, _>(ctx.heap, wasm_ptr.into(), &ctx.gas)? + }; + + let result = eth_call( + eth_adapter, + call_cache, + &ctx.logger, + &ctx.block_ptr, + call, + abis, + )?; + match result { + Some(tokens) => Ok(asc_new(ctx.heap, tokens.as_slice(), &ctx.gas)?), + None => Ok(AscPtr::null()), + } +} + +/// Returns `Ok(None)` if the call was reverted. +fn eth_call( + eth_adapter: &EthereumAdapter, + call_cache: Arc, + logger: &Logger, + block_ptr: &BlockPtr, + unresolved_call: UnresolvedContractCall, + abis: &[Arc], +) -> Result>, HostExportError> { + let start_time = Instant::now(); + + // Obtain the path to the contract ABI + let contract = abis + .iter() + .find(|abi| abi.name == unresolved_call.contract_name) + .with_context(|| { + format!( + "Could not find ABI for contract \"{}\", try adding it to the 'abis' section \ + of the subgraph manifest", + unresolved_call.contract_name + ) + })? + .contract + .clone(); + + let function = match unresolved_call.function_signature { + // Behavior for apiVersion < 0.0.4: look up function by name; for overloaded + // functions this always picks the same overloaded variant, which is incorrect + // and may lead to encoding/decoding errors + None => contract + .function(unresolved_call.function_name.as_str()) + .with_context(|| { + format!( + "Unknown function \"{}::{}\" called from WASM runtime", + unresolved_call.contract_name, unresolved_call.function_name + ) + })?, + + // Behavior for apiVersion >= 0.0.04: look up function by signature of + // the form `functionName(uint256,string) returns (bytes32,string)`; this + // correctly picks the correct variant of an overloaded function + Some(ref function_signature) => contract + .functions_by_name(unresolved_call.function_name.as_str()) + .with_context(|| { + format!( + "Unknown function \"{}::{}\" called from WASM runtime", + unresolved_call.contract_name, unresolved_call.function_name + ) + })? + .iter() + .find(|f| function_signature == &f.signature()) + .with_context(|| { + format!( + "Unknown function \"{}::{}\" with signature `{}` \ + called from WASM runtime", + unresolved_call.contract_name, + unresolved_call.function_name, + function_signature, + ) + })?, + }; + + let call = EthereumContractCall { + address: unresolved_call.contract_address, + block_ptr: block_ptr.cheap_clone(), + function: function.clone(), + args: unresolved_call.function_args.clone(), + }; + + // Run Ethereum call in tokio runtime + let logger1 = logger.clone(); + let call_cache = call_cache.clone(); + let result = match graph::block_on( + eth_adapter.contract_call(&logger1, call, call_cache).compat() + ) { + Ok(tokens) => Ok(Some(tokens)), + Err(EthereumContractCallError::Revert(reason)) => { + info!(logger, "Contract call reverted"; "reason" => reason); + Ok(None) + } + + // Any error reported by the Ethereum node could be due to the block no longer being on + // the main chain. This is very unespecific but we don't want to risk failing a + // subgraph due to a transient error such as a reorg. + Err(EthereumContractCallError::Web3Error(e)) => Err(HostExportError::PossibleReorg(anyhow::anyhow!( + "Ethereum node returned an error when calling function \"{}\" of contract \"{}\": {}", + unresolved_call.function_name, + unresolved_call.contract_name, + e + ))), + + // Also retry on timeouts. + Err(EthereumContractCallError::Timeout) => Err(HostExportError::PossibleReorg(anyhow::anyhow!( + "Ethereum node did not respond when calling function \"{}\" of contract \"{}\"", + unresolved_call.function_name, + unresolved_call.contract_name, + ))), + + Err(e) => Err(HostExportError::Unknown(anyhow::anyhow!( + "Failed to call function \"{}\" of contract \"{}\": {}", + unresolved_call.function_name, + unresolved_call.contract_name, + e + ))), + }; + + trace!(logger, "Contract call finished"; + "address" => &unresolved_call.contract_address.to_string(), + "contract" => &unresolved_call.contract_name, + "function" => &unresolved_call.function_name, + "function_signature" => &unresolved_call.function_signature, + "time" => format!("{}ms", start_time.elapsed().as_millis())); + + result +} + +#[derive(Clone, Debug)] +pub struct UnresolvedContractCall { + pub contract_name: String, + pub contract_address: Address, + pub function_name: String, + pub function_signature: Option, + pub function_args: Vec, +} + +impl AscIndexId for AscUnresolvedContractCall { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::SmartContractCall; +} diff --git a/chain/ethereum/src/tests.rs b/chain/ethereum/src/tests.rs new file mode 100644 index 0000000..eee594f --- /dev/null +++ b/chain/ethereum/src/tests.rs @@ -0,0 +1,97 @@ +use std::sync::Arc; + +use graph::{ + blockchain::{block_stream::BlockWithTriggers, BlockPtr}, + prelude::{ + web3::types::{Address, Bytes, Log, H160, H256, U64}, + EthereumCall, + }, +}; + +use crate::{ + chain::BlockFinality, + trigger::{EthereumBlockTriggerType, EthereumTrigger}, +}; + +#[test] +fn test_trigger_ordering() { + let block1 = EthereumTrigger::Block( + BlockPtr::from((H256::random(), 1u64)), + EthereumBlockTriggerType::Every, + ); + + let block2 = EthereumTrigger::Block( + BlockPtr::from((H256::random(), 0u64)), + EthereumBlockTriggerType::WithCallTo(Address::random()), + ); + + let mut call1 = EthereumCall::default(); + call1.transaction_index = 1; + let call1 = EthereumTrigger::Call(Arc::new(call1)); + + let mut call2 = EthereumCall::default(); + call2.transaction_index = 2; + let call2 = EthereumTrigger::Call(Arc::new(call2)); + + let mut call3 = EthereumCall::default(); + call3.transaction_index = 3; + let call3 = EthereumTrigger::Call(Arc::new(call3)); + + // Call with the same tx index as call2 + let mut call4 = EthereumCall::default(); + call4.transaction_index = 2; + let call4 = EthereumTrigger::Call(Arc::new(call4)); + + fn create_log(tx_index: u64, log_index: u64) -> Arc { + Arc::new(Log { + address: H160::default(), + topics: vec![], + data: Bytes::default(), + block_hash: Some(H256::zero()), + block_number: Some(U64::zero()), + transaction_hash: Some(H256::zero()), + transaction_index: Some(tx_index.into()), + log_index: Some(log_index.into()), + transaction_log_index: Some(log_index.into()), + log_type: Some("".into()), + removed: Some(false), + }) + } + + // Event with transaction_index 1 and log_index 0; + // should be the first element after sorting + let log1 = EthereumTrigger::Log(create_log(1, 0), None); + + // Event with transaction_index 1 and log_index 1; + // should be the second element after sorting + let log2 = EthereumTrigger::Log(create_log(1, 1), None); + + // Event with transaction_index 2 and log_index 5; + // should come after call1 and before call2 after sorting + let log3 = EthereumTrigger::Log(create_log(2, 5), None); + + let triggers = vec![ + // Call triggers; these should be in the order 1, 2, 4, 3 after sorting + call3.clone(), + call1.clone(), + call2.clone(), + call4.clone(), + // Block triggers; these should appear at the end after sorting + // but with their order unchanged + block2.clone(), + block1.clone(), + // Event triggers + log3.clone(), + log2.clone(), + log1.clone(), + ]; + + // Test that `BlockWithTriggers` sorts the triggers. + let block_with_triggers = + BlockWithTriggers::::new(BlockFinality::Final(Default::default()), triggers); + + assert_eq!( + block_with_triggers.trigger_data, + vec![log1, log2, call1, log3, call2, call4, call3, block2, block1] + ); +} diff --git a/chain/ethereum/src/transport.rs b/chain/ethereum/src/transport.rs new file mode 100644 index 0000000..2d4302c --- /dev/null +++ b/chain/ethereum/src/transport.rs @@ -0,0 +1,88 @@ +use jsonrpc_core::types::Call; +use jsonrpc_core::Value; + +use web3::transports::{http, ipc, ws}; +use web3::RequestId; + +use graph::prelude::*; +use graph::url::Url; +use std::future::Future; + +/// Abstraction over the different web3 transports. +#[derive(Clone, Debug)] +pub enum Transport { + RPC(http::Http), + IPC(ipc::Ipc), + WS(ws::WebSocket), +} + +impl Transport { + /// Creates an IPC transport. + #[cfg(unix)] + pub async fn new_ipc(ipc: &str) -> Self { + ipc::Ipc::new(ipc) + .await + .map(|transport| Transport::IPC(transport)) + .expect("Failed to connect to Ethereum IPC") + } + + /// Creates a WebSocket transport. + pub async fn new_ws(ws: &str) -> Self { + ws::WebSocket::new(ws) + .await + .map(|transport| Transport::WS(transport)) + .expect("Failed to connect to Ethereum WS") + } + + /// Creates a JSON-RPC over HTTP transport. + /// + /// Note: JSON-RPC over HTTP doesn't always support subscribing to new + /// blocks (one such example is Infura's HTTP endpoint). + pub fn new_rpc(rpc: Url, headers: ::http::HeaderMap) -> Self { + // Unwrap: This only fails if something is wrong with the system's TLS config. + let client = reqwest::Client::builder() + .default_headers(headers) + .build() + .unwrap(); + Transport::RPC(http::Http::with_client(client, rpc)) + } +} + +impl web3::Transport for Transport { + type Out = Box> + Send + Unpin>; + + fn prepare(&self, method: &str, params: Vec) -> (RequestId, Call) { + match self { + Transport::RPC(http) => http.prepare(method, params), + Transport::IPC(ipc) => ipc.prepare(method, params), + Transport::WS(ws) => ws.prepare(method, params), + } + } + + fn send(&self, id: RequestId, request: Call) -> Self::Out { + match self { + Transport::RPC(http) => Box::new(http.send(id, request)), + Transport::IPC(ipc) => Box::new(ipc.send(id, request)), + Transport::WS(ws) => Box::new(ws.send(id, request)), + } + } +} + +impl web3::BatchTransport for Transport { + type Batch = Box< + dyn Future>, web3::error::Error>> + + Send + + Unpin, + >; + + fn send_batch(&self, requests: T) -> Self::Batch + where + T: IntoIterator, + { + match self { + Transport::RPC(http) => Box::new(http.send_batch(requests)), + Transport::IPC(ipc) => Box::new(ipc.send_batch(requests)), + Transport::WS(ws) => Box::new(ws.send_batch(requests)), + } + } +} diff --git a/chain/ethereum/src/trigger.rs b/chain/ethereum/src/trigger.rs new file mode 100644 index 0000000..9d3a4a8 --- /dev/null +++ b/chain/ethereum/src/trigger.rs @@ -0,0 +1,433 @@ +use graph::blockchain::TriggerData; +use graph::data::subgraph::API_VERSION_0_0_2; +use graph::data::subgraph::API_VERSION_0_0_6; +use graph::data::subgraph::API_VERSION_0_0_7; +use graph::prelude::ethabi::ethereum_types::H160; +use graph::prelude::ethabi::ethereum_types::H256; +use graph::prelude::ethabi::ethereum_types::U128; +use graph::prelude::ethabi::ethereum_types::U256; +use graph::prelude::ethabi::ethereum_types::U64; +use graph::prelude::ethabi::Address; +use graph::prelude::ethabi::Bytes; +use graph::prelude::ethabi::LogParam; +use graph::prelude::web3::types::Block; +use graph::prelude::web3::types::Log; +use graph::prelude::web3::types::Transaction; +use graph::prelude::web3::types::TransactionReceipt; +use graph::prelude::BlockNumber; +use graph::prelude::BlockPtr; +use graph::prelude::{CheapClone, EthereumCall}; +use graph::runtime::asc_new; +use graph::runtime::gas::GasCounter; +use graph::runtime::AscHeap; +use graph::runtime::AscPtr; +use graph::runtime::DeterministicHostError; +use graph::semver::Version; +use graph_runtime_wasm::module::ToAscPtr; +use std::convert::TryFrom; +use std::ops::Deref; +use std::{cmp::Ordering, sync::Arc}; + +use crate::runtime::abi::AscEthereumBlock; +use crate::runtime::abi::AscEthereumBlock_0_0_6; +use crate::runtime::abi::AscEthereumCall; +use crate::runtime::abi::AscEthereumCall_0_0_3; +use crate::runtime::abi::AscEthereumEvent; +use crate::runtime::abi::AscEthereumEvent_0_0_7; +use crate::runtime::abi::AscEthereumTransaction_0_0_1; +use crate::runtime::abi::AscEthereumTransaction_0_0_2; +use crate::runtime::abi::AscEthereumTransaction_0_0_6; + +// ETHDEP: This should be defined in only one place. +type LightEthereumBlock = Block; + +pub enum MappingTrigger { + Log { + block: Arc, + transaction: Arc, + log: Arc, + params: Vec, + receipt: Option>, + }, + Call { + block: Arc, + transaction: Arc, + call: Arc, + inputs: Vec, + outputs: Vec, + }, + Block { + block: Arc, + }, +} + +// Logging the block is too verbose, so this strips the block from the trigger for Debug. +impl std::fmt::Debug for MappingTrigger { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + #[derive(Debug)] + enum MappingTriggerWithoutBlock { + Log { + _transaction: Arc, + _log: Arc, + _params: Vec, + }, + Call { + _transaction: Arc, + _call: Arc, + _inputs: Vec, + _outputs: Vec, + }, + Block, + } + + let trigger_without_block = match self { + MappingTrigger::Log { + block: _, + transaction, + log, + params, + receipt: _, + } => MappingTriggerWithoutBlock::Log { + _transaction: transaction.cheap_clone(), + _log: log.cheap_clone(), + _params: params.clone(), + }, + MappingTrigger::Call { + block: _, + transaction, + call, + inputs, + outputs, + } => MappingTriggerWithoutBlock::Call { + _transaction: transaction.cheap_clone(), + _call: call.cheap_clone(), + _inputs: inputs.clone(), + _outputs: outputs.clone(), + }, + MappingTrigger::Block { block: _ } => MappingTriggerWithoutBlock::Block, + }; + + write!(f, "{:?}", trigger_without_block) + } +} + +impl ToAscPtr for MappingTrigger { + fn to_asc_ptr( + self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + Ok(match self { + MappingTrigger::Log { + block, + transaction, + log, + params, + receipt, + } => { + let api_version = heap.api_version(); + let ethereum_event_data = EthereumEventData { + block: EthereumBlockData::from(block.as_ref()), + transaction: EthereumTransactionData::from(transaction.deref()), + address: log.address, + log_index: log.log_index.unwrap_or(U256::zero()), + transaction_log_index: log.log_index.unwrap_or(U256::zero()), + log_type: log.log_type.clone(), + params, + }; + if api_version >= API_VERSION_0_0_7 { + asc_new::< + AscEthereumEvent_0_0_7< + AscEthereumTransaction_0_0_6, + AscEthereumBlock_0_0_6, + >, + _, + _, + >(heap, &(ethereum_event_data, receipt.as_deref()), gas)? + .erase() + } else if api_version >= API_VERSION_0_0_6 { + asc_new::< + AscEthereumEvent, + _, + _, + >(heap, ðereum_event_data, gas)? + .erase() + } else if api_version >= API_VERSION_0_0_2 { + asc_new::< + AscEthereumEvent, + _, + _, + >(heap, ðereum_event_data, gas)? + .erase() + } else { + asc_new::< + AscEthereumEvent, + _, + _, + >(heap, ðereum_event_data, gas)? + .erase() + } + } + MappingTrigger::Call { + block, + transaction, + call, + inputs, + outputs, + } => { + let call = EthereumCallData { + to: call.to, + from: call.from, + block: EthereumBlockData::from(block.as_ref()), + transaction: EthereumTransactionData::from(transaction.deref()), + inputs, + outputs, + }; + if heap.api_version() >= Version::new(0, 0, 6) { + asc_new::< + AscEthereumCall_0_0_3, + _, + _, + >(heap, &call, gas)? + .erase() + } else if heap.api_version() >= Version::new(0, 0, 3) { + asc_new::< + AscEthereumCall_0_0_3, + _, + _, + >(heap, &call, gas)? + .erase() + } else { + asc_new::(heap, &call, gas)?.erase() + } + } + MappingTrigger::Block { block } => { + let block = EthereumBlockData::from(block.as_ref()); + if heap.api_version() >= Version::new(0, 0, 6) { + asc_new::(heap, &block, gas)?.erase() + } else { + asc_new::(heap, &block, gas)?.erase() + } + } + }) + } +} + +#[derive(Clone, Debug)] +pub enum EthereumTrigger { + Block(BlockPtr, EthereumBlockTriggerType), + Call(Arc), + Log(Arc, Option>), +} + +impl PartialEq for EthereumTrigger { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Block(a_ptr, a_kind), Self::Block(b_ptr, b_kind)) => { + a_ptr == b_ptr && a_kind == b_kind + } + + (Self::Call(a), Self::Call(b)) => a == b, + + (Self::Log(a, a_receipt), Self::Log(b, b_receipt)) => { + a.transaction_hash == b.transaction_hash + && a.log_index == b.log_index + && a_receipt == b_receipt + } + + _ => false, + } + } +} + +impl Eq for EthereumTrigger {} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EthereumBlockTriggerType { + Every, + WithCallTo(Address), +} + +impl EthereumTrigger { + pub fn block_number(&self) -> BlockNumber { + match self { + EthereumTrigger::Block(block_ptr, _) => block_ptr.number, + EthereumTrigger::Call(call) => call.block_number, + EthereumTrigger::Log(log, _) => { + i32::try_from(log.block_number.unwrap().as_u64()).unwrap() + } + } + } + + pub fn block_hash(&self) -> H256 { + match self { + EthereumTrigger::Block(block_ptr, _) => block_ptr.hash_as_h256(), + EthereumTrigger::Call(call) => call.block_hash, + EthereumTrigger::Log(log, _) => log.block_hash.unwrap(), + } + } +} + +impl Ord for EthereumTrigger { + fn cmp(&self, other: &Self) -> Ordering { + match (self, other) { + // Keep the order when comparing two block triggers + (Self::Block(..), Self::Block(..)) => Ordering::Equal, + + // Block triggers always come last + (Self::Block(..), _) => Ordering::Greater, + (_, Self::Block(..)) => Ordering::Less, + + // Calls are ordered by their tx indexes + (Self::Call(a), Self::Call(b)) => a.transaction_index.cmp(&b.transaction_index), + + // Events are ordered by their log index + (Self::Log(a, _), Self::Log(b, _)) => a.log_index.cmp(&b.log_index), + + // Calls vs. events are logged by their tx index; + // if they are from the same transaction, events come first + (Self::Call(a), Self::Log(b, _)) + if a.transaction_index == b.transaction_index.unwrap().as_u64() => + { + Ordering::Greater + } + (Self::Log(a, _), Self::Call(b)) + if a.transaction_index.unwrap().as_u64() == b.transaction_index => + { + Ordering::Less + } + (Self::Call(a), Self::Log(b, _)) => a + .transaction_index + .cmp(&b.transaction_index.unwrap().as_u64()), + (Self::Log(a, _), Self::Call(b)) => a + .transaction_index + .unwrap() + .as_u64() + .cmp(&b.transaction_index), + } + } +} + +impl PartialOrd for EthereumTrigger { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl TriggerData for EthereumTrigger { + fn error_context(&self) -> std::string::String { + let transaction_id = match self { + EthereumTrigger::Log(log, _) => log.transaction_hash, + EthereumTrigger::Call(call) => call.transaction_hash, + EthereumTrigger::Block(..) => None, + }; + + match transaction_id { + Some(tx_hash) => format!( + "block #{} ({}), transaction {:x}", + self.block_number(), + self.block_hash(), + tx_hash + ), + None => String::new(), + } + } +} + +/// Ethereum block data. +#[derive(Clone, Debug, Default)] +pub struct EthereumBlockData { + pub hash: H256, + pub parent_hash: H256, + pub uncles_hash: H256, + pub author: H160, + pub state_root: H256, + pub transactions_root: H256, + pub receipts_root: H256, + pub number: U64, + pub gas_used: U256, + pub gas_limit: U256, + pub timestamp: U256, + pub difficulty: U256, + pub total_difficulty: U256, + pub size: Option, + pub base_fee_per_gas: Option, +} + +impl<'a, T> From<&'a Block> for EthereumBlockData { + fn from(block: &'a Block) -> EthereumBlockData { + EthereumBlockData { + hash: block.hash.unwrap(), + parent_hash: block.parent_hash, + uncles_hash: block.uncles_hash, + author: block.author, + state_root: block.state_root, + transactions_root: block.transactions_root, + receipts_root: block.receipts_root, + number: block.number.unwrap(), + gas_used: block.gas_used, + gas_limit: block.gas_limit, + timestamp: block.timestamp, + difficulty: block.difficulty, + total_difficulty: block.total_difficulty.unwrap_or_default(), + size: block.size, + base_fee_per_gas: block.base_fee_per_gas, + } + } +} + +/// Ethereum transaction data. +#[derive(Clone, Debug)] +pub struct EthereumTransactionData { + pub hash: H256, + pub index: U128, + pub from: H160, + pub to: Option, + pub value: U256, + pub gas_limit: U256, + pub gas_price: U256, + pub input: Bytes, + pub nonce: U256, +} + +impl From<&'_ Transaction> for EthereumTransactionData { + fn from(tx: &Transaction) -> EthereumTransactionData { + // unwrap: this is always `Some` for txns that have been mined + // (see https://github.com/tomusdrw/rust-web3/pull/407) + let from = tx.from.unwrap(); + EthereumTransactionData { + hash: tx.hash, + index: tx.transaction_index.unwrap().as_u64().into(), + from, + to: tx.to, + value: tx.value, + gas_limit: tx.gas, + gas_price: tx.gas_price.unwrap_or(U256::zero()), // EIP-1559 made this optional. + input: tx.input.0.clone(), + nonce: tx.nonce.clone(), + } + } +} + +/// An Ethereum event logged from a specific contract address and block. +#[derive(Debug, Clone)] +pub struct EthereumEventData { + pub address: Address, + pub log_index: U256, + pub transaction_log_index: U256, + pub log_type: Option, + pub block: EthereumBlockData, + pub transaction: EthereumTransactionData, + pub params: Vec, +} + +/// An Ethereum call executed within a transaction within a block to a contract address. +#[derive(Debug, Clone)] +pub struct EthereumCallData { + pub from: Address, + pub to: Address, + pub block: EthereumBlockData, + pub transaction: EthereumTransactionData, + pub inputs: Vec, + pub outputs: Vec, +} diff --git a/chain/ethereum/tests/full-text.graphql b/chain/ethereum/tests/full-text.graphql new file mode 100644 index 0000000..8fd9451 --- /dev/null +++ b/chain/ethereum/tests/full-text.graphql @@ -0,0 +1,25 @@ +# Schema used for unit tests. + +type _Schema_ + @fulltext( + name: "bandSearch" + language: en + algorithm: rank + include: [ + { + entity: "Band" + fields: [{ name: "name" }, { name: "description" }, { name: "bio" }] + } + ] + ) + +type Band @entity { + id: ID! + name: String! + description: String! + bio: String + wallet: Address + labels: [Label!]! + discography: [Album!]! + members: [Musician!]! +} diff --git a/chain/ethereum/tests/ipfs-on-ethereum-contracts.ts b/chain/ethereum/tests/ipfs-on-ethereum-contracts.ts new file mode 100644 index 0000000..ac5f70a --- /dev/null +++ b/chain/ethereum/tests/ipfs-on-ethereum-contracts.ts @@ -0,0 +1,27 @@ +export enum ValueKind { + STRING = 0, + INT = 1, + BIGDECIMAL = 2, + BOOL = 3, + ARRAY = 4, + NULL = 5, + BYTES = 6, + BIGINT = 7, +} + +export type ValuePayload = u64; + +export class Value { + constructor(public kind: ValueKind, public data: ValuePayload) {} +} + +declare namespace ipfs { + function cat(hash: String): void + function map(hash: String, callback: String, userData: Value, flags: String[]): void +} + +export function foo(): void { + let value = new Value(0, 0) + ipfs.cat("") + ipfs.map("", "", value, []) +} diff --git a/chain/ethereum/tests/ipfs-on-ethereum-contracts.wasm b/chain/ethereum/tests/ipfs-on-ethereum-contracts.wasm new file mode 100644 index 0000000..21f37cc Binary files /dev/null and b/chain/ethereum/tests/ipfs-on-ethereum-contracts.wasm differ diff --git a/chain/ethereum/tests/manifest.rs b/chain/ethereum/tests/manifest.rs new file mode 100644 index 0000000..2c82f1f --- /dev/null +++ b/chain/ethereum/tests/manifest.rs @@ -0,0 +1,797 @@ +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use graph::data::subgraph::schema::SubgraphError; +use graph::data::subgraph::{SPEC_VERSION_0_0_4, SPEC_VERSION_0_0_7}; +use graph::data_source::DataSourceTemplate; +use graph::prelude::{ + anyhow, async_trait, serde_yaml, tokio, DeploymentHash, Entity, Link, Logger, SubgraphManifest, + SubgraphManifestValidationError, UnvalidatedSubgraphManifest, +}; +use graph::{ + blockchain::NodeCapabilities as _, + components::{ + link_resolver::{JsonValueStream, LinkResolver as LinkResolverTrait}, + store::EntityType, + }, + data::subgraph::SubgraphFeature, +}; + +use graph_chain_ethereum::{Chain, NodeCapabilities}; +use semver::Version; +use test_store::LOGGER; + +const GQL_SCHEMA: &str = "type Thing @entity { id: ID! }"; +const GQL_SCHEMA_FULLTEXT: &str = include_str!("full-text.graphql"); +const MAPPING_WITH_IPFS_FUNC_WASM: &[u8] = include_bytes!("ipfs-on-ethereum-contracts.wasm"); +const ABI: &str = "[{\"type\":\"function\", \"inputs\": [{\"name\": \"i\",\"type\": \"uint256\"}],\"name\":\"get\",\"outputs\": [{\"type\": \"address\",\"name\": \"o\"}]}]"; +const FILE: &str = "{}"; +const FILE_CID: &str = "bafkreigkhuldxkyfkoaye4rgcqcwr45667vkygd45plwq6hawy7j4rbdky"; + +#[derive(Default, Debug, Clone)] +struct TextResolver { + texts: HashMap>, +} + +impl TextResolver { + fn add(&mut self, link: &str, text: &impl AsRef<[u8]>) { + self.texts.insert( + link.to_owned(), + text.as_ref().into_iter().cloned().collect(), + ); + } +} + +#[async_trait] +impl LinkResolverTrait for TextResolver { + fn with_timeout(&self, _timeout: Duration) -> Box { + Box::new(self.clone()) + } + + fn with_retries(&self) -> Box { + Box::new(self.clone()) + } + + async fn cat(&self, _logger: &Logger, link: &Link) -> Result, anyhow::Error> { + self.texts + .get(&link.link) + .ok_or(anyhow!("No text for {}", &link.link)) + .map(Clone::clone) + } + + async fn get_block(&self, _logger: &Logger, _link: &Link) -> Result, anyhow::Error> { + unimplemented!() + } + + async fn json_stream( + &self, + _logger: &Logger, + _link: &Link, + ) -> Result { + unimplemented!() + } +} + +async fn resolve_manifest( + text: &str, + max_spec_version: Version, +) -> SubgraphManifest { + let mut resolver = TextResolver::default(); + let id = DeploymentHash::new("Qmmanifest").unwrap(); + + resolver.add(id.as_str(), &text); + resolver.add("/ipfs/Qmschema", &GQL_SCHEMA); + resolver.add("/ipfs/Qmabi", &ABI); + resolver.add("/ipfs/Qmmapping", &MAPPING_WITH_IPFS_FUNC_WASM); + resolver.add(FILE_CID, &FILE); + + let resolver: Arc = Arc::new(resolver); + + let raw = serde_yaml::from_str(text).unwrap(); + SubgraphManifest::resolve_from_raw(id, raw, &resolver, &LOGGER, max_spec_version) + .await + .expect("Parsing simple manifest works") +} + +async fn resolve_unvalidated(text: &str) -> UnvalidatedSubgraphManifest { + let mut resolver = TextResolver::default(); + let id = DeploymentHash::new("Qmmanifest").unwrap(); + + resolver.add(id.as_str(), &text); + resolver.add("/ipfs/Qmschema", &GQL_SCHEMA); + + let resolver: Arc = Arc::new(resolver); + + let raw = serde_yaml::from_str(text).unwrap(); + UnvalidatedSubgraphManifest::resolve(id, raw, &resolver, &LOGGER, SPEC_VERSION_0_0_4.clone()) + .await + .expect("Parsing simple manifest works") +} + +// Some of these manifest tests should be made chain-independent, but for +// now we just run them for the ethereum `Chain` + +#[tokio::test] +async fn simple_manifest() { + const YAML: &str = " +dataSources: [] +schema: + file: + /: /ipfs/Qmschema +specVersion: 0.0.2 +"; + + let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; + + assert_eq!("Qmmanifest", manifest.id.as_str()); + assert!(manifest.graft.is_none()); +} + +#[tokio::test] +async fn ipfs_manifest() { + let yaml = " +schema: + file: + /: /ipfs/Qmschema +dataSources: [] +templates: + - name: IpfsSource + kind: file/ipfs + mapping: + apiVersion: 0.0.6 + language: wasm/assemblyscript + entities: + - TestEntity + file: + /: /ipfs/Qmmapping + handler: handleFile +specVersion: 0.0.7 +"; + + let manifest = resolve_manifest(&yaml, SPEC_VERSION_0_0_7).await; + + assert_eq!("Qmmanifest", manifest.id.as_str()); + assert_eq!(manifest.data_sources.len(), 0); + let data_source = match &manifest.templates[0] { + DataSourceTemplate::Offchain(ds) => ds, + DataSourceTemplate::Onchain(_) => unreachable!(), + }; + assert_eq!(data_source.kind, "file/ipfs"); +} + +#[tokio::test] +async fn graft_manifest() { + const YAML: &str = " +dataSources: [] +schema: + file: + /: /ipfs/Qmschema +graft: + base: Qmbase + block: 12345 +specVersion: 0.0.2 +"; + + let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; + + assert_eq!("Qmmanifest", manifest.id.as_str()); + let graft = manifest.graft.expect("The manifest has a graft base"); + assert_eq!("Qmbase", graft.base.as_str()); + assert_eq!(12345, graft.block); +} + +#[test] +fn graft_failed_subgraph() { + const YAML: &str = " +dataSources: [] +schema: + file: + /: /ipfs/Qmschema +graft: + base: Qmbase + block: 0 +specVersion: 0.0.2 +"; + + test_store::run_test_sequentially(|store| async move { + let subgraph_store = store.subgraph_store(); + + let unvalidated = resolve_unvalidated(YAML).await; + let subgraph = DeploymentHash::new("Qmbase").unwrap(); + + // Creates base subgraph at block 0 (genesis). + let deployment = test_store::create_test_subgraph(&subgraph, GQL_SCHEMA).await; + + // Adds an example entity. + let mut thing = Entity::new(); + thing.set("id", "datthing"); + test_store::insert_entities(&deployment, vec![(EntityType::from("Thing"), thing)]) + .await + .unwrap(); + + let error = SubgraphError { + subgraph_id: deployment.hash.clone(), + message: "deterministic error".to_string(), + block_ptr: Some(test_store::BLOCKS[1].clone()), + handler: None, + deterministic: true, + }; + + // Fails the base subgraph at block 1 (and advances the pointer). + test_store::transact_errors( + &store, + &deployment, + test_store::BLOCKS[1].clone(), + vec![error], + ) + .await + .unwrap(); + + // Make sure there are no GraftBaseInvalid errors. + // + // This is allowed because: + // - base: failed at block 1 + // - graft: starts at block 0 + // + // Meaning that the graft will fail just like it's parent + // but it started at a valid previous block. + assert!( + unvalidated + .validate(subgraph_store.clone(), true) + .await + .expect_err("Validation must fail") + .into_iter() + .find(|e| matches!(e, SubgraphManifestValidationError::GraftBaseInvalid(_))) + .is_none(), + "There shouldn't be a GraftBaseInvalid error" + ); + + // Resolve the graft normally. + let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; + + assert_eq!("Qmmanifest", manifest.id.as_str()); + let graft = manifest.graft.expect("The manifest has a graft base"); + assert_eq!("Qmbase", graft.base.as_str()); + assert_eq!(0, graft.block); + }) +} + +#[test] +fn graft_invalid_manifest() { + const YAML: &str = " +dataSources: [] +schema: + file: + /: /ipfs/Qmschema +graft: + base: Qmbase + block: 1 +specVersion: 0.0.2 +"; + + test_store::run_test_sequentially(|store| async move { + let subgraph_store = store.subgraph_store(); + + let unvalidated = resolve_unvalidated(YAML).await; + let subgraph = DeploymentHash::new("Qmbase").unwrap(); + + // + // Validation against subgraph that hasn't synced anything fails + // + let deployment = test_store::create_test_subgraph(&subgraph, GQL_SCHEMA).await; + // This check is awkward since the test manifest has other problems + // that the validation complains about as setting up a valid manifest + // would be a bit more work; we just want to make sure that + // graft-related checks work + let msg = unvalidated + .validate(subgraph_store.clone(), true) + .await + .expect_err("Validation must fail") + .into_iter() + .find(|e| matches!(e, SubgraphManifestValidationError::GraftBaseInvalid(_))) + .expect("There must be a GraftBaseInvalid error") + .to_string(); + assert_eq!( + "the graft base is invalid: failed to graft onto `Qmbase` since \ + it has not processed any blocks", + msg + ); + + let mut thing = Entity::new(); + thing.set("id", "datthing"); + test_store::insert_entities(&deployment, vec![(EntityType::from("Thing"), thing)]) + .await + .unwrap(); + + // Validation against subgraph that has not reached the graft point fails + let unvalidated = resolve_unvalidated(YAML).await; + let msg = unvalidated + .validate(subgraph_store.clone(), true) + .await + .expect_err("Validation must fail") + .into_iter() + .find(|e| matches!(e, SubgraphManifestValidationError::GraftBaseInvalid(_))) + .expect("There must be a GraftBaseInvalid error") + .to_string(); + assert_eq!( + "the graft base is invalid: failed to graft onto `Qmbase` \ + at block 1 since it has only processed block 0", + msg + ); + + let error = SubgraphError { + subgraph_id: deployment.hash.clone(), + message: "deterministic error".to_string(), + block_ptr: Some(test_store::BLOCKS[1].clone()), + handler: None, + deterministic: true, + }; + + test_store::transact_errors( + &store, + &deployment, + test_store::BLOCKS[1].clone(), + vec![error], + ) + .await + .unwrap(); + + // This check is bit awkward, but we just want to be sure there is a + // GraftBaseInvalid error. + // + // The validation error happens because: + // - base: failed at block 1 + // - graft: starts at block 1 + // + // Since we start grafts at N + 1, we can't allow a graft to be created + // at the failed block. They (developers) should choose a previous valid + // block. + let unvalidated = resolve_unvalidated(YAML).await; + let msg = unvalidated + .validate(subgraph_store, true) + .await + .expect_err("Validation must fail") + .into_iter() + .find(|e| matches!(e, SubgraphManifestValidationError::GraftBaseInvalid(_))) + .expect("There must be a GraftBaseInvalid error") + .to_string(); + assert_eq!( + "the graft base is invalid: failed to graft onto `Qmbase` \ + at block 1 since it's not healthy. You can graft it starting at block 0 backwards", + msg + ); + }) +} + +#[tokio::test] +async fn parse_call_handlers() { + const YAML: &str = " +dataSources: + - kind: ethereum/contract + name: Factory + network: mainnet + source: + abi: Factory + startBlock: 9562480 + mapping: + kind: ethereum/events + apiVersion: 0.0.4 + language: wasm/assemblyscript + entities: + - TestEntity + file: + /: /ipfs/Qmmapping + abis: + - name: Factory + file: + /: /ipfs/Qmabi + callHandlers: + - function: get(address) + handler: handleget +schema: + file: + /: /ipfs/Qmschema +specVersion: 0.0.2 +"; + + let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; + let onchain_data_sources = manifest + .data_sources + .iter() + .filter_map(|ds| ds.as_onchain().cloned()) + .collect::>(); + let required_capabilities = NodeCapabilities::from_data_sources(&onchain_data_sources); + + assert_eq!("Qmmanifest", manifest.id.as_str()); + assert_eq!(true, required_capabilities.traces); +} + +#[test] +fn undeclared_grafting_feature_causes_feature_validation_error() { + const YAML: &str = " +specVersion: 0.0.4 +dataSources: [] +schema: + file: + /: /ipfs/Qmschema +graft: + base: Qmbase + block: 1 +"; + test_store::run_test_sequentially(|store| async move { + let store = store.subgraph_store(); + let unvalidated = resolve_unvalidated(YAML).await; + let error_msg = unvalidated + .validate(store.clone(), true) + .await + .expect_err("Validation must fail") + .into_iter() + .find(|e| { + matches!( + e, + SubgraphManifestValidationError::FeatureValidationError(_) + ) + }) + .expect("There must be a FeatureValidation error") + .to_string(); + assert_eq!( + "The feature `grafting` is used by the subgraph but it is not declared in the manifest.", + error_msg + ) + }) +} + +#[test] +fn declared_grafting_feature_causes_no_feature_validation_errors() { + const YAML: &str = " +specVersion: 0.0.4 +features: + - grafting +dataSources: [] +schema: + file: + /: /ipfs/Qmschema +graft: + base: Qmbase + block: 1 +"; + test_store::run_test_sequentially(|store| async move { + let store = store.subgraph_store(); + let unvalidated = resolve_unvalidated(YAML).await; + assert!(unvalidated + .validate(store.clone(), true) + .await + .expect_err("Validation must fail") + .into_iter() + .find(|e| { + matches!( + e, + SubgraphManifestValidationError::FeatureValidationError(_) + ) + }) + .is_none()); + let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; + assert!(manifest.features.contains(&SubgraphFeature::Grafting)) + }) +} + +#[test] +fn declared_non_fatal_errors_feature_causes_no_feature_validation_errors() { + const YAML: &str = " +specVersion: 0.0.4 +features: + - nonFatalErrors +dataSources: [] +schema: + file: + /: /ipfs/Qmschema +"; + test_store::run_test_sequentially(|store| async move { + let store = store.subgraph_store(); + let unvalidated = resolve_unvalidated(YAML).await; + assert!(unvalidated + .validate(store.clone(), true) + .await + .expect_err("Validation must fail") + .into_iter() + .find(|e| { + matches!( + e, + SubgraphManifestValidationError::FeatureValidationError(_) + ) + }) + .is_none()); + + let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; + assert!(manifest.features.contains(&SubgraphFeature::NonFatalErrors)) + }); +} + +#[test] +fn declared_full_text_search_feature_causes_no_feature_validation_errors() { + const YAML: &str = " +specVersion: 0.0.4 +features: + - fullTextSearch +dataSources: [] +schema: + file: + /: /ipfs/Qmschema +"; + + test_store::run_test_sequentially(|store| async move { + let store = store.subgraph_store(); + let unvalidated: UnvalidatedSubgraphManifest = { + let mut resolver = TextResolver::default(); + let id = DeploymentHash::new("Qmmanifest").unwrap(); + resolver.add(id.as_str(), &YAML); + resolver.add("/ipfs/Qmabi", &ABI); + resolver.add("/ipfs/Qmschema", &GQL_SCHEMA_FULLTEXT); + + let resolver: Arc = Arc::new(resolver); + + let raw = serde_yaml::from_str(YAML).unwrap(); + UnvalidatedSubgraphManifest::resolve( + id, + raw, + &resolver, + &LOGGER, + SPEC_VERSION_0_0_4.clone(), + ) + .await + .expect("Parsing simple manifest works") + }; + + assert!(unvalidated + .validate(store.clone(), true) + .await + .expect_err("Validation must fail") + .into_iter() + .find(|e| { + matches!( + e, + SubgraphManifestValidationError::FeatureValidationError(_) + ) + }) + .is_none()); + + let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; + assert!(manifest.features.contains(&SubgraphFeature::FullTextSearch)) + }); +} + +#[test] +fn undeclared_full_text_search_feature_causes_no_feature_validation_errors() { + const YAML: &str = " +specVersion: 0.0.4 + +dataSources: [] +schema: + file: + /: /ipfs/Qmschema +"; + + test_store::run_test_sequentially(|store| async move { + let store = store.subgraph_store(); + let unvalidated: UnvalidatedSubgraphManifest = { + let mut resolver = TextResolver::default(); + let id = DeploymentHash::new("Qmmanifest").unwrap(); + resolver.add(id.as_str(), &YAML); + resolver.add("/ipfs/Qmabi", &ABI); + resolver.add("/ipfs/Qmschema", &GQL_SCHEMA_FULLTEXT); + + let resolver: Arc = Arc::new(resolver); + + let raw = serde_yaml::from_str(YAML).unwrap(); + UnvalidatedSubgraphManifest::resolve( + id, + raw, + &resolver, + &LOGGER, + SPEC_VERSION_0_0_4.clone(), + ) + .await + .expect("Parsing simple manifest works") + }; + + let error_msg = unvalidated + .validate(store.clone(), true) + .await + .expect_err("Validation must fail") + .into_iter() + .find(|e| { + matches!( + e, + SubgraphManifestValidationError::FeatureValidationError(_) + ) + }) + .expect("There must be a FeatureValidationError") + .to_string(); + + assert_eq!( + "The feature `fullTextSearch` is used by the subgraph but it is not declared in the manifest.", + error_msg + ); + }); +} + +#[test] +fn undeclared_ipfs_on_ethereum_contracts_feature_causes_feature_validation_error() { + const YAML: &str = " +specVersion: 0.0.4 +schema: + file: + /: /ipfs/Qmschema +dataSources: + - kind: ethereum/contract + name: Factory + network: mainnet + source: + abi: Factory + startBlock: 9562480 + mapping: + kind: ethereum/events + apiVersion: 0.0.4 + language: wasm/assemblyscript + entities: + - TestEntity + file: + /: /ipfs/Qmmapping + abis: + - name: Factory + file: + /: /ipfs/Qmabi + callHandlers: + - function: get(address) + handler: handleget +"; + + test_store::run_test_sequentially(|store| async move { + let store = store.subgraph_store(); + let unvalidated: UnvalidatedSubgraphManifest = { + let mut resolver = TextResolver::default(); + let id = DeploymentHash::new("Qmmanifest").unwrap(); + resolver.add(id.as_str(), &YAML); + resolver.add("/ipfs/Qmabi", &ABI); + resolver.add("/ipfs/Qmschema", &GQL_SCHEMA); + resolver.add("/ipfs/Qmmapping", &MAPPING_WITH_IPFS_FUNC_WASM); + + let resolver: Arc = Arc::new(resolver); + + let raw = serde_yaml::from_str(YAML).unwrap(); + UnvalidatedSubgraphManifest::resolve( + id, + raw, + &resolver, + &LOGGER, + SPEC_VERSION_0_0_4.clone(), + ) + .await + .expect("Parsing simple manifest works") + }; + + let error_msg = unvalidated + .validate(store.clone(), true) + .await + .expect_err("Validation must fail") + .into_iter() + .find(|e| { + matches!( + e, + SubgraphManifestValidationError::FeatureValidationError(_) + ) + }) + .expect("There must be a FeatureValidationError") + .to_string(); + + assert_eq!( + "The feature `ipfsOnEthereumContracts` is used by the subgraph but it is not declared in the manifest.", + error_msg + ); + }); +} + +#[test] +fn declared_ipfs_on_ethereum_contracts_feature_causes_no_errors() { + const YAML: &str = " +specVersion: 0.0.4 +schema: + file: + /: /ipfs/Qmschema +features: + - ipfsOnEthereumContracts +dataSources: + - kind: ethereum/contract + name: Factory + network: mainnet + source: + abi: Factory + startBlock: 9562480 + mapping: + kind: ethereum/events + apiVersion: 0.0.4 + language: wasm/assemblyscript + entities: + - TestEntity + file: + /: /ipfs/Qmmapping + abis: + - name: Factory + file: + /: /ipfs/Qmabi + callHandlers: + - function: get(address) + handler: handleget +"; + + test_store::run_test_sequentially(|store| async move { + let store = store.subgraph_store(); + let unvalidated: UnvalidatedSubgraphManifest = { + let mut resolver = TextResolver::default(); + let id = DeploymentHash::new("Qmmanifest").unwrap(); + resolver.add(id.as_str(), &YAML); + resolver.add("/ipfs/Qmabi", &ABI); + resolver.add("/ipfs/Qmschema", &GQL_SCHEMA); + resolver.add("/ipfs/Qmmapping", &MAPPING_WITH_IPFS_FUNC_WASM); + + let resolver: Arc = Arc::new(resolver); + + let raw = serde_yaml::from_str(YAML).unwrap(); + UnvalidatedSubgraphManifest::resolve( + id, + raw, + &resolver, + &LOGGER, + SPEC_VERSION_0_0_4.clone(), + ) + .await + .expect("Parsing simple manifest works") + }; + + assert!(unvalidated + .validate(store.clone(), true) + .await + .expect_err("Validation must fail") + .into_iter() + .find(|e| { + matches!( + e, + SubgraphManifestValidationError::FeatureValidationError(_) + ) + }) + .is_none()); + }); +} + +#[test] +fn can_detect_features_in_subgraphs_with_spec_version_lesser_than_0_0_4() { + const YAML: &str = " +specVersion: 0.0.2 +features: + - nonFatalErrors +dataSources: [] +schema: + file: + /: /ipfs/Qmschema +"; + test_store::run_test_sequentially(|store| async move { + let store = store.subgraph_store(); + let unvalidated = resolve_unvalidated(YAML).await; + assert!(unvalidated + .validate(store.clone(), true) + .await + .expect_err("Validation must fail") + .into_iter() + .find(|e| { + matches!( + e, + SubgraphManifestValidationError::FeatureValidationError(_) + ) + }) + .is_none()); + + let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; + assert!(manifest.features.contains(&SubgraphFeature::NonFatalErrors)) + }); +} diff --git a/chain/near/Cargo.toml b/chain/near/Cargo.toml new file mode 100644 index 0000000..e61b560 --- /dev/null +++ b/chain/near/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "graph-chain-near" +version = "0.27.0" +edition = "2021" + +[build-dependencies] +tonic-build = { version = "0.7.1", features = ["prost"] } + +[dependencies] +base64 = "0.13" +graph = { path = "../../graph" } +prost = "0.10.1" +prost-types = "0.10.1" +serde = "1.0" + +graph-runtime-wasm = { path = "../../runtime/wasm" } +graph-runtime-derive = { path = "../../runtime/derive" } + +[dev-dependencies] +diesel = { version = "1.4.7", features = ["postgres", "serde_json", "numeric", "r2d2"] } diff --git a/chain/near/build.rs b/chain/near/build.rs new file mode 100644 index 0000000..73c33ef --- /dev/null +++ b/chain/near/build.rs @@ -0,0 +1,7 @@ +fn main() { + println!("cargo:rerun-if-changed=proto"); + tonic_build::configure() + .out_dir("src/protobuf") + .compile(&["proto/codec.proto"], &["proto"]) + .expect("Failed to compile Firehose NEAR proto(s)"); +} diff --git a/chain/near/proto/codec.proto b/chain/near/proto/codec.proto new file mode 100644 index 0000000..22a0267 --- /dev/null +++ b/chain/near/proto/codec.proto @@ -0,0 +1,521 @@ +syntax = "proto3"; + +package sf.near.codec.v1; + +option go_package = "github.com/streamingfast/sf-near/pb/sf/near/codec/v1;pbcodec"; + +message Block { + string author = 1; + BlockHeader header = 2; + repeated ChunkHeader chunk_headers = 3; + repeated IndexerShard shards = 4; + repeated StateChangeWithCause state_changes = 5; +} + +// HeaderOnlyBlock is a standard [Block] structure where all other fields are +// removed so that hydrating that object from a [Block] bytes payload will +// drastically reduced allocated memory required to hold the full block. +// +// This can be used to unpack a [Block] when only the [BlockHeader] information +// is required and greatly reduced required memory. +message HeaderOnlyBlock { + BlockHeader header = 2; +} + +message StateChangeWithCause { + StateChangeValue value = 1; + StateChangeCause cause = 2; +} + +message StateChangeCause { + oneof cause { + NotWritableToDisk not_writable_to_disk = 1; + InitialState initial_state = 2; + TransactionProcessing transaction_processing = 3; + ActionReceiptProcessingStarted action_receipt_processing_started = 4; + ActionReceiptGasReward action_receipt_gas_reward = 5; + ReceiptProcessing receipt_processing = 6; + PostponedReceipt postponed_receipt = 7; + UpdatedDelayedReceipts updated_delayed_receipts = 8; + ValidatorAccountsUpdate validator_accounts_update = 9; + Migration migration = 10; + } + + message NotWritableToDisk {} + message InitialState {} + message TransactionProcessing {CryptoHash tx_hash = 1;} + message ActionReceiptProcessingStarted {CryptoHash receipt_hash = 1;} + message ActionReceiptGasReward {CryptoHash tx_hash = 1;} + message ReceiptProcessing {CryptoHash tx_hash = 1;} + message PostponedReceipt {CryptoHash tx_hash = 1;} + message UpdatedDelayedReceipts {} + message ValidatorAccountsUpdate {} + message Migration {} +} + +message StateChangeValue { + oneof value { + AccountUpdate account_update = 1; + AccountDeletion account_deletion = 2; + AccessKeyUpdate access_key_update = 3; + AccessKeyDeletion access_key_deletion = 4; + DataUpdate data_update = 5; + DataDeletion data_deletion = 6; + ContractCodeUpdate contract_code_update = 7; + ContractCodeDeletion contract_deletion = 8; + } + + message AccountUpdate {string account_id = 1; Account account = 2;} + message AccountDeletion {string account_id = 1;} + message AccessKeyUpdate { + string account_id = 1; + PublicKey public_key = 2; + AccessKey access_key = 3; + } + message AccessKeyDeletion { + string account_id = 1; + PublicKey public_key = 2; + } + message DataUpdate { + string account_id = 1; + bytes key = 2; + bytes value = 3; + } + message DataDeletion { + string account_id = 1; + bytes key = 2; + } + message ContractCodeUpdate { + string account_id = 1; + bytes code = 2; + } + message ContractCodeDeletion { + string account_id = 1; + } +} + +message Account { + BigInt amount = 1; + BigInt locked = 2; + CryptoHash code_hash = 3; + uint64 storage_usage = 4; +} + +message BlockHeader { + uint64 height = 1; + uint64 prev_height = 2; + CryptoHash epoch_id = 3; + CryptoHash next_epoch_id = 4; + CryptoHash hash = 5; + CryptoHash prev_hash = 6; + CryptoHash prev_state_root = 7; + CryptoHash chunk_receipts_root = 8; + CryptoHash chunk_headers_root = 9; + CryptoHash chunk_tx_root = 10; + CryptoHash outcome_root = 11; + uint64 chunks_included = 12; + CryptoHash challenges_root = 13; + uint64 timestamp = 14; + uint64 timestamp_nanosec = 15; + CryptoHash random_value = 16; + repeated ValidatorStake validator_proposals = 17; + repeated bool chunk_mask = 18; + BigInt gas_price = 19; + uint64 block_ordinal = 20; + BigInt total_supply = 21; + repeated SlashedValidator challenges_result = 22; + uint64 last_final_block_height = 23; + CryptoHash last_final_block = 24; + uint64 last_ds_final_block_height = 25; + CryptoHash last_ds_final_block = 26; + CryptoHash next_bp_hash = 27; + CryptoHash block_merkle_root = 28; + bytes epoch_sync_data_hash = 29; + repeated Signature approvals = 30; + Signature signature = 31; + uint32 latest_protocol_version = 32; +} + +message BigInt { + bytes bytes = 1; +} +message CryptoHash { + bytes bytes = 1; +} + +enum CurveKind { + ED25519 = 0; + SECP256K1 = 1; +} + +message Signature { + CurveKind type = 1; + bytes bytes = 2; +} + +message PublicKey { + CurveKind type = 1; + bytes bytes = 2; +} + +message ValidatorStake { + string account_id = 1; + PublicKey public_key = 2; + BigInt stake = 3; +} + +message SlashedValidator { + string account_id = 1; + bool is_double_sign = 2; +} + +message ChunkHeader { + bytes chunk_hash = 1; + bytes prev_block_hash = 2; + bytes outcome_root = 3; + bytes prev_state_root = 4; + bytes encoded_merkle_root = 5; + uint64 encoded_length = 6; + uint64 height_created = 7; + uint64 height_included = 8; + uint64 shard_id = 9; + uint64 gas_used = 10; + uint64 gas_limit = 11; + BigInt validator_reward = 12; + BigInt balance_burnt = 13; + bytes outgoing_receipts_root = 14; + bytes tx_root = 15; + repeated ValidatorStake validator_proposals = 16; + Signature signature = 17; +} + +message IndexerShard { + uint64 shard_id = 1; + IndexerChunk chunk = 2; + repeated IndexerExecutionOutcomeWithReceipt receipt_execution_outcomes = 3; +} + +message IndexerExecutionOutcomeWithReceipt { + ExecutionOutcomeWithId execution_outcome = 1; + Receipt receipt = 2; +} + +message IndexerChunk { + string author = 1; + ChunkHeader header = 2; + repeated IndexerTransactionWithOutcome transactions = 3; + repeated Receipt receipts = 4; +} + +message IndexerTransactionWithOutcome { + SignedTransaction transaction = 1; + IndexerExecutionOutcomeWithOptionalReceipt outcome = 2; +} + +message SignedTransaction { + string signer_id = 1; + PublicKey public_key = 2; + uint64 nonce = 3; + string receiver_id = 4; + repeated Action actions = 5; + Signature signature = 6; + CryptoHash hash = 7; +} + +message IndexerExecutionOutcomeWithOptionalReceipt { + ExecutionOutcomeWithId execution_outcome = 1; + Receipt receipt = 2; +} + +message Receipt { + string predecessor_id = 1; + string receiver_id = 2; + CryptoHash receipt_id = 3; + + oneof receipt { + ReceiptAction action = 10; + ReceiptData data = 11; + } +} + +message ReceiptData { + CryptoHash data_id = 1; + bytes data = 2; +} + +message ReceiptAction { + string signer_id = 1; + PublicKey signer_public_key = 2; + BigInt gas_price = 3; + repeated DataReceiver output_data_receivers = 4; + repeated CryptoHash input_data_ids = 5; + repeated Action actions = 6; +} + +message DataReceiver { + CryptoHash data_id = 1; + string receiver_id = 2; +} + +message ExecutionOutcomeWithId { + MerklePath proof = 1; + CryptoHash block_hash = 2; + CryptoHash id = 3; + ExecutionOutcome outcome = 4; +} + +message ExecutionOutcome { + repeated string logs = 1; + repeated CryptoHash receipt_ids = 2; + uint64 gas_burnt = 3; + BigInt tokens_burnt = 4; + string executor_id = 5; + oneof status { + UnknownExecutionStatus unknown = 20; + FailureExecutionStatus failure = 21; + SuccessValueExecutionStatus success_value = 22; + SuccessReceiptIdExecutionStatus success_receipt_id = 23; + } + ExecutionMetadata metadata = 6; +} + +enum ExecutionMetadata { + ExecutionMetadataV1 = 0; +} + +message SuccessValueExecutionStatus { + bytes value = 1; +} + +message SuccessReceiptIdExecutionStatus { + CryptoHash id = 1; +} + +message UnknownExecutionStatus {} +message FailureExecutionStatus { + oneof failure { + ActionError action_error = 1; + InvalidTxError invalid_tx_error = 2; + } +} + +message ActionError { + uint64 index = 1; + oneof kind { + AccountAlreadyExistsErrorKind account_already_exist = 21; + AccountDoesNotExistErrorKind account_does_not_exist = 22; + CreateAccountOnlyByRegistrarErrorKind create_account_only_by_registrar = 23; + CreateAccountNotAllowedErrorKind create_account_not_allowed = 24; + ActorNoPermissionErrorKind actor_no_permission =25; + DeleteKeyDoesNotExistErrorKind delete_key_does_not_exist = 26; + AddKeyAlreadyExistsErrorKind add_key_already_exists = 27; + DeleteAccountStakingErrorKind delete_account_staking = 28; + LackBalanceForStateErrorKind lack_balance_for_state = 29; + TriesToUnstakeErrorKind tries_to_unstake = 30; + TriesToStakeErrorKind tries_to_stake = 31; + InsufficientStakeErrorKind insufficient_stake = 32; + FunctionCallErrorKind function_call = 33; + NewReceiptValidationErrorKind new_receipt_validation = 34; + OnlyImplicitAccountCreationAllowedErrorKind only_implicit_account_creation_allowed = 35; + DeleteAccountWithLargeStateErrorKind delete_account_with_large_state = 36; + } +} + +message AccountAlreadyExistsErrorKind { + string account_id = 1; +} + +message AccountDoesNotExistErrorKind { + string account_id = 1; +} + +/// A top-level account ID can only be created by registrar. +message CreateAccountOnlyByRegistrarErrorKind{ + string account_id = 1; + string registrar_account_id = 2; + string predecessor_id = 3; +} + +message CreateAccountNotAllowedErrorKind{ + string account_id = 1; + string predecessor_id = 2; +} + +message ActorNoPermissionErrorKind{ + string account_id = 1; + string actor_id = 2; +} + +message DeleteKeyDoesNotExistErrorKind{ + string account_id = 1; + PublicKey public_key = 2; +} + +message AddKeyAlreadyExistsErrorKind{ + string account_id = 1; + PublicKey public_key = 2; +} + +message DeleteAccountStakingErrorKind{ + string account_id = 1; +} + +message LackBalanceForStateErrorKind{ + string account_id = 1; + BigInt balance = 2; +} + +message TriesToUnstakeErrorKind{ + string account_id = 1; +} + +message TriesToStakeErrorKind{ + string account_id = 1; + BigInt stake = 2; + BigInt locked = 3; + BigInt balance = 4; +} + +message InsufficientStakeErrorKind{ + string account_id = 1; + BigInt stake = 2; + BigInt minimum_stake = 3; +} + +message FunctionCallErrorKind { + FunctionCallErrorSer error = 1; +} + +enum FunctionCallErrorSer { //todo: add more detail? + CompilationError = 0; + LinkError = 1; + MethodResolveError = 2; + WasmTrap = 3; + WasmUnknownError = 4; + HostError = 5; + _EVMError = 6; + ExecutionError = 7; +} + +message NewReceiptValidationErrorKind { + ReceiptValidationError error = 1; +} + +enum ReceiptValidationError { //todo: add more detail? + InvalidPredecessorId = 0; + InvalidReceiverAccountId = 1; + InvalidSignerAccountId = 2; + InvalidDataReceiverId = 3; + ReturnedValueLengthExceeded = 4; + NumberInputDataDependenciesExceeded = 5; + ActionsValidationError = 6; +} + +message OnlyImplicitAccountCreationAllowedErrorKind{ + string account_id = 1; +} + +message DeleteAccountWithLargeStateErrorKind{ + string account_id = 1; +} + +enum InvalidTxError { //todo: add more detail? + InvalidAccessKeyError = 0; + InvalidSignerId = 1; + SignerDoesNotExist = 2; + InvalidNonce = 3; + NonceTooLarge = 4; + InvalidReceiverId = 5; + InvalidSignature = 6; + NotEnoughBalance = 7; + LackBalanceForState = 8; + CostOverflow = 9; + InvalidChain = 10; + Expired = 11; + ActionsValidation = 12; + TransactionSizeExceeded = 13; +} + +message MerklePath { + repeated MerklePathItem path = 1; +} + +message MerklePathItem { + CryptoHash hash = 1; + Direction direction = 2; +} + +enum Direction { + left = 0; + right = 1; +} + +message Action { + oneof action { + CreateAccountAction create_account = 1; + DeployContractAction deploy_contract = 2; + FunctionCallAction function_call = 3; + TransferAction transfer = 4; + StakeAction stake = 5; + AddKeyAction add_key = 6; + DeleteKeyAction delete_key = 7; + DeleteAccountAction delete_account = 8; + } +} + +message CreateAccountAction { +} + +message DeployContractAction { + bytes code = 1; +} + +message FunctionCallAction { + string method_name = 1; + bytes args = 2; + uint64 gas = 3; + BigInt deposit = 4; +} + +message TransferAction { + BigInt deposit = 1; +} + +message StakeAction { + BigInt stake = 1; + PublicKey public_key = 2; +} + +message AddKeyAction { + PublicKey public_key = 1; + AccessKey access_key = 2; +} + +message DeleteKeyAction { + PublicKey public_key = 1; +} + +message DeleteAccountAction { + string beneficiary_id = 1; +} + +message AccessKey { + uint64 nonce = 1; + AccessKeyPermission permission = 2; +} + +message AccessKeyPermission { + oneof permission { + FunctionCallPermission function_call = 1; + FullAccessPermission full_access = 2; + } +} + +message FunctionCallPermission { + BigInt allowance = 1; + string receiver_id = 2; + repeated string method_names = 3; +} + +message FullAccessPermission { +} diff --git a/chain/near/src/adapter.rs b/chain/near/src/adapter.rs new file mode 100644 index 0000000..1b39692 --- /dev/null +++ b/chain/near/src/adapter.rs @@ -0,0 +1,355 @@ +use std::collections::HashSet; + +use crate::capabilities::NodeCapabilities; +use crate::data_source::PartialAccounts; +use crate::{data_source::DataSource, Chain}; +use graph::blockchain as bc; +use graph::firehose::{BasicReceiptFilter, PrefixSuffixPair}; +use graph::prelude::*; +use prost::Message; +use prost_types::Any; + +const BASIC_RECEIPT_FILTER_TYPE_URL: &str = + "type.googleapis.com/sf.near.transform.v1.BasicReceiptFilter"; + +#[derive(Clone, Debug, Default)] +pub struct TriggerFilter { + pub(crate) block_filter: NearBlockFilter, + pub(crate) receipt_filter: NearReceiptFilter, +} + +impl bc::TriggerFilter for TriggerFilter { + fn extend<'a>(&mut self, data_sources: impl Iterator + Clone) { + let TriggerFilter { + block_filter, + receipt_filter, + } = self; + + block_filter.extend(NearBlockFilter::from_data_sources(data_sources.clone())); + receipt_filter.extend(NearReceiptFilter::from_data_sources(data_sources)); + } + + fn node_capabilities(&self) -> NodeCapabilities { + NodeCapabilities {} + } + + fn extend_with_template( + &mut self, + _data_source: impl Iterator::DataSourceTemplate>, + ) { + } + + fn to_firehose_filter(self) -> Vec { + let TriggerFilter { + block_filter: block, + receipt_filter: receipt, + } = self; + + if block.trigger_every_block { + return vec![]; + } + + if receipt.is_empty() { + return vec![]; + } + + let filter = BasicReceiptFilter { + accounts: receipt.accounts.into_iter().collect(), + prefix_and_suffix_pairs: receipt + .partial_accounts + .iter() + .map(|(prefix, suffix)| PrefixSuffixPair { + prefix: prefix.clone().unwrap_or("".to_string()), + suffix: suffix.clone().unwrap_or("".to_string()), + }) + .collect(), + }; + + vec![Any { + type_url: BASIC_RECEIPT_FILTER_TYPE_URL.into(), + value: filter.encode_to_vec(), + }] + } +} + +pub(crate) type Account = String; + +/// NearReceiptFilter requires the account to be set, it will match every receipt where `source.account` is the recipient. +/// see docs: https://thegraph.com/docs/en/supported-networks/near/ +#[derive(Clone, Debug, Default)] +pub(crate) struct NearReceiptFilter { + pub accounts: HashSet, + pub partial_accounts: HashSet<(Option, Option)>, +} + +impl NearReceiptFilter { + pub fn matches(&self, account: &String) -> bool { + let NearReceiptFilter { + accounts, + partial_accounts, + } = self; + + if accounts.contains(account) { + return true; + } + + partial_accounts.iter().any(|partial| match partial { + (Some(prefix), Some(suffix)) => { + account.starts_with(prefix) && account.ends_with(suffix) + } + (Some(prefix), None) => account.starts_with(prefix), + (None, Some(suffix)) => account.ends_with(suffix), + (None, None) => unreachable!(), + }) + } + + pub fn is_empty(&self) -> bool { + let NearReceiptFilter { + accounts, + partial_accounts, + } = self; + + accounts.is_empty() && partial_accounts.is_empty() + } + + pub fn from_data_sources<'a>(iter: impl IntoIterator) -> Self { + struct Source { + account: Option, + partial_accounts: Option, + } + + // Select any ds with either partial or exact accounts. + let sources: Vec = iter + .into_iter() + .filter(|data_source| { + (data_source.source.account.is_some() || data_source.source.accounts.is_some()) + && !data_source.mapping.receipt_handlers.is_empty() + }) + .map(|ds| Source { + account: ds.source.account.clone(), + partial_accounts: ds.source.accounts.clone(), + }) + .collect(); + + // Handle exact matches + let accounts: Vec = sources + .iter() + .filter(|s| s.account.is_some()) + .map(|s| s.account.as_ref().cloned().unwrap()) + .collect(); + + // Parse all the partial accounts, produces all possible combinations of the values + // eg: + // prefix [a,b] and suffix [d] would produce [a,d], [b,d] + // prefix [a] and suffix [c,d] would produce [a,c], [a,d] + // prefix [] and suffix [c, d] would produce [None, c], [None, d] + // prefix [a,b] and suffix [] would produce [a, None], [b, None] + let partial_accounts: Vec<(Option, Option)> = sources + .iter() + .filter(|s| s.partial_accounts.is_some()) + .map(|s| { + let partials = s.partial_accounts.as_ref().unwrap(); + + let mut pairs: Vec<(Option, Option)> = vec![]; + let prefixes: Vec> = if partials.prefixes.is_empty() { + vec![None] + } else { + partials + .prefixes + .iter() + .filter(|s| !s.is_empty()) + .map(|s| Some(s.clone())) + .collect() + }; + + let suffixes: Vec> = if partials.suffixes.is_empty() { + vec![None] + } else { + partials + .suffixes + .iter() + .filter(|s| !s.is_empty()) + .map(|s| Some(s.clone())) + .collect() + }; + + for prefix in prefixes.into_iter() { + for suffix in suffixes.iter() { + pairs.push((prefix.clone(), suffix.clone())) + } + } + + pairs + }) + .flatten() + .collect(); + + Self { + accounts: HashSet::from_iter(accounts), + partial_accounts: HashSet::from_iter(partial_accounts), + } + } + + pub fn extend(&mut self, other: NearReceiptFilter) { + let NearReceiptFilter { + accounts, + partial_accounts, + } = self; + + accounts.extend(other.accounts); + partial_accounts.extend(other.partial_accounts); + } +} + +/// NearBlockFilter will match every block regardless of source being set. +/// see docs: https://thegraph.com/docs/en/supported-networks/near/ +#[derive(Clone, Debug, Default)] +pub(crate) struct NearBlockFilter { + pub trigger_every_block: bool, +} + +impl NearBlockFilter { + pub fn from_data_sources<'a>(iter: impl IntoIterator) -> Self { + Self { + trigger_every_block: iter + .into_iter() + .any(|data_source| !data_source.mapping.block_handlers.is_empty()), + } + } + + pub fn extend(&mut self, other: NearBlockFilter) { + self.trigger_every_block = self.trigger_every_block || other.trigger_every_block; + } +} + +#[cfg(test)] +mod test { + use std::collections::HashSet; + + use super::NearBlockFilter; + use crate::adapter::{TriggerFilter, BASIC_RECEIPT_FILTER_TYPE_URL}; + use graph::{ + blockchain::TriggerFilter as _, + firehose::{BasicReceiptFilter, PrefixSuffixPair}, + }; + use prost::Message; + use prost_types::Any; + + #[test] + fn near_trigger_empty_filter() { + let filter = TriggerFilter { + block_filter: NearBlockFilter { + trigger_every_block: false, + }, + receipt_filter: super::NearReceiptFilter { + accounts: HashSet::new(), + partial_accounts: HashSet::new(), + }, + }; + assert_eq!(filter.to_firehose_filter(), vec![]); + } + + #[test] + fn near_trigger_filter_match_all_block() { + let filter = TriggerFilter { + block_filter: NearBlockFilter { + trigger_every_block: true, + }, + receipt_filter: super::NearReceiptFilter { + accounts: HashSet::from_iter(vec!["acc1".into(), "acc2".into(), "acc3".into()]), + partial_accounts: HashSet::new(), + }, + }; + + let filter = filter.to_firehose_filter(); + assert_eq!(filter.len(), 0); + } + + #[test] + fn near_trigger_filter() { + let filter = TriggerFilter { + block_filter: NearBlockFilter { + trigger_every_block: false, + }, + receipt_filter: super::NearReceiptFilter { + accounts: HashSet::from_iter(vec!["acc1".into(), "acc2".into(), "acc3".into()]), + partial_accounts: HashSet::new(), + }, + }; + + let filter = filter.to_firehose_filter(); + assert_eq!(filter.len(), 1); + + let firehose_filter = decode_filter(filter); + + assert_eq!( + firehose_filter.accounts, + vec![ + String::from("acc1"), + String::from("acc2"), + String::from("acc3") + ], + ); + } + + #[test] + fn near_trigger_partial_filter() { + let filter = TriggerFilter { + block_filter: NearBlockFilter { + trigger_every_block: false, + }, + receipt_filter: super::NearReceiptFilter { + accounts: HashSet::from_iter(vec!["acc1".into()]), + partial_accounts: HashSet::from_iter(vec![ + (Some("acc1".into()), None), + (None, Some("acc2".into())), + (Some("acc3".into()), Some("acc4".into())), + ]), + }, + }; + + let filter = filter.to_firehose_filter(); + assert_eq!(filter.len(), 1); + + let firehose_filter = decode_filter(filter); + assert_eq!(firehose_filter.accounts, vec![String::from("acc1"),],); + + let expected_pairs = vec![ + PrefixSuffixPair { + prefix: "acc3".to_string(), + suffix: "acc4".to_string(), + }, + PrefixSuffixPair { + prefix: "".to_string(), + suffix: "acc2".to_string(), + }, + PrefixSuffixPair { + prefix: "acc1".to_string(), + suffix: "".to_string(), + }, + ]; + + let pairs = firehose_filter.prefix_and_suffix_pairs; + assert_eq!(pairs.len(), 3); + assert_eq!( + true, + expected_pairs.iter().all(|x| pairs.contains(x)), + "{:?}", + pairs + ); + } + + fn decode_filter(firehose_filter: Vec) -> BasicReceiptFilter { + let firehose_filter = firehose_filter[0].clone(); + assert_eq!( + firehose_filter.type_url, + String::from(BASIC_RECEIPT_FILTER_TYPE_URL), + ); + let mut bytes = &firehose_filter.value[..]; + let mut firehose_filter = + BasicReceiptFilter::decode(&mut bytes).expect("unable to parse basic receipt filter"); + firehose_filter.accounts.sort(); + + firehose_filter + } +} diff --git a/chain/near/src/capabilities.rs b/chain/near/src/capabilities.rs new file mode 100644 index 0000000..0d84c9c --- /dev/null +++ b/chain/near/src/capabilities.rs @@ -0,0 +1,37 @@ +use graph::{anyhow::Error, impl_slog_value}; +use std::cmp::{Ordering, PartialOrd}; +use std::fmt; +use std::str::FromStr; + +use crate::data_source::DataSource; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NodeCapabilities {} + +impl PartialOrd for NodeCapabilities { + fn partial_cmp(&self, _other: &Self) -> Option { + None + } +} + +impl FromStr for NodeCapabilities { + type Err = Error; + + fn from_str(_s: &str) -> Result { + Ok(NodeCapabilities {}) + } +} + +impl fmt::Display for NodeCapabilities { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("near") + } +} + +impl_slog_value!(NodeCapabilities, "{}"); + +impl graph::blockchain::NodeCapabilities for NodeCapabilities { + fn from_data_sources(_data_sources: &[DataSource]) -> Self { + NodeCapabilities {} + } +} diff --git a/chain/near/src/chain.rs b/chain/near/src/chain.rs new file mode 100644 index 0000000..226a915 --- /dev/null +++ b/chain/near/src/chain.rs @@ -0,0 +1,912 @@ +use graph::blockchain::BlockchainKind; +use graph::cheap_clone::CheapClone; +use graph::data::subgraph::UnifiedMappingApiVersion; +use graph::firehose::{FirehoseEndpoint, FirehoseEndpoints}; +use graph::prelude::{MetricsRegistry, TryFutureExt}; +use graph::{ + anyhow, + anyhow::Result, + blockchain::{ + block_stream::{ + BlockStreamEvent, BlockWithTriggers, FirehoseError, + FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, + }, + firehose_block_stream::FirehoseBlockStream, + BlockHash, BlockPtr, Blockchain, IngestorError, RuntimeAdapter as RuntimeAdapterTrait, + }, + components::store::DeploymentLocator, + firehose::{self as firehose, ForkStep}, + prelude::{async_trait, o, BlockNumber, ChainStore, Error, Logger, LoggerFactory}, +}; +use prost::Message; +use std::sync::Arc; + +use crate::adapter::TriggerFilter; +use crate::capabilities::NodeCapabilities; +use crate::data_source::{DataSourceTemplate, UnresolvedDataSourceTemplate}; +use crate::runtime::RuntimeAdapter; +use crate::trigger::{self, NearTrigger}; +use crate::{ + codec, + data_source::{DataSource, UnresolvedDataSource}, +}; +use graph::blockchain::block_stream::{BlockStream, BlockStreamBuilder, FirehoseCursor}; + +pub struct NearStreamBuilder {} + +#[async_trait] +impl BlockStreamBuilder for NearStreamBuilder { + async fn build_firehose( + &self, + chain: &Chain, + deployment: DeploymentLocator, + block_cursor: FirehoseCursor, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc<::TriggerFilter>, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>> { + let adapter = chain + .triggers_adapter(&deployment, &NodeCapabilities {}, unified_api_version) + .expect(&format!("no adapter for network {}", chain.name,)); + + let firehose_endpoint = match chain.firehose_endpoints.random() { + Some(e) => e.clone(), + None => return Err(anyhow::format_err!("no firehose endpoint available")), + }; + + let logger = chain + .logger_factory + .subgraph_logger(&deployment) + .new(o!("component" => "FirehoseBlockStream")); + + let firehose_mapper = Arc::new(FirehoseMapper { + endpoint: firehose_endpoint.cheap_clone(), + }); + + Ok(Box::new(FirehoseBlockStream::new( + deployment.hash, + firehose_endpoint, + subgraph_current_block, + block_cursor, + firehose_mapper, + adapter, + filter, + start_blocks, + logger, + chain.metrics_registry.clone(), + ))) + } + + async fn build_polling( + &self, + _chain: Arc, + _deployment: DeploymentLocator, + _start_blocks: Vec, + _subgraph_current_block: Option, + _filter: Arc<::TriggerFilter>, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>> { + todo!() + } +} + +pub struct Chain { + logger_factory: LoggerFactory, + name: String, + firehose_endpoints: Arc, + chain_store: Arc, + metrics_registry: Arc, + block_stream_builder: Arc>, +} + +impl std::fmt::Debug for Chain { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "chain: near") + } +} + +impl Chain { + pub fn new( + logger_factory: LoggerFactory, + name: String, + chain_store: Arc, + firehose_endpoints: FirehoseEndpoints, + metrics_registry: Arc, + block_stream_builder: Arc>, + ) -> Self { + Chain { + logger_factory, + name, + firehose_endpoints: Arc::new(firehose_endpoints), + chain_store, + metrics_registry, + block_stream_builder, + } + } +} + +#[async_trait] +impl Blockchain for Chain { + const KIND: BlockchainKind = BlockchainKind::Near; + + type Block = codec::Block; + + type DataSource = DataSource; + + type UnresolvedDataSource = UnresolvedDataSource; + + type DataSourceTemplate = DataSourceTemplate; + + type UnresolvedDataSourceTemplate = UnresolvedDataSourceTemplate; + + type TriggerData = crate::trigger::NearTrigger; + + type MappingTrigger = crate::trigger::NearTrigger; + + type TriggerFilter = crate::adapter::TriggerFilter; + + type NodeCapabilities = crate::capabilities::NodeCapabilities; + + fn triggers_adapter( + &self, + _loc: &DeploymentLocator, + _capabilities: &Self::NodeCapabilities, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + let adapter = TriggersAdapter {}; + Ok(Arc::new(adapter)) + } + + async fn new_firehose_block_stream( + &self, + deployment: DeploymentLocator, + block_cursor: FirehoseCursor, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + self.block_stream_builder + .build_firehose( + self, + deployment, + block_cursor, + start_blocks, + subgraph_current_block, + filter, + unified_api_version, + ) + .await + } + + async fn new_polling_block_stream( + &self, + _deployment: DeploymentLocator, + _start_blocks: Vec, + _subgraph_current_block: Option, + _filter: Arc, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + panic!("NEAR does not support polling block stream") + } + + fn chain_store(&self) -> Arc { + self.chain_store.clone() + } + + async fn block_pointer_from_number( + &self, + logger: &Logger, + number: BlockNumber, + ) -> Result { + let firehose_endpoint = match self.firehose_endpoints.random() { + Some(e) => e.clone(), + None => return Err(anyhow::format_err!("no firehose endpoint available").into()), + }; + + firehose_endpoint + .block_ptr_for_number::(logger, number) + .map_err(Into::into) + .await + } + + fn runtime_adapter(&self) -> Arc> { + Arc::new(RuntimeAdapter {}) + } + + fn is_firehose_supported(&self) -> bool { + true + } +} + +pub struct TriggersAdapter {} + +#[async_trait] +impl TriggersAdapterTrait for TriggersAdapter { + async fn scan_triggers( + &self, + _from: BlockNumber, + _to: BlockNumber, + _filter: &TriggerFilter, + ) -> Result>, Error> { + panic!("Should never be called since not used by FirehoseBlockStream") + } + + async fn triggers_in_block( + &self, + _logger: &Logger, + block: codec::Block, + filter: &TriggerFilter, + ) -> Result, Error> { + // TODO: Find the best place to introduce an `Arc` and avoid this clone. + let shared_block = Arc::new(block.clone()); + + let TriggerFilter { + block_filter, + receipt_filter, + } = filter; + + // Filter non-successful or non-action receipts. + let receipts = block.shards.iter().flat_map(|shard| { + shard + .receipt_execution_outcomes + .iter() + .filter_map(|outcome| { + if !outcome + .execution_outcome + .as_ref()? + .outcome + .as_ref()? + .status + .as_ref()? + .is_success() + { + return None; + } + if !matches!( + outcome.receipt.as_ref()?.receipt, + Some(codec::receipt::Receipt::Action(_)) + ) { + return None; + } + + let receipt = outcome.receipt.as_ref()?.clone(); + if !receipt_filter.matches(&receipt.receiver_id) { + return None; + } + + Some(trigger::ReceiptWithOutcome { + outcome: outcome.execution_outcome.as_ref()?.clone(), + receipt, + block: shared_block.cheap_clone(), + }) + }) + }); + + let mut trigger_data: Vec<_> = receipts + .map(|r| NearTrigger::Receipt(Arc::new(r))) + .collect(); + + if block_filter.trigger_every_block { + trigger_data.push(NearTrigger::Block(shared_block.cheap_clone())); + } + + Ok(BlockWithTriggers::new(block, trigger_data)) + } + + async fn is_on_main_chain(&self, _ptr: BlockPtr) -> Result { + panic!("Should never be called since not used by FirehoseBlockStream") + } + + async fn ancestor_block( + &self, + _ptr: BlockPtr, + _offset: BlockNumber, + ) -> Result, Error> { + panic!("Should never be called since FirehoseBlockStream cannot resolve it") + } + + /// Panics if `block` is genesis. + /// But that's ok since this is only called when reverting `block`. + async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { + // FIXME (NEAR): Might not be necessary for NEAR support for now + Ok(Some(BlockPtr { + hash: BlockHash::from(vec![0xff; 32]), + number: block.number.saturating_sub(1), + })) + } +} + +pub struct FirehoseMapper { + endpoint: Arc, +} + +#[async_trait] +impl FirehoseMapperTrait for FirehoseMapper { + async fn to_block_stream_event( + &self, + logger: &Logger, + response: &firehose::Response, + adapter: &Arc>, + filter: &TriggerFilter, + ) -> Result, FirehoseError> { + let step = ForkStep::from_i32(response.step).unwrap_or_else(|| { + panic!( + "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", + response.step + ) + }); + + let any_block = response + .block + .as_ref() + .expect("block payload information should always be present"); + + // Right now, this is done in all cases but in reality, with how the BlockStreamEvent::Revert + // is defined right now, only block hash and block number is necessary. However, this information + // is not part of the actual bstream::BlockResponseV2 payload. As such, we need to decode the full + // block which is useless. + // + // Check about adding basic information about the block in the bstream::BlockResponseV2 or maybe + // define a slimmed down stuct that would decode only a few fields and ignore all the rest. + let block = codec::Block::decode(any_block.value.as_ref())?; + + use ForkStep::*; + match step { + StepNew => Ok(BlockStreamEvent::ProcessBlock( + adapter.triggers_in_block(logger, block, filter).await?, + FirehoseCursor::from(response.cursor.clone()), + )), + + StepUndo => { + let parent_ptr = block + .header() + .parent_ptr() + .expect("Genesis block should never be reverted"); + + Ok(BlockStreamEvent::Revert( + parent_ptr, + FirehoseCursor::from(response.cursor.clone()), + )) + } + + StepIrreversible => { + panic!("irreversible step is not handled and should not be requested in the Firehose request") + } + + StepUnknown => { + panic!("unknown step should not happen in the Firehose response") + } + } + } + + async fn block_ptr_for_number( + &self, + logger: &Logger, + number: BlockNumber, + ) -> Result { + self.endpoint + .block_ptr_for_number::(logger, number) + .await + } + + async fn final_block_ptr_for( + &self, + logger: &Logger, + block: &codec::Block, + ) -> Result { + let final_block_number = block.header().last_final_block_height as BlockNumber; + + self.endpoint + .block_ptr_for_number::(logger, final_block_number) + .await + } +} + +#[cfg(test)] +mod test { + use std::{collections::HashSet, sync::Arc, vec}; + + use graph::{ + blockchain::{block_stream::BlockWithTriggers, DataSource as _, TriggersAdapter as _}, + prelude::{tokio, Link}, + semver::Version, + slog::{self, o, Logger}, + }; + + use crate::{ + adapter::{NearReceiptFilter, TriggerFilter}, + codec::{ + self, execution_outcome, receipt, Block, BlockHeader, DataReceiver, ExecutionOutcome, + ExecutionOutcomeWithId, IndexerExecutionOutcomeWithReceipt, IndexerShard, + ReceiptAction, SuccessValueExecutionStatus, + }, + data_source::{DataSource, Mapping, PartialAccounts, ReceiptHandler, NEAR_KIND}, + trigger::{NearTrigger, ReceiptWithOutcome}, + Chain, + }; + + use super::TriggersAdapter; + + #[test] + fn validate_empty() { + let ds = new_data_source(None, None); + let errs = ds.validate(); + assert_eq!(errs.len(), 1, "{:?}", ds); + assert_eq!(errs[0].to_string(), "subgraph source address is required"); + } + + #[test] + fn validate_empty_account_none_partial() { + let ds = new_data_source(None, Some(PartialAccounts::default())); + let errs = ds.validate(); + assert_eq!(errs.len(), 1, "{:?}", ds); + assert_eq!(errs[0].to_string(), "subgraph source address is required"); + } + + #[test] + fn validate_empty_account() { + let ds = new_data_source( + None, + Some(PartialAccounts { + prefixes: vec![], + suffixes: vec!["x.near".to_string()], + }), + ); + let errs = ds.validate(); + assert_eq!(errs.len(), 0, "{:?}", ds); + } + + #[test] + fn validate_empty_prefix_and_suffix_values() { + let ds = new_data_source( + None, + Some(PartialAccounts { + prefixes: vec!["".to_string()], + suffixes: vec!["".to_string()], + }), + ); + let errs: Vec = ds + .validate() + .into_iter() + .map(|err| err.to_string()) + .collect(); + assert_eq!(errs.len(), 2, "{:?}", ds); + + let expected_errors = vec![ + "partial account prefixes can't have empty values".to_string(), + "partial account suffixes can't have empty values".to_string(), + ]; + assert_eq!( + true, + expected_errors.iter().all(|err| errs.contains(err)), + "{:?}", + errs + ); + } + + #[test] + fn validate_empty_partials() { + let ds = new_data_source(Some("x.near".to_string()), None); + let errs = ds.validate(); + assert_eq!(errs.len(), 0, "{:?}", ds); + } + + #[test] + fn receipt_filter_from_ds() { + struct Case { + name: String, + account: Option, + partial_accounts: Option, + expected: HashSet<(Option, Option)>, + } + + let cases = vec![ + Case { + name: "2 prefix && 1 suffix".into(), + account: None, + partial_accounts: Some(PartialAccounts { + prefixes: vec!["a".to_string(), "b".to_string()], + suffixes: vec!["d".to_string()], + }), + expected: HashSet::from_iter(vec![ + (Some("a".to_string()), Some("d".to_string())), + (Some("b".to_string()), Some("d".to_string())), + ]), + }, + Case { + name: "1 prefix && 2 suffix".into(), + account: None, + partial_accounts: Some(PartialAccounts { + prefixes: vec!["a".to_string()], + suffixes: vec!["c".to_string(), "d".to_string()], + }), + expected: HashSet::from_iter(vec![ + (Some("a".to_string()), Some("c".to_string())), + (Some("a".to_string()), Some("d".to_string())), + ]), + }, + Case { + name: "no prefix".into(), + account: None, + partial_accounts: Some(PartialAccounts { + prefixes: vec![], + suffixes: vec!["c".to_string(), "d".to_string()], + }), + expected: HashSet::from_iter(vec![ + (None, Some("c".to_string())), + (None, Some("d".to_string())), + ]), + }, + Case { + name: "no suffix".into(), + account: None, + partial_accounts: Some(PartialAccounts { + prefixes: vec!["a".to_string(), "b".to_string()], + suffixes: vec![], + }), + expected: HashSet::from_iter(vec![ + (Some("a".to_string()), None), + (Some("b".to_string()), None), + ]), + }, + ]; + + for case in cases.into_iter() { + let ds1 = new_data_source(case.account, None); + let ds2 = new_data_source(None, case.partial_accounts); + + let receipt = NearReceiptFilter::from_data_sources(vec![&ds1, &ds2]); + assert_eq!( + receipt.partial_accounts.len(), + case.expected.len(), + "name: {}\npartial_accounts: {:?}", + case.name, + receipt.partial_accounts, + ); + assert_eq!( + true, + case.expected + .iter() + .all(|x| receipt.partial_accounts.contains(&x)), + "name: {}\npartial_accounts: {:?}", + case.name, + receipt.partial_accounts, + ); + } + } + + #[test] + fn data_source_match_and_decode() { + struct Request { + account: String, + matches: bool, + } + struct Case { + name: String, + account: Option, + partial_accounts: Option, + expected: Vec, + } + + let cases = vec![ + Case { + name: "2 prefix && 1 suffix".into(), + account: None, + partial_accounts: Some(PartialAccounts { + prefixes: vec!["a".to_string(), "b".to_string()], + suffixes: vec!["d".to_string()], + }), + expected: vec![ + Request { + account: "ssssssd".to_string(), + matches: false, + }, + Request { + account: "asasdasdas".to_string(), + matches: false, + }, + Request { + account: "asd".to_string(), + matches: true, + }, + Request { + account: "bsd".to_string(), + matches: true, + }, + ], + }, + Case { + name: "1 prefix && 2 suffix".into(), + account: None, + partial_accounts: Some(PartialAccounts { + prefixes: vec!["a".to_string()], + suffixes: vec!["c".to_string(), "d".to_string()], + }), + expected: vec![ + Request { + account: "ssssssd".to_string(), + matches: false, + }, + Request { + account: "asasdasdas".to_string(), + matches: false, + }, + Request { + account: "asdc".to_string(), + matches: true, + }, + Request { + account: "absd".to_string(), + matches: true, + }, + ], + }, + Case { + name: "no prefix with exact match".into(), + account: Some("bsda".to_string()), + partial_accounts: Some(PartialAccounts { + prefixes: vec![], + suffixes: vec!["c".to_string(), "d".to_string()], + }), + expected: vec![ + Request { + account: "ssssss".to_string(), + matches: false, + }, + Request { + account: "asasdasdas".to_string(), + matches: false, + }, + Request { + account: "asdasdasdasdc".to_string(), + matches: true, + }, + Request { + account: "bsd".to_string(), + matches: true, + }, + Request { + account: "bsda".to_string(), + matches: true, + }, + ], + }, + Case { + name: "no suffix with exact match".into(), + account: Some("zbsd".to_string()), + partial_accounts: Some(PartialAccounts { + prefixes: vec!["a".to_string(), "b".to_string()], + suffixes: vec![], + }), + expected: vec![ + Request { + account: "ssssssd".to_string(), + matches: false, + }, + Request { + account: "zasdasdas".to_string(), + matches: false, + }, + Request { + account: "asa".to_string(), + matches: true, + }, + Request { + account: "bsb".to_string(), + matches: true, + }, + Request { + account: "zbsd".to_string(), + matches: true, + }, + ], + }, + ]; + + let logger = Logger::root(slog::Discard, o!()); + for case in cases.into_iter() { + let ds = new_data_source(case.account, case.partial_accounts); + let filter = NearReceiptFilter::from_data_sources(vec![&ds]); + + for req in case.expected { + let res = filter.matches(&req.account); + assert_eq!( + res, req.matches, + "name: {} request:{} failed", + case.name, req.account + ); + + let block = Arc::new(new_success_block(11, &req.account)); + let receipt = Arc::new(new_receipt_with_outcome(&req.account, block.clone())); + let res = ds + .match_and_decode(&NearTrigger::Receipt(receipt.clone()), &block, &logger) + .expect("unable to process block"); + assert_eq!( + req.matches, + res.is_some(), + "case name: {} req: {}", + case.name, + req.account + ); + } + } + } + + #[tokio::test] + async fn test_trigger_filter_empty() { + let account1: String = "account1".into(); + + let adapter = TriggersAdapter {}; + + let logger = Logger::root(slog::Discard, o!()); + let block1 = new_success_block(1, &account1); + + let filter = TriggerFilter::default(); + + let block_with_triggers: BlockWithTriggers = adapter + .triggers_in_block(&logger, block1, &filter) + .await + .expect("failed to execute triggers_in_block"); + assert_eq!(block_with_triggers.trigger_count(), 0); + } + + #[tokio::test] + async fn test_trigger_filter_every_block() { + let account1: String = "account1".into(); + + let adapter = TriggersAdapter {}; + + let logger = Logger::root(slog::Discard, o!()); + let block1 = new_success_block(1, &account1); + + let filter = TriggerFilter { + block_filter: crate::adapter::NearBlockFilter { + trigger_every_block: true, + }, + ..Default::default() + }; + + let block_with_triggers: BlockWithTriggers = adapter + .triggers_in_block(&logger, block1, &filter) + .await + .expect("failed to execute triggers_in_block"); + assert_eq!(block_with_triggers.trigger_count(), 1); + + let height: Vec = heights_from_triggers(&block_with_triggers); + assert_eq!(height, vec![1]); + } + + #[tokio::test] + async fn test_trigger_filter_every_receipt() { + let account1: String = "account1".into(); + + let adapter = TriggersAdapter {}; + + let logger = Logger::root(slog::Discard, o!()); + let block1 = new_success_block(1, &account1); + + let filter = TriggerFilter { + receipt_filter: NearReceiptFilter { + accounts: HashSet::from_iter(vec![account1]), + partial_accounts: HashSet::new(), + }, + ..Default::default() + }; + + let block_with_triggers: BlockWithTriggers = adapter + .triggers_in_block(&logger, block1, &filter) + .await + .expect("failed to execute triggers_in_block"); + assert_eq!(block_with_triggers.trigger_count(), 1); + + let height: Vec = heights_from_triggers(&block_with_triggers); + assert_eq!(height.len(), 0); + } + + fn heights_from_triggers(block: &BlockWithTriggers) -> Vec { + block + .trigger_data + .clone() + .into_iter() + .filter_map(|x| match x { + crate::trigger::NearTrigger::Block(b) => b.header.clone().map(|x| x.height), + _ => None, + }) + .collect() + } + + fn new_success_block(height: u64, receiver_id: &String) -> codec::Block { + codec::Block { + header: Some(BlockHeader { + height, + hash: Some(codec::CryptoHash { bytes: vec![0; 32] }), + ..Default::default() + }), + shards: vec![IndexerShard { + receipt_execution_outcomes: vec![IndexerExecutionOutcomeWithReceipt { + receipt: Some(crate::codec::Receipt { + receipt: Some(receipt::Receipt::Action(ReceiptAction { + output_data_receivers: vec![DataReceiver { + receiver_id: receiver_id.clone(), + ..Default::default() + }], + ..Default::default() + })), + receiver_id: receiver_id.clone(), + ..Default::default() + }), + execution_outcome: Some(ExecutionOutcomeWithId { + outcome: Some(ExecutionOutcome { + status: Some(execution_outcome::Status::SuccessValue( + SuccessValueExecutionStatus::default(), + )), + + ..Default::default() + }), + ..Default::default() + }), + }], + ..Default::default() + }], + ..Default::default() + } + } + + fn new_data_source( + account: Option, + partial_accounts: Option, + ) -> DataSource { + DataSource { + kind: NEAR_KIND.to_string(), + network: None, + name: "asd".to_string(), + source: crate::data_source::Source { + account, + start_block: 10, + accounts: partial_accounts, + }, + mapping: Mapping { + api_version: Version::parse("1.0.0").expect("unable to parse version"), + language: "".to_string(), + entities: vec![], + block_handlers: vec![], + receipt_handlers: vec![ReceiptHandler { + handler: "asdsa".to_string(), + }], + runtime: Arc::new(vec![]), + link: Link::default(), + }, + context: Arc::new(None), + creation_block: None, + } + } + + fn new_receipt_with_outcome(receiver_id: &String, block: Arc) -> ReceiptWithOutcome { + ReceiptWithOutcome { + outcome: ExecutionOutcomeWithId { + outcome: Some(ExecutionOutcome { + status: Some(execution_outcome::Status::SuccessValue( + SuccessValueExecutionStatus::default(), + )), + + ..Default::default() + }), + ..Default::default() + }, + receipt: codec::Receipt { + receipt: Some(receipt::Receipt::Action(ReceiptAction { + output_data_receivers: vec![DataReceiver { + receiver_id: receiver_id.clone(), + ..Default::default() + }], + ..Default::default() + })), + receiver_id: receiver_id.clone(), + ..Default::default() + }, + block, + } + } +} diff --git a/chain/near/src/codec.rs b/chain/near/src/codec.rs new file mode 100644 index 0000000..854e9dc --- /dev/null +++ b/chain/near/src/codec.rs @@ -0,0 +1,110 @@ +#[rustfmt::skip] +#[path = "protobuf/sf.near.codec.v1.rs"] +pub mod pbcodec; + +use graph::{ + blockchain::Block as BlockchainBlock, + blockchain::BlockPtr, + prelude::{hex, web3::types::H256, BlockNumber}, +}; +use std::convert::TryFrom; +use std::fmt::LowerHex; + +pub use pbcodec::*; + +impl From<&CryptoHash> for H256 { + fn from(input: &CryptoHash) -> Self { + H256::from_slice(&input.bytes) + } +} + +impl LowerHex for &CryptoHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&hex::encode(&self.bytes)) + } +} + +impl BlockHeader { + pub fn parent_ptr(&self) -> Option { + match (self.prev_hash.as_ref(), self.prev_height) { + (Some(hash), number) => Some(BlockPtr::from((H256::from(hash), number))), + _ => None, + } + } +} + +impl<'a> From<&'a BlockHeader> for BlockPtr { + fn from(b: &'a BlockHeader) -> BlockPtr { + BlockPtr::from((H256::from(b.hash.as_ref().unwrap()), b.height)) + } +} + +impl Block { + pub fn header(&self) -> &BlockHeader { + self.header.as_ref().unwrap() + } + + pub fn ptr(&self) -> BlockPtr { + BlockPtr::from(self.header()) + } + + pub fn parent_ptr(&self) -> Option { + self.header().parent_ptr() + } +} + +impl<'a> From<&'a Block> for BlockPtr { + fn from(b: &'a Block) -> BlockPtr { + BlockPtr::from(b.header()) + } +} + +impl BlockchainBlock for Block { + fn number(&self) -> i32 { + BlockNumber::try_from(self.header().height).unwrap() + } + + fn ptr(&self) -> BlockPtr { + self.into() + } + + fn parent_ptr(&self) -> Option { + self.parent_ptr() + } +} + +impl HeaderOnlyBlock { + pub fn header(&self) -> &BlockHeader { + self.header.as_ref().unwrap() + } +} + +impl<'a> From<&'a HeaderOnlyBlock> for BlockPtr { + fn from(b: &'a HeaderOnlyBlock) -> BlockPtr { + BlockPtr::from(b.header()) + } +} + +impl BlockchainBlock for HeaderOnlyBlock { + fn number(&self) -> i32 { + BlockNumber::try_from(self.header().height).unwrap() + } + + fn ptr(&self) -> BlockPtr { + self.into() + } + + fn parent_ptr(&self) -> Option { + self.header().parent_ptr() + } +} + +impl execution_outcome::Status { + pub fn is_success(&self) -> bool { + use execution_outcome::Status::*; + match self { + Unknown(_) | Failure(_) => false, + SuccessValue(_) | SuccessReceiptId(_) => true, + } + } +} diff --git a/chain/near/src/data_source.rs b/chain/near/src/data_source.rs new file mode 100644 index 0000000..c0fa5c6 --- /dev/null +++ b/chain/near/src/data_source.rs @@ -0,0 +1,476 @@ +use graph::blockchain::{Block, TriggerWithHandler}; +use graph::components::store::StoredDynamicDataSource; +use graph::data::subgraph::DataSourceContext; +use graph::prelude::SubgraphManifestValidationError; +use graph::{ + anyhow::{anyhow, Error}, + blockchain::{self, Blockchain}, + prelude::{ + async_trait, info, BlockNumber, CheapClone, DataSourceTemplateInfo, Deserialize, Link, + LinkResolver, Logger, + }, + semver, +}; +use std::{convert::TryFrom, sync::Arc}; + +use crate::chain::Chain; +use crate::trigger::{NearTrigger, ReceiptWithOutcome}; + +pub const NEAR_KIND: &str = "near"; + +/// Runtime representation of a data source. +#[derive(Clone, Debug)] +pub struct DataSource { + pub kind: String, + pub network: Option, + pub name: String, + pub(crate) source: Source, + pub mapping: Mapping, + pub context: Arc>, + pub creation_block: Option, +} + +impl blockchain::DataSource for DataSource { + fn address(&self) -> Option<&[u8]> { + self.source.account.as_ref().map(String::as_bytes) + } + + fn start_block(&self) -> BlockNumber { + self.source.start_block + } + + fn match_and_decode( + &self, + trigger: &::TriggerData, + block: &Arc<::Block>, + _logger: &Logger, + ) -> Result>, Error> { + if self.source.start_block > block.number() { + return Ok(None); + } + + fn account_matches(ds: &DataSource, receipt: &Arc) -> bool { + if Some(&receipt.receipt.receiver_id) == ds.source.account.as_ref() { + return true; + } + + if let Some(partial_accounts) = &ds.source.accounts { + let matches_prefix = if partial_accounts.prefixes.is_empty() { + true + } else { + partial_accounts + .prefixes + .iter() + .any(|prefix| receipt.receipt.receiver_id.starts_with(prefix)) + }; + + let matches_suffix = if partial_accounts.suffixes.is_empty() { + true + } else { + partial_accounts + .suffixes + .iter() + .any(|suffix| receipt.receipt.receiver_id.ends_with(suffix)) + }; + + if matches_prefix && matches_suffix { + return true; + } + } + + false + } + + let handler = match trigger { + // A block trigger matches if a block handler is present. + NearTrigger::Block(_) => match self.handler_for_block() { + Some(handler) => &handler.handler, + None => return Ok(None), + }, + + // A receipt trigger matches if the receiver matches `source.account` and a receipt + // handler is present. + NearTrigger::Receipt(receipt) => { + if !account_matches(self, receipt) { + return Ok(None); + } + + match self.handler_for_receipt() { + Some(handler) => &handler.handler, + None => return Ok(None), + } + } + }; + + Ok(Some(TriggerWithHandler::::new( + trigger.cheap_clone(), + handler.to_owned(), + block.ptr(), + ))) + } + + fn name(&self) -> &str { + &self.name + } + + fn kind(&self) -> &str { + &self.kind + } + + fn network(&self) -> Option<&str> { + self.network.as_ref().map(|s| s.as_str()) + } + + fn context(&self) -> Arc> { + self.context.cheap_clone() + } + + fn creation_block(&self) -> Option { + self.creation_block + } + + fn is_duplicate_of(&self, other: &Self) -> bool { + let DataSource { + kind, + network, + name, + source, + mapping, + context, + + // The creation block is ignored for detection duplicate data sources. + // Contract ABI equality is implicit in `source` and `mapping.abis` equality. + creation_block: _, + } = self; + + // mapping_request_sender, host_metrics, and (most of) host_exports are operational structs + // used at runtime but not needed to define uniqueness; each runtime host should be for a + // unique data source. + kind == &other.kind + && network == &other.network + && name == &other.name + && source == &other.source + && mapping.block_handlers == other.mapping.block_handlers + && context == &other.context + } + + fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { + // FIXME (NEAR): Implement me! + todo!() + } + + fn from_stored_dynamic_data_source( + _template: &DataSourceTemplate, + _stored: StoredDynamicDataSource, + ) -> Result { + // FIXME (NEAR): Implement me correctly + todo!() + } + + fn validate(&self) -> Vec { + let mut errors = Vec::new(); + + if self.kind != NEAR_KIND { + errors.push(anyhow!( + "data source has invalid `kind`, expected {} but found {}", + NEAR_KIND, + self.kind + )) + } + + // Validate that there is a `source` address if there are receipt handlers + let no_source_address = self.address().is_none(); + + // Validate that there are no empty PartialAccount. + let no_partial_addresses = match &self.source.accounts { + None => true, + Some(addrs) => addrs.is_empty(), + }; + + let has_receipt_handlers = !self.mapping.receipt_handlers.is_empty(); + + // Validate not both address and partial addresses are empty. + if (no_source_address && no_partial_addresses) && has_receipt_handlers { + errors.push(SubgraphManifestValidationError::SourceAddressRequired.into()); + }; + + // Validate empty lines not allowed in suffix or prefix + if let Some(partial_accounts) = self.source.accounts.as_ref() { + if partial_accounts.prefixes.iter().any(|x| x.is_empty()) { + errors.push(anyhow!("partial account prefixes can't have empty values")) + } + + if partial_accounts.suffixes.iter().any(|x| x.is_empty()) { + errors.push(anyhow!("partial account suffixes can't have empty values")) + } + } + + // Validate that there are no more than one of both block handlers and receipt handlers + if self.mapping.block_handlers.len() > 1 { + errors.push(anyhow!("data source has duplicated block handlers")); + } + if self.mapping.receipt_handlers.len() > 1 { + errors.push(anyhow!("data source has duplicated receipt handlers")); + } + + errors + } + + fn api_version(&self) -> semver::Version { + self.mapping.api_version.clone() + } + + fn runtime(&self) -> Option>> { + Some(self.mapping.runtime.cheap_clone()) + } +} + +impl DataSource { + fn from_manifest( + kind: String, + network: Option, + name: String, + source: Source, + mapping: Mapping, + context: Option, + ) -> Result { + // Data sources in the manifest are created "before genesis" so they have no creation block. + let creation_block = None; + + Ok(DataSource { + kind, + network, + name, + source, + mapping, + context: Arc::new(context), + creation_block, + }) + } + + fn handler_for_block(&self) -> Option<&MappingBlockHandler> { + self.mapping.block_handlers.first() + } + + fn handler_for_receipt(&self) -> Option<&ReceiptHandler> { + self.mapping.receipt_handlers.first() + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub struct UnresolvedDataSource { + pub kind: String, + pub network: Option, + pub name: String, + pub(crate) source: Source, + pub mapping: UnresolvedMapping, + pub context: Option, +} + +#[async_trait] +impl blockchain::UnresolvedDataSource for UnresolvedDataSource { + async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + _manifest_idx: u32, + ) -> Result { + let UnresolvedDataSource { + kind, + network, + name, + source, + mapping, + context, + } = self; + + info!(logger, "Resolve data source"; "name" => &name, "source_account" => format_args!("{:?}", source.account), "source_start_block" => source.start_block); + + let mapping = mapping.resolve(resolver, logger).await?; + + DataSource::from_manifest(kind, network, name, source, mapping, context) + } +} + +impl TryFrom> for DataSource { + type Error = Error; + + fn try_from(_info: DataSourceTemplateInfo) -> Result { + Err(anyhow!("Near subgraphs do not support templates")) + + // How this might be implemented if/when Near gets support for templates: + // let DataSourceTemplateInfo { + // template, + // params, + // context, + // creation_block, + // } = info; + + // let account = params + // .get(0) + // .with_context(|| { + // format!( + // "Failed to create data source from template `{}`: account parameter is missing", + // template.name + // ) + // })? + // .clone(); + + // Ok(DataSource { + // kind: template.kind, + // network: template.network, + // name: template.name, + // source: Source { + // account, + // start_block: 0, + // }, + // mapping: template.mapping, + // context: Arc::new(context), + // creation_block: Some(creation_block), + // }) + } +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +pub struct BaseDataSourceTemplate { + pub kind: String, + pub network: Option, + pub name: String, + pub mapping: M, +} + +pub type UnresolvedDataSourceTemplate = BaseDataSourceTemplate; +pub type DataSourceTemplate = BaseDataSourceTemplate; + +#[async_trait] +impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { + async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + _manifest_idx: u32, + ) -> Result { + let UnresolvedDataSourceTemplate { + kind, + network, + name, + mapping, + } = self; + + info!(logger, "Resolve data source template"; "name" => &name); + + Ok(DataSourceTemplate { + kind, + network, + name, + mapping: mapping.resolve(resolver, logger).await?, + }) + } +} + +impl blockchain::DataSourceTemplate for DataSourceTemplate { + fn name(&self) -> &str { + &self.name + } + + fn api_version(&self) -> semver::Version { + self.mapping.api_version.clone() + } + + fn runtime(&self) -> Option>> { + Some(self.mapping.runtime.cheap_clone()) + } + + fn manifest_idx(&self) -> u32 { + unreachable!("near does not support dynamic data sources") + } +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UnresolvedMapping { + pub api_version: String, + pub language: String, + pub entities: Vec, + #[serde(default)] + pub block_handlers: Vec, + #[serde(default)] + pub receipt_handlers: Vec, + pub file: Link, +} + +impl UnresolvedMapping { + pub async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + ) -> Result { + let UnresolvedMapping { + api_version, + language, + entities, + block_handlers, + receipt_handlers, + file: link, + } = self; + + let api_version = semver::Version::parse(&api_version)?; + + info!(logger, "Resolve mapping"; "link" => &link.link); + let module_bytes = resolver.cat(logger, &link).await?; + + Ok(Mapping { + api_version, + language, + entities, + block_handlers, + receipt_handlers, + runtime: Arc::new(module_bytes), + link, + }) + } +} + +#[derive(Clone, Debug)] +pub struct Mapping { + pub api_version: semver::Version, + pub language: String, + pub entities: Vec, + pub block_handlers: Vec, + pub receipt_handlers: Vec, + pub runtime: Arc>, + pub link: Link, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct MappingBlockHandler { + pub handler: String, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct ReceiptHandler { + pub(crate) handler: String, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize, Default)] +pub(crate) struct PartialAccounts { + #[serde(default)] + pub(crate) prefixes: Vec, + #[serde(default)] + pub(crate) suffixes: Vec, +} + +impl PartialAccounts { + pub fn is_empty(&self) -> bool { + self.prefixes.is_empty() && self.suffixes.is_empty() + } +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub(crate) struct Source { + // A data source that does not have an account or accounts can only have block handlers. + pub(crate) account: Option, + #[serde(rename = "startBlock", default)] + pub(crate) start_block: BlockNumber, + pub(crate) accounts: Option, +} diff --git a/chain/near/src/lib.rs b/chain/near/src/lib.rs new file mode 100644 index 0000000..2ab7dd8 --- /dev/null +++ b/chain/near/src/lib.rs @@ -0,0 +1,11 @@ +mod adapter; +mod capabilities; +mod chain; +pub mod codec; +mod data_source; +mod runtime; +mod trigger; + +pub use crate::chain::Chain; +pub use crate::chain::NearStreamBuilder; +pub use codec::HeaderOnlyBlock; diff --git a/chain/near/src/protobuf/sf.near.codec.v1.rs b/chain/near/src/protobuf/sf.near.codec.v1.rs new file mode 100644 index 0000000..4a58d01 --- /dev/null +++ b/chain/near/src/protobuf/sf.near.codec.v1.rs @@ -0,0 +1,850 @@ +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Block { + #[prost(string, tag="1")] + pub author: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub header: ::core::option::Option, + #[prost(message, repeated, tag="3")] + pub chunk_headers: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="4")] + pub shards: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="5")] + pub state_changes: ::prost::alloc::vec::Vec, +} +/// HeaderOnlyBlock is a standard \[Block\] structure where all other fields are +/// removed so that hydrating that object from a \[Block\] bytes payload will +/// drastically reduced allocated memory required to hold the full block. +/// +/// This can be used to unpack a \[Block\] when only the \[BlockHeader\] information +/// is required and greatly reduced required memory. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct HeaderOnlyBlock { + #[prost(message, optional, tag="2")] + pub header: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StateChangeWithCause { + #[prost(message, optional, tag="1")] + pub value: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub cause: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StateChangeCause { + #[prost(oneof="state_change_cause::Cause", tags="1, 2, 3, 4, 5, 6, 7, 8, 9, 10")] + pub cause: ::core::option::Option, +} +/// Nested message and enum types in `StateChangeCause`. +pub mod state_change_cause { + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct NotWritableToDisk { + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct InitialState { + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct TransactionProcessing { + #[prost(message, optional, tag="1")] + pub tx_hash: ::core::option::Option, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct ActionReceiptProcessingStarted { + #[prost(message, optional, tag="1")] + pub receipt_hash: ::core::option::Option, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct ActionReceiptGasReward { + #[prost(message, optional, tag="1")] + pub tx_hash: ::core::option::Option, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct ReceiptProcessing { + #[prost(message, optional, tag="1")] + pub tx_hash: ::core::option::Option, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct PostponedReceipt { + #[prost(message, optional, tag="1")] + pub tx_hash: ::core::option::Option, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct UpdatedDelayedReceipts { + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct ValidatorAccountsUpdate { + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Migration { + } + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Cause { + #[prost(message, tag="1")] + NotWritableToDisk(NotWritableToDisk), + #[prost(message, tag="2")] + InitialState(InitialState), + #[prost(message, tag="3")] + TransactionProcessing(TransactionProcessing), + #[prost(message, tag="4")] + ActionReceiptProcessingStarted(ActionReceiptProcessingStarted), + #[prost(message, tag="5")] + ActionReceiptGasReward(ActionReceiptGasReward), + #[prost(message, tag="6")] + ReceiptProcessing(ReceiptProcessing), + #[prost(message, tag="7")] + PostponedReceipt(PostponedReceipt), + #[prost(message, tag="8")] + UpdatedDelayedReceipts(UpdatedDelayedReceipts), + #[prost(message, tag="9")] + ValidatorAccountsUpdate(ValidatorAccountsUpdate), + #[prost(message, tag="10")] + Migration(Migration), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StateChangeValue { + #[prost(oneof="state_change_value::Value", tags="1, 2, 3, 4, 5, 6, 7, 8")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `StateChangeValue`. +pub mod state_change_value { + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct AccountUpdate { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub account: ::core::option::Option, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct AccountDeletion { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct AccessKeyUpdate { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub public_key: ::core::option::Option, + #[prost(message, optional, tag="3")] + pub access_key: ::core::option::Option, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct AccessKeyDeletion { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub public_key: ::core::option::Option, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct DataUpdate { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(bytes="vec", tag="2")] + pub key: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="3")] + pub value: ::prost::alloc::vec::Vec, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct DataDeletion { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(bytes="vec", tag="2")] + pub key: ::prost::alloc::vec::Vec, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct ContractCodeUpdate { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(bytes="vec", tag="2")] + pub code: ::prost::alloc::vec::Vec, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct ContractCodeDeletion { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + } + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Value { + #[prost(message, tag="1")] + AccountUpdate(AccountUpdate), + #[prost(message, tag="2")] + AccountDeletion(AccountDeletion), + #[prost(message, tag="3")] + AccessKeyUpdate(AccessKeyUpdate), + #[prost(message, tag="4")] + AccessKeyDeletion(AccessKeyDeletion), + #[prost(message, tag="5")] + DataUpdate(DataUpdate), + #[prost(message, tag="6")] + DataDeletion(DataDeletion), + #[prost(message, tag="7")] + ContractCodeUpdate(ContractCodeUpdate), + #[prost(message, tag="8")] + ContractDeletion(ContractCodeDeletion), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Account { + #[prost(message, optional, tag="1")] + pub amount: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub locked: ::core::option::Option, + #[prost(message, optional, tag="3")] + pub code_hash: ::core::option::Option, + #[prost(uint64, tag="4")] + pub storage_usage: u64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockHeader { + #[prost(uint64, tag="1")] + pub height: u64, + #[prost(uint64, tag="2")] + pub prev_height: u64, + #[prost(message, optional, tag="3")] + pub epoch_id: ::core::option::Option, + #[prost(message, optional, tag="4")] + pub next_epoch_id: ::core::option::Option, + #[prost(message, optional, tag="5")] + pub hash: ::core::option::Option, + #[prost(message, optional, tag="6")] + pub prev_hash: ::core::option::Option, + #[prost(message, optional, tag="7")] + pub prev_state_root: ::core::option::Option, + #[prost(message, optional, tag="8")] + pub chunk_receipts_root: ::core::option::Option, + #[prost(message, optional, tag="9")] + pub chunk_headers_root: ::core::option::Option, + #[prost(message, optional, tag="10")] + pub chunk_tx_root: ::core::option::Option, + #[prost(message, optional, tag="11")] + pub outcome_root: ::core::option::Option, + #[prost(uint64, tag="12")] + pub chunks_included: u64, + #[prost(message, optional, tag="13")] + pub challenges_root: ::core::option::Option, + #[prost(uint64, tag="14")] + pub timestamp: u64, + #[prost(uint64, tag="15")] + pub timestamp_nanosec: u64, + #[prost(message, optional, tag="16")] + pub random_value: ::core::option::Option, + #[prost(message, repeated, tag="17")] + pub validator_proposals: ::prost::alloc::vec::Vec, + #[prost(bool, repeated, tag="18")] + pub chunk_mask: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="19")] + pub gas_price: ::core::option::Option, + #[prost(uint64, tag="20")] + pub block_ordinal: u64, + #[prost(message, optional, tag="21")] + pub total_supply: ::core::option::Option, + #[prost(message, repeated, tag="22")] + pub challenges_result: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="23")] + pub last_final_block_height: u64, + #[prost(message, optional, tag="24")] + pub last_final_block: ::core::option::Option, + #[prost(uint64, tag="25")] + pub last_ds_final_block_height: u64, + #[prost(message, optional, tag="26")] + pub last_ds_final_block: ::core::option::Option, + #[prost(message, optional, tag="27")] + pub next_bp_hash: ::core::option::Option, + #[prost(message, optional, tag="28")] + pub block_merkle_root: ::core::option::Option, + #[prost(bytes="vec", tag="29")] + pub epoch_sync_data_hash: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="30")] + pub approvals: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="31")] + pub signature: ::core::option::Option, + #[prost(uint32, tag="32")] + pub latest_protocol_version: u32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BigInt { + #[prost(bytes="vec", tag="1")] + pub bytes: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CryptoHash { + #[prost(bytes="vec", tag="1")] + pub bytes: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Signature { + #[prost(enumeration="CurveKind", tag="1")] + pub r#type: i32, + #[prost(bytes="vec", tag="2")] + pub bytes: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PublicKey { + #[prost(enumeration="CurveKind", tag="1")] + pub r#type: i32, + #[prost(bytes="vec", tag="2")] + pub bytes: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ValidatorStake { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub public_key: ::core::option::Option, + #[prost(message, optional, tag="3")] + pub stake: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SlashedValidator { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(bool, tag="2")] + pub is_double_sign: bool, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ChunkHeader { + #[prost(bytes="vec", tag="1")] + pub chunk_hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="2")] + pub prev_block_hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="3")] + pub outcome_root: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="4")] + pub prev_state_root: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="5")] + pub encoded_merkle_root: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="6")] + pub encoded_length: u64, + #[prost(uint64, tag="7")] + pub height_created: u64, + #[prost(uint64, tag="8")] + pub height_included: u64, + #[prost(uint64, tag="9")] + pub shard_id: u64, + #[prost(uint64, tag="10")] + pub gas_used: u64, + #[prost(uint64, tag="11")] + pub gas_limit: u64, + #[prost(message, optional, tag="12")] + pub validator_reward: ::core::option::Option, + #[prost(message, optional, tag="13")] + pub balance_burnt: ::core::option::Option, + #[prost(bytes="vec", tag="14")] + pub outgoing_receipts_root: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="15")] + pub tx_root: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="16")] + pub validator_proposals: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="17")] + pub signature: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IndexerShard { + #[prost(uint64, tag="1")] + pub shard_id: u64, + #[prost(message, optional, tag="2")] + pub chunk: ::core::option::Option, + #[prost(message, repeated, tag="3")] + pub receipt_execution_outcomes: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IndexerExecutionOutcomeWithReceipt { + #[prost(message, optional, tag="1")] + pub execution_outcome: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub receipt: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IndexerChunk { + #[prost(string, tag="1")] + pub author: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub header: ::core::option::Option, + #[prost(message, repeated, tag="3")] + pub transactions: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="4")] + pub receipts: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IndexerTransactionWithOutcome { + #[prost(message, optional, tag="1")] + pub transaction: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub outcome: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SignedTransaction { + #[prost(string, tag="1")] + pub signer_id: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub public_key: ::core::option::Option, + #[prost(uint64, tag="3")] + pub nonce: u64, + #[prost(string, tag="4")] + pub receiver_id: ::prost::alloc::string::String, + #[prost(message, repeated, tag="5")] + pub actions: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="6")] + pub signature: ::core::option::Option, + #[prost(message, optional, tag="7")] + pub hash: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IndexerExecutionOutcomeWithOptionalReceipt { + #[prost(message, optional, tag="1")] + pub execution_outcome: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub receipt: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Receipt { + #[prost(string, tag="1")] + pub predecessor_id: ::prost::alloc::string::String, + #[prost(string, tag="2")] + pub receiver_id: ::prost::alloc::string::String, + #[prost(message, optional, tag="3")] + pub receipt_id: ::core::option::Option, + #[prost(oneof="receipt::Receipt", tags="10, 11")] + pub receipt: ::core::option::Option, +} +/// Nested message and enum types in `Receipt`. +pub mod receipt { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Receipt { + #[prost(message, tag="10")] + Action(super::ReceiptAction), + #[prost(message, tag="11")] + Data(super::ReceiptData), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReceiptData { + #[prost(message, optional, tag="1")] + pub data_id: ::core::option::Option, + #[prost(bytes="vec", tag="2")] + pub data: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReceiptAction { + #[prost(string, tag="1")] + pub signer_id: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub signer_public_key: ::core::option::Option, + #[prost(message, optional, tag="3")] + pub gas_price: ::core::option::Option, + #[prost(message, repeated, tag="4")] + pub output_data_receivers: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="5")] + pub input_data_ids: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="6")] + pub actions: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DataReceiver { + #[prost(message, optional, tag="1")] + pub data_id: ::core::option::Option, + #[prost(string, tag="2")] + pub receiver_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ExecutionOutcomeWithId { + #[prost(message, optional, tag="1")] + pub proof: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub block_hash: ::core::option::Option, + #[prost(message, optional, tag="3")] + pub id: ::core::option::Option, + #[prost(message, optional, tag="4")] + pub outcome: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ExecutionOutcome { + #[prost(string, repeated, tag="1")] + pub logs: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(message, repeated, tag="2")] + pub receipt_ids: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="3")] + pub gas_burnt: u64, + #[prost(message, optional, tag="4")] + pub tokens_burnt: ::core::option::Option, + #[prost(string, tag="5")] + pub executor_id: ::prost::alloc::string::String, + #[prost(enumeration="ExecutionMetadata", tag="6")] + pub metadata: i32, + #[prost(oneof="execution_outcome::Status", tags="20, 21, 22, 23")] + pub status: ::core::option::Option, +} +/// Nested message and enum types in `ExecutionOutcome`. +pub mod execution_outcome { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Status { + #[prost(message, tag="20")] + Unknown(super::UnknownExecutionStatus), + #[prost(message, tag="21")] + Failure(super::FailureExecutionStatus), + #[prost(message, tag="22")] + SuccessValue(super::SuccessValueExecutionStatus), + #[prost(message, tag="23")] + SuccessReceiptId(super::SuccessReceiptIdExecutionStatus), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SuccessValueExecutionStatus { + #[prost(bytes="vec", tag="1")] + pub value: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SuccessReceiptIdExecutionStatus { + #[prost(message, optional, tag="1")] + pub id: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct UnknownExecutionStatus { +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FailureExecutionStatus { + #[prost(oneof="failure_execution_status::Failure", tags="1, 2")] + pub failure: ::core::option::Option, +} +/// Nested message and enum types in `FailureExecutionStatus`. +pub mod failure_execution_status { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Failure { + #[prost(message, tag="1")] + ActionError(super::ActionError), + #[prost(enumeration="super::InvalidTxError", tag="2")] + InvalidTxError(i32), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ActionError { + #[prost(uint64, tag="1")] + pub index: u64, + #[prost(oneof="action_error::Kind", tags="21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36")] + pub kind: ::core::option::Option, +} +/// Nested message and enum types in `ActionError`. +pub mod action_error { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Kind { + #[prost(message, tag="21")] + AccountAlreadyExist(super::AccountAlreadyExistsErrorKind), + #[prost(message, tag="22")] + AccountDoesNotExist(super::AccountDoesNotExistErrorKind), + #[prost(message, tag="23")] + CreateAccountOnlyByRegistrar(super::CreateAccountOnlyByRegistrarErrorKind), + #[prost(message, tag="24")] + CreateAccountNotAllowed(super::CreateAccountNotAllowedErrorKind), + #[prost(message, tag="25")] + ActorNoPermission(super::ActorNoPermissionErrorKind), + #[prost(message, tag="26")] + DeleteKeyDoesNotExist(super::DeleteKeyDoesNotExistErrorKind), + #[prost(message, tag="27")] + AddKeyAlreadyExists(super::AddKeyAlreadyExistsErrorKind), + #[prost(message, tag="28")] + DeleteAccountStaking(super::DeleteAccountStakingErrorKind), + #[prost(message, tag="29")] + LackBalanceForState(super::LackBalanceForStateErrorKind), + #[prost(message, tag="30")] + TriesToUnstake(super::TriesToUnstakeErrorKind), + #[prost(message, tag="31")] + TriesToStake(super::TriesToStakeErrorKind), + #[prost(message, tag="32")] + InsufficientStake(super::InsufficientStakeErrorKind), + #[prost(message, tag="33")] + FunctionCall(super::FunctionCallErrorKind), + #[prost(message, tag="34")] + NewReceiptValidation(super::NewReceiptValidationErrorKind), + #[prost(message, tag="35")] + OnlyImplicitAccountCreationAllowed(super::OnlyImplicitAccountCreationAllowedErrorKind), + #[prost(message, tag="36")] + DeleteAccountWithLargeState(super::DeleteAccountWithLargeStateErrorKind), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AccountAlreadyExistsErrorKind { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AccountDoesNotExistErrorKind { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, +} +//// A top-level account ID can only be created by registrar. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CreateAccountOnlyByRegistrarErrorKind { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(string, tag="2")] + pub registrar_account_id: ::prost::alloc::string::String, + #[prost(string, tag="3")] + pub predecessor_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CreateAccountNotAllowedErrorKind { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(string, tag="2")] + pub predecessor_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ActorNoPermissionErrorKind { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(string, tag="2")] + pub actor_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteKeyDoesNotExistErrorKind { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub public_key: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AddKeyAlreadyExistsErrorKind { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub public_key: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteAccountStakingErrorKind { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LackBalanceForStateErrorKind { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub balance: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TriesToUnstakeErrorKind { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TriesToStakeErrorKind { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub stake: ::core::option::Option, + #[prost(message, optional, tag="3")] + pub locked: ::core::option::Option, + #[prost(message, optional, tag="4")] + pub balance: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct InsufficientStakeErrorKind { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub stake: ::core::option::Option, + #[prost(message, optional, tag="3")] + pub minimum_stake: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FunctionCallErrorKind { + #[prost(enumeration="FunctionCallErrorSer", tag="1")] + pub error: i32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NewReceiptValidationErrorKind { + #[prost(enumeration="ReceiptValidationError", tag="1")] + pub error: i32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OnlyImplicitAccountCreationAllowedErrorKind { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteAccountWithLargeStateErrorKind { + #[prost(string, tag="1")] + pub account_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MerklePath { + #[prost(message, repeated, tag="1")] + pub path: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MerklePathItem { + #[prost(message, optional, tag="1")] + pub hash: ::core::option::Option, + #[prost(enumeration="Direction", tag="2")] + pub direction: i32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Action { + #[prost(oneof="action::Action", tags="1, 2, 3, 4, 5, 6, 7, 8")] + pub action: ::core::option::Option, +} +/// Nested message and enum types in `Action`. +pub mod action { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Action { + #[prost(message, tag="1")] + CreateAccount(super::CreateAccountAction), + #[prost(message, tag="2")] + DeployContract(super::DeployContractAction), + #[prost(message, tag="3")] + FunctionCall(super::FunctionCallAction), + #[prost(message, tag="4")] + Transfer(super::TransferAction), + #[prost(message, tag="5")] + Stake(super::StakeAction), + #[prost(message, tag="6")] + AddKey(super::AddKeyAction), + #[prost(message, tag="7")] + DeleteKey(super::DeleteKeyAction), + #[prost(message, tag="8")] + DeleteAccount(super::DeleteAccountAction), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CreateAccountAction { +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeployContractAction { + #[prost(bytes="vec", tag="1")] + pub code: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FunctionCallAction { + #[prost(string, tag="1")] + pub method_name: ::prost::alloc::string::String, + #[prost(bytes="vec", tag="2")] + pub args: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="3")] + pub gas: u64, + #[prost(message, optional, tag="4")] + pub deposit: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransferAction { + #[prost(message, optional, tag="1")] + pub deposit: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StakeAction { + #[prost(message, optional, tag="1")] + pub stake: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub public_key: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AddKeyAction { + #[prost(message, optional, tag="1")] + pub public_key: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub access_key: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteKeyAction { + #[prost(message, optional, tag="1")] + pub public_key: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteAccountAction { + #[prost(string, tag="1")] + pub beneficiary_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AccessKey { + #[prost(uint64, tag="1")] + pub nonce: u64, + #[prost(message, optional, tag="2")] + pub permission: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AccessKeyPermission { + #[prost(oneof="access_key_permission::Permission", tags="1, 2")] + pub permission: ::core::option::Option, +} +/// Nested message and enum types in `AccessKeyPermission`. +pub mod access_key_permission { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Permission { + #[prost(message, tag="1")] + FunctionCall(super::FunctionCallPermission), + #[prost(message, tag="2")] + FullAccess(super::FullAccessPermission), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FunctionCallPermission { + #[prost(message, optional, tag="1")] + pub allowance: ::core::option::Option, + #[prost(string, tag="2")] + pub receiver_id: ::prost::alloc::string::String, + #[prost(string, repeated, tag="3")] + pub method_names: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FullAccessPermission { +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum CurveKind { + Ed25519 = 0, + Secp256k1 = 1, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ExecutionMetadata { + V1 = 0, +} +///todo: add more detail? +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum FunctionCallErrorSer { + CompilationError = 0, + LinkError = 1, + MethodResolveError = 2, + WasmTrap = 3, + WasmUnknownError = 4, + HostError = 5, + EvmError = 6, + ExecutionError = 7, +} +///todo: add more detail? +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ReceiptValidationError { + InvalidPredecessorId = 0, + InvalidReceiverAccountId = 1, + InvalidSignerAccountId = 2, + InvalidDataReceiverId = 3, + ReturnedValueLengthExceeded = 4, + NumberInputDataDependenciesExceeded = 5, + ActionsValidationError = 6, +} +///todo: add more detail? +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum InvalidTxError { + InvalidAccessKeyError = 0, + InvalidSignerId = 1, + SignerDoesNotExist = 2, + InvalidNonce = 3, + NonceTooLarge = 4, + InvalidReceiverId = 5, + InvalidSignature = 6, + NotEnoughBalance = 7, + LackBalanceForState = 8, + CostOverflow = 9, + InvalidChain = 10, + Expired = 11, + ActionsValidation = 12, + TransactionSizeExceeded = 13, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum Direction { + Left = 0, + Right = 1, +} diff --git a/chain/near/src/runtime/abi.rs b/chain/near/src/runtime/abi.rs new file mode 100644 index 0000000..fdd3be1 --- /dev/null +++ b/chain/near/src/runtime/abi.rs @@ -0,0 +1,637 @@ +use crate::codec; +use crate::trigger::ReceiptWithOutcome; +use graph::anyhow::anyhow; +use graph::runtime::gas::GasCounter; +use graph::runtime::{asc_new, AscHeap, AscPtr, DeterministicHostError, ToAscObj}; +use graph_runtime_wasm::asc_abi::class::{Array, AscEnum, EnumPayload, Uint8Array}; + +pub(crate) use super::generated::*; + +impl ToAscObj for codec::Block { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscBlock { + author: asc_new(heap, &self.author, gas)?, + header: asc_new(heap, self.header(), gas)?, + chunks: asc_new(heap, &self.chunk_headers, gas)?, + }) + } +} + +impl ToAscObj for codec::BlockHeader { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let chunk_mask = Array::new(self.chunk_mask.as_ref(), heap, gas)?; + + Ok(AscBlockHeader { + height: self.height, + prev_height: self.prev_height, + epoch_id: asc_new(heap, self.epoch_id.as_ref().unwrap(), gas)?, + next_epoch_id: asc_new(heap, self.next_epoch_id.as_ref().unwrap(), gas)?, + hash: asc_new(heap, self.hash.as_ref().unwrap(), gas)?, + prev_hash: asc_new(heap, self.prev_hash.as_ref().unwrap(), gas)?, + prev_state_root: asc_new(heap, self.prev_state_root.as_ref().unwrap(), gas)?, + chunk_receipts_root: asc_new(heap, self.chunk_receipts_root.as_ref().unwrap(), gas)?, + chunk_headers_root: asc_new(heap, self.chunk_headers_root.as_ref().unwrap(), gas)?, + chunk_tx_root: asc_new(heap, self.chunk_tx_root.as_ref().unwrap(), gas)?, + outcome_root: asc_new(heap, self.outcome_root.as_ref().unwrap(), gas)?, + chunks_included: self.chunks_included, + challenges_root: asc_new(heap, self.challenges_root.as_ref().unwrap(), gas)?, + timestamp_nanosec: self.timestamp_nanosec, + random_value: asc_new(heap, self.random_value.as_ref().unwrap(), gas)?, + validator_proposals: asc_new(heap, &self.validator_proposals, gas)?, + chunk_mask: AscPtr::alloc_obj(chunk_mask, heap, gas)?, + gas_price: asc_new(heap, self.gas_price.as_ref().unwrap(), gas)?, + block_ordinal: self.block_ordinal, + total_supply: asc_new(heap, self.total_supply.as_ref().unwrap(), gas)?, + challenges_result: asc_new(heap, &self.challenges_result, gas)?, + last_final_block: asc_new(heap, self.last_final_block.as_ref().unwrap(), gas)?, + last_ds_final_block: asc_new(heap, self.last_ds_final_block.as_ref().unwrap(), gas)?, + next_bp_hash: asc_new(heap, self.next_bp_hash.as_ref().unwrap(), gas)?, + block_merkle_root: asc_new(heap, self.block_merkle_root.as_ref().unwrap(), gas)?, + epoch_sync_data_hash: asc_new(heap, self.epoch_sync_data_hash.as_slice(), gas)?, + approvals: asc_new(heap, &self.approvals, gas)?, + signature: asc_new(heap, &self.signature.as_ref().unwrap(), gas)?, + latest_protocol_version: self.latest_protocol_version, + }) + } +} + +impl ToAscObj for codec::ChunkHeader { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscChunkHeader { + chunk_hash: asc_new(heap, self.chunk_hash.as_slice(), gas)?, + signature: asc_new(heap, &self.signature.as_ref().unwrap(), gas)?, + prev_block_hash: asc_new(heap, self.prev_block_hash.as_slice(), gas)?, + prev_state_root: asc_new(heap, self.prev_state_root.as_slice(), gas)?, + encoded_merkle_root: asc_new(heap, self.encoded_merkle_root.as_slice(), gas)?, + encoded_length: self.encoded_length, + height_created: self.height_created, + height_included: self.height_included, + shard_id: self.shard_id, + gas_used: self.gas_used, + gas_limit: self.gas_limit, + balance_burnt: asc_new(heap, self.balance_burnt.as_ref().unwrap(), gas)?, + outgoing_receipts_root: asc_new(heap, self.outgoing_receipts_root.as_slice(), gas)?, + tx_root: asc_new(heap, self.tx_root.as_slice(), gas)?, + validator_proposals: asc_new(heap, &self.validator_proposals, gas)?, + + _padding: 0, + }) + } +} + +impl ToAscObj for Vec { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); + let content = content?; + Ok(AscChunkHeaderArray(Array::new(&*content, heap, gas)?)) + } +} + +impl ToAscObj for ReceiptWithOutcome { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscReceiptWithOutcome { + outcome: asc_new(heap, &self.outcome, gas)?, + receipt: asc_new(heap, &self.receipt, gas)?, + block: asc_new(heap, self.block.as_ref(), gas)?, + }) + } +} + +impl ToAscObj for codec::Receipt { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let action = match self.receipt.as_ref().unwrap() { + codec::receipt::Receipt::Action(action) => action, + codec::receipt::Receipt::Data(_) => { + return Err(DeterministicHostError::from(anyhow!( + "Data receipt are now allowed" + ))); + } + }; + + Ok(AscActionReceipt { + id: asc_new(heap, &self.receipt_id.as_ref().unwrap(), gas)?, + predecessor_id: asc_new(heap, &self.predecessor_id, gas)?, + receiver_id: asc_new(heap, &self.receiver_id, gas)?, + signer_id: asc_new(heap, &action.signer_id, gas)?, + signer_public_key: asc_new(heap, action.signer_public_key.as_ref().unwrap(), gas)?, + gas_price: asc_new(heap, action.gas_price.as_ref().unwrap(), gas)?, + output_data_receivers: asc_new(heap, &action.output_data_receivers, gas)?, + input_data_ids: asc_new(heap, &action.input_data_ids, gas)?, + actions: asc_new(heap, &action.actions, gas)?, + }) + } +} + +impl ToAscObj for codec::Action { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let (kind, payload) = match self.action.as_ref().unwrap() { + codec::action::Action::CreateAccount(action) => ( + AscActionKind::CreateAccount, + asc_new(heap, action, gas)?.to_payload(), + ), + codec::action::Action::DeployContract(action) => ( + AscActionKind::DeployContract, + asc_new(heap, action, gas)?.to_payload(), + ), + codec::action::Action::FunctionCall(action) => ( + AscActionKind::FunctionCall, + asc_new(heap, action, gas)?.to_payload(), + ), + codec::action::Action::Transfer(action) => ( + AscActionKind::Transfer, + asc_new(heap, action, gas)?.to_payload(), + ), + codec::action::Action::Stake(action) => ( + AscActionKind::Stake, + asc_new(heap, action, gas)?.to_payload(), + ), + codec::action::Action::AddKey(action) => ( + AscActionKind::AddKey, + asc_new(heap, action, gas)?.to_payload(), + ), + codec::action::Action::DeleteKey(action) => ( + AscActionKind::DeleteKey, + asc_new(heap, action, gas)?.to_payload(), + ), + codec::action::Action::DeleteAccount(action) => ( + AscActionKind::DeleteAccount, + asc_new(heap, action, gas)?.to_payload(), + ), + }; + + Ok(AscActionEnum(AscEnum { + kind, + _padding: 0, + payload: EnumPayload(payload), + })) + } +} + +impl ToAscObj for Vec { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); + let content = content?; + Ok(AscActionEnumArray(Array::new(&*content, heap, gas)?)) + } +} + +impl ToAscObj for codec::CreateAccountAction { + fn to_asc_obj( + &self, + _heap: &mut H, + _gas: &GasCounter, + ) -> Result { + Ok(AscCreateAccountAction {}) + } +} + +impl ToAscObj for codec::DeployContractAction { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscDeployContractAction { + code: asc_new(heap, self.code.as_slice(), gas)?, + }) + } +} + +impl ToAscObj for codec::FunctionCallAction { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscFunctionCallAction { + method_name: asc_new(heap, &self.method_name, gas)?, + args: asc_new(heap, self.args.as_slice(), gas)?, + gas: self.gas, + deposit: asc_new(heap, self.deposit.as_ref().unwrap(), gas)?, + _padding: 0, + }) + } +} + +impl ToAscObj for codec::TransferAction { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscTransferAction { + deposit: asc_new(heap, self.deposit.as_ref().unwrap(), gas)?, + }) + } +} + +impl ToAscObj for codec::StakeAction { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscStakeAction { + stake: asc_new(heap, self.stake.as_ref().unwrap(), gas)?, + public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas)?, + }) + } +} + +impl ToAscObj for codec::AddKeyAction { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscAddKeyAction { + public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas)?, + access_key: asc_new(heap, self.access_key.as_ref().unwrap(), gas)?, + }) + } +} + +impl ToAscObj for codec::AccessKey { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscAccessKey { + nonce: self.nonce, + permission: asc_new(heap, self.permission.as_ref().unwrap(), gas)?, + _padding: 0, + }) + } +} + +impl ToAscObj for codec::AccessKeyPermission { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let (kind, payload) = match self.permission.as_ref().unwrap() { + codec::access_key_permission::Permission::FunctionCall(permission) => ( + AscAccessKeyPermissionKind::FunctionCall, + asc_new(heap, permission, gas)?.to_payload(), + ), + codec::access_key_permission::Permission::FullAccess(permission) => ( + AscAccessKeyPermissionKind::FullAccess, + asc_new(heap, permission, gas)?.to_payload(), + ), + }; + + Ok(AscAccessKeyPermissionEnum(AscEnum { + _padding: 0, + kind, + payload: EnumPayload(payload), + })) + } +} + +impl ToAscObj for codec::FunctionCallPermission { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscFunctionCallPermission { + // The `allowance` field is one of the few fields that can actually be None for real + allowance: match self.allowance.as_ref() { + Some(allowance) => asc_new(heap, allowance, gas)?, + None => AscPtr::null(), + }, + receiver_id: asc_new(heap, &self.receiver_id, gas)?, + method_names: asc_new(heap, &self.method_names, gas)?, + }) + } +} + +impl ToAscObj for codec::FullAccessPermission { + fn to_asc_obj( + &self, + _heap: &mut H, + _gas: &GasCounter, + ) -> Result { + Ok(AscFullAccessPermission {}) + } +} + +impl ToAscObj for codec::DeleteKeyAction { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscDeleteKeyAction { + public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas)?, + }) + } +} + +impl ToAscObj for codec::DeleteAccountAction { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscDeleteAccountAction { + beneficiary_id: asc_new(heap, &self.beneficiary_id, gas)?, + }) + } +} + +impl ToAscObj for codec::DataReceiver { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscDataReceiver { + data_id: asc_new(heap, self.data_id.as_ref().unwrap(), gas)?, + receiver_id: asc_new(heap, &self.receiver_id, gas)?, + }) + } +} + +impl ToAscObj for Vec { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); + let content = content?; + Ok(AscDataReceiverArray(Array::new(&*content, heap, gas)?)) + } +} + +impl ToAscObj for codec::ExecutionOutcomeWithId { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let outcome = self.outcome.as_ref().unwrap(); + + Ok(AscExecutionOutcome { + proof: asc_new(heap, &self.proof.as_ref().unwrap().path, gas)?, + block_hash: asc_new(heap, self.block_hash.as_ref().unwrap(), gas)?, + id: asc_new(heap, self.id.as_ref().unwrap(), gas)?, + logs: asc_new(heap, &outcome.logs, gas)?, + receipt_ids: asc_new(heap, &outcome.receipt_ids, gas)?, + gas_burnt: outcome.gas_burnt, + tokens_burnt: asc_new(heap, outcome.tokens_burnt.as_ref().unwrap(), gas)?, + executor_id: asc_new(heap, &outcome.executor_id, gas)?, + status: asc_new(heap, outcome.status.as_ref().unwrap(), gas)?, + }) + } +} + +impl ToAscObj for codec::execution_outcome::Status { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let (kind, payload) = match self { + codec::execution_outcome::Status::SuccessValue(value) => { + let bytes = &value.value; + + ( + AscSuccessStatusKind::Value, + asc_new(heap, bytes.as_slice(), gas)?.to_payload(), + ) + } + codec::execution_outcome::Status::SuccessReceiptId(receipt_id) => ( + AscSuccessStatusKind::ReceiptId, + asc_new(heap, receipt_id.id.as_ref().unwrap(), gas)?.to_payload(), + ), + codec::execution_outcome::Status::Failure(_) => { + return Err(DeterministicHostError::from(anyhow!( + "Failure execution status are not allowed" + ))); + } + codec::execution_outcome::Status::Unknown(_) => { + return Err(DeterministicHostError::from(anyhow!( + "Unknown execution status are not allowed" + ))); + } + }; + + Ok(AscSuccessStatusEnum(AscEnum { + _padding: 0, + kind, + payload: EnumPayload(payload), + })) + } +} + +impl ToAscObj for codec::MerklePathItem { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscMerklePathItem { + hash: asc_new(heap, self.hash.as_ref().unwrap(), gas)?, + direction: match self.direction { + 0 => AscDirection::Left, + 1 => AscDirection::Right, + x => { + return Err(DeterministicHostError::from(anyhow!( + "Invalid direction value {}", + x + ))) + } + }, + }) + } +} + +impl ToAscObj for Vec { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); + let content = content?; + Ok(AscMerklePathItemArray(Array::new(&*content, heap, gas)?)) + } +} + +impl ToAscObj for codec::Signature { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscSignature { + kind: match self.r#type { + 0 => 0, + 1 => 1, + value => { + return Err(DeterministicHostError::from(anyhow!( + "Invalid signature type {}", + value, + ))) + } + }, + bytes: asc_new(heap, self.bytes.as_slice(), gas)?, + }) + } +} + +impl ToAscObj for Vec { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); + let content = content?; + Ok(AscSignatureArray(Array::new(&*content, heap, gas)?)) + } +} + +impl ToAscObj for codec::PublicKey { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscPublicKey { + kind: match self.r#type { + 0 => 0, + 1 => 1, + value => { + return Err(DeterministicHostError::from(anyhow!( + "Invalid public key type {}", + value, + ))) + } + }, + bytes: asc_new(heap, self.bytes.as_slice(), gas)?, + }) + } +} + +impl ToAscObj for codec::ValidatorStake { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscValidatorStake { + account_id: asc_new(heap, &self.account_id, gas)?, + public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas)?, + stake: asc_new(heap, self.stake.as_ref().unwrap(), gas)?, + }) + } +} + +impl ToAscObj for Vec { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); + let content = content?; + Ok(AscValidatorStakeArray(Array::new(&*content, heap, gas)?)) + } +} + +impl ToAscObj for codec::SlashedValidator { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscSlashedValidator { + account_id: asc_new(heap, &self.account_id, gas)?, + is_double_sign: self.is_double_sign, + }) + } +} + +impl ToAscObj for Vec { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); + let content = content?; + Ok(AscSlashedValidatorArray(Array::new(&*content, heap, gas)?)) + } +} + +impl ToAscObj for codec::CryptoHash { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + self.bytes.to_asc_obj(heap, gas) + } +} + +impl ToAscObj for Vec { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); + let content = content?; + Ok(AscCryptoHashArray(Array::new(&*content, heap, gas)?)) + } +} + +impl ToAscObj for codec::BigInt { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + // Bytes are reversed to align with BigInt bytes endianess + let reversed: Vec = self.bytes.iter().rev().map(|x| *x).collect(); + + reversed.to_asc_obj(heap, gas) + } +} diff --git a/chain/near/src/runtime/generated.rs b/chain/near/src/runtime/generated.rs new file mode 100644 index 0000000..153eb8b --- /dev/null +++ b/chain/near/src/runtime/generated.rs @@ -0,0 +1,620 @@ +use graph::runtime::{ + AscIndexId, AscPtr, AscType, AscValue, DeterministicHostError, IndexForAscTypeId, +}; +use graph::semver::Version; +use graph_runtime_derive::AscType; +use graph_runtime_wasm::asc_abi::class::{Array, AscBigInt, AscEnum, AscString, Uint8Array}; + +pub(crate) type AscCryptoHash = Uint8Array; +pub(crate) type AscAccountId = AscString; +pub(crate) type AscBlockHeight = u64; +pub(crate) type AscBalance = AscBigInt; +pub(crate) type AscGas = u64; +pub(crate) type AscShardId = u64; +pub(crate) type AscNumBlocks = u64; +pub(crate) type AscProtocolVersion = u32; + +pub struct AscDataReceiverArray(pub(crate) Array>); + +impl AscType for AscDataReceiverArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl AscIndexId for AscDataReceiverArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearArrayDataReceiver; +} + +pub struct AscCryptoHashArray(pub(crate) Array>); + +impl AscType for AscCryptoHashArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl AscIndexId for AscCryptoHashArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearArrayCryptoHash; +} + +pub struct AscActionEnumArray(pub(crate) Array>); + +impl AscType for AscActionEnumArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl AscIndexId for AscActionEnumArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearArrayActionEnum; +} + +pub struct AscMerklePathItemArray(pub(crate) Array>); + +impl AscType for AscMerklePathItemArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl AscIndexId for AscMerklePathItemArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearArrayMerklePathItem; +} + +pub struct AscValidatorStakeArray(pub(crate) Array>); + +impl AscType for AscValidatorStakeArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl AscIndexId for AscValidatorStakeArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearArrayValidatorStake; +} + +pub struct AscSlashedValidatorArray(pub(crate) Array>); + +impl AscType for AscSlashedValidatorArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl AscIndexId for AscSlashedValidatorArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearArraySlashedValidator; +} + +pub struct AscSignatureArray(pub(crate) Array>); + +impl AscType for AscSignatureArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl AscIndexId for AscSignatureArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearArraySignature; +} + +pub struct AscChunkHeaderArray(pub(crate) Array>); + +impl AscType for AscChunkHeaderArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl AscIndexId for AscChunkHeaderArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearArrayChunkHeader; +} + +pub struct AscAccessKeyPermissionEnum(pub(crate) AscEnum); + +impl AscType for AscAccessKeyPermissionEnum { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(AscEnum::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl AscIndexId for AscAccessKeyPermissionEnum { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearAccessKeyPermissionEnum; +} + +pub struct AscActionEnum(pub(crate) AscEnum); + +impl AscType for AscActionEnum { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(AscEnum::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl AscIndexId for AscActionEnum { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearActionEnum; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscPublicKey { + pub kind: i32, + pub bytes: AscPtr, +} + +impl AscIndexId for AscPublicKey { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearPublicKey; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscSignature { + pub kind: i32, + pub bytes: AscPtr, +} + +impl AscIndexId for AscSignature { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearSignature; +} + +#[repr(u32)] +#[derive(AscType, Copy, Clone)] +pub(crate) enum AscAccessKeyPermissionKind { + FunctionCall, + FullAccess, +} + +impl AscValue for AscAccessKeyPermissionKind {} + +impl Default for AscAccessKeyPermissionKind { + fn default() -> Self { + Self::FunctionCall + } +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscFunctionCallPermission { + pub allowance: AscPtr, + pub receiver_id: AscPtr, + pub method_names: AscPtr>>, +} + +impl AscIndexId for AscFunctionCallPermission { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearFunctionCallPermission; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscFullAccessPermission {} + +impl AscIndexId for AscFullAccessPermission { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearFullAccessPermission; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscAccessKey { + pub nonce: u64, + pub permission: AscPtr, + + // It seems that is impossible to correctly order fields in this struct + // so that Rust packs it tighly without padding. So we add 4 bytes of padding + // ourself. + // + // This is a bit problematic because AssemblyScript actually is ok with 12 bytes + // and is fully packed. Seems like a differences between alignment for `repr(C)` and + // AssemblyScript. + pub(crate) _padding: u32, +} + +impl AscIndexId for AscAccessKey { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearAccessKey; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscDataReceiver { + pub data_id: AscPtr, + pub receiver_id: AscPtr, +} + +impl AscIndexId for AscDataReceiver { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearDataReceiver; +} + +#[repr(u32)] +#[derive(AscType, Copy, Clone)] +pub(crate) enum AscActionKind { + CreateAccount, + DeployContract, + FunctionCall, + Transfer, + Stake, + AddKey, + DeleteKey, + DeleteAccount, +} + +impl AscValue for AscActionKind {} + +impl Default for AscActionKind { + fn default() -> Self { + Self::CreateAccount + } +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscCreateAccountAction {} + +impl AscIndexId for AscCreateAccountAction { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearCreateAccountAction; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscDeployContractAction { + pub code: AscPtr, +} + +impl AscIndexId for AscDeployContractAction { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearDeployContractAction; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscFunctionCallAction { + pub method_name: AscPtr, + pub args: AscPtr, + pub gas: u64, + pub deposit: AscPtr, + + // It seems that is impossible to correctly order fields in this struct + // so that Rust packs it tighly without padding. So we add 4 bytes of padding + // ourself. + // + // This is a bit problematic because AssemblyScript actually is ok with 20 bytes + // and is fully packed. Seems like a differences between alignment for `repr(C)` and + // AssemblyScript. + pub(crate) _padding: u32, +} + +impl AscIndexId for AscFunctionCallAction { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearFunctionCallAction; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscTransferAction { + pub deposit: AscPtr, +} + +impl AscIndexId for AscTransferAction { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearTransferAction; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscStakeAction { + pub stake: AscPtr, + pub public_key: AscPtr, +} + +impl AscIndexId for AscStakeAction { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearStakeAction; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscAddKeyAction { + pub public_key: AscPtr, + pub access_key: AscPtr, +} + +impl AscIndexId for AscAddKeyAction { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearAddKeyAction; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscDeleteKeyAction { + pub public_key: AscPtr, +} + +impl AscIndexId for AscDeleteKeyAction { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearDeleteKeyAction; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscDeleteAccountAction { + pub beneficiary_id: AscPtr, +} + +impl AscIndexId for AscDeleteAccountAction { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearDeleteAccountAction; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscActionReceipt { + pub predecessor_id: AscPtr, + pub receiver_id: AscPtr, + pub id: AscPtr, + pub signer_id: AscPtr, + pub signer_public_key: AscPtr, + pub gas_price: AscPtr, + pub output_data_receivers: AscPtr, + pub input_data_ids: AscPtr, + pub actions: AscPtr, +} + +impl AscIndexId for AscActionReceipt { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearActionReceipt; +} + +#[repr(u32)] +#[derive(AscType, Copy, Clone)] +pub(crate) enum AscSuccessStatusKind { + Value, + ReceiptId, +} + +impl AscValue for AscSuccessStatusKind {} + +impl Default for AscSuccessStatusKind { + fn default() -> Self { + Self::Value + } +} + +pub struct AscSuccessStatusEnum(pub(crate) AscEnum); + +impl AscType for AscSuccessStatusEnum { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(Self(AscEnum::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl AscIndexId for AscSuccessStatusEnum { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearSuccessStatusEnum; +} + +#[repr(u32)] +#[derive(AscType, Copy, Clone)] +pub(crate) enum AscDirection { + Left, + Right, +} + +impl AscValue for AscDirection {} + +impl Default for AscDirection { + fn default() -> Self { + Self::Left + } +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscMerklePathItem { + pub hash: AscPtr, + pub direction: AscDirection, +} + +impl AscIndexId for AscMerklePathItem { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearMerklePathItem; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscExecutionOutcome { + pub gas_burnt: u64, + pub proof: AscPtr, + pub block_hash: AscPtr, + pub id: AscPtr, + pub logs: AscPtr>>, + pub receipt_ids: AscPtr, + pub tokens_burnt: AscPtr, + pub executor_id: AscPtr, + pub status: AscPtr, +} + +impl AscIndexId for AscExecutionOutcome { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearExecutionOutcome; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscSlashedValidator { + pub account_id: AscPtr, + pub is_double_sign: bool, +} + +impl AscIndexId for AscSlashedValidator { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearSlashedValidator; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscBlockHeader { + pub height: AscBlockHeight, + pub prev_height: AscBlockHeight, + pub block_ordinal: AscNumBlocks, + pub epoch_id: AscPtr, + pub next_epoch_id: AscPtr, + pub chunks_included: u64, + pub hash: AscPtr, + pub prev_hash: AscPtr, + pub timestamp_nanosec: u64, + pub prev_state_root: AscPtr, + pub chunk_receipts_root: AscPtr, + pub chunk_headers_root: AscPtr, + pub chunk_tx_root: AscPtr, + pub outcome_root: AscPtr, + pub challenges_root: AscPtr, + pub random_value: AscPtr, + pub validator_proposals: AscPtr, + pub chunk_mask: AscPtr>, + pub gas_price: AscPtr, + pub total_supply: AscPtr, + pub challenges_result: AscPtr, + pub last_final_block: AscPtr, + pub last_ds_final_block: AscPtr, + pub next_bp_hash: AscPtr, + pub block_merkle_root: AscPtr, + pub epoch_sync_data_hash: AscPtr, + pub approvals: AscPtr, + pub signature: AscPtr, + pub latest_protocol_version: AscProtocolVersion, +} + +impl AscIndexId for AscBlockHeader { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearBlockHeader; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscValidatorStake { + pub account_id: AscPtr, + pub public_key: AscPtr, + pub stake: AscPtr, +} + +impl AscIndexId for AscValidatorStake { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearValidatorStake; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscChunkHeader { + pub encoded_length: u64, + pub gas_used: AscGas, + pub gas_limit: AscGas, + pub shard_id: AscShardId, + pub height_created: AscBlockHeight, + pub height_included: AscBlockHeight, + pub chunk_hash: AscPtr, + pub signature: AscPtr, + pub prev_block_hash: AscPtr, + pub prev_state_root: AscPtr, + pub encoded_merkle_root: AscPtr, + pub balance_burnt: AscPtr, + pub outgoing_receipts_root: AscPtr, + pub tx_root: AscPtr, + pub validator_proposals: AscPtr, + + // It seems that is impossible to correctly order fields in this struct + // so that Rust packs it tighly without padding. So we add 4 bytes of padding + // ourself. + // + // This is a bit problematic because AssemblyScript actually is ok with 84 bytes + // and is fully packed. Seems like a differences between alignment for `repr(C)` and + // AssemblyScript. + pub(crate) _padding: u32, +} + +impl AscIndexId for AscChunkHeader { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearChunkHeader; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscBlock { + pub author: AscPtr, + pub header: AscPtr, + pub chunks: AscPtr, +} + +impl AscIndexId for AscBlock { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearBlock; +} + +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscReceiptWithOutcome { + pub outcome: AscPtr, + pub receipt: AscPtr, + pub block: AscPtr, +} + +impl AscIndexId for AscReceiptWithOutcome { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::NearReceiptWithOutcome; +} diff --git a/chain/near/src/runtime/mod.rs b/chain/near/src/runtime/mod.rs new file mode 100644 index 0000000..f44391c --- /dev/null +++ b/chain/near/src/runtime/mod.rs @@ -0,0 +1,6 @@ +pub use runtime_adapter::RuntimeAdapter; + +pub mod abi; +pub mod runtime_adapter; + +mod generated; diff --git a/chain/near/src/runtime/runtime_adapter.rs b/chain/near/src/runtime/runtime_adapter.rs new file mode 100644 index 0000000..c5fa9e1 --- /dev/null +++ b/chain/near/src/runtime/runtime_adapter.rs @@ -0,0 +1,11 @@ +use crate::{data_source::DataSource, Chain}; +use blockchain::HostFn; +use graph::{anyhow::Error, blockchain}; + +pub struct RuntimeAdapter {} + +impl blockchain::RuntimeAdapter for RuntimeAdapter { + fn host_fns(&self, _ds: &DataSource) -> Result, Error> { + Ok(vec![]) + } +} diff --git a/chain/near/src/trigger.rs b/chain/near/src/trigger.rs new file mode 100644 index 0000000..049cce5 --- /dev/null +++ b/chain/near/src/trigger.rs @@ -0,0 +1,501 @@ +use graph::blockchain::Block; +use graph::blockchain::TriggerData; +use graph::cheap_clone::CheapClone; +use graph::prelude::hex; +use graph::prelude::web3::types::H256; +use graph::prelude::BlockNumber; +use graph::runtime::{asc_new, gas::GasCounter, AscHeap, AscPtr, DeterministicHostError}; +use graph_runtime_wasm::module::ToAscPtr; +use std::{cmp::Ordering, sync::Arc}; + +use crate::codec; + +// Logging the block is too verbose, so this strips the block from the trigger for Debug. +impl std::fmt::Debug for NearTrigger { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + #[derive(Debug)] + pub enum MappingTriggerWithoutBlock<'a> { + Block, + + Receipt { + outcome: &'a codec::ExecutionOutcomeWithId, + receipt: &'a codec::Receipt, + }, + } + + let trigger_without_block = match self { + NearTrigger::Block(_) => MappingTriggerWithoutBlock::Block, + NearTrigger::Receipt(receipt) => MappingTriggerWithoutBlock::Receipt { + outcome: &receipt.outcome, + receipt: &receipt.receipt, + }, + }; + + write!(f, "{:?}", trigger_without_block) + } +} + +impl ToAscPtr for NearTrigger { + fn to_asc_ptr( + self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + Ok(match self { + NearTrigger::Block(block) => asc_new(heap, block.as_ref(), gas)?.erase(), + NearTrigger::Receipt(receipt) => asc_new(heap, receipt.as_ref(), gas)?.erase(), + }) + } +} + +#[derive(Clone)] +pub enum NearTrigger { + Block(Arc), + Receipt(Arc), +} + +impl CheapClone for NearTrigger { + fn cheap_clone(&self) -> NearTrigger { + match self { + NearTrigger::Block(block) => NearTrigger::Block(block.cheap_clone()), + NearTrigger::Receipt(receipt) => NearTrigger::Receipt(receipt.cheap_clone()), + } + } +} + +impl PartialEq for NearTrigger { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Block(a_ptr), Self::Block(b_ptr)) => a_ptr == b_ptr, + (Self::Receipt(a), Self::Receipt(b)) => a.receipt.receipt_id == b.receipt.receipt_id, + + (Self::Block(_), Self::Receipt(_)) | (Self::Receipt(_), Self::Block(_)) => false, + } + } +} + +impl Eq for NearTrigger {} + +impl NearTrigger { + pub fn block_number(&self) -> BlockNumber { + match self { + NearTrigger::Block(block) => block.number(), + NearTrigger::Receipt(receipt) => receipt.block.number(), + } + } + + pub fn block_hash(&self) -> H256 { + match self { + NearTrigger::Block(block) => block.ptr().hash_as_h256(), + NearTrigger::Receipt(receipt) => receipt.block.ptr().hash_as_h256(), + } + } +} + +impl Ord for NearTrigger { + fn cmp(&self, other: &Self) -> Ordering { + match (self, other) { + // Keep the order when comparing two block triggers + (Self::Block(..), Self::Block(..)) => Ordering::Equal, + + // Block triggers always come last + (Self::Block(..), _) => Ordering::Greater, + (_, Self::Block(..)) => Ordering::Less, + + // Execution outcomes have no intrinsic ordering information, so we keep the order in + // which they are included in the `receipt_execution_outcomes` field of `IndexerShard`. + (Self::Receipt(..), Self::Receipt(..)) => Ordering::Equal, + } + } +} + +impl PartialOrd for NearTrigger { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl TriggerData for NearTrigger { + fn error_context(&self) -> std::string::String { + match self { + NearTrigger::Block(..) => { + format!("Block #{} ({})", self.block_number(), self.block_hash()) + } + NearTrigger::Receipt(receipt) => { + format!( + "receipt id {}, block #{} ({})", + hex::encode(&receipt.receipt.receipt_id.as_ref().unwrap().bytes), + self.block_number(), + self.block_hash() + ) + } + } + } +} + +pub struct ReceiptWithOutcome { + // REVIEW: Do we want to actually also have those two below behind an `Arc` wrapper? + pub outcome: codec::ExecutionOutcomeWithId, + pub receipt: codec::Receipt, + pub block: Arc, +} + +#[cfg(test)] +mod tests { + use std::convert::TryFrom; + + use super::*; + + use graph::{ + anyhow::anyhow, + data::subgraph::API_VERSION_0_0_5, + prelude::{hex, BigInt}, + runtime::gas::GasCounter, + util::mem::init_slice, + }; + + #[test] + fn block_trigger_to_asc_ptr() { + let mut heap = BytesHeap::new(API_VERSION_0_0_5); + let trigger = NearTrigger::Block(Arc::new(block())); + + let result = trigger.to_asc_ptr(&mut heap, &GasCounter::default()); + assert!(result.is_ok()); + } + + #[test] + fn receipt_trigger_to_asc_ptr() { + let mut heap = BytesHeap::new(API_VERSION_0_0_5); + let trigger = NearTrigger::Receipt(Arc::new(ReceiptWithOutcome { + block: Arc::new(block()), + outcome: execution_outcome_with_id().unwrap(), + receipt: receipt().unwrap(), + })); + + let result = trigger.to_asc_ptr(&mut heap, &GasCounter::default()); + assert!(result.is_ok()); + } + + fn block() -> codec::Block { + codec::Block { + author: "test".to_string(), + header: Some(codec::BlockHeader { + height: 2, + prev_height: 1, + epoch_id: hash("01"), + next_epoch_id: hash("02"), + hash: hash("01"), + prev_hash: hash("00"), + prev_state_root: hash("bb00010203"), + chunk_receipts_root: hash("bb00010203"), + chunk_headers_root: hash("bb00010203"), + chunk_tx_root: hash("bb00010203"), + outcome_root: hash("cc00010203"), + chunks_included: 1, + challenges_root: hash("aa"), + timestamp: 100, + timestamp_nanosec: 0, + random_value: hash("010203"), + validator_proposals: vec![], + chunk_mask: vec![], + gas_price: big_int(10), + block_ordinal: 0, + total_supply: big_int(1_000), + challenges_result: vec![], + last_final_block: hash("00"), + last_final_block_height: 0, + last_ds_final_block: hash("00"), + last_ds_final_block_height: 0, + next_bp_hash: hash("bb"), + block_merkle_root: hash("aa"), + epoch_sync_data_hash: vec![0x00, 0x01], + approvals: vec![], + signature: signature("00"), + latest_protocol_version: 0, + }), + chunk_headers: vec![chunk_header().unwrap()], + shards: vec![codec::IndexerShard { + shard_id: 0, + chunk: Some(codec::IndexerChunk { + author: "near".to_string(), + header: chunk_header(), + transactions: vec![codec::IndexerTransactionWithOutcome { + transaction: Some(codec::SignedTransaction { + signer_id: "signer".to_string(), + public_key: public_key("aabb"), + nonce: 1, + receiver_id: "receiver".to_string(), + actions: vec![], + signature: signature("ff"), + hash: hash("bb"), + }), + outcome: Some(codec::IndexerExecutionOutcomeWithOptionalReceipt { + execution_outcome: execution_outcome_with_id(), + receipt: receipt(), + }), + }], + receipts: vec![receipt().unwrap()], + }), + receipt_execution_outcomes: vec![codec::IndexerExecutionOutcomeWithReceipt { + execution_outcome: execution_outcome_with_id(), + receipt: receipt(), + }], + }], + state_changes: vec![], + } + } + + fn receipt() -> Option { + Some(codec::Receipt { + predecessor_id: "genesis.near".to_string(), + receiver_id: "near".to_string(), + receipt_id: hash("dead"), + receipt: Some(codec::receipt::Receipt::Action(codec::ReceiptAction { + signer_id: "near".to_string(), + signer_public_key: public_key("aa"), + gas_price: big_int(2), + output_data_receivers: vec![], + input_data_ids: vec![], + actions: vec![ + codec::Action { + action: Some(codec::action::Action::CreateAccount( + codec::CreateAccountAction {}, + )), + }, + codec::Action { + action: Some(codec::action::Action::DeployContract( + codec::DeployContractAction { + code: vec![0x01, 0x02], + }, + )), + }, + codec::Action { + action: Some(codec::action::Action::FunctionCall( + codec::FunctionCallAction { + method_name: "func".to_string(), + args: vec![0x01, 0x02], + gas: 1000, + deposit: big_int(100), + }, + )), + }, + codec::Action { + action: Some(codec::action::Action::Transfer(codec::TransferAction { + deposit: big_int(100), + })), + }, + codec::Action { + action: Some(codec::action::Action::Stake(codec::StakeAction { + stake: big_int(100), + public_key: public_key("aa"), + })), + }, + codec::Action { + action: Some(codec::action::Action::AddKey(codec::AddKeyAction { + public_key: public_key("aa"), + access_key: Some(codec::AccessKey { + nonce: 1, + permission: Some(codec::AccessKeyPermission { + permission: Some( + codec::access_key_permission::Permission::FunctionCall( + codec::FunctionCallPermission { + // allowance can be None, so let's test this out here + allowance: None, + receiver_id: "receiver".to_string(), + method_names: vec!["sayGm".to_string()], + }, + ), + ), + }), + }), + })), + }, + codec::Action { + action: Some(codec::action::Action::AddKey(codec::AddKeyAction { + public_key: public_key("aa"), + access_key: Some(codec::AccessKey { + nonce: 1, + permission: Some(codec::AccessKeyPermission { + permission: Some( + codec::access_key_permission::Permission::FullAccess( + codec::FullAccessPermission {}, + ), + ), + }), + }), + })), + }, + codec::Action { + action: Some(codec::action::Action::DeleteKey(codec::DeleteKeyAction { + public_key: public_key("aa"), + })), + }, + codec::Action { + action: Some(codec::action::Action::DeleteAccount( + codec::DeleteAccountAction { + beneficiary_id: "suicided.near".to_string(), + }, + )), + }, + ], + })), + }) + } + + fn chunk_header() -> Option { + Some(codec::ChunkHeader { + chunk_hash: vec![0x00], + prev_block_hash: vec![0x01], + outcome_root: vec![0x02], + prev_state_root: vec![0x03], + encoded_merkle_root: vec![0x04], + encoded_length: 1, + height_created: 2, + height_included: 3, + shard_id: 4, + gas_used: 5, + gas_limit: 6, + validator_reward: big_int(7), + balance_burnt: big_int(7), + outgoing_receipts_root: vec![0x07], + tx_root: vec![0x08], + validator_proposals: vec![codec::ValidatorStake { + account_id: "account".to_string(), + public_key: public_key("aa"), + stake: big_int(10), + }], + signature: signature("ff"), + }) + } + + fn execution_outcome_with_id() -> Option { + Some(codec::ExecutionOutcomeWithId { + proof: Some(codec::MerklePath { path: vec![] }), + block_hash: hash("aa"), + id: hash("beef"), + outcome: execution_outcome(), + }) + } + + fn execution_outcome() -> Option { + Some(codec::ExecutionOutcome { + logs: vec!["string".to_string()], + receipt_ids: vec![], + gas_burnt: 1, + tokens_burnt: big_int(2), + executor_id: "near".to_string(), + metadata: 0, + status: Some(codec::execution_outcome::Status::SuccessValue( + codec::SuccessValueExecutionStatus { value: vec![0x00] }, + )), + }) + } + + fn big_int(input: u64) -> Option { + let value = + BigInt::try_from(input).expect(format!("Invalid BigInt value {}", input).as_ref()); + let bytes = value.to_signed_bytes_le(); + + Some(codec::BigInt { bytes }) + } + + fn hash(input: &str) -> Option { + Some(codec::CryptoHash { + bytes: hex::decode(input).expect(format!("Invalid hash value {}", input).as_ref()), + }) + } + + fn public_key(input: &str) -> Option { + Some(codec::PublicKey { + r#type: 0, + bytes: hex::decode(input).expect(format!("Invalid PublicKey value {}", input).as_ref()), + }) + } + + fn signature(input: &str) -> Option { + Some(codec::Signature { + r#type: 0, + bytes: hex::decode(input).expect(format!("Invalid Signature value {}", input).as_ref()), + }) + } + + struct BytesHeap { + api_version: graph::semver::Version, + memory: Vec, + } + + impl BytesHeap { + fn new(api_version: graph::semver::Version) -> Self { + Self { + api_version, + memory: vec![], + } + } + } + + impl AscHeap for BytesHeap { + fn raw_new( + &mut self, + bytes: &[u8], + _gas: &GasCounter, + ) -> Result { + self.memory.extend_from_slice(bytes); + Ok((self.memory.len() - bytes.len()) as u32) + } + + fn read_u32(&self, offset: u32, gas: &GasCounter) -> Result { + let mut data = [std::mem::MaybeUninit::::uninit(); 4]; + let init = self.read(offset, &mut data, gas)?; + Ok(u32::from_le_bytes(init.try_into().unwrap())) + } + + fn read<'a>( + &self, + offset: u32, + buffer: &'a mut [std::mem::MaybeUninit], + _gas: &GasCounter, + ) -> Result<&'a mut [u8], DeterministicHostError> { + let memory_byte_count = self.memory.len(); + if memory_byte_count == 0 { + return Err(DeterministicHostError::from(anyhow!( + "No memory is allocated" + ))); + } + + let start_offset = offset as usize; + let end_offset_exclusive = start_offset + buffer.len(); + + if start_offset >= memory_byte_count { + return Err(DeterministicHostError::from(anyhow!( + "Start offset {} is outside of allocated memory, max offset is {}", + start_offset, + memory_byte_count - 1 + ))); + } + + if end_offset_exclusive > memory_byte_count { + return Err(DeterministicHostError::from(anyhow!( + "End of offset {} is outside of allocated memory, max offset is {}", + end_offset_exclusive, + memory_byte_count - 1 + ))); + } + + let src = &self.memory[start_offset..end_offset_exclusive]; + + Ok(init_slice(src, buffer)) + } + + fn api_version(&self) -> graph::semver::Version { + self.api_version.clone() + } + + fn asc_type_id( + &mut self, + type_id_index: graph::runtime::IndexForAscTypeId, + ) -> Result { + // Not totally clear what is the purpose of this method, why not a default implementation here? + Ok(type_id_index as u32) + } + } +} diff --git a/chain/substreams/Cargo.toml b/chain/substreams/Cargo.toml new file mode 100644 index 0000000..bc9bdbc --- /dev/null +++ b/chain/substreams/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "graph-chain-substreams" +version = "0.27.0" +edition = "2021" + +[build-dependencies] +tonic-build = { version = "0.7.2", features = ["prost"] } + +[dependencies] +async-stream = "0.3" +envconfig = "0.10.0" +futures = "0.1.21" +http = "0.2.4" +jsonrpc-core = "18.0.0" +graph = { path = "../../graph" } +graph-runtime-wasm = { path = "../../runtime/wasm" } +lazy_static = "1.2.0" +serde = "1.0" +prost = "0.10.4" +prost-types = "0.10.1" +dirs-next = "2.0" +anyhow = "1.0" +tiny-keccak = "1.5.0" +hex = "0.4.3" +semver = "1.0.12" + +itertools = "0.10.3" + +[dev-dependencies] +graph-core = { path = "../../core" } +tokio = { version = "1", features = ["full"]} diff --git a/chain/substreams/build.rs b/chain/substreams/build.rs new file mode 100644 index 0000000..8998174 --- /dev/null +++ b/chain/substreams/build.rs @@ -0,0 +1,7 @@ +fn main() { + println!("cargo:rerun-if-changed=proto"); + tonic_build::configure() + .out_dir("src/protobuf") + .compile(&["codec.proto"], &["proto"]) + .expect("Failed to compile Substreams entity proto(s)"); +} diff --git a/chain/substreams/examples/README.md b/chain/substreams/examples/README.md new file mode 100644 index 0000000..afd1882 --- /dev/null +++ b/chain/substreams/examples/README.md @@ -0,0 +1,13 @@ +## Substreams example + +1. Set environmental variables +```bash +$> export SUBSTREAMS_API_TOKEN=your_sf_token +$> export SUBSTREAMS_ENDPOINT=your_sf_endpoint # you can also not define this one and use the default specified endpoint +$> export SUBSTREAMS_PACKAGE=path_to_your_spkg +``` + +2. Run `substreams` example +```bash +cargo run -p graph-chain-substreams --example substreams [module_name] # for graph entities run `graph_out` +``` diff --git a/chain/substreams/examples/substreams.rs b/chain/substreams/examples/substreams.rs new file mode 100644 index 0000000..bc7af4a --- /dev/null +++ b/chain/substreams/examples/substreams.rs @@ -0,0 +1,107 @@ +use anyhow::{format_err, Context, Error}; +use graph::blockchain::block_stream::BlockStreamEvent; +use graph::blockchain::substreams_block_stream::SubstreamsBlockStream; +use graph::prelude::{info, tokio, DeploymentHash, Registry}; +use graph::tokio_stream::StreamExt; +use graph::{ + env::env_var, + firehose::FirehoseEndpoint, + log::logger, + substreams::{self}, +}; +use graph_chain_substreams::mapper::Mapper; +use graph_core::MetricsRegistry; +use prost::Message; +use std::env; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Error> { + let module_name = env::args().nth(1).unwrap(); + + let token_env = env_var("SUBSTREAMS_API_TOKEN", "".to_string()); + let mut token: Option = None; + if token_env.len() > 0 { + token = Some(token_env); + } + + let endpoint = env_var( + "SUBSTREAMS_ENDPOINT", + "https://api-dev.streamingfast.io".to_string(), + ); + + let package_file = env_var("SUBSTREAMS_PACKAGE", "".to_string()); + if package_file == "" { + panic!("Environment variable SUBSTREAMS_PACKAGE must be set"); + } + + let package = read_package(&package_file)?; + + let logger = logger(true); + // Set up Prometheus registry + let prometheus_registry = Arc::new(Registry::new()); + let metrics_registry = Arc::new(MetricsRegistry::new( + logger.clone(), + prometheus_registry.clone(), + )); + + let firehose = Arc::new(FirehoseEndpoint::new( + "substreams", + &endpoint, + token, + false, + false, + 1, + )); + + let mut stream: SubstreamsBlockStream = + SubstreamsBlockStream::new( + DeploymentHash::new("substreams".to_string()).unwrap(), + firehose.clone(), + None, + None, + Arc::new(Mapper {}), + package.modules.clone(), + module_name.to_string(), + vec![12369621], + vec![], + logger.clone(), + metrics_registry, + ); + + loop { + match stream.next().await { + None => { + break; + } + Some(event) => match event { + Err(_) => {} + Ok(block_stream_event) => match block_stream_event { + BlockStreamEvent::Revert(_, _) => {} + BlockStreamEvent::ProcessBlock(block_with_trigger, _) => { + let changes = block_with_trigger.block; + for change in changes.entity_changes { + info!(&logger, "----- Entity -----"); + info!( + &logger, + "name: {} operation: {}", change.entity, change.operation + ); + for field in change.fields { + info!(&logger, "field: {}, type: {}", field.name, field.value_type); + info!(&logger, "new value: {}", hex::encode(field.new_value)); + info!(&logger, "old value: {}", hex::encode(field.old_value)); + } + } + } + }, + }, + } + } + + Ok(()) +} + +fn read_package(file: &str) -> Result { + let content = std::fs::read(file).context(format_err!("read package {}", file))?; + substreams::Package::decode(content.as_ref()).context("decode command") +} diff --git a/chain/substreams/proto/codec.proto b/chain/substreams/proto/codec.proto new file mode 100644 index 0000000..4320cb7 --- /dev/null +++ b/chain/substreams/proto/codec.proto @@ -0,0 +1,42 @@ +syntax = "proto3"; + +package substreams.entity.v1; + +message EntitiesChanges { + bytes block_id = 1; + uint64 block_number = 2; + bytes prev_block_id = 3; + uint64 prev_block_number = 4; + repeated EntityChange entityChanges = 5; +} + +message EntityChange { + string entity = 1; + bytes id = 2; + uint64 ordinal = 3; + enum Operation { + UNSET = 0; // Protobuf default should not be used, this is used so that the consume can ensure that the value was actually specified + CREATE = 1; + UPDATE = 2; + DELETE = 3; + } + Operation operation = 4; + repeated Field fields = 5; +} + +message Field { + string name = 1; + enum Type { + UNSET = 0; // Protobuf default should not be used, this is used so that the consume can ensure that the value was actually specified + BIGDECIMAL = 1; + BIGINT = 2; + INT = 3; // int32 + BYTES = 4; + STRING = 5; + } + Type value_type = 2; + bytes new_value = 3; + bool new_value_null = 4; + bytes old_value = 5; + bool old_value_null = 6; +} diff --git a/chain/substreams/src/block_stream.rs b/chain/substreams/src/block_stream.rs new file mode 100644 index 0000000..ef8409a --- /dev/null +++ b/chain/substreams/src/block_stream.rs @@ -0,0 +1,80 @@ +use anyhow::Result; +use std::sync::Arc; + +use graph::{ + blockchain::{ + block_stream::{ + BlockStream, BlockStreamBuilder as BlockStreamBuilderTrait, FirehoseCursor, + }, + substreams_block_stream::SubstreamsBlockStream, + }, + components::store::DeploymentLocator, + data::subgraph::UnifiedMappingApiVersion, + prelude::{async_trait, BlockNumber, BlockPtr}, + slog::o, +}; + +use crate::{mapper::Mapper, Chain, TriggerFilter}; + +pub struct BlockStreamBuilder {} + +impl BlockStreamBuilder { + pub fn new() -> Self { + Self {} + } +} + +#[async_trait] +/// Substreams doesn't actually use Firehose, the configuration for firehose and the grpc substream +/// is very similar, so we can re-use the configuration and the builder for it. +/// This is probably something to improve but for now it works. +impl BlockStreamBuilderTrait for BlockStreamBuilder { + async fn build_firehose( + &self, + chain: &Chain, + deployment: DeploymentLocator, + block_cursor: FirehoseCursor, + _start_blocks: Vec, + _subgraph_current_block: Option, + filter: Arc, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>> { + let firehose_endpoint = match chain.endpoints.random() { + Some(e) => e.clone(), + None => return Err(anyhow::format_err!("no firehose endpoint available")), + }; + + let mapper = Arc::new(Mapper {}); + + let logger = chain + .logger_factory + .subgraph_logger(&deployment) + .new(o!("component" => "SubstreamsBlockStream")); + + Ok(Box::new(SubstreamsBlockStream::new( + deployment.hash, + firehose_endpoint, + None, + block_cursor.as_ref().clone(), + mapper, + filter.modules.clone(), + filter.module_name.clone(), + filter.start_block.map(|x| vec![x]).unwrap_or(vec![]), + vec![], + logger, + chain.metrics_registry.clone(), + ))) + } + + async fn build_polling( + &self, + _chain: Arc, + _deployment: DeploymentLocator, + _start_blocks: Vec, + _subgraph_current_block: Option, + _filter: Arc, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>> { + unimplemented!("polling block stream is not support for substreams") + } +} diff --git a/chain/substreams/src/chain.rs b/chain/substreams/src/chain.rs new file mode 100644 index 0000000..68a7389 --- /dev/null +++ b/chain/substreams/src/chain.rs @@ -0,0 +1,194 @@ +use crate::{data_source::*, Block, TriggerData, TriggerFilter, TriggersAdapter}; +use anyhow::Error; +use core::fmt; +use graph::firehose::FirehoseEndpoints; +use graph::prelude::{BlockHash, LoggerFactory, MetricsRegistry}; +use graph::{ + blockchain::{ + self, + block_stream::{BlockStream, BlockStreamBuilder, FirehoseCursor}, + BlockPtr, Blockchain, BlockchainKind, IngestorError, RuntimeAdapter as RuntimeAdapterTrait, + }, + components::store::DeploymentLocator, + data::subgraph::UnifiedMappingApiVersion, + impl_slog_value, + prelude::{async_trait, BlockNumber, ChainStore}, + slog::Logger, +}; +use std::{str::FromStr, sync::Arc}; + +impl blockchain::Block for Block { + fn ptr(&self) -> BlockPtr { + return BlockPtr { + hash: BlockHash(Box::from(self.block_id.clone())), + number: self.block_number as i32, + }; + } + + fn parent_ptr(&self) -> Option { + Some(BlockPtr { + hash: BlockHash(Box::from(self.prev_block_id.clone())), + number: self.prev_block_number as i32, + }) + } +} + +pub struct Chain { + chain_store: Arc, + block_stream_builder: Arc>, + + pub(crate) logger_factory: LoggerFactory, + pub(crate) endpoints: FirehoseEndpoints, + pub(crate) metrics_registry: Arc, +} + +impl Chain { + pub fn new( + logger_factory: LoggerFactory, + endpoints: FirehoseEndpoints, + metrics_registry: Arc, + chain_store: Arc, + block_stream_builder: Arc>, + ) -> Self { + Self { + logger_factory, + endpoints, + metrics_registry, + chain_store, + block_stream_builder, + } + } +} + +impl std::fmt::Debug for Chain { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "chain: substreams") + } +} + +#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Ord, Eq)] +pub struct NodeCapabilities {} + +impl FromStr for NodeCapabilities { + type Err = Error; + + fn from_str(_s: &str) -> Result { + Ok(NodeCapabilities {}) + } +} + +impl fmt::Display for NodeCapabilities { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("substream") + } +} + +impl_slog_value!(NodeCapabilities, "{}"); + +impl graph::blockchain::NodeCapabilities for NodeCapabilities { + fn from_data_sources(_data_sources: &[DataSource]) -> Self { + NodeCapabilities {} + } +} + +#[async_trait] +impl Blockchain for Chain { + const KIND: BlockchainKind = BlockchainKind::Substreams; + + type Block = Block; + type DataSource = DataSource; + type UnresolvedDataSource = UnresolvedDataSource; + + type DataSourceTemplate = NoopDataSourceTemplate; + type UnresolvedDataSourceTemplate = NoopDataSourceTemplate; + + /// Trigger data as parsed from the triggers adapter. + type TriggerData = TriggerData; + + /// Decoded trigger ready to be processed by the mapping. + /// New implementations should have this be the same as `TriggerData`. + type MappingTrigger = TriggerData; + + /// Trigger filter used as input to the triggers adapter. + type TriggerFilter = TriggerFilter; + + type NodeCapabilities = NodeCapabilities; + + fn triggers_adapter( + &self, + _log: &DeploymentLocator, + _capabilities: &Self::NodeCapabilities, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + Ok(Arc::new(TriggersAdapter {})) + } + + async fn new_firehose_block_stream( + &self, + deployment: DeploymentLocator, + block_cursor: FirehoseCursor, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + self.block_stream_builder + .build_firehose( + self, + deployment, + block_cursor, + start_blocks, + subgraph_current_block, + filter, + unified_api_version, + ) + .await + } + + async fn new_polling_block_stream( + &self, + _deployment: DeploymentLocator, + _start_blocks: Vec, + _subgraph_current_block: Option, + _filter: Arc, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error> { + unimplemented!("this should never be called for substreams") + } + + fn chain_store(&self) -> Arc { + self.chain_store.clone() + } + + async fn block_pointer_from_number( + &self, + _logger: &Logger, + number: BlockNumber, + ) -> Result { + // This is the same thing TriggersAdapter does, not sure if it's going to work but + // we also don't yet have a good way of getting this value until we sort out the + // chain store. + // TODO(filipe): Fix this once the chain_store is correctly setup for substreams. + Ok(BlockPtr { + hash: BlockHash::from(vec![0xff; 32]), + number, + }) + } + fn runtime_adapter(&self) -> Arc> { + Arc::new(RuntimeAdapter {}) + } + + fn is_firehose_supported(&self) -> bool { + true + } +} + +pub struct RuntimeAdapter {} +impl RuntimeAdapterTrait for RuntimeAdapter { + fn host_fns( + &self, + _ds: &::DataSource, + ) -> Result, Error> { + todo!() + } +} diff --git a/chain/substreams/src/codec.rs b/chain/substreams/src/codec.rs new file mode 100644 index 0000000..31781ba --- /dev/null +++ b/chain/substreams/src/codec.rs @@ -0,0 +1,5 @@ +#[rustfmt::skip] +#[path = "protobuf/substreams.entity.v1.rs"] +mod pbsubstreamsentity; + +pub use pbsubstreamsentity::*; diff --git a/chain/substreams/src/data_source.rs b/chain/substreams/src/data_source.rs new file mode 100644 index 0000000..d4b7370 --- /dev/null +++ b/chain/substreams/src/data_source.rs @@ -0,0 +1,409 @@ +use std::sync::Arc; + +use anyhow::{anyhow, Error}; +use graph::{ + blockchain, + cheap_clone::CheapClone, + components::link_resolver::LinkResolver, + prelude::{async_trait, BlockNumber, DataSourceTemplateInfo, Link}, + slog::Logger, +}; + +use prost::Message; +use serde::Deserialize; + +use crate::{chain::Chain, Block, TriggerData}; + +pub const SUBSTREAMS_KIND: &str = "substreams"; + +const DYNAMIC_DATA_SOURCE_ERROR: &str = "Substreams do not support dynamic data sources"; +const TEMPLATE_ERROR: &str = "Substreams do not support templates"; + +const ALLOWED_MAPPING_KIND: [&'static str; 1] = ["substreams/graph-entities"]; + +#[derive(Clone, Debug, PartialEq)] +/// Represents the DataSource portion of the manifest once it has been parsed +/// and the substream spkg has been downloaded + parsed. +pub struct DataSource { + pub kind: String, + pub network: Option, + pub name: String, + pub(crate) source: Source, + pub mapping: Mapping, + pub context: Arc>, + pub initial_block: Option, +} + +impl TryFrom> for DataSource { + type Error = anyhow::Error; + + fn try_from(_value: DataSourceTemplateInfo) -> Result { + Err(anyhow!("Substreams does not support templates")) + } +} + +impl blockchain::DataSource for DataSource { + fn address(&self) -> Option<&[u8]> { + None + } + + fn start_block(&self) -> BlockNumber { + self.initial_block.unwrap_or(0) + } + + fn name(&self) -> &str { + &self.name + } + + fn kind(&self) -> &str { + &self.kind + } + + fn network(&self) -> Option<&str> { + self.network.as_ref().map(|s| s.as_str()) + } + + fn context(&self) -> Arc> { + self.context.cheap_clone() + } + + fn creation_block(&self) -> Option { + None + } + + fn api_version(&self) -> semver::Version { + self.mapping.api_version.clone() + } + + // runtime is not needed for substreams, it will cause the host creation to be skipped. + fn runtime(&self) -> Option>> { + None + } + + // match_and_decode only seems to be used on the default trigger processor which substreams + // bypasses so it should be fine to leave it unimplemented. + fn match_and_decode( + &self, + _trigger: &TriggerData, + _block: &Arc, + _logger: &Logger, + ) -> Result>, Error> { + unimplemented!() + } + + fn is_duplicate_of(&self, _other: &Self) -> bool { + todo!() + } + + fn as_stored_dynamic_data_source(&self) -> graph::components::store::StoredDynamicDataSource { + unimplemented!("{}", DYNAMIC_DATA_SOURCE_ERROR) + } + + fn validate(&self) -> Vec { + let mut errs = vec![]; + + if &self.kind != SUBSTREAMS_KIND { + errs.push(anyhow!( + "data source has invalid `kind`, expected {} but found {}", + SUBSTREAMS_KIND, + self.kind + )) + } + + if self.name.is_empty() { + errs.push(anyhow!("name cannot be empty")); + } + + if !ALLOWED_MAPPING_KIND.contains(&self.mapping.kind.as_str()) { + errs.push(anyhow!( + "mapping kind has to be one of {:?}, found {}", + ALLOWED_MAPPING_KIND, + self.mapping.kind + )) + } + + errs + } + + fn from_stored_dynamic_data_source( + _template: &::DataSourceTemplate, + _stored: graph::components::store::StoredDynamicDataSource, + ) -> Result { + Err(anyhow!(DYNAMIC_DATA_SOURCE_ERROR)) + } +} + +#[derive(Clone, Debug, Default, PartialEq)] +/// Module name comes from the manifest, package is the parsed spkg file. +pub struct Source { + pub module_name: String, + pub package: graph::substreams::Package, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Mapping { + pub api_version: semver::Version, + pub kind: String, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +/// Raw representation of the data source for deserialization purposes. +pub struct UnresolvedDataSource { + pub kind: String, + pub network: Option, + pub name: String, + pub(crate) source: UnresolvedSource, + pub mapping: UnresolvedMapping, +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +/// Text api_version, before parsing and validation. +pub struct UnresolvedMapping { + pub api_version: String, + pub kind: String, +} + +#[async_trait] +impl blockchain::UnresolvedDataSource for UnresolvedDataSource { + async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + _manifest_idx: u32, + ) -> Result { + let content = resolver.cat(logger, &self.source.package.file).await?; + + let package = graph::substreams::Package::decode(content.as_ref())?; + + let initial_block: Option = match package.modules { + Some(ref modules) => modules.modules.iter().map(|x| x.initial_block).min(), + None => None, + }; + + let initial_block: Option = initial_block + .map_or(Ok(None), |x: u64| TryInto::::try_into(x).map(Some)) + .map_err(anyhow::Error::from)?; + + Ok(DataSource { + kind: SUBSTREAMS_KIND.into(), + network: self.network, + name: self.name, + source: Source { + module_name: self.source.package.module_name, + package, + }, + mapping: Mapping { + api_version: semver::Version::parse(&self.mapping.api_version)?, + kind: self.mapping.kind, + }, + context: Arc::new(None), + initial_block, + }) + } +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +/// Source is a part of the manifest and this is needed for parsing. +pub struct UnresolvedSource { + package: UnresolvedPackage, +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +/// The unresolved Package section of the manifest. +pub struct UnresolvedPackage { + pub module_name: String, + pub file: Link, +} + +#[derive(Debug, Clone, Default, Deserialize)] +/// This is necessary for the Blockchain trait associated types, substreams do not support +/// data source templates so this is a noop and is not expected to be called. +pub struct NoopDataSourceTemplate {} + +impl blockchain::DataSourceTemplate for NoopDataSourceTemplate { + fn name(&self) -> &str { + unimplemented!("{}", TEMPLATE_ERROR); + } + + fn api_version(&self) -> semver::Version { + unimplemented!("{}", TEMPLATE_ERROR); + } + + fn runtime(&self) -> Option>> { + unimplemented!("{}", TEMPLATE_ERROR); + } + + fn manifest_idx(&self) -> u32 { + todo!() + } +} + +#[async_trait] +impl blockchain::UnresolvedDataSourceTemplate for NoopDataSourceTemplate { + async fn resolve( + self, + _resolver: &Arc, + _logger: &Logger, + _manifest_idx: u32, + ) -> Result { + unimplemented!("{}", TEMPLATE_ERROR) + } +} + +#[cfg(test)] +mod test { + use std::{str::FromStr, sync::Arc}; + + use anyhow::Error; + use graph::{ + blockchain::{DataSource as _, UnresolvedDataSource as _}, + components::link_resolver::LinkResolver, + prelude::{async_trait, serde_yaml, JsonValueStream, Link}, + slog::{o, Discard, Logger}, + }; + + use crate::{DataSource, Mapping, UnresolvedDataSource, UnresolvedMapping, SUBSTREAMS_KIND}; + + const EMPTY_PACKAGE: graph::substreams::Package = graph::substreams::Package { + proto_files: vec![], + version: 0, + modules: None, + module_meta: vec![], + package_meta: vec![], + }; + + #[test] + fn parse_data_source() { + let ds: UnresolvedDataSource = serde_yaml::from_str(TEMPLATE_DATA_SOURCE).unwrap(); + let expected = UnresolvedDataSource { + kind: SUBSTREAMS_KIND.into(), + network: Some("mainnet".into()), + name: "Uniswap".into(), + source: crate::UnresolvedSource { + package: crate::UnresolvedPackage { + module_name: "output".into(), + file: Link { + link: "/ipfs/QmbHnhUFZa6qqqRyubUYhXntox1TCBxqryaBM1iNGqVJzT".into(), + }, + }, + }, + mapping: UnresolvedMapping { + api_version: "0.0.7".into(), + kind: "substreams/graph-entities".into(), + }, + }; + assert_eq!(ds, expected); + } + + #[tokio::test] + async fn data_source_conversion() { + let ds: UnresolvedDataSource = serde_yaml::from_str(TEMPLATE_DATA_SOURCE).unwrap(); + let link_resolver: Arc = Arc::new(NoopLinkResolver {}); + let logger = Logger::root(Discard, o!()); + let ds: DataSource = ds.resolve(&link_resolver, &logger, 0).await.unwrap(); + let expected = DataSource { + kind: SUBSTREAMS_KIND.into(), + network: Some("mainnet".into()), + name: "Uniswap".into(), + source: crate::Source { + module_name: "output".into(), + package: EMPTY_PACKAGE, + }, + mapping: Mapping { + api_version: semver::Version::from_str("0.0.7").unwrap(), + kind: "substreams/graph-entities".into(), + }, + context: Arc::new(None), + initial_block: None, + }; + assert_eq!(ds, expected); + } + + #[test] + fn data_source_validation() { + let mut ds = gen_data_source(); + assert_eq!(true, ds.validate().is_empty()); + + ds.network = None; + assert_eq!(true, ds.validate().is_empty()); + + ds.kind = "asdasd".into(); + ds.name = "".into(); + ds.mapping.kind = "asdasd".into(); + let errs: Vec = ds.validate().into_iter().map(|e| e.to_string()).collect(); + assert_eq!( + errs, + vec![ + "data source has invalid `kind`, expected substreams but found asdasd", + "name cannot be empty", + "mapping kind has to be one of [\"substreams/graph-entities\"], found asdasd" + ] + ); + } + + fn gen_data_source() -> DataSource { + DataSource { + kind: SUBSTREAMS_KIND.into(), + network: Some("mainnet".into()), + name: "Uniswap".into(), + source: crate::Source { + module_name: "".to_string(), + package: EMPTY_PACKAGE, + }, + mapping: Mapping { + api_version: semver::Version::from_str("0.0.7").unwrap(), + kind: "substreams/graph-entities".into(), + }, + context: Arc::new(None), + initial_block: None, + } + } + + const TEMPLATE_DATA_SOURCE: &str = r#" + kind: substreams + name: Uniswap + network: mainnet + source: + package: + moduleName: output + file: + /: /ipfs/QmbHnhUFZa6qqqRyubUYhXntox1TCBxqryaBM1iNGqVJzT + # This IPFs path would be generated from a local path at deploy time + mapping: + kind: substreams/graph-entities + apiVersion: 0.0.7 + "#; + + #[derive(Debug)] + struct NoopLinkResolver {} + + #[async_trait] + impl LinkResolver for NoopLinkResolver { + fn with_timeout(&self, _timeout: std::time::Duration) -> Box { + unimplemented!() + } + + fn with_retries(&self) -> Box { + unimplemented!() + } + + async fn cat(&self, _logger: &Logger, _link: &Link) -> Result, Error> { + Ok(vec![]) + } + + async fn get_block(&self, _logger: &Logger, _link: &Link) -> Result, Error> { + unimplemented!() + } + + async fn json_stream( + &self, + _logger: &Logger, + _link: &Link, + ) -> Result { + unimplemented!() + } + } +} diff --git a/chain/substreams/src/lib.rs b/chain/substreams/src/lib.rs new file mode 100644 index 0000000..68fa97c --- /dev/null +++ b/chain/substreams/src/lib.rs @@ -0,0 +1,16 @@ +mod block_stream; +mod chain; +mod codec; +mod data_source; +mod trigger; + +pub mod mapper; + +pub use block_stream::BlockStreamBuilder; +pub use chain::*; +pub use codec::EntitiesChanges as Block; +pub use data_source::*; +pub use trigger::*; + +pub use codec::field::Type as FieldType; +pub use codec::Field; diff --git a/chain/substreams/src/mapper.rs b/chain/substreams/src/mapper.rs new file mode 100644 index 0000000..48c61a8 --- /dev/null +++ b/chain/substreams/src/mapper.rs @@ -0,0 +1,77 @@ +use crate::{Block, Chain, TriggerData}; +use graph::blockchain::block_stream::SubstreamsError::{ + MultipleModuleOutputError, UnexpectedStoreDeltaOutput, +}; +use graph::blockchain::block_stream::{ + BlockStreamEvent, BlockWithTriggers, FirehoseCursor, SubstreamsError, SubstreamsMapper, +}; +use graph::prelude::{async_trait, BlockNumber, BlockPtr, Logger}; +use graph::substreams::module_output::Data; +use graph::substreams::{BlockScopedData, ForkStep}; +use prost::Message; + +pub struct Mapper {} + +#[async_trait] +impl SubstreamsMapper for Mapper { + async fn to_block_stream_event( + &self, + _logger: &Logger, + block_scoped_data: &BlockScopedData, + ) -> Result>, SubstreamsError> { + let step = ForkStep::from_i32(block_scoped_data.step).unwrap_or_else(|| { + panic!( + "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", + block_scoped_data.step + ) + }); + + if block_scoped_data.outputs.len() == 0 { + return Ok(None); + } + + if block_scoped_data.outputs.len() > 1 { + return Err(MultipleModuleOutputError()); + } + + //todo: handle step + let module_output = &block_scoped_data.outputs[0]; + let cursor = &block_scoped_data.cursor; + + match module_output.data.as_ref().unwrap() { + Data::MapOutput(msg) => { + let changes: Block = Message::decode(msg.value.as_slice()).unwrap(); + + use ForkStep::*; + match step { + StepIrreversible | StepNew => Ok(Some(BlockStreamEvent::ProcessBlock( + // Even though the trigger processor for substreams doesn't care about TriggerData + // there are a bunch of places in the runner that check if trigger data + // empty and skip processing if so. This will prolly breakdown + // close to head so we will need to improve things. + + // TODO(filipe): Fix once either trigger data can be empty + // or we move the changes into trigger data. + BlockWithTriggers::new(changes, vec![TriggerData {}]), + FirehoseCursor::from(cursor.clone()), + ))), + StepUndo => { + let parent_ptr = BlockPtr { + hash: changes.prev_block_id.clone().into(), + number: changes.prev_block_number as BlockNumber, + }; + + Ok(Some(BlockStreamEvent::Revert( + parent_ptr, + FirehoseCursor::from(cursor.clone()), + ))) + } + StepUnknown => { + panic!("unknown step should not happen in the Firehose response") + } + } + } + Data::StoreDeltas(_) => Err(UnexpectedStoreDeltaOutput()), + } + } +} diff --git a/chain/substreams/src/protobuf/substreams.entity.v1.rs b/chain/substreams/src/protobuf/substreams.entity.v1.rs new file mode 100644 index 0000000..94354c7 --- /dev/null +++ b/chain/substreams/src/protobuf/substreams.entity.v1.rs @@ -0,0 +1,68 @@ +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EntitiesChanges { + #[prost(bytes="vec", tag="1")] + pub block_id: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(bytes="vec", tag="3")] + pub prev_block_id: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="4")] + pub prev_block_number: u64, + #[prost(message, repeated, tag="5")] + pub entity_changes: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EntityChange { + #[prost(string, tag="1")] + pub entity: ::prost::alloc::string::String, + #[prost(bytes="vec", tag="2")] + pub id: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="3")] + pub ordinal: u64, + #[prost(enumeration="entity_change::Operation", tag="4")] + pub operation: i32, + #[prost(message, repeated, tag="5")] + pub fields: ::prost::alloc::vec::Vec, +} +/// Nested message and enum types in `EntityChange`. +pub mod entity_change { + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] + #[repr(i32)] + pub enum Operation { + /// Protobuf default should not be used, this is used so that the consume can ensure that the value was actually specified + Unset = 0, + Create = 1, + Update = 2, + Delete = 3, + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Field { + #[prost(string, tag="1")] + pub name: ::prost::alloc::string::String, + #[prost(enumeration="field::Type", tag="2")] + pub value_type: i32, + #[prost(bytes="vec", tag="3")] + pub new_value: ::prost::alloc::vec::Vec, + #[prost(bool, tag="4")] + pub new_value_null: bool, + #[prost(bytes="vec", tag="5")] + pub old_value: ::prost::alloc::vec::Vec, + #[prost(bool, tag="6")] + pub old_value_null: bool, +} +/// Nested message and enum types in `Field`. +pub mod field { + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] + #[repr(i32)] + pub enum Type { + /// Protobuf default should not be used, this is used so that the consume can ensure that the value was actually specified + Unset = 0, + Bigdecimal = 1, + Bigint = 2, + /// int32 + Int = 3, + Bytes = 4, + String = 5, + } +} diff --git a/chain/substreams/src/trigger.rs b/chain/substreams/src/trigger.rs new file mode 100644 index 0000000..22e0ace --- /dev/null +++ b/chain/substreams/src/trigger.rs @@ -0,0 +1,449 @@ +use std::{collections::HashMap, sync::Arc}; + +use anyhow::Error; +use graph::{ + blockchain::{self, block_stream::BlockWithTriggers, BlockPtr}, + components::{ + store::{DeploymentLocator, EntityKey, SubgraphFork}, + subgraph::{MappingError, ProofOfIndexingEvent, SharedProofOfIndexing}, + }, + data::store::scalar::Bytes, + data_source, + prelude::{ + anyhow, async_trait, BigDecimal, BigInt, BlockHash, BlockNumber, BlockState, Entity, + RuntimeHostBuilder, Value, + }, + slog::Logger, + substreams::Modules, +}; +use graph_runtime_wasm::module::ToAscPtr; +use lazy_static::__Deref; + +use crate::codec::Field; +use crate::{ + codec::{entity_change::Operation, field::Type}, + Block, Chain, NodeCapabilities, NoopDataSourceTemplate, +}; + +#[derive(Eq, PartialEq, PartialOrd, Ord, Debug)] +pub struct TriggerData {} + +impl blockchain::TriggerData for TriggerData { + // TODO(filipe): Can this be improved with some data from the block? + fn error_context(&self) -> String { + "Failed to process substreams block".to_string() + } +} + +impl ToAscPtr for TriggerData { + // substreams doesn't rely on wasm on the graph-node so this is not needed. + fn to_asc_ptr( + self, + _heap: &mut H, + _gas: &graph::runtime::gas::GasCounter, + ) -> Result, graph::runtime::DeterministicHostError> { + unimplemented!() + } +} + +#[derive(Debug, Clone, Default)] +pub struct TriggerFilter { + pub(crate) modules: Option, + pub(crate) module_name: String, + pub(crate) start_block: Option, + pub(crate) data_sources_len: u8, +} + +// TriggerFilter should bypass all triggers and just rely on block since all the data received +// should already have been processed. +impl blockchain::TriggerFilter for TriggerFilter { + fn extend_with_template(&mut self, _data_source: impl Iterator) { + } + + /// this function is not safe to call multiple times, only one DataSource is supported for + /// + fn extend<'a>( + &mut self, + mut data_sources: impl Iterator + Clone, + ) { + let Self { + modules, + module_name, + start_block, + data_sources_len, + } = self; + + if *data_sources_len >= 1 { + return; + } + + if let Some(ref ds) = data_sources.next() { + *data_sources_len = 1; + *modules = ds.source.package.modules.clone(); + *module_name = ds.source.module_name.clone(); + *start_block = ds.initial_block; + } + } + + fn node_capabilities(&self) -> NodeCapabilities { + NodeCapabilities {} + } + + fn to_firehose_filter(self) -> Vec { + unimplemented!("this should never be called for this type") + } +} + +pub struct TriggersAdapter {} + +#[async_trait] +impl blockchain::TriggersAdapter for TriggersAdapter { + async fn ancestor_block( + &self, + _ptr: BlockPtr, + _offset: BlockNumber, + ) -> Result, Error> { + unimplemented!() + } + + async fn scan_triggers( + &self, + _from: BlockNumber, + _to: BlockNumber, + _filter: &TriggerFilter, + ) -> Result>, Error> { + unimplemented!() + } + + async fn triggers_in_block( + &self, + _logger: &Logger, + _block: Block, + _filter: &TriggerFilter, + ) -> Result, Error> { + unimplemented!() + } + + async fn is_on_main_chain(&self, _ptr: BlockPtr) -> Result { + unimplemented!() + } + + async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { + // This seems to work for a lot of the firehose chains. + Ok(Some(BlockPtr { + hash: BlockHash::from(vec![0xff; 32]), + number: block.number.saturating_sub(1), + })) + } +} + +fn write_poi_event( + proof_of_indexing: &SharedProofOfIndexing, + poi_event: &ProofOfIndexingEvent, + causality_region: &str, + logger: &Logger, +) { + if let Some(proof_of_indexing) = proof_of_indexing { + let mut proof_of_indexing = proof_of_indexing.deref().borrow_mut(); + proof_of_indexing.write(logger, causality_region, poi_event); + } +} + +pub struct TriggerProcessor { + pub locator: DeploymentLocator, +} + +impl TriggerProcessor { + pub fn new(locator: DeploymentLocator) -> Self { + Self { locator } + } +} + +#[async_trait] +impl graph::prelude::TriggerProcessor for TriggerProcessor +where + T: RuntimeHostBuilder, +{ + async fn process_trigger( + &self, + logger: &Logger, + _hosts: &[Arc], + block: &Arc, + _trigger: &data_source::TriggerData, + mut state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &str, + _debug_fork: &Option>, + _subgraph_metrics: &Arc, + ) -> Result, MappingError> { + for entity_change in block.entity_changes.iter() { + match entity_change.operation() { + Operation::Unset => { + // Potentially an issue with the server side or + // we are running an outdated version. In either case we should abort. + return Err(MappingError::Unknown(anyhow!("Detected UNSET entity operation, either a server error or there's a new type of operation and we're running an outdated protobuf"))); + } + Operation::Create | Operation::Update => { + // TODO(filipe): Remove this once the substreams GRPC has been fixed. + let entity_type: &str = { + let letter: String = entity_change.entity[0..1].to_uppercase(); + &(letter + &entity_change.entity[1..]) + }; + let entity_id: String = String::from_utf8(entity_change.id.clone()) + .map_err(|e| MappingError::Unknown(anyhow::Error::from(e)))?; + let key = EntityKey::data(entity_type.to_string(), entity_id.clone()); + let mut data: HashMap = HashMap::from_iter(vec![]); + for field in entity_change.fields.iter() { + let value: Value = decode_entity_change(&field, &entity_change.entity)?; + *data + .entry(field.name.as_str().to_owned()) + .or_insert(Value::Null) = value; + } + + write_poi_event( + proof_of_indexing, + &ProofOfIndexingEvent::SetEntity { + entity_type: &entity_type, + id: &entity_id, + data: &data, + }, + causality_region, + logger, + ); + + state.entity_cache.set(key, Entity::from(data))?; + } + Operation::Delete => { + let entity_type: &str = &entity_change.entity; + let entity_id: String = String::from_utf8(entity_change.id.clone()) + .map_err(|e| MappingError::Unknown(anyhow::Error::from(e)))?; + let key = EntityKey::data(entity_type.to_string(), entity_id.clone()); + + state.entity_cache.remove(key); + + write_poi_event( + proof_of_indexing, + &ProofOfIndexingEvent::RemoveEntity { + entity_type, + id: &entity_id, + }, + causality_region, + logger, + ) + } + } + } + + Ok(state) + } +} + +fn decode_entity_change(field: &Field, entity: &String) -> Result { + match field.value_type() { + Type::Unset => { + return Err(MappingError::Unknown(anyhow!( + "Invalid field type, the protobuf probably needs updating" + ))) + } + Type::Bigdecimal => match BigDecimal::parse_bytes(field.new_value.as_ref()) { + Some(bd) => Ok(Value::BigDecimal(bd)), + None => { + return Err(MappingError::Unknown(anyhow!( + "Unable to parse BigDecimal for entity {}", + entity + ))) + } + }, + Type::Bigint => Ok(Value::BigInt(BigInt::from_signed_bytes_be( + field.new_value.as_ref(), + ))), + Type::Int => { + let mut bytes: [u8; 8] = [0; 8]; + bytes.copy_from_slice(field.new_value.as_ref()); + Ok(Value::Int(i64::from_be_bytes(bytes) as i32)) + } + Type::Bytes => Ok(Value::Bytes(Bytes::from(field.new_value.as_ref()))), + Type::String => Ok(Value::String( + String::from_utf8(field.new_value.clone()) + .map_err(|e| MappingError::Unknown(anyhow::Error::from(e)))?, + )), + } +} + +#[cfg(test)] +mod test { + use std::str::FromStr; + + use crate::codec::field::Type as FieldType; + use crate::codec::Field; + use crate::trigger::decode_entity_change; + use graph::{ + data::store::scalar::Bytes, + prelude::{BigDecimal, BigInt, Value}, + }; + + #[test] + fn validate_substreams_field_types() { + struct Case { + field: Field, + entity: String, + expected_new_value: Value, + } + + let cases = vec![ + Case { + field: Field { + name: "setting string value".to_string(), + value_type: FieldType::String as i32, + new_value: Vec::from( + "d4325ee72c39999e778a9908f5fb0803f78e30c441a5f2ce5c65eee0e0eba59d", + ), + new_value_null: false, + old_value: Vec::from("".to_string()), + old_value_null: true, + }, + entity: "Block".to_string(), + expected_new_value: Value::String( + "d4325ee72c39999e778a9908f5fb0803f78e30c441a5f2ce5c65eee0e0eba59d".to_string(), + ), + }, + Case { + field: Field { + name: "settings bytes value".to_string(), + value_type: FieldType::Bytes as i32, + new_value: hex::decode( + "445247fe150195bd866516594e087e1728294aa831613f4d48b8ec618908519f", + ) + .unwrap(), + new_value_null: false, + old_value: Vec::from("".to_string()), + old_value_null: true, + }, + entity: "Block".to_string(), + expected_new_value: Value::Bytes( + Bytes::from_str( + "0x445247fe150195bd866516594e087e1728294aa831613f4d48b8ec618908519f", + ) + .unwrap(), + ), + }, + Case { + field: Field { + name: "setting int value for block 12369760".to_string(), + value_type: FieldType::Int as i32, + new_value: hex::decode("0000000000bcbf60").unwrap(), + new_value_null: false, + old_value: vec![], + old_value_null: true, + }, + entity: "Block".to_string(), + expected_new_value: Value::Int(12369760), + }, + Case { + field: Field { + name: "setting int value for block 12369622".to_string(), + value_type: FieldType::Int as i32, + new_value: hex::decode("0000000000bcbed6").unwrap(), + new_value_null: false, + old_value: vec![], + old_value_null: true, + }, + entity: "Block".to_string(), + expected_new_value: Value::Int(12369622), + }, + Case { + field: Field { + name: "setting int value for block 12369623".to_string(), + value_type: FieldType::Int as i32, + new_value: hex::decode("0000000000bcbed7").unwrap(), + new_value_null: false, + old_value: vec![], + old_value_null: true, + }, + entity: "Block".to_string(), + expected_new_value: Value::Int(12369623), + }, + Case { + field: Field { + name: "setting big int transactions count of 123".to_string(), + value_type: FieldType::Bigint as i32, + new_value: hex::decode("7b").unwrap(), + new_value_null: false, + old_value: vec![], + old_value_null: true, + }, + entity: "Block".to_string(), + expected_new_value: Value::BigInt(BigInt::from(123u64)), + }, + Case { + field: Field { + name: "setting big int transactions count of 302".to_string(), + value_type: FieldType::Bigint as i32, + new_value: hex::decode("012e").unwrap(), + new_value_null: false, + old_value: vec![], + old_value_null: true, + }, + entity: "Block".to_string(), + expected_new_value: Value::BigInt(BigInt::from(302u64)), + }, + Case { + field: Field { + name: "setting big int transactions count of 209".to_string(), + value_type: FieldType::Bigint as i32, + new_value: hex::decode("00d1").unwrap(), + new_value_null: false, + old_value: vec![], + old_value_null: true, + }, + entity: "Block".to_string(), + expected_new_value: Value::BigInt(BigInt::from(209u64)), + }, + Case { + field: Field { + name: "setting big decimal value".to_string(), + value_type: FieldType::Bigdecimal as i32, + new_value: hex::decode("3133363633312e35").unwrap(), + new_value_null: false, + old_value: vec![], + old_value_null: true, + }, + entity: "Block".to_string(), + expected_new_value: Value::BigDecimal(BigDecimal::from(136631.5)), + }, + Case { + field: Field { + name: "setting big decimal value 2".to_string(), + value_type: FieldType::Bigdecimal as i32, + new_value: hex::decode("3133303730392e30").unwrap(), + new_value_null: false, + old_value: vec![], + old_value_null: true, + }, + entity: "Block".to_string(), + expected_new_value: Value::BigDecimal(BigDecimal::from(130709.0)), + }, + Case { + field: Field { + name: "setting big decimal value 3".to_string(), + value_type: FieldType::Bigdecimal as i32, + new_value: hex::decode("39373839322e36").unwrap(), + new_value_null: false, + old_value: vec![], + old_value_null: true, + }, + entity: "Block".to_string(), + expected_new_value: Value::BigDecimal(BigDecimal::new(BigInt::from(978926u64), -1)), + }, + ]; + + for case in cases.into_iter() { + let value: Value = decode_entity_change(&case.field, &case.entity).unwrap(); + assert_eq!( + case.expected_new_value, value, + "failed case: {}", + case.field.name + ) + } + } +} diff --git a/core/Cargo.toml b/core/Cargo.toml new file mode 100644 index 0000000..94f1681 --- /dev/null +++ b/core/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "graph-core" +version = "0.27.0" +edition = "2021" + +[dependencies] +async-trait = "0.1.50" +atomic_refcell = "0.1.8" +async-stream = "0.3" +bytes = "1.0" +futures01 = { package="futures", version="0.1.31" } +futures = { version="0.3.4", features=["compat"] } +graph = { path = "../graph" } +# This dependency is temporary. The multiblockchain refactoring is not +# finished as long as this dependency exists +graph-chain-arweave = { path = "../chain/arweave" } +graph-chain-ethereum = { path = "../chain/ethereum" } +graph-chain-near = { path = "../chain/near" } +graph-chain-cosmos = { path = "../chain/cosmos" } +graph-chain-substreams = { path = "../chain/substreams" } +lazy_static = "1.2.0" +lru_time_cache = "0.11" +semver = "1.0.12" +serde = "1.0" +serde_json = "1.0" +serde_yaml = "0.8" +# Switch to crates.io once tower 0.5 is released +tower = { git = "https://github.com/tower-rs/tower.git", features = ["util", "limit"] } +graph-runtime-wasm = { path = "../runtime/wasm" } +cid = "0.8.6" +anyhow = "1.0" + +[dev-dependencies] +tower-test = { git = "https://github.com/tower-rs/tower.git" } +graph-mock = { path = "../mock" } +test-store = { path = "../store/test-store" } +hex = "0.4.3" +graphql-parser = "0.4.0" +pretty_assertions = "1.3.0" +anyhow = "1.0" diff --git a/core/src/lib.rs b/core/src/lib.rs new file mode 100644 index 0000000..08ac3fc --- /dev/null +++ b/core/src/lib.rs @@ -0,0 +1,9 @@ +pub mod polling_monitor; + +mod link_resolver; +mod metrics; +mod subgraph; + +pub use crate::link_resolver::LinkResolver; +pub use crate::metrics::MetricsRegistry; +pub use crate::subgraph::{SubgraphAssignmentProvider, SubgraphInstanceManager, SubgraphRegistrar}; diff --git a/core/src/link_resolver.rs b/core/src/link_resolver.rs new file mode 100644 index 0000000..9b2be6f --- /dev/null +++ b/core/src/link_resolver.rs @@ -0,0 +1,406 @@ +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use anyhow::anyhow; +use async_trait::async_trait; +use bytes::BytesMut; +use futures01::{stream::poll_fn, try_ready}; +use futures03::stream::FuturesUnordered; +use graph::env::EnvVars; +use graph::util::futures::RetryConfigNoTimeout; +use lru_time_cache::LruCache; +use serde_json::Value; + +use graph::{ + ipfs_client::{IpfsClient, StatApi}, + prelude::{LinkResolver as LinkResolverTrait, *}, +}; + +fn retry_policy( + always_retry: bool, + op: &'static str, + logger: &Logger, +) -> RetryConfigNoTimeout { + // Even if retries were not requested, networking errors are still retried until we either get + // a valid HTTP response or a timeout. + if always_retry { + retry(op, logger).no_limit() + } else { + retry(op, logger) + .no_limit() + .when(|res: &Result<_, reqwest::Error>| match res { + Ok(_) => false, + Err(e) => !(e.is_status() || e.is_timeout()), + }) + } + .no_timeout() // The timeout should be set in the internal future. +} + +/// The IPFS APIs don't have a quick "do you have the file" function. Instead, we +/// just rely on whether an API times out. That makes sense for IPFS, but not for +/// our application. We want to be able to quickly select from a potential list +/// of clients where hopefully one already has the file, and just get the file +/// from that. +/// +/// The strategy here then is to use a stat API as a proxy for "do you have the +/// file". Whichever client has or gets the file first wins. This API is a good +/// choice, because it doesn't involve us actually starting to download the file +/// from each client, which would be wasteful of bandwidth and memory in the +/// case multiple clients respond in a timely manner. In addition, we may make +/// good use of the stat returned. +async fn select_fastest_client_with_stat( + clients: Arc>>, + logger: Logger, + api: StatApi, + path: String, + timeout: Duration, + do_retry: bool, +) -> Result<(u64, Arc), Error> { + let mut err: Option = None; + + let mut stats: FuturesUnordered<_> = clients + .iter() + .enumerate() + .map(|(i, c)| { + let c = c.cheap_clone(); + let path = path.clone(); + retry_policy(do_retry, "IPFS stat", &logger).run(move || { + let path = path.clone(); + let c = c.cheap_clone(); + async move { + c.stat_size(api, path, timeout) + .map_ok(move |s| (s, i)) + .await + } + }) + }) + .collect(); + + while let Some(result) = stats.next().await { + match result { + Ok((stat, index)) => { + return Ok((stat, clients[index].cheap_clone())); + } + Err(e) => err = Some(e.into()), + } + } + + Err(err.unwrap_or_else(|| { + anyhow!( + "No IPFS clients were supplied to handle the call to object.stat. File: {}", + path + ) + })) +} + +// Returns an error if the stat is bigger than `max_file_bytes` +fn restrict_file_size(path: &str, size: u64, max_file_bytes: usize) -> Result<(), Error> { + if size > max_file_bytes as u64 { + return Err(anyhow!( + "IPFS file {} is too large. It can be at most {} bytes but is {} bytes", + path, + max_file_bytes, + size + )); + } + Ok(()) +} + +#[derive(Clone)] +pub struct LinkResolver { + clients: Arc>>, + cache: Arc>>>, + timeout: Duration, + retry: bool, + env_vars: Arc, +} + +impl LinkResolver { + pub fn new(clients: Vec, env_vars: Arc) -> Self { + Self { + clients: Arc::new(clients.into_iter().map(Arc::new).collect()), + cache: Arc::new(Mutex::new(LruCache::with_capacity( + env_vars.mappings.max_ipfs_cache_size as usize, + ))), + timeout: env_vars.mappings.ipfs_timeout, + retry: false, + env_vars, + } + } +} + +impl Debug for LinkResolver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LinkResolver") + .field("timeout", &self.timeout) + .field("retry", &self.retry) + .field("env_vars", &self.env_vars) + .finish() + } +} + +impl CheapClone for LinkResolver { + fn cheap_clone(&self) -> Self { + self.clone() + } +} + +#[async_trait] +impl LinkResolverTrait for LinkResolver { + fn with_timeout(&self, timeout: Duration) -> Box { + let mut s = self.cheap_clone(); + s.timeout = timeout; + Box::new(s) + } + + fn with_retries(&self) -> Box { + let mut s = self.cheap_clone(); + s.retry = true; + Box::new(s) + } + + /// Supports links of the form `/ipfs/ipfs_hash` or just `ipfs_hash`. + async fn cat(&self, logger: &Logger, link: &Link) -> Result, Error> { + // Discard the `/ipfs/` prefix (if present) to get the hash. + let path = link.link.trim_start_matches("/ipfs/").to_owned(); + + if let Some(data) = self.cache.lock().unwrap().get(&path) { + trace!(logger, "IPFS cache hit"; "hash" => &path); + return Ok(data.clone()); + } + trace!(logger, "IPFS cache miss"; "hash" => &path); + + let (size, client) = select_fastest_client_with_stat( + self.clients.cheap_clone(), + logger.cheap_clone(), + StatApi::Files, + path.clone(), + self.timeout, + self.retry, + ) + .await?; + + let max_cache_file_size = self.env_vars.mappings.max_ipfs_cache_file_size; + let max_file_size = self.env_vars.mappings.max_ipfs_file_bytes; + restrict_file_size(&path, size, max_file_size)?; + + let req_path = path.clone(); + let timeout = self.timeout; + let data = retry_policy(self.retry, "ipfs.cat", &logger) + .run(move || { + let path = req_path.clone(); + let client = client.clone(); + async move { Ok(client.cat_all(&path, timeout).await?.to_vec()) } + }) + .await?; + + // The size reported by `files/stat` is not guaranteed to be exact, so check the limit again. + restrict_file_size(&path, data.len() as u64, max_file_size)?; + + // Only cache files if they are not too large + if data.len() <= max_cache_file_size { + let mut cache = self.cache.lock().unwrap(); + if !cache.contains_key(&path) { + cache.insert(path.to_owned(), data.clone()); + } + } else { + debug!(logger, "File too large for cache"; + "path" => path, + "size" => data.len() + ); + } + + Ok(data) + } + + async fn get_block(&self, logger: &Logger, link: &Link) -> Result, Error> { + trace!(logger, "IPFS block get"; "hash" => &link.link); + let (size, client) = select_fastest_client_with_stat( + self.clients.cheap_clone(), + logger.cheap_clone(), + StatApi::Block, + link.link.clone(), + self.timeout, + self.retry, + ) + .await?; + + let max_file_size = self.env_vars.mappings.max_ipfs_file_bytes; + restrict_file_size(&link.link, size, max_file_size)?; + + let link = link.link.clone(); + let data = retry_policy(self.retry, "ipfs.getBlock", &logger) + .run(move || { + let link = link.clone(); + let client = client.clone(); + async move { + let data = client.get_block(link.clone()).await?.to_vec(); + Result::, reqwest::Error>::Ok(data) + } + }) + .await?; + + Ok(data) + } + + async fn json_stream(&self, logger: &Logger, link: &Link) -> Result { + // Discard the `/ipfs/` prefix (if present) to get the hash. + let path = link.link.trim_start_matches("/ipfs/"); + + let (size, client) = select_fastest_client_with_stat( + self.clients.cheap_clone(), + logger.cheap_clone(), + StatApi::Files, + path.to_string(), + self.timeout, + self.retry, + ) + .await?; + + let max_file_size = self.env_vars.mappings.max_ipfs_map_file_size; + restrict_file_size(path, size, max_file_size)?; + + let mut stream = client.cat(path, None).await?.fuse().boxed().compat(); + + let mut buf = BytesMut::with_capacity(1024); + + // Count the number of lines we've already successfully deserialized. + // We need that to adjust the line number in error messages from serde_json + // to translate from line numbers in the snippet we are deserializing + // to the line number in the overall file + let mut count = 0; + + let stream: JsonValueStream = Box::pin( + poll_fn(move || -> Poll, Error> { + loop { + if let Some(offset) = buf.iter().position(|b| *b == b'\n') { + let line_bytes = buf.split_to(offset + 1); + count += 1; + if line_bytes.len() > 1 { + let line = std::str::from_utf8(&line_bytes)?; + let res = match serde_json::from_str::(line) { + Ok(v) => Ok(Async::Ready(Some(JsonStreamValue { + value: v, + line: count, + }))), + Err(e) => { + // Adjust the line number in the serde error. This + // is fun because we can only get at the full error + // message, and not the error message without line number + let msg = e.to_string(); + let msg = msg.split(" at line ").next().unwrap(); + Err(anyhow!( + "{} at line {} column {}: '{}'", + msg, + e.line() + count - 1, + e.column(), + line + )) + } + }; + return res; + } + } else { + // We only get here if there is no complete line in buf, and + // it is therefore ok to immediately pass an Async::NotReady + // from stream through. + // If we get a None from poll, but still have something in buf, + // that means the input was not terminated with a newline. We + // add that so that the last line gets picked up in the next + // run through the loop. + match try_ready!(stream.poll().map_err(|e| anyhow::anyhow!("{}", e))) { + Some(b) => buf.extend_from_slice(&b), + None if buf.len() > 0 => buf.extend_from_slice(&[b'\n']), + None => return Ok(Async::Ready(None)), + } + } + } + }) + .compat(), + ); + + Ok(stream) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use graph::env::EnvVars; + use serde_json::json; + + #[tokio::test] + async fn max_file_size() { + let mut env_vars = EnvVars::default(); + env_vars.mappings.max_ipfs_file_bytes = 200; + + let file: &[u8] = &[0u8; 201]; + let client = IpfsClient::localhost(); + let resolver = super::LinkResolver::new(vec![client.clone()], Arc::new(env_vars)); + + let logger = Logger::root(slog::Discard, o!()); + + let link = client.add(file.into()).await.unwrap().hash; + let err = LinkResolver::cat(&resolver, &logger, &Link { link: link.clone() }) + .await + .unwrap_err(); + assert_eq!( + err.to_string(), + format!( + "IPFS file {} is too large. It can be at most 200 bytes but is 212 bytes", + link + ) + ); + } + + async fn json_round_trip(text: &'static str, env_vars: EnvVars) -> Result, Error> { + let client = IpfsClient::localhost(); + let resolver = super::LinkResolver::new(vec![client.clone()], Arc::new(env_vars)); + + let logger = Logger::root(slog::Discard, o!()); + let link = client.add(text.as_bytes().into()).await.unwrap().hash; + + let stream = LinkResolver::json_stream(&resolver, &logger, &Link { link }).await?; + stream.map_ok(|sv| sv.value).try_collect().await + } + + #[tokio::test] + async fn read_json_stream() { + let values = json_round_trip("\"with newline\"\n", EnvVars::default()).await; + assert_eq!(vec![json!("with newline")], values.unwrap()); + + let values = json_round_trip("\"without newline\"", EnvVars::default()).await; + assert_eq!(vec![json!("without newline")], values.unwrap()); + + let values = json_round_trip("\"two\" \n \"things\"", EnvVars::default()).await; + assert_eq!(vec![json!("two"), json!("things")], values.unwrap()); + + let values = json_round_trip( + "\"one\"\n \"two\" \n [\"bad\" \n \"split\"]", + EnvVars::default(), + ) + .await; + assert_eq!( + "EOF while parsing a list at line 4 column 0: ' [\"bad\" \n'", + values.unwrap_err().to_string() + ); + } + + #[tokio::test] + async fn ipfs_map_file_size() { + let file = "\"small test string that trips the size restriction\""; + let mut env_vars = EnvVars::default(); + env_vars.mappings.max_ipfs_map_file_size = file.len() - 1; + + let err = json_round_trip(file, env_vars).await.unwrap_err(); + + assert!(err.to_string().contains(" is too large")); + + env_vars = EnvVars::default(); + let values = json_round_trip(file, env_vars).await; + assert_eq!( + vec!["small test string that trips the size restriction"], + values.unwrap() + ); + } +} diff --git a/core/src/metrics/mod.rs b/core/src/metrics/mod.rs new file mode 100644 index 0000000..047d6b2 --- /dev/null +++ b/core/src/metrics/mod.rs @@ -0,0 +1,3 @@ +mod registry; + +pub use registry::MetricsRegistry; diff --git a/core/src/metrics/registry.rs b/core/src/metrics/registry.rs new file mode 100644 index 0000000..12f77ce --- /dev/null +++ b/core/src/metrics/registry.rs @@ -0,0 +1,331 @@ +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; + +use graph::components::metrics::{counter_with_labels, gauge_with_labels}; +use graph::prelude::{MetricsRegistry as MetricsRegistryTrait, *}; + +#[derive(Clone)] +pub struct MetricsRegistry { + logger: Logger, + registry: Arc, + register_errors: Box, + unregister_errors: Box, + registered_metrics: Box, + + /// Global metrics are lazily initialized and identified by + /// the `Desc.id` that hashes the name and const label values + global_counters: Arc>>, + global_counter_vecs: Arc>>, + global_gauges: Arc>>, + global_gauge_vecs: Arc>>, + global_histogram_vecs: Arc>>, +} + +impl MetricsRegistry { + pub fn new(logger: Logger, registry: Arc) -> Self { + // Generate internal metrics + let register_errors = Self::gen_register_errors_counter(registry.clone()); + let unregister_errors = Self::gen_unregister_errors_counter(registry.clone()); + let registered_metrics = Self::gen_registered_metrics_gauge(registry.clone()); + + MetricsRegistry { + logger: logger.new(o!("component" => String::from("MetricsRegistry"))), + registry, + register_errors, + unregister_errors, + registered_metrics, + global_counters: Arc::new(RwLock::new(HashMap::new())), + global_counter_vecs: Arc::new(RwLock::new(HashMap::new())), + global_gauges: Arc::new(RwLock::new(HashMap::new())), + global_gauge_vecs: Arc::new(RwLock::new(HashMap::new())), + global_histogram_vecs: Arc::new(RwLock::new(HashMap::new())), + } + } + + fn gen_register_errors_counter(registry: Arc) -> Box { + let opts = Opts::new( + String::from("metrics_register_errors"), + String::from("Counts Prometheus metrics register errors"), + ); + let counter = Box::new( + Counter::with_opts(opts).expect("failed to create `metrics_register_errors` counter"), + ); + registry + .register(counter.clone()) + .expect("failed to register `metrics_register_errors` counter"); + counter + } + + fn gen_unregister_errors_counter(registry: Arc) -> Box { + let opts = Opts::new( + String::from("metrics_unregister_errors"), + String::from("Counts Prometheus metrics unregister errors"), + ); + let counter = Box::new( + Counter::with_opts(opts).expect("failed to create `metrics_unregister_errors` counter"), + ); + registry + .register(counter.clone()) + .expect("failed to register `metrics_unregister_errors` counter"); + counter + } + + fn gen_registered_metrics_gauge(registry: Arc) -> Box { + let opts = Opts::new( + String::from("registered_metrics"), + String::from("Tracks the number of registered metrics on the node"), + ); + let gauge = + Box::new(Gauge::with_opts(opts).expect("failed to create `registered_metrics` gauge")); + registry + .register(gauge.clone()) + .expect("failed to register `registered_metrics` gauge"); + gauge + } + + fn global_counter_vec_internal( + &self, + name: &str, + help: &str, + deployment: Option<&str>, + variable_labels: &[&str], + ) -> Result { + let opts = Opts::new(name, help); + let opts = match deployment { + None => opts, + Some(deployment) => opts.const_label("deployment", deployment), + }; + let counters = CounterVec::new(opts, variable_labels)?; + let id = counters.desc().first().unwrap().id; + let maybe_counter = self.global_counter_vecs.read().unwrap().get(&id).cloned(); + if let Some(counters) = maybe_counter { + Ok(counters) + } else { + self.register(name, Box::new(counters.clone())); + self.global_counter_vecs + .write() + .unwrap() + .insert(id, counters.clone()); + Ok(counters) + } + } +} + +impl MetricsRegistryTrait for MetricsRegistry { + fn register(&self, name: &str, c: Box) { + let err = match self.registry.register(c).err() { + None => { + self.registered_metrics.inc(); + return; + } + Some(err) => { + self.register_errors.inc(); + err + } + }; + match err { + PrometheusError::AlreadyReg => { + error!( + self.logger, + "registering metric [{}] failed because it was already registered", name, + ); + } + PrometheusError::InconsistentCardinality { expect, got } => { + error!( + self.logger, + "registering metric [{}] failed due to inconsistent caridinality, expected = {} got = {}", + name, + expect, + got, + ); + } + PrometheusError::Msg(msg) => { + error!( + self.logger, + "registering metric [{}] failed because: {}", name, msg, + ); + } + PrometheusError::Io(err) => { + error!( + self.logger, + "registering metric [{}] failed due to io error: {}", name, err, + ); + } + PrometheusError::Protobuf(err) => { + error!( + self.logger, + "registering metric [{}] failed due to protobuf error: {}", name, err + ); + } + }; + } + + fn global_counter( + &self, + name: &str, + help: &str, + const_labels: HashMap, + ) -> Result { + let counter = counter_with_labels(name, help, const_labels)?; + let id = counter.desc().first().unwrap().id; + let maybe_counter = self.global_counters.read().unwrap().get(&id).cloned(); + if let Some(counter) = maybe_counter { + Ok(counter) + } else { + self.register(name, Box::new(counter.clone())); + self.global_counters + .write() + .unwrap() + .insert(id, counter.clone()); + Ok(counter) + } + } + + fn global_counter_vec( + &self, + name: &str, + help: &str, + variable_labels: &[&str], + ) -> Result { + self.global_counter_vec_internal(name, help, None, variable_labels) + } + + fn global_deployment_counter_vec( + &self, + name: &str, + help: &str, + subgraph: &str, + variable_labels: &[&str], + ) -> Result { + self.global_counter_vec_internal(name, help, Some(subgraph), variable_labels) + } + + fn global_gauge( + &self, + name: &str, + help: &str, + const_labels: HashMap, + ) -> Result { + let gauge = gauge_with_labels(name, help, const_labels)?; + let id = gauge.desc().first().unwrap().id; + let maybe_gauge = self.global_gauges.read().unwrap().get(&id).cloned(); + if let Some(gauge) = maybe_gauge { + Ok(gauge.clone()) + } else { + self.register(name, Box::new(gauge.clone())); + self.global_gauges + .write() + .unwrap() + .insert(id, gauge.clone()); + Ok(gauge) + } + } + + fn global_gauge_vec( + &self, + name: &str, + help: &str, + variable_labels: &[&str], + ) -> Result { + let opts = Opts::new(name, help); + let gauges = GaugeVec::new(opts, variable_labels)?; + let id = gauges.desc().first().unwrap().id; + let maybe_gauge = self.global_gauge_vecs.read().unwrap().get(&id).cloned(); + if let Some(gauges) = maybe_gauge { + Ok(gauges) + } else { + self.register(name, Box::new(gauges.clone())); + self.global_gauge_vecs + .write() + .unwrap() + .insert(id, gauges.clone()); + Ok(gauges) + } + } + + fn global_histogram_vec( + &self, + name: &str, + help: &str, + variable_labels: &[&str], + ) -> Result { + let opts = HistogramOpts::new(name, help); + let histograms = HistogramVec::new(opts, variable_labels)?; + let id = histograms.desc().first().unwrap().id; + let maybe_histogram = self.global_histogram_vecs.read().unwrap().get(&id).cloned(); + if let Some(histograms) = maybe_histogram { + Ok(histograms) + } else { + self.register(name, Box::new(histograms.clone())); + self.global_histogram_vecs + .write() + .unwrap() + .insert(id, histograms.clone()); + Ok(histograms) + } + } + + fn unregister(&self, metric: Box) { + match self.registry.unregister(metric) { + Ok(_) => { + self.registered_metrics.dec(); + } + Err(e) => { + self.unregister_errors.inc(); + error!(self.logger, "Unregistering metric failed = {:?}", e,); + } + }; + } +} + +#[test] +fn global_counters_are_shared() { + use graph::log; + + let logger = log::logger(false); + let prom_reg = Arc::new(Registry::new()); + let registry = MetricsRegistry::new(logger, prom_reg.clone()); + + fn check_counters( + registry: &MetricsRegistry, + name: &str, + const_labels: HashMap, + ) { + let c1 = registry + .global_counter(name, "help me", const_labels.clone()) + .expect("first test counter"); + let c2 = registry + .global_counter(name, "help me", const_labels) + .expect("second test counter"); + let desc1 = c1.desc(); + let desc2 = c2.desc(); + let d1 = desc1.first().unwrap(); + let d2 = desc2.first().unwrap(); + + // Registering the same metric with the same name and + // const labels twice works and returns the same metric (logically) + assert_eq!(d1.id, d2.id, "counters: {}", name); + + // They share the reported values + c1.inc_by(7.0); + c2.inc_by(2.0); + assert_eq!(9.0, c1.get(), "counters: {}", name); + assert_eq!(9.0, c2.get(), "counters: {}", name); + } + + check_counters(®istry, "nolabels", HashMap::new()); + + let const_labels = { + let mut map = HashMap::new(); + map.insert("pool".to_owned(), "main".to_owned()); + map + }; + check_counters(®istry, "pool", const_labels); + + let const_labels = { + let mut map = HashMap::new(); + map.insert("pool".to_owned(), "replica0".to_owned()); + map + }; + check_counters(®istry, "pool", const_labels); +} diff --git a/core/src/polling_monitor/ipfs_service.rs b/core/src/polling_monitor/ipfs_service.rs new file mode 100644 index 0000000..05d904f --- /dev/null +++ b/core/src/polling_monitor/ipfs_service.rs @@ -0,0 +1,131 @@ +use anyhow::{anyhow, Error}; +use bytes::Bytes; +use cid::Cid; +use futures::{Future, FutureExt}; +use graph::{ + cheap_clone::CheapClone, + ipfs_client::{IpfsClient, StatApi}, + tokio::sync::Semaphore, +}; +use std::{pin::Pin, sync::Arc, task::Poll, time::Duration}; +use tower::Service; + +const CLOUDFLARE_TIMEOUT: u16 = 524; +const GATEWAY_TIMEOUT: u16 = 504; + +/// Reference type, clones will refer to the same service. +#[derive(Clone)] +pub struct IpfsService { + client: IpfsClient, + max_file_size: u64, + timeout: Duration, + concurrency_limiter: Arc, +} + +impl CheapClone for IpfsService { + fn cheap_clone(&self) -> Self { + Self { + client: self.client.cheap_clone(), + max_file_size: self.max_file_size, + timeout: self.timeout, + concurrency_limiter: self.concurrency_limiter.cheap_clone(), + } + } +} + +impl IpfsService { + pub fn new( + client: IpfsClient, + max_file_size: u64, + timeout: Duration, + concurrency_limit: u16, + ) -> Self { + Self { + client, + max_file_size, + timeout, + concurrency_limiter: Arc::new(Semaphore::new(concurrency_limit as usize)), + } + } + + async fn call(&self, cid: Cid) -> Result, Error> { + let multihash = cid.hash().code(); + if !SAFE_MULTIHASHES.contains(&multihash) { + return Err(anyhow!("CID multihash {} is not allowed", multihash)); + } + + let cid_str = cid.to_string(); + let size = match self + .client + .stat_size(StatApi::Files, cid_str, self.timeout) + .await + { + Ok(size) => size, + Err(e) => match e.status().map(|e| e.as_u16()) { + Some(GATEWAY_TIMEOUT) | Some(CLOUDFLARE_TIMEOUT) => return Ok(None), + _ if e.is_timeout() => return Ok(None), + _ => return Err(e.into()), + }, + }; + + if size > self.max_file_size { + return Err(anyhow!( + "IPFS file {} is too large. It can be at most {} bytes but is {} bytes", + cid.to_string(), + self.max_file_size, + size + )); + } + + Ok(self + .client + .cat_all(&cid.to_string(), self.timeout) + .await + .map(Some)?) + } +} + +impl Service for IpfsService { + type Response = (Cid, Option); + type Error = (Cid, Error); + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + // The permit is acquired and immediately dropped, as tower does not yet allow returning it. + // So this is only indicative of capacity being available. + Pin::new(&mut self.concurrency_limiter.acquire().boxed()) + .poll(cx) + .map_ok(|_| ()) + .map_err(|_| unreachable!("semaphore is never closed")) + } + + fn call(&mut self, cid: Cid) -> Self::Future { + let this = self.cheap_clone(); + async move { + let _permit = this.concurrency_limiter.acquire().await; + this.call(cid).await.map(|x| (cid, x)).map_err(|e| (cid, e)) + } + .boxed() + } +} + +// Multihashes that are collision resistant. This is not complete but covers the commonly used ones. +// Code table: https://github.com/multiformats/multicodec/blob/master/table.csv +// rust-multihash code enum: https://github.com/multiformats/rust-multihash/blob/master/src/multihash_impl.rs +const SAFE_MULTIHASHES: [u64; 15] = [ + 0x0, // Identity + 0x12, // SHA2-256 (32-byte hash size) + 0x13, // SHA2-512 (64-byte hash size) + 0x17, // SHA3-224 (28-byte hash size) + 0x16, // SHA3-256 (32-byte hash size) + 0x15, // SHA3-384 (48-byte hash size) + 0x14, // SHA3-512 (64-byte hash size) + 0x1a, // Keccak-224 (28-byte hash size) + 0x1b, // Keccak-256 (32-byte hash size) + 0x1c, // Keccak-384 (48-byte hash size) + 0x1d, // Keccak-512 (64-byte hash size) + 0xb220, // BLAKE2b-256 (32-byte hash size) + 0xb240, // BLAKE2b-512 (64-byte hash size) + 0xb260, // BLAKE2s-256 (32-byte hash size) + 0x1e, // BLAKE3-256 (32-byte hash size) +]; diff --git a/core/src/polling_monitor/metrics.rs b/core/src/polling_monitor/metrics.rs new file mode 100644 index 0000000..c7cfa89 --- /dev/null +++ b/core/src/polling_monitor/metrics.rs @@ -0,0 +1,62 @@ +use std::sync::Arc; + +use graph::{ + prelude::{DeploymentHash, MetricsRegistry}, + prometheus::{Counter, Gauge}, +}; + +pub struct PollingMonitorMetrics { + pub requests: Counter, + pub errors: Counter, + pub not_found: Counter, + pub queue_depth: Gauge, +} + +impl PollingMonitorMetrics { + pub fn new(registry: Arc, subgraph_hash: &DeploymentHash) -> Self { + let requests = registry + .new_deployment_counter( + "polling_monitor_requests", + "counts the total requests made to the service being polled", + subgraph_hash.as_str(), + ) + .unwrap(); + let not_found = registry + .new_deployment_counter( + "polling_monitor_not_found", + "counts 'not found' responses returned from the service being polled", + subgraph_hash.as_str(), + ) + .unwrap(); + let errors = registry + .new_deployment_counter( + "polling_monitor_errors", + "counts errors returned from the service being polled", + subgraph_hash.as_str(), + ) + .unwrap(); + let queue_depth = registry + .new_deployment_gauge( + "polling_monitor_queue_depth", + "size of the queue of polling requests", + subgraph_hash.as_str(), + ) + .unwrap(); + Self { + requests, + errors, + not_found, + queue_depth: queue_depth.into(), + } + } + + #[cfg(test)] + pub(crate) fn mock() -> Self { + Self { + requests: Counter::new("x", " ").unwrap(), + errors: Counter::new("y", " ").unwrap(), + not_found: Counter::new("z", " ").unwrap(), + queue_depth: Gauge::new("w", " ").unwrap(), + } + } +} diff --git a/core/src/polling_monitor/mod.rs b/core/src/polling_monitor/mod.rs new file mode 100644 index 0000000..cb8dbf4 --- /dev/null +++ b/core/src/polling_monitor/mod.rs @@ -0,0 +1,263 @@ +pub mod ipfs_service; +mod metrics; + +use std::fmt::Display; +use std::sync::Arc; + +use futures::stream; +use futures::stream::StreamExt; +use graph::cheap_clone::CheapClone; +use graph::parking_lot::Mutex; +use graph::prelude::tokio; +use graph::slog::{debug, Logger}; +use graph::util::monitored::MonitoredVecDeque as VecDeque; +use tokio::sync::{mpsc, watch}; +use tower::{Service, ServiceExt}; + +pub use self::metrics::PollingMonitorMetrics; + +/// Spawn a monitor that actively polls a service. Whenever the service has capacity, the monitor +/// pulls object ids from the queue and polls the service. If the object is not present or in case +/// of error, the object id is pushed to the back of the queue to be polled again. +/// +/// The service returns the request ID along with errors or responses. The response is an +/// `Option`, to represent the object not being found. +pub fn spawn_monitor( + service: S, + response_sender: mpsc::Sender<(ID, Response)>, + logger: Logger, + metrics: PollingMonitorMetrics, +) -> PollingMonitor +where + ID: Display + Send + 'static, + S: Service), Error = (ID, E)> + Send + 'static, + E: Display + Send + 'static, + S::Future: Send, +{ + let queue = Arc::new(Mutex::new(VecDeque::new( + metrics.queue_depth.clone(), + metrics.requests.clone(), + ))); + let (wake_up_queue, queue_woken) = watch::channel(()); + + let queue_to_stream = { + let queue = queue.cheap_clone(); + stream::unfold((), move |()| { + let queue = queue.cheap_clone(); + let mut queue_woken = queue_woken.clone(); + async move { + loop { + let id = queue.lock().pop_front(); + match id { + Some(id) => break Some((id, ())), + None => match queue_woken.changed().await { + // Queue woken, check it. + Ok(()) => {} + + // The `PollingMonitor` has been dropped, cancel this task. + Err(_) => break None, + }, + }; + } + } + }) + }; + + { + let queue = queue.cheap_clone(); + graph::spawn(async move { + let mut responses = service.call_all(queue_to_stream).unordered().boxed(); + while let Some(response) = responses.next().await { + match response { + Ok((id, Some(response))) => { + let send_result = response_sender.send((id, response)).await; + if send_result.is_err() { + // The receiver has been dropped, cancel this task. + break; + } + } + + // Object not found, push the id to the back of the queue. + Ok((id, None)) => { + metrics.not_found.inc(); + queue.lock().push_back(id); + } + + // Error polling, log it and push the id to the back of the queue. + Err((id, e)) => { + debug!(logger, "error polling"; + "error" => format!("{:#}", e), + "object_id" => id.to_string()); + metrics.errors.inc(); + queue.lock().push_back(id); + } + } + } + }); + } + + PollingMonitor { + queue, + wake_up_queue, + } +} + +/// Handle for adding objects to be monitored. +pub struct PollingMonitor { + queue: Arc>>, + + // This serves two purposes, to wake up the monitor when an item arrives on an empty queue, and + // to stop the montior task when this handle is dropped. + wake_up_queue: watch::Sender<()>, +} + +impl PollingMonitor { + /// Add an object id to the polling queue. New requests have priority and are pushed to the + /// front of the queue. + pub fn monitor(&self, id: ID) { + let mut queue = self.queue.lock(); + if queue.is_empty() { + // If the send fails, the response receiver has been dropped, so this handle is useless. + let _ = self.wake_up_queue.send(()); + } + queue.push_front(id); + } +} + +#[cfg(test)] +mod tests { + use anyhow::anyhow; + use futures::{Future, FutureExt, TryFutureExt}; + use graph::log; + use std::{pin::Pin, task::Poll}; + use tower_test::mock; + + use super::*; + + struct MockService(mock::Mock<&'static str, Option<&'static str>>); + + impl Service<&'static str> for MockService { + type Response = (&'static str, Option<&'static str>); + + type Error = (&'static str, anyhow::Error); + + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + self.0.poll_ready(cx).map_err(|_| unreachable!()) + } + + fn call(&mut self, req: &'static str) -> Self::Future { + self.0 + .call(req) + .map_ok(move |x| (req, x)) + .map_err(move |e| (req, anyhow!(e.to_string()))) + .boxed() + } + } + + async fn send_response(handle: &mut mock::Handle, res: U) { + handle.next_request().await.unwrap().1.send_response(res) + } + + #[tokio::test] + async fn polling_monitor_simple() { + let (svc, mut handle) = mock::pair(); + let (tx, mut rx) = mpsc::channel(10); + let monitor = spawn_monitor( + MockService(svc), + tx, + log::discard(), + PollingMonitorMetrics::mock(), + ); + + // Basic test, single file is immediately available. + monitor.monitor("req-0"); + send_response(&mut handle, Some("res-0")).await; + assert_eq!(rx.recv().await, Some(("req-0", "res-0"))); + } + + #[tokio::test] + async fn polling_monitor_unordered() { + let (svc, mut handle) = mock::pair(); + let (tx, mut rx) = mpsc::channel(10); + let monitor = spawn_monitor( + MockService(svc), + tx, + log::discard(), + PollingMonitorMetrics::mock(), + ); + + // Test unorderedness of the response stream, and the LIFO semantics of `monitor`. + // + // `req-1` has priority since it is the last request, but `req-0` is responded first. + monitor.monitor("req-0"); + monitor.monitor("req-1"); + let req_1 = handle.next_request().await.unwrap().1; + let req_0 = handle.next_request().await.unwrap().1; + req_0.send_response(Some("res-0")); + assert_eq!(rx.recv().await, Some(("req-0", "res-0"))); + req_1.send_response(Some("res-1")); + assert_eq!(rx.recv().await, Some(("req-1", "res-1"))); + } + + #[tokio::test] + async fn polling_monitor_failed_push_to_back() { + let (svc, mut handle) = mock::pair(); + let (tx, mut rx) = mpsc::channel(10); + + // Limit service to one request at a time. + let svc = tower::limit::ConcurrencyLimit::new(MockService(svc), 1); + let monitor = spawn_monitor(svc, tx, log::discard(), PollingMonitorMetrics::mock()); + + // Test that objects not found go on the back of the queue. + monitor.monitor("req-0"); + monitor.monitor("req-1"); + send_response(&mut handle, None).await; + send_response(&mut handle, Some("res-0")).await; + assert_eq!(rx.recv().await, Some(("req-0", "res-0"))); + send_response(&mut handle, Some("res-1")).await; + assert_eq!(rx.recv().await, Some(("req-1", "res-1"))); + + // Test that failed requests go on the back of the queue. + monitor.monitor("req-0"); + monitor.monitor("req-1"); + let req = handle.next_request().await.unwrap().1; + req.send_error(anyhow!("e")); + send_response(&mut handle, Some("res-0")).await; + assert_eq!(rx.recv().await, Some(("req-0", "res-0"))); + send_response(&mut handle, Some("res-1")).await; + assert_eq!(rx.recv().await, Some(("req-1", "res-1"))); + } + + #[tokio::test] + async fn polling_monitor_cancelation() { + let (svc, _handle) = mock::pair(); + let (tx, mut rx) = mpsc::channel(10); + let monitor = spawn_monitor( + MockService(svc), + tx, + log::discard(), + PollingMonitorMetrics::mock(), + ); + + // Cancelation on monitor drop. + drop(monitor); + assert_eq!(rx.recv().await, None); + + let (svc, mut handle) = mock::pair(); + let (tx, rx) = mpsc::channel(10); + let monitor = spawn_monitor( + MockService(svc), + tx, + log::discard(), + PollingMonitorMetrics::mock(), + ); + + // Cancelation on receiver drop. + monitor.monitor("req-0"); + drop(rx); + send_response(&mut handle, Some("res-0")).await; + assert!(handle.next_request().await.is_none()); + } +} diff --git a/core/src/subgraph/context.rs b/core/src/subgraph/context.rs new file mode 100644 index 0000000..653d6c8 --- /dev/null +++ b/core/src/subgraph/context.rs @@ -0,0 +1,195 @@ +pub mod instance; + +use crate::polling_monitor::{ + ipfs_service::IpfsService, spawn_monitor, PollingMonitor, PollingMonitorMetrics, +}; +use anyhow::{self, Error}; +use bytes::Bytes; +use cid::Cid; +use graph::{ + blockchain::Blockchain, + components::{ + store::{DeploymentId, SubgraphFork}, + subgraph::{MappingError, SharedProofOfIndexing}, + }, + data_source::{offchain, DataSource, TriggerData}, + prelude::{ + BlockNumber, BlockState, CancelGuard, DeploymentHash, MetricsRegistry, RuntimeHostBuilder, + SubgraphInstanceMetrics, TriggerProcessor, + }, + slog::Logger, + tokio::sync::mpsc, +}; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; + +use self::instance::SubgraphInstance; + +pub type SharedInstanceKeepAliveMap = Arc>>; + +// The context keeps track of mutable in-memory state that is retained across blocks. +// +// Currently most of the changes are applied in `runner.rs`, but ideally more of that would be +// refactored into the context so it wouldn't need `pub` fields. The entity cache should probaby +// also be moved here. +pub(crate) struct IndexingContext +where + T: RuntimeHostBuilder, + C: Blockchain, +{ + instance: SubgraphInstance, + pub instances: SharedInstanceKeepAliveMap, + pub filter: C::TriggerFilter, + pub offchain_monitor: OffchainMonitor, + trigger_processor: Box>, +} + +impl> IndexingContext { + pub fn new( + instance: SubgraphInstance, + instances: SharedInstanceKeepAliveMap, + filter: C::TriggerFilter, + offchain_monitor: OffchainMonitor, + trigger_processor: Box>, + ) -> Self { + Self { + instance, + instances, + filter, + offchain_monitor, + trigger_processor, + } + } + + pub async fn process_trigger( + &self, + logger: &Logger, + block: &Arc, + trigger: &TriggerData, + state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &str, + debug_fork: &Option>, + subgraph_metrics: &Arc, + ) -> Result, MappingError> { + self.process_trigger_in_hosts( + logger, + &self.instance.hosts(), + block, + trigger, + state, + proof_of_indexing, + causality_region, + debug_fork, + subgraph_metrics, + ) + .await + } + + pub async fn process_trigger_in_hosts( + &self, + logger: &Logger, + hosts: &[Arc], + block: &Arc, + trigger: &TriggerData, + state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &str, + debug_fork: &Option>, + subgraph_metrics: &Arc, + ) -> Result, MappingError> { + self.trigger_processor + .process_trigger( + logger, + hosts, + block, + trigger, + state, + proof_of_indexing, + causality_region, + debug_fork, + subgraph_metrics, + ) + .await + } + + // Removes data sources hosts with a creation block greater or equal to `reverted_block`, so + // that they are no longer candidates for `process_trigger`. + // + // This does not currently affect the `offchain_monitor` or the `filter`, so they will continue + // to include data sources that have been reverted. This is not ideal for performance, but it + // does not affect correctness since triggers that have no matching host will be ignored by + // `process_trigger`. + pub fn revert_data_sources(&mut self, reverted_block: BlockNumber) { + self.instance.revert_data_sources(reverted_block) + } + + pub fn add_dynamic_data_source( + &mut self, + logger: &Logger, + data_source: DataSource, + ) -> Result>, Error> { + let source = data_source.as_offchain().map(|ds| ds.source.clone()); + let host = self.instance.add_dynamic_data_source(logger, data_source)?; + + if host.is_some() { + if let Some(source) = source { + self.offchain_monitor.add_source(&source)?; + } + } + + Ok(host) + } +} + +pub(crate) struct OffchainMonitor { + ipfs_monitor: PollingMonitor, + ipfs_monitor_rx: mpsc::Receiver<(Cid, Bytes)>, +} + +impl OffchainMonitor { + pub fn new( + logger: Logger, + registry: Arc, + subgraph_hash: &DeploymentHash, + ipfs_service: IpfsService, + ) -> Self { + let (ipfs_monitor_tx, ipfs_monitor_rx) = mpsc::channel(10); + let ipfs_monitor = spawn_monitor( + ipfs_service, + ipfs_monitor_tx, + logger, + PollingMonitorMetrics::new(registry, subgraph_hash), + ); + Self { + ipfs_monitor, + ipfs_monitor_rx, + } + } + + fn add_source(&mut self, source: &offchain::Source) -> Result<(), Error> { + match source { + offchain::Source::Ipfs(cid) => self.ipfs_monitor.monitor(cid.clone()), + }; + Ok(()) + } + + pub fn ready_offchain_events(&mut self) -> Result, Error> { + use graph::tokio::sync::mpsc::error::TryRecvError; + + let mut triggers = vec![]; + loop { + match self.ipfs_monitor_rx.try_recv() { + Ok((cid, data)) => triggers.push(offchain::TriggerData { + source: offchain::Source::Ipfs(cid), + data: Arc::new(data), + }), + Err(TryRecvError::Disconnected) => { + anyhow::bail!("ipfs monitor unexpectedly terminated") + } + Err(TryRecvError::Empty) => break, + } + } + Ok(triggers) + } +} diff --git a/core/src/subgraph/context/instance.rs b/core/src/subgraph/context/instance.rs new file mode 100644 index 0000000..a9e3ade --- /dev/null +++ b/core/src/subgraph/context/instance.rs @@ -0,0 +1,166 @@ +use futures01::sync::mpsc::Sender; +use graph::{ + blockchain::Blockchain, + data_source::{DataSource, DataSourceTemplate}, + prelude::*, +}; +use std::collections::HashMap; + +use super::OffchainMonitor; + +pub(crate) struct SubgraphInstance> { + subgraph_id: DeploymentHash, + network: String, + host_builder: T, + templates: Arc>>, + host_metrics: Arc, + + /// Runtime hosts, one for each data source mapping. + /// + /// The runtime hosts are created and added in the same order the + /// data sources appear in the subgraph manifest. Incoming block + /// stream events are processed by the mappings in this same order. + hosts: Vec>, + + /// Maps the hash of a module to a channel to the thread in which the module is instantiated. + module_cache: HashMap<[u8; 32], Sender>, +} + +impl SubgraphInstance +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + pub fn from_manifest( + logger: &Logger, + manifest: SubgraphManifest, + host_builder: T, + host_metrics: Arc, + offchain_monitor: &mut OffchainMonitor, + ) -> Result { + let subgraph_id = manifest.id.clone(); + let network = manifest.network_name(); + let templates = Arc::new(manifest.templates); + + let mut this = SubgraphInstance { + host_builder, + subgraph_id, + network, + hosts: Vec::new(), + module_cache: HashMap::new(), + templates, + host_metrics, + }; + + // Create a new runtime host for each data source in the subgraph manifest; + // we use the same order here as in the subgraph manifest to make the + // event processing behavior predictable + for ds in manifest.data_sources { + // TODO: This is duplicating code from `IndexingContext::add_dynamic_data_source` and + // `SubgraphInstance::add_dynamic_data_source`. Ideally this should be refactored into + // `IndexingContext`. + + let runtime = ds.runtime(); + let module_bytes = match runtime { + None => continue, + Some(ref module_bytes) => module_bytes, + }; + + if let DataSource::Offchain(ds) = &ds { + offchain_monitor.add_source(&ds.source)?; + } + + let host = this.new_host(logger.cheap_clone(), ds, module_bytes)?; + this.hosts.push(Arc::new(host)); + } + + Ok(this) + } + + // module_bytes is the same as data_source.runtime().unwrap(), this is to ensure that this + // function is only called for data_sources for which data_source.runtime().is_some() is true. + fn new_host( + &mut self, + logger: Logger, + data_source: DataSource, + module_bytes: &Arc>, + ) -> Result { + let mapping_request_sender = { + let module_hash = tiny_keccak::keccak256(module_bytes.as_ref()); + if let Some(sender) = self.module_cache.get(&module_hash) { + sender.clone() + } else { + let sender = T::spawn_mapping( + module_bytes.as_ref(), + logger, + self.subgraph_id.clone(), + self.host_metrics.cheap_clone(), + )?; + self.module_cache.insert(module_hash, sender.clone()); + sender + } + }; + self.host_builder.build( + self.network.clone(), + self.subgraph_id.clone(), + data_source, + self.templates.cheap_clone(), + mapping_request_sender, + self.host_metrics.cheap_clone(), + ) + } + + pub(super) fn add_dynamic_data_source( + &mut self, + logger: &Logger, + data_source: DataSource, + ) -> Result>, Error> { + // Protect against creating more than the allowed maximum number of data sources + if let Some(max_data_sources) = ENV_VARS.subgraph_max_data_sources { + if self.hosts.len() >= max_data_sources { + anyhow::bail!( + "Limit of {} data sources per subgraph exceeded", + max_data_sources, + ); + } + } + + // `hosts` will remain ordered by the creation block. + // See also 8f1bca33-d3b7-4035-affc-fd6161a12448. + assert!( + self.hosts.last().and_then(|h| h.creation_block_number()) + <= data_source.creation_block() + ); + + let module_bytes = match &data_source.runtime() { + None => return Ok(None), + Some(ref module_bytes) => module_bytes.cheap_clone(), + }; + + let host = Arc::new(self.new_host(logger.clone(), data_source, &module_bytes)?); + + Ok(if self.hosts.contains(&host) { + None + } else { + self.hosts.push(host.clone()); + Some(host) + }) + } + + pub(super) fn revert_data_sources(&mut self, reverted_block: BlockNumber) { + // `hosts` is ordered by the creation block. + // See also 8f1bca33-d3b7-4035-affc-fd6161a12448. + while self + .hosts + .last() + .filter(|h| h.creation_block_number() >= Some(reverted_block)) + .is_some() + { + self.hosts.pop(); + } + } + + pub(super) fn hosts(&self) -> &[Arc] { + &self.hosts + } +} diff --git a/core/src/subgraph/error.rs b/core/src/subgraph/error.rs new file mode 100644 index 0000000..b313125 --- /dev/null +++ b/core/src/subgraph/error.rs @@ -0,0 +1,28 @@ +use graph::data::subgraph::schema::SubgraphError; +use graph::prelude::{thiserror, Error, StoreError}; + +#[derive(thiserror::Error, Debug)] +pub enum BlockProcessingError { + #[error("{0:#}")] + Unknown(#[from] Error), + + // The error had a deterministic cause but, for a possibly non-deterministic reason, we chose to + // halt processing due to the error. + #[error("{0}")] + Deterministic(SubgraphError), + + #[error("subgraph stopped while processing triggers")] + Canceled, +} + +impl BlockProcessingError { + pub fn is_deterministic(&self) -> bool { + matches!(self, BlockProcessingError::Deterministic(_)) + } +} + +impl From for BlockProcessingError { + fn from(e: StoreError) -> Self { + BlockProcessingError::Unknown(e.into()) + } +} diff --git a/core/src/subgraph/inputs.rs b/core/src/subgraph/inputs.rs new file mode 100644 index 0000000..191dc69 --- /dev/null +++ b/core/src/subgraph/inputs.rs @@ -0,0 +1,31 @@ +use graph::{ + blockchain::{Blockchain, TriggersAdapter}, + components::{ + store::{DeploymentLocator, SubgraphFork, WritableStore}, + subgraph::ProofOfIndexingVersion, + }, + data::subgraph::{SubgraphFeature, UnifiedMappingApiVersion}, + data_source::DataSourceTemplate, + prelude::BlockNumber, +}; +use std::collections::BTreeSet; +use std::sync::Arc; + +pub struct IndexingInputs { + pub deployment: DeploymentLocator, + pub features: BTreeSet, + pub start_blocks: Vec, + pub stop_block: Option, + pub store: Arc, + pub debug_fork: Option>, + pub triggers_adapter: Arc>, + pub chain: Arc, + pub templates: Arc>>, + pub unified_api_version: UnifiedMappingApiVersion, + pub static_filters: bool, + pub poi_version: ProofOfIndexingVersion, + pub network: String, + + // Correspondence between data source or template position in the manifest and name. + pub manifest_idx_and_name: Vec<(u32, String)>, +} diff --git a/core/src/subgraph/instance_manager.rs b/core/src/subgraph/instance_manager.rs new file mode 100644 index 0000000..785f848 --- /dev/null +++ b/core/src/subgraph/instance_manager.rs @@ -0,0 +1,397 @@ +use crate::polling_monitor::ipfs_service::IpfsService; +use crate::subgraph::context::{IndexingContext, SharedInstanceKeepAliveMap}; +use crate::subgraph::inputs::IndexingInputs; +use crate::subgraph::loader::load_dynamic_data_sources; +use crate::subgraph::runner::SubgraphRunner; +use graph::blockchain::block_stream::BlockStreamMetrics; +use graph::blockchain::Blockchain; +use graph::blockchain::NodeCapabilities; +use graph::blockchain::{BlockchainKind, TriggerFilter}; +use graph::components::subgraph::ProofOfIndexingVersion; +use graph::data::subgraph::SPEC_VERSION_0_0_6; +use graph::prelude::{SubgraphInstanceManager as SubgraphInstanceManagerTrait, *}; +use graph::{blockchain::BlockchainMap, components::store::DeploymentLocator}; +use graph_runtime_wasm::module::ToAscPtr; +use graph_runtime_wasm::RuntimeHostBuilder; +use tokio::task; + +use super::context::OffchainMonitor; +use super::SubgraphTriggerProcessor; + +pub struct SubgraphInstanceManager { + logger_factory: LoggerFactory, + subgraph_store: Arc, + chains: Arc, + metrics_registry: Arc, + manager_metrics: SubgraphInstanceManagerMetrics, + instances: SharedInstanceKeepAliveMap, + link_resolver: Arc, + ipfs_service: IpfsService, + static_filters: bool, +} + +#[async_trait] +impl SubgraphInstanceManagerTrait for SubgraphInstanceManager { + async fn start_subgraph( + self: Arc, + loc: DeploymentLocator, + manifest: serde_yaml::Mapping, + stop_block: Option, + ) { + let logger = self.logger_factory.subgraph_logger(&loc); + let err_logger = logger.clone(); + let instance_manager = self.cheap_clone(); + + let subgraph_start_future = async move { + match BlockchainKind::from_manifest(&manifest)? { + BlockchainKind::Arweave => { + instance_manager + .start_subgraph_inner::( + logger, + loc, + manifest, + stop_block, + Box::new(SubgraphTriggerProcessor {}), + ) + .await + } + BlockchainKind::Ethereum => { + instance_manager + .start_subgraph_inner::( + logger, + loc, + manifest, + stop_block, + Box::new(SubgraphTriggerProcessor {}), + ) + .await + } + BlockchainKind::Near => { + instance_manager + .start_subgraph_inner::( + logger, + loc, + manifest, + stop_block, + Box::new(SubgraphTriggerProcessor {}), + ) + .await + } + BlockchainKind::Cosmos => { + instance_manager + .start_subgraph_inner::( + logger, + loc, + manifest, + stop_block, + Box::new(SubgraphTriggerProcessor {}), + ) + .await + } + BlockchainKind::Substreams => { + instance_manager + .start_subgraph_inner::( + logger, + loc.cheap_clone(), + manifest, + stop_block, + Box::new(graph_chain_substreams::TriggerProcessor::new(loc)), + ) + .await + } + } + }; + // Perform the actual work of starting the subgraph in a separate + // task. If the subgraph is a graft or a copy, starting it will + // perform the actual work of grafting/copying, which can take + // hours. Running it in the background makes sure the instance + // manager does not hang because of that work. + graph::spawn(async move { + match subgraph_start_future.await { + Ok(()) => self.manager_metrics.subgraph_count.inc(), + Err(err) => error!( + err_logger, + "Failed to start subgraph"; + "error" => format!("{:#}", err), + "code" => LogCode::SubgraphStartFailure + ), + } + }); + } + + fn stop_subgraph(&self, loc: DeploymentLocator) { + let logger = self.logger_factory.subgraph_logger(&loc); + info!(logger, "Stop subgraph"); + + // Drop the cancel guard to shut down the subgraph now + let mut instances = self.instances.write().unwrap(); + instances.remove(&loc.id); + + self.manager_metrics.subgraph_count.dec(); + } +} + +impl SubgraphInstanceManager { + pub fn new( + logger_factory: &LoggerFactory, + subgraph_store: Arc, + chains: Arc, + metrics_registry: Arc, + link_resolver: Arc, + ipfs_service: IpfsService, + static_filters: bool, + ) -> Self { + let logger = logger_factory.component_logger("SubgraphInstanceManager", None); + let logger_factory = logger_factory.with_parent(logger.clone()); + + SubgraphInstanceManager { + logger_factory, + subgraph_store, + chains, + manager_metrics: SubgraphInstanceManagerMetrics::new(metrics_registry.cheap_clone()), + metrics_registry, + instances: SharedInstanceKeepAliveMap::default(), + link_resolver, + ipfs_service, + static_filters, + } + } + + async fn start_subgraph_inner( + self: Arc, + logger: Logger, + deployment: DeploymentLocator, + manifest: serde_yaml::Mapping, + stop_block: Option, + tp: Box>>, + ) -> Result<(), Error> + where + ::MappingTrigger: ToAscPtr, + { + let subgraph_store = self.subgraph_store.cheap_clone(); + let registry = self.metrics_registry.cheap_clone(); + let store = self + .subgraph_store + .cheap_clone() + .writable(logger.clone(), deployment.id) + .await?; + + // Start the subgraph deployment before reading dynamic data + // sources; if the subgraph is a graft or a copy, starting it will + // do the copying and dynamic data sources won't show up until after + // that is done + store.start_subgraph_deployment(&logger).await?; + + let (manifest, manifest_idx_and_name) = { + info!(logger, "Resolve subgraph files using IPFS"); + + let mut manifest = SubgraphManifest::resolve_from_raw( + deployment.hash.cheap_clone(), + manifest, + // Allow for infinite retries for subgraph definition files. + &Arc::from(self.link_resolver.with_retries()), + &logger, + ENV_VARS.max_spec_version.clone(), + ) + .await + .context("Failed to resolve subgraph from IPFS")?; + + // We cannot include static data sources in the map because a static data source and a + // template may have the same name in the manifest. + let ds_len = manifest.data_sources.len() as u32; + let manifest_idx_and_name: Vec<(u32, String)> = manifest + .templates + .iter() + .map(|t| t.name().to_owned()) + .enumerate() + .map(|(idx, name)| (ds_len + idx as u32, name)) + .collect(); + + let data_sources = load_dynamic_data_sources( + store.clone(), + logger.clone(), + &manifest, + manifest_idx_and_name.clone(), + ) + .await + .context("Failed to load dynamic data sources")?; + + info!(logger, "Successfully resolved subgraph files using IPFS"); + + // Add dynamic data sources to the subgraph + manifest.data_sources.extend(data_sources); + + info!( + logger, + "Data source count at start: {}", + manifest.data_sources.len() + ); + + (manifest, manifest_idx_and_name) + }; + + let onchain_data_sources = manifest + .data_sources + .iter() + .filter_map(|d| d.as_onchain().cloned()) + .collect::>(); + let required_capabilities = C::NodeCapabilities::from_data_sources(&onchain_data_sources); + let network = manifest.network_name(); + + let chain = self + .chains + .get::(network.clone()) + .with_context(|| format!("no chain configured for network {}", network))? + .clone(); + + // Obtain filters from the manifest + let mut filter = C::TriggerFilter::from_data_sources(onchain_data_sources.iter()); + + if self.static_filters { + filter.extend_with_template( + manifest + .templates + .iter() + .filter_map(|ds| ds.as_onchain()) + .cloned(), + ); + } + + let start_blocks = manifest.start_blocks(); + + let templates = Arc::new(manifest.templates.clone()); + + // Obtain the debug fork from the subgraph store + let debug_fork = self + .subgraph_store + .debug_fork(&deployment.hash, logger.clone())?; + + // Create a subgraph instance from the manifest; this moves + // ownership of the manifest and host builder into the new instance + let stopwatch_metrics = StopwatchMetrics::new( + logger.clone(), + deployment.hash.clone(), + "process", + self.metrics_registry.clone(), + ); + + let unified_mapping_api_version = manifest.unified_mapping_api_version()?; + let triggers_adapter = chain.triggers_adapter(&deployment, &required_capabilities, unified_mapping_api_version).map_err(|e| + anyhow!( + "expected triggers adapter that matches deployment {} with required capabilities: {}: {}", + &deployment, + &required_capabilities, e))?.clone(); + + let subgraph_metrics = Arc::new(SubgraphInstanceMetrics::new( + registry.cheap_clone(), + deployment.hash.as_str(), + stopwatch_metrics.clone(), + )); + let subgraph_metrics_unregister = subgraph_metrics.clone(); + let host_metrics = Arc::new(HostMetrics::new( + registry.cheap_clone(), + deployment.hash.as_str(), + stopwatch_metrics.clone(), + )); + let block_stream_metrics = Arc::new(BlockStreamMetrics::new( + registry.cheap_clone(), + &deployment.hash, + manifest.network_name(), + store.shard().to_string(), + stopwatch_metrics, + )); + + let mut offchain_monitor = OffchainMonitor::new( + logger.cheap_clone(), + registry.cheap_clone(), + &manifest.id, + self.ipfs_service.cheap_clone(), + ); + + // Initialize deployment_head with current deployment head. Any sort of trouble in + // getting the deployment head ptr leads to initializing with 0 + let deployment_head = store.block_ptr().map(|ptr| ptr.number).unwrap_or(0) as f64; + block_stream_metrics.deployment_head.set(deployment_head); + + let host_builder = graph_runtime_wasm::RuntimeHostBuilder::new( + chain.runtime_adapter(), + self.link_resolver.cheap_clone(), + subgraph_store.ens_lookup(), + ); + + let features = manifest.features.clone(); + let unified_api_version = manifest.unified_mapping_api_version()?; + let poi_version = if manifest.spec_version.ge(&SPEC_VERSION_0_0_6) { + ProofOfIndexingVersion::Fast + } else { + ProofOfIndexingVersion::Legacy + }; + + let instance = super::context::instance::SubgraphInstance::from_manifest( + &logger, + manifest, + host_builder, + host_metrics.clone(), + &mut offchain_monitor, + )?; + + let inputs = IndexingInputs { + deployment: deployment.clone(), + features, + start_blocks, + stop_block, + store, + debug_fork, + triggers_adapter, + chain, + templates, + unified_api_version, + static_filters: self.static_filters, + manifest_idx_and_name, + poi_version, + network, + }; + + // The subgraph state tracks the state of the subgraph instance over time + let ctx = IndexingContext::new( + instance, + self.instances.cheap_clone(), + filter, + offchain_monitor, + tp, + ); + + let metrics = RunnerMetrics { + subgraph: subgraph_metrics, + host: host_metrics, + stream: block_stream_metrics, + }; + + // Keep restarting the subgraph until it terminates. The subgraph + // will usually only run once, but is restarted whenever a block + // creates dynamic data sources. This allows us to recreate the + // block stream and include events for the new data sources going + // forward; this is easier than updating the existing block stream. + // + // This is a long-running and unfortunately a blocking future (see #905), so it is run in + // its own thread. It is also run with `task::unconstrained` because we have seen deadlocks + // occur without it, possibly caused by our use of legacy futures and tokio versions in the + // codebase and dependencies, which may not play well with the tokio 1.0 cooperative + // scheduling. It is also logical in terms of performance to run this with `unconstrained`, + // it has a dedicated OS thread so the OS will handle the preemption. See + // https://github.com/tokio-rs/tokio/issues/3493. + graph::spawn_thread(deployment.to_string(), move || { + let runner = SubgraphRunner::new(inputs, ctx, logger.cheap_clone(), metrics); + if let Err(e) = graph::block_on(task::unconstrained(runner.run())) { + error!( + &logger, + "Subgraph instance failed to run: {}", + format!("{:#}", e) + ); + } + subgraph_metrics_unregister.unregister(registry); + }); + + Ok(()) + } +} diff --git a/core/src/subgraph/loader.rs b/core/src/subgraph/loader.rs new file mode 100644 index 0000000..3e59319 --- /dev/null +++ b/core/src/subgraph/loader.rs @@ -0,0 +1,47 @@ +use std::time::Instant; + +use graph::blockchain::Blockchain; +use graph::components::store::WritableStore; +use graph::data_source::DataSource; +use graph::prelude::*; + +pub async fn load_dynamic_data_sources( + store: Arc, + logger: Logger, + manifest: &SubgraphManifest, + manifest_idx_and_name: Vec<(u32, String)>, +) -> Result>, Error> { + let start_time = Instant::now(); + + let mut data_sources: Vec> = vec![]; + + for stored in store + .load_dynamic_data_sources(manifest_idx_and_name) + .await? + { + let template = manifest + .templates + .iter() + .find(|template| template.manifest_idx() == stored.manifest_idx) + .ok_or_else(|| anyhow!("no template with idx `{}` was found", stored.manifest_idx))?; + + let ds = DataSource::from_stored_dynamic_data_source(&template, stored)?; + + // The data sources are ordered by the creation block. + // See also 8f1bca33-d3b7-4035-affc-fd6161a12448. + anyhow::ensure!( + data_sources.last().and_then(|d| d.creation_block()) <= ds.creation_block(), + "Assertion failure: new data source has lower creation block than existing ones" + ); + + data_sources.push(ds); + } + + trace!( + logger, + "Loaded dynamic data sources"; + "ms" => start_time.elapsed().as_millis() + ); + + Ok(data_sources) +} diff --git a/core/src/subgraph/mod.rs b/core/src/subgraph/mod.rs new file mode 100644 index 0000000..490c45f --- /dev/null +++ b/core/src/subgraph/mod.rs @@ -0,0 +1,16 @@ +mod context; +mod error; +mod inputs; +mod instance_manager; +mod loader; +mod provider; +mod registrar; +mod runner; +mod state; +mod stream; +mod trigger_processor; + +pub use self::instance_manager::SubgraphInstanceManager; +pub use self::provider::SubgraphAssignmentProvider; +pub use self::registrar::SubgraphRegistrar; +pub use self::trigger_processor::*; diff --git a/core/src/subgraph/provider.rs b/core/src/subgraph/provider.rs new file mode 100644 index 0000000..c513cb3 --- /dev/null +++ b/core/src/subgraph/provider.rs @@ -0,0 +1,90 @@ +use std::collections::HashSet; +use std::sync::Mutex; + +use async_trait::async_trait; + +use graph::{ + components::store::{DeploymentId, DeploymentLocator}, + prelude::{SubgraphAssignmentProvider as SubgraphAssignmentProviderTrait, *}, +}; + +pub struct SubgraphAssignmentProvider { + logger_factory: LoggerFactory, + subgraphs_running: Arc>>, + link_resolver: Arc, + instance_manager: Arc, +} + +impl SubgraphAssignmentProvider { + pub fn new( + logger_factory: &LoggerFactory, + link_resolver: Arc, + instance_manager: I, + ) -> Self { + let logger = logger_factory.component_logger("SubgraphAssignmentProvider", None); + let logger_factory = logger_factory.with_parent(logger.clone()); + + // Create the subgraph provider + SubgraphAssignmentProvider { + logger_factory, + subgraphs_running: Arc::new(Mutex::new(HashSet::new())), + link_resolver: link_resolver.with_retries().into(), + instance_manager: Arc::new(instance_manager), + } + } +} + +#[async_trait] +impl SubgraphAssignmentProviderTrait for SubgraphAssignmentProvider { + async fn start( + &self, + loc: DeploymentLocator, + stop_block: Option, + ) -> Result<(), SubgraphAssignmentProviderError> { + let logger = self.logger_factory.subgraph_logger(&loc); + + // If subgraph ID already in set + if !self.subgraphs_running.lock().unwrap().insert(loc.id) { + info!(logger, "Subgraph deployment is already running"); + + return Err(SubgraphAssignmentProviderError::AlreadyRunning( + loc.hash.clone(), + )); + } + + let file_bytes = self + .link_resolver + .cat(&logger, &loc.hash.to_ipfs_link()) + .await + .map_err(SubgraphAssignmentProviderError::ResolveError)?; + + let raw: serde_yaml::Mapping = serde_yaml::from_slice(&file_bytes) + .map_err(|e| SubgraphAssignmentProviderError::ResolveError(e.into()))?; + + self.instance_manager + .cheap_clone() + .start_subgraph(loc, raw, stop_block) + .await; + + Ok(()) + } + + async fn stop( + &self, + deployment: DeploymentLocator, + ) -> Result<(), SubgraphAssignmentProviderError> { + // If subgraph ID was in set + if self + .subgraphs_running + .lock() + .unwrap() + .remove(&deployment.id) + { + // Shut down subgraph processing + self.instance_manager.stop_subgraph(deployment); + Ok(()) + } else { + Err(SubgraphAssignmentProviderError::NotRunning(deployment)) + } + } +} diff --git a/core/src/subgraph/registrar.rs b/core/src/subgraph/registrar.rs new file mode 100644 index 0000000..8c4a215 --- /dev/null +++ b/core/src/subgraph/registrar.rs @@ -0,0 +1,634 @@ +use std::collections::HashSet; +use std::time::Instant; + +use async_trait::async_trait; +use graph::blockchain::Blockchain; +use graph::blockchain::BlockchainKind; +use graph::blockchain::BlockchainMap; +use graph::components::store::{DeploymentId, DeploymentLocator, SubscriptionManager}; +use graph::data::subgraph::schema::DeploymentCreate; +use graph::data::subgraph::Graft; +use graph::prelude::{ + CreateSubgraphResult, SubgraphAssignmentProvider as SubgraphAssignmentProviderTrait, + SubgraphRegistrar as SubgraphRegistrarTrait, *, +}; + +pub struct SubgraphRegistrar { + logger: Logger, + logger_factory: LoggerFactory, + resolver: Arc, + provider: Arc

, + store: Arc, + subscription_manager: Arc, + chains: Arc, + node_id: NodeId, + version_switching_mode: SubgraphVersionSwitchingMode, + assignment_event_stream_cancel_guard: CancelGuard, // cancels on drop +} + +impl SubgraphRegistrar +where + P: SubgraphAssignmentProviderTrait, + S: SubgraphStore, + SM: SubscriptionManager, +{ + pub fn new( + logger_factory: &LoggerFactory, + resolver: Arc, + provider: Arc

, + store: Arc, + subscription_manager: Arc, + chains: Arc, + node_id: NodeId, + version_switching_mode: SubgraphVersionSwitchingMode, + ) -> Self { + let logger = logger_factory.component_logger("SubgraphRegistrar", None); + let logger_factory = logger_factory.with_parent(logger.clone()); + + let resolver = resolver.with_retries(); + + SubgraphRegistrar { + logger, + logger_factory, + resolver: resolver.into(), + provider, + store, + subscription_manager, + chains, + node_id, + version_switching_mode, + assignment_event_stream_cancel_guard: CancelGuard::new(), + } + } + + pub fn start(&self) -> impl Future { + let logger_clone1 = self.logger.clone(); + let logger_clone2 = self.logger.clone(); + let provider = self.provider.clone(); + let node_id = self.node_id.clone(); + let assignment_event_stream_cancel_handle = + self.assignment_event_stream_cancel_guard.handle(); + + // The order of the following three steps is important: + // - Start assignment event stream + // - Read assignments table and start assigned subgraphs + // - Start processing assignment event stream + // + // Starting the event stream before reading the assignments table ensures that no + // assignments are missed in the period of time between the table read and starting event + // processing. + // Delaying the start of event processing until after the table has been read and processed + // ensures that Remove events happen after the assigned subgraphs have been started, not + // before (otherwise a subgraph could be left running due to a race condition). + // + // The discrepancy between the start time of the event stream and the table read can result + // in some extraneous events on start up. Examples: + // - The event stream sees an Add event for subgraph A, but the table query finds that + // subgraph A is already in the table. + // - The event stream sees a Remove event for subgraph B, but the table query finds that + // subgraph B has already been removed. + // The `handle_assignment_events` function handles these cases by ignoring AlreadyRunning + // (on subgraph start) or NotRunning (on subgraph stop) error types, which makes the + // operations idempotent. + + // Start event stream + let assignment_event_stream = self.assignment_events(); + + // Deploy named subgraphs found in store + self.start_assigned_subgraphs().and_then(move |()| { + // Spawn a task to handle assignment events. + // Blocking due to store interactions. Won't be blocking after #905. + graph::spawn_blocking( + assignment_event_stream + .compat() + .map_err(SubgraphAssignmentProviderError::Unknown) + .map_err(CancelableError::Error) + .cancelable(&assignment_event_stream_cancel_handle, || { + Err(CancelableError::Cancel) + }) + .compat() + .for_each(move |assignment_event| { + assert_eq!(assignment_event.node_id(), &node_id); + handle_assignment_event( + assignment_event, + provider.clone(), + logger_clone1.clone(), + ) + .boxed() + .compat() + }) + .map_err(move |e| match e { + CancelableError::Cancel => panic!("assignment event stream canceled"), + CancelableError::Error(e) => { + error!(logger_clone2, "Assignment event stream failed: {}", e); + panic!("assignment event stream failed: {}", e); + } + }) + .compat(), + ); + + Ok(()) + }) + } + + pub fn assignment_events(&self) -> impl Stream + Send { + let store = self.store.clone(); + let node_id = self.node_id.clone(); + let logger = self.logger.clone(); + + self.subscription_manager + .subscribe(FromIterator::from_iter([SubscriptionFilter::Assignment])) + .map_err(|()| anyhow!("Entity change stream failed")) + .map(|event| { + // We're only interested in the SubgraphDeploymentAssignment change; we + // know that there is at least one, as that is what we subscribed to + let filter = SubscriptionFilter::Assignment; + let assignments = event + .changes + .iter() + .filter(|change| filter.matches(change)) + .map(|change| match change { + EntityChange::Data { .. } => unreachable!(), + EntityChange::Assignment { + deployment, + operation, + } => (deployment.clone(), operation.clone()), + }) + .collect::>(); + stream::iter_ok(assignments) + }) + .flatten() + .and_then( + move |(deployment, operation)| -> Result + Send>, _> { + trace!(logger, "Received assignment change"; + "deployment" => %deployment, + "operation" => format!("{:?}", operation), + ); + + match operation { + EntityChangeOperation::Set => { + store + .assigned_node(&deployment) + .map_err(|e| { + anyhow!("Failed to get subgraph assignment entity: {}", e) + }) + .map(|assigned| -> Box + Send> { + if let Some(assigned) = assigned { + if assigned == node_id { + // Start subgraph on this node + debug!(logger, "Deployment assignee is this node, broadcasting add event"; "assigned_to" => assigned, "node_id" => &node_id); + Box::new(stream::once(Ok(AssignmentEvent::Add { + deployment, + node_id: node_id.clone(), + }))) + } else { + // Ensure it is removed from this node + debug!(logger, "Deployment assignee is not this node, broadcasting remove event"; "assigned_to" => assigned, "node_id" => &node_id); + Box::new(stream::once(Ok(AssignmentEvent::Remove { + deployment, + node_id: node_id.clone(), + }))) + } + } else { + // Was added/updated, but is now gone. + debug!(logger, "Deployment has not assignee, we will get a separate remove event later"; "node_id" => &node_id); + Box::new(stream::empty()) + } + }) + } + EntityChangeOperation::Removed => { + // Send remove event without checking node ID. + // If node ID does not match, then this is a no-op when handled in + // assignment provider. + Ok(Box::new(stream::once(Ok(AssignmentEvent::Remove { + deployment, + node_id: node_id.clone(), + })))) + } + } + }, + ) + .flatten() + } + + fn start_assigned_subgraphs(&self) -> impl Future { + let provider = self.provider.clone(); + let logger = self.logger.clone(); + let node_id = self.node_id.clone(); + + future::result(self.store.assignments(&self.node_id)) + .map_err(|e| anyhow!("Error querying subgraph assignments: {}", e)) + .and_then(move |deployments| { + // This operation should finish only after all subgraphs are + // started. We wait for the spawned tasks to complete by giving + // each a `sender` and waiting for all of them to be dropped, so + // the receiver terminates without receiving anything. + let deployments = HashSet::::from_iter(deployments); + let deployments_len = deployments.len(); + let (sender, receiver) = futures01::sync::mpsc::channel::<()>(1); + for id in deployments { + let sender = sender.clone(); + let logger = logger.clone(); + + graph::spawn( + start_subgraph(id, provider.clone(), logger).map(move |()| drop(sender)), + ); + } + drop(sender); + receiver.collect().then(move |_| { + info!(logger, "Started all assigned subgraphs"; + "count" => deployments_len, "node_id" => &node_id); + future::ok(()) + }) + }) + } +} + +#[async_trait] +impl SubgraphRegistrarTrait for SubgraphRegistrar +where + P: SubgraphAssignmentProviderTrait, + S: SubgraphStore, + SM: SubscriptionManager, +{ + async fn create_subgraph( + &self, + name: SubgraphName, + ) -> Result { + let id = self.store.create_subgraph(name.clone())?; + + debug!(self.logger, "Created subgraph"; "subgraph_name" => name.to_string()); + + Ok(CreateSubgraphResult { id }) + } + + async fn create_subgraph_version( + &self, + name: SubgraphName, + hash: DeploymentHash, + node_id: NodeId, + debug_fork: Option, + start_block_override: Option, + graft_block_override: Option, + ) -> Result { + // We don't have a location for the subgraph yet; that will be + // assigned when we deploy for real. For logging purposes, make up a + // fake locator + let logger = self + .logger_factory + .subgraph_logger(&DeploymentLocator::new(DeploymentId(0), hash.clone())); + + let raw: serde_yaml::Mapping = { + let file_bytes = self + .resolver + .cat(&logger, &hash.to_ipfs_link()) + .await + .map_err(|e| { + SubgraphRegistrarError::ResolveError( + SubgraphManifestResolveError::ResolveError(e), + ) + })?; + + serde_yaml::from_slice(&file_bytes) + .map_err(|e| SubgraphRegistrarError::ResolveError(e.into()))? + }; + + let kind = BlockchainKind::from_manifest(&raw).map_err(|e| { + SubgraphRegistrarError::ResolveError(SubgraphManifestResolveError::ResolveError(e)) + })?; + + let deployment_locator = match kind { + BlockchainKind::Arweave => { + create_subgraph_version::( + &logger, + self.store.clone(), + self.chains.cheap_clone(), + name.clone(), + hash.cheap_clone(), + start_block_override, + graft_block_override, + raw, + node_id, + debug_fork, + self.version_switching_mode, + &self.resolver, + ) + .await? + } + BlockchainKind::Ethereum => { + create_subgraph_version::( + &logger, + self.store.clone(), + self.chains.cheap_clone(), + name.clone(), + hash.cheap_clone(), + start_block_override, + graft_block_override, + raw, + node_id, + debug_fork, + self.version_switching_mode, + &self.resolver, + ) + .await? + } + BlockchainKind::Near => { + create_subgraph_version::( + &logger, + self.store.clone(), + self.chains.cheap_clone(), + name.clone(), + hash.cheap_clone(), + start_block_override, + graft_block_override, + raw, + node_id, + debug_fork, + self.version_switching_mode, + &self.resolver, + ) + .await? + } + BlockchainKind::Cosmos => { + create_subgraph_version::( + &logger, + self.store.clone(), + self.chains.cheap_clone(), + name.clone(), + hash.cheap_clone(), + start_block_override, + graft_block_override, + raw, + node_id, + debug_fork, + self.version_switching_mode, + &self.resolver, + ) + .await? + } + BlockchainKind::Substreams => { + create_subgraph_version::( + &logger, + self.store.clone(), + self.chains.cheap_clone(), + name.clone(), + hash.cheap_clone(), + start_block_override, + graft_block_override, + raw, + node_id, + debug_fork, + self.version_switching_mode, + &self.resolver, + ) + .await? + } + }; + + debug!( + &logger, + "Wrote new subgraph version to store"; + "subgraph_name" => name.to_string(), + "subgraph_hash" => hash.to_string(), + ); + + Ok(deployment_locator) + } + + async fn remove_subgraph(&self, name: SubgraphName) -> Result<(), SubgraphRegistrarError> { + self.store.clone().remove_subgraph(name.clone())?; + + debug!(self.logger, "Removed subgraph"; "subgraph_name" => name.to_string()); + + Ok(()) + } + + /// Reassign a subgraph deployment to a different node. + /// + /// Reassigning to a nodeId that does not match any reachable graph-nodes will effectively pause the + /// subgraph syncing process. + async fn reassign_subgraph( + &self, + hash: &DeploymentHash, + node_id: &NodeId, + ) -> Result<(), SubgraphRegistrarError> { + let locations = self.store.locators(hash)?; + let deployment = match locations.len() { + 0 => return Err(SubgraphRegistrarError::DeploymentNotFound(hash.to_string())), + 1 => locations[0].clone(), + _ => { + return Err(SubgraphRegistrarError::StoreError( + anyhow!( + "there are {} different deployments with id {}", + locations.len(), + hash.as_str() + ) + .into(), + )) + } + }; + self.store.reassign_subgraph(&deployment, node_id)?; + + Ok(()) + } +} + +async fn handle_assignment_event( + event: AssignmentEvent, + provider: Arc, + logger: Logger, +) -> Result<(), CancelableError> { + let logger = logger.to_owned(); + + debug!(logger, "Received assignment event: {:?}", event); + + match event { + AssignmentEvent::Add { + deployment, + node_id: _, + } => Ok(start_subgraph(deployment, provider.clone(), logger).await), + AssignmentEvent::Remove { + deployment, + node_id: _, + } => match provider.stop(deployment).await { + Ok(()) => Ok(()), + Err(SubgraphAssignmentProviderError::NotRunning(_)) => Ok(()), + Err(e) => Err(CancelableError::Error(e)), + }, + } +} + +async fn start_subgraph( + deployment: DeploymentLocator, + provider: Arc, + logger: Logger, +) { + let logger = logger + .new(o!("subgraph_id" => deployment.hash.to_string(), "sgd" => deployment.id.to_string())); + + trace!(logger, "Start subgraph"); + + let start_time = Instant::now(); + let result = provider.start(deployment.clone(), None).await; + + debug!( + logger, + "Subgraph started"; + "start_ms" => start_time.elapsed().as_millis() + ); + + match result { + Ok(()) => (), + Err(SubgraphAssignmentProviderError::AlreadyRunning(_)) => (), + Err(e) => { + // Errors here are likely an issue with the subgraph. + error!( + logger, + "Subgraph instance failed to start"; + "error" => e.to_string() + ); + } + } +} + +/// Resolves the subgraph's earliest block +async fn resolve_start_block( + manifest: &SubgraphManifest, + chain: &impl Blockchain, + logger: &Logger, +) -> Result, SubgraphRegistrarError> { + // If the minimum start block is 0 (i.e. the genesis block), + // return `None` to start indexing from the genesis block. Otherwise + // return a block pointer for the block with number `min_start_block - 1`. + match manifest + .start_blocks() + .into_iter() + .min() + .expect("cannot identify minimum start block because there are no data sources") + { + 0 => Ok(None), + min_start_block => chain + .block_pointer_from_number(logger, min_start_block - 1) + .await + .map(Some) + .map_err(move |_| { + SubgraphRegistrarError::ManifestValidationError(vec![ + SubgraphManifestValidationError::BlockNotFound(min_start_block.to_string()), + ]) + }), + } +} + +/// Resolves the manifest's graft base block +async fn resolve_graft_block( + base: &Graft, + chain: &impl Blockchain, + logger: &Logger, +) -> Result { + chain + .block_pointer_from_number(logger, base.block) + .await + .map_err(|_| { + SubgraphRegistrarError::ManifestValidationError(vec![ + SubgraphManifestValidationError::BlockNotFound(format!( + "graft base block {} not found", + base.block + )), + ]) + }) +} + +async fn create_subgraph_version( + logger: &Logger, + store: Arc, + chains: Arc, + name: SubgraphName, + deployment: DeploymentHash, + start_block_override: Option, + graft_block_override: Option, + raw: serde_yaml::Mapping, + node_id: NodeId, + debug_fork: Option, + version_switching_mode: SubgraphVersionSwitchingMode, + resolver: &Arc, +) -> Result { + let unvalidated = UnvalidatedSubgraphManifest::::resolve( + deployment, + raw, + &resolver, + &logger, + ENV_VARS.max_spec_version.clone(), + ) + .map_err(SubgraphRegistrarError::ResolveError) + .await?; + + let manifest = unvalidated + .validate(store.cheap_clone(), true) + .await + .map_err(SubgraphRegistrarError::ManifestValidationError)?; + + let network_name = manifest.network_name(); + + let chain = chains + .get::(network_name.clone()) + .map_err(SubgraphRegistrarError::NetworkNotSupported)? + .cheap_clone(); + + let logger = logger.clone(); + let store = store.clone(); + let deployment_store = store.clone(); + + if !store.subgraph_exists(&name)? { + debug!( + logger, + "Subgraph not found, could not create_subgraph_version"; + "subgraph_name" => name.to_string() + ); + return Err(SubgraphRegistrarError::NameNotFound(name.to_string())); + } + + let start_block = match start_block_override { + Some(block) => Some(block), + None => resolve_start_block(&manifest, &*chain, &logger).await?, + }; + + let base_block = match &manifest.graft { + None => None, + Some(graft) => Some(( + graft.base.clone(), + match graft_block_override { + Some(block) => block, + None => resolve_graft_block(&graft, &*chain, &logger).await?, + }, + )), + }; + + info!( + logger, + "Set subgraph start block"; + "block" => format!("{:?}", start_block), + ); + + info!( + logger, + "Graft base"; + "base" => format!("{:?}", base_block.as_ref().map(|(subgraph,_)| subgraph.to_string())), + "block" => format!("{:?}", base_block.as_ref().map(|(_,ptr)| ptr.number)) + ); + + // Apply the subgraph versioning and deployment operations, + // creating a new subgraph deployment if one doesn't exist. + let deployment = DeploymentCreate::new(&manifest, start_block) + .graft(base_block) + .debug(debug_fork); + deployment_store + .create_subgraph_deployment( + name, + &manifest.schema, + deployment, + node_id, + network_name, + version_switching_mode, + ) + .map_err(|e| SubgraphRegistrarError::SubgraphDeploymentError(e)) +} diff --git a/core/src/subgraph/runner.rs b/core/src/subgraph/runner.rs new file mode 100644 index 0000000..9ae886c --- /dev/null +++ b/core/src/subgraph/runner.rs @@ -0,0 +1,1001 @@ +use crate::subgraph::context::IndexingContext; +use crate::subgraph::error::BlockProcessingError; +use crate::subgraph::inputs::IndexingInputs; +use crate::subgraph::state::IndexingState; +use crate::subgraph::stream::new_block_stream; +use atomic_refcell::AtomicRefCell; +use graph::blockchain::block_stream::{BlockStreamEvent, BlockWithTriggers, FirehoseCursor}; +use graph::blockchain::{Block, Blockchain, TriggerFilter as _}; +use graph::components::store::{EmptyStore, EntityKey, StoredDynamicDataSource}; +use graph::components::{ + store::ModificationsAndCache, + subgraph::{CausalityRegion, MappingError, ProofOfIndexing, SharedProofOfIndexing}, +}; +use graph::data::store::scalar::Bytes; +use graph::data::subgraph::{ + schema::{SubgraphError, SubgraphHealth, POI_OBJECT}, + SubgraphFeature, +}; +use graph::data_source::{offchain, DataSource, TriggerData}; +use graph::prelude::*; +use graph::util::{backoff::ExponentialBackoff, lfu_cache::LfuCache}; +use std::convert::TryFrom; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +const MINUTE: Duration = Duration::from_secs(60); + +const SKIP_PTR_UPDATES_THRESHOLD: Duration = Duration::from_secs(60 * 5); + +pub(crate) struct SubgraphRunner +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + ctx: IndexingContext, + state: IndexingState, + inputs: Arc>, + logger: Logger, + metrics: RunnerMetrics, +} + +impl SubgraphRunner +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + pub fn new( + inputs: IndexingInputs, + ctx: IndexingContext, + logger: Logger, + metrics: RunnerMetrics, + ) -> Self { + Self { + inputs: Arc::new(inputs), + ctx, + state: IndexingState { + should_try_unfail_non_deterministic: true, + synced: false, + skip_ptr_updates_timer: Instant::now(), + backoff: ExponentialBackoff::new(MINUTE * 2, ENV_VARS.subgraph_error_retry_ceil), + entity_lfu_cache: LfuCache::new(), + }, + logger, + metrics, + } + } + + pub async fn run(mut self) -> Result<(), Error> { + // If a subgraph failed for deterministic reasons, before start indexing, we first + // revert the deployment head. It should lead to the same result since the error was + // deterministic. + if let Some(current_ptr) = self.inputs.store.block_ptr() { + if let Some(parent_ptr) = self + .inputs + .triggers_adapter + .parent_ptr(¤t_ptr) + .await? + { + // This reverts the deployment head to the parent_ptr if + // deterministic errors happened. + // + // There's no point in calling it if we have no current or parent block + // pointers, because there would be: no block to revert to or to search + // errors from (first execution). + let _outcome = self + .inputs + .store + .unfail_deterministic_error(¤t_ptr, &parent_ptr) + .await?; + } + } + + loop { + debug!(self.logger, "Starting or restarting subgraph"); + + let block_stream_canceler = CancelGuard::new(); + let block_stream_cancel_handle = block_stream_canceler.handle(); + + let mut block_stream = new_block_stream(&self.inputs, &self.ctx.filter) + .await? + .map_err(CancelableError::Error) + .cancelable(&block_stream_canceler, || Err(CancelableError::Cancel)); + + // Keep the stream's cancel guard around to be able to shut it down when the subgraph + // deployment is unassigned + self.ctx + .instances + .write() + .unwrap() + .insert(self.inputs.deployment.id, block_stream_canceler); + + debug!(self.logger, "Starting block stream"); + + // Process events from the stream as long as no restart is needed + loop { + let event = { + let _section = self.metrics.stream.stopwatch.start_section("scan_blocks"); + + block_stream.next().await + }; + + // TODO: move cancel handle to the Context + // This will require some code refactor in how the BlockStream is created + match self + .handle_stream_event(event, &block_stream_cancel_handle) + .await? + { + Action::Continue => continue, + Action::Stop => { + info!(self.logger, "Stopping subgraph"); + self.inputs.store.flush().await?; + return Ok(()); + } + Action::Restart => break, + }; + } + } + } + + /// Processes a block and returns the updated context and a boolean flag indicating + /// whether new dynamic data sources have been added to the subgraph. + async fn process_block( + &mut self, + block_stream_cancel_handle: &CancelHandle, + block: BlockWithTriggers, + firehose_cursor: FirehoseCursor, + ) -> Result { + let triggers = block.trigger_data; + let block = Arc::new(block.block); + let block_ptr = block.ptr(); + + let logger = self.logger.new(o!( + "block_number" => format!("{:?}", block_ptr.number), + "block_hash" => format!("{}", block_ptr.hash) + )); + + if triggers.len() == 1 { + debug!(&logger, "1 candidate trigger in this block"); + } else { + debug!( + &logger, + "{} candidate triggers in this block", + triggers.len() + ); + } + + let proof_of_indexing = if self.inputs.store.supports_proof_of_indexing().await? { + Some(Arc::new(AtomicRefCell::new(ProofOfIndexing::new( + block_ptr.number, + self.inputs.poi_version, + )))) + } else { + None + }; + + // Causality region for onchain triggers. + let causality_region = CausalityRegion::from_network(&self.inputs.network); + + // Process events one after the other, passing in entity operations + // collected previously to every new event being processed + let mut block_state = match self + .process_triggers( + &proof_of_indexing, + &block, + triggers.into_iter().map(TriggerData::Onchain), + &causality_region, + ) + .await + { + // Triggers processed with no errors or with only deterministic errors. + Ok(block_state) => block_state, + + // Some form of unknown or non-deterministic error ocurred. + Err(MappingError::Unknown(e)) => return Err(BlockProcessingError::Unknown(e)), + Err(MappingError::PossibleReorg(e)) => { + info!(logger, + "Possible reorg detected, retrying"; + "error" => format!("{:#}", e), + ); + + // In case of a possible reorg, we want this function to do nothing and restart the + // block stream so it has a chance to detect the reorg. + // + // The state is unchanged at this point, except for having cleared the entity cache. + // Losing the cache is a bit annoying but not an issue for correctness. + // + // See also b21fa73b-6453-4340-99fb-1a78ec62efb1. + return Ok(Action::Restart); + } + }; + + // If new data sources have been created, and static filters are not in use, it is necessary + // to restart the block stream with the new filters. + let needs_restart = block_state.has_created_data_sources() && !self.inputs.static_filters; + + // This loop will: + // 1. Instantiate created data sources. + // 2. Process those data sources for the current block. + // Until no data sources are created or MAX_DATA_SOURCES is hit. + + // Note that this algorithm processes data sources spawned on the same block _breadth + // first_ on the tree implied by the parent-child relationship between data sources. Only a + // very contrived subgraph would be able to observe this. + while block_state.has_created_data_sources() { + // Instantiate dynamic data sources, removing them from the block state. + let (data_sources, runtime_hosts) = + self.create_dynamic_data_sources(block_state.drain_created_data_sources())?; + + let filter = C::TriggerFilter::from_data_sources( + data_sources.iter().filter_map(DataSource::as_onchain), + ); + + // Reprocess the triggers from this block that match the new data sources + let block_with_triggers = self + .inputs + .triggers_adapter + .triggers_in_block(&logger, block.as_ref().clone(), &filter) + .await?; + + let triggers = block_with_triggers.trigger_data; + + if triggers.len() == 1 { + info!( + &logger, + "1 trigger found in this block for the new data sources" + ); + } else if triggers.len() > 1 { + info!( + &logger, + "{} triggers found in this block for the new data sources", + triggers.len() + ); + } + + // Add entity operations for the new data sources to the block state + // and add runtimes for the data sources to the subgraph instance. + self.persist_dynamic_data_sources(&mut block_state.entity_cache, data_sources); + + // Process the triggers in each host in the same order the + // corresponding data sources have been created. + for trigger in triggers { + block_state = self + .ctx + .process_trigger_in_hosts( + &logger, + &runtime_hosts, + &block, + &TriggerData::Onchain(trigger), + block_state, + &proof_of_indexing, + &causality_region, + &self.inputs.debug_fork, + &self.metrics.subgraph, + ) + .await + .map_err(|e| { + // This treats a `PossibleReorg` as an ordinary error which will fail the subgraph. + // This can cause an unnecessary subgraph failure, to fix it we need to figure out a + // way to revert the effect of `create_dynamic_data_sources` so we may return a + // clean context as in b21fa73b-6453-4340-99fb-1a78ec62efb1. + match e { + MappingError::PossibleReorg(e) | MappingError::Unknown(e) => { + BlockProcessingError::Unknown(e) + } + } + })?; + } + } + + let has_errors = block_state.has_errors(); + let is_non_fatal_errors_active = self + .inputs + .features + .contains(&SubgraphFeature::NonFatalErrors); + + // Apply entity operations and advance the stream + + // Avoid writing to store if block stream has been canceled + if block_stream_cancel_handle.is_canceled() { + return Err(BlockProcessingError::Canceled); + } + + if let Some(proof_of_indexing) = proof_of_indexing { + let proof_of_indexing = Arc::try_unwrap(proof_of_indexing).unwrap().into_inner(); + update_proof_of_indexing( + proof_of_indexing, + &self.metrics.host.stopwatch, + &mut block_state.entity_cache, + ) + .await?; + } + + let section = self + .metrics + .host + .stopwatch + .start_section("as_modifications"); + let ModificationsAndCache { + modifications: mut mods, + data_sources, + entity_lfu_cache: cache, + } = block_state + .entity_cache + .as_modifications() + .map_err(|e| BlockProcessingError::Unknown(e.into()))?; + section.end(); + + // Check for offchain events and process them, including their entity modifications in the + // set to be transacted. + let offchain_events = self.ctx.offchain_monitor.ready_offchain_events()?; + let (offchain_mods, offchain_to_remove) = + self.handle_offchain_triggers(offchain_events).await?; + mods.extend(offchain_mods); + + // Put the cache back in the state, asserting that the placeholder cache was not used. + assert!(self.state.entity_lfu_cache.is_empty()); + self.state.entity_lfu_cache = cache; + + if !mods.is_empty() { + info!(&logger, "Applying {} entity operation(s)", mods.len()); + } + + let err_count = block_state.deterministic_errors.len(); + for (i, e) in block_state.deterministic_errors.iter().enumerate() { + let message = format!("{:#}", e).replace("\n", "\t"); + error!(&logger, "Subgraph error {}/{}", i + 1, err_count; + "error" => message, + "code" => LogCode::SubgraphSyncingFailure + ); + } + + // Transact entity operations into the store and update the + // subgraph's block stream pointer + let _section = self.metrics.host.stopwatch.start_section("transact_block"); + let start = Instant::now(); + + let store = &self.inputs.store; + + // If a deterministic error has happened, make the PoI to be the only entity that'll be stored. + if has_errors && !is_non_fatal_errors_active { + let is_poi_entity = + |entity_mod: &EntityModification| entity_mod.entity_ref().entity_type.is_poi(); + mods.retain(is_poi_entity); + // Confidence check + assert!( + mods.len() == 1, + "There should be only one PoI EntityModification" + ); + } + + let BlockState { + deterministic_errors, + .. + } = block_state; + + let first_error = deterministic_errors.first().cloned(); + + store + .transact_block_operations( + block_ptr, + firehose_cursor, + mods, + &self.metrics.host.stopwatch, + data_sources, + deterministic_errors, + self.inputs.manifest_idx_and_name.clone(), + offchain_to_remove, + ) + .await + .context("Failed to transact block operations")?; + + // For subgraphs with `nonFatalErrors` feature disabled, we consider + // any error as fatal. + // + // So we do an early return to make the subgraph stop processing blocks. + // + // In this scenario the only entity that is stored/transacted is the PoI, + // all of the others are discarded. + if has_errors && !is_non_fatal_errors_active { + // Only the first error is reported. + return Err(BlockProcessingError::Deterministic(first_error.unwrap())); + } + + let elapsed = start.elapsed().as_secs_f64(); + self.metrics + .subgraph + .block_ops_transaction_duration + .observe(elapsed); + + // To prevent a buggy pending version from replacing a current version, if errors are + // present the subgraph will be unassigned. + if has_errors && !ENV_VARS.disable_fail_fast && !store.is_deployment_synced().await? { + store + .unassign_subgraph() + .map_err(|e| BlockProcessingError::Unknown(e.into()))?; + + // Use `Canceled` to avoiding setting the subgraph health to failed, an error was + // just transacted so it will be already be set to unhealthy. + return Err(BlockProcessingError::Canceled); + } + + match needs_restart { + true => Ok(Action::Restart), + false => Ok(Action::Continue), + } + } + + async fn process_triggers( + &mut self, + proof_of_indexing: &SharedProofOfIndexing, + block: &Arc, + triggers: impl Iterator>, + causality_region: &str, + ) -> Result, MappingError> { + let mut block_state = BlockState::new( + self.inputs.store.clone(), + std::mem::take(&mut self.state.entity_lfu_cache), + ); + + for trigger in triggers { + block_state = self + .ctx + .process_trigger( + &self.logger, + block, + &trigger, + block_state, + proof_of_indexing, + causality_region, + &self.inputs.debug_fork, + &self.metrics.subgraph, + ) + .await + .map_err(move |mut e| { + let error_context = trigger.error_context(); + if !error_context.is_empty() { + e = e.context(error_context); + } + e.context("failed to process trigger".to_string()) + })?; + } + Ok(block_state) + } + + fn create_dynamic_data_sources( + &mut self, + created_data_sources: Vec>, + ) -> Result<(Vec>, Vec>), Error> { + let mut data_sources = vec![]; + let mut runtime_hosts = vec![]; + + for info in created_data_sources { + // Try to instantiate a data source from the template + let data_source = DataSource::try_from(info)?; + + // Try to create a runtime host for the data source + let host = self + .ctx + .add_dynamic_data_source(&self.logger, data_source.clone())?; + + match host { + Some(host) => { + data_sources.push(data_source); + runtime_hosts.push(host); + } + None => { + warn!( + self.logger, + "no runtime hosted created, there is already a runtime host instantiated for \ + this data source"; + "name" => &data_source.name(), + "address" => &data_source.address() + .map(|address| hex::encode(address)) + .unwrap_or("none".to_string()), + ) + } + } + } + + Ok((data_sources, runtime_hosts)) + } + + fn persist_dynamic_data_sources( + &mut self, + entity_cache: &mut EntityCache, + data_sources: Vec>, + ) { + if !data_sources.is_empty() { + debug!( + self.logger, + "Creating {} dynamic data source(s)", + data_sources.len() + ); + } + + // Add entity operations to the block state in order to persist + // the dynamic data sources + for data_source in data_sources.iter() { + debug!( + self.logger, + "Persisting data_source"; + "name" => &data_source.name(), + "address" => &data_source.address().map(|address| hex::encode(address)).unwrap_or("none".to_string()), + ); + entity_cache.add_data_source(data_source); + } + + // Merge filters from data sources into the block stream builder + self.ctx + .filter + .extend(data_sources.iter().filter_map(|ds| ds.as_onchain())); + } +} + +impl SubgraphRunner +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + async fn handle_stream_event( + &mut self, + event: Option, CancelableError>>, + cancel_handle: &CancelHandle, + ) -> Result { + let action = match event { + Some(Ok(BlockStreamEvent::ProcessBlock(block, cursor))) => { + self.handle_process_block(block, cursor, cancel_handle) + .await? + } + Some(Ok(BlockStreamEvent::Revert(revert_to_ptr, cursor))) => { + self.handle_revert(revert_to_ptr, cursor).await? + } + // Log and drop the errors from the block_stream + // The block stream will continue attempting to produce blocks + Some(Err(e)) => self.handle_err(e, cancel_handle).await?, + // If the block stream ends, that means that there is no more indexing to do. + // Typically block streams produce indefinitely, but tests are an example of finite block streams. + None => Action::Stop, + }; + + Ok(action) + } + + async fn handle_offchain_triggers( + &mut self, + triggers: Vec, + ) -> Result<(Vec, Vec), Error> { + let mut mods = vec![]; + let mut offchain_to_remove = vec![]; + + for trigger in triggers { + // Using an `EmptyStore` and clearing the cache for each trigger is a makeshift way to + // get causality region isolation. + let schema = self.inputs.store.input_schema(); + let mut block_state = BlockState::::new(EmptyStore::new(schema), LfuCache::new()); + + // PoI ignores offchain events. + let proof_of_indexing = None; + let causality_region = ""; + + // We'll eventually need to do better here, but using an empty block works for now. + let block = Arc::default(); + block_state = self + .ctx + .process_trigger( + &self.logger, + &block, + &TriggerData::Offchain(trigger), + block_state, + &proof_of_indexing, + &causality_region, + &self.inputs.debug_fork, + &self.metrics.subgraph, + ) + .await + .map_err(move |err| { + let err = match err { + // Ignoring `PossibleReorg` isn't so bad since the subgraph will retry + // non-deterministic errors. + MappingError::PossibleReorg(e) | MappingError::Unknown(e) => e, + }; + err.context("failed to process trigger".to_string()) + })?; + + anyhow::ensure!( + !block_state.has_created_data_sources(), + "Attempted to create data source in offchain data source handler. This is not yet supported.", + ); + + mods.extend(block_state.entity_cache.as_modifications()?.modifications); + offchain_to_remove.extend(block_state.offchain_to_remove); + } + + Ok((mods, offchain_to_remove)) + } +} + +#[derive(Debug)] +enum Action { + Continue, + Stop, + Restart, +} + +#[async_trait] +trait StreamEventHandler { + async fn handle_process_block( + &mut self, + block: BlockWithTriggers, + cursor: FirehoseCursor, + cancel_handle: &CancelHandle, + ) -> Result; + async fn handle_revert( + &mut self, + revert_to_ptr: BlockPtr, + cursor: FirehoseCursor, + ) -> Result; + async fn handle_err( + &mut self, + err: CancelableError, + cancel_handle: &CancelHandle, + ) -> Result; +} + +#[async_trait] +impl StreamEventHandler for SubgraphRunner +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + async fn handle_process_block( + &mut self, + block: BlockWithTriggers, + cursor: FirehoseCursor, + cancel_handle: &CancelHandle, + ) -> Result { + let block_ptr = block.ptr(); + self.metrics + .stream + .deployment_head + .set(block_ptr.number as f64); + + if block.trigger_count() > 0 { + self.metrics + .subgraph + .block_trigger_count + .observe(block.trigger_count() as f64); + } + + if block.trigger_count() == 0 + && self.state.skip_ptr_updates_timer.elapsed() <= SKIP_PTR_UPDATES_THRESHOLD + && !self.state.synced + && !close_to_chain_head( + &block_ptr, + self.inputs.chain.chain_store().cached_head_ptr().await?, + // The "skip ptr updates timer" is ignored when a subgraph is at most 1000 blocks + // behind the chain head. + 1000, + ) + { + return Ok(Action::Continue); + } else { + self.state.skip_ptr_updates_timer = Instant::now(); + } + + let start = Instant::now(); + + let res = self.process_block(&cancel_handle, block, cursor).await; + + let elapsed = start.elapsed().as_secs_f64(); + self.metrics + .subgraph + .block_processing_duration + .observe(elapsed); + + match res { + Ok(action) => { + // Once synced, no need to try to update the status again. + if !self.state.synced + && close_to_chain_head( + &block_ptr, + self.inputs.chain.chain_store().cached_head_ptr().await?, + // We consider a subgraph synced when it's at most 1 block behind the + // chain head. + 1, + ) + { + // Updating the sync status is an one way operation. + // This state change exists: not synced -> synced + // This state change does NOT: synced -> not synced + self.inputs.store.deployment_synced()?; + + // Stop trying to update the sync status. + self.state.synced = true; + + // Stop recording time-to-sync metrics. + self.metrics.stream.stopwatch.disable(); + } + + // Keep trying to unfail subgraph for everytime it advances block(s) until it's + // health is not Failed anymore. + if self.state.should_try_unfail_non_deterministic { + // If the deployment head advanced, we can unfail + // the non-deterministic error (if there's any). + let outcome = self + .inputs + .store + .unfail_non_deterministic_error(&block_ptr)?; + + if let UnfailOutcome::Unfailed = outcome { + // Stop trying to unfail. + self.state.should_try_unfail_non_deterministic = false; + self.metrics.stream.deployment_failed.set(0.0); + self.state.backoff.reset(); + } + } + + if let Some(stop_block) = &self.inputs.stop_block { + if block_ptr.number >= *stop_block { + info!(self.logger, "stop block reached for subgraph"); + return Ok(Action::Stop); + } + } + + if matches!(action, Action::Restart) { + // Cancel the stream for real + self.ctx + .instances + .write() + .unwrap() + .remove(&self.inputs.deployment.id); + + // And restart the subgraph + return Ok(Action::Restart); + } + + return Ok(Action::Continue); + } + Err(BlockProcessingError::Canceled) => { + debug!(self.logger, "Subgraph block stream shut down cleanly"); + return Ok(Action::Stop); + } + + // Handle unexpected stream errors by marking the subgraph as failed. + Err(e) => { + // Clear entity cache when a subgraph fails. + // + // This is done to be safe and sure that there's no state that's + // out of sync from the database. + // + // Without it, POI changes on failure would be kept in the entity cache + // and be transacted incorrectly in the next run. + self.state.entity_lfu_cache = LfuCache::new(); + + self.metrics.stream.deployment_failed.set(1.0); + + let message = format!("{:#}", e).replace("\n", "\t"); + let err = anyhow!("{}, code: {}", message, LogCode::SubgraphSyncingFailure); + let deterministic = e.is_deterministic(); + + let error = SubgraphError { + subgraph_id: self.inputs.deployment.hash.clone(), + message, + block_ptr: Some(block_ptr), + handler: None, + deterministic, + }; + + match deterministic { + true => { + // Fail subgraph: + // - Change status/health. + // - Save the error to the database. + self.inputs + .store + .fail_subgraph(error) + .await + .context("Failed to set subgraph status to `failed`")?; + + return Err(err); + } + false => { + // Shouldn't fail subgraph if it's already failed for non-deterministic + // reasons. + // + // If we don't do this check we would keep adding the same error to the + // database. + let should_fail_subgraph = + self.inputs.store.health().await? != SubgraphHealth::Failed; + + if should_fail_subgraph { + // Fail subgraph: + // - Change status/health. + // - Save the error to the database. + self.inputs + .store + .fail_subgraph(error) + .await + .context("Failed to set subgraph status to `failed`")?; + } + + // Retry logic below: + + // Cancel the stream for real. + self.ctx + .instances + .write() + .unwrap() + .remove(&self.inputs.deployment.id); + + let message = format!("{:#}", e).replace("\n", "\t"); + error!(self.logger, "Subgraph failed with non-deterministic error: {}", message; + "attempt" => self.state.backoff.attempt, + "retry_delay_s" => self.state.backoff.delay().as_secs()); + + // Sleep before restarting. + self.state.backoff.sleep_async().await; + + self.state.should_try_unfail_non_deterministic = true; + + // And restart the subgraph. + return Ok(Action::Restart); + } + } + } + } + } + + async fn handle_revert( + &mut self, + revert_to_ptr: BlockPtr, + cursor: FirehoseCursor, + ) -> Result { + // Current deployment head in the database / WritableAgent Mutex cache. + // + // Safe unwrap because in a Revert event we're sure the subgraph has + // advanced at least once. + let subgraph_ptr = self.inputs.store.block_ptr().unwrap(); + if revert_to_ptr.number >= subgraph_ptr.number { + info!(&self.logger, "Block to revert is higher than subgraph pointer, nothing to do"; "subgraph_ptr" => &subgraph_ptr, "revert_to_ptr" => &revert_to_ptr); + return Ok(Action::Continue); + } + + info!(&self.logger, "Reverting block to get back to main chain"; "subgraph_ptr" => &subgraph_ptr, "revert_to_ptr" => &revert_to_ptr); + + if let Err(e) = self + .inputs + .store + .revert_block_operations(revert_to_ptr, cursor) + .await + { + error!(&self.logger, "Could not revert block. Retrying"; "error" => %e); + + // Exit inner block stream consumption loop and go up to loop that restarts subgraph + return Ok(Action::Restart); + } + + self.metrics + .stream + .reverted_blocks + .set(subgraph_ptr.number as f64); + self.metrics + .stream + .deployment_head + .set(subgraph_ptr.number as f64); + + // Revert the in-memory state: + // - Revert any dynamic data sources. + // - Clear the entity cache. + self.ctx.revert_data_sources(subgraph_ptr.number); + self.state.entity_lfu_cache = LfuCache::new(); + + Ok(Action::Continue) + } + + async fn handle_err( + &mut self, + err: CancelableError, + cancel_handle: &CancelHandle, + ) -> Result { + if cancel_handle.is_canceled() { + debug!(&self.logger, "Subgraph block stream shut down cleanly"); + return Ok(Action::Stop); + } + + debug!( + &self.logger, + "Block stream produced a non-fatal error"; + "error" => format!("{}", err), + ); + + Ok(Action::Continue) + } +} + +/// Transform the proof of indexing changes into entity updates that will be +/// inserted when as_modifications is called. +async fn update_proof_of_indexing( + proof_of_indexing: ProofOfIndexing, + stopwatch: &StopwatchMetrics, + entity_cache: &mut EntityCache, +) -> Result<(), Error> { + let _section_guard = stopwatch.start_section("update_proof_of_indexing"); + + let mut proof_of_indexing = proof_of_indexing.take(); + + for (causality_region, stream) in proof_of_indexing.drain() { + // Create the special POI entity key specific to this causality_region + let entity_key = EntityKey { + entity_type: POI_OBJECT.to_owned(), + entity_id: causality_region.into(), + }; + + // Grab the current digest attribute on this entity + let prev_poi = + entity_cache + .get(&entity_key) + .map_err(Error::from)? + .map(|entity| match entity.get("digest") { + Some(Value::Bytes(b)) => b.clone(), + _ => panic!("Expected POI entity to have a digest and for it to be bytes"), + }); + + // Finish the POI stream, getting the new POI value. + let updated_proof_of_indexing = stream.pause(prev_poi.as_deref()); + let updated_proof_of_indexing: Bytes = (&updated_proof_of_indexing[..]).into(); + + // Put this onto an entity with the same digest attribute + // that was expected before when reading. + let new_poi_entity = entity! { + id: entity_key.entity_id.to_string(), + digest: updated_proof_of_indexing, + }; + + entity_cache.set(entity_key, new_poi_entity)?; + } + + Ok(()) +} + +/// Checks if the Deployment BlockPtr is at least X blocks behind to the chain head. +fn close_to_chain_head( + deployment_head_ptr: &BlockPtr, + chain_head_ptr: Option, + n: BlockNumber, +) -> bool { + matches!((deployment_head_ptr, &chain_head_ptr), (b1, Some(b2)) if b1.number >= (b2.number - n)) +} + +#[test] +fn test_close_to_chain_head() { + let offset = 1; + + let block_0 = BlockPtr::try_from(( + "bd34884280958002c51d3f7b5f853e6febeba33de0f40d15b0363006533c924f", + 0, + )) + .unwrap(); + let block_1 = BlockPtr::try_from(( + "8511fa04b64657581e3f00e14543c1d522d5d7e771b54aa3060b662ade47da13", + 1, + )) + .unwrap(); + let block_2 = BlockPtr::try_from(( + "b98fb783b49de5652097a989414c767824dff7e7fd765a63b493772511db81c1", + 2, + )) + .unwrap(); + + assert!(!close_to_chain_head(&block_0, None, offset)); + assert!(!close_to_chain_head(&block_2, None, offset)); + + assert!(!close_to_chain_head( + &block_0, + Some(block_2.clone()), + offset + )); + + assert!(close_to_chain_head(&block_1, Some(block_2.clone()), offset)); + assert!(close_to_chain_head(&block_2, Some(block_2.clone()), offset)); +} diff --git a/core/src/subgraph/state.rs b/core/src/subgraph/state.rs new file mode 100644 index 0000000..0d5edd8 --- /dev/null +++ b/core/src/subgraph/state.rs @@ -0,0 +1,22 @@ +use graph::{ + components::store::EntityKey, + prelude::Entity, + util::{backoff::ExponentialBackoff, lfu_cache::LfuCache}, +}; +use std::time::Instant; + +pub struct IndexingState { + /// `true` -> `false` on the first run + pub should_try_unfail_non_deterministic: bool, + /// `false` -> `true` once it reaches chain head + pub synced: bool, + /// Backoff used for the retry mechanism on non-deterministic errors + pub backoff: ExponentialBackoff, + /// Related to field above `backoff` + /// + /// Resets to `Instant::now` every time: + /// - The time THRESHOLD is passed + /// - Or the subgraph has triggers for the block + pub skip_ptr_updates_timer: Instant, + pub entity_lfu_cache: LfuCache>, +} diff --git a/core/src/subgraph/stream.rs b/core/src/subgraph/stream.rs new file mode 100644 index 0000000..9733f0d --- /dev/null +++ b/core/src/subgraph/stream.rs @@ -0,0 +1,46 @@ +use crate::subgraph::inputs::IndexingInputs; +use graph::blockchain::block_stream::{BlockStream, BufferedBlockStream}; +use graph::blockchain::Blockchain; +use graph::prelude::Error; +use std::sync::Arc; + +const BUFFERED_BLOCK_STREAM_SIZE: usize = 100; +const BUFFERED_FIREHOSE_STREAM_SIZE: usize = 1; + +pub async fn new_block_stream( + inputs: &IndexingInputs, + filter: &C::TriggerFilter, +) -> Result>, Error> { + let is_firehose = inputs.chain.is_firehose_supported(); + + let buffer_size = match is_firehose { + true => BUFFERED_FIREHOSE_STREAM_SIZE, + false => BUFFERED_BLOCK_STREAM_SIZE, + }; + + let current_ptr = inputs.store.block_ptr(); + + let block_stream = match is_firehose { + true => inputs.chain.new_firehose_block_stream( + inputs.deployment.clone(), + inputs.store.block_cursor(), + inputs.start_blocks.clone(), + current_ptr, + Arc::new(filter.clone()), + inputs.unified_api_version.clone(), + ), + false => inputs.chain.new_polling_block_stream( + inputs.deployment.clone(), + inputs.start_blocks.clone(), + current_ptr, + Arc::new(filter.clone()), + inputs.unified_api_version.clone(), + ), + } + .await?; + + Ok(BufferedBlockStream::spawn_from_stream( + block_stream, + buffer_size, + )) +} diff --git a/core/src/subgraph/trigger_processor.rs b/core/src/subgraph/trigger_processor.rs new file mode 100644 index 0000000..6122faa --- /dev/null +++ b/core/src/subgraph/trigger_processor.rs @@ -0,0 +1,101 @@ +use async_trait::async_trait; +use graph::blockchain::Blockchain; +use graph::cheap_clone::CheapClone; +use graph::components::store::SubgraphFork; +use graph::components::subgraph::{MappingError, SharedProofOfIndexing}; +use graph::data_source::{MappingTrigger, TriggerData, TriggerWithHandler}; +use graph::prelude::tokio::time::Instant; +use graph::prelude::{ + BlockState, RuntimeHost, RuntimeHostBuilder, SubgraphInstanceMetrics, TriggerProcessor, +}; +use graph::slog::Logger; +use std::sync::Arc; + +pub struct SubgraphTriggerProcessor {} + +#[async_trait] +impl TriggerProcessor for SubgraphTriggerProcessor +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + async fn process_trigger( + &self, + logger: &Logger, + hosts: &[Arc], + block: &Arc, + trigger: &TriggerData, + mut state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &str, + debug_fork: &Option>, + subgraph_metrics: &Arc, + ) -> Result, MappingError> { + let error_count = state.deterministic_errors.len(); + + let mut host_mapping: Vec<(&T::Host, TriggerWithHandler>)> = vec![]; + + { + let _section = subgraph_metrics.stopwatch.start_section("match_and_decode"); + + for host in hosts { + let mapping_trigger = match host.match_and_decode(trigger, block, logger)? { + // Trigger matches and was decoded as a mapping trigger. + Some(mapping_trigger) => mapping_trigger, + + // Trigger does not match, do not process it. + None => continue, + }; + + host_mapping.push((&host, mapping_trigger)); + } + } + + if host_mapping.is_empty() { + return Ok(state); + } + + if let Some(proof_of_indexing) = proof_of_indexing { + proof_of_indexing + .borrow_mut() + .start_handler(causality_region); + } + + for (host, mapping_trigger) in host_mapping { + let start = Instant::now(); + state = host + .process_mapping_trigger( + logger, + mapping_trigger.block_ptr(), + mapping_trigger, + state, + proof_of_indexing.cheap_clone(), + debug_fork, + ) + .await?; + let elapsed = start.elapsed().as_secs_f64(); + subgraph_metrics.observe_trigger_processing_duration(elapsed); + + if host.data_source().as_offchain().is_some() { + // Remove this offchain data source since it has just been processed. + state + .offchain_to_remove + .push(host.data_source().as_stored_dynamic_data_source()); + } + } + + if let Some(proof_of_indexing) = proof_of_indexing { + if state.deterministic_errors.len() != error_count { + assert!(state.deterministic_errors.len() == error_count + 1); + + // If a deterministic error has happened, write a new + // ProofOfIndexingEvent::DeterministicError to the SharedProofOfIndexing. + proof_of_indexing + .borrow_mut() + .write_deterministic_error(&logger, causality_region); + } + } + + Ok(state) + } +} diff --git a/core/tests/interfaces.rs b/core/tests/interfaces.rs new file mode 100644 index 0000000..45f8109 --- /dev/null +++ b/core/tests/interfaces.rs @@ -0,0 +1,1514 @@ +// Tests for graphql interfaces. + +use pretty_assertions::assert_eq; + +use graph::{components::store::EntityType, data::graphql::object}; +use graph::{data::query::QueryTarget, prelude::*}; +use test_store::*; + +// `entities` is `(entity, type)`. +async fn insert_and_query( + subgraph_id: &str, + schema: &str, + entities: Vec<(&str, Entity)>, + query: &str, +) -> Result { + let subgraph_id = DeploymentHash::new(subgraph_id).unwrap(); + let deployment = create_test_subgraph(&subgraph_id, schema).await; + + let entities = entities + .into_iter() + .map(|(entity_type, data)| (EntityType::new(entity_type.to_owned()), data)) + .collect(); + + insert_entities(&deployment, entities).await?; + + let document = graphql_parser::parse_query(query).unwrap().into_static(); + let target = QueryTarget::Deployment(subgraph_id, Default::default()); + let query = Query::new(document, None); + Ok(execute_subgraph_query(query, target) + .await + .first() + .unwrap() + .duplicate()) +} + +/// Extract the data from a `QueryResult`, and panic if it has errors +macro_rules! extract_data { + ($result: expr) => { + match $result.to_result() { + Err(errors) => panic!("Unexpected errors return for query: {:#?}", errors), + Ok(data) => data, + } + }; +} + +#[tokio::test] +async fn one_interface_zero_entities() { + let subgraph_id = "oneInterfaceZeroEntities"; + let schema = "interface Legged { legs: Int } + type Animal implements Legged @entity { id: ID!, legs: Int }"; + + let query = "query { leggeds(first: 100) { legs } }"; + + let res = insert_and_query(subgraph_id, schema, vec![], query) + .await + .unwrap(); + + let data = extract_data!(res).unwrap(); + let exp = object! { leggeds: Vec::::new() }; + assert_eq!(data, exp); +} + +#[tokio::test] +async fn one_interface_one_entity() { + let subgraph_id = "oneInterfaceOneEntity"; + let schema = "interface Legged { legs: Int } + type Animal implements Legged @entity { id: ID!, legs: Int }"; + + let entity = ( + "Animal", + Entity::from(vec![("id", Value::from("1")), ("legs", Value::from(3))]), + ); + + // Collection query. + let query = "query { leggeds(first: 100) { legs } }"; + let res = insert_and_query(subgraph_id, schema, vec![entity], query) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + let exp = object! { leggeds: vec![ object!{ legs: 3 }]}; + assert_eq!(data, exp); + + // Query by ID. + let query = "query { legged(id: \"1\") { legs } }"; + let res = insert_and_query(subgraph_id, schema, vec![], query) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + let exp = object! { legged: object! { legs: 3 }}; + assert_eq!(data, exp); +} + +#[tokio::test] +async fn one_interface_one_entity_typename() { + let subgraph_id = "oneInterfaceOneEntityTypename"; + let schema = "interface Legged { legs: Int } + type Animal implements Legged @entity { id: ID!, legs: Int }"; + + let entity = ( + "Animal", + Entity::from(vec![("id", Value::from("1")), ("legs", Value::from(3))]), + ); + + let query = "query { leggeds(first: 100) { __typename } }"; + + let res = insert_and_query(subgraph_id, schema, vec![entity], query) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + let exp = object! { leggeds: vec![ object!{ __typename: "Animal" } ]}; + assert_eq!(data, exp); +} + +#[tokio::test] +async fn one_interface_multiple_entities() { + let subgraph_id = "oneInterfaceMultipleEntities"; + let schema = "interface Legged { legs: Int } + type Animal implements Legged @entity { id: ID!, legs: Int } + type Furniture implements Legged @entity { id: ID!, legs: Int } + "; + + let animal = ( + "Animal", + Entity::from(vec![("id", Value::from("1")), ("legs", Value::from(3))]), + ); + let furniture = ( + "Furniture", + Entity::from(vec![("id", Value::from("2")), ("legs", Value::from(4))]), + ); + + let query = "query { leggeds(first: 100, orderBy: legs) { legs } }"; + + let res = insert_and_query(subgraph_id, schema, vec![animal, furniture], query) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + let exp = object! { leggeds: vec![ object! { legs: 3 }, object! { legs: 4 }]}; + assert_eq!(data, exp); + + // Test for support issue #32. + let query = "query { legged(id: \"2\") { legs } }"; + let res = insert_and_query(subgraph_id, schema, vec![], query) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + let exp = object! { legged: object! { legs: 4 }}; + assert_eq!(data, exp); +} + +#[tokio::test] +async fn reference_interface() { + let subgraph_id = "ReferenceInterface"; + let schema = "type Leg @entity { id: ID! } + interface Legged { leg: Leg } + type Animal implements Legged @entity { id: ID!, leg: Leg }"; + + let query = "query { leggeds(first: 100) { leg { id } } }"; + + let leg = ("Leg", Entity::from(vec![("id", Value::from("1"))])); + let animal = ( + "Animal", + Entity::from(vec![("id", Value::from("1")), ("leg", Value::from("1"))]), + ); + + let res = insert_and_query(subgraph_id, schema, vec![leg, animal], query) + .await + .unwrap(); + + let data = extract_data!(res).unwrap(); + let exp = object! { leggeds: vec![ object!{ leg: object! { id: "1" } }] }; + assert_eq!(data, exp); +} + +#[tokio::test] +async fn reference_interface_derived() { + // Test the different ways in which interface implementations + // can reference another entity + let subgraph_id = "ReferenceInterfaceDerived"; + let schema = " + type Transaction @entity { + id: ID!, + buyEvent: BuyEvent!, + sellEvents: [SellEvent!]!, + giftEvent: [GiftEvent!]! @derivedFrom(field: \"transaction\"), + } + + interface Event { + id: ID!, + transaction: Transaction! + } + + type BuyEvent implements Event @entity { + id: ID!, + # Derived, but only one buyEvent per Transaction + transaction: Transaction! @derivedFrom(field: \"buyEvent\") + } + + type SellEvent implements Event @entity { + id: ID! + # Derived, many sellEvents per Transaction + transaction: Transaction! @derivedFrom(field: \"sellEvents\") + } + + type GiftEvent implements Event @entity { + id: ID!, + # Store the transaction directly + transaction: Transaction! + }"; + + let query = "query { events { id transaction { id } } }"; + + let buy = ("BuyEvent", Entity::from(vec![("id", "buy".into())])); + let sell1 = ("SellEvent", Entity::from(vec![("id", "sell1".into())])); + let sell2 = ("SellEvent", Entity::from(vec![("id", "sell2".into())])); + let gift = ( + "GiftEvent", + Entity::from(vec![("id", "gift".into()), ("transaction", "txn".into())]), + ); + let txn = ( + "Transaction", + Entity::from(vec![ + ("id", "txn".into()), + ("buyEvent", "buy".into()), + ("sellEvents", vec!["sell1", "sell2"].into()), + ]), + ); + + let entities = vec![buy, sell1, sell2, gift, txn]; + let res = insert_and_query(subgraph_id, schema, entities.clone(), query) + .await + .unwrap(); + + let data = extract_data!(res).unwrap(); + let exp = object! { + events: vec![ + object! { id: "buy", transaction: object! { id: "txn" } }, + object! { id: "gift", transaction: object! { id: "txn" } }, + object! { id: "sell1", transaction: object! { id: "txn" } }, + object! { id: "sell2", transaction: object! { id: "txn" } } + ] + }; + assert_eq!(data, exp); +} + +#[tokio::test] +async fn follow_interface_reference_invalid() { + let subgraph_id = "FollowInterfaceReferenceInvalid"; + let schema = "interface Legged { legs: Int! } + type Animal implements Legged @entity { + id: ID! + legs: Int! + parent: Legged + }"; + + let query = "query { legged(id: \"child\") { parent { id } } }"; + + let res = insert_and_query(subgraph_id, schema, vec![], query) + .await + .unwrap(); + + // Depending on whether `ENABLE_GRAPHQL_VALIDATIONS` is set or not, we + // get different errors + match &res.to_result().unwrap_err()[0] { + QueryError::ExecutionError(QueryExecutionError::ValidationError(_, error_message)) => { + assert_eq!( + error_message, + "Cannot query field \"parent\" on type \"Legged\"." + ); + } + QueryError::ExecutionError(QueryExecutionError::UnknownField(_, type_name, field_name)) => { + assert_eq!(type_name, "Legged"); + assert_eq!(field_name, "parent"); + } + e => panic!("error `{}` is not the expected one", e), + } +} + +#[tokio::test] +async fn follow_interface_reference() { + let subgraph_id = "FollowInterfaceReference"; + let schema = "interface Legged { id: ID!, legs: Int! } + type Animal implements Legged @entity { + id: ID! + legs: Int! + parent: Legged + }"; + + let query = "query { legged(id: \"child\") { ... on Animal { parent { id } } } }"; + + let parent = ( + "Animal", + Entity::from(vec![ + ("id", Value::from("parent")), + ("legs", Value::from(4)), + ("parent", Value::Null), + ]), + ); + let child = ( + "Animal", + Entity::from(vec![ + ("id", Value::from("child")), + ("legs", Value::from(3)), + ("parent", Value::String("parent".into())), + ]), + ); + + let res = insert_and_query(subgraph_id, schema, vec![parent, child], query) + .await + .unwrap(); + + let data = extract_data!(res).unwrap(); + let exp = object! { + legged: object! { parent: object! { id: "parent" } } + }; + assert_eq!(data, exp) +} + +#[tokio::test] +async fn conflicting_implementors_id() { + let subgraph_id = "ConflictingImplementorsId"; + let schema = "interface Legged { legs: Int } + type Animal implements Legged @entity { id: ID!, legs: Int } + type Furniture implements Legged @entity { id: ID!, legs: Int } + "; + + let animal = ( + "Animal", + Entity::from(vec![("id", Value::from("1")), ("legs", Value::from(3))]), + ); + let furniture = ( + "Furniture", + Entity::from(vec![("id", Value::from("1")), ("legs", Value::from(3))]), + ); + + let query = "query { leggeds(first: 100) { legs } }"; + + let res = insert_and_query(subgraph_id, schema, vec![animal, furniture], query).await; + + let msg = res.unwrap_err().to_string(); + // We don't know in which order the two entities get inserted; the two + // error messages only differ in who gets inserted first + const EXPECTED1: &str = + "tried to set entity of type `Furniture` with ID \"1\" but an entity of type `Animal`, \ + which has an interface in common with `Furniture`, exists with the same ID"; + const EXPECTED2: &str = + "tried to set entity of type `Animal` with ID \"1\" but an entity of type `Furniture`, \ + which has an interface in common with `Animal`, exists with the same ID"; + + assert!(msg == EXPECTED1 || msg == EXPECTED2); +} + +#[tokio::test] +async fn derived_interface_relationship() { + let subgraph_id = "DerivedInterfaceRelationship"; + let schema = "interface ForestDweller { id: ID!, forest: Forest } + type Animal implements ForestDweller @entity { id: ID!, forest: Forest } + type Forest @entity { id: ID!, dwellers: [ForestDweller]! @derivedFrom(field: \"forest\") } + "; + + let forest = ("Forest", Entity::from(vec![("id", Value::from("1"))])); + let animal = ( + "Animal", + Entity::from(vec![("id", Value::from("1")), ("forest", Value::from("1"))]), + ); + + let query = "query { forests(first: 100) { dwellers(first: 100) { id } } }"; + + let res = insert_and_query(subgraph_id, schema, vec![forest, animal], query) + .await + .unwrap(); + let data = extract_data!(res); + assert_eq!( + data.unwrap().to_string(), + "{forests: [{dwellers: [{id: \"1\"}]}]}" + ); +} + +#[tokio::test] +async fn two_interfaces() { + let subgraph_id = "TwoInterfaces"; + let schema = "interface IFoo { foo: String! } + interface IBar { bar: Int! } + + type A implements IFoo @entity { id: ID!, foo: String! } + type B implements IBar @entity { id: ID!, bar: Int! } + + type AB implements IFoo & IBar @entity { id: ID!, foo: String!, bar: Int! } + "; + + let a = ( + "A", + Entity::from(vec![("id", Value::from("1")), ("foo", Value::from("bla"))]), + ); + let b = ( + "B", + Entity::from(vec![("id", Value::from("1")), ("bar", Value::from(100))]), + ); + let ab = ( + "AB", + Entity::from(vec![ + ("id", Value::from("2")), + ("foo", Value::from("ble")), + ("bar", Value::from(200)), + ]), + ); + + let query = "query { + ibars(first: 100, orderBy: bar) { bar } + ifoos(first: 100, orderBy: foo) { foo } + }"; + let res = insert_and_query(subgraph_id, schema, vec![a, b, ab], query) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + let exp = object! { + ibars: vec![ object! { bar: 100 }, object! { bar: 200 }], + ifoos: vec![ object! { foo: "bla" }, object! { foo: "ble" } ] + }; + assert_eq!(data, exp); +} + +#[tokio::test] +async fn interface_non_inline_fragment() { + let subgraph_id = "interfaceNonInlineFragment"; + let schema = "interface Legged { legs: Int } + type Animal implements Legged @entity { id: ID!, name: String, legs: Int }"; + + let entity = ( + "Animal", + Entity::from(vec![ + ("id", Value::from("1")), + ("name", Value::from("cow")), + ("legs", Value::from(3)), + ]), + ); + + // Query only the fragment. + let query = "query { leggeds { ...frag } } fragment frag on Animal { name }"; + let res = insert_and_query(subgraph_id, schema, vec![entity], query) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + let exp = object! { leggeds: vec![ object! { name: "cow" } ]}; + assert_eq!(data, exp); + + // Query the fragment and something else. + let query = "query { leggeds { legs, ...frag } } fragment frag on Animal { name }"; + let res = insert_and_query(subgraph_id, schema, vec![], query) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + let exp = object! { leggeds: vec![ object!{ legs: 3, name: "cow" } ]}; + assert_eq!(data, exp); +} + +#[tokio::test] +async fn interface_inline_fragment() { + let subgraph_id = "interfaceInlineFragment"; + let schema = "interface Legged { legs: Int } + type Animal implements Legged @entity { id: ID!, name: String, legs: Int } + type Bird implements Legged @entity { id: ID!, airspeed: Int, legs: Int }"; + + let animal = ( + "Animal", + Entity::from(vec![ + ("id", Value::from("1")), + ("name", Value::from("cow")), + ("legs", Value::from(4)), + ]), + ); + let bird = ( + "Bird", + Entity::from(vec![ + ("id", Value::from("2")), + ("airspeed", Value::from(24)), + ("legs", Value::from(2)), + ]), + ); + + let query = + "query { leggeds(orderBy: legs) { ... on Animal { name } ...on Bird { airspeed } } }"; + let res = insert_and_query(subgraph_id, schema, vec![animal, bird], query) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + let exp = object! { leggeds: vec![ object!{ airspeed: 24 }, object! { name: "cow" }]}; + assert_eq!(data, exp); +} + +#[tokio::test] +async fn interface_inline_fragment_with_subquery() { + let subgraph_id = "InterfaceInlineFragmentWithSubquery"; + let schema = " + interface Legged { legs: Int } + type Parent @entity { + id: ID! + } + type Animal implements Legged @entity { + id: ID! + name: String + legs: Int + parent: Parent + } + type Bird implements Legged @entity { + id: ID! + airspeed: Int + legs: Int + parent: Parent + } + "; + + let mama_cow = ( + "Parent", + Entity::from(vec![("id", Value::from("mama_cow"))]), + ); + let cow = ( + "Animal", + Entity::from(vec![ + ("id", Value::from("1")), + ("name", Value::from("cow")), + ("legs", Value::from(4)), + ("parent", Value::from("mama_cow")), + ]), + ); + + let mama_bird = ( + "Parent", + Entity::from(vec![("id", Value::from("mama_bird"))]), + ); + let bird = ( + "Bird", + Entity::from(vec![ + ("id", Value::from("2")), + ("airspeed", Value::from(5)), + ("legs", Value::from(2)), + ("parent", Value::from("mama_bird")), + ]), + ); + + let query = "query { leggeds(orderBy: legs) { legs ... on Bird { airspeed parent { id } } } }"; + let res = insert_and_query( + subgraph_id, + schema, + vec![cow, mama_cow, bird, mama_bird], + query, + ) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + let exp = object! { + leggeds: vec![ object!{ legs: 2, airspeed: 5, parent: object! { id: "mama_bird" } }, + object!{ legs: 4 }] + }; + assert_eq!(data, exp); +} + +#[tokio::test] +async fn invalid_fragment() { + let subgraph_id = "InvalidFragment"; + let schema = "interface Legged { legs: Int! } + type Animal implements Legged @entity { + id: ID! + name: String! + legs: Int! + parent: Legged + }"; + + let query = "query { legged(id: \"child\") { ...{ name } } }"; + + let res = insert_and_query(subgraph_id, schema, vec![], query) + .await + .unwrap(); + + match &res.to_result().unwrap_err()[0] { + QueryError::ExecutionError(QueryExecutionError::ValidationError(_, error_message)) => { + assert_eq!( + error_message, + "Cannot query field \"name\" on type \"Legged\"." + ); + } + QueryError::ExecutionError(QueryExecutionError::UnknownField(_, type_name, field_name)) => { + assert_eq!(type_name, "Legged"); + assert_eq!(field_name, "name"); + } + e => panic!("error {} is not the expected one", e), + } +} + +#[tokio::test] +async fn alias() { + let subgraph_id = "Alias"; + let schema = "interface Legged { id: ID!, legs: Int! } + type Animal implements Legged @entity { + id: ID! + legs: Int! + parent: Legged + }"; + + let query = "query { + l: legged(id: \"child\") { + ... on Animal { + p: parent { + i: id, + t: __typename, + __typename + } + } + } + }"; + + let parent = ( + "Animal", + Entity::from(vec![ + ("id", Value::from("parent")), + ("legs", Value::from(4)), + ("parent", Value::Null), + ]), + ); + let child = ( + "Animal", + Entity::from(vec![ + ("id", Value::from("child")), + ("legs", Value::from(3)), + ("parent", Value::String("parent".into())), + ]), + ); + + let res = insert_and_query(subgraph_id, schema, vec![parent, child], query) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + assert_eq!( + data, + object! { + l: object! { + p: object! { + i: "parent", + t: "Animal", + __typename: "Animal" + } + } + } + ) +} + +#[tokio::test] +async fn fragments_dont_panic() { + let subgraph_id = "FragmentsDontPanic"; + let schema = " + type Parent @entity { + id: ID! + child: Child + } + + type Child @entity { + id: ID! + } + "; + + let query = " + query { + parents { + ...on Parent { + child { + id + } + } + ...Frag + child { + id + } + } + } + + fragment Frag on Parent { + child { + id + } + } + "; + + // The panic manifests if two parents exist. + let parent = ( + "Parent", + entity!( + id: "p", + child: "c", + ), + ); + let parent2 = ( + "Parent", + entity!( + id: "p2", + child: Value::Null, + ), + ); + let child = ( + "Child", + entity!( + id:"c" + ), + ); + + let res = insert_and_query(subgraph_id, schema, vec![parent, parent2, child], query) + .await + .unwrap(); + + let data = extract_data!(res).unwrap(); + assert_eq!( + data, + object! { + parents: vec![ + object! { + child: object! { + id: "c", + } + }, + object! { + child: r::Value::Null + } + ] + } + ) +} + +// See issue #1816 +#[tokio::test] +async fn fragments_dont_duplicate_data() { + let subgraph_id = "FragmentsDupe"; + let schema = " + type Parent @entity { + id: ID! + children: [Child!]! + } + + type Child @entity { + id: ID! + } + "; + + let query = " + query { + parents { + ...Frag + children { + id + } + } + } + + fragment Frag on Parent { + children { + id + } + } + "; + + // This bug manifests if two parents exist. + let parent = ( + "Parent", + entity!( + id: "p", + children: vec!["c"] + ), + ); + let parent2 = ( + "Parent", + entity!( + id: "b", + children: Vec::::new() + ), + ); + let child = ( + "Child", + entity!( + id:"c" + ), + ); + + let res = insert_and_query(subgraph_id, schema, vec![parent, parent2, child], query) + .await + .unwrap(); + + let data = extract_data!(res).unwrap(); + assert_eq!( + data, + object! { + parents: vec![ + object! { + children: Vec::::new() + }, + object! { + children: vec![ + object! { + id: "c", + } + ] + } + ] + } + ) +} + +// See also: e0d6da3e-60cf-41a5-b83c-b60a7a766d4a +#[tokio::test] +async fn redundant_fields() { + let subgraph_id = "RedundantFields"; + let schema = "interface Legged { id: ID!, parent: Legged } + type Animal implements Legged @entity { + id: ID! + parent: Legged + }"; + + let query = "query { + leggeds { + parent { id } + ...on Animal { + parent { id } + } + } + }"; + + let parent = ( + "Animal", + entity!( + id: "parent", + parent: Value::Null, + ), + ); + let child = ( + "Animal", + entity!( + id: "child", + parent: "parent", + ), + ); + + let res = insert_and_query(subgraph_id, schema, vec![parent, child], query) + .await + .unwrap(); + + let data = extract_data!(res).unwrap(); + assert_eq!( + data, + object! { + leggeds: vec![ + object! { + parent: object! { + id: "parent", + }, + }, + object! { + parent: r::Value::Null + } + ] + } + ) +} + +#[tokio::test] +async fn fragments_merge_selections() { + let subgraph_id = "FragmentsMergeSelections"; + let schema = " + type Parent @entity { + id: ID! + children: [Child!]! + } + + type Child @entity { + id: ID! + foo: Int! + } + "; + + let query = " + query { + parents { + ...Frag + children { + id + } + } + } + + fragment Frag on Parent { + children { + foo + } + } + "; + + let parent = ( + "Parent", + entity!( + id: "p", + children: vec!["c"] + ), + ); + let child = ( + "Child", + entity!( + id: "c", + foo: 1, + ), + ); + + let res = insert_and_query(subgraph_id, schema, vec![parent, child], query) + .await + .unwrap(); + + let data = extract_data!(res).unwrap(); + assert_eq!( + data, + object! { + parents: vec![ + object! { + children: vec![ + object! { + foo: 1, + id: "c", + } + ] + } + ] + } + ) +} + +#[tokio::test] +async fn merge_fields_not_in_interface() { + let subgraph_id = "MergeFieldsNotInInterface"; + let schema = "interface Iface { id: ID! } + type Animal implements Iface @entity { + id: ID! + human: Iface! + } + type Human implements Iface @entity { + id: ID! + animal: Iface! + } + "; + + let query = "query { + ifaces { + ...on Animal { + id + friend: human { + id + } + } + ...on Human { + id + friend: animal { + id + } + } + } + }"; + + let animal = ( + "Animal", + entity!( + id: "cow", + human: "fred", + ), + ); + let human = ( + "Human", + entity!( + id: "fred", + animal: "cow", + ), + ); + + let res = insert_and_query(subgraph_id, schema, vec![animal, human], query) + .await + .unwrap(); + + let data = extract_data!(res).unwrap(); + assert_eq!( + data, + object! { + ifaces: vec![ + object! { + id: "cow", + friend: object! { + id: "fred", + }, + }, + object! { + id: "fred", + friend: object! { + id: "cow", + }, + }, + ] + } + ) +} + +#[tokio::test] +async fn nested_interface_fragments() { + let subgraph_id = "NestedInterfaceFragments"; + let schema = "interface I1face { id: ID!, foo1: Foo! } + interface I2face { id: ID!, foo2: Foo! } + interface I3face { id: ID!, foo3: Foo! } + type Foo @entity { + id: ID! + } + type One implements I1face @entity { + id: ID! + foo1: Foo! + } + type Two implements I1face & I2face @entity { + id: ID! + foo1: Foo! + foo2: Foo! + } + type Three implements I1face & I2face & I3face @entity { + id: ID! + foo1: Foo! + foo2: Foo! + foo3: Foo! + }"; + + let query = "query { + i1Faces { + __typename + foo1 { + id + } + ...on I2face { + foo2 { + id + } + } + ...on I3face { + foo3 { + id + } + } + } + }"; + + let foo = ( + "Foo", + entity!( + id: "foo", + ), + ); + let one = ( + "One", + entity!( + id: "1", + foo1: "foo", + ), + ); + let two = ( + "Two", + entity!( + id: "2", + foo1: "foo", + foo2: "foo", + ), + ); + let three = ( + "Three", + entity!( + id: "3", + foo1: "foo", + foo2: "foo", + foo3: "foo" + ), + ); + + let res = insert_and_query(subgraph_id, schema, vec![foo, one, two, three], query) + .await + .unwrap(); + + let data = extract_data!(res).unwrap(); + assert_eq!( + data, + object! { + i1Faces: vec![ + object! { + __typename: "One", + foo1: object! { + id: "foo", + }, + }, + object! { + __typename: "Two", + foo1: object! { + id: "foo", + }, + foo2: object! { + id: "foo", + }, + }, + object! { + __typename: "Three", + foo1: object! { + id: "foo", + }, + foo2: object! { + id: "foo", + }, + foo3: object! { + id: "foo", + }, + }, + ] + } + ) +} + +#[tokio::test] +async fn nested_interface_fragments_overlapping() { + let subgraph_id = "NestedInterfaceFragmentsOverlapping"; + let schema = "interface I1face { id: ID!, foo1: Foo! } + interface I2face { id: ID!, foo1: Foo! } + type Foo @entity { + id: ID! + } + type One implements I1face @entity { + id: ID! + foo1: Foo! + } + type Two implements I1face & I2face @entity { + id: ID! + foo1: Foo! + }"; + + let query = "query { + i1Faces { + __typename + ...on I2face { + foo1 { + id + } + } + } + }"; + + let foo = ( + "Foo", + entity!( + id: "foo", + ), + ); + let one = ( + "One", + entity!( + id: "1", + foo1: "foo", + ), + ); + let two = ( + "Two", + entity!( + id: "2", + foo1: "foo", + ), + ); + let res = insert_and_query(subgraph_id, schema, vec![foo, one, two], query) + .await + .unwrap(); + + let data = extract_data!(res).unwrap(); + assert_eq!( + data, + object! { + i1Faces: vec![ + object! { + __typename: "One" + }, + object! { + __typename: "Two", + foo1: object! { + id: "foo", + }, + }, + ] + } + ); + + let query = "query { + i1Faces { + __typename + foo1 { + id + } + ...on I2face { + foo1 { + id + } + } + } + }"; + + let res = insert_and_query(subgraph_id, schema, vec![], query) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + assert_eq!( + data, + object! { + i1Faces: vec![ + object! { + __typename: "One", + foo1: object! { + id: "foo" + } + }, + object! { + __typename: "Two", + foo1: object! { + id: "foo", + }, + }, + ] + } + ); +} + +#[tokio::test] +async fn enums() { + use r::Value::Enum; + let subgraph_id = "enums"; + let schema = r#" + enum Direction { + NORTH + EAST + SOUTH + WEST + } + + type Trajectory @entity { + id: ID! + direction: Direction! + meters: Int! + }"#; + + let entities = vec![ + ( + "Trajectory", + Entity::from(vec![ + ("id", Value::from("1")), + ("direction", Value::from("EAST")), + ("meters", Value::from(10)), + ]), + ), + ( + "Trajectory", + Entity::from(vec![ + ("id", Value::from("2")), + ("direction", Value::from("NORTH")), + ("meters", Value::from(15)), + ]), + ), + ]; + let query = "query { trajectories { id, direction, meters } }"; + + let res = insert_and_query(subgraph_id, schema, entities, query) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + assert_eq!( + data, + object! { + trajectories: vec![ + object!{ + id: "1", + direction: Enum("EAST".to_string()), + meters: 10, + }, + object!{ + id: "2", + direction: Enum("NORTH".to_string()), + meters: 15, + }, + ]} + ); +} + +#[tokio::test] +async fn enum_list_filters() { + use r::Value::Enum; + let subgraph_id = "enum_list_filters"; + let schema = r#" + enum Direction { + NORTH + EAST + SOUTH + WEST + } + + type Trajectory @entity { + id: ID! + direction: Direction! + meters: Int! + }"#; + + let entities = vec![ + ( + "Trajectory", + Entity::from(vec![ + ("id", Value::from("1")), + ("direction", Value::from("EAST")), + ("meters", Value::from(10)), + ]), + ), + ( + "Trajectory", + Entity::from(vec![ + ("id", Value::from("2")), + ("direction", Value::from("NORTH")), + ("meters", Value::from(15)), + ]), + ), + ( + "Trajectory", + Entity::from(vec![ + ("id", Value::from("3")), + ("direction", Value::from("WEST")), + ("meters", Value::from(20)), + ]), + ), + ]; + + let query = "query { trajectories(where: { direction_in: [NORTH, EAST] }) { id, direction } }"; + let res = insert_and_query(subgraph_id, schema, entities, query) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + assert_eq!( + data, + object! { + trajectories: vec![ + object!{ + id: "1", + direction: Enum("EAST".to_string()), + }, + object!{ + id: "2", + direction: Enum("NORTH".to_string()), + }, + ]} + ); + + let query = "query { trajectories(where: { direction_not_in: [EAST] }) { id, direction } }"; + let res = insert_and_query(subgraph_id, schema, vec![], query) + .await + .unwrap(); + let data = extract_data!(res).unwrap(); + assert_eq!( + data, + object! { + trajectories: vec![ + object!{ + id: "2", + direction: Enum("NORTH".to_string()), + }, + object!{ + id: "3", + direction: Enum("WEST".to_string()), + }, + ]} + ); +} + +#[tokio::test] +async fn recursive_fragment() { + // Depending on whether `ENABLE_GRAPHQL_VALIDATIONS` is set or not, we + // get different error messages + const FOO_ERRORS: [&str; 2] = [ + "Cannot spread fragment \"FooFrag\" within itself.", + "query has fragment cycle including `FooFrag`", + ]; + const FOO_BAR_ERRORS: [&str; 2] = [ + "Cannot spread fragment \"BarFrag\" within itself via \"FooFrag\".", + "query has fragment cycle including `BarFrag`", + ]; + let subgraph_id = "RecursiveFragment"; + let schema = " + type Foo @entity { + id: ID! + foo: Foo! + bar: Bar! + } + + type Bar @entity { + id: ID! + foo: Foo! + } + "; + + let self_recursive = " + query { + foos { + ...FooFrag + } + } + + fragment FooFrag on Foo { + id + foo { + ...FooFrag + } + } + "; + let res = insert_and_query(subgraph_id, schema, vec![], self_recursive) + .await + .unwrap(); + let data = res.to_result().unwrap_err()[0].to_string(); + assert!(FOO_ERRORS.contains(&data.as_str())); + + let co_recursive = " + query { + foos { + ...BarFrag + } + } + + fragment BarFrag on Bar { + id + foo { + ...FooFrag + } + } + + fragment FooFrag on Foo { + id + bar { + ...BarFrag + } + } + "; + let res = insert_and_query(subgraph_id, schema, vec![], co_recursive) + .await + .unwrap(); + let data = res.to_result().unwrap_err()[0].to_string(); + assert!(FOO_BAR_ERRORS.contains(&data.as_str())); +} + +#[tokio::test] +async fn mixed_mutability() { + let subgraph_id = "MixedMutability"; + let schema = "interface Event { id: String! } + type Mutable implements Event @entity { id: String!, name: String! } + type Immutable implements Event @entity(immutable: true) { id: String!, name: String! }"; + + let query = "query { events { id } }"; + + let entities = vec![ + ("Mutable", entity! { id: "mut0", name: "mut0" }), + ("Immutable", entity! { id: "immo0", name: "immo0" }), + ]; + + { + // We need to remove the subgraph since it contains immutable + // entities, and the trick the other tests use does not work for + // this. They rely on the EntityCache filtering out entity changes + // that are already in the store + let id = DeploymentHash::new(subgraph_id).unwrap(); + remove_subgraph(&id); + } + let res = insert_and_query(subgraph_id, schema, entities, query) + .await + .unwrap(); + + let data = extract_data!(res).unwrap(); + let exp = object! { events: vec![ object!{ id: "immo0" }, object! { id: "mut0" } ] }; + assert_eq!(data, exp); +} + +#[tokio::test] +async fn derived_interface_bytes() { + let subgraph_id = "DerivedInterfaceBytes"; + let schema = r#" type Pool { + id: Bytes!, + trades: [Trade!]! @derivedFrom(field: "pool") + } + + interface Trade { + id: Bytes! + pool: Pool! + } + + type Sell implements Trade @entity { + id: Bytes! + pool: Pool! + } + type Buy implements Trade @entity { + id: Bytes! + pool: Pool! + }"#; + + let query = "query { pools { trades { id } } }"; + + let entities = vec![ + ("Pool", entity! { id: "0xf001" }), + ("Sell", entity! { id: "0xc0", pool: "0xf001"}), + ("Buy", entity! { id: "0xb0", pool: "0xf001"}), + ]; + + let res = insert_and_query(subgraph_id, schema, entities, query) + .await + .unwrap(); + + let data = extract_data!(res).unwrap(); + let exp = object! { pools: vec![ object!{ trades: vec![ object! { id: "0xb0" }, object! { id: "0xc0" }] } ] }; + assert_eq!(data, exp); +} diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..d15c8f4 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,101 @@ +# Full build with debuginfo for graph-node +# +# The expectation if that the docker build uses the parent directory as PWD +# by running something like the following +# docker build --target STAGE -f docker/Dockerfile . + +FROM golang:buster as envsubst + +# v1.2.0 +ARG ENVSUBST_COMMIT_SHA=16035fe3571ad42c7796bf554f978bb2df64231b + +RUN go install github.com/a8m/envsubst/cmd/envsubst@$ENVSUBST_COMMIT_SHA \ + && strip -g /go/bin/envsubst + +FROM rust:buster as graph-node-build + +ARG COMMIT_SHA=unknown +ARG REPO_NAME=unknown +ARG BRANCH_NAME=unknown +ARG TAG_NAME=unknown + +ADD . /graph-node + +RUN cd /graph-node \ + && apt-get update && apt-get install -y cmake \ + && rustup component add rustfmt \ + && RUSTFLAGS="-g" cargo install --locked --path node \ + && cargo clean \ + && objcopy --only-keep-debug /usr/local/cargo/bin/graph-node /usr/local/cargo/bin/graph-node.debug \ + && strip -g /usr/local/cargo/bin/graph-node \ + && strip -g /usr/local/cargo/bin/graphman \ + && cd /usr/local/cargo/bin \ + && objcopy --add-gnu-debuglink=graph-node.debug graph-node \ + && echo "REPO_NAME='$REPO_NAME'" > /etc/image-info \ + && echo "TAG_NAME='$TAG_NAME'" >> /etc/image-info \ + && echo "BRANCH_NAME='$BRANCH_NAME'" >> /etc/image-info \ + && echo "COMMIT_SHA='$COMMIT_SHA'" >> /etc/image-info \ + && echo "CARGO_VERSION='$(cargo --version)'" >> /etc/image-info \ + && echo "RUST_VERSION='$(rustc --version)'" >> /etc/image-info + +# Debug image to access core dumps +FROM graph-node-build as graph-node-debug +RUN apt-get update \ + && apt-get install -y curl gdb postgresql-client + +COPY docker/Dockerfile /Dockerfile +COPY docker/bin/* /usr/local/bin/ + +# The graph-node runtime image with only the executable +FROM debian:buster-slim as graph-node +ENV RUST_LOG "" +ENV GRAPH_LOG "" +ENV EARLY_LOG_CHUNK_SIZE "" +ENV ETHEREUM_RPC_PARALLEL_REQUESTS "" +ENV ETHEREUM_BLOCK_CHUNK_SIZE "" + +ENV postgres_host "" +ENV postgres_user "" +ENV postgres_pass "" +ENV postgres_db "" +# The full URL to the IPFS node +ENV ipfs "" +# The etherum network(s) to connect to. Set this to a space-separated +# list of the networks where each entry has the form NAME:URL +ENV ethereum "" +# The role the node should have, one of index-node, query-node, or +# combined-node +ENV node_role "combined-node" +# The name of this node +ENV node_id "default" +# The ethereum network polling interval (in milliseconds) +ENV ethereum_polling_interval "" + +# The location of an optional configuration file for graph-node, as +# described in ../docs/config.md +# Using a configuration file is experimental, and the file format may +# change in backwards-incompatible ways +ENV GRAPH_NODE_CONFIG "" + +# Disable core dumps; this is useful for query nodes with large caches. Set +# this to anything to disable coredumps (via 'ulimit -c 0') +ENV disable_core_dumps "" + +# HTTP port +EXPOSE 8000 +# WebSocket port +EXPOSE 8001 +# JSON-RPC port +EXPOSE 8020 +# Indexing status port +EXPOSE 8030 + +RUN apt-get update \ + && apt-get install -y libpq-dev ca-certificates netcat + +ADD docker/wait_for docker/start /usr/local/bin/ +COPY --from=graph-node-build /usr/local/cargo/bin/graph-node /usr/local/cargo/bin/graphman /usr/local/bin/ +COPY --from=graph-node-build /etc/image-info /etc/image-info +COPY --from=envsubst /go/bin/envsubst /usr/local/bin/ +COPY docker/Dockerfile /Dockerfile +CMD start diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..743a612 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,96 @@ +# Graph Node Docker Image + +Preconfigured Docker image for running a Graph Node. + +## Usage + +```sh +docker run -it \ + -e postgres_host= \ + -e postgres_port= \ + -e postgres_user= \ + -e postgres_pass= \ + -e postgres_db= \ + -e ipfs=: \ + -e ethereum=: \ + graphprotocol/graph-node:latest +``` + +### Example usage + +```sh +docker run -it \ + -e postgres_host=host.docker.internal \ + -e postgres_port=5432 \ + -e postgres_user=graph-node \ + -e postgres_pass=oh-hello \ + -e postgres_db=graph-node \ + -e ipfs=host.docker.internal:5001 \ + -e ethereum=mainnet:http://localhost:8545/ \ + graphprotocol/graph-node:latest +``` + +## Docker Compose + +The Docker Compose setup requires an Ethereum network name and node +to connect to. By default, it will use `mainnet:http://host.docker.internal:8545` +in order to connect to an Ethereum node running on your host machine. +You can replace this with anything else in `docker-compose.yaml`. + +> **Note for Linux users:** On Linux, `host.docker.internal` is not +> currently supported. Instead, you will have to replace it with the +> IP address of your Docker host (from the perspective of the Graph +> Node container). +> To do this, run: +> +> ``` +> CONTAINER_ID=$(docker container ls | grep graph-node | cut -d' ' -f1) +> docker exec $CONTAINER_ID /bin/bash -c 'apt install -y iproute2 && ip route' | awk '/^default via /{print $3}' +> ``` +> +> This will print the host's IP address. Then, put it into `docker-compose.yml`: +> +> ``` +> sed -i -e 's/host.docker.internal//g' docker-compose.yml +> ``` + +After you have set up an Ethereum node—e.g. Ganache or Parity—simply +clone this repository and run + +```sh +docker-compose up +``` + +This will start IPFS, Postgres and Graph Node in Docker and create persistent +data directories for IPFS and Postgres in `./data/ipfs` and `./data/postgres`. You +can access these via: + +- Graph Node: + - GraphiQL: `http://localhost:8000/` + - HTTP: `http://localhost:8000/subgraphs/name/` + - WebSockets: `ws://localhost:8001/subgraphs/name/` + - Admin: `http://localhost:8020/` +- IPFS: + - `127.0.0.1:5001` or `/ip4/127.0.0.1/tcp/5001` +- Postgres: + - `postgresql://graph-node:let-me-in@localhost:5432/graph-node` + +Once this is up and running, you can use +[`graph-cli`](https://github.com/graphprotocol/graph-cli) to create and +deploy your subgraph to the running Graph Node. + +### Running Graph Node on an Macbook M1 + +We do not currently build native images for Macbook M1, which can lead to processes being killed due to out-of-memory errors (code 137). Based on the example `docker-compose.yml` is possible to rebuild the image for your M1 by running the following, then running `docker-compose up` as normal: + +> **Important** Increase memory limits for the docker engine running on your machine. Otherwise docker build command will fail due to out of memory error. To do that, open docker-desktop and go to Resources/advanced/memory. +``` +# Remove the original image +docker rmi graphprotocol/graph-node:latest + +# Build the image +./docker/build.sh + +# Tag the newly created image +docker tag graph-node graphprotocol/graph-node:latest +``` diff --git a/docker/bin/create b/docker/bin/create new file mode 100755 index 0000000..9d9a4eb --- /dev/null +++ b/docker/bin/create @@ -0,0 +1,11 @@ +#! /bin/bash + +if [ $# != 1 ]; then + echo "usage: create " + exit 1 +fi + +api="http://index-node.default/" + +data=$(printf '{"jsonrpc": "2.0", "method": "subgraph_create", "params": {"name":"%s"}, "id":"1"}' "$1") +curl -s -H "content-type: application/json" --data "$data" "$api" diff --git a/docker/bin/debug b/docker/bin/debug new file mode 100755 index 0000000..87649f1 --- /dev/null +++ b/docker/bin/debug @@ -0,0 +1,9 @@ +#! /bin/bash + +if [ -f "$1" ] +then + exec rust-gdb -c "$1" /usr/local/cargo/bin/graph-node +else + echo "usage: debug " + exit 1 +fi diff --git a/docker/bin/deploy b/docker/bin/deploy new file mode 100755 index 0000000..f0c9833 --- /dev/null +++ b/docker/bin/deploy @@ -0,0 +1,12 @@ +#! /bin/bash + +if [ $# != 3 ]; then + echo "usage: deploy " + exit 1 +fi + +api="http://index-node.default/" + +echo "Deploying $1 (deployment $2)" +data=$(printf '{"jsonrpc": "2.0", "method": "subgraph_deploy", "params": {"name":"%s", "ipfs_hash":"%s", "node_id":"%s"}, "id":"1"}' "$1" "$2" "$3") +curl -s -H "content-type: application/json" --data "$data" "$api" diff --git a/docker/bin/reassign b/docker/bin/reassign new file mode 100755 index 0000000..a8eb703 --- /dev/null +++ b/docker/bin/reassign @@ -0,0 +1,12 @@ +#! /bin/bash + +if [ $# -lt 3 ]; then + echo "usage: reassign " + exit 1 +fi + +api="http://index-node.default/" + +echo Assigning to "$3" +data=$(printf '{"jsonrpc": "2.0", "method": "subgraph_reassign", "params": {"name":"%s", "ipfs_hash":"%s", "node_id":"%s"}, "id":"1"}' "$1" "$2" "$3") +curl -s -H "content-type: application/json" --data "$data" "$api" diff --git a/docker/bin/remove b/docker/bin/remove new file mode 100755 index 0000000..872fc33 --- /dev/null +++ b/docker/bin/remove @@ -0,0 +1,11 @@ +#! /bin/bash + +if [ $# != 1 ]; then + echo "usage: remove " + exit 1 +fi + +api="http://index-node.default/" + +data=$(printf '{"jsonrpc": "2.0", "method": "subgraph_remove", "params": {"name":"%s"}, "id":"1"}' "$1") +curl -s -H "content-type: application/json" --data "$data" "$api" diff --git a/docker/build.sh b/docker/build.sh new file mode 100755 index 0000000..5dd67ea --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,26 @@ +#! /bin/bash + +# This file is only here to ease testing/development. Official images are +# built using the 'cloudbuild.yaml' file + +type -p podman > /dev/null && docker=podman || docker=docker + +cd $(dirname $0)/.. + +if [ -d .git ] +then + COMMIT_SHA=$(git rev-parse HEAD) + TAG_NAME=$(git tag --points-at HEAD) + REPO_NAME="Checkout of $(git remote get-url origin) at $(git describe --dirty)" + BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) +fi +for stage in graph-node-build graph-node graph-node-debug +do + $docker build --target $stage \ + --build-arg "COMMIT_SHA=$COMMIT_SHA" \ + --build-arg "REPO_NAME=$REPO_NAME" \ + --build-arg "BRANCH_NAME=$BRANCH_NAME" \ + --build-arg "TAG_NAME=$TAG_NAME" \ + -t $stage \ + -f docker/Dockerfile . +done diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml new file mode 100644 index 0000000..39cf285 --- /dev/null +++ b/docker/cloudbuild.yaml @@ -0,0 +1,53 @@ +options: + machineType: "N1_HIGHCPU_32" +timeout: 1800s +steps: +- name: 'gcr.io/cloud-builders/docker' + args: ['build', '--target', 'graph-node-build', + '--build-arg', 'COMMIT_SHA=$COMMIT_SHA', + '--build-arg', 'REPO_NAME=$REPO_NAME', + '--build-arg', 'BRANCH_NAME=$BRANCH_NAME', + '--build-arg', 'TAG_NAME=$TAG_NAME', + '-t', 'gcr.io/$PROJECT_ID/graph-node-build:$SHORT_SHA', + '-f', 'docker/Dockerfile', '.'] +- name: 'gcr.io/cloud-builders/docker' + args: ['build', '--target', 'graph-node', + '--build-arg', 'COMMIT_SHA=$COMMIT_SHA', + '--build-arg', 'REPO_NAME=$REPO_NAME', + '--build-arg', 'BRANCH_NAME=$BRANCH_NAME', + '--build-arg', 'TAG_NAME=$TAG_NAME', + '-t', 'gcr.io/$PROJECT_ID/graph-node:$SHORT_SHA', + '-f', 'docker/Dockerfile', '.'] +- name: 'gcr.io/cloud-builders/docker' + args: ['build', '--target', 'graph-node-debug', + '--build-arg', 'COMMIT_SHA=$COMMIT_SHA', + '--build-arg', 'REPO_NAME=$REPO_NAME', + '--build-arg', 'BRANCH_NAME=$BRANCH_NAME', + '--build-arg', 'TAG_NAME=$TAG_NAME', + '-t', 'gcr.io/$PROJECT_ID/graph-node-debug:$SHORT_SHA', + '-f', 'docker/Dockerfile', '.'] +- name: 'gcr.io/cloud-builders/docker' + args: ['tag', + 'gcr.io/$PROJECT_ID/graph-node:$SHORT_SHA', + 'lutter/graph-node:$SHORT_SHA'] +- name: 'gcr.io/cloud-builders/docker' + entrypoint: 'bash' + args: ['docker/tag.sh'] + secretEnv: ['PASSWORD'] + env: + - 'SHORT_SHA=$SHORT_SHA' + - 'TAG_NAME=$TAG_NAME' + - 'PROJECT_ID=$PROJECT_ID' + - 'DOCKER_HUB_USER=$_DOCKER_HUB_USER' + - 'BRANCH_NAME=$BRANCH_NAME' +images: + - 'gcr.io/$PROJECT_ID/graph-node-build:$SHORT_SHA' + - 'gcr.io/$PROJECT_ID/graph-node:$SHORT_SHA' + - 'gcr.io/$PROJECT_ID/graph-node-debug:$SHORT_SHA' +substitutions: + # The owner of the access token whose encrypted value is in PASSWORD + _DOCKER_HUB_USER: "lutter" +secrets: + - kmsKeyName: projects/the-graph-staging/locations/global/keyRings/docker/cryptoKeys/docker-hub-push + secretEnv: + PASSWORD: 'CiQAdfFldbmUiHgGP1lPq6bAOfd+VQ/dFwyohB1IQwiwQg03ZE8STQDvWKpv6eJHVUN1YoFC5FcooJrH+Stvx9oMD7jBjgxEH5ngIiAysWP3E4Pgxt/73xnaanbM1EQ94eVFKCiY0GaEKFNu0BJx22vCYmU4' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..60c188b --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,45 @@ +version: '3' +services: + graph-node: + image: graphprotocol/graph-node + ports: + - '8000:8000' + - '8001:8001' + - '8020:8020' + - '8030:8030' + - '8040:8040' + depends_on: + - ipfs + - postgres + extra_hosts: + - host.docker.internal:host-gateway + environment: + postgres_host: postgres + postgres_user: graph-node + postgres_pass: let-me-in + postgres_db: graph-node + ipfs: 'ipfs:5001' + ethereum: 'mainnet:http://host.docker.internal:8545' + GRAPH_LOG: info + ipfs: + image: ipfs/go-ipfs:v0.10.0 + ports: + - '5001:5001' + volumes: + - ./data/ipfs:/data/ipfs + postgres: + image: postgres + ports: + - '5432:5432' + command: + [ + "postgres", + "-cshared_preload_libraries=pg_stat_statements" + ] + environment: + POSTGRES_USER: graph-node + POSTGRES_PASSWORD: let-me-in + POSTGRES_DB: graph-node + PGDATA: "/data/postgres" + volumes: + - ./data/postgres:/var/lib/postgresql/data diff --git a/docker/hooks/post_checkout b/docker/hooks/post_checkout new file mode 100755 index 0000000..f1b6f18 --- /dev/null +++ b/docker/hooks/post_checkout @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e +set -x + +echo "Setting SOURCE_BRANCH to ${SOURCE_BRANCH}" + +sed -i "s@^ENV SOURCE_BRANCH \"master\"@ENV SOURCE_BRANCH \"${SOURCE_BRANCH}\"@g" Dockerfile diff --git a/docker/setup.sh b/docker/setup.sh new file mode 100755 index 0000000..5823ad0 --- /dev/null +++ b/docker/setup.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -e + +if ! which docker 2>&1 > /dev/null; then + echo "Please install 'docker' first" + exit 1 +fi + +if ! which docker-compose 2>&1 > /dev/null; then + echo "Please install 'docker-compose' first" + exit 1 +fi + +if ! which jq 2>&1 > /dev/null; then + echo "Please install 'jq' first" + exit 1 +fi + +# Create the graph-node container +docker-compose up --no-start graph-node + +# Start graph-node so we can inspect it +docker-compose start graph-node + +# Identify the container ID +CONTAINER_ID=$(docker container ls | grep graph-node | cut -d' ' -f1) + +# Inspect the container to identify the host IP address +HOST_IP=$(docker inspect "$CONTAINER_ID" | jq -r .[0].NetworkSettings.Networks[].Gateway) + +echo "Host IP: $HOST_IP" + +# Inject the host IP into docker-compose.yml +sed -i -e "s/host.docker.internal/$HOST_IP/g" docker-compose.yml + +function stop_graph_node { + # Ensure graph-node is stopped + docker-compose stop graph-node +} + +trap stop_graph_node EXIT diff --git a/docker/start b/docker/start new file mode 100755 index 0000000..435fbe5 --- /dev/null +++ b/docker/start @@ -0,0 +1,140 @@ +#!/bin/bash + +save_coredumps() { + graph_dir=/var/lib/graph + datestamp=$(date +"%Y-%m-%dT%H:%M:%S") + ls /core.* >& /dev/null && have_cores=yes || have_cores=no + if [ -d "$graph_dir" -a "$have_cores" = yes ] + then + core_dir=$graph_dir/cores + mkdir -p $core_dir + exec >> "$core_dir"/messages 2>&1 + echo "${HOSTNAME##*-} Saving core dump on ${HOSTNAME} at ${datestamp}" + + dst="$core_dir/$datestamp-${HOSTNAME}" + mkdir "$dst" + cp /usr/local/bin/graph-node "$dst" + cp /proc/loadavg "$dst" + [ -f /Dockerfile ] && cp /Dockerfile "$dst" + tar czf "$dst/etc.tgz" /etc/ + dmesg -e > "$dst/dmesg" + # Capture environment variables, but filter out passwords + env | sort | sed -r -e 's/^(postgres_pass|ELASTICSEARCH_PASSWORD)=.*$/\1=REDACTED/' > "$dst/env" + + for f in /core.* + do + echo "${HOSTNAME##*-} Found core dump $f" + mv "$f" "$dst" + done + echo "${HOSTNAME##*-} Saving done" + fi +} + +wait_for_ipfs() { + # Take the IPFS URL in $1 apart and extract host and port. If no explicit + # host is given, use 443 for https, and 80 otherwise + if [[ "$1" =~ ^((https?)://)?([^:/]+)(:([0-9]+))? ]] + then + proto=${BASH_REMATCH[2]:-http} + host=${BASH_REMATCH[3]} + port=${BASH_REMATCH[5]} + if [ -z "$port" ] + then + [ "$proto" = "https" ] && port=443 || port=80 + fi + wait_for "$host:$port" -t 120 + else + echo "invalid IPFS URL: $1" + exit 1 + fi +} + +run_graph_node() { + if [ -n "$GRAPH_NODE_CONFIG" ] + then + # Start with a configuration file; we don't do a check whether + # postgres is up in this case, though we probably should + wait_for_ipfs "$ipfs" + sleep 5 + exec graph-node \ + --node-id "$node_id" \ + --config "$GRAPH_NODE_CONFIG" \ + --ipfs "$ipfs" \ + ${fork_base:+ --fork-base "$fork_base"} + else + unset GRAPH_NODE_CONFIG + postgres_port=${postgres_port:-5432} + postgres_url="postgresql://$postgres_user:$postgres_pass@$postgres_host:$postgres_port/$postgres_db?sslmode=prefer" + + wait_for_ipfs "$ipfs" + wait_for "$postgres_host:$postgres_port" -t 120 + sleep 5 + + exec graph-node \ + --node-id "$node_id" \ + --postgres-url "$postgres_url" \ + --ethereum-rpc $ethereum \ + --ipfs "$ipfs" \ + ${fork_base:+ --fork-base "$fork_base"} + fi +} + +start_query_node() { + # Query nodes are never the block ingestor + export DISABLE_BLOCK_INGESTOR=true + run_graph_node +} + +start_index_node() { + run_graph_node +} + +start_combined_node() { + run_graph_node +} + +# Only the index node with the name set in BLOCK_INGESTOR should ingest +# blocks. For historical reasons, that name is set to the unmangled version +# of `node_id` and we need to check whether we are the block ingestor +# before possibly mangling the node_id. +if [[ ${node_id} != "${BLOCK_INGESTOR}" ]]; then + export DISABLE_BLOCK_INGESTOR=true +fi + +# Allow operators to opt out of legacy character +# restrictions on the node ID by setting enablement +# variable to a non-zero length string: +if [ -z "$GRAPH_NODE_ID_USE_LITERAL_VALUE" ] +then + node_id="${node_id//-/_}" +fi + +if [ -n "$disable_core_dumps" ] +then + ulimit -c 0 +fi + +trap save_coredumps EXIT + +export PGAPPNAME="${node_id:-$HOSTNAME}" + +# Set custom poll interval +if [ -n "$ethereum_polling_interval" ]; then + export ETHEREUM_POLLING_INTERVAL=$ethereum_polling_interval +fi + +case "${node_role:-combined-node}" in + query-node) + start_query_node + ;; + index-node) + start_index_node + ;; + combined-node) + start_combined_node + ;; + *) + echo "Unknown mode for start-node: $1" + echo "usage: start (combined-node|query-node|index-node)" + exit 1 +esac diff --git a/docker/tag.sh b/docker/tag.sh new file mode 100644 index 0000000..032ab54 --- /dev/null +++ b/docker/tag.sh @@ -0,0 +1,28 @@ +#! /bin/bash + +# This script is used by cloud build to push Docker images into Docker hub + +tag_and_push() { + tag=$1 + docker tag gcr.io/$PROJECT_ID/graph-node:$SHORT_SHA \ + graphprotocol/graph-node:$tag + docker push graphprotocol/graph-node:$tag + + docker tag gcr.io/$PROJECT_ID/graph-node-debug:$SHORT_SHA \ + graphprotocol/graph-node-debug:$tag + docker push graphprotocol/graph-node-debug:$tag +} + +echo "Logging into Docker Hub" +echo $PASSWORD | docker login --username="$DOCKER_HUB_USER" --password-stdin + +set -ex + +tag_and_push "$SHORT_SHA" + +# Builds of tags set the tag in Docker Hub, too +[ -n "$TAG_NAME" ] && tag_and_push "$TAG_NAME" +# Builds for tags vN.N.N become the 'latest' +[[ "$TAG_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && tag_and_push latest + +exit 0 diff --git a/docker/wait_for b/docker/wait_for new file mode 100755 index 0000000..eb0865d --- /dev/null +++ b/docker/wait_for @@ -0,0 +1,83 @@ +#!/bin/sh + +# POSIX compatible clone of wait-for-it.sh +# This copy is from https://github.com/eficode/wait-for/commits/master +# at commit 8d9b4446 + +TIMEOUT=15 +QUIET=0 + +echoerr() { + if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi +} + +usage() { + exitcode="$1" + cat << USAGE >&2 +Usage: + $cmdname host:port [-t timeout] [-- command args] + -q | --quiet Do not output any status messages + -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit "$exitcode" +} + +wait_for() { + for i in `seq $TIMEOUT` ; do + nc -z "$HOST" "$PORT" > /dev/null 2>&1 + + result=$? + if [ $result -eq 0 ] ; then + if [ $# -gt 0 ] ; then + exec "$@" + fi + exit 0 + fi + sleep 1 + done + echo "Operation timed out" >&2 + exit 1 +} + +while [ $# -gt 0 ] +do + case "$1" in + *:* ) + HOST=$(printf "%s\n" "$1"| cut -d : -f 1) + PORT=$(printf "%s\n" "$1"| cut -d : -f 2) + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -t) + TIMEOUT="$2" + if [ "$TIMEOUT" = "" ]; then break; fi + shift 2 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + break + ;; + --help) + usage 0 + ;; + *) + echoerr "Unknown argument: $1" + usage 1 + ;; + esac +done + +if [ "$HOST" = "" -o "$PORT" = "" ]; then + echoerr "Error: you need to provide a host and port to test." + usage 2 +fi + +wait_for "$@" diff --git a/docs/adding-a-new-api-version.md b/docs/adding-a-new-api-version.md new file mode 100644 index 0000000..1a7b495 --- /dev/null +++ b/docs/adding-a-new-api-version.md @@ -0,0 +1,37 @@ +# Adding a new `apiVersion` + +This document explains how to coordinate an `apiVersion` upgrade +across all impacted projects: + +1. [`graph-node`](https:github.com/graphprotocol/graph-node) +2. [`graph-ts`](https:github.com/graphprotocol/graph-ts) +3. [`graph-cli`](https:github.com/graphprotocol/graph-cli) +4. `graph-docs` + +## Steps + +Those steps should be taken after all relevant `graph-node` changes +have been rolled out to production (hosted-service): + +1. Update the default value of the `GRAPH_MAX_API_VERSION` environment + variable, currently located at this file: `graph/src/data/subgraph/mod.rs`. + If you're setting it up somewhere manually, you should change there + as well, or just remove it. + +2. Update `graph-node` minor version and create a new release. + +3. Update `graph-ts` version and create a new release. + +4. For `graph-cli`: + + 1. Write migrations for the new `apiVersion`. + 2. Update the version restriction on the `build` and `deploy` + commands to match the new `graph-ts` and `apiVersion` versions. + 3. Update the `graph-cli` version in `package.json`. + 4. Update `graph-ts` and `graph-cli` version numbers on scaffolded code and examples. + 5. Recompile all the examples by running `=$ npm install` inside + each example directory. + 6. Update `graph-cli`\'s version and create a new release. + 7. Release in NPM + +5. Update `graph-docs` with the new `apiVersion` content. diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 0000000..8020334 --- /dev/null +++ b/docs/config.md @@ -0,0 +1,279 @@ +# Advanced Graph Node configuration + +A TOML configuration file can be used to set more complex configurations than those exposed in the +CLI. The location of the file is passed with the `--config` command line switch. When using a +configuration file, it is not possible to use the options `--postgres-url`, +`--postgres-secondary-hosts`, and `--postgres-host-weights`. + +The TOML file consists of four sections: +* `[chains]` sets the endpoints to blockchain clients. +* `[store]` describes the available databases. +* `[ingestor]` sets the name of the node responsible for block ingestion. +* `[deployment]` describes how to place newly deployed subgraphs. + +Some of these sections support environment variable expansion out of the box, +most notably Postgres connection strings. The official `graph-node` Docker image +includes [`envsubst`](https://github.com/a8m/envsubst) for more complex use +cases. + +## Configuring Multiple Databases + +For most use cases, a single Postgres database is sufficient to support a +`graph-node` instance. When a `graph-node` instance outgrows a single +Postgres database, it is possible to split the storage of `graph-node`'s +data across multiple Postgres databases. All databases together form the +store of the `graph-node` instance. Each individual database is called a +_shard_. + +The `[store]` section must always have a primary shard configured, which +must be called `primary`. Each shard can have additional read replicas that +are used for responding to queries. Only queries are processed by read +replicas. Indexing and block ingestion will always use the main database. + +Any number of additional shards, with their own read replicas, can also be +configured. When read replicas are used, query traffic is split between the +main database and the replicas according to their weights. In the example +below, for the primary shard, no queries will be sent to the main database, +and the replicas will receive 50% of the traffic each. In the `vip` shard, +50% of the traffic goes to the main database, and 50% to the replica. + +```toml +[store] +[store.primary] +connection = "postgresql://graph:${PGPASSWORD}@primary/graph" +weight = 0 +pool_size = 10 +[store.primary.replicas.repl1] +connection = "postgresql://graph:${PGPASSWORD}@primary-repl1/graph" +weight = 1 +[store.primary.replicas.repl2] +connection = "postgresql://graph:${PGPASSWORD}@primary-repl2/graph" +weight = 1 + +[store.vip] +connection = "postgresql://graph:${PGPASSWORD}@${VIP_MAIN}/graph" +weight = 1 +pool_size = 10 +[store.vip.replicas.repl1] +connection = "postgresql://graph:${PGPASSWORD}@${VIP_REPL1}/graph" +weight = 1 +``` + +The `connection` string must be a valid [libpq connection +string](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING). Before +passing the connection string to Postgres, environment variables embedded +in the string are expanded. + +### Setting the `pool_size` + +Each shard must indicate how many database connections each `graph-node` +instance should keep in its connection pool for that database. For +replicas, the pool size defaults to the pool size of the main database, but +can also be set explicitly. Such a setting replaces the setting from the +main database. + +The `pool_size` can either be a number like in the example above, in which +case any `graph-node` instance will use a connection pool of that size, or a set +of rules that uses different sizes for different `graph-node` instances, +keyed off the `node_id` set on the command line. When using rules, the +`pool_size` is set like this: + +```toml +pool_size = [ + { node = "index_node_general_.*", size = 20 }, + { node = "index_node_special_.*", size = 30 }, + { node = "query_node_.*", size = 80 } +] +``` + +Each rule consists of a regular expression `node` and the size that should +be used if the current instance's `node_id` matches that regular +expression. You can use the command `graphman config pools` to check how +many connections each `graph-node` instance will use, and how many database +connections will be opened by all `graph-node` instance. The rules are +checked in the order in which they are written, and the first one that +matches is used. It is an error if no rule matches. + +It is highly recommended to run `graphman config pools $all_nodes` every +time the configuration is changed to make sure that the connection pools +are what is expected. Here, `$all_nodes` should be a list of all the node +names that will use this configuration file. + +## Configuring Ethereum Providers + +The `[chains]` section controls the ethereum providers that `graph-node` +connects to, and where blocks and other metadata for each chain are +stored. The section consists of the name of the node doing block ingestion +(currently not used), and a list of chains. The configuration for a chain +`name` is specified in the section `[chains.]`, and consists of the +`shard` where chain data is stored and a list of providers for that +chain. For each provider, the following information must be given: + +* `label`: a label that is used when logging information about that + provider (not implemented yet) +* `transport`: one of `rpc`, `ws`, and `ipc`. Defaults to `rpc`. +* `url`: the URL for the provider +* `features`: an array of features that the provider supports, either empty + or any combination of `traces` and `archive` +* `headers`: HTTP headers to be added on every request. Defaults to none. +* `limit`: the maximum number of subgraphs that can use this provider. + Defaults to unlimited. At least one provider should be unlimited, + otherwise `graph-node` might not be able to handle all subgraphs. The + tracking for this is approximate, and a small amount of deviation from + this value should be expected. The deviation will be less than 10. + +The following example configures two chains, `mainnet` and `kovan`, where +blocks for `mainnet` are stored in the `vip` shard and blocks for `kovan` +are stored in the primary shard. The `mainnet` chain can use two different +providers, whereas `kovan` only has one provider. + +```toml +[chains] +ingestor = "block_ingestor_node" +[chains.mainnet] +shard = "vip" +provider = [ + { label = "mainnet1", url = "http://..", features = [], headers = { Authorization = "Bearer foo" } }, + { label = "mainnet2", url = "http://..", features = [ "archive", "traces" ] } +] +[chains.kovan] +shard = "primary" +provider = [ { label = "kovan", url = "http://..", features = [] } ] +``` + +### Controlling the number of subgraphs using a provider + +**This feature is experimental and might be removed in a future release** + +Each provider can set a limit for the number of subgraphs that can use this +provider. The measurement of the number of subgraphs using a provider is +approximate and can differ from the true number by a small amount +(generally less than 10) + +The limit is set through rules that match on the node name. If a node's +name does not match any rule, the corresponding provider can be used for an +unlimited number of subgraphs. It is recommended that at least one provider +is generally unlimited. The limit is set in the following way: + +```toml +[chains.mainnet] +shard = "vip" +provider = [ + { label = "mainnet-0", url = "http://..", features = [] }, + { label = "mainnet-1", url = "http://..", features = [], + match = [ + { name = "some_node_.*", limit = 10 }, + { name = "other_node_.*", limit = 0 } ] } ] +``` + +Nodes named `some_node_.*` will use `mainnet-1` for at most 10 subgraphs, +and `mainnet-0` for everything else, nodes named `other_node_.*` will never +use `mainnet-1` and always `mainnet-0`. Any node whose name does not match +one of these patterns will use `mainnet-0` and `mainnet-1` for an unlimited +number of subgraphs. + +## Controlling Deployment + +When `graph-node` receives a request to deploy a new subgraph deployment, +it needs to decide in which shard to store the data for the deployment, and +which of any number of nodes connected to the store should index the +deployment. That decision is based on a number of rules defined in the +`[deployment]` section. Deployment rules can match on the subgraph name and +the network that the deployment is indexing. + +Rules are evaluated in order, and the first rule that matches determines +where the deployment is placed. The `match` element of a rule can have a +`name`, a [regular expression](https://docs.rs/regex/1.4.2/regex/#syntax) +that is matched against the subgraph name for the deployment, and a +`network` name that is compared to the network that the new deployment +indexes. The `network` name can either be a string, or a list of strings. + +The last rule must not have a `match` statement to make sure that there is +always some shard and some indexer that will work on a deployment. + +The rule indicates the name of the `shard` where the data for the +deployment should be stored, which defaults to `primary`, and a list of +`indexers`. For the matching rule, one indexer is chosen from the +`indexers` list so that deployments are spread evenly across all the nodes +mentioned in `indexers`. The names for the indexers must be the same names +that are passed with `--node-id` when those index nodes are started. + +Instead of a fixed `shard`, it is also possible to use a list of `shards`; +in that case, the system uses the shard from the given list with the fewest +active deployments in it. + +```toml +[deployment] +[[deployment.rule]] +match = { name = "(vip|important)/.*" } +shard = "vip" +indexers = [ "index_node_vip_0", "index_node_vip_1" ] +[[deployment.rule]] +match = { network = "kovan" } +# No shard, so we use the default shard called 'primary' +indexers = [ "index_node_kovan_0" ] +[[deployment.rule]] +match = { network = [ "xdai", "poa-core" ] } +indexers = [ "index_node_other_0" ] +[[deployment.rule]] +# There's no 'match', so any subgraph matches +shards = [ "sharda", "shardb" ] +indexers = [ + "index_node_community_0", + "index_node_community_1", + "index_node_community_2", + "index_node_community_3", + "index_node_community_4", + "index_node_community_5" + ] + +``` + +## Query nodes + +Nodes can be configured to explicitly be query nodes by including the +following in the configuration file: +```toml +[general] +query = "" +``` + +Any node whose `--node-id` matches the regular expression will be set up to +only respond to queries. For now, that only means that the node will not +try to connect to any of the configured Ethereum providers. + +## Basic Setup + +The following file is equivalent to using the `--postgres-url` command line +option: +```toml +[store] +[store.primary] +connection="<.. postgres-url argument ..>" +[deployment] +[[deployment.rule]] +indexers = [ "<.. list of all indexing nodes ..>" ] +``` + +## Validating configuration files + +A configuration file can be checked for validity by passing the `--check-config` +flag to `graph-node`. The command +```shell +graph-node --config $CONFIG_FILE --check-config +``` +will read the configuration file and print information about syntax errors or, for +valid files, a JSON representation of the configuration. + +## Simulating deployment placement + +Given a configuration file, placement of newly deployed subgraphs can be +simulated with +```shell +graphman --config $CONFIG_FILE config place some/subgraph mainnet +``` +The command will not make any changes, but simply print where that subgraph +would be placed. The output will indicate the database shard that will hold +the subgraph's data, and a list of indexing nodes that could be used for +indexing that subgraph. During deployment, `graph-node` chooses the indexing +nodes with the fewest subgraphs currently assigned from that list. diff --git a/docs/environment-variables.md b/docs/environment-variables.md new file mode 100644 index 0000000..e52e529 --- /dev/null +++ b/docs/environment-variables.md @@ -0,0 +1,201 @@ +# Environment Variables + +**Warning**: the names of some of these environment variables will be changed at +some point in the near future. + +This page lists the environment variables used by `graph-node` and what effect +they have. Some environment variables can be used instead of command line flags. +Those are not listed here, please consult `graph-node --help` for details on +those. + +## JSON-RPC configuration for EVM chains + +- `ETHEREUM_REORG_THRESHOLD`: Maximum expected reorg size, if a larger reorg + happens, subgraphs might process inconsistent data. Defaults to 250. +- `ETHEREUM_POLLING_INTERVAL`: how often to poll Ethereum for new blocks (in ms, + defaults to 500ms) +- `GRAPH_ETHEREUM_TARGET_TRIGGERS_PER_BLOCK_RANGE`: The ideal amount of triggers + to be processed in a batch. If this is too small it may cause too many requests + to the ethereum node, if it is too large it may cause unreasonably expensive + calls to the ethereum node and excessive memory usage (defaults to 100). +- `ETHEREUM_TRACE_STREAM_STEP_SIZE`: `graph-node` queries traces for a given + block range when a subgraph defines call handlers or block handlers with a + call filter. The value of this variable controls the number of blocks to scan + in a single RPC request for traces from the Ethereum node. Defaults to 50. +- `DISABLE_BLOCK_INGESTOR`: set to `true` to disable block ingestion. Leave + unset or set to `false` to leave block ingestion enabled. +- `ETHEREUM_BLOCK_BATCH_SIZE`: number of Ethereum blocks to request in parallel. + Also limits other parallel requests such such as trace_filter. Defaults to 10. +- `GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE`: Maximum number of blocks to scan for + triggers in each request (defaults to 1000). +- `GRAPH_ETHEREUM_MAX_EVENT_ONLY_RANGE`: Maximum range size for `eth.getLogs` + requests that dont filter on contract address, only event signature (defaults to 500). +- `GRAPH_ETHEREUM_JSON_RPC_TIMEOUT`: Timeout for Ethereum JSON-RPC requests. +- `GRAPH_ETHEREUM_REQUEST_RETRIES`: Number of times to retry JSON-RPC requests + made against Ethereum. This is used for requests that will not fail the + subgraph if the limit is reached, but will simply restart the syncing step, + so it can be low. This limit guards against scenarios such as requesting a + block hash that has been reorged. Defaults to 10. +- `GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS`: + The maximum number of concurrent requests made against Ethereum for + requesting transaction receipts during block ingestion. + Defaults to 1,000. +- `GRAPH_ETHEREUM_FETCH_TXN_RECEIPTS_IN_BATCHES`: Set to `true` to + disable fetching receipts from the Ethereum node concurrently during + block ingestion. This will use fewer, batched requests. This is always set to `true` + on MacOS to avoid DNS issues. +- `GRAPH_ETHEREUM_CLEANUP_BLOCKS` : Set to `true` to clean up unneeded + blocks from the cache in the database. When this is `false` or unset (the + default), blocks will never be removed from the block cache. This setting + should only be used during development to reduce the size of the + database. In production environments, it will cause multiple downloads of + the same blocks and therefore slow the system down. This setting can not + be used if the store uses more than one shard. +- `GRAPH_ETHEREUM_GENESIS_BLOCK_NUMBER`: Specify genesis block number. If the flag + is not set, the default value will be `0`. + +## Running mapping handlers + +- `GRAPH_MAPPING_HANDLER_TIMEOUT`: amount of time a mapping handler is allowed to + take (in seconds, default is unlimited) +- `GRAPH_ENTITY_CACHE_SIZE`: Size of the entity cache, in kilobytes. Defaults to 10000 which is 10MB. +- `GRAPH_MAX_API_VERSION`: Maximum `apiVersion` supported, if a developer tries to create a subgraph + with a higher `apiVersion` than this in their mappings, they'll receive an error. Defaults to `0.0.7`. +- `GRAPH_MAX_SPEC_VERSION`: Maximum `specVersion` supported. if a developer tries to create a subgraph + with a higher `apiVersion` than this, they'll receive an error. Defaults to `0.0.5`. +- `GRAPH_RUNTIME_MAX_STACK_SIZE`: Maximum stack size for the WASM runtime, if exceeded the execution + stops and an error is thrown. Defaults to 512KiB. + +## IPFS + +- `GRAPH_IPFS_TIMEOUT`: timeout for IPFS, which includes requests for manifest files + and from mappings (in seconds, default is 30). +- `GRAPH_MAX_IPFS_FILE_BYTES`: maximum size for a file that can be retrieved (in bytes, default is 256 MiB). +- `GRAPH_MAX_IPFS_MAP_FILE_SIZE`: maximum size of files that can be processed + with `ipfs.map`. When a file is processed through `ipfs.map`, the entities + generated from that are kept in memory until the entire file is done + processing. This setting therefore limits how much memory a call to `ipfs.map` + may use (in bytes, defaults to 256MB). +- `GRAPH_MAX_IPFS_CACHE_SIZE`: maximum number of files cached (defaults to 50). +- `GRAPH_MAX_IPFS_CACHE_FILE_SIZE`: maximum size of each cached file (in bytes, defaults to 1MiB). +- `GRAPH_MAX_IPFS_CONCURRENT_REQUESTS`: maximum concurrent requests to IPFS from file data sources (defaults to 100). + +## GraphQL + +- `GRAPH_GRAPHQL_QUERY_TIMEOUT`: maximum execution time for a graphql query, in + seconds. Default is unlimited. +- `SUBSCRIPTION_THROTTLE_INTERVAL`: while a subgraph is syncing, subscriptions + to that subgraph get updated at most this often, in ms. Default is 1000ms. +- `GRAPH_GRAPHQL_MAX_COMPLEXITY`: maximum complexity for a graphql query. See + [here](https://developer.github.com/v4/guides/resource-limitations) for what + that means. Default is unlimited. Typical introspection queries have a + complexity of just over 1 million, so setting a value below that may interfere + with introspection done by graphql clients. +- `GRAPH_GRAPHQL_MAX_DEPTH`: maximum depth of a graphql query. Default (and + maximum) is 255. +- `GRAPH_GRAPHQL_MAX_FIRST`: maximum value that can be used for the `first` + argument in GraphQL queries. If not provided, `first` defaults to 100. The + default value for `GRAPH_GRAPHQL_MAX_FIRST` is 1000. +- `GRAPH_GRAPHQL_MAX_SKIP`: maximum value that can be used for the `skip` + argument in GraphQL queries. The default value for + `GRAPH_GRAPHQL_MAX_SKIP` is unlimited. +- `GRAPH_GRAPHQL_WARN_RESULT_SIZE` and `GRAPH_GRAPHQL_ERROR_RESULT_SIZE`: + if a GraphQL result is larger than these sizes in bytes, log a warning + respectively abort query execution and return an error. The size of the + result is checked while the response is being constructed, so that + execution does not take more memory than what is configured. The default + value for both is unlimited. +- `GRAPH_GRAPHQL_MAX_OPERATIONS_PER_CONNECTION`: maximum number of GraphQL + operations per WebSocket connection. Any operation created after the limit + will return an error to the client. Default: 1000. +- `GRAPH_GRAPHQL_HTTP_PORT` : Port for the GraphQL HTTP server +- `GRAPH_GRAPHQL_WS_PORT` : Port for the GraphQL WebSocket server +- `GRAPH_SQL_STATEMENT_TIMEOUT`: the maximum number of seconds an + individual SQL query is allowed to take during GraphQL + execution. Default: unlimited +- `GRAPH_DISABLE_SUBSCRIPTION_NOTIFICATIONS`: disables the internal + mechanism that is used to trigger updates on GraphQL subscriptions. When + this variable is set to any value, `graph-node` will still accept GraphQL + subscriptions, but they won't receive any updates. +- `ENABLE_GRAPHQL_VALIDATIONS`: enables GraphQL validations, based on the GraphQL specification. + This will validate and ensure every query executes follows the execution rules. +- `SILENT_GRAPHQL_VALIDATIONS`: If `ENABLE_GRAPHQL_VALIDATIONS` is enabled, you are also able to just + silently print the GraphQL validation errors, without failing the actual query. Note: queries + might still fail as part of the later stage validations running, during GraphQL engine execution. + +### GraphQL caching + +- `GRAPH_CACHED_SUBGRAPH_IDS`: when set to `*`, cache all subgraphs (default behavior). Otherwise, a comma-separated list of subgraphs for which to cache queries. +- `GRAPH_QUERY_CACHE_BLOCKS`: How many recent blocks per network should be kept in the query cache. This should be kept small since the lookup time and the cache memory usage are proportional to this value. Set to 0 to disable the cache. Defaults to 1. +- `GRAPH_QUERY_CACHE_MAX_MEM`: Maximum total memory to be used by the query cache, in MB. The total amount of memory used for caching will be twice this value - once for recent blocks, divided evenly among the `GRAPH_QUERY_CACHE_BLOCKS`, and once for frequent queries against older blocks. The default is plenty for most loads, particularly if `GRAPH_QUERY_CACHE_BLOCKS` is kept small. Defaults to 1000, which corresponds to 1GB. +- `GRAPH_QUERY_CACHE_STALE_PERIOD`: Number of queries after which a cache entry can be considered stale. Defaults to 100. + +## Miscellaneous + +- `GRAPH_NODE_ID`: sets the node ID, allowing to run multiple Graph Nodes + in parallel and deploy to specific nodes; each ID must be unique among the set + of nodes. A single node should have the same value between consecutive restarts. + Subgraphs get assigned to node IDs and are not reassigned to other nodes automatically. +- `GRAPH_NODE_ID_USE_LITERAL_VALUE`: (Docker only) Use the literal `node_id` + provided to the docker start script instead of replacing hyphens (-) in names + with underscores (\_). Changing this for an existing `graph-node` + installation requires also changing the assigned node IDs in the + `subgraphs.subgraph_deployment_assignment` table in the database. This can be + done with GraphMan or via the PostgreSQL command line. +- `GRAPH_LOG`: control log levels, the same way that `RUST_LOG` is described + [here](https://docs.rs/env_logger/0.6.0/env_logger/) +- `THEGRAPH_STORE_POSTGRES_DIESEL_URL`: postgres instance used when running + tests. Set to `postgresql://:@:/` +- `GRAPH_KILL_IF_UNRESPONSIVE`: If set, the process will be killed if unresponsive. +- `GRAPH_LOG_QUERY_TIMING`: Control whether the process logs details of + processing GraphQL and SQL queries. The value is a comma separated list + of `sql`,`gql`, and `cache`. If `gql` is present in the list, each + GraphQL query made against the node is logged at level `info`. The log + message contains the subgraph that was queried, the query, its variables, + the amount of time the query took, and a unique `query_id`. If `sql` is + present, the SQL queries that a GraphQL query causes are logged. The log + message contains the subgraph, the query, its bind variables, the amount + of time it took to execute the query, the number of entities found by the + query, and the `query_id` of the GraphQL query that caused the SQL + query. These SQL queries are marked with `component: GraphQlRunner` There + are additional SQL queries that get logged when `sql` is given. These are + queries caused by mappings when processing blocks for a subgraph, and + queries caused by subscriptions. If `cache` is present in addition to + `gql`, also logs information for each toplevel GraphQL query field + whether that could be retrieved from cache or not. Defaults to no + logging. +- `GRAPH_LOG_TIME_FORMAT`: Custom log time format.Default value is `%b %d %H:%M:%S%.3f`. More information [here](https://docs.rs/chrono/latest/chrono/#formatting-and-parsing). +- `STORE_CONNECTION_POOL_SIZE`: How many simultaneous connections to allow to the store. + Due to implementation details, this value may not be strictly adhered to. Defaults to 10. +- `GRAPH_LOG_POI_EVENTS`: Logs Proof of Indexing events deterministically. + This may be useful for debugging. +- `GRAPH_LOAD_WINDOW_SIZE`, `GRAPH_LOAD_BIN_SIZE`: Load can be + automatically throttled if load measurements over a time period of + `GRAPH_LOAD_WINDOW_SIZE` seconds exceed a threshold. Measurements within + each window are binned into bins of `GRAPH_LOAD_BIN_SIZE` seconds. The + variables default to 300s and 1s +- `GRAPH_LOAD_THRESHOLD`: If wait times for getting database connections go + above this threshold, throttle queries until the wait times fall below + the threshold. Value is in milliseconds, and defaults to 0 which + turns throttling and any associated statistics collection off. +- `GRAPH_LOAD_JAIL_THRESHOLD`: When the system is overloaded, any query + that causes more than this fraction of the effort will be rejected for as + long as the process is running (i.e., even after the overload situation + is resolved) If this variable is not set, no queries will ever be jailed, + but they will still be subject to normal load management when the system + is overloaded. +- `GRAPH_LOAD_SIMULATE`: Perform all the steps that the load manager would + given the other load management configuration settings, but never + actually decline to run a query, instead log about load management + decisions. Set to `true` to turn simulation on, defaults to `false` +- `GRAPH_STORE_CONNECTION_TIMEOUT`: How long to wait to connect to a + database before assuming the database is down in ms. Defaults to 5000ms. +- `EXPERIMENTAL_SUBGRAPH_VERSION_SWITCHING_MODE`: default is `instant`, set + to `synced` to only switch a named subgraph to a new deployment once it + has synced, making the new deployment the "Pending" version. +- `GRAPH_REMOVE_UNUSED_INTERVAL`: How long to wait before removing an + unused deployment. The system periodically checks and marks deployments + that are not used by any subgraphs any longer. Once a deployment has been + identified as unused, `graph-node` will wait at least this long before + actually deleting the data (value is in minutes, defaults to 360, i.e. 6 + hours) diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..213a540 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,486 @@ +# Getting Started +> **Note:** This project is heavily a WIP, and until it reaches v1.0, the API is subject to change in breaking ways without notice. + +## 0 Introduction + +This page explains everything you need to know to run a local Graph Node, including links to other reference pages. First, we describe what The Graph is and then explain how to get started. + +### 0.1 What Is The Graph? + +The Graph is a decentralized protocol for indexing and querying data from blockchains, which makes it possible to query for data that is difficult or impossible to do directly. Currently, we only work with Ethereum. + +For example, with the popular Cryptokitties decentralized application (dApp) that implements the [ERC-721 Non-Fungible Token (NFT)](https://github.com/ethereum/eips/issues/721) standard, it is relatively straightforward to ask the following questions: +> *How many cryptokitties does a specific Ethereum account own?* +> *When was a particular cryptokitty born?* + +These read patterns are directly supported by the methods exposed by the [contract](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyCore.sol): the [`balanceOf`](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyOwnership.sol#L64) and [`getKitty`](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyCore.sol#L91) methods for these two examples. + +However, other questions are more difficult to answer: +> *Who are the owners of the cryptokitties born between January and February of 2018?* + +To answer this question, you need to process all [`Birth` events](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyBase.sol#L15) and then call the [`ownerOf` method](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyOwnership.sol#L144) for each cryptokitty born. An alternate approach could involve processing all (`Transfer` events) and filtering based on the most recent transfer for each cryptokitty. + +Even for this relatively simple question, it would take hours or even days for a dApp running in a browser to find an answer. Indexing and caching data off blockchains is hard. There are also edge cases around finality, chain reorganizations, uncled blocks, etc., which make it even more difficult to display deterministic data to the end user. + +The Graph solves this issue by providing an open source node implementation, [Graph Node](../README.md), which handles indexing and caching of blockchain data. The entire community can contribute to and utilize this tool. In the current implementation, it exposes functionality through a GraphQL API for end users. + +### 0.2 How Does It Work? + +The Graph must be run alongside a running IPFS node, Ethereum node, and a store (Postgres, in this initial implementation). + +![Data Flow Diagram](images/TheGraph_DataFlowDiagram.png) + +The high-level dataflow for a dApp using The Graph is as follows: +1. The dApp creates/modifies data on Ethereum through a transaction to a smart contract. +2. The smart contract emits one or more events (logs) while processing this transaction. +3. The Graph Node listens for specific events and triggers handlers in a user-defined mapping. +4. The mapping is a WASM module that runs in a WASM runtime. It creates one or more store transactions in response to Ethereum events. +5. The store is updated along with the indexes. +6. The dApp queries the Graph Node for data indexed from the blockchain using the node's [GraphQL endpoint](https://graphql.org/learn/). The Graph Node, in turn, translates the GraphQL queries into queries for its underlying store to fetch this data. This makes use of the store's indexing capabilities. +7. The dApp displays this data in a user-friendly format, which an end-user leverages when making new transactions against the Ethereum blockchain. +8. And, this cycle repeats. + +### 0.3 What's Needed to Build a Graph Node? +Three repositories are relevant to building on The Graph: +1. [Graph Node](../README.md) – A server implementation for indexing, caching, and serving queries against data from Ethereum. +2. [Graph CLI](https://github.com/graphprotocol/graph-cli) – A CLI for building and compiling projects that are deployed to the Graph Node. +3. [Graph TypeScript Library](https://github.com/graphprotocol/graph-ts) – TypeScript/AssemblyScript library for writing subgraph mappings to be deployed to The Graph. + +### 0.4 Getting Started Overview +Below, we outline the required steps to build a subgraph from scratch, which will serve queries from a GraphQL endpoint. The three major steps are: + +1. [Define the subgraph](#1-define-the-subgraph) + 1. [Define the data sources and create a manifest](#11-define-the-data-sources-and-create-a-manifest) + + 2. [Create the GraphQL schema](#12-create-the-graphql-schema-for-the-data-source) + + 3. [Create a subgraph project and generate types](#13-create-a-subgraph-project-and-generate-types) + + 4. [Write the mappings](#14-writing-mappings) +2. Deploy the subgraph + 1. [Start up an IPFS node](#21-start-up-ipfs) + + 2. [Create the Postgres database](#22-create-the-postgres-db) + + 3. [Start the Graph Node and Connect to an Etheruem node](#23-starting-the-graph-node-and-connecting-to-an-etheruem-node) + + 4. [Deploy the subgraph](#24-deploying-the-subgraph) +3. Query the subgraph + 1. [Query the newly deployed GraphQL API](#3-query-the-local-graph-node) + +Now, let's dig in! + +## 1 Define the Subgraph +When we refer to a subgraph, we reference the entire project that is indexing a chosen set of data. + +To start, create a repository for this project. + +### 1.1 Define the Data Sources and Create a Manifest + +When building a subgraph, you must first decide what blockchain data you want the Graph Node to index. These are known as `dataSources`, which are datasets derived from a blockchain, i.e., an Ethereum smart contract. + +The subgraph is defined by a YAML file known as the **subgraph manifest**. This file should always be named `subgraph.yaml`. View the full specification for the subgraph manifest [here](subgraph-manifest.md). It contains a schema, data sources, and mappings that are used to deploy the GraphQL endpoint. + +Let's go through an example to display what a subgraph manifest looks like. In this case, we use the common ERC721 contract and look at the `Transfer` event because it is familiar to many developers. Below, we define a subgraph manifest with one contract under `dataSources`, which is a smart contract implementing the ERC721 interface: +```yaml +specVersion: 0.0.1 +description: ERC-721 Example +repository: https://github.com//erc721-example +schema: + file: ./schema.graphql +dataSources: +- kind: ethereum/contract + name: MyERC721Contract + network: mainnet + source: + address: "0x06012c8cf97BEaD5deAe237070F9587f8E7A266d" + abi: ERC721 + mapping: + kind: ethereum/events + apiVersion: 0.0.1 + language: wasm/assemblyscript + entities: + - Token + abis: + - name: ERC721 + file: ./abis/ERC721ABI.json + eventHandlers: + - event: Transfer(address,address,uint256) + handler: handleTransfer + file: ./mapping.ts +``` +We point out a few important facts from this example to supplement the [subgraph manifest spec](subgraph-manifest.md): + +* The name `ERC721` under `source > abi` must match the name displayed underneath `abis > name`. +* The event `Transfer(address,address,uint256)` under `eventHandlers` must match what is in the ABI. The name `handleTransfer` under `eventHandlers > handler` must match the name of the mapping function, which we explain in section 1.4. +* Ensure that you have the correct contract address under `source > address`. This is also the case when indexing testnet contracts as well because you might switch back and forth. +* You can define multiple data sources under dataSources. Within a datasource, you can also have multiple `entities` and `events`. See [this subgraph](https://github.com/graphprotocol/decentraland-subgraph/blob/master/subgraph.yaml) for an example. +* If at any point the Graph CLI outputs 'Failed to copy subgraph files', it probably means you have a typo in the manifest. + +#### 1.1.1 Obtain the Contract ABIs +The ABI JSON file must contain the correct ABI to source all the events or any contract state you wish to ingest into the Graph Node. There are a few ways to obtain an ABI for the contract: +* If you are building your own project, you likely have access to your most current ABIs of your smart contracts. +* If you are building a subgraph for a public project, you can download that project to your computer and generate the ABI by using [`truffle compile`](https://truffleframework.com/docs/truffle/overview) or `solc` to compile. This creates the ABI files that you can then transfer to your subgraph `/abi` folder. +* Sometimes, you can also find the ABI on [Etherscan](https://etherscan.io), but this is not always reliable because the uploaded ABI may be out of date. Make sure you have the correct ABI. Otherwise, you will not be able to start a Graph Node. + +If you run into trouble here, double-check the ABI and ensure that the event signatures exist *exactly* as you expect them by examining the smart contract code you are sourcing. Also, note with the ABI, you only need the array for the ABI. Compiling the contracts locally results in a `.json` file that contains the complete ABI nested within the `.json` file under the key `abi`. + +An example `abi` for the `Transfer` event is shown below and would be stored in the `/abi` folder with the name `ERC721ABI.json`: + +```json + [{ + "anonymous": false, + "inputs": [ + { + "indexed": true, + "name": "_from", + "type": "address" + }, + { + "indexed": true, + "name": "_to", + "type": "address" + }, + { + "indexed": true, + "name": "_tokenId", + "type": "uint256" + } + ], + "name": "Transfer", + "type": "event" + }] + ``` + +Once you create this `subgraph.yaml` file, move to the next section. + +### 1.2 Create the GraphQL Schema for the Data Source +GraphQL schemas are defined using the GraphQL interface definition language (IDL). If you have never written a GraphQL schema, we recommend checking out a [quick primer](https://graphql.org/learn/schema/#type-language) on the GraphQL type system. + +With The Graph, rather than defining the top-level `Query` type, you simply define entity types. Then, the Graph Node will generate top-level fields for querying single instances and collections of that entity type. Each entity type is required to be annotated with an `@entity` directive. + +As you see in the example `subgraph.yaml` manifest above, it contains one entity named `Token`. Let's define what that would look like for the GraphQL schema: + +Define a Token entity type: +```graphql +type Token @entity { + id: ID! + currentOwner: Address! +} +``` + +This `entity` tracks a single ERC721 token on Ethereum by its ID and the current owner. The **`ID` field is required** and stores values of the ID type, which are strings. The `ID` must be a unique value so that it can be placed into the store. For an ERC721 token, the unique ID could be the token ID because that value is unique to that coin. + +The exclamation mark represents the fact that that field must be set when the entity is stored in the database, i.e., it cannot be `null`. See the [Schema API](graphql-api.md#3-schema) for a complete reference on defining the schema for The Graph. + +When you complete the schema, add its path to the top-level `schema` key in the subgraph manifest. See the code below for an example: + +```yaml +specVersion: 0.0.1 +schema: + file: ./schema.graphql +``` + +### 1.3 Create a Subgraph Project and Generate Types +Once you have the `subgraph.yaml` manifest and the `./schema.graphql` file, you are ready to use the Graph CLI to set up the subgraph directory. The Graph CLI is a command-line tool that contains helpful commands for deploying the subgraphs. Before continuing with this guide, please go to the [Graph CLI README](https://github.com/graphprotocol/graph-cli/) and follow the instructions up to Step 7 for setting up the subgraph directory. + +Once you run `yarn codegen` as outlined in the [Graph CLI README](https://github.com/graphprotocol/graph-cli/), you are ready to create the mappings. + +`yarn codegen` looks at the contract ABIs defined in the subgraph manifest and generates TypeScript classes for the smart contracts the mappings script will interface with, which includes the types of public methods and events. In reality, the classes are AssemblyScript but more on that later. + +Classes are also generated based on the types defined in the GraphQL schema. These generated classes are incredibly useful for writing correct mappings. This allows you to autocomplete Ethererum events as well as improve developer productivity using the TypeScript language support in your favorite editor or IDE. + +### 1.4 Write the Mappings + +The mappings that you write will perform transformations on the Ethereum data you are sourcing, and it will dictate how this data is loaded into the Graph Node. Mappings can be very simple but can become complex. It depends on how much abstraction you want between the data and the underlying Ethereum contract. + +Mappings are written in a subset of TypeScript called AssemblyScript, which can be compiled down to WASM. AssemblyScript is stricter than normal TypeScript but follows the same backbone. A few TypeScript/JavaScript features that are not supported in AssemblyScript include plain old Javascript objects (POJOs), untyped arrays, untyped maps, union types, the `any` type, and variadic functions. In addition, `switch` statements also work differently. See the [AssemblyScript wiki](https://github.com/AssemblyScript/assemblyscript/wiki) for a full reference on AssemblyScript features. + +In the mapping file, create export functions named after the event handlers in the subgraph manifest. Each handler should accept a single parameter called `event` with a type corresponding to the name of the event that is being handled. This type was generated for you in the previous step, 1.3. + +```typescript +export function handleTransfer(event: Transfer): void { + // Event handler logic goes here +} +``` + +As mentioned, AssemblyScript does not have untyped maps or POJOs, so classes are generated to represent the types defined in the GraphQL schema. The generated type classes handle property type conversions for you, so AssemblyScript's requirement of strictly typed functions is satisfied without the extra work of converting each property explicitly. + +Let's look at an example. Continuing with our previous token example, let's write a mapping that tracks the owner of a particular ERC721 token. + +```typescript + +// This is an example event type generated by `graph-cli` +// from an Ethereum smart contract ABI +import { Transfer } from './types/abis/SomeContract' + +// This is an example of an entity type generated from a +// subgraph's GraphQL schema +import { Token } from './types/schema' + +export function handleTransfer(event: Transfer): void { + let tokenID = event.params.tokenID.toHex() + let token = new Token(tokenID) + token.currentOwner = event.params.to + + token.save() +} +``` +A few things to note from this code: +* We create a new entity named `token`, which is stored in the Graph Node database. +* We create an ID for that token, which must be unique, and then create an entity with `new Token(tokenID)`. We get the token ID from the event emitted by Ethereum, which was turned into an AssemblyScript type by the [Graph TypeScript Library](https://github.com/graphprotocol/graph-ts). We access it at `event.params.tokenId`. Note that you must set `ID` as a string and call `toHex()` on the `tokenID` to turn it into a hex string. +* This entity is updated by the `Transfer` event emitted by the ERC721 contract. +* The current owner is gathered from the event with `event.params.to`. It is set as an Address by the Token class. +* Event handlers functions always return `void`. +* `token.save()` is used to set the Token entity. `.save()` comes from `graph-ts` just like the entity type (`Token` in this example). It is used for setting the value(s) of a particular entity's attribute(s) in the store. There is also a `.load()` function, which will be explained in 1.4.1. + +#### 1.4.1 Use the `save`, `load`, and `remove` entity functions + +The only way that entities may be added to The Graph is by calling `.save()`, which may be called multiple times in an event handler. `.save()` will only set the entity attributes that have explicitly been set on the `entity`. Attributes that are not explicitly set or are unset by calling `Entity.unset()` will not be overwritten. This means you can safely update one field of an entity and not worry about overwriting other fields not referenced in the mapping. + +The definition for `.save()` is: + +```typescript +entity.save() // Entity is representative of the entity type being updated. In our example above, it is Token. +``` + + `.load()` expects the entity type and ID of the entity. Use `.load()` to retrieve information previously added with `.save()`. + +The definition for `.load()` is: + + ```typescript +entity.load() // Entity is representative of the entity type being updated. In our example above, it is Token. +``` + +Once again, all these functions come from the [Graph TypeScript Library](https://github.com/graphprotocol/graph-ts). + +Let's look at the ERC721 token as an example for using `token.load()`. Above, we showed how to use `token.save()`. Now, let's consider that you have another event handler that needs to retrieve the currentOwner of an ERC721 token. To do this within an event handler, you would write the following: + +```typescript + let token = token.load(tokenID.toHex()) + if (token !== null) { + let owner = token.currentOwner + } +``` + +You now have the `owner` data, and you can use that in the mapping to set the owner value to a new entity. + +There is also `.remove()`, which allows you to erase an entry that exists in the store. You simply pass the entity and ID: + +```typescript +entity.remove(ID) +``` + +#### 1.4.2 Call into the Contract Storage to Get Data + +You can also obtain data that is stored in one of the included ABI contracts. Any state variable that is marked `public` or any `view` function can be accessed. Below shows how you obtain the token +symbol of an ERC721 token, which is a state variable of the smart contract. You would add this inside of the event handler function. + +```typescript + let tokenContract = ERC721.bind(event.address); + let tokenSymbol = tokenContract.symbol(); +``` + +Note, we are using an ERC721 class generated from the ABI, which we call bind on. This is gathered from the subgraph manifest here: +```yaml + source: + address: "0x06012c8cf97BEaD5deAe237070F9587f8E7A266d" + abi: ERC721 +``` + +The class is imported from the ABI's TypeScript file generated via `yarn codegen`. + +## 2 Deploy the Subgraph + +### 2.1 Start Up an IPFS Node +To deploy the subgraph to the Graph Node, the subgraph will first need to be built and stored on IPFS, along with all linked files. + +To run an IPFS daemon locally, execute the following: +1. Download and install IPFS. +2. Run `ipfs init`. +3. Run `ipfs daemon`. + +If you encounter problems, follow the instructions from the [IPFS website](https://ipfs.io/docs/getting-started/). + +To confirm the subgraph is stored on IPFS, pass that subgraph ID into `ipfs cat` to view the subgraph manifest with file paths replaced by IPLD links. + +### 2.2 Create the Postgres database + +Ensure that you have Postgres installed. Navigate to a location where you want to save the `.postgres` folder. The desktop is fine since this folder can be used for many different subgraphs. Then, run the following commands: + +``` +initdb -D .postgres +pg_ctl -D .postgres -l logfile start +createdb +``` +Name the database something relevant to the project so that you always know how to access it. + +### 2.3 Start the Graph Node and Connect to an Ethereum Node + +When you start the Graph Node, you need to specify which Ethereum network it should connect to. There are three common ways to do this: + * Infura + * A local Ethereum node + * Ganache + +The Ethereum Network (Mainnet, Ropsten, Rinkeby, etc.) must be passed as a flag in the command that starts the Graph Node as laid out in the following subsections. + +#### 2.3.1 Infura + +[Infura](https://infura.io/) is supported and is the simplest way to connect to an Ethereum node because you do not have to set up your own geth or parity node. However, it does sync slower than being connected to your own node. The following flags are passed to start the Graph Node and indicate you want to use Infura: + +```sh +cargo run -p graph-node --release -- \ + --postgres-url postgresql://<:PASSWORD>@localhost:5432/ \ + --ethereum-rpc :https://mainnet.infura.io \ + --ipfs 127.0.0.1:5001 \ + --debug +``` + +Also, note that the Postgres database may not have a password at all. If that is the case, the Postgres connection URL can be passed as follows: + +` --postgres-url postgresql://@localhost:5432/ \ ` + +#### 2.3.2 Local Geth or Parity Node + +This is the speediest way to get mainnet or testnet data. The problem is that if you do not already have a synced [geth](https://geth.ethereum.org/docs/getting-started) or [parity](https://github.com/paritytech/parity-ethereum) node, you will have to sync one, which takes a very long time and takes up a lot of space. Additionally, note that geth `fast sync` works. So, if you are starting from scratch, this is the fastest way to get caught up, but expect at least 12 hours of syncing on a modern laptop with a good internet connection to sync geth. Normal mode geth or parity will take much longer. Use the following geth command to start syncing: + +`geth --syncmode "fast" --rpc --ws --wsorigins="*" --rpcvhosts="*" --cache 1024` + +Once you have the local node fully synced, run the following command: + +```sh +cargo run -p graph-node --release -- \ + --postgres-url postgresql://<:PASSWORD>@localhost:5432/ \ + --ethereum-rpc :127.0.0.1:8545 \ + --ipfs 127.0.0.1:5001 \ + --debug +``` + +This assumes the local node is on the default `8545` port. If you are on a different port, change it. + +Switching back and forth between sourcing data from Infura and your own local nodes is fine. The Graph Node picks up where it left off. + +#### 2.3.3 Ganache + +**IMPORTANT: Ganache fixed the [issue](https://github.com/trufflesuite/ganache/issues/907) that prevented things from working properly. However, it did not release the new version. Follow the steps in this [issue](https://github.com/graphprotocol/graph-node/issues/375) to run the fixed version locally.** + +[Ganache](https://github.com/trufflesuite/ganache-cli) can be used as well and is preferable for quick testing. This might be an option if you are simply testing out the contracts for quick iterations. Of course, if you close Ganache, then the Graph Node will no longer have any data to source. Ganache is best for short-term projects such as hackathons. Also, it is useful for testing to see that the schema and mappings are working properly before working on the mainnet. + +You can connect the Graph Node to Ganache the same way you connected to a local geth or parity node in the previous section, 2.3.2. Note, however, that Ganache normally runs on port `9545` instead of `8545`. + +#### 2.3.4 Local Parity Testnet + +To set up a local testnet that will allow you to rapidly test the project, download the parity software if you do not already have it. + +This command will work for a one-line install: + +`bash <(curl https://get.parity.io -L)` + +Next, you want to make an account that you can unlock and make transactions on for the parity dev chain. Run the following command: + +`parity account new --chain dev` + +Create a password that you will remember. Take note of the account that gets output. Now, you also have to make that password a text file and pass it into the next command. The desktop is a good location for it. If the password is `123`, only put the numbers in the text file. Do not include any quotes. + +Then, run this command: + +`parity --config dev --unsafe-expose --jsonrpc-cors="all" --unlock --password ~/Desktop/password.txt` + +The chain should start and will be accessible by default on `localhost:8545`. It is a chain with 0 block time and instant transactions, making testing very fast. Passing `unsafe-expose` and `--jsonrpc-cors="all"` as flags allows MetaMask to connect. The `unlock` flag gives parity the ability to send transactions with that account. You can also import the account to MetaMask, which allows you to interact with the test chain directly in your browser. With MetaMask, you need to import the account with the private testnet Ether. The base account that the normal configuration of parity gives you is +`0x00a329c0648769A73afAc7F9381E08FB43dBEA72`. + +The private key is: +``` +4d5db4107d237df6a3d58ee5f70ae63d73d7658d4026f2eefd2f204c81682cb7 (note this is the private key given along with the parity dev chain, so it is okay to share) +``` +Use MetaMask ---> import account ---> private key. + +All the extra information for customization of a parity dev chain is located [here](https://wiki.parity.io/Private-development-chain#customizing-the-development-chain). + +You now have an Ethereum account with a ton of Ether and should be able to set up the migrations on this network and use Truffle. Now, send some Ether to the previous account that was created and unlocked. This way, you can run `truffle migrate` with this account. + +#### 2.3.5 Syncing with a Public Testnet + +If you want to sync using a public testnet such as Kovan, Rinkeby, or Ropsten, just make sure the local node is a testnet node or that you are hitting the correct Infura testnet endpoint. + +### 2.4 Deploy the Subgraph + +When you deploy the subgraph to the Graph Node, it will start ingesting all the subgraph events from the blockchain, transforming that data with the subgraph mappings and storing it in the Graph Node. Note that a running subgraph can safely be stopped and restarted, picking up where it left off. + +Now that the infrastructure is set up, you can run `yarn create-subgraph` and then `yarn deploy` in the subgraph directory. These commands should have been added to `package.json` in section 1.3 when we took a moment to go through the set up for [Graph CLI documentation](https://github.com/graphprotocol/graph-cli). This builds the subgraph and creates the WASM files in the `dist/` folder. Next, it uploads the `dist/ +` files to IPFS and deploys it to the Graph Node. The subgraph is now fully running. + +The `watch` flag allows the subgraph to continually restart every time you save an update to the `manifest`, `schema`, or `mappings`. If you are making many edits or have a subgraph that has been syncing for a few hours, leave this flag off. + +Depending on how many events have been emitted by your smart contracts, it could take less than a minute to get fully caught up. If it is a large contract, it could take hours. For example, ENS takes about 12 to 14 hours to register every single ENS domain. + +## 3 Query the Local Graph Node +With the subgraph deployed to the locally running Graph Node, visit http://127.0.0.1:8000/ to open up a [GraphiQL](https://github.com/graphql/graphiql) interface where you can explore the deployed GraphQL API for the subgraph by issuing queries and viewing the schema. + +We provide a few simple examples below, but please see the [Query API](graphql-api.md#1-queries) for a complete reference on how to query the subgraph's entities. + +Query the `Token` entities: +```graphql +{ + tokens(first: 100) { + id + currentOwner + } +} +``` +Notice that `tokens` is plural and that it will return at most 100 entities. + +Later, when you have deployed the subgraph with this entity, you can query for a specific value, such as the token ID: + +```graphql +{ + token(first: 100, id: "c2dac230ed4ced84ad0ca5dfb3ff8592d59cef7ff2983450113d74a47a12") { + currentOwner + } +} +``` + +You can also sort, filter, or paginate query results. The query below would organize all tokens by their ID and return the current owner of each token. + +```graphql +{ + tokens(first: 100, orderBy: id) { + currentOwner + } +} +``` + +GraphQL provides a ton of functionality. Once again, check out the [Query API](graphql-api.md#1-queries) to find out how to use all supported query features. + +## 4 Changing the Schema, Mappings, and Manifest, and Launching a New Subgraph + +When you first start building the subgraph, it is likely that you will make a few changes to the manifest, mappings, or schema. If you update any of them, rerun `yarn codegen` and `yarn deploy`. This will post the new files on IPFS and deploy the new subgraph. Note that the Graph Node can track multiple subgraphs, so you can do this as many times as you like. + +## 5 Common Patterns for Building Subgraphs + +### 5.1 Removing Elements of an Array in a Subgraph + +Using the AssemblyScript built-in functions for arrays is the way to go. Find the source code [here](https://github.com/AssemblyScript/assemblyscript/blob/18826798074c9fb02243dff76b1a938570a8eda7/std/assembly/array.ts). Using `.indexOf()` to find the element and then using `.splice()` is one way to do so. See this [file](https://github.com/graphprotocol/aragon-subgraph/blob/master/individual-dao-subgraph/mappings/ACL.ts) from the Aragon subgraph for a working implementation. + +### 5.2 Getting Data from Multiple Versions of Your Contracts + +If you have launched multiple versions of your smart contracts onto Ethereum, it is very easy to source data from all of them. This simply requires you to add all versions of the contracts to the `subgraph.yaml` file and handle the events from each contract. Design your schema to consider both versions, and handle any changes to the event signatures that are emitted from each version. See the [0x Subgraph](https://github.com/graphprotocol/0x-subgraph/tree/master/src/mappings) for an implementation of multiple versions of smart contracts being ingested by a subgraph. + +## 5 Example Subgraphs + +Here is a list of current subgraphs that we have open sourced: +* https://github.com/graphprotocol/ens-subgraph +* https://github.com/graphprotocol/decentraland-subgraph +* https://github.com/graphprotocol/adchain-subgraph +* https://github.com/graphprotocol/0x-subgraph +* https://github.com/graphprotocol/aragon-subgraph +* https://github.com/graphprotocol/dharma-subgraph +* https://github.com/daostack/subgraph +* https://github.com/graphprotocol/dydx-subgraph +* https://github.com/livepeer/livepeerjs/tree/master/packages/subgraph +* https://github.com/graphprotocol/augur-subgraph + +## Contributions + +All feedback and contributions in the form of issues and pull requests are welcome! + diff --git a/docs/images/TheGraph_DataFlowDiagram.png b/docs/images/TheGraph_DataFlowDiagram.png new file mode 100644 index 0000000..2891b34 Binary files /dev/null and b/docs/images/TheGraph_DataFlowDiagram.png differ diff --git a/docs/implementation/README.md b/docs/implementation/README.md new file mode 100644 index 0000000..441c5f2 --- /dev/null +++ b/docs/implementation/README.md @@ -0,0 +1,11 @@ +# Implementation Notes + +The files in this directory explain some higher-level concepts about the +implementation of `graph-node`. Explanations that are tied more closely to +the code should go into comments. + +* [Metadata storage](./metadata.md) +* [Schema Generation](./schema-generation.md) +* [Time-travel Queries](./time-travel.md) +* [SQL Query Generation](./sql-query-generation.md) +* [Adding support for a new chain](./add-chain.md) diff --git a/docs/implementation/add-chain.md b/docs/implementation/add-chain.md new file mode 100644 index 0000000..eea6168 --- /dev/null +++ b/docs/implementation/add-chain.md @@ -0,0 +1,279 @@ +# Adding support for a new chain + +## Context + +`graph-node` started as a project that could only index EVM compatible chains, eg: `ethereum`, `xdai`, etc. + +It was known from the start that with growth we would like `graph-node` to be able to index other chains like `NEAR`, `Solana`, `Cosmos`, list goes on... + +However to do it, several refactors were necessary, because the code had a great amount of assumptions based of how Ethereum works. + +At first there was a [RFC](https://github.com/graphprotocol/rfcs/blob/10aaae30fdf82f0dd2ccdf4bbecf7ec6bbfb703b/rfcs/0005-multi-blockchain-support.md) for a design overview, then actual PRs such as: + +- https://github.com/graphprotocol/graph-node/pull/2272 +- https://github.com/graphprotocol/graph-node/pull/2292 +- https://github.com/graphprotocol/graph-node/pull/2399 +- https://github.com/graphprotocol/graph-node/pull/2411 +- https://github.com/graphprotocol/graph-node/pull/2453 +- https://github.com/graphprotocol/graph-node/pull/2463 +- https://github.com/graphprotocol/graph-node/pull/2755 + +All new chains, besides the EVM compatible ones, are integrated using [StreamingFast](https://www.streamingfast.io/)'s [Firehose](https://firehose.streamingfast.io/). The integration consists of chain specific `protobuf` files with the type definitions. + +## How to do it? + +The `graph-node` repository contains multiple Rust crates in it, this section will be divided in each of them that needs to be modified/created. + +> It's important to remember that this document is static and may not be up to date with the current implementation. Be aware too that it won't contain all that's needed, it's mostly listing the main areas that need change. + +### chain + +You'll need to create a new crate in the [chain folder](https://github.com/graphprotocol/graph-node/tree/1cd7936f9143f317feb51be1fc199122761fcbb1/chain) with an appropriate name and the same `version` as the rest of the other ones. + +> Note: you'll probably have to add something like `graph-chain-{{CHAIN_NAME}} = { path = "../chain/{{CHAIN_NAME}}" }` to the `[dependencies]` section of a few other `Cargo.toml` files + +It's here that you add the `protobuf` definitions with the specific types for the chain you're integrating with. Examples: + +- [Ethereum](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/chain/ethereum/proto/codec.proto) +- [NEAR](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/chain/near/proto/codec.proto) +- [Cosmos](https://github.com/graphprotocol/graph-node/blob/caa54c1039d3c282ac31bb0e96cb277dbf82f793/chain/cosmos/proto/type.proto) + +To compile those we use a crate called `tonic`, it will require a [`build.rs` file](https://doc.rust-lang.org/cargo/reference/build-scripts.html) like the one in the other folders/chains, eg: + +```rust +fn main() { + println!("cargo:rerun-if-changed=proto"); + tonic_build::configure() + .out_dir("src/protobuf") + .compile(&["proto/codec.proto"], &["proto"]) + .expect("Failed to compile Firehose CoolChain proto(s)"); +} +``` + +You'll also need a `src/codec.rs` to extract the data from the generated Rust code, much like [this one](https://github.com/graphprotocol/graph-node/blob/caa54c1039d3c282ac31bb0e96cb277dbf82f793/chain/cosmos/src/codec.rs). + +Besides this source file, there should also be a `TriggerFilter`, `NodeCapabilities` and `RuntimeAdapter`, here are a few empty examples: + +`src/adapter.rs` +```rust +use crate::capabilities::NodeCapabilities; +use crate::{data_source::DataSource, Chain}; +use graph::blockchain as bc; +use graph::prelude::*; + +#[derive(Clone, Debug, Default)] +pub struct TriggerFilter {} + +impl bc::TriggerFilter for TriggerFilter { + fn extend<'a>(&mut self, _data_sources: impl Iterator + Clone) {} + + fn node_capabilities(&self) -> NodeCapabilities { + NodeCapabilities {} + } + + fn extend_with_template( + &mut self, + _data_source: impl Iterator::DataSourceTemplate>, + ) { + } + + fn to_firehose_filter(self) -> Vec { + vec![] + } +} +``` + +`src/capabilities.rs` +```rust +use std::cmp::PartialOrd; +use std::fmt; +use std::str::FromStr; + +use anyhow::Error; +use graph::impl_slog_value; + +use crate::DataSource; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd)] +pub struct NodeCapabilities {} + +impl FromStr for NodeCapabilities { + type Err = Error; + + fn from_str(_s: &str) -> Result { + Ok(NodeCapabilities {}) + } +} + +impl fmt::Display for NodeCapabilities { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("{{CHAIN_NAME}}") + } +} + +impl_slog_value!(NodeCapabilities, "{}"); + +impl graph::blockchain::NodeCapabilities for NodeCapabilities { + fn from_data_sources(_data_sources: &[DataSource]) -> Self { + NodeCapabilities {} + } +} +``` + +`src/runtime/runtime_adapter.rs` +```rust +use crate::{Chain, DataSource}; +use anyhow::Result; +use blockchain::HostFn; +use graph::blockchain; + +pub struct RuntimeAdapter {} + +impl blockchain::RuntimeAdapter for RuntimeAdapter { + fn host_fns(&self, _ds: &DataSource) -> Result> { + Ok(vec![]) + } +} +``` + +The chain specific type definitions should also be available for the `runtime`. Since it comes mostly from the `protobuf` files, there's a [generation tool](https://github.com/streamingfast/graph-as-to-rust) made by StreamingFast that you can use to create the `src/runtime/generated.rs`. + +You'll also have to implement `ToAscObj` for those types, that usually is made in a `src/runtime/abi.rs` file. + +Another thing that will be needed is the `DataSource` types for the [subgraph manifest](https://thegraph.com/docs/en/developer/create-subgraph-hosted/#the-subgraph-manifest). + +`src/data_source.rs` +```rust +#[derive(Clone, Debug)] +pub struct DataSource { + // example fields: + pub kind: String, + pub network: Option, + pub name: String, + pub source: Source, + pub mapping: Mapping, + pub context: Arc>, + pub creation_block: Option, + /*...*/ +} + +impl blockchain::DataSource for DataSource { /*...*/ } + +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub struct UnresolvedDataSource { + pub kind: String, + pub network: Option, + pub name: String, + pub source: Source, + pub mapping: UnresolvedMapping, + pub context: Option, +} + +#[async_trait] +impl blockchain::UnresolvedDataSource for UnresolvedDataSource { /*...*/ } + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +pub struct BaseDataSourceTemplate { + pub kind: String, + pub network: Option, + pub name: String, + pub mapping: M, +} + +pub type UnresolvedDataSourceTemplate = BaseDataSourceTemplate; +pub type DataSourceTemplate = BaseDataSourceTemplate; + +#[async_trait] +impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { /*...*/ } + +impl blockchain::DataSourceTemplate for DataSourceTemplate { /*...*/ } +``` + +And at last, the type that will glue them all, the `Chain` itself. + +`src/chain.rs` +```rust +pub struct Chain { /*...*/ } + +#[async_trait] +impl Blockchain for Chain { + const KIND: BlockchainKind = BlockchainKind::CoolChain; + + type Block = codec::...; + + type DataSource = DataSource; + + // ... + + type TriggerFilter = TriggerFilter; + + type NodeCapabilities = NodeCapabilities; + + type RuntimeAdapter = RuntimeAdapter; +} + +pub struct TriggersAdapter { /*...*/ } + +#[async_trait] +impl TriggersAdapterTrait for TriggersAdapter { /*...*/ } + +pub struct FirehoseMapper { + endpoint: Arc, +} + +#[async_trait] +impl FirehoseMapperTrait for FirehoseMapper { /*...*/ } +``` + +### node + +The `src/main.rs` file should be able to handle the connection to the new chain via Firehose for the startup, similar to [this](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/node/src/main.rs#L255). + +### graph + +Two changes are required here: + +1. [BlockchainKind](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/graph/src/blockchain/mod.rs#L309) needs to have a new variant for the chain you're integrating with. +2. And the [IndexForAscTypeId](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/graph/src/runtime/mod.rs#L147) should have the new variants for the chain specific types of the `runtime`. + +### server + +You'll just have to handle the new `BlockchainKind` in the [index-node/src/resolver.rs](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/server/index-node/src/resolver.rs#L361). + +### core + +Just like in the `server` crate, you'll just have to handle the new `BlockchainKind` in the [SubgraphInstanceManager](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/core/src/subgraph/instance_manager.rs#L41). + +## Example Integrations (PRs) + +- NEAR by StreamingFast + - https://github.com/graphprotocol/graph-node/pull/2820 +- Cosmos by Figment + - https://github.com/graphprotocol/graph-node/pull/3212 + - https://github.com/graphprotocol/graph-node/pull/3543 +- Solana by StreamingFast + - https://github.com/graphprotocol/graph-node/pull/3210 + +## What else? + +Besides making `graph-node` support the new chain, [graph-cli](https://github.com/graphprotocol/graph-cli) and [graph-ts](https://github.com/graphprotocol/graph-ts) should also include the new types and enable the new functionality so that subgraph developers can use it. + +For now this document doesn't include how to do that integration, here are a few PRs that might help you with that: + +- NEAR + - `graph-cli` + - https://github.com/graphprotocol/graph-cli/pull/760 + - https://github.com/graphprotocol/graph-cli/pull/783 + - `graph-ts` + - https://github.com/graphprotocol/graph-ts/pull/210 + - https://github.com/graphprotocol/graph-ts/pull/217 +- Cosmos + - `graph-cli` + - https://github.com/graphprotocol/graph-cli/pull/827 + - https://github.com/graphprotocol/graph-cli/pull/851 + - https://github.com/graphprotocol/graph-cli/pull/888 + - `graph-ts` + - https://github.com/graphprotocol/graph-ts/pull/250 + - https://github.com/graphprotocol/graph-ts/pull/273 + +Also this document doesn't include the multi-blockchain part required for The Graph Network, which at this current moment is in progress, for now the network only supports Ethereum `mainnet`. diff --git a/docs/implementation/metadata.md b/docs/implementation/metadata.md new file mode 100644 index 0000000..3183b51 --- /dev/null +++ b/docs/implementation/metadata.md @@ -0,0 +1,142 @@ +# Metadata and how it is stored + +## Mapping subgraph names to deployments + +### `subgraphs.subgraph` + +List of all known subgraph names. Maintained in the primary, but there is a background job that periodically copies the table from the primary to all other shards. Those copies are used for queries when the primary is down. + +| Column | Type | Use | +|-------------------|--------------|-------------------------------------------| +| `id` | `text!` | primary key, UUID | +| `name` | `text!` | user-chosen name | +| `current_version` | `text` | `subgraph_version.id` for current version | +| `pending_version` | `text` | `subgraph_version.id` for pending version | +| `created_at` | `numeric!` | UNIX timestamp | +| `vid` | `int8!` | unused | +| `block_range` | `int4range!` | unused | + +The `id` is used by the hosted explorer to reference the subgraph. + + +### `subgraphs.subgraph_version` + +Mapping of subgraph names from `subgraph` to IPFS hashes. Maintained in the primary, but there is a background job that periodically copies the table from the primary to all other shards. Those copies are used for queries when the primary is down. + +| Column | Type | Use | +|---------------|--------------|-------------------------| +| `id` | `text!` | primary key, UUID | +| `subgraph` | `text!` | `subgraph.id` | +| `deployment` | `text!` | IPFS hash of deployment | +| `created_at` | `numeric` | UNIX timestamp | +| `vid` | `int8!` | unused | +| `block_range` | `int4range!` | unused | + + +## Managing a deployment + +Directory of all deployments. Maintained in the primary, but there is a background job that periodically copies the table from the primary to all other shards. Those copies are used for queries when the primary is down. + +### `deployment_schemas` + +| Column | Type | Use | +|--------------|----------------|----------------------------------------------| +| `id` | `int4!` | primary key | +| `subgraph` | `text!` | IPFS hash of deployment | +| `name` | `text!` | name of `sgdNNN` schema | +| `version` | `int4!` | version of data layout in `sgdNNN` | +| `shard` | `text!` | database shard holding data | +| `network` | `text!` | network/chain used | +| `active` | `boolean!` | whether to query this copy of the deployment | +| `created_at` | `timestamptz!` | | + +There can be multiple copies of the same deployment, but at most one per shard. The `active` flag indicates which of these copies will be used for queries; `graph-node` makes sure that there is always exactly one for each IPFS hash. + +### `subgraph_deployment` + +Details about a deployment to track sync progress etc. Maintained in the +shard alongside the deployment's data in `sgdNNN`. The table should only +contain frequently changing data, but for historical reasons contains also +static data. + +| Column | Type | Use | +|--------------------------------------|------------|----------------------------------------------| +| `id` | `integer!` | primary key, same as `deployment_schemas.id` | +| `deployment` | `text!` | IPFS hash | +| `failed` | `boolean!` | | +| `synced` | `boolean!` | | +| `earliest_ethereum_block_hash` | `bytea` | start block from manifest (to be removed) | +| `earliest_ethereum_block_number` | `numeric` | | +| `latest_ethereum_block_hash` | `bytea` | current subgraph head | +| `latest_ethereum_block_number` | `numeric` | | +| `entity_count` | `numeric!` | total number of entities | +| `graft_base` | `text` | IPFS hash of graft base | +| `graft_block_hash` | `bytea` | graft block | +| `graft_block_number` | `numeric` | | +| `reorg_count` | `integer!` | | +| `current_reorg_depth` | `integer!` | | +| `max_reorg_depth` | `integer!` | | +| `fatal_error` | `text` | | +| `non_fatal_errors` | `text[]` | | +| `health` | `health!` | | +| `last_healthy_ethereum_block_hash` | `bytea` | | +| `last_healthy_ethereum_block_number` | `numeric` | | +| `firehose_cursor` | `text` | | +| `debug_fork` | `text` | | + +The columns `reorg_count`, `current_reorg_depth`, and `max_reorg_depth` are +set during indexing. They are used to determine whether a reorg happened +while a query was running, and whether that reorg could have affected the +query. + +### `subgraph_manifest` + +Details about a deployment that rarely change. Maintained in the +shard alongside the deployment's data in `sgdNNN`. + +| Column | Type | Use | +|-------------------------|------------|------------------------------------------------------| +| `id` | `integer!` | primary key, same as `deployment_schemas.id` | +| `spec_version` | `text!` | | +| `description` | `text` | | +| `repository` | `text` | | +| `schema` | `text!` | GraphQL schema | +| `features` | `text[]!` | | +| `graph_node_version_id` | `integer` | | +| `use_bytea_prefix` | `boolean!` | | +| `start_block_hash` | `bytea` | Parent of the smallest start block from the manifest | +| `start_block_number` | `int4` | | + +### `subgraph_deployment_assignment` + +Tracks which index node is indexing a deployment. Maintained in the primary, +but there is a background job that periodically copies the table from the +primary to all other shards. + +| Column | Type | Use | +|---------|-------|---------------------------------------------| +| id | int4! | primary key, ref to `deployment_schemas.id` | +| node_id | text! | name of index node | + +This table could simply be a column on `deployment_schemas`. + +### `dynamic_ethereum_contract_data_source` + +Stores the dynamic data sources for all subgraphs (will be turned into a +table that lives in each subgraph's namespace `sgdNNN` soon) + +### `subgraph_error` + +Stores details about errors that subgraphs encounter during indexing. + +### Copying of deployments + +The tables `active_copies` in the primary, and `subgraphs.copy_state` and +`subgraphs.copy_table_state` are used to track which deployments need to be +copied and how far copying has progressed to make sure that copying works +correctly across index node restarts. + +### Influencing query generation + +The table `subgraphs.table_stats` stores which tables for a deployment +should have the 'account-like' optimization turned on. diff --git a/docs/implementation/schema-generation.md b/docs/implementation/schema-generation.md new file mode 100644 index 0000000..c8bba83 --- /dev/null +++ b/docs/implementation/schema-generation.md @@ -0,0 +1,125 @@ +# Schema Generation + +This document describes how we go from a GraphQL schema to a relational +table definition in Postgres. + +Schema generation follows a few simple rules: + +* the data for a subgraph is entirely stored in a Postgres namespace whose + name is `sgdNNNN`. The mapping between namespace name and deployment id is + kept in `deployment_schemas` +* the data for each entity type is stored in a table whose structure follows + the declaration of the type in the GraphQL schema +* enums in the GraphQL schema are stored as enum types in Postgres +* interfaces are not stored in the database, only the concrete types that + implement the interface are stored + +Any table for an entity type has the following structure: + +```sql + create table sgd42.account( + vid int8 serial primary key, + id text not null, -- or bytea + .. attributes .. + block_range int4range not null + ) +``` + +The `vid` is used in some situations to uniquely identify the specific +version of an entity. The `block_range` is used to enable [time-travel +queries](./time-travel.md). + +The attributes of the GraphQL type correspond directly to columns in the +generated table. The types of these columns are + +* the `id` column can have type `ID`, `String`, and `Bytes`, where `ID` is + an alias for `String` for historical reasons. +* if the attribute has a primitive type, the column has the SQL type that + most closely mirrors the GraphQL type. `BigDecimal` and `BigInt` are + stored as `numeric`, `Bytes` is stored as `bytea`, etc. +* if the attribute references another entity, the column has the type of the + `id` type of the referenced entity type. We do not use foreign key + constraints to allow storing an entity that references an entity that will + only be created later. Foreign key constraint violations will therefore + only be detected when a query is issued, or simply lead to the reference + missing from the query result. +* if the attribute has an enum type, we generate a SQL enum type and use + that as the type of the column. +* if the attribute has a list type, like `[String]`, the corresponding + column uses an array type. We do not allow nested arrays like `[[String]]` + in GraphQL, so arrays will only ever contain entries of a primitive type. + +### Immutable entities + +Entity types declared with a plain `@entity` in the GraphQL schema are +mutable, and the above table design enables selecting one of many versions +of the same entity, depending on the block height at which the query is +run. In a lot of cases, the subgraph author knows that entities will never +be mutated, e.g., because they are just a direct copy of immutable chain data, +like a transfer. In those cases, we know that the upper end of the block +range will always be infinite and don't need to store that explicitly. + +When an entity type is declared with `@entity(immutable: true)` in the +GraphQL schema, we do not generate a `block_range` column in the +corresponding table. Instead, we generate a column `block$ int not null`, +so that the check whether a row is visible at block `B` simplifies to +`block$ <= B`. + +Furthermore, since each entity can only have one version, we also add a +constraint `unique(id)` to such tables, and can avoid expensive GiST +indexes in favor of simple BTree indexes since the `block$` column is an +integer. + +## Indexing + +We do not know ahead of time which queries will be issued and therefore +build indexes extensively. This leads to serious overindexing, but both +reducing the overindexing and making it possible to generate custom indexes +are open issues at this time. + +We generate the following indexes for each table: + +* for mutable entity types + * an exclusion index over `(id, block_range)` that ensures that the + versions for the same entity `id` have disjoint block ranges + * a BRIN index on `(lower(block_range), COALESCE(upper(block_range), + 2147483647), vid)` that helps speed up some operations, especially + reversion, in tables that have good data locality, for example, tables + where entities are never updated or deleted +* for immutable entity types + * a unique index on `id` + * a BRIN index on `(block$, vid)` +* for each attribute, an index called `attr_N_M_..` where `N` is the number + of the entity type in the GraphQL schema, and `M` is the number of the + attribute within that type. For attributes of a primitive type, the index + is a BTree index. For attributes that reference other entities, the index + is a GiST index on `(attribute, block_range)` + +### Indexes on String Attributes + +In some cases, `String` attributes are used to store large pieces of text, +text that is longer than the limit that Postgres imposes on individual index +entries. For such attributes, we therefore index `left(attribute, +STRING_PREFIX_SIZE)`. When we generate queries, query generation makes sure +that this index is usable by adding additional clauses to the query that use +`left(attribute, STRING_PREFIX_SIZE)` in the query. For example, if a query +was looking for entities where the `name` equals `"Hamming"`, the query +would contain a clause `left(name, STRING_PREFIX_SIZE) = 'Hamming'`. + +## Known Issues + +- Storing arrays as array attributes in Postgres can have catastrophically + bad performance if the size of the array is not bounded by a relatively + small number. +- Overindexing leads to large amounts of storage used for indexes, and, of + course, slows down writes. +- At the same time, indexes are not always usable. For example, a BTree + index on `name` is not usable for sorting entities, since we always add + `id` to the `order by` clause, i.e., when a user asks for entities ordered + by `name`, we actually include `order by name, id` in the SQL query to + guarantee an unambiguous ordering. Incremental sorting in Postgres 13 + might help with that. +- Lack of support for custom indexes makes it hard to transfer manually + created indexes between different versions of the same subgraph. By + convention, manually created indexes should have a name that starts with + `manual_`. diff --git a/docs/implementation/sql-query-generation.md b/docs/implementation/sql-query-generation.md new file mode 100644 index 0000000..e88602a --- /dev/null +++ b/docs/implementation/sql-query-generation.md @@ -0,0 +1,499 @@ +## SQL Query Generation + +### Goal + +For a GraphQL query of the form + +```graphql +query { + parents(filter) { + id + children(filter) { + id + } + } +} +``` + +we want to generate only two SQL queries: one to get the parents, and one to +get the children for all those parents. The fact that `children` is nested +under `parents` requires that we add a filter to the `children` query that +restricts children to those that are related to the parents we fetched in +the first query to get the parents. How exactly we filter the `children` +query depends on how the relationship between parents and children is +modeled in the GraphQL schema, and on whether one (or both) of the types +involved are interfaces. + +The rest of this writeup is concerned with how to generate the query for +`children`, assuming we already retrieved the list of all parents. + +The bulk of the implementation of this feature can be found in +`graphql/src/store/prefetch.rs`, `store/postgres/src/relational.rs`, and +`store/postgres/src/relational_queries.rs` + + +### Handling first/skip + +We never get all the `children` for a parent; instead we always have a +`first` and `skip` argument in the children filter. Those arguments need to +be applied to each parent individually by ranking the children for each +parent according to the order defined by the `children` query. If the same +child matches multiple parents, we need to make sure that it is considered +separately for each parent as it might appear at different ranks for +different parents. In SQL, we use a lateral join, essentially a for loop. +For children that store the id of their parent in `parent_id`, we'd run the +following query: + +```sql +select c.*, p.id + from unnest({parent_ids}) as p(id) + cross join lateral + (select * + from children c + where c.parent_id = p.id + and .. other conditions on c .. + order by c.{sort_key}, c.id + limit {first} + offset {skip}) c + order by p.id, c.{sort_key}, c.id +``` + +Note that we order children by the sort key the user specified, followed by +the `id` to guarantee an unambiguous order even if the sort key is a +non-unique column. Unfortunately, we do not know which attributes of an +entity are unique and which ones aren't. + +### Handling parent/child relationships + +How we get the children for a set of parents depends on how the relationship +between the two is modeled. The interesting parameters there are whether +parents store a list or a single child, and whether that field is derived, +together with the same for children. + +There are a total of 16 combinations of these four boolean variables; four +of them, when both parent and child derive their fields, are not +permissible. It also doesn't matter whether the child derives its parent +field: when the parent field is not derived, we need to use that since that +is the only place that contains the parent -> child relationship. When the +parent field is derived, the child field can not be a derived field. + +That leaves us with eight combinations of whether the parent and child store +a list or a scalar value, and whether the parent is derived. For details on +the GraphQL schema for each row in this table, see the section at the end. +The `Join cond` indicates how we can find the children for a given parent. +The table refers to the four different kinds of join condition we might need +as types A, B, C, and D. + +| Case | Parent list? | Parent derived? | Child list? | Join cond | Type | +|------|--------------|-----------------|-------------|----------------------------|------| +| 1 | TRUE | TRUE | TRUE | child.parents ∋ parent.id | A | +| 2 | FALSE | TRUE | TRUE | child.parents ∋ parent.id | A | +| 3 | TRUE | TRUE | FALSE | child.parent = parent.id | B | +| 4 | FALSE | TRUE | FALSE | child.parent = parent.id | B | +| 5 | TRUE | FALSE | TRUE | child.id ∈ parent.children | C | +| 6 | TRUE | FALSE | FALSE | child.id ∈ parent.children | C | +| 7 | FALSE | FALSE | TRUE | child.id = parent.child | D | +| 8 | FALSE | FALSE | FALSE | child.id = parent.child | D | + +In addition to how the data about the parent/child relationship is stored, +the multiplicity of the parent/child relationship also influences query +generation: if each parent can have at most a single child, queries can be +much simpler than if we have to account for multiple children per parent, +which requires paginating them. We also need to detect cases where the +mappings created multiple children per parent. We do this by adding a clause +`limit {parent_ids.len} + 1` to the query, so that if there is one parent +with multiple children, we will select it, but still protect ourselves +against mappings that produce catastrophically bad data with huge numbers of +children per parent. The GraphQL execution logic will detect that there is a +parent with multiple children, and generate an error. + +When we query children, we already have a list of all parents from running a +previous query. To find the children, we need to have the id of the parent +that child is related to, and, when the parent stores the ids of its +children directly (types C and D) the child ids for each parent id. + +The following queries all produce a relation that has the same columns as +the table holding children, plus a column holding the id of the parent that +the child belongs to. + +#### Type A + +Use when child stores a list of parents + +Data needed to generate: + +- children: name of child table +- parent_ids: list of parent ids +- parent_field: name of parents field (array) in child table +- single: boolean to indicate whether a parent has at most one child or + not + +The implementation uses an `EntityLink::Direct` for joins of this type. + +##### Multiple children per parent +```sql +select c.*, p.id as parent_id + from unnest({parent_ids}) as p(id) + cross join lateral + (select * + from children c + where p.id = any(c.{parent_field}) + and .. other conditions on c .. + order by c.{sort_key} + limit {first} offset {skip}) c + order by c.{sort_key} +``` + +##### Single child per parent +```sql +select c.*, p.id as parent_id + from unnest({parent_ids}) as p(id), + children c + where c.{parent_field} @> array[p.id] + and .. other conditions on c .. + limit {parent_ids.len} + 1 +``` + +#### Type B + +Use when child stores a single parent + +Data needed to generate: + +- children: name of child table +- parent_ids: list of parent ids +- parent_field: name of parent field (scalar) in child table +- single: boolean to indicate whether a parent has at most one child or + not + +The implementation uses an `EntityLink::Direct` for joins of this type. + +##### Multiple children per parent +```sql +select c.*, p.id as parent_id + from unnest({parent_ids}) as p(id) + cross join lateral + (select * + from children c + where p.id = c.{parent_field} + and .. other conditions on c .. + order by c.{sort_key} + limit {first} offset {skip}) c + order by c.{sort_key} +``` + +##### Single child per parent + +```sql +select c.*, c.{parent_field} as parent_id + from children c + where c.{parent_field} = any({parent_ids}) + and .. other conditions on c .. + limit {parent_ids.len} + 1 +``` + +Alternatively, this is worth a try, too: +```sql +select c.*, c.{parent_field} as parent_id + from unnest({parent_ids}) as p(id), children c + where c.{parent_field} = p.id + and .. other conditions on c .. + limit {parent_ids.len} + 1 +``` + +#### Type C + +Use when the parent stores a list of its children. + +Data needed to generate: + +- children: name of child table +- parent_ids: list of parent ids +- child\_id_matrix: array of arrays where `child_id_matrix[i]` is an array + containing the ids of the children for `parent_id[i]` + +The implementation uses a `EntityLink::Parent` for joins of this type. + +##### Multiple children per parent + +```sql +select c.*, p.id as parent_id + from rows from (unnest({parent_ids}), reduce_dim({child_id_matrix})) + as p(id, child_ids) + cross join lateral + (select * + from children c + where c.id = any(p.child_ids) + and .. other conditions on c .. + order by c.{sort_key} + limit {first} offset {skip}) c + order by c.{sort_key} +``` + +Note that `reduce_dim` is a custom function that is not part of [ANSI +SQL:2016](https://en.wikipedia.org/wiki/SQL:2016) but is needed as there is +no standard way to decompose a matrix into a table where each row contains +one row of the matrix. The `ROWS FROM` construct is also not part of ANSI +SQL. + +##### Single child per parent + +Not possible with relations of this type + +#### Type D + +Use when parent is not a list and not derived + +Data needed to generate: + +- children: name of child table +- parent_ids: list of parent ids +- child_ids: list of the id of the child for each parent such that + `child_ids[i]` is the id of the child for `parent_id[i]` + +The implementation uses a `EntityLink::Parent` for joins of this type. + +##### Multiple children per parent + +Not possible with relations of this type + +##### Single child per parent + +```sql +select c.*, p.id as parent_id + from rows from (unnest({parent_ids}), unnest({child_ids})) as p(id, child_id), + children c + where c.id = p.child_id + and .. other conditions on c .. +``` + +If the list of unique `child_ids` is small enough, we also add a where +clause `c.id = any({ unique child_ids })`. The list is small enough if it +contains fewer than `TYPED_CHILDREN_SET_SIZE` (default: 150) unique child +ids. + + +The `ROWS FROM` construct is not part of ANSI SQL. + +### Handling interfaces + +If the GraphQL type of the children is an interface, we need to take +special care to form correct queries. Whether the parents are +implementations of an interface or not does not matter, as we will have a +full list of parents already loaded into memory when we build the query for +the children. Whether the GraphQL type of the parents is an interface may +influence from which parent attribute we get child ids for queries of type +C and D. + +When the GraphQL type of the children is an interface, we resolve the +interface type into the concrete types implementing it, produce a query for +each concrete child type and combine those queries via `union all`. + +Since implementations of the same interface will generally differ in the +schema they use, we can not form a `union all` of all the data in the +tables for these concrete types, but have to first query only attributes +that we know will be common to all entities implementing the interface, +most notably the `vid` (a unique identifier that identifies the precise +version of an entity), and then later fill in the details of each entity by +converting it directly to JSON. A second reason to pass entities as JSON +from the database is that it is impossible with Diesel to execute queries +where the number and types of the columns of the result are not known at +compile time. + +We need to to be careful though to not convert to JSONB too early, as that +is slow when done for large numbers of rows. Deferring conversion is +responsible for some of the complexity in these queries. + +That means that when we deal with children that are an interface, we will +first select only the following columns from each concrete child type +(where exactly they come from depends on how the parent/child relationship +is modeled) + +```sql +select '{__typename}' as entity, c.vid, c.id, c.{sort_key}, p.id as parent_id +``` + +and then use that data to fill in the complete details of each concrete +entity. The query `type_query(children)` is the query from the previous +section according to the concrete type of `children`, but without the +`select`, `limit`, `offset` or `order by` clauses. The overall structure of +this query then is + +```sql +with matches as ( + select '{children.object}' as entity, c.vid, c.id, + c.{sort_key}, p.id as parent_id + from .. type_query(children) .. + union all + .. range over all child types .. + order by {sort_key} + limit {first} offset {skip}) +select m.*, to_jsonb(c.*) as data + from matches m, {children.table} c + where c.vid = m.vid and m.entity = '{children.object}' + union all + .. range over all child tables .. + order by {sort_key} +``` + +The list `all_parent_ids` must contain the ids of all the parents for which +we want to find children. + +We have one `children` object for each concrete GraphQL type that we need +to query, where `children.table` is the name of the database table in which +these entities are stored, and `children.object` is the GraphQL typename +for these children. + +The code uses an `EntityCollection::Window` containing multiple +`EntityWindow` instances to represent the most general form of querying for +the children of a set of parents, the query given above. + +When there is only one window, we can simplify the above query. The +simplification basically inlines the `matches` CTE. That is important as +CTE's in Postgres before Postgres 12 are optimization fences, even when +they are only used once. We therefore reduce the two queries that Postgres +executes above to one for the fairly common case that the children are not +an interface. For each type of parent/child relationship, the resulting +query is essentially the same as the one given in the section +`Handling parent/child relationships`, except that the `select` clause is +changed to `select '{window.child_type}' as entity, to_jsonb(c.*) as data`: + +```sql +select '..' as entity, to_jsonb(e.*) as data, p.id as parent_id + from {expand_parents} + cross join lateral + (select * + from children c + where {linked_children} + and .. other conditions on c .. + order by c.{sort_key} + limit {first} offset {skip}) c + order by c.{sort_key} +``` + +Toplevel queries, i.e., queries where we have no parents, and therefore do +not restrict the children we return by parent ids are represented in the +code by an `EntityCollection::All`. If the GraphQL type of the children is +an interface with multiple implementers, we can simplify the query by +avoiding ranking and just using an ordinary `order by` clause: + +```sql +with matches as ( + -- Get uniform info for all matching children + select '{entity_type}' as entity, id, vid, {sort_key} + from {entity_table} c + where {query_filter} + union all + ... range over all entity types + order by {sort_key} offset {query.skip} limit {query.first}) +-- Get the full entity for each match +select m.entity, to_jsonb(c.*) as data, c.id, c.{sort_key} + from matches m, {entity_table} c + where c.vid = m.vid and m.entity = '{entity_type}' + union all + ... range over all entity types + -- Make sure we return the children for each parent in the correct order + order by c.{sort_key}, c.id +``` + +And finally, for the very common case of a toplevel GraphQL query for a +concrete type, not an interface, we can further simplify this, again by +essentially inlining the `matches` CTE to: + +```sql +select '{entity_type}' as entity, to_jsonb(c.*) as data + from {entity_table} c + where query.filter() + order by {query.order} offset {query.skip} limit {query.first} +``` + +## Boring list of possible GraphQL models + +These are the eight ways in which a parent/child relationship can be +modeled. For brevity, the `id` attribute on each parent and child type has +been left out. + +This list assumes that parent and child types are concrete types, i.e., that +any interfaces involved in this query have already been resolved into their +implementations and we are dealing with one pair of concrete parent/child +types. + +```graphql +# Case 1 +type Parent { + children: [Child] @derived +} + +type Child { + parents: [Parent] +} + +# Case 2 +type Parent { + child: Child @derived +} + +type Child { + parents: [Parent] +} + +# Case 3 +type Parent { + children: [Child] @derived +} + +type Child { + parent: Parent +} + +# Case 4 +type Parent { + child: Child @derived +} + +type Child { + parent: Parent +} + +# Case 5 +type Parent { + children: [Child] +} + +type Child { + # doesn't matter +} + +# Case 6 +type Parent { + children: [Child] +} + +type Child { + # doesn't matter +} + +# Case 7 +type Parent { + child: Child +} + +type Child { + # doesn't matter +} + +# Case 8 +type Parent { + child: Child +} + +type Child { + # doesn't matter +} +``` + +## Resources + +* [PostgreSQL Manual](https://www.postgresql.org/docs/12/index.html) +* [Browsable SQL Grammar](https://jakewheat.github.io/sql-overview/sql-2016-foundation-grammar.html) +* [Wikipedia entry on ANSI SQL:2016](https://en.wikipedia.org/wiki/SQL:2016) The actual standard is not freely available diff --git a/docs/implementation/time-travel.md b/docs/implementation/time-travel.md new file mode 100644 index 0000000..15433e3 --- /dev/null +++ b/docs/implementation/time-travel.md @@ -0,0 +1,133 @@ +# Time-travel queries + +Time-travel queries make it possible to query the state of a subgraph at +a given block. Assume that a subgraph has an `Account` entity defined +as + +```graphql +type Account @entity { + id ID! + balance BigInt +} +``` + +The corresponding database table for that will have the form +```sql +create table account( + vid int8 primary key, + id text not null, + balance numeric, + block_range int4range not null, + exclude using gist(id with =, block_range with &&) +); +``` + +The `account` table will contain one entry for each version of each account; +that means that for the account with `id = "1"`, there will be multiple rows +in the database, but with different block ranges. The exclusion constraint +makes sure that the block ranges for any given entity do not overlap. + +The block range indicates from which block (inclusive) to which block +(exclusive) an entity version is valid. The most recent version of an entity +has a block range with an unlimited upper bound. The block range `[7, 15)` +means that this version of the entity should be used for queries that want +to know the state of the account if the query asks for block heights between +7 and 14. + +A nice benefit of this approach is that we do not modify the data for +existing entities. The only attribute of an entity that can ever be modified +is the range of blocks for which a specific entity version is valid. This +will become particularly significant once zHeap is fully integrated into +Postgres (anticipated for Postgres 14) + +For background on ranges in Postgres, see the +[rangetypes](https://www.postgresql.org/docs/9.6/rangetypes.html) and +[range operators](https://www.postgresql.org/docs/9.6/functions-range.html) +chapters in the documentation. + +### Immutable entities + +For entity types declared with `@entity(immutable: true)`, the table has a +`block$ int not null` column instead of a `block_range` column, where the +`block$` column stores the value that would be stored in +`lower(block_range)`. Since the upper bound of the block range for an +immutable entity is always infinite, a test like `block_range @> $B`, which +is equivalent to `lower(block_range) <= $B and upper(block_range) > $B`, +can be simplified to `block$ <= $B`. + +The operations in the next section are adjusted accordingly for immutable +entities. + +## Operations + +For all operations, we assume that we perform them for block number `B`; +for most of them we only focus on how the `block_range` is used and +manipulated. The values for omitted columns should be clear from context. + +For deletion and update, we only modify the current version, + +### Querying for a point-in-time + +Any query that selects entities will have a condition added to it that +requires that the `block_range` for the entity must include `B`: + +```sql + select .. from account + where .. + and block_range @> $B +``` + +### Create entity + +Creating an entity consist of writing an entry with a block range marking +it valid from `B` to infinity: + +```sql + insert into account(id, block_range, ...) + values ($ID, '[$B,]', ...); +``` + +### Delete entity + +Only the current version of an entity can be deleted. For that version, +deleting it consists of clamping the block range at `B`: + +```sql + update account + set block_range = int4range(lower(block_range), $B) + where id = $ID and block_range @> $INTMAX; +``` + +Note that this operation is not allowed for immutable entities. + +### Update entity + +Only the current version of an entity can be updated. An update is performed +as a deletion followed by an insertion. + +Note that this operation is not allowed for immutable entities. + +### Rolling back + +When we need to revert entity changes that happened for blocks with numbers +higher than `B`, we delete all entities which would only be valid 'in the +future', and then open the range of the one entity entry for which the +range contains `B`, thereby marking it as the current version: + +```sql + delete from account lower(block_range) >= $B; + + update account + set block_range = int4range(lower(block_range), NULL) + where block_range @> $B; +``` + +## Notes + +- It is important to note that the block number does not uniquely identify a + block, only the block hash does. But within our database, at any given + moment in time, we can identify the block for a given block number and + subgraph by following the chain starting at the subgraph's block pointer + back. In practice, the query to do that is expensive for blocks far away + from the subgraph's head block, but so far we have not had a need for + that. diff --git a/docs/maintenance.md b/docs/maintenance.md new file mode 100644 index 0000000..350b415 --- /dev/null +++ b/docs/maintenance.md @@ -0,0 +1,51 @@ +# Common maintenance tasks + +This document explains how to perform common maintenance tasks using +`graphman`. The `graphman` command is included in the official containers, +and you can `docker exec` into your `graph-node` container to run it. It +requires a [configuration +file](https://github.com/graphprotocol/graph-node/blob/master/docs/config.md). If +you are not using one already, [these +instructions](https://github.com/graphprotocol/graph-node/blob/master/docs/config.md#basic-setup) +show how to create a minimal configuration file that works for `graphman`. + +The command pays attention to the `GRAPH_LOG` environment variable, and +will print normal `graph-node` logs on stdout. You can turn them off by +doing `unset GRAPH_LOG` before running `graphman`. + +A simple way to check that `graphman` is set up correctly is to run +`graphman info some/subgraph`. If that subgraph exists, the command will +print basic information about it, like the namespace in Postgres that +contains the data for the underlying deployment. + +## Removing unused deployments + +When a new version of a subgraph is deployed, the new deployment displaces +the old one when it finishes syncing. At that point, the system will not +use the old deployment anymore, but its data is still in the database. + +These unused deployments can be removed by running `graphman unused record` +which compiles a list of unused deployments. That list can then be +inspected with `graphman unused list -e`. The data for these unused +deployments can then be removed with `graphman unused remove` which will +only remove the deployments that have previously marked for removal with +`record`. + +## Removing a subgraph + +The command `graphman remove some/subgraph` will remove the mapping from +the given name to the underlying deployment. If no other subgraph name uses +that deployment, it becomes eligible for removal, and the steps for +removing unused deployments will delete its data. + +## Modifying assignments + +Each deployment is assigned to a specific `graph-node` instance for +indexing. It is possible to change the `graph-node` instance that indexes a +given subgraph with `graphman reassign`. To permanently stop indexing it, +use `graphman unassign`. Unfortunately, `graphman` does not currently allow +creating an assignment for an unassigned deployment; it is possible to +assign a deployment to a node that does not exist, which will also stop +indexing it, for example by assigning it to a node `paused_`. Indexing can then be resumed by reassigning the deployment to an +existing node. diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 0000000..7a545b1 --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,69 @@ +graph-node provides the following metrics via Prometheus endpoint on 8040 port by default: +- `deployment_block_processing_duration` +Measures **duration of block processing** for a subgraph deployment +- `deployment_block_trigger_count` +Measures the **number of triggers in each** block for a subgraph deployment +- `deployment_count` +Counts the number of deployments currently being indexed by the graph-node. +- `deployment_eth_rpc_errors` +Counts **eth** **rpc request errors** for a subgraph deployment +- `deployment_eth_rpc_request_duration` +Measures **eth** **rpc request duration** for a subgraph deployment +- `deployment_failed` +Boolean gauge to indicate **whether the deployment has failed** (1 == failed) +- `deployment_handler_execution_time` +Measures the **execution time for handlers** +- `deployment_head` +Track the **head block number** for a deployment. Example: + +```protobuf +deployment_head{deployment="QmaeWFYbPwmXEk7UuACmkqgPq2Pba5t2RYdJtEyvAUmrxg",network="mumbai",shard="primary"} 19509077 +``` + +- `deployment_host_fn_execution_time` +Measures the **execution time for host functions** +- `deployment_reverted_blocks` +Track the **last reverted block** for a subgraph deployment +- `deployment_sync_secs` +total **time spent syncing** +- `deployment_transact_block_operations_duration` +Measures **duration of commiting all the entity operations** in a block and **updating the subgraph pointer** +- `deployment_trigger_processing_duration` +Measures **duration of trigger processing** for a subgraph deployment +- `eth_rpc_errors` +Counts **eth rpc request errors** +- `eth_rpc_request_duration` +Measures **eth rpc request duration** +- `ethereum_chain_head_number` +Block **number of the most recent block synced from Ethereum**. Example: + +```protobuf +ethereum_chain_head_number{network="mumbai"} 20045294 +``` + +- `metrics_register_errors` +Counts **Prometheus metrics register errors** +- `metrics_unregister_errors` +Counts **Prometheus metrics unregister errors** +- `query_cache_status_count` +Count **toplevel GraphQL fields executed** and their cache status +- `query_effort_ms` +Moving **average of time spent running queries** +- `query_execution_time` +**Execution time for successful GraphQL queries** +- `query_result_max` +the **maximum size of a query result** (in CacheWeight) +- `query_result_size` +the **size of the result of successful GraphQL queries** (in CacheWeight) +- `query_semaphore_wait_ms` +Moving **average of time spent on waiting for postgres query semaphore** +- `query_kill_rate` +The rate at which the load manager kills queries +- `registered_metrics` +Tracks the **number of registered metrics** on the node +- `store_connection_checkout_count` +The **number of Postgres connections** currently **checked out** +- `store_connection_error_count` +The **number of Postgres connections errors** +- `store_connection_wait_time_ms` +**Average connection wait time** diff --git a/docs/subgraph-manifest.md b/docs/subgraph-manifest.md new file mode 100644 index 0000000..d1ba64d --- /dev/null +++ b/docs/subgraph-manifest.md @@ -0,0 +1,158 @@ +# Subgraph Manifest +##### v.0.0.4 + +## 1.1 Overview +The subgraph manifest specifies all the information required to index and query a specific subgraph. This is the entry point to your subgraph. + +The subgraph manifest, and all the files linked from it, is what is deployed to IPFS and hashed to produce a subgraph ID that can be referenced and used to retrieve your subgraph in The Graph. + +## 1.2 Format +Any data format that has a well-defined 1:1 mapping with the [IPLD Canonical Format](https://github.com/ipld/specs/) may be used to define a subgraph manifest. This includes YAML and JSON. Examples in this document are in YAML. + +## 1.3 Top-Level API + +| Field | Type | Description | +| --- | --- | --- | +| **specVersion** | *String* | A Semver version indicating which version of this API is being used.| +| **schema** | [*Schema*](#14-schema) | The GraphQL schema of this subgraph.| +| **description** | *String* | An optional description of the subgraph's purpose. | +| **repository** | *String* | An optional link to where the subgraph lives. | +| **graft** | optional [*Graft Base*](#18-graft-base) | An optional base to graft onto. | +| **dataSources**| [*Data Source Spec*](#15-data-source)| Each data source spec defines the data that will be ingested as well as the transformation logic to derive the state of the subgraph's entities based on the source data.| +| **templates** | [*Data Source Templates Spec*](#17-data-source-templates) | Each data source template defines a data source that can be created dynamically from the mappings. | +| **features** | optional [*[String]*](#19-features) | A list of feature names used by the subgraph. | + +## 1.4 Schema + +| Field | Type | Description | +| --- | --- | --- | +| **file**| [*Path*](#16-path) | The path of the GraphQL IDL file, either local or on IPFS. | + +## 1.5 Data Source + +| Field | Type | Description | +| --- | --- | --- | +| **kind** | *String | The type of data source. Possible values: *ethereum/contract*.| +| **name** | *String* | The name of the source data. Will be used to generate APIs in the mapping and also for self-documentation purposes. | +| **network** | *String* | For blockchains, this describes which network the subgraph targets. For Ethereum, this can be any of "mainnet", "rinkeby", "kovan", "ropsten", "goerli", "poa-core", "poa-sokol", "xdai", "matic", "mumbai", "fantom", "bsc" or "clover". Developers could look for an up to date list in the graph-cli [*code*](https://github.com/graphprotocol/graph-cli/blob/master/src/commands/init.js#L43-L57).| +| **source** | [*EthereumContractSource*](#151-ethereumcontractsource) | The source data on a blockchain such as Ethereum. | +| **mapping** | [*Mapping*](#152-mapping) | The transformation logic applied to the data prior to being indexed. | + +### 1.5.1 EthereumContractSource + +| Field | Type | Description | +| --- | --- | --- | +| **address** | *String* | The address of the source data in its respective blockchain. | +| **abi** | *String* | The name of the ABI for this Ethereum contract. See `abis` in the `mapping` manifest. | +| **startBlock** | optional *BigInt* | The block to start indexing this data source from. | + + +### 1.5.2 Mapping +The `mapping` field may be one of the following supported mapping manifests: + - [Ethereum Mapping](#1521-ethereum-mapping) + +#### 1.5.2.1 Ethereum Mapping + +| Field | Type | Description | +| --- | --- | --- | +| **kind** | *String* | Must be "ethereum/events" for Ethereum Events Mapping. | +| **apiVersion** | *String* | Semver string of the version of the Mappings API that will be used by the mapping script. | +| **language** | *String* | The language of the runtime for the Mapping API. Possible values: *wasm/assemblyscript*. | +| **entities** | *[String]* | A list of entities that will be ingested as part of this mapping. Must correspond to names of entities in the GraphQL IDL. | +| **abis** | *ABI* | ABIs for the contract classes that should be generated in the Mapping ABI. Name is also used to reference the ABI elsewhere in the manifest. | +| **eventHandlers** | optional *EventHandler* | Handlers for specific events, which will be defined in the mapping script. | +| **callHandlers** | optional *CallHandler* | A list of functions that will trigger a handler and the name of the corresponding handlers in the mapping. | +| **blockHandlers** | optional *BlockHandler* | Defines block filters and handlers to process matching blocks. | +| **file** | [*Path*](#16-path) | The path of the mapping script. | + +> **Note:** Each mapping is required to supply one or more handler type, available types: `EventHandler`, `CallHandler`, or `BlockHandler`. + +#### 1.5.2.2 EventHandler + +| Field | Type | Description | +| --- | --- | --- | +| **event** | *String* | An identifier for an event that will be handled in the mapping script. For Ethereum contracts, this must be the full event signature to distinguish from events that may share the same name. No alias types can be used. For example, uint will not work, uint256 must be used.| +| **handler** | *String* | The name of an exported function in the mapping script that should handle the specified event. | +| **topic0** | optional *String* | A `0x` prefixed hex string. If provided, events whose topic0 is equal to this value will be processed by the given handler. When topic0 is provided, _only_ the topic0 value will be matched, and not the hash of the event signature. This is useful for processing anonymous events in Solidity, which can have their topic0 set to anything. By default, topic0 is equal to the hash of the event signature. | + +#### 1.5.2.3 CallHandler + +| Field | Type | Description | +| --- | --- | --- | +| **function** | *String* | An identifier for a function that will be handled in the mapping script. For Ethereum contracts, this is the normalized function signature to filter calls by. | +| **handler** | *String* | The name of an exported function in the mapping script that should handle the specified event. | + +#### 1.5.2.4 BlockHandler + +| Field | Type | Description | +| --- | --- | --- | +| **handler** | *String* | The name of an exported function in the mapping script that should handle the specified event. | +| **filter** | optional *BlockHandlerFilter* | Definition of the filter to apply. If none is supplied, the handler will be called on every block. | + +#### 1.5.2.4.1 BlockHandlerFilter + +| Field | Type | Description | +| --- | --- | --- | +| **kind** | *String* | The selected block handler filter. Only option for now: `call`: This will only run the handler if the block contains at least one call to the data source contract. | + +## 1.6 Path +A path has one field `path`, which either refers to a path of a file on the local dev machine or an [IPLD link](https://github.com/ipld/specs/). + +When using the Graph-CLI, local paths may be used during development, and then, the tool will take care of deploying linked files to IPFS and replacing the local paths with IPLD links at deploy time. + +| Field | Type | Description | +| --- | --- | --- | +| **path** | *String or [IPLD Link](https://github.com/ipld/specs/)* | A path to a local file or IPLD link. | + +## 1.7 Data Source Templates +A data source template has all of the fields of a normal data source, except it does not include a contract address under `source`. The address is a parameter that can later be provided when creating a dynamic data source from the template. +```yml +# ... +templates: + - name: Exchange + kind: ethereum/contract + network: mainnet + source: + abi: Exchange + mapping: + kind: ethereum/events + apiVersion: 0.0.1 + language: wasm/assemblyscript + file: ./src/mappings/exchange.ts + entities: + - Exchange + abis: + - name: Exchange + file: ./abis/exchange.json + eventHandlers: + - event: TokenPurchase(address,uint256,uint256) + handler: handleTokenPurchase +``` + +## 1.8 Graft Base +A subgraph can be _grafted_ on top of another subgraph, meaning that, rather than starting to index the subgraph from the genesis block, the subgraph is initialized with a copy of the given base subgraph, and indexing resumes from the given block. + +| Field | Type | Description | +| --- | --- | --- | +| **base** | *String* | The subgraph ID of the base subgraph | +| **block** | *BigInt* | The block number up to which to use data from the base subgraph | + +## 1.9 Features + +Starting from `specVersion` `0.0.4`, a subgraph must declare all _feature_ names it uses to be +considered valid. + +A Graph Node instance will **reject** a subgraph deployment if: +- the `specVersion` is equal to or higher than `0.0.4` **AND** +- it hasn't explicitly declared a feature it uses. + +No validation errors will happen if a feature is declared but not used. + +These are the currently available features and their names: + +| Feature | Name | +| --- | --- | +| Non-fatal errors | `nonFatalErrors` | +| Full-text Search | `fullTextSearch` | +| Grafting | `grafting` | +| IPFS on Ethereum Contracts | `ipfsOnEthereumContracts` | diff --git a/graph/Cargo.toml b/graph/Cargo.toml new file mode 100644 index 0000000..6062689 --- /dev/null +++ b/graph/Cargo.toml @@ -0,0 +1,73 @@ +[package] +name = "graph" +version = "0.27.0" +edition = "2021" + +[dependencies] +anyhow = "1.0" +async-trait = "0.1.50" +async-stream = "0.3" +atomic_refcell = "0.1.8" +bigdecimal = { version = "0.1.0", features = ["serde"] } +bytes = "1.0.1" +cid = "0.8.3" +diesel = { version = "1.4.8", features = ["postgres", "serde_json", "numeric", "r2d2", "chrono"] } +diesel_derives = "1.4" +chrono = "0.4.22" +envconfig = "0.10.0" +Inflector = "0.11.3" +isatty = "0.1.9" +reqwest = { version = "0.11.2", features = ["json", "stream", "multipart"] } +ethabi = "17.2" +hex = "0.4.3" +http = "0.2.3" +futures = "0.1.21" +graphql-parser = "0.4.0" +lazy_static = "1.4.0" +num-bigint = { version = "^0.2.6", features = ["serde"] } +num_cpus = "1.13.1" +num-traits = "0.2.15" +rand = "0.8.4" +semver = { version = "1.0.12", features = ["serde"] } +serde = { version = "1.0.126", features = ["rc"] } +serde_derive = "1.0.125" +serde_json = { version = "1.0", features = ["arbitrary_precision"] } +serde_yaml = "0.8" +slog = { version = "2.7.0", features = ["release_max_level_trace", "max_level_trace"] } +stable-hash_legacy = { version = "0.3.3", package = "stable-hash" } +stable-hash = { version = "0.4.2"} +strum = "0.21.0" +strum_macros = "0.21.1" +slog-async = "2.5.0" +slog-envlogger = "2.1.0" +slog-term = "2.7.0" +petgraph = "0.6.2" +tiny-keccak = "1.5.0" +tokio = { version = "1.16.1", features = ["time", "sync", "macros", "test-util", "rt-multi-thread", "parking_lot"] } +tokio-stream = { version = "0.1.9", features = ["sync"] } +tokio-retry = "0.3.0" +url = "2.2.1" +prometheus = "0.13.2" +priority-queue = "0.7.0" +tonic = { version = "0.7.1", features = ["tls-roots","compression"] } +prost = "0.10.4" +prost-types = "0.10.1" + +futures03 = { version = "0.3.1", package = "futures", features = ["compat"] } +wasmparser = "0.78.2" +thiserror = "1.0.25" +parking_lot = "0.12.1" +itertools = "0.10.3" + +# Our fork contains patches to make some fields optional for Celo and Fantom compatibility. +# Without the "arbitrary_precision" feature, we get the error `data did not match any variant of untagged enum Response`. +web3 = { git = "https://github.com/graphprotocol/rust-web3", branch = "graph-patches-onto-0.18", features = ["arbitrary_precision"] } +serde_plain = "1.0.0" + +[dev-dependencies] +test-store = { path = "../store/test-store" } +clap = { version = "3.2.22", features = ["derive", "env"] } +maplit = "1.0.2" + +[build-dependencies] +tonic-build = { version = "0.7.2", features = ["prost","compression"] } diff --git a/graph/build.rs b/graph/build.rs new file mode 100644 index 0000000..67e9920 --- /dev/null +++ b/graph/build.rs @@ -0,0 +1,20 @@ +fn main() { + println!("cargo:rerun-if-changed=proto"); + tonic_build::configure() + .out_dir("src/firehose") + .compile( + &[ + "proto/firehose.proto", + "proto/ethereum/transforms.proto", + "proto/near/transforms.proto", + "proto/cosmos/transforms.proto", + ], + &["proto"], + ) + .expect("Failed to compile Firehose proto(s)"); + + tonic_build::configure() + .out_dir("src/substreams") + .compile(&["proto/substreams.proto"], &["proto"]) + .expect("Failed to compile Substreams proto(s)"); +} diff --git a/graph/examples/stress.rs b/graph/examples/stress.rs new file mode 100644 index 0000000..0475437 --- /dev/null +++ b/graph/examples/stress.rs @@ -0,0 +1,695 @@ +use std::alloc::{GlobalAlloc, Layout, System}; +use std::collections::{BTreeMap, HashMap}; +use std::iter::FromIterator; +use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use clap::Parser; +use graph::data::value::{Object, Word}; +use graph::object; +use graph::prelude::{lazy_static, q, r, BigDecimal, BigInt, QueryResult}; +use rand::SeedableRng; +use rand::{rngs::SmallRng, Rng}; + +use graph::util::cache_weight::CacheWeight; +use graph::util::lfu_cache::LfuCache; + +// Use a custom allocator that tracks how much memory the program +// has allocated overall + +struct Counter; + +static ALLOCATED: AtomicUsize = AtomicUsize::new(0); + +lazy_static! { + // Set 'MAP_MEASURE' to something to use the `CacheWeight` defined here + // in the `btree` module for `BTreeMap`. If this is not set, use the + // estimate from `graph::util::cache_weight` + static ref MAP_MEASURE: bool = std::env::var("MAP_MEASURE").ok().is_some(); + + // When running the `valuemap` test for BTreeMap, put maps into the + // values of the generated maps + static ref NESTED_MAP: bool = std::env::var("NESTED_MAP").ok().is_some(); +} +// Yes, a global variable. It gets set at the beginning of `main` +static mut PRINT_SAMPLES: bool = false; + +/// Helpers to estimate the size of a `BTreeMap`. Everything in this module, +/// except for `node_size()` is copied from `std::collections::btree`. +/// +/// It is not possible to know how many nodes a BTree has, as +/// `BTreeMap` does not expose its depth or any other detail about +/// the true size of the BTree. We estimate that size, assuming the +/// average case, i.e., a BTree where every node has the average +/// between the minimum and maximum number of entries per node, i.e., +/// the average of (B-1) and (2*B-1) entries, which we call +/// `NODE_FILL`. The number of leaf nodes in the tree is then the +/// number of entries divided by `NODE_FILL`, and the number of +/// interior nodes can be determined by dividing the number of nodes +/// at the child level by `NODE_FILL` + +/// The other difficulty is that the structs with which `BTreeMap` +/// represents internal and leaf nodes are not public, so we can't +/// get their size with `std::mem::size_of`; instead, we base our +/// estimates of their size on the current `std` code, assuming that +/// these structs will not change + +mod btree { + use std::mem; + use std::{mem::MaybeUninit, ptr::NonNull}; + + const B: usize = 6; + const CAPACITY: usize = 2 * B - 1; + + /// Assume BTree nodes are this full (average of minimum and maximum fill) + const NODE_FILL: usize = ((B - 1) + (2 * B - 1)) / 2; + + type BoxedNode = NonNull>; + + struct InternalNode { + _data: LeafNode, + + /// The pointers to the children of this node. `len + 1` of these are considered + /// initialized and valid, except that near the end, while the tree is held + /// through borrow type `Dying`, some of these pointers are dangling. + _edges: [MaybeUninit>; 2 * B], + } + + struct LeafNode { + /// We want to be covariant in `K` and `V`. + _parent: Option>>, + + /// This node's index into the parent node's `edges` array. + /// `*node.parent.edges[node.parent_idx]` should be the same thing as `node`. + /// This is only guaranteed to be initialized when `parent` is non-null. + _parent_idx: MaybeUninit, + + /// The number of keys and values this node stores. + _len: u16, + + /// The arrays storing the actual data of the node. Only the first `len` elements of each + /// array are initialized and valid. + _keys: [MaybeUninit; CAPACITY], + _vals: [MaybeUninit; CAPACITY], + } + + pub fn node_size(map: &std::collections::BTreeMap) -> usize { + // Measure the size of internal and leaf nodes directly - that's why + // we copied all this code from `std` + let ln_sz = mem::size_of::>(); + let in_sz = mem::size_of::>(); + + // Estimate the number of internal and leaf nodes based on the only + // thing we can measure about a BTreeMap, the number of entries in + // it, and use our `NODE_FILL` assumption to estimate how the tree + // is structured. We try to be very good for small maps, since + // that's what we use most often in our code. This estimate is only + // for the indirect weight of the `BTreeMap` + let (leaves, int_nodes) = if map.is_empty() { + // An empty tree has no indirect weight + (0, 0) + } else if map.len() <= CAPACITY { + // We only have the root node + (1, 0) + } else { + // Estimate based on our `NODE_FILL` assumption + let leaves = map.len() / NODE_FILL + 1; + let mut prev_level = leaves / NODE_FILL + 1; + let mut int_nodes = prev_level; + while prev_level > 1 { + int_nodes += prev_level; + prev_level = prev_level / NODE_FILL + 1; + } + (leaves, int_nodes) + }; + + let sz = leaves * ln_sz + int_nodes * in_sz; + + if unsafe { super::PRINT_SAMPLES } { + println!( + " btree: leaves={} internal={} sz={} ln_sz={} in_sz={} len={}", + leaves, + int_nodes, + sz, + ln_sz, + in_sz, + map.len() + ); + } + sz + } +} + +struct MapMeasure(BTreeMap); + +impl Default for MapMeasure { + fn default() -> MapMeasure { + MapMeasure(BTreeMap::new()) + } +} + +impl CacheWeight for MapMeasure { + fn indirect_weight(&self) -> usize { + if *MAP_MEASURE { + let kv_sz = self + .0 + .iter() + .map(|(key, value)| key.indirect_weight() + value.indirect_weight()) + .sum::(); + let node_sz = btree::node_size(&self.0); + kv_sz + node_sz + } else { + self.0.indirect_weight() + } + } +} + +unsafe impl GlobalAlloc for Counter { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let ret = System.alloc(layout); + if !ret.is_null() { + ALLOCATED.fetch_add(layout.size(), SeqCst); + } + ret + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + System.dealloc(ptr, layout); + ALLOCATED.fetch_sub(layout.size(), SeqCst); + } +} + +#[global_allocator] +static A: Counter = Counter; + +// Setup to make checking different data types and how they interact +// with cache size easier + +/// The template of an object we want to cache +trait Template: CacheWeight { + // Create a new test object + fn create(size: usize, rng: Option<&mut SmallRng>) -> Self; + + // Return a sample of this test object of the given `size`. There's no + // fixed definition of 'size', other than that smaller sizes will + // take less memory than larger ones + fn sample(&self, size: usize, rng: Option<&mut SmallRng>) -> Box; +} + +/// Template for testing caching of `String` +impl Template for String { + fn create(size: usize, _rng: Option<&mut SmallRng>) -> Self { + let mut s = String::with_capacity(size); + for _ in 0..size { + s.push('x'); + } + s + } + fn sample(&self, size: usize, _rng: Option<&mut SmallRng>) -> Box { + Box::new(self[0..size].into()) + } +} + +/// Template for testing caching of `String` +impl Template for Word { + fn create(size: usize, _rng: Option<&mut SmallRng>) -> Self { + let mut s = String::with_capacity(size); + for _ in 0..size { + s.push('x'); + } + Word::from(s) + } + + fn sample(&self, size: usize, _rng: Option<&mut SmallRng>) -> Box { + Box::new(self[0..size].into()) + } +} + +/// Template for testing caching of `Vec` +impl Template for Vec { + fn create(size: usize, _rng: Option<&mut SmallRng>) -> Self { + Vec::from_iter(0..size) + } + fn sample(&self, size: usize, _rng: Option<&mut SmallRng>) -> Box { + Box::new(self[0..size].into()) + } +} + +impl Template for BigInt { + fn create(size: usize, rng: Option<&mut SmallRng>) -> Self { + let f = match rng { + Some(rng) => { + let mag = rng.gen_range(1..100); + if rng.gen_bool(0.5) { + mag + } else { + -mag + } + } + None => 1, + }; + BigInt::from(3u64).pow(size as u8) * BigInt::from(f) + } + + fn sample(&self, size: usize, rng: Option<&mut SmallRng>) -> Box { + Box::new(Self::create(size, rng)) + } +} + +impl Template for BigDecimal { + fn create(size: usize, mut rng: Option<&mut SmallRng>) -> Self { + let f = match rng.as_deref_mut() { + Some(rng) => { + let mag = rng.gen_range(1i32..100); + if rng.gen_bool(0.5) { + mag + } else { + -mag + } + } + None => 1, + }; + let exp = match rng { + Some(rng) => rng.gen_range(-100..=100), + None => 1, + }; + let bi = BigInt::from(3u64).pow(size as u8) * BigInt::from(f); + BigDecimal::new(bi, exp) + } + + fn sample(&self, size: usize, rng: Option<&mut SmallRng>) -> Box { + Box::new(Self::create(size, rng)) + } +} + +/// Template for testing caching of `HashMap` +impl Template for HashMap { + fn create(size: usize, _rng: Option<&mut SmallRng>) -> Self { + let mut map = HashMap::new(); + for i in 0..size { + map.insert(format!("key{}", i), format!("value{}", i)); + } + map + } + + fn sample(&self, size: usize, _rng: Option<&mut SmallRng>) -> Box { + Box::new(HashMap::from_iter( + self.iter() + .take(size) + .map(|(k, v)| (k.to_owned(), v.to_owned())), + )) + } +} + +fn make_object(size: usize, mut rng: Option<&mut SmallRng>) -> Object { + let mut obj = Vec::new(); + let modulus = if *NESTED_MAP { 8 } else { 7 }; + + for i in 0..size { + let kind = rng + .as_deref_mut() + .map(|rng| rng.gen_range(0..modulus)) + .unwrap_or(i % modulus); + + let value = match kind { + 0 => r::Value::Boolean(i % 11 > 5), + 1 => r::Value::Int((i as i32).into()), + 2 => r::Value::Null, + 3 => r::Value::Float(i as f64 / 17.0), + 4 => r::Value::Enum(format!("enum{}", i)), + 5 => r::Value::String(format!("0x0000000000000000000000000000000000000000{}", i)), + 6 => { + let vals = (0..(i % 51)).map(|i| r::Value::String(format!("list{}", i))); + r::Value::List(Vec::from_iter(vals)) + } + 7 => { + let mut obj = Vec::new(); + for j in 0..(i % 51) { + obj.push((format!("key{}", j), r::Value::String(format!("value{}", j)))); + } + r::Value::Object(Object::from_iter(obj)) + } + _ => unreachable!(), + }; + + let key = rng.as_deref_mut().map(|rng| rng.gen()).unwrap_or(i) % modulus; + obj.push((format!("val{}", key), value)); + } + Object::from_iter(obj) +} + +fn make_domains(size: usize, _rng: Option<&mut SmallRng>) -> Object { + let owner = object! { + owner: object! { + id: "0xe8d391ef649a6652b9047735f6c0d48b6ae751df", + name: "36190.eth" + } + }; + + let domains: Vec<_> = (0..size).map(|_| owner.clone()).collect(); + Object::from_iter([("domains".to_string(), r::Value::List(domains))]) +} + +/// Template for testing caching of `Object` +impl Template for Object { + fn create(size: usize, rng: Option<&mut SmallRng>) -> Self { + make_object(size, rng) + } + + fn sample(&self, size: usize, rng: Option<&mut SmallRng>) -> Box { + // If the user specified '--fixed', don't build a new map every call + // since that can be slow + if rng.is_none() { + Box::new(Object::from_iter( + self.iter() + .take(size) + .map(|(k, v)| (k.to_owned(), v.to_owned())), + )) + } else { + Box::new(make_object(size, rng)) + } + } +} + +/// Template for testing caching of `QueryResult` +impl Template for QueryResult { + fn create(size: usize, rng: Option<&mut SmallRng>) -> Self { + QueryResult::new(make_domains(size, rng)) + } + + fn sample(&self, size: usize, rng: Option<&mut SmallRng>) -> Box { + // If the user specified '--fixed', don't build a new map every call + // since that can be slow + if rng.is_none() { + Box::new(QueryResult::new(Object::from_iter( + self.data() + .unwrap() + .iter() + .take(size) + .map(|(k, v)| (k.to_owned(), v.to_owned())), + ))) + } else { + Box::new(QueryResult::new(make_domains(size, rng))) + } + } +} + +type ValueMap = MapMeasure; + +impl ValueMap { + fn make_map(size: usize, mut rng: Option<&mut SmallRng>) -> Self { + let mut map = BTreeMap::new(); + let modulus = if *NESTED_MAP { 9 } else { 8 }; + + for i in 0..size { + let kind = rng + .as_deref_mut() + .map(|rng| rng.gen_range(0..modulus)) + .unwrap_or(i % modulus); + + let value = match kind { + 0 => q::Value::Boolean(i % 11 > 5), + 1 => q::Value::Int((i as i32).into()), + 2 => q::Value::Null, + 3 => q::Value::Float(i as f64 / 17.0), + 4 => q::Value::Enum(format!("enum{}", i)), + 5 => q::Value::String(format!("string{}", i)), + 6 => q::Value::Variable(format!("var{}", i)), + 7 => { + let vals = (0..(i % 51)).map(|i| q::Value::String(format!("list{}", i))); + q::Value::List(Vec::from_iter(vals)) + } + 8 => { + let mut map = BTreeMap::new(); + for j in 0..(i % 51) { + map.insert(format!("key{}", j), q::Value::String(format!("value{}", j))); + } + q::Value::Object(map) + } + _ => unreachable!(), + }; + + let key = rng.as_deref_mut().map(|rng| rng.gen()).unwrap_or(i) % modulus; + map.insert(format!("val{}", key), value); + } + MapMeasure(map) + } +} + +/// Template for testing roughly a GraphQL response, i.e., a `BTreeMap` +impl Template for ValueMap { + fn create(size: usize, rng: Option<&mut SmallRng>) -> Self { + Self::make_map(size, rng) + } + + fn sample(&self, size: usize, rng: Option<&mut SmallRng>) -> Box { + // If the user specified '--fixed', don't build a new map every call + // since that can be slow + if rng.is_none() { + Box::new(MapMeasure(BTreeMap::from_iter( + self.0 + .iter() + .take(size) + .map(|(k, v)| (k.to_owned(), v.to_owned())), + ))) + } else { + Box::new(Self::make_map(size, rng)) + } + } +} + +type UsizeMap = MapMeasure; + +impl UsizeMap { + fn make_map(size: usize, mut rng: Option<&mut SmallRng>) -> Self { + let mut map = BTreeMap::new(); + for i in 0..size { + let key = rng.as_deref_mut().map(|rng| rng.gen()).unwrap_or(2 * i); + map.insert(key, i * 3); + } + MapMeasure(map) + } +} + +/// Template for testing roughly a GraphQL response, i.e., a `BTreeMap` +impl Template for UsizeMap { + fn create(size: usize, rng: Option<&mut SmallRng>) -> Self { + Self::make_map(size, rng) + } + + fn sample(&self, size: usize, rng: Option<&mut SmallRng>) -> Box { + // If the user specified '--fixed', don't build a new map every call + // since that can be slow + if rng.is_none() { + Box::new(MapMeasure(BTreeMap::from_iter( + self.0 + .iter() + .take(size) + .map(|(k, v)| (k.to_owned(), v.to_owned())), + ))) + } else { + Box::new(Self::make_map(size, rng)) + } + } +} + +/// Wrapper around our template objects; we always put them behind an `Arc` +/// so that dropping the template object frees the entire object rather than +/// leaving part of it in the cache. That's also how the production code +/// uses this cache: objects always wrapped in an `Arc` +struct Entry(Arc); + +impl Default for Entry { + fn default() -> Self { + Self(Arc::new(T::create(0, None))) + } +} + +impl CacheWeight for Entry { + fn indirect_weight(&self) -> usize { + // Account for the two pointers the Arc uses for keeping reference + // counts. Including that in the weight is only correct in this + // test, since we know we never have more than one reference throug + // the `Arc` + self.0.weight() + 2 * std::mem::size_of::() + } +} + +impl From for Entry { + fn from(templ: T) -> Self { + Self(Arc::new(templ)) + } +} + +// Command line arguments +#[derive(Parser)] +#[clap(name = "stress", about = "Stress test for the LFU Cache")] +struct Opt { + /// Number of cache evictions and insertions + #[clap(short, long, default_value = "1000")] + niter: usize, + /// Print this many intermediate messages + #[clap(short, long, default_value = "10")] + print_count: usize, + /// Use objects of size 0 up to this size, chosen unifromly randomly + /// unless `--fixed` is given + #[clap(short, long, default_value = "1024")] + obj_size: usize, + #[clap(short, long, default_value = "1000000")] + cache_size: usize, + #[clap(short, long, default_value = "vec")] + template: String, + #[clap(short, long)] + samples: bool, + /// Always use objects of size `--obj-size` + #[clap(short, long)] + fixed: bool, + /// The seed of the random number generator. A seed of 0 means that all + /// samples are taken from the same template object, and only differ in + /// size + #[clap(long)] + seed: Option, +} + +fn maybe_rng<'a>(opt: &'a Opt, rng: &'a mut SmallRng) -> Option<&'a mut SmallRng> { + if opt.seed == Some(0) { + None + } else { + Some(rng) + } +} + +fn stress(opt: &Opt) { + let mut rng = match opt.seed { + None => SmallRng::from_entropy(), + Some(seed) => SmallRng::seed_from_u64(seed), + }; + + let mut cache: LfuCache> = LfuCache::new(); + let template = T::create(opt.obj_size, maybe_rng(opt, &mut rng)); + + println!("type: {}", std::any::type_name::()); + println!( + "obj weight: {} iterations: {} cache_size: {}\n", + template.weight(), + opt.niter, + opt.cache_size + ); + + let base_mem = ALLOCATED.load(SeqCst); + let print_mod = opt.niter / opt.print_count + 1; + let mut should_print = true; + let mut print_header = true; + let mut sample_weight: usize = 0; + let mut sample_alloc: usize = 0; + let mut evict_count: usize = 0; + let mut evict_duration = Duration::from_secs(0); + + let start = Instant::now(); + for key in 0..opt.niter { + should_print = should_print || key % print_mod == 0; + let before_mem = ALLOCATED.load(SeqCst); + let start_evict = Instant::now(); + if let Some(stats) = cache.evict(opt.cache_size) { + evict_duration += start_evict.elapsed(); + let after_mem = ALLOCATED.load(SeqCst); + evict_count += 1; + if should_print { + if print_header { + println!("evict: weight that was removed from cache"); + println!("drop: allocated memory that was freed"); + println!("slip: evicted - dropped"); + println!("room: configured cache size - cache weight"); + println!("heap: allocated heap_size / configured cache_size"); + println!("mem: memory allocated for cache + all entries\n"); + print_header = false; + } + + let dropped = before_mem - after_mem; + let evicted_count = stats.evicted_count; + let evicted = stats.evicted_weight; + let slip = evicted as i64 - dropped as i64; + let room = opt.cache_size as i64 - stats.new_weight as i64; + let heap = (after_mem - base_mem) as f64 / opt.cache_size as f64; + let mem = after_mem - base_mem; + println!( + "evict: [{evicted_count:3}]{evicted:6} drop: {dropped:6} slip: {slip:4} \ + room: {room:6} heap: {heap:0.2} mem: {mem:8}" + ); + should_print = false; + } + } + let size = if opt.fixed || opt.obj_size == 0 { + opt.obj_size + } else { + rng.gen_range(0..opt.obj_size) + }; + let before = ALLOCATED.load(SeqCst); + let sample = template.sample(size, maybe_rng(opt, &mut rng)); + let weight = sample.weight(); + let alloc = ALLOCATED.load(SeqCst) - before; + sample_weight += weight; + sample_alloc += alloc; + if opt.samples { + println!("sample: weight {:6} alloc {:6}", weight, alloc,); + } + cache.insert(key, Entry::from(*sample)); + // Do a few random reads from the cache + for _attempt in 0..5 { + let read = rng.gen_range(0..=key); + let _v = cache.get(&read); + } + } + + println!( + "\ncache: entries: {} evictions: {} took {}ms out of {}ms", + cache.len(), + evict_count, + evict_duration.as_millis(), + start.elapsed().as_millis() + ); + if sample_alloc == sample_weight { + println!( + "samples: weight {} alloc {} weight/alloc precise", + sample_weight, sample_alloc + ); + } else { + let heap_factor = sample_alloc as f64 / sample_weight as f64; + println!( + "samples: weight {} alloc {} weight/alloc {:0.2}", + sample_weight, sample_alloc, heap_factor + ); + } +} + +/// This program constructs a template object of size `obj_size` and then +/// inserts a sample of size up to `obj_size` into the cache `niter` times. +/// The cache is limited to `cache_size` total weight, and we call `evict` +/// before each insertion into the cache. +/// +/// After each `evict`, we check how much heap we have currently allocated +/// and print that roughly `print_count` times over the run of the program. +/// The most important measure is the `heap_factor`, which is the ratio of +/// memory used on the heap since we started inserting into the cache to +/// the target `cache_size` +pub fn main() { + let opt = Opt::from_args(); + unsafe { PRINT_SAMPLES = opt.samples } + + // Use different Cacheables to see how the cache manages memory with + // different types of cache entries. + match opt.template.as_str() { + "bigdecimal" => stress::(&opt), + "bigint" => stress::(&opt), + "hashmap" => stress::>(&opt), + "object" => stress::(&opt), + "result" => stress::(&opt), + "string" => stress::(&opt), + "usizemap" => stress::(&opt), + "valuemap" => stress::(&opt), + "vec" => stress::>(&opt), + "word" => stress::(&opt), + _ => println!("unknown value `{}` for --template", opt.template), + } +} diff --git a/graph/proto/cosmos/transforms.proto b/graph/proto/cosmos/transforms.proto new file mode 100644 index 0000000..e37edd3 --- /dev/null +++ b/graph/proto/cosmos/transforms.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package sf.cosmos.transform.v1; + +option go_package = "github.com/figment-networks/proto-cosmos/pb/sf/cosmos/transform/v1;pbtransform"; + +message EventTypeFilter { + repeated string event_types = 1; +} diff --git a/graph/proto/ethereum/transforms.proto b/graph/proto/ethereum/transforms.proto new file mode 100644 index 0000000..3d24fc9 --- /dev/null +++ b/graph/proto/ethereum/transforms.proto @@ -0,0 +1,64 @@ +syntax = "proto3"; + +package sf.ethereum.transform.v1; +option go_package = "github.com/streamingfast/firehose-ethereum/types/pb/sf/ethereum/transform/v1;pbtransform"; + +// CombinedFilter is a combination of "LogFilters" and "CallToFilters" +// +// It transforms the requested stream in two ways: +// 1. STRIPPING +// The block data is stripped from all transactions that don't +// match any of the filters. +// +// 2. SKIPPING +// If an "block index" covers a range containing a +// block that does NOT match any of the filters, the block will be +// skipped altogether, UNLESS send_all_block_headers is enabled +// In that case, the block would still be sent, but without any +// transactionTrace +// +// The SKIPPING feature only applies to historical blocks, because +// the "block index" is always produced after the merged-blocks files +// are produced. Therefore, the "live" blocks are never filtered out. +// +message CombinedFilter { + repeated LogFilter log_filters = 1; + repeated CallToFilter call_filters = 2; + + // Always send all blocks. if they don't match any log_filters or call_filters, + // all the transactions will be filtered out, sending only the header. + bool send_all_block_headers = 3; +} + +// MultiLogFilter concatenates the results of each LogFilter (inclusive OR) +message MultiLogFilter { + repeated LogFilter log_filters = 1; +} + +// LogFilter will match calls where *BOTH* +// * the contract address that emits the log is one in the provided addresses -- OR addresses list is empty -- +// * the event signature (topic.0) is one of the provided event_signatures -- OR event_signatures is empty -- +// +// a LogFilter with both empty addresses and event_signatures lists is invalid and will fail. +message LogFilter { + repeated bytes addresses = 1; + repeated bytes event_signatures = 2; // corresponds to the keccak of the event signature which is stores in topic.0 +} + +// MultiCallToFilter concatenates the results of each CallToFilter (inclusive OR) +message MultiCallToFilter { + repeated CallToFilter call_filters = 1; +} + +// CallToFilter will match calls where *BOTH* +// * the contract address (TO) is one in the provided addresses -- OR addresses list is empty -- +// * the method signature (in 4-bytes format) is one of the provided signatures -- OR signatures is empty -- +// +// a CallToFilter with both empty addresses and signatures lists is invalid and will fail. +message CallToFilter { + repeated bytes addresses = 1; + repeated bytes signatures = 2; +} + +message LightBlock { +} diff --git a/graph/proto/firehose.proto b/graph/proto/firehose.proto new file mode 100644 index 0000000..b806028 --- /dev/null +++ b/graph/proto/firehose.proto @@ -0,0 +1,110 @@ +syntax = "proto3"; + +package sf.firehose.v1; + +import "google/protobuf/any.proto"; + +option go_package = "github.com/streamingfast/pbgo/sf/firehose/v1;pbfirehose"; + +service Stream { + rpc Blocks(Request) returns (stream Response); +} + +// For historical segments, forks are not passed +message Request { + // Controls where the stream of blocks will start. + // + // The stream will start **inclusively** at the requested block num. + // + // When not provided, starts at first streamable block of the chain. Not all + // chain starts at the same block number, so you might get an higher block than + // requested when using default value of 0. + // + // Can be negative, will be resolved relative to the chain head block, assuming + // a chain at head block #100, then using `-50` as the value will start at block + // #50. If it resolves before first streamable block of chain, we assume start + // of chain. + // + // If `start_cursor` is passed, this value is ignored and the stream instead starts + // immediately after the Block pointed by the opaque `start_cursor` value. + int64 start_block_num = 1; + + // Controls where the stream of blocks will start which will be immediately after + // the Block pointed by this opaque cursor. + // + // Obtain this value from a previously received from `Response.cursor`. + // + // This value takes precedence over `start_block_num`. + string start_cursor = 13; + + // When non-zero, controls where the stream of blocks will stop. + // + // The stream will close **after** that block has passed so the boundary is + // **inclusive**. + uint64 stop_block_num = 5; + + // Filter the steps you want to see. If not specified, defaults to all steps. + // + // Most common steps will be [STEP_IRREVERSIBLE], or [STEP_NEW, STEP_UNDO, STEP_IRREVERSIBLE]. + repeated ForkStep fork_steps = 8; + + // The CEL filter expression used to include transactions, specific to the target protocol, + // works in combination with `exclude_filter_expr` value. + string include_filter_expr = 10; + + // The CEL filter expression used to exclude transactions, specific to the target protocol, works + // in combination with `include_filter_expr` value. + string exclude_filter_expr = 11; + + // **Warning** Experimental API, controls how blocks are trimmed for extraneous information before + // being sent back. The actual trimming is chain dependent. + //BlockDetails details = 15; + reserved 15; + + // controls how many confirmations will consider a given block as final (STEP_IRREVERSIBLE). Warning, if any reorg goes beyond that number of confirmations, the request will stall forever + //uint64 confirmations = 16; + reserved 16; + + + //- EOS "handoffs:3" + //- EOS "lib" + //- EOS "confirms:3" + //- ETH "confirms:200" + //- ETH "confirms:7" + //- SOL "commmitement:finalized" + //- SOL "confirms:200" + string irreversibility_condition = 17; + + repeated google.protobuf.Any transforms = 18; +} + +message Response { + // Chain specific block payload, one of: + // - sf.eosio.codec.v1.Block + // - sf.ethereum.codec.v1.Block + // - sf.near.codec.v1.Block + // - sf.solana.codec.v1.Block + google.protobuf.Any block = 1; + ForkStep step = 6; + string cursor = 10; +} + +enum ForkStep { + STEP_UNKNOWN = 0; + // Block is new head block of the chain, that is linear with the previous block + STEP_NEW = 1; + // Block is now forked and should be undone, it's not the head block of the chain anymore + STEP_UNDO = 2; + // Removed, was STEP_REDO + reserved 3; + // Block is now irreversible and can be committed to (finality is chain specific, see chain documentation for more details) + STEP_IRREVERSIBLE = 4; + // Removed, was STEP_STALLED + reserved 5 ; +} + +// TODO: move to ethereum specific transforms +enum BlockDetails { + BLOCK_DETAILS_FULL = 0; + BLOCK_DETAILS_LIGHT = 1; +} \ No newline at end of file diff --git a/graph/proto/near/transforms.proto b/graph/proto/near/transforms.proto new file mode 100644 index 0000000..6dfe138 --- /dev/null +++ b/graph/proto/near/transforms.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package sf.near.transform.v1; +option go_package = "github.com/streamingfast/sf-near/pb/sf/near/transform/v1;pbtransform"; + +message BasicReceiptFilter { + repeated string accounts = 1; + repeated PrefixSuffixPair prefix_and_suffix_pairs = 2; +} + +// PrefixSuffixPair applies a logical AND to prefix and suffix when both fields are non-empty. +// * {prefix="hello",suffix="world"} will match "hello.world" but not "hello.friend" +// * {prefix="hello",suffix=""} will match both "hello.world" and "hello.friend" +// * {prefix="",suffix="world"} will match both "hello.world" and "good.day.world" +// * {prefix="",suffix=""} is invalid +// +// Note that the suffix will usually have a TLD, ex: "mydomain.near" or "mydomain.testnet" +message PrefixSuffixPair { + string prefix = 1; + string suffix = 2; +} diff --git a/graph/proto/substreams.proto b/graph/proto/substreams.proto new file mode 100644 index 0000000..564d0cb --- /dev/null +++ b/graph/proto/substreams.proto @@ -0,0 +1,267 @@ +syntax = "proto3"; + +package sf.substreams.v1; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/v1;pbsubstreams"; + +import "google/protobuf/any.proto"; +import "google/protobuf/descriptor.proto"; +import "google/protobuf/timestamp.proto"; + +// FIXME: I copied over and inlined most of the substreams files, this is bad and we need a better way to +// generate that, outside of doing this copying over. We should check maybe `buf` or a pre-populated +// package. + +service Stream { + rpc Blocks(Request) returns (stream Response); +} + +message Request { + int64 start_block_num = 1; + string start_cursor = 2; + uint64 stop_block_num = 3; + repeated ForkStep fork_steps = 4; + string irreversibility_condition = 5; + + Modules modules = 6; + repeated string output_modules = 7; + repeated string initial_store_snapshot_for_modules = 8; +} + +message Response { + oneof message { + ModulesProgress progress = 1; // Progress of data preparation, before sending in the stream of `data` events. + InitialSnapshotData snapshot_data = 2; + InitialSnapshotComplete snapshot_complete = 3; + BlockScopedData data = 4; + } +} + +enum ForkStep { + STEP_UNKNOWN = 0; + // Block is new head block of the chain, that is linear with the previous block + STEP_NEW = 1; + // Block is now forked and should be undone, it's not the head block of the chain anymore + STEP_UNDO = 2; + // Removed, was STEP_REDO + reserved 3; + // Block is now irreversible and can be committed to (finality is chain specific, see chain documentation for more details) + STEP_IRREVERSIBLE = 4; + // Removed, was STEP_STALLED + reserved 5; +} + +message InitialSnapshotComplete { + string cursor = 1; +} + +message InitialSnapshotData { + string module_name = 1; + StoreDeltas deltas = 2; + uint64 sent_keys = 4; + uint64 total_keys = 3; +} + +message BlockScopedData { + repeated ModuleOutput outputs = 1; + Clock clock = 3; + ForkStep step = 6; + string cursor = 10; +} + +message ModuleOutput { + string name = 1; + oneof data { + google.protobuf.Any map_output = 2; + StoreDeltas store_deltas = 3; + } + repeated string logs = 4; + + // LogsTruncated is a flag that tells you if you received all the logs or if they + // were truncated because you logged too much (fixed limit currently is set to 128 KiB). + bool logs_truncated = 5; +} + +message ModulesProgress { + repeated ModuleProgress modules = 1; +} + +message ModuleProgress { + string name = 1; + + oneof type { + ProcessedRange processed_ranges = 2; + InitialState initial_state = 3; + ProcessedBytes processed_bytes = 4; + Failed failed = 5; + } + + message ProcessedRange { + repeated BlockRange processed_ranges = 1; + } + message InitialState { + uint64 available_up_to_block = 2; + } + message ProcessedBytes { + uint64 total_bytes_read = 1; + uint64 total_bytes_written = 2; + } + message Failed { + string reason = 1; + repeated string logs = 2; + // FailureLogsTruncated is a flag that tells you if you received all the logs or if they + // were truncated because you logged too much (fixed limit currently is set to 128 KiB). + bool logs_truncated = 3; + } +} + +message BlockRange { + uint64 start_block = 1; + uint64 end_block = 2; +} + +message StoreDeltas { + repeated StoreDelta deltas = 1; +} + +message StoreDelta { + enum Operation { + UNSET = 0; + CREATE = 1; + UPDATE = 2; + DELETE = 3; + } + Operation operation = 1; + uint64 ordinal = 2; + string key = 3; + bytes old_value = 4; + bytes new_value = 5; +} + +message Output { + uint64 block_num = 1; + string block_id = 2; + google.protobuf.Timestamp timestamp = 4; + google.protobuf.Any value = 10; +} + +message Modules { + repeated Module modules = 1; + repeated Binary binaries = 2; +} + +// Binary represents some code compiled to its binary form. +message Binary { + string type = 1; + bytes content = 2; +} + +message Module { + string name = 1; + oneof kind { + KindMap kind_map = 2; + KindStore kind_store = 3; + }; + + uint32 binary_index = 4; + string binary_entrypoint = 5; + + repeated Input inputs = 6; + Output output = 7; + + uint64 initial_block = 8; + + message KindMap { + string output_type = 1; + } + + message KindStore { + // The `update_policy` determines the functions available to mutate the store + // (like `set()`, `set_if_not_exists()` or `sum()`, etc..) in + // order to ensure that parallel operations are possible and deterministic + // + // Say a store cumulates keys from block 0 to 1M, and a second store + // cumulates keys from block 1M to 2M. When we want to use this + // store as a dependency for a downstream module, we will merge the + // two stores according to this policy. + UpdatePolicy update_policy = 1; + string value_type = 2; + + enum UpdatePolicy { + UPDATE_POLICY_UNSET = 0; + // Provides a store where you can `set()` keys, and the latest key wins + UPDATE_POLICY_SET = 1; + // Provides a store where you can `set_if_not_exists()` keys, and the first key wins + UPDATE_POLICY_SET_IF_NOT_EXISTS = 2; + // Provides a store where you can `add_*()` keys, where two stores merge by summing its values. + UPDATE_POLICY_ADD = 3; + // Provides a store where you can `min_*()` keys, where two stores merge by leaving the minimum value. + UPDATE_POLICY_MIN = 4; + // Provides a store where you can `max_*()` keys, where two stores merge by leaving the maximum value. + UPDATE_POLICY_MAX = 5; + } + + } + + message Input { + oneof input { + Source source = 1; + Map map = 2; + Store store = 3; + } + + message Source { + string type = 1; // ex: "sf.ethereum.type.v1.Block" + } + message Map { + string module_name = 1; // ex: "block_to_pairs" + } + message Store { + string module_name = 1; + Mode mode = 2; + + enum Mode { + UNSET = 0; + GET = 1; + DELTAS = 2; + } + } + } + + message Output { + string type = 1; + } +} + +message Clock { + string id = 1; + uint64 number = 2; + google.protobuf.Timestamp timestamp = 3; +} + +message Package { + // Needs to be one so this file can be used _directly_ as a + // buf `Image` andor a ProtoSet for grpcurl and other tools + repeated google.protobuf.FileDescriptorProto proto_files = 1; + reserved 2; // In case protosets add a field some day. + reserved 3; // In case protosets add a field some day. + reserved 4; // In case protosets add a field some day. + + uint64 version = 5; + sf.substreams.v1.Modules modules = 6; + repeated ModuleMetadata module_meta = 7; + repeated PackageMetadata package_meta = 8; +} + +message PackageMetadata { + string version = 1; + string url = 2; + string name = 3; + string doc = 4; +} + +message ModuleMetadata { + // Corresponds to the index in `Package.metadata.package_meta` + uint64 package_index = 1; + string doc = 2; +} diff --git a/graph/src/blockchain/block_stream.rs b/graph/src/blockchain/block_stream.rs new file mode 100644 index 0000000..801ba99 --- /dev/null +++ b/graph/src/blockchain/block_stream.rs @@ -0,0 +1,485 @@ +use anyhow::Error; +use async_stream::stream; +use futures03::Stream; +use std::fmt; +use std::sync::Arc; +use thiserror::Error; +use tokio::sync::mpsc::{self, Receiver, Sender}; + +use super::{Block, BlockPtr, Blockchain}; +use crate::anyhow::Result; +use crate::components::store::{BlockNumber, DeploymentLocator}; +use crate::data::subgraph::UnifiedMappingApiVersion; +use crate::firehose; +use crate::substreams::BlockScopedData; +use crate::{prelude::*, prometheus::labels}; + +pub struct BufferedBlockStream { + inner: Pin, Error>> + Send>>, +} + +impl BufferedBlockStream { + pub fn spawn_from_stream( + stream: Box>, + size_hint: usize, + ) -> Box> { + let (sender, receiver) = mpsc::channel::, Error>>(size_hint); + crate::spawn(async move { BufferedBlockStream::stream_blocks(stream, sender).await }); + + Box::new(BufferedBlockStream::new(receiver)) + } + + pub fn new(mut receiver: Receiver, Error>>) -> Self { + let inner = stream! { + loop { + let event = match receiver.recv().await { + Some(evt) => evt, + None => return, + }; + + yield event + } + }; + + Self { + inner: Box::pin(inner), + } + } + + pub async fn stream_blocks( + mut stream: Box>, + sender: Sender, Error>>, + ) -> Result<(), Error> { + while let Some(event) = stream.next().await { + match sender.send(event).await { + Ok(_) => continue, + Err(err) => { + return Err(anyhow!( + "buffered blockstream channel is closed, stopping. Err: {}", + err + )) + } + } + } + + Ok(()) + } +} + +impl BlockStream for BufferedBlockStream {} + +impl Stream for BufferedBlockStream { + type Item = Result, Error>; + + fn poll_next( + mut self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.inner.poll_next_unpin(cx) + } +} + +pub trait BlockStream: + Stream, Error>> + Unpin + Send +{ +} + +/// BlockStreamBuilder is an abstraction that would separate the logic for building streams from the blockchain trait +#[async_trait] +pub trait BlockStreamBuilder: Send + Sync { + async fn build_firehose( + &self, + chain: &C, + deployment: DeploymentLocator, + block_cursor: FirehoseCursor, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>>; + + async fn build_polling( + &self, + chain: Arc, + deployment: DeploymentLocator, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>>; +} + +#[derive(Debug, Clone)] +pub struct FirehoseCursor(Option); + +impl FirehoseCursor { + #[allow(non_upper_case_globals)] + pub const None: Self = FirehoseCursor(None); + + pub fn is_none(&self) -> bool { + self.0.is_none() + } +} + +impl fmt::Display for FirehoseCursor { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + f.write_str(&self.0.as_deref().unwrap_or("")) + } +} + +impl From for FirehoseCursor { + fn from(cursor: String) -> Self { + // Treat a cursor of "" as None, not absolutely necessary for correctness since the firehose + // treats both as the same, but makes it a little clearer. + if cursor == "" { + FirehoseCursor::None + } else { + FirehoseCursor(Some(cursor)) + } + } +} + +impl From> for FirehoseCursor { + fn from(cursor: Option) -> Self { + match cursor { + None => FirehoseCursor::None, + Some(s) => FirehoseCursor::from(s), + } + } +} + +impl AsRef> for FirehoseCursor { + fn as_ref(&self) -> &Option { + &self.0 + } +} + +#[derive(Debug)] +pub struct BlockWithTriggers { + pub block: C::Block, + pub trigger_data: Vec, +} + +impl Clone for BlockWithTriggers +where + C::TriggerData: Clone, +{ + fn clone(&self) -> Self { + Self { + block: self.block.clone(), + trigger_data: self.trigger_data.clone(), + } + } +} + +impl BlockWithTriggers { + pub fn new(block: C::Block, mut trigger_data: Vec) -> Self { + // This is where triggers get sorted. + trigger_data.sort(); + Self { + block, + trigger_data, + } + } + + pub fn trigger_count(&self) -> usize { + self.trigger_data.len() + } + + pub fn ptr(&self) -> BlockPtr { + self.block.ptr() + } + + pub fn parent_ptr(&self) -> Option { + self.block.parent_ptr() + } +} + +#[async_trait] +pub trait TriggersAdapter: Send + Sync { + // Return the block that is `offset` blocks before the block pointed to + // by `ptr` from the local cache. An offset of 0 means the block itself, + // an offset of 1 means the block's parent etc. If the block is not in + // the local cache, return `None` + async fn ancestor_block( + &self, + ptr: BlockPtr, + offset: BlockNumber, + ) -> Result, Error>; + + // Returns a sequence of blocks in increasing order of block number. + // Each block will include all of its triggers that match the given `filter`. + // The sequence may omit blocks that contain no triggers, + // but all returned blocks must part of a same chain starting at `chain_base`. + // At least one block will be returned, even if it contains no triggers. + // `step_size` is the suggested number blocks to be scanned. + async fn scan_triggers( + &self, + from: BlockNumber, + to: BlockNumber, + filter: &C::TriggerFilter, + ) -> Result>, Error>; + + // Used for reprocessing blocks when creating a data source. + async fn triggers_in_block( + &self, + logger: &Logger, + block: C::Block, + filter: &C::TriggerFilter, + ) -> Result, Error>; + + /// Return `true` if the block with the given hash and number is on the + /// main chain, i.e., the chain going back from the current chain head. + async fn is_on_main_chain(&self, ptr: BlockPtr) -> Result; + + /// Get pointer to parent of `block`. This is called when reverting `block`. + async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error>; +} + +#[async_trait] +pub trait FirehoseMapper: Send + Sync { + async fn to_block_stream_event( + &self, + logger: &Logger, + response: &firehose::Response, + adapter: &Arc>, + filter: &C::TriggerFilter, + ) -> Result, FirehoseError>; + + /// Returns the [BlockPtr] value for this given block number. This is the block pointer + /// of the longuest according to Firehose view of the blockchain state. + /// + /// This is a thin wrapper around [FirehoseEndpoint#block_ptr_for_number] to make + /// it chain agnostic and callable from chain agnostic [FirehoseBlockStream]. + async fn block_ptr_for_number( + &self, + logger: &Logger, + number: BlockNumber, + ) -> Result; + + /// Returns the closest final block ptr to the block ptr received. + /// On probablitics chain like Ethereum, final is determined by + /// the confirmations threshold configured for the Firehose stack (currently + /// hard-coded to 200). + /// + /// On some other chain like NEAR, the actual final block number is determined + /// from the block itself since it contains information about which block number + /// is final against the current block. + /// + /// To take an example, assuming we are on Ethereum, the final block pointer + /// for block #10212 would be the determined final block #10012 (10212 - 200 = 10012). + async fn final_block_ptr_for( + &self, + logger: &Logger, + block: &C::Block, + ) -> Result; +} + +#[async_trait] +pub trait SubstreamsMapper: Send + Sync { + async fn to_block_stream_event( + &self, + logger: &Logger, + response: &BlockScopedData, + // adapter: &Arc>, + // filter: &C::TriggerFilter, + ) -> Result>, SubstreamsError>; +} + +#[derive(Error, Debug)] +pub enum FirehoseError { + /// We were unable to decode the received block payload into the chain specific Block struct (e.g. chain_ethereum::pb::Block) + #[error("received gRPC block payload cannot be decoded: {0}")] + DecodingError(#[from] prost::DecodeError), + + /// Some unknown error occurred + #[error("unknown error")] + UnknownError(#[from] anyhow::Error), +} + +#[derive(Error, Debug)] +pub enum SubstreamsError { + /// We were unable to decode the received block payload into the chain specific Block struct (e.g. chain_ethereum::pb::Block) + #[error("received gRPC block payload cannot be decoded: {0}")] + DecodingError(#[from] prost::DecodeError), + + /// Some unknown error occurred + #[error("unknown error")] + UnknownError(#[from] anyhow::Error), + + #[error("multiple module output error")] + MultipleModuleOutputError(), + + #[error("unexpected store delta output")] + UnexpectedStoreDeltaOutput(), +} + +#[derive(Debug)] +pub enum BlockStreamEvent { + // The payload is the block the subgraph should revert to, so it becomes the new subgraph head. + Revert(BlockPtr, FirehoseCursor), + + ProcessBlock(BlockWithTriggers, FirehoseCursor), +} + +impl Clone for BlockStreamEvent +where + C::TriggerData: Clone, +{ + fn clone(&self) -> Self { + match self { + Self::Revert(arg0, arg1) => Self::Revert(arg0.clone(), arg1.clone()), + Self::ProcessBlock(arg0, arg1) => Self::ProcessBlock(arg0.clone(), arg1.clone()), + } + } +} + +#[derive(Clone)] +pub struct BlockStreamMetrics { + pub deployment_head: Box, + pub deployment_failed: Box, + pub reverted_blocks: Gauge, + pub stopwatch: StopwatchMetrics, +} + +impl BlockStreamMetrics { + pub fn new( + registry: Arc, + deployment_id: &DeploymentHash, + network: String, + shard: String, + stopwatch: StopwatchMetrics, + ) -> Self { + let reverted_blocks = registry + .new_deployment_gauge( + "deployment_reverted_blocks", + "Track the last reverted block for a subgraph deployment", + deployment_id.as_str(), + ) + .expect("Failed to create `deployment_reverted_blocks` gauge"); + let labels = labels! { + String::from("deployment") => deployment_id.to_string(), + String::from("network") => network, + String::from("shard") => shard + }; + let deployment_head = registry + .new_gauge( + "deployment_head", + "Track the head block number for a deployment", + labels.clone(), + ) + .expect("failed to create `deployment_head` gauge"); + let deployment_failed = registry + .new_gauge( + "deployment_failed", + "Boolean gauge to indicate whether the deployment has failed (1 == failed)", + labels, + ) + .expect("failed to create `deployment_failed` gauge"); + Self { + deployment_head, + deployment_failed, + reverted_blocks, + stopwatch, + } + } +} + +/// Notifications about the chain head advancing. The block ingestor sends +/// an update on this stream whenever the head of the underlying chain +/// changes. The updates have no payload, receivers should call +/// `Store::chain_head_ptr` to check what the latest block is. +pub type ChainHeadUpdateStream = Box + Send + Unpin>; + +pub trait ChainHeadUpdateListener: Send + Sync + 'static { + /// Subscribe to chain head updates for the given network. + fn subscribe(&self, network: String, logger: Logger) -> ChainHeadUpdateStream; +} + +#[cfg(test)] +mod test { + use std::{collections::HashSet, task::Poll}; + + use anyhow::Error; + use futures03::{Stream, StreamExt, TryStreamExt}; + + use crate::{ + blockchain::mock::{MockBlock, MockBlockchain}, + ext::futures::{CancelableError, SharedCancelGuard, StreamExtension}, + }; + + use super::{ + BlockStream, BlockStreamEvent, BlockWithTriggers, BufferedBlockStream, FirehoseCursor, + }; + + #[derive(Debug)] + struct TestStream { + number: u64, + } + + impl BlockStream for TestStream {} + + impl Stream for TestStream { + type Item = Result, Error>; + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.number += 1; + Poll::Ready(Some(Ok(BlockStreamEvent::ProcessBlock( + BlockWithTriggers:: { + block: MockBlock { + number: self.number - 1, + }, + trigger_data: vec![], + }, + FirehoseCursor::None, + )))) + } + } + + #[tokio::test] + async fn consume_stream() { + let initial_block = 100; + let buffer_size = 5; + + let stream = Box::new(TestStream { + number: initial_block, + }); + let guard = SharedCancelGuard::new(); + + let mut stream = BufferedBlockStream::spawn_from_stream(stream, buffer_size) + .map_err(CancelableError::Error) + .cancelable(&guard, || Err(CancelableError::Cancel)); + + let mut blocks = HashSet::::new(); + let mut count = 0; + loop { + match stream.next().await { + None if blocks.len() == 0 => panic!("None before blocks"), + Some(Err(CancelableError::Cancel)) => { + assert!(guard.is_canceled(), "Guard shouldn't be called yet"); + + break; + } + Some(Ok(BlockStreamEvent::ProcessBlock(block_triggers, _))) => { + let block = block_triggers.block; + blocks.insert(block.clone()); + count += 1; + + if block.number > initial_block + buffer_size as u64 { + guard.cancel(); + } + } + _ => panic!("Should not happen"), + }; + } + assert!( + blocks.len() > buffer_size, + "should consume at least a full buffer, consumed {}", + count + ); + assert_eq!(count, blocks.len(), "should not have duplicated blocks"); + } +} diff --git a/graph/src/blockchain/firehose_block_ingestor.rs b/graph/src/blockchain/firehose_block_ingestor.rs new file mode 100644 index 0000000..965272c --- /dev/null +++ b/graph/src/blockchain/firehose_block_ingestor.rs @@ -0,0 +1,168 @@ +use std::{marker::PhantomData, sync::Arc, time::Duration}; + +use crate::{ + blockchain::Block as BlockchainBlock, + components::store::ChainStore, + firehose::{self, decode_firehose_block, FirehoseEndpoint}, + prelude::{error, info, Logger}, + util::backoff::ExponentialBackoff, +}; +use anyhow::{Context, Error}; +use futures03::StreamExt; +use slog::trace; +use tonic::Streaming; + +pub struct FirehoseBlockIngestor +where + M: prost::Message + BlockchainBlock + Default + 'static, +{ + chain_store: Arc, + endpoint: Arc, + logger: Logger, + + phantom: PhantomData, +} + +impl FirehoseBlockIngestor +where + M: prost::Message + BlockchainBlock + Default + 'static, +{ + pub fn new( + chain_store: Arc, + endpoint: Arc, + logger: Logger, + ) -> FirehoseBlockIngestor { + FirehoseBlockIngestor { + chain_store, + endpoint, + logger, + phantom: PhantomData {}, + } + } + + pub async fn run(self) { + use firehose::ForkStep::*; + + let mut latest_cursor = self.fetch_head_cursor().await; + let mut backoff = + ExponentialBackoff::new(Duration::from_millis(250), Duration::from_secs(30)); + + loop { + info!( + self.logger, + "Blockstream disconnected, connecting"; "endpoint uri" => format_args!("{}", self.endpoint), "cursor" => format_args!("{}", latest_cursor), + ); + + let result = self + .endpoint + .clone() + .stream_blocks(firehose::Request { + // Starts at current HEAD block of the chain (viewed from Firehose side) + start_block_num: -1, + start_cursor: latest_cursor.clone(), + fork_steps: vec![StepNew as i32, StepUndo as i32], + ..Default::default() + }) + .await; + + match result { + Ok(stream) => { + info!(self.logger, "Blockstream connected, consuming blocks"); + + // Consume the stream of blocks until an error is hit + latest_cursor = self.process_blocks(latest_cursor, stream).await + } + Err(e) => { + error!(self.logger, "Unable to connect to endpoint: {:?}", e); + } + } + + // If we reach this point, we must wait a bit before retrying + backoff.sleep_async().await; + } + } + + async fn fetch_head_cursor(&self) -> String { + let mut backoff = + ExponentialBackoff::new(Duration::from_millis(250), Duration::from_secs(30)); + loop { + match self.chain_store.clone().chain_head_cursor() { + Ok(cursor) => return cursor.unwrap_or_else(|| "".to_string()), + Err(e) => { + error!(self.logger, "Fetching chain head cursor failed: {:?}", e); + + backoff.sleep_async().await; + } + } + } + } + + /// Consumes the incoming stream of blocks infinitely until it hits an error. In which case + /// the error is logged right away and the latest available cursor is returned + /// upstream for future consumption. + async fn process_blocks( + &self, + cursor: String, + mut stream: Streaming, + ) -> String { + use firehose::ForkStep; + use firehose::ForkStep::*; + + let mut latest_cursor = cursor; + + while let Some(message) = stream.next().await { + match message { + Ok(v) => { + let step = ForkStep::from_i32(v.step) + .expect("Fork step should always match to known value"); + + let result = match step { + StepNew => self.process_new_block(&v).await, + StepUndo => { + trace!(self.logger, "Received undo block to ingest, skipping"); + Ok(()) + } + StepIrreversible | StepUnknown => panic!( + "We explicitly requested StepNew|StepUndo but received something else" + ), + }; + + if let Err(e) = result { + error!(self.logger, "Process block failed: {:?}", e); + break; + } + + latest_cursor = v.cursor; + } + Err(e) => { + info!( + self.logger, + "An error occurred while streaming blocks: {}", e + ); + break; + } + } + } + + error!( + self.logger, + "Stream blocks complete unexpectedly, expecting stream to always stream blocks" + ); + latest_cursor + } + + async fn process_new_block(&self, response: &firehose::Response) -> Result<(), Error> { + let block = decode_firehose_block::(response) + .context("Mapping firehose block to blockchain::Block")?; + + trace!(self.logger, "Received new block to ingest {}", block.ptr()); + + self.chain_store + .clone() + .set_chain_head(block, response.cursor.clone()) + .await + .context("Updating chain head")?; + + Ok(()) + } +} diff --git a/graph/src/blockchain/firehose_block_stream.rs b/graph/src/blockchain/firehose_block_stream.rs new file mode 100644 index 0000000..bcf4c85 --- /dev/null +++ b/graph/src/blockchain/firehose_block_stream.rs @@ -0,0 +1,507 @@ +use super::block_stream::{BlockStream, BlockStreamEvent, FirehoseMapper}; +use super::{Blockchain, TriggersAdapter}; +use crate::blockchain::block_stream::FirehoseCursor; +use crate::blockchain::TriggerFilter; +use crate::firehose::ForkStep::*; +use crate::prelude::*; +use crate::util::backoff::ExponentialBackoff; +use crate::{firehose, firehose::FirehoseEndpoint}; +use async_stream::try_stream; +use futures03::{Stream, StreamExt}; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; +use tonic::Status; + +struct FirehoseBlockStreamMetrics { + deployment: DeploymentHash, + provider: String, + restarts: CounterVec, + connect_duration: GaugeVec, + time_between_responses: HistogramVec, + responses: CounterVec, +} + +impl FirehoseBlockStreamMetrics { + pub fn new( + registry: Arc, + deployment: DeploymentHash, + provider: String, + ) -> Self { + Self { + deployment, + provider, + + restarts: registry + .global_counter_vec( + "deployment_firehose_blockstream_restarts", + "Counts the number of times a Firehose block stream is (re)started", + vec!["deployment", "provider", "success"].as_slice(), + ) + .unwrap(), + + connect_duration: registry + .global_gauge_vec( + "deployment_firehose_blockstream_connect_duration", + "Measures the time it takes to connect a Firehose block stream", + vec!["deployment", "provider"].as_slice(), + ) + .unwrap(), + + time_between_responses: registry + .global_histogram_vec( + "deployment_firehose_blockstream_time_between_responses", + "Measures the time between receiving and processing Firehose stream responses", + vec!["deployment", "provider"].as_slice(), + ) + .unwrap(), + + responses: registry + .global_counter_vec( + "deployment_firehose_blockstream_responses", + "Counts the number of responses received from a Firehose block stream", + vec!["deployment", "provider", "kind"].as_slice(), + ) + .unwrap(), + } + } + + fn observe_successful_connection(&self, time: &mut Instant) { + self.restarts + .with_label_values(&[&self.deployment, &self.provider, "true"]) + .inc(); + self.connect_duration + .with_label_values(&[&self.deployment, &self.provider]) + .set(time.elapsed().as_secs_f64()); + + // Reset last connection timestamp + *time = Instant::now(); + } + + fn observe_failed_connection(&self, time: &mut Instant) { + self.restarts + .with_label_values(&[&self.deployment, &self.provider, "false"]) + .inc(); + self.connect_duration + .with_label_values(&[&self.deployment, &self.provider]) + .set(time.elapsed().as_secs_f64()); + + // Reset last connection timestamp + *time = Instant::now(); + } + + fn observe_response(&self, kind: &str, time: &mut Instant) { + self.time_between_responses + .with_label_values(&[&self.deployment, &self.provider]) + .observe(time.elapsed().as_secs_f64()); + self.responses + .with_label_values(&[&self.deployment, &self.provider, kind]) + .inc(); + + // Reset last response timestamp + *time = Instant::now(); + } +} + +pub struct FirehoseBlockStream { + stream: Pin, Error>> + Send>>, +} + +impl FirehoseBlockStream +where + C: Blockchain, +{ + pub fn new( + deployment: DeploymentHash, + endpoint: Arc, + subgraph_current_block: Option, + cursor: FirehoseCursor, + mapper: Arc, + adapter: Arc>, + filter: Arc, + start_blocks: Vec, + logger: Logger, + registry: Arc, + ) -> Self + where + F: FirehoseMapper + 'static, + { + let manifest_start_block_num = start_blocks + .into_iter() + .min() + // Firehose knows where to start the stream for the specific chain, 0 here means + // start at Genesis block. + .unwrap_or(0); + + let metrics = + FirehoseBlockStreamMetrics::new(registry, deployment, endpoint.provider.clone()); + + FirehoseBlockStream { + stream: Box::pin(stream_blocks( + endpoint, + cursor, + mapper, + adapter, + filter, + manifest_start_block_num, + subgraph_current_block, + logger, + metrics, + )), + } + } +} + +fn stream_blocks>( + endpoint: Arc, + mut latest_cursor: FirehoseCursor, + mapper: Arc, + adapter: Arc>, + filter: Arc, + manifest_start_block_num: BlockNumber, + subgraph_current_block: Option, + logger: Logger, + metrics: FirehoseBlockStreamMetrics, +) -> impl Stream, Error>> { + let mut subgraph_current_block = subgraph_current_block; + let mut start_block_num = subgraph_current_block + .as_ref() + .map(|ptr| { + // Firehose start block is inclusive while the subgraph_current_block is where the actual + // subgraph is currently at. So to process the actual next block, we must start one block + // further in the chain. + ptr.block_number() + 1 as BlockNumber + }) + .unwrap_or(manifest_start_block_num); + + // Sanity check when starting from a subgraph block ptr directly. When + // this happens, we must ensure that Firehose first picked block directly follows the + // subgraph block ptr. So we check that Firehose first picked block's parent is + // equal to subgraph block ptr. + // + // This can happen for example when rewinding, unfailing a deterministic error or + // when switching from RPC to Firehose on Ethereum. + // + // What could go wrong is that the subgraph block ptr points to a forked block but + // since Firehose only accepts `block_number`, it could pick right away the canonical + // block of the longuest chain creating inconsistencies in the data (because it would + // not revert the forked the block). + // + // If a Firehose cursor is present, it's used to resume the stream and as such, there is no need to + // perform the chain continuity check. + // + // If there was no cursor, now we need to check if the subgraph current block is set to something. + // When the graph node deploys a new subgraph, it always create a subgraph ptr for this subgraph, the + // initial subgraph block pointer points to the parent block of the manifest's start block, which is usually + // equivalent (but not always) to manifest's start block number - 1. + // + // Hence, we only need to check the chain continuity if the subgraph current block ptr is higher or equal + // to the subgraph manifest's start block number. Indeed, only in this case (and when there is no firehose + // cursor) it means the subgraph was started and advanced with something else than Firehose and as such, + // chain continuity check needs to be performed. + let mut check_subgraph_continuity = must_check_subgraph_continuity( + &logger, + &subgraph_current_block, + &latest_cursor, + manifest_start_block_num, + ); + if check_subgraph_continuity { + debug!(&logger, "Going to check continuity of chain on first block"); + } + + // Back off exponentially whenever we encounter a connection error or a stream with bad data + let mut backoff = ExponentialBackoff::new(Duration::from_millis(500), Duration::from_secs(45)); + + // This attribute is needed because `try_stream!` seems to break detection of `skip_backoff` assignments + #[allow(unused_assignments)] + let mut skip_backoff = false; + + try_stream! { + loop { + info!( + &logger, + "Blockstream disconnected, connecting"; + "endpoint_uri" => format_args!("{}", endpoint), + "start_block" => start_block_num, + "cursor" => latest_cursor.to_string(), + ); + + // We just reconnected, assume that we want to back off on errors + skip_backoff = false; + + let mut request = firehose::Request { + start_block_num: start_block_num as i64, + start_cursor: latest_cursor.to_string(), + fork_steps: vec![StepNew as i32, StepUndo as i32], + ..Default::default() + }; + + if endpoint.filters_enabled { + request.transforms = filter.as_ref().clone().to_firehose_filter(); + } + + let mut connect_start = Instant::now(); + let req = endpoint.clone().stream_blocks(request); + let result = tokio::time::timeout(Duration::from_secs(120), req).await.map_err(|x| x.into()).and_then(|x| x); + + match result { + Ok(stream) => { + info!(&logger, "Blockstream connected"); + + // Track the time it takes to set up the block stream + metrics.observe_successful_connection(&mut connect_start); + + let mut last_response_time = Instant::now(); + let mut expected_stream_end = false; + + for await response in stream { + match process_firehose_response( + response, + &mut check_subgraph_continuity, + manifest_start_block_num, + subgraph_current_block.as_ref(), + mapper.as_ref(), + &adapter, + &filter, + &logger, + ).await { + Ok(BlockResponse::Proceed(event, cursor)) => { + // Reset backoff because we got a good value from the stream + backoff.reset(); + + metrics.observe_response("proceed", &mut last_response_time); + + yield event; + + latest_cursor = FirehoseCursor::from(cursor); + }, + Ok(BlockResponse::Rewind(revert_to)) => { + // Reset backoff because we got a good value from the stream + backoff.reset(); + + metrics.observe_response("rewind", &mut last_response_time); + + // It's totally correct to pass the None as the cursor here, if we are here, there + // was no cursor before anyway, so it's totally fine to pass `None` + yield BlockStreamEvent::Revert(revert_to.clone(), FirehoseCursor::None); + + latest_cursor = FirehoseCursor::None; + + // We have to reconnect (see below) but we don't wait to wait before doing + // that, so skip the optional backing off at the end of the loop + skip_backoff = true; + + // We must restart the stream to ensure we now send block from revert_to point + // and we add + 1 to start block num because Firehose is inclusive and as such, + // we need to move to "next" block. + start_block_num = revert_to.number + 1; + subgraph_current_block = Some(revert_to); + expected_stream_end = true; + break; + }, + Err(err) => { + // We have an open connection but there was an error processing the Firehose + // response. We will reconnect the stream after this; this is the case where + // we actually _want_ to back off in case we keep running into the same error. + // An example of this situation is if we get invalid block or transaction data + // that cannot be decoded properly. + + metrics.observe_response("error", &mut last_response_time); + + error!(logger, "{:#}", err); + expected_stream_end = true; + break; + } + } + } + + if !expected_stream_end { + error!(logger, "Stream blocks complete unexpectedly, expecting stream to always stream blocks"); + } + }, + Err(e) => { + // We failed to connect and will try again; this is another + // case where we actually _want_ to back off in case we keep + // having connection errors. + + metrics.observe_failed_connection(&mut connect_start); + + error!(logger, "Unable to connect to endpoint: {:#}", e); + } + } + + // If we reach this point, we must wait a bit before retrying, unless `skip_backoff` is true + if !skip_backoff { + backoff.sleep_async().await; + } + } + } +} + +enum BlockResponse { + Proceed(BlockStreamEvent, String), + Rewind(BlockPtr), +} + +async fn process_firehose_response>( + result: Result, + check_subgraph_continuity: &mut bool, + manifest_start_block_num: BlockNumber, + subgraph_current_block: Option<&BlockPtr>, + mapper: &F, + adapter: &Arc>, + filter: &C::TriggerFilter, + logger: &Logger, +) -> Result, Error> { + let response = result.context("An error occurred while streaming blocks")?; + + let event = mapper + .to_block_stream_event(logger, &response, adapter, filter) + .await + .context("Mapping block to BlockStreamEvent failed")?; + + if *check_subgraph_continuity { + info!(logger, "Firehose started from a subgraph pointer without an existing cursor, ensuring chain continuity"); + + if let BlockStreamEvent::ProcessBlock(ref block, _) = event { + let previous_block_ptr = block.parent_ptr(); + if previous_block_ptr.is_some() && previous_block_ptr.as_ref() != subgraph_current_block + { + warn!(&logger, + "Firehose selected first streamed block's parent should match subgraph start block, reverting to last know final chain segment"; + "subgraph_current_block" => &subgraph_current_block.unwrap(), + "firehose_start_block" => &previous_block_ptr.unwrap(), + ); + + let mut revert_to = mapper + .final_block_ptr_for(logger, &block.block) + .await + .context("Could not fetch final block to revert to")?; + + if revert_to.number < manifest_start_block_num { + warn!(&logger, "We would return before subgraph manifest's start block, limiting rewind to manifest's start block"); + + // We must revert up to parent's of manifest start block to ensure we delete everything "including" the start + // block that was processed. + let mut block_num = manifest_start_block_num - 1; + if block_num < 0 { + block_num = 0; + } + + revert_to = mapper + .block_ptr_for_number(logger, block_num) + .await + .context("Could not fetch manifest start block to revert to")?; + } + + return Ok(BlockResponse::Rewind(revert_to)); + } + } + + info!( + logger, + "Subgraph chain continuity is respected, proceeding normally" + ); + *check_subgraph_continuity = false; + } + + Ok(BlockResponse::Proceed(event, response.cursor)) +} + +impl Stream for FirehoseBlockStream { + type Item = Result, Error>; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.stream.poll_next_unpin(cx) + } +} + +impl BlockStream for FirehoseBlockStream {} + +fn must_check_subgraph_continuity( + logger: &Logger, + subgraph_current_block: &Option, + subgraph_cursor: &FirehoseCursor, + subgraph_manifest_start_block_number: i32, +) -> bool { + match subgraph_current_block { + Some(current_block) if subgraph_cursor.is_none() => { + debug!(&logger, "Checking if subgraph current block is after manifest start block"; + "subgraph_current_block_number" => current_block.number, + "manifest_start_block_number" => subgraph_manifest_start_block_number, + ); + + current_block.number >= subgraph_manifest_start_block_number + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use crate::blockchain::{ + block_stream::FirehoseCursor, firehose_block_stream::must_check_subgraph_continuity, + BlockPtr, + }; + use slog::{o, Logger}; + + #[test] + fn check_continuity() { + let logger = Logger::root(slog::Discard, o!()); + let no_current_block: Option = None; + let no_cursor = FirehoseCursor::None; + let some_cursor = FirehoseCursor::from("abc".to_string()); + let some_current_block = |number: i32| -> Option { + Some(BlockPtr { + hash: vec![0xab, 0xcd].into(), + number, + }) + }; + + // Nothing + + assert_eq!( + must_check_subgraph_continuity(&logger, &no_current_block, &no_cursor, 10), + false, + ); + + // No cursor, subgraph current block ptr <, ==, > than manifest start block num + + assert_eq!( + must_check_subgraph_continuity(&logger, &some_current_block(9), &no_cursor, 10), + false, + ); + + assert_eq!( + must_check_subgraph_continuity(&logger, &some_current_block(10), &no_cursor, 10), + true, + ); + + assert_eq!( + must_check_subgraph_continuity(&logger, &some_current_block(11), &no_cursor, 10), + true, + ); + + // Some cursor, subgraph current block ptr <, ==, > than manifest start block num + + assert_eq!( + must_check_subgraph_continuity(&logger, &no_current_block, &some_cursor, 10), + false, + ); + + assert_eq!( + must_check_subgraph_continuity(&logger, &some_current_block(9), &some_cursor, 10), + false, + ); + + assert_eq!( + must_check_subgraph_continuity(&logger, &some_current_block(10), &some_cursor, 10), + false, + ); + + assert_eq!( + must_check_subgraph_continuity(&logger, &some_current_block(11), &some_cursor, 10), + false, + ); + } +} diff --git a/graph/src/blockchain/mock.rs b/graph/src/blockchain/mock.rs new file mode 100644 index 0000000..5902795 --- /dev/null +++ b/graph/src/blockchain/mock.rs @@ -0,0 +1,339 @@ +use crate::{ + components::{link_resolver::LinkResolver, store::BlockNumber}, + prelude::DataSourceTemplateInfo, +}; +use anyhow::Error; +use async_trait::async_trait; +use core::fmt; +use serde::Deserialize; +use std::{convert::TryFrom, sync::Arc}; + +use super::{ + block_stream::{self, FirehoseCursor}, + HostFn, IngestorError, TriggerWithHandler, +}; + +use super::{ + block_stream::BlockWithTriggers, Block, BlockPtr, Blockchain, BlockchainKind, DataSource, + DataSourceTemplate, NodeCapabilities, RuntimeAdapter, TriggerData, TriggerFilter, + TriggersAdapter, UnresolvedDataSource, UnresolvedDataSourceTemplate, +}; + +#[derive(Debug)] +pub struct MockBlockchain; + +#[derive(Clone, Hash, Eq, PartialEq, Debug, Default)] +pub struct MockBlock { + pub number: u64, +} + +impl Block for MockBlock { + fn ptr(&self) -> BlockPtr { + todo!() + } + + fn parent_ptr(&self) -> Option { + todo!() + } +} + +#[derive(Clone)] +pub struct MockDataSource; + +impl TryFrom> for MockDataSource { + type Error = Error; + + fn try_from(_value: DataSourceTemplateInfo) -> Result { + todo!() + } +} + +impl DataSource for MockDataSource { + fn address(&self) -> Option<&[u8]> { + todo!() + } + + fn start_block(&self) -> crate::components::store::BlockNumber { + todo!() + } + + fn name(&self) -> &str { + todo!() + } + + fn kind(&self) -> &str { + todo!() + } + + fn network(&self) -> Option<&str> { + todo!() + } + + fn context(&self) -> std::sync::Arc> { + todo!() + } + + fn creation_block(&self) -> Option { + todo!() + } + + fn api_version(&self) -> semver::Version { + todo!() + } + + fn runtime(&self) -> Option>> { + todo!() + } + + fn match_and_decode( + &self, + _trigger: &C::TriggerData, + _block: &std::sync::Arc, + _logger: &slog::Logger, + ) -> Result>, anyhow::Error> { + todo!() + } + + fn is_duplicate_of(&self, _other: &Self) -> bool { + todo!() + } + + fn as_stored_dynamic_data_source(&self) -> crate::components::store::StoredDynamicDataSource { + todo!() + } + + fn from_stored_dynamic_data_source( + _template: &C::DataSourceTemplate, + _stored: crate::components::store::StoredDynamicDataSource, + ) -> Result { + todo!() + } + + fn validate(&self) -> Vec { + todo!() + } +} + +#[derive(Clone, Default, Deserialize)] +pub struct MockUnresolvedDataSource; + +#[async_trait] +impl UnresolvedDataSource for MockUnresolvedDataSource { + async fn resolve( + self, + _resolver: &Arc, + _logger: &slog::Logger, + _manifest_idx: u32, + ) -> Result { + todo!() + } +} + +#[derive(Debug, Clone)] +pub struct MockDataSourceTemplate; + +impl DataSourceTemplate for MockDataSourceTemplate { + fn api_version(&self) -> semver::Version { + todo!() + } + + fn runtime(&self) -> Option>> { + todo!() + } + + fn name(&self) -> &str { + todo!() + } + + fn manifest_idx(&self) -> u32 { + todo!() + } +} + +#[derive(Clone, Default, Deserialize)] +pub struct MockUnresolvedDataSourceTemplate; + +#[async_trait] +impl UnresolvedDataSourceTemplate for MockUnresolvedDataSourceTemplate { + async fn resolve( + self, + _resolver: &Arc, + _logger: &slog::Logger, + _manifest_idx: u32, + ) -> Result { + todo!() + } +} + +pub struct MockTriggersAdapter; + +#[async_trait] +impl TriggersAdapter for MockTriggersAdapter { + async fn ancestor_block( + &self, + _ptr: BlockPtr, + _offset: BlockNumber, + ) -> Result, Error> { + todo!() + } + + async fn scan_triggers( + &self, + _from: crate::components::store::BlockNumber, + _to: crate::components::store::BlockNumber, + _filter: &C::TriggerFilter, + ) -> Result>, Error> { + todo!() + } + + async fn triggers_in_block( + &self, + _logger: &slog::Logger, + _block: C::Block, + _filter: &C::TriggerFilter, + ) -> Result, Error> { + todo!() + } + + async fn is_on_main_chain(&self, _ptr: BlockPtr) -> Result { + todo!() + } + + async fn parent_ptr(&self, _block: &BlockPtr) -> Result, Error> { + todo!() + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub struct MockTriggerData; + +impl TriggerData for MockTriggerData { + fn error_context(&self) -> String { + todo!() + } +} + +#[derive(Debug)] +pub struct MockMappingTrigger {} + +#[derive(Clone, Default)] +pub struct MockTriggerFilter; + +impl TriggerFilter for MockTriggerFilter { + fn extend<'a>(&mut self, _data_sources: impl Iterator + Clone) { + todo!() + } + + fn node_capabilities(&self) -> C::NodeCapabilities { + todo!() + } + + fn extend_with_template( + &mut self, + _data_source: impl Iterator::DataSourceTemplate>, + ) { + todo!() + } + + fn to_firehose_filter(self) -> Vec { + todo!() + } +} + +#[derive(Debug)] +pub struct MockNodeCapabilities; + +impl fmt::Display for MockNodeCapabilities { + fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { + todo!() + } +} + +impl NodeCapabilities for MockNodeCapabilities { + fn from_data_sources(_data_sources: &[C::DataSource]) -> Self { + todo!() + } +} + +pub struct MockRuntimeAdapter; + +impl RuntimeAdapter for MockRuntimeAdapter { + fn host_fns(&self, _ds: &C::DataSource) -> Result, Error> { + todo!() + } +} + +#[async_trait] +impl Blockchain for MockBlockchain { + const KIND: BlockchainKind = BlockchainKind::Ethereum; + + type Block = MockBlock; + + type DataSource = MockDataSource; + + type UnresolvedDataSource = MockUnresolvedDataSource; + + type DataSourceTemplate = MockDataSourceTemplate; + + type UnresolvedDataSourceTemplate = MockUnresolvedDataSourceTemplate; + + type TriggerData = MockTriggerData; + + type MappingTrigger = MockMappingTrigger; + + type TriggerFilter = MockTriggerFilter; + + type NodeCapabilities = MockNodeCapabilities; + + fn triggers_adapter( + &self, + _loc: &crate::components::store::DeploymentLocator, + _capabilities: &Self::NodeCapabilities, + _unified_api_version: crate::data::subgraph::UnifiedMappingApiVersion, + ) -> Result>, anyhow::Error> { + todo!() + } + + async fn new_firehose_block_stream( + &self, + _deployment: crate::components::store::DeploymentLocator, + _block_cursor: FirehoseCursor, + _start_blocks: Vec, + _subgraph_current_block: Option, + _filter: std::sync::Arc, + _unified_api_version: crate::data::subgraph::UnifiedMappingApiVersion, + ) -> Result>, anyhow::Error> { + todo!() + } + + async fn new_polling_block_stream( + &self, + _deployment: crate::components::store::DeploymentLocator, + _start_blocks: Vec, + _subgraph_current_block: Option, + _filter: std::sync::Arc, + _unified_api_version: crate::data::subgraph::UnifiedMappingApiVersion, + ) -> Result>, anyhow::Error> { + todo!() + } + + fn chain_store(&self) -> std::sync::Arc { + todo!() + } + + async fn block_pointer_from_number( + &self, + _logger: &slog::Logger, + _number: crate::components::store::BlockNumber, + ) -> Result { + todo!() + } + + fn runtime_adapter(&self) -> std::sync::Arc> { + todo!() + } + + fn is_firehose_supported(&self) -> bool { + todo!() + } +} diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs new file mode 100644 index 0000000..dc18103 --- /dev/null +++ b/graph/src/blockchain/mod.rs @@ -0,0 +1,390 @@ +//! The `blockchain` module exports the necessary traits and data structures to integrate a +//! blockchain into Graph Node. A blockchain is represented by an implementation of the `Blockchain` +//! trait which is the centerpiece of this module. + +pub mod block_stream; +pub mod firehose_block_ingestor; +pub mod firehose_block_stream; +pub mod mock; +pub mod polling_block_stream; +pub mod substreams_block_stream; +mod types; + +// Try to reexport most of the necessary types +use crate::{ + cheap_clone::CheapClone, + components::store::{DeploymentLocator, StoredDynamicDataSource}, + data::subgraph::UnifiedMappingApiVersion, + data_source, + prelude::DataSourceContext, + runtime::{gas::GasCounter, AscHeap, HostExportError}, +}; +use crate::{ + components::{ + store::{BlockNumber, ChainStore}, + subgraph::DataSourceTemplateInfo, + }, + prelude::{thiserror::Error, LinkResolver}, +}; +use anyhow::{anyhow, Context, Error}; +use async_trait::async_trait; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use slog::Logger; +use std::{ + any::Any, + collections::HashMap, + convert::TryFrom, + fmt::{self, Debug}, + str::FromStr, + sync::Arc, +}; +use web3::types::H256; + +pub use block_stream::{ChainHeadUpdateListener, ChainHeadUpdateStream, TriggersAdapter}; +pub use types::{BlockHash, BlockPtr, ChainIdentifier}; + +use self::block_stream::{BlockStream, FirehoseCursor}; + +pub trait TriggersAdapterSelector: Sync + Send { + fn triggers_adapter( + &self, + loc: &DeploymentLocator, + capabilities: &C::NodeCapabilities, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error>; +} + +pub trait Block: Send + Sync { + fn ptr(&self) -> BlockPtr; + fn parent_ptr(&self) -> Option; + + fn number(&self) -> i32 { + self.ptr().number + } + + fn hash(&self) -> BlockHash { + self.ptr().hash + } + + fn parent_hash(&self) -> Option { + self.parent_ptr().map(|ptr| ptr.hash) + } + + /// The data that should be stored for this block in the `ChainStore` + fn data(&self) -> Result { + Ok(serde_json::Value::Null) + } +} + +#[async_trait] +// This is only `Debug` because some tests require that +pub trait Blockchain: Debug + Sized + Send + Sync + Unpin + 'static { + const KIND: BlockchainKind; + const ALIASES: &'static [&'static str] = &[]; + + // The `Clone` bound is used when reprocessing a block, because `triggers_in_block` requires an + // owned `Block`. It would be good to come up with a way to remove this bound. + type Block: Block + Clone + Debug + Default; + type DataSource: DataSource; + type UnresolvedDataSource: UnresolvedDataSource; + + type DataSourceTemplate: DataSourceTemplate + Clone; + type UnresolvedDataSourceTemplate: UnresolvedDataSourceTemplate + Clone; + + /// Trigger data as parsed from the triggers adapter. + type TriggerData: TriggerData + Ord + Send + Sync + Debug; + + /// Decoded trigger ready to be processed by the mapping. + /// New implementations should have this be the same as `TriggerData`. + type MappingTrigger: Send + Sync + Debug; + + /// Trigger filter used as input to the triggers adapter. + type TriggerFilter: TriggerFilter; + + type NodeCapabilities: NodeCapabilities + std::fmt::Display; + + fn triggers_adapter( + &self, + log: &DeploymentLocator, + capabilities: &Self::NodeCapabilities, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error>; + + async fn new_firehose_block_stream( + &self, + deployment: DeploymentLocator, + block_cursor: FirehoseCursor, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error>; + + async fn new_polling_block_stream( + &self, + deployment: DeploymentLocator, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>, Error>; + + fn chain_store(&self) -> Arc; + + async fn block_pointer_from_number( + &self, + logger: &Logger, + number: BlockNumber, + ) -> Result; + + fn runtime_adapter(&self) -> Arc>; + + fn is_firehose_supported(&self) -> bool; +} + +#[derive(Error, Debug)] +pub enum IngestorError { + /// The Ethereum node does not know about this block for some reason, probably because it + /// disappeared in a chain reorg. + #[error("Block data unavailable, block was likely uncled (block hash = {0:?})")] + BlockUnavailable(H256), + + /// The Ethereum node does not know about this block for some reason, probably because it + /// disappeared in a chain reorg. + #[error("Receipt for tx {1:?} unavailable, block was likely uncled (block hash = {0:?})")] + ReceiptUnavailable(H256, H256), + + /// An unexpected error occurred. + #[error("Ingestor error: {0:#}")] + Unknown(#[from] Error), +} + +impl From for IngestorError { + fn from(e: web3::Error) -> Self { + IngestorError::Unknown(anyhow::anyhow!(e)) + } +} + +pub trait TriggerFilter: Default + Clone + Send + Sync { + fn from_data_sources<'a>( + data_sources: impl Iterator + Clone, + ) -> Self { + let mut this = Self::default(); + this.extend(data_sources); + this + } + + fn extend_with_template(&mut self, data_source: impl Iterator); + + fn extend<'a>(&mut self, data_sources: impl Iterator + Clone); + + fn node_capabilities(&self) -> C::NodeCapabilities; + + fn to_firehose_filter(self) -> Vec; +} + +pub trait DataSource: + 'static + Sized + Send + Sync + Clone + TryFrom, Error = anyhow::Error> +{ + fn address(&self) -> Option<&[u8]>; + fn start_block(&self) -> BlockNumber; + fn name(&self) -> &str; + fn kind(&self) -> &str; + fn network(&self) -> Option<&str>; + fn context(&self) -> Arc>; + fn creation_block(&self) -> Option; + fn api_version(&self) -> semver::Version; + fn runtime(&self) -> Option>>; + + /// Checks if `trigger` matches this data source, and if so decodes it into a `MappingTrigger`. + /// A return of `Ok(None)` mean the trigger does not match. + /// + /// Performance note: This is very hot code, because in the worst case it could be called a + /// quadratic T*D times where T is the total number of triggers in the chain and D is the number + /// of data sources in the subgraph. So it could be called billions, or even trillions, of times + /// in the sync time of a subgraph. + /// + /// This is typicaly reduced by the triggers being pre-filtered in the block stream. But with + /// dynamic data sources the block stream does not filter on the dynamic parameters, so the + /// matching should efficently discard false positives. + fn match_and_decode( + &self, + trigger: &C::TriggerData, + block: &Arc, + logger: &Logger, + ) -> Result>, Error>; + + fn is_duplicate_of(&self, other: &Self) -> bool; + + fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource; + + fn from_stored_dynamic_data_source( + template: &C::DataSourceTemplate, + stored: StoredDynamicDataSource, + ) -> Result; + + /// Used as part of manifest validation. If there are no errors, return an empty vector. + fn validate(&self) -> Vec; +} + +#[async_trait] +pub trait UnresolvedDataSourceTemplate: + 'static + Sized + Send + Sync + DeserializeOwned + Default +{ + async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + manifest_idx: u32, + ) -> Result; +} + +pub trait DataSourceTemplate: Send + Sync + Debug { + fn api_version(&self) -> semver::Version; + fn runtime(&self) -> Option>>; + fn name(&self) -> &str; + fn manifest_idx(&self) -> u32; +} + +#[async_trait] +pub trait UnresolvedDataSource: + 'static + Sized + Send + Sync + DeserializeOwned +{ + async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + manifest_idx: u32, + ) -> Result; +} + +pub trait TriggerData { + /// If there is an error when processing this trigger, this will called to add relevant context. + /// For example an useful return is: `"block # (), transaction ". + fn error_context(&self) -> String; +} + +pub struct HostFnCtx<'a> { + pub logger: Logger, + pub block_ptr: BlockPtr, + pub heap: &'a mut dyn AscHeap, + pub gas: GasCounter, +} + +/// Host fn that receives one u32 argument and returns an u32. +/// The name for an AS fuction is in the format `.`. +#[derive(Clone)] +pub struct HostFn { + pub name: &'static str, + pub func: Arc Result>, +} + +impl CheapClone for HostFn { + fn cheap_clone(&self) -> Self { + HostFn { + name: self.name, + func: self.func.cheap_clone(), + } + } +} + +pub trait RuntimeAdapter: Send + Sync { + fn host_fns(&self, ds: &C::DataSource) -> Result, Error>; +} + +pub trait NodeCapabilities { + fn from_data_sources(data_sources: &[C::DataSource]) -> Self; +} + +/// Blockchain technologies supported by Graph Node. +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum BlockchainKind { + /// Arweave chains that are compatible. + Arweave, + + /// Ethereum itself or chains that are compatible. + Ethereum, + + /// NEAR chains (Mainnet, Testnet) or chains that are compatible + Near, + + /// Cosmos chains + Cosmos, + + Substreams, +} + +impl fmt::Display for BlockchainKind { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let value = match self { + BlockchainKind::Arweave => "arweave", + BlockchainKind::Ethereum => "ethereum", + BlockchainKind::Near => "near", + BlockchainKind::Cosmos => "cosmos", + BlockchainKind::Substreams => "substreams", + }; + write!(f, "{}", value) + } +} + +impl FromStr for BlockchainKind { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s { + "arweave" => Ok(BlockchainKind::Arweave), + "ethereum" => Ok(BlockchainKind::Ethereum), + "near" => Ok(BlockchainKind::Near), + "cosmos" => Ok(BlockchainKind::Cosmos), + "substreams" => Ok(BlockchainKind::Substreams), + _ => Err(anyhow!("unknown blockchain kind {}", s)), + } + } +} + +impl BlockchainKind { + pub fn from_manifest(manifest: &serde_yaml::Mapping) -> Result { + use serde_yaml::Value; + + // The `kind` field of the first data source in the manifest. + // + // Split by `/` to, for example, read 'ethereum' in 'ethereum/contracts'. + manifest + .get(&Value::String("dataSources".to_owned())) + .and_then(|ds| ds.as_sequence()) + .and_then(|ds| ds.first()) + .and_then(|ds| ds.as_mapping()) + .and_then(|ds| ds.get(&Value::String("kind".to_owned()))) + .and_then(|kind| kind.as_str()) + .and_then(|kind| kind.split('/').next()) + .context("invalid manifest") + .and_then(BlockchainKind::from_str) + } +} + +/// A collection of blockchains, keyed by `BlockchainKind` and network. +#[derive(Default, Debug, Clone)] +pub struct BlockchainMap(HashMap<(BlockchainKind, String), Arc>); + +impl BlockchainMap { + pub fn new() -> Self { + Self::default() + } + + pub fn insert(&mut self, network: String, chain: Arc) { + self.0.insert((C::KIND, network), chain); + } + + pub fn get(&self, network: String) -> Result, Error> { + self.0 + .get(&(C::KIND, network.clone())) + .with_context(|| format!("no network {} found on chain {}", network, C::KIND))? + .cheap_clone() + .downcast() + .map_err(|_| anyhow!("unable to downcast, wrong type for blockchain {}", C::KIND)) + } +} + +pub type TriggerWithHandler = data_source::TriggerWithHandler<::MappingTrigger>; diff --git a/graph/src/blockchain/polling_block_stream.rs b/graph/src/blockchain/polling_block_stream.rs new file mode 100644 index 0000000..daebeef --- /dev/null +++ b/graph/src/blockchain/polling_block_stream.rs @@ -0,0 +1,611 @@ +use anyhow::Error; +use futures03::{stream::Stream, Future, FutureExt}; +use std::cmp; +use std::collections::VecDeque; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; + +use super::block_stream::{ + BlockStream, BlockStreamEvent, BlockWithTriggers, ChainHeadUpdateStream, FirehoseCursor, + TriggersAdapter, +}; +use super::{Block, BlockPtr, Blockchain}; + +use crate::components::store::BlockNumber; +use crate::data::subgraph::UnifiedMappingApiVersion; +use crate::prelude::*; + +// A high number here forces a slow start. +const STARTING_PREVIOUS_TRIGGERS_PER_BLOCK: f64 = 1_000_000.0; + +enum BlockStreamState +where + C: Blockchain, +{ + /// Starting or restarting reconciliation. + /// + /// Valid next states: Reconciliation + BeginReconciliation, + + /// The BlockStream is reconciling the subgraph store state with the chain store state. + /// + /// Valid next states: YieldingBlocks, Idle, BeginReconciliation (in case of revert) + Reconciliation(Pin, Error>> + Send>>), + + /// The BlockStream is emitting blocks that must be processed in order to bring the subgraph + /// store up to date with the chain store. + /// + /// Valid next states: BeginReconciliation + YieldingBlocks(Box>>), + + /// The BlockStream experienced an error and is pausing before attempting to produce + /// blocks again. + /// + /// Valid next states: BeginReconciliation + RetryAfterDelay(Pin> + Send>>), + + /// The BlockStream has reconciled the subgraph store and chain store states. + /// No more work is needed until a chain head update. + /// + /// Valid next states: BeginReconciliation + Idle, +} + +/// A single next step to take in reconciling the state of the subgraph store with the state of the +/// chain store. +enum ReconciliationStep +where + C: Blockchain, +{ + /// Revert(to) the block the subgraph should be reverted to, so it becomes the new subgraph + /// head. + Revert(BlockPtr), + + /// Move forwards, processing one or more blocks. Second element is the block range size. + ProcessDescendantBlocks(Vec>, BlockNumber), + + /// This step is a no-op, but we need to check again for a next step. + Retry, + + /// Subgraph pointer now matches chain head pointer. + /// Reconciliation is complete. + Done, +} + +struct PollingBlockStreamContext +where + C: Blockchain, +{ + chain_store: Arc, + adapter: Arc>, + node_id: NodeId, + subgraph_id: DeploymentHash, + // This is not really a block number, but the (unsigned) difference + // between two block numbers + reorg_threshold: BlockNumber, + filter: Arc, + start_blocks: Vec, + logger: Logger, + previous_triggers_per_block: f64, + // Not a BlockNumber, but the difference between two block numbers + previous_block_range_size: BlockNumber, + // Not a BlockNumber, but the difference between two block numbers + max_block_range_size: BlockNumber, + target_triggers_per_block_range: u64, + unified_api_version: UnifiedMappingApiVersion, + current_block: Option, +} + +impl Clone for PollingBlockStreamContext { + fn clone(&self) -> Self { + Self { + chain_store: self.chain_store.cheap_clone(), + adapter: self.adapter.clone(), + node_id: self.node_id.clone(), + subgraph_id: self.subgraph_id.clone(), + reorg_threshold: self.reorg_threshold, + filter: self.filter.clone(), + start_blocks: self.start_blocks.clone(), + logger: self.logger.clone(), + previous_triggers_per_block: self.previous_triggers_per_block, + previous_block_range_size: self.previous_block_range_size, + max_block_range_size: self.max_block_range_size, + target_triggers_per_block_range: self.target_triggers_per_block_range, + unified_api_version: self.unified_api_version.clone(), + current_block: self.current_block.clone(), + } + } +} + +pub struct PollingBlockStream { + state: BlockStreamState, + consecutive_err_count: u32, + chain_head_update_stream: ChainHeadUpdateStream, + ctx: PollingBlockStreamContext, +} + +// This is the same as `ReconciliationStep` but without retries. +enum NextBlocks +where + C: Blockchain, +{ + /// Blocks and range size + Blocks(VecDeque>, BlockNumber), + + // The payload is block the subgraph should be reverted to, so it becomes the new subgraph head. + Revert(BlockPtr), + Done, +} + +impl PollingBlockStream +where + C: Blockchain, +{ + pub fn new( + chain_store: Arc, + chain_head_update_stream: ChainHeadUpdateStream, + adapter: Arc>, + node_id: NodeId, + subgraph_id: DeploymentHash, + filter: Arc, + start_blocks: Vec, + reorg_threshold: BlockNumber, + logger: Logger, + max_block_range_size: BlockNumber, + target_triggers_per_block_range: u64, + unified_api_version: UnifiedMappingApiVersion, + start_block: Option, + ) -> Self { + Self { + state: BlockStreamState::BeginReconciliation, + consecutive_err_count: 0, + chain_head_update_stream, + ctx: PollingBlockStreamContext { + current_block: start_block, + chain_store, + adapter, + node_id, + subgraph_id, + reorg_threshold, + logger, + filter, + start_blocks, + previous_triggers_per_block: STARTING_PREVIOUS_TRIGGERS_PER_BLOCK, + previous_block_range_size: 1, + max_block_range_size, + target_triggers_per_block_range, + unified_api_version, + }, + } + } +} + +impl PollingBlockStreamContext +where + C: Blockchain, +{ + /// Perform reconciliation steps until there are blocks to yield or we are up-to-date. + async fn next_blocks(&self) -> Result, Error> { + let ctx = self.clone(); + + loop { + match ctx.get_next_step().await? { + ReconciliationStep::ProcessDescendantBlocks(next_blocks, range_size) => { + return Ok(NextBlocks::Blocks( + next_blocks.into_iter().collect(), + range_size, + )); + } + ReconciliationStep::Retry => { + continue; + } + ReconciliationStep::Done => { + return Ok(NextBlocks::Done); + } + ReconciliationStep::Revert(parent_ptr) => { + return Ok(NextBlocks::Revert(parent_ptr)) + } + } + } + } + + /// Determine the next reconciliation step. Does not modify Store or ChainStore. + async fn get_next_step(&self) -> Result, Error> { + let ctx = self.clone(); + let start_blocks = self.start_blocks.clone(); + let max_block_range_size = self.max_block_range_size; + + // Get pointers from database for comparison + let head_ptr_opt = ctx.chain_store.chain_head_ptr().await?; + let subgraph_ptr = self.current_block.clone(); + + // If chain head ptr is not set yet + let head_ptr = match head_ptr_opt { + Some(head_ptr) => head_ptr, + + // Don't do any reconciliation until the chain store has more blocks + None => { + return Ok(ReconciliationStep::Done); + } + }; + + trace!( + ctx.logger, "Chain head pointer"; + "hash" => format!("{:?}", head_ptr.hash), + "number" => &head_ptr.number + ); + trace!( + ctx.logger, "Subgraph pointer"; + "hash" => format!("{:?}", subgraph_ptr.as_ref().map(|block| &block.hash)), + "number" => subgraph_ptr.as_ref().map(|block| &block.number), + ); + + // Make sure not to include genesis in the reorg threshold. + let reorg_threshold = ctx.reorg_threshold.min(head_ptr.number); + + // Only continue if the subgraph block ptr is behind the head block ptr. + // subgraph_ptr > head_ptr shouldn't happen, but if it does, it's safest to just stop. + if let Some(ptr) = &subgraph_ptr { + if ptr.number >= head_ptr.number { + return Ok(ReconciliationStep::Done); + } + } + + // Subgraph ptr is behind head ptr. + // Let's try to move the subgraph ptr one step in the right direction. + // First question: which direction should the ptr be moved? + // + // We will use a different approach to deciding the step direction depending on how far + // the subgraph ptr is behind the head ptr. + // + // Normally, we need to worry about chain reorganizations -- situations where the + // Ethereum client discovers a new longer chain of blocks different from the one we had + // processed so far, forcing us to rollback one or more blocks we had already + // processed. + // We can't assume that blocks we receive are permanent. + // + // However, as a block receives more and more confirmations, eventually it becomes safe + // to assume that that block will be permanent. + // The probability of a block being "uncled" approaches zero as more and more blocks + // are chained on after that block. + // Eventually, the probability is so low, that a block is effectively permanent. + // The "effectively permanent" part is what makes blockchains useful. + // See here for more discussion: + // https://blog.ethereum.org/2016/05/09/on-settlement-finality/ + // + // Accordingly, if the subgraph ptr is really far behind the head ptr, then we can + // trust that the Ethereum node knows what the real, permanent block is for that block + // number. + // We'll define "really far" to mean "greater than reorg_threshold blocks". + // + // If the subgraph ptr is not too far behind the head ptr (i.e. less than + // reorg_threshold blocks behind), then we have to allow for the possibility that the + // block might be on the main chain now, but might become uncled in the future. + // + // Most importantly: Our ability to make this assumption (or not) will determine what + // Ethereum RPC calls can give us accurate data without race conditions. + // (This is mostly due to some unfortunate API design decisions on the Ethereum side) + if subgraph_ptr.is_none() + || (head_ptr.number - subgraph_ptr.as_ref().unwrap().number) > reorg_threshold + { + // Since we are beyond the reorg threshold, the Ethereum node knows what block has + // been permanently assigned this block number. + // This allows us to ask the node: does subgraph_ptr point to a block that was + // permanently accepted into the main chain, or does it point to a block that was + // uncled? + let is_on_main_chain = match &subgraph_ptr { + Some(ptr) => ctx.adapter.is_on_main_chain(ptr.clone()).await?, + None => true, + }; + if !is_on_main_chain { + // The subgraph ptr points to a block that was uncled. + // We need to revert this block. + // + // Note: We can safely unwrap the subgraph ptr here, because + // if it was `None`, `is_on_main_chain` would be true. + let from = subgraph_ptr.unwrap(); + let parent = self.parent_ptr(&from, "is_on_main_chain").await?; + + return Ok(ReconciliationStep::Revert(parent)); + } + + // The subgraph ptr points to a block on the main chain. + // This means that the last block we processed does not need to be + // reverted. + // Therefore, our direction of travel will be forward, towards the + // chain head. + + // As an optimization, instead of advancing one block, we will use an + // Ethereum RPC call to find the first few blocks that have event(s) we + // are interested in that lie within the block range between the subgraph ptr + // and either the next data source start_block or the reorg threshold. + // Note that we use block numbers here. + // This is an artifact of Ethereum RPC limitations. + // It is only safe to use block numbers because we are beyond the reorg + // threshold. + + // Start with first block after subgraph ptr; if the ptr is None, + // then we start with the genesis block + let from = subgraph_ptr.map_or(0, |ptr| ptr.number + 1); + + // Get the next subsequent data source start block to ensure the block + // range is aligned with data source. This is not necessary for + // correctness, but it avoids an ineffecient situation such as the range + // being 0..100 and the start block for a data source being 99, then + // `calls_in_block_range` would request unecessary traces for the blocks + // 0 to 98 because the start block is within the range. + let next_start_block: BlockNumber = start_blocks + .into_iter() + .filter(|block_num| block_num > &from) + .min() + .unwrap_or(BLOCK_NUMBER_MAX); + + // End either just before the the next data source start_block or just + // prior to the reorg threshold. It isn't safe to go farther than the + // reorg threshold due to race conditions. + let to_limit = cmp::min(head_ptr.number - reorg_threshold, next_start_block - 1); + + // Calculate the range size according to the target number of triggers, + // respecting the global maximum and also not increasing too + // drastically from the previous block range size. + // + // An example of the block range dynamics: + // - Start with a block range of 1, target of 1000. + // - Scan 1 block: + // 0 triggers found, max_range_size = 10, range_size = 10 + // - Scan 10 blocks: + // 2 triggers found, 0.2 per block, range_size = 1000 / 0.2 = 5000 + // - Scan 5000 blocks: + // 10000 triggers found, 2 per block, range_size = 1000 / 2 = 500 + // - Scan 500 blocks: + // 1000 triggers found, 2 per block, range_size = 1000 / 2 = 500 + let range_size_upper_limit = + max_block_range_size.min(ctx.previous_block_range_size * 10); + let range_size = if ctx.previous_triggers_per_block == 0.0 { + range_size_upper_limit + } else { + (self.target_triggers_per_block_range as f64 / ctx.previous_triggers_per_block) + .max(1.0) + .min(range_size_upper_limit as f64) as BlockNumber + }; + let to = cmp::min(from + range_size - 1, to_limit); + + info!( + ctx.logger, + "Scanning blocks [{}, {}]", from, to; + "range_size" => range_size + ); + + let blocks = self.adapter.scan_triggers(from, to, &self.filter).await?; + + Ok(ReconciliationStep::ProcessDescendantBlocks( + blocks, range_size, + )) + } else { + // The subgraph ptr is not too far behind the head ptr. + // This means a few things. + // + // First, because we are still within the reorg threshold, + // we can't trust the Ethereum RPC methods that use block numbers. + // Block numbers in this region are not yet immutable pointers to blocks; + // the block associated with a particular block number on the Ethereum node could + // change under our feet at any time. + // + // Second, due to how the BlockIngestor is designed, we get a helpful guarantee: + // the head block and at least its reorg_threshold most recent ancestors will be + // present in the block store. + // This allows us to work locally in the block store instead of relying on + // Ethereum RPC calls, so that we are not subject to the limitations of the RPC + // API. + + // To determine the step direction, we need to find out if the subgraph ptr refers + // to a block that is an ancestor of the head block. + // We can do so by walking back up the chain from the head block to the appropriate + // block number, and checking to see if the block we found matches the + // subgraph_ptr. + + let subgraph_ptr = + subgraph_ptr.expect("subgraph block pointer should not be `None` here"); + + // Precondition: subgraph_ptr.number < head_ptr.number + // Walk back to one block short of subgraph_ptr.number + let offset = head_ptr.number - subgraph_ptr.number - 1; + + // In principle this block should be in the store, but we have seen this error for deep + // reorgs in ropsten. + let head_ancestor_opt = self.adapter.ancestor_block(head_ptr, offset).await?; + + match head_ancestor_opt { + None => { + // Block is missing in the block store. + // This generally won't happen often, but can happen if the head ptr has + // been updated since we retrieved the head ptr, and the block store has + // been garbage collected. + // It's easiest to start over at this point. + Ok(ReconciliationStep::Retry) + } + Some(head_ancestor) => { + // We stopped one block short, so we'll compare the parent hash to the + // subgraph ptr. + if head_ancestor.parent_hash().as_ref() == Some(&subgraph_ptr.hash) { + // The subgraph ptr is an ancestor of the head block. + // We cannot use an RPC call here to find the first interesting block + // due to the race conditions previously mentioned, + // so instead we will advance the subgraph ptr by one block. + // Note that head_ancestor is a child of subgraph_ptr. + let block = self + .adapter + .triggers_in_block(&self.logger, head_ancestor, &self.filter) + .await?; + Ok(ReconciliationStep::ProcessDescendantBlocks(vec![block], 1)) + } else { + let parent = self.parent_ptr(&subgraph_ptr, "nonfinal").await?; + + // The subgraph ptr is not on the main chain. + // We will need to step back (possibly repeatedly) one block at a time + // until we are back on the main chain. + Ok(ReconciliationStep::Revert(parent)) + } + } + } + } + } + + async fn parent_ptr(&self, block_ptr: &BlockPtr, reason: &str) -> Result { + let ptr = + self.adapter.parent_ptr(block_ptr).await?.ok_or_else(|| { + anyhow!("Failed to get parent pointer for {block_ptr} ({reason})") + })?; + + Ok(ptr) + } +} + +impl BlockStream for PollingBlockStream {} + +impl Stream for PollingBlockStream { + type Item = Result, Error>; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let result = loop { + match &mut self.state { + BlockStreamState::BeginReconciliation => { + // Start the reconciliation process by asking for blocks + let ctx = self.ctx.clone(); + let fut = async move { ctx.next_blocks().await }; + self.state = BlockStreamState::Reconciliation(fut.boxed()); + } + + // Waiting for the reconciliation to complete or yield blocks + BlockStreamState::Reconciliation(next_blocks_future) => { + match next_blocks_future.poll_unpin(cx) { + Poll::Ready(Ok(next_block_step)) => match next_block_step { + NextBlocks::Blocks(next_blocks, block_range_size) => { + // We had only one error, so we infer that reducing the range size is + // what fixed it. Reduce the max range size to prevent future errors. + // See: 018c6df4-132f-4acc-8697-a2d64e83a9f0 + if self.consecutive_err_count == 1 { + // Reduce the max range size by 10%, but to no less than 10. + self.ctx.max_block_range_size = + (self.ctx.max_block_range_size * 9 / 10).max(10); + } + self.consecutive_err_count = 0; + + let total_triggers = + next_blocks.iter().map(|b| b.trigger_count()).sum::(); + self.ctx.previous_triggers_per_block = + total_triggers as f64 / block_range_size as f64; + self.ctx.previous_block_range_size = block_range_size; + if total_triggers > 0 { + debug!( + self.ctx.logger, + "Processing {} triggers", total_triggers + ); + } + + // Switch to yielding state until next_blocks is depleted + self.state = + BlockStreamState::YieldingBlocks(Box::new(next_blocks)); + + // Yield the first block in next_blocks + continue; + } + // Reconciliation completed. We're caught up to chain head. + NextBlocks::Done => { + // Reset error count + self.consecutive_err_count = 0; + + // Switch to idle + self.state = BlockStreamState::Idle; + + // Poll for chain head update + continue; + } + NextBlocks::Revert(parent_ptr) => { + self.ctx.current_block = Some(parent_ptr.clone()); + + self.state = BlockStreamState::BeginReconciliation; + break Poll::Ready(Some(Ok(BlockStreamEvent::Revert( + parent_ptr, + FirehoseCursor::None, + )))); + } + }, + Poll::Pending => break Poll::Pending, + Poll::Ready(Err(e)) => { + // Reset the block range size in an attempt to recover from the error. + // See also: 018c6df4-132f-4acc-8697-a2d64e83a9f0 + self.ctx.previous_triggers_per_block = + STARTING_PREVIOUS_TRIGGERS_PER_BLOCK; + self.consecutive_err_count += 1; + + // Pause before trying again + let secs = (5 * self.consecutive_err_count).max(120) as u64; + + self.state = BlockStreamState::RetryAfterDelay(Box::pin( + tokio::time::sleep(Duration::from_secs(secs)).map(Ok), + )); + + break Poll::Ready(Some(Err(e))); + } + } + } + + // Yielding blocks from reconciliation process + BlockStreamState::YieldingBlocks(ref mut next_blocks) => { + match next_blocks.pop_front() { + // Yield one block + Some(next_block) => { + self.ctx.current_block = Some(next_block.block.ptr()); + + break Poll::Ready(Some(Ok(BlockStreamEvent::ProcessBlock( + next_block, + FirehoseCursor::None, + )))); + } + + // Done yielding blocks + None => { + self.state = BlockStreamState::BeginReconciliation; + } + } + } + + // Pausing after an error, before looking for more blocks + BlockStreamState::RetryAfterDelay(ref mut delay) => match delay.as_mut().poll(cx) { + Poll::Ready(Ok(..)) | Poll::Ready(Err(_)) => { + self.state = BlockStreamState::BeginReconciliation; + } + + Poll::Pending => { + break Poll::Pending; + } + }, + + // Waiting for a chain head update + BlockStreamState::Idle => { + match Pin::new(self.chain_head_update_stream.as_mut()).poll_next(cx) { + // Chain head was updated + Poll::Ready(Some(())) => { + self.state = BlockStreamState::BeginReconciliation; + } + + // Chain head update stream ended + Poll::Ready(None) => { + // Should not happen + return Poll::Ready(Some(Err(anyhow::anyhow!( + "chain head update stream ended unexpectedly" + )))); + } + + Poll::Pending => break Poll::Pending, + } + } + } + }; + + result + } +} diff --git a/graph/src/blockchain/substreams_block_stream.rs b/graph/src/blockchain/substreams_block_stream.rs new file mode 100644 index 0000000..291e14b --- /dev/null +++ b/graph/src/blockchain/substreams_block_stream.rs @@ -0,0 +1,323 @@ +use super::block_stream::SubstreamsMapper; +use crate::blockchain::block_stream::{BlockStream, BlockStreamEvent}; +use crate::blockchain::Blockchain; +use crate::firehose::FirehoseEndpoint; +use crate::prelude::*; +use crate::substreams::response::Message; +use crate::substreams::ForkStep::{StepNew, StepUndo}; +use crate::substreams::{Modules, Request, Response}; +use crate::util::backoff::ExponentialBackoff; +use async_stream::try_stream; +use futures03::{Stream, StreamExt}; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; +use tonic::Status; + +struct SubstreamsBlockStreamMetrics { + deployment: DeploymentHash, + provider: String, + restarts: CounterVec, + connect_duration: GaugeVec, + time_between_responses: HistogramVec, + responses: CounterVec, +} + +impl SubstreamsBlockStreamMetrics { + pub fn new( + registry: Arc, + deployment: DeploymentHash, + provider: String, + ) -> Self { + Self { + deployment, + provider, + + restarts: registry + .global_counter_vec( + "deployment_substreams_blockstream_restarts", + "Counts the number of times a Substreams block stream is (re)started", + vec!["deployment", "provider", "success"].as_slice(), + ) + .unwrap(), + + connect_duration: registry + .global_gauge_vec( + "deployment_substreams_blockstream_connect_duration", + "Measures the time it takes to connect a Substreams block stream", + vec!["deployment", "provider"].as_slice(), + ) + .unwrap(), + + time_between_responses: registry + .global_histogram_vec( + "deployment_substreams_blockstream_time_between_responses", + "Measures the time between receiving and processing Substreams stream responses", + vec!["deployment", "provider"].as_slice(), + ) + .unwrap(), + + responses: registry + .global_counter_vec( + "deployment_substreams_blockstream_responses", + "Counts the number of responses received from a Substreams block stream", + vec!["deployment", "provider", "kind"].as_slice(), + ) + .unwrap(), + } + } + + fn observe_successful_connection(&self, time: &mut Instant) { + self.restarts + .with_label_values(&[&self.deployment, &self.provider, "true"]) + .inc(); + self.connect_duration + .with_label_values(&[&self.deployment, &self.provider]) + .set(time.elapsed().as_secs_f64()); + + // Reset last connection timestamp + *time = Instant::now(); + } + + fn observe_failed_connection(&self, time: &mut Instant) { + self.restarts + .with_label_values(&[&self.deployment, &self.provider, "false"]) + .inc(); + self.connect_duration + .with_label_values(&[&self.deployment, &self.provider]) + .set(time.elapsed().as_secs_f64()); + + // Reset last connection timestamp + *time = Instant::now(); + } + + fn observe_response(&self, kind: &str, time: &mut Instant) { + self.time_between_responses + .with_label_values(&[&self.deployment, &self.provider]) + .observe(time.elapsed().as_secs_f64()); + self.responses + .with_label_values(&[&self.deployment, &self.provider, kind]) + .inc(); + + // Reset last response timestamp + *time = Instant::now(); + } +} + +pub struct SubstreamsBlockStream { + //fixme: not sure if this is ok to be set as public, maybe + // we do not want to expose the stream to the caller + stream: Pin, Error>> + Send>>, +} + +impl SubstreamsBlockStream +where + C: Blockchain, +{ + pub fn new( + deployment: DeploymentHash, + endpoint: Arc, + subgraph_current_block: Option, + cursor: Option, + mapper: Arc, + modules: Option, + module_name: String, + start_blocks: Vec, + end_blocks: Vec, + logger: Logger, + registry: Arc, + ) -> Self + where + F: SubstreamsMapper + 'static, + { + let manifest_start_block_num = start_blocks.into_iter().min().unwrap_or(0); + + let manifest_end_block_num = end_blocks.into_iter().min().unwrap_or(0); + + let metrics = + SubstreamsBlockStreamMetrics::new(registry, deployment, endpoint.provider.clone()); + + SubstreamsBlockStream { + stream: Box::pin(stream_blocks( + endpoint, + cursor, + mapper, + modules, + module_name, + manifest_start_block_num, + manifest_end_block_num, + subgraph_current_block, + logger, + metrics, + )), + } + } +} + +fn stream_blocks>( + endpoint: Arc, + cursor: Option, + mapper: Arc, + modules: Option, + module_name: String, + manifest_start_block_num: BlockNumber, + manifest_end_block_num: BlockNumber, + _subgraph_current_block: Option, + logger: Logger, + metrics: SubstreamsBlockStreamMetrics, +) -> impl Stream, Error>> { + let mut latest_cursor = cursor.unwrap_or_else(|| "".to_string()); + + let start_block_num = manifest_start_block_num as i64; + let stop_block_num = manifest_end_block_num as u64; + + let request = Request { + start_block_num, + start_cursor: latest_cursor.clone(), + stop_block_num, + fork_steps: vec![StepNew as i32, StepUndo as i32], + irreversibility_condition: "".to_string(), + modules, + output_modules: vec![module_name], + ..Default::default() + }; + + // Back off exponentially whenever we encounter a connection error or a stream with bad data + let mut backoff = ExponentialBackoff::new(Duration::from_millis(500), Duration::from_secs(45)); + + // This attribute is needed because `try_stream!` seems to break detection of `skip_backoff` assignments + #[allow(unused_assignments)] + let mut skip_backoff = false; + + try_stream! { + loop { + info!( + &logger, + "Blockstreams disconnected, connecting"; + "endpoint_uri" => format_args!("{}", endpoint), + "start_block" => start_block_num, + "cursor" => &latest_cursor, + ); + + // We just reconnected, assume that we want to back off on errors + skip_backoff = false; + + let mut connect_start = Instant::now(); + let result = endpoint.clone().substreams(request.clone()).await; + + match result { + Ok(stream) => { + info!(&logger, "Blockstreams connected"); + + // Track the time it takes to set up the block stream + metrics.observe_successful_connection(&mut connect_start); + + let mut last_response_time = Instant::now(); + let mut expected_stream_end = false; + + for await response in stream{ + match process_substreams_response( + response, + mapper.as_ref(), + &logger, + ).await { + Ok(block_response) => { + match block_response { + None => {} + Some(BlockResponse::Proceed(event, cursor)) => { + // Reset backoff because we got a good value from the stream + backoff.reset(); + + metrics.observe_response("proceed", &mut last_response_time); + + yield event; + + latest_cursor = cursor; + } + } + }, + Err(err) => { + info!(&logger, "received err"); + // We have an open connection but there was an error processing the Firehose + // response. We will reconnect the stream after this; this is the case where + // we actually _want_ to back off in case we keep running into the same error. + // An example of this situation is if we get invalid block or transaction data + // that cannot be decoded properly. + + metrics.observe_response("error", &mut last_response_time); + + error!(logger, "{:#}", err); + expected_stream_end = true; + break; + } + } + } + + if !expected_stream_end { + error!(logger, "Stream blocks complete unexpectedly, expecting stream to always stream blocks"); + } + }, + Err(e) => { + // We failed to connect and will try again; this is another + // case where we actually _want_ to back off in case we keep + // having connection errors. + + metrics.observe_failed_connection(&mut connect_start); + + error!(logger, "Unable to connect to endpoint: {:?}", e); + } + } + + // If we reach this point, we must wait a bit before retrying, unless `skip_backoff` is true + if !skip_backoff { + backoff.sleep_async().await; + } + } + } +} + +enum BlockResponse { + Proceed(BlockStreamEvent, String), +} + +async fn process_substreams_response>( + result: Result, + mapper: &F, + logger: &Logger, +) -> Result>, Error> { + let response = match result { + Ok(v) => v, + Err(e) => return Err(anyhow!("An error occurred while streaming blocks: {:?}", e)), + }; + + match response.message { + Some(Message::Data(block_scoped_data)) => { + match mapper + .to_block_stream_event(logger, &block_scoped_data) + .await + .context("Mapping block to BlockStreamEvent failed")? + { + Some(event) => Ok(Some(BlockResponse::Proceed( + event, + block_scoped_data.cursor.to_string(), + ))), + None => Ok(None), + } + } + None => { + warn!(&logger, "Got None on substream message"); + Ok(None) + } + _ => Ok(None), + } +} + +impl Stream for SubstreamsBlockStream { + type Item = Result, Error>; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.stream.poll_next_unpin(cx) + } +} + +impl BlockStream for SubstreamsBlockStream {} diff --git a/graph/src/blockchain/types.rs b/graph/src/blockchain/types.rs new file mode 100644 index 0000000..de7bbce --- /dev/null +++ b/graph/src/blockchain/types.rs @@ -0,0 +1,282 @@ +use anyhow::anyhow; +use std::convert::TryFrom; +use std::{fmt, str::FromStr}; +use web3::types::{Block, H256}; + +use crate::data::graphql::IntoValue; +use crate::object; +use crate::prelude::{r, BigInt, TryFromValue, ValueMap}; +use crate::util::stable_hash_glue::{impl_stable_hash, AsBytes}; +use crate::{cheap_clone::CheapClone, components::store::BlockNumber}; + +/// A simple marker for byte arrays that are really block hashes +#[derive(Clone, Default, PartialEq, Eq, Hash)] +pub struct BlockHash(pub Box<[u8]>); + +impl_stable_hash!(BlockHash(transparent: AsBytes)); + +impl BlockHash { + pub fn as_slice(&self) -> &[u8] { + &self.0 + } + + /// Encodes the block hash into a hexadecimal string **without** a "0x" + /// prefix. Hashes are stored in the database in this format when the + /// schema uses `text` columns, which is a legacy and such columns + /// should be changed to use `bytea` + pub fn hash_hex(&self) -> String { + hex::encode(&self.0) + } + + pub fn zero() -> Self { + Self::from(H256::zero()) + } +} + +impl fmt::Display for BlockHash { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "0x{}", hex::encode(&self.0)) + } +} + +impl fmt::Debug for BlockHash { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "0x{}", hex::encode(&self.0)) + } +} + +impl fmt::LowerHex for BlockHash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&hex::encode(&self.0)) + } +} + +impl CheapClone for BlockHash {} + +impl From for BlockHash { + fn from(hash: H256) -> Self { + BlockHash(hash.as_bytes().into()) + } +} + +impl From> for BlockHash { + fn from(bytes: Vec) -> Self { + BlockHash(bytes.as_slice().into()) + } +} + +impl TryFrom<&str> for BlockHash { + type Error = anyhow::Error; + + fn try_from(hash: &str) -> Result { + let hash = hash.trim_start_matches("0x"); + let hash = hex::decode(hash)?; + + Ok(BlockHash(hash.as_slice().into())) + } +} + +impl FromStr for BlockHash { + type Err = anyhow::Error; + + fn from_str(hash: &str) -> Result { + Self::try_from(hash) + } +} + +/// A block hash and block number from a specific Ethereum block. +/// +/// Block numbers are signed 32 bit integers +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct BlockPtr { + pub hash: BlockHash, + pub number: BlockNumber, +} + +impl CheapClone for BlockPtr {} + +impl_stable_hash!(BlockPtr { hash, number }); + +impl BlockPtr { + pub fn new(hash: BlockHash, number: BlockNumber) -> Self { + Self { hash, number } + } + + /// Encodes the block hash into a hexadecimal string **without** a "0x" prefix. + /// Hashes are stored in the database in this format. + pub fn hash_hex(&self) -> String { + self.hash.hash_hex() + } + + /// Block number to be passed into the store. Panics if it does not fit in an i32. + pub fn block_number(&self) -> BlockNumber { + self.number + } + + // FIXME: + // + // workaround for arweave + pub fn hash_as_h256(&self) -> H256 { + H256::from_slice(&self.hash_slice()[..32]) + } + + pub fn hash_slice(&self) -> &[u8] { + self.hash.0.as_ref() + } +} + +impl fmt::Display for BlockPtr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "#{} ({})", self.number, self.hash_hex()) + } +} + +impl fmt::Debug for BlockPtr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "#{} ({})", self.number, self.hash_hex()) + } +} + +impl slog::Value for BlockPtr { + fn serialize( + &self, + record: &slog::Record, + key: slog::Key, + serializer: &mut dyn slog::Serializer, + ) -> slog::Result { + slog::Value::serialize(&self.to_string(), record, key, serializer) + } +} + +impl From> for BlockPtr { + fn from(b: Block) -> BlockPtr { + BlockPtr::from((b.hash.unwrap(), b.number.unwrap().as_u64())) + } +} + +impl<'a, T> From<&'a Block> for BlockPtr { + fn from(b: &'a Block) -> BlockPtr { + BlockPtr::from((b.hash.unwrap(), b.number.unwrap().as_u64())) + } +} + +impl From<(Vec, i32)> for BlockPtr { + fn from((bytes, number): (Vec, i32)) -> Self { + BlockPtr { + hash: BlockHash::from(bytes), + number, + } + } +} + +impl From<(H256, i32)> for BlockPtr { + fn from((hash, number): (H256, i32)) -> BlockPtr { + BlockPtr { + hash: hash.into(), + number, + } + } +} + +impl From<(Vec, u64)> for BlockPtr { + fn from((bytes, number): (Vec, u64)) -> Self { + let number = i32::try_from(number).unwrap(); + BlockPtr { + hash: BlockHash::from(bytes), + number, + } + } +} + +impl From<(H256, u64)> for BlockPtr { + fn from((hash, number): (H256, u64)) -> BlockPtr { + let number = i32::try_from(number).unwrap(); + + BlockPtr::from((hash, number)) + } +} + +impl From<(H256, i64)> for BlockPtr { + fn from((hash, number): (H256, i64)) -> BlockPtr { + if number < 0 { + panic!("block number out of range: {}", number); + } + + BlockPtr::from((hash, number as u64)) + } +} + +impl TryFrom<(&str, i64)> for BlockPtr { + type Error = anyhow::Error; + + fn try_from((hash, number): (&str, i64)) -> Result { + let hash = hash.trim_start_matches("0x"); + let hash = BlockHash::from_str(hash)?; + + Ok(BlockPtr::new(hash, number as i32)) + } +} + +impl TryFrom<(&[u8], i64)> for BlockPtr { + type Error = anyhow::Error; + + fn try_from((bytes, number): (&[u8], i64)) -> Result { + let hash = if bytes.len() == H256::len_bytes() { + H256::from_slice(bytes) + } else { + return Err(anyhow!( + "invalid H256 value `{}` has {} bytes instead of {}", + hex::encode(bytes), + bytes.len(), + H256::len_bytes() + )); + }; + Ok(BlockPtr::from((hash, number))) + } +} + +impl TryFromValue for BlockPtr { + fn try_from_value(value: &r::Value) -> Result { + match value { + r::Value::Object(o) => { + let number = o.get_required::("number")?.to_u64() as BlockNumber; + let hash = o.get_required::("hash")?; + + Ok(BlockPtr::new(hash, number)) + } + _ => Err(anyhow!( + "failed to parse non-object value into BlockPtr: {:?}", + value + )), + } + } +} + +impl IntoValue for BlockPtr { + fn into_value(self) -> r::Value { + object! { + __typename: "Block", + hash: self.hash_hex(), + number: format!("{}", self.number), + } + } +} + +impl From for H256 { + fn from(ptr: BlockPtr) -> Self { + ptr.hash_as_h256() + } +} + +impl From for BlockNumber { + fn from(ptr: BlockPtr) -> Self { + ptr.number + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +/// A collection of attributes that (kind of) uniquely identify a blockchain. +pub struct ChainIdentifier { + pub net_version: String, + pub genesis_block_hash: BlockHash, +} diff --git a/graph/src/cheap_clone.rs b/graph/src/cheap_clone.rs new file mode 100644 index 0000000..7deff5c --- /dev/null +++ b/graph/src/cheap_clone.rs @@ -0,0 +1,38 @@ +use slog::Logger; +use std::future::Future; +use std::rc::Rc; +use std::sync::Arc; +use tonic::transport::Channel; + +/// Things that are fast to clone in the context of an application such as Graph Node +/// +/// The purpose of this API is to reduce the number of calls to .clone() which need to +/// be audited for performance. +/// +/// As a rule of thumb, only constant-time Clone impls should also implement CheapClone. +/// Eg: +/// ✔ Arc +/// ✗ Vec +/// ✔ u128 +/// ✗ String +pub trait CheapClone: Clone { + #[inline] + fn cheap_clone(&self) -> Self { + self.clone() + } +} + +impl CheapClone for Rc {} +impl CheapClone for Arc {} +impl CheapClone for Box {} +impl CheapClone for std::pin::Pin {} +impl CheapClone for Option {} +impl CheapClone for Logger {} + +// Pool is implemented as a newtype over Arc, +// So it is CheapClone. +impl CheapClone for diesel::r2d2::Pool {} + +impl CheapClone for futures03::future::Shared {} + +impl CheapClone for Channel {} diff --git a/graph/src/components/ethereum/mod.rs b/graph/src/components/ethereum/mod.rs new file mode 100644 index 0000000..45f1f5d --- /dev/null +++ b/graph/src/components/ethereum/mod.rs @@ -0,0 +1,6 @@ +mod types; + +pub use self::types::{ + evaluate_transaction_status, EthereumBlock, EthereumBlockWithCalls, EthereumCall, + LightEthereumBlock, LightEthereumBlockExt, +}; diff --git a/graph/src/components/ethereum/types.rs b/graph/src/components/ethereum/types.rs new file mode 100644 index 0000000..ca823aa --- /dev/null +++ b/graph/src/components/ethereum/types.rs @@ -0,0 +1,172 @@ +use serde::{Deserialize, Serialize}; +use std::{convert::TryFrom, sync::Arc}; +use web3::types::{ + Action, Address, Block, Bytes, Log, Res, Trace, Transaction, TransactionReceipt, H256, U256, + U64, +}; + +use crate::{blockchain::BlockPtr, prelude::BlockNumber}; + +pub type LightEthereumBlock = Block; + +pub trait LightEthereumBlockExt { + fn number(&self) -> BlockNumber; + fn transaction_for_log(&self, log: &Log) -> Option; + fn transaction_for_call(&self, call: &EthereumCall) -> Option; + fn parent_ptr(&self) -> Option; + fn format(&self) -> String; + fn block_ptr(&self) -> BlockPtr; +} + +impl LightEthereumBlockExt for LightEthereumBlock { + fn number(&self) -> BlockNumber { + BlockNumber::try_from(self.number.unwrap().as_u64()).unwrap() + } + + fn transaction_for_log(&self, log: &Log) -> Option { + log.transaction_hash + .and_then(|hash| self.transactions.iter().find(|tx| tx.hash == hash)) + .cloned() + } + + fn transaction_for_call(&self, call: &EthereumCall) -> Option { + call.transaction_hash + .and_then(|hash| self.transactions.iter().find(|tx| tx.hash == hash)) + .cloned() + } + + fn parent_ptr(&self) -> Option { + match self.number() { + 0 => None, + n => Some(BlockPtr::from((self.parent_hash, n - 1))), + } + } + + fn format(&self) -> String { + format!( + "{} ({})", + self.number + .map_or(String::from("none"), |number| format!("#{}", number)), + self.hash + .map_or(String::from("-"), |hash| format!("{:x}", hash)) + ) + } + + fn block_ptr(&self) -> BlockPtr { + BlockPtr::from((self.hash.unwrap(), self.number.unwrap().as_u64())) + } +} + +#[derive(Clone, Debug)] +pub struct EthereumBlockWithCalls { + pub ethereum_block: EthereumBlock, + /// The calls in this block; `None` means we haven't checked yet, + /// `Some(vec![])` means that we checked and there were none + pub calls: Option>, +} + +impl EthereumBlockWithCalls { + /// Given an `EthereumCall`, check within receipts if that transaction was successful. + pub fn transaction_for_call_succeeded(&self, call: &EthereumCall) -> anyhow::Result { + let call_transaction_hash = call.transaction_hash.ok_or(anyhow::anyhow!( + "failed to find a transaction for this call" + ))?; + + let receipt = self + .ethereum_block + .transaction_receipts + .iter() + .find(|txn| txn.transaction_hash == call_transaction_hash) + .ok_or(anyhow::anyhow!( + "failed to find the receipt for this transaction" + ))?; + + Ok(evaluate_transaction_status(receipt.status)) + } +} + +/// Evaluates if a given transaction was successful. +/// +/// Returns `true` on success and `false` on failure. +/// If a receipt does not have a status value (EIP-658), assume the transaction was successful. +pub fn evaluate_transaction_status(receipt_status: Option) -> bool { + receipt_status + .map(|status| !status.is_zero()) + .unwrap_or(true) +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] +pub struct EthereumBlock { + pub block: Arc, + pub transaction_receipts: Vec>, +} + +#[derive(Debug, Default, Clone, PartialEq)] +pub struct EthereumCall { + pub from: Address, + pub to: Address, + pub value: U256, + pub gas_used: U256, + pub input: Bytes, + pub output: Bytes, + pub block_number: BlockNumber, + pub block_hash: H256, + pub transaction_hash: Option, + pub transaction_index: u64, +} + +impl EthereumCall { + pub fn try_from_trace(trace: &Trace) -> Option { + // The parity-ethereum tracing api returns traces for operations which had execution errors. + // Filter errorful traces out, since call handlers should only run on successful CALLs. + if trace.error.is_some() { + return None; + } + // We are only interested in traces from CALLs + let call = match &trace.action { + // Contract to contract value transfers compile to the CALL opcode + // and have no input. Call handlers are for triggering on explicit method calls right now. + Action::Call(call) if call.input.0.len() >= 4 => call, + _ => return None, + }; + let (output, gas_used) = match &trace.result { + Some(Res::Call(result)) => (result.output.clone(), result.gas_used), + _ => return None, + }; + + // The only traces without transactions are those from Parity block reward contracts, we + // don't support triggering on that. + let transaction_index = trace.transaction_position? as u64; + + Some(EthereumCall { + from: call.from, + to: call.to, + value: call.value, + gas_used, + input: call.input.clone(), + output, + block_number: trace.block_number as BlockNumber, + block_hash: trace.block_hash, + transaction_hash: trace.transaction_hash, + transaction_index, + }) + } +} + +impl From for BlockPtr { + fn from(b: EthereumBlock) -> BlockPtr { + BlockPtr::from((b.block.hash.unwrap(), b.block.number.unwrap().as_u64())) + } +} + +impl<'a> From<&'a EthereumBlock> for BlockPtr { + fn from(b: &'a EthereumBlock) -> BlockPtr { + BlockPtr::from((b.block.hash.unwrap(), b.block.number.unwrap().as_u64())) + } +} + +impl<'a> From<&'a EthereumCall> for BlockPtr { + fn from(call: &'a EthereumCall) -> BlockPtr { + BlockPtr::from((call.block_hash, call.block_number)) + } +} diff --git a/graph/src/components/graphql.rs b/graph/src/components/graphql.rs new file mode 100644 index 0000000..78c5472 --- /dev/null +++ b/graph/src/components/graphql.rs @@ -0,0 +1,59 @@ +use futures::prelude::*; + +use crate::data::query::{CacheStatus, Query, QueryTarget}; +use crate::data::subscription::{Subscription, SubscriptionError, SubscriptionResult}; +use crate::data::{graphql::effort::LoadManager, query::QueryResults}; +use crate::prelude::DeploymentHash; + +use async_trait::async_trait; +use std::sync::Arc; +use std::time::Duration; + +/// Future for subscription results. +pub type SubscriptionResultFuture = + Box + Send>; + +pub enum GraphQlTarget { + SubgraphName(String), + Deployment(DeploymentHash), +} + +/// A component that can run GraphqL queries against a [Store](../store/trait.Store.html). +#[async_trait] +pub trait GraphQlRunner: Send + Sync + 'static { + /// Runs a GraphQL query and returns its result. + async fn run_query(self: Arc, query: Query, target: QueryTarget) -> QueryResults; + + /// Runs a GraphqL query up to the given complexity. Overrides the global complexity limit. + async fn run_query_with_complexity( + self: Arc, + query: Query, + target: QueryTarget, + max_complexity: Option, + max_depth: Option, + max_first: Option, + max_skip: Option, + ) -> QueryResults; + + /// Runs a GraphQL subscription and returns a stream of results. + async fn run_subscription( + self: Arc, + subscription: Subscription, + target: QueryTarget, + ) -> Result; + + fn load_manager(&self) -> Arc; + + fn metrics(&self) -> Arc; +} + +pub trait GraphQLMetrics: Send + Sync + 'static { + fn observe_query_execution(&self, duration: Duration, results: &QueryResults); + fn observe_query_parsing(&self, duration: Duration, results: &QueryResults); + fn observe_query_validation(&self, duration: Duration, id: &DeploymentHash); +} + +#[async_trait] +pub trait QueryLoadManager: Send + Sync { + fn record_work(&self, shape_hash: u64, duration: Duration, cache_status: CacheStatus); +} diff --git a/graph/src/components/link_resolver.rs b/graph/src/components/link_resolver.rs new file mode 100644 index 0000000..42f6a26 --- /dev/null +++ b/graph/src/components/link_resolver.rs @@ -0,0 +1,44 @@ +use std::pin::Pin; +use std::time::Duration; + +use async_trait::async_trait; +use futures03::prelude::Stream; +use serde_json::Value; +use slog::Logger; + +use crate::data::subgraph::Link; +use crate::prelude::Error; +use std::fmt::Debug; + +/// The values that `json_stream` returns. The struct contains the deserialized +/// JSON value from the input stream, together with the line number from which +/// the value was read. +pub struct JsonStreamValue { + pub value: Value, + pub line: usize, +} + +pub type JsonValueStream = + Pin> + Send + 'static>>; + +/// Resolves links to subgraph manifests and resources referenced by them. +#[async_trait] +pub trait LinkResolver: Send + Sync + 'static + Debug { + /// Updates the timeout used by the resolver. + fn with_timeout(&self, timeout: Duration) -> Box; + + /// Enables infinite retries. + fn with_retries(&self) -> Box; + + /// Fetches the link contents as bytes. + async fn cat(&self, logger: &Logger, link: &Link) -> Result, Error>; + + /// Fetches the IPLD block contents as bytes. + async fn get_block(&self, logger: &Logger, link: &Link) -> Result, Error>; + + /// Read the contents of `link` and deserialize them into a stream of JSON + /// values. The values must each be on a single line; newlines are significant + /// as they are used to split the file contents and each line is deserialized + /// separately. + async fn json_stream(&self, logger: &Logger, link: &Link) -> Result; +} diff --git a/graph/src/components/metrics/aggregate.rs b/graph/src/components/metrics/aggregate.rs new file mode 100644 index 0000000..a8f0822 --- /dev/null +++ b/graph/src/components/metrics/aggregate.rs @@ -0,0 +1,63 @@ +use std::time::Duration; + +use crate::prelude::*; + +pub struct Aggregate { + /// Number of values. + count: Gauge, + + /// Sum over all values. + sum: Gauge, + + /// Moving average over the values. + avg: Gauge, + + /// Latest value. + cur: Gauge, +} + +impl Aggregate { + pub fn new(name: &str, subgraph: &str, help: &str, registry: Arc) -> Self { + let make_gauge = |suffix: &str| { + registry + .new_deployment_gauge( + &format!("{}_{}", name, suffix), + &format!("{} ({})", help, suffix), + subgraph, + ) + .unwrap_or_else(|_| { + panic!( + "failed to register metric `{}_{}` for {}", + name, suffix, subgraph + ) + }) + }; + + Aggregate { + count: make_gauge("count"), + sum: make_gauge("sum"), + avg: make_gauge("avg"), + cur: make_gauge("cur"), + } + } + + pub fn update(&self, x: f64) { + // Update count + self.count.inc(); + let n = self.count.get(); + + // Update sum + self.sum.add(x); + + // Update current value + self.cur.set(x); + + // Update aggregate value. + let avg = self.avg.get(); + self.avg.set(avg + (x - avg) / n); + } + + pub fn update_duration(&self, x: Duration) { + self.update(x.as_secs_f64()) + } +} diff --git a/graph/src/components/metrics/mod.rs b/graph/src/components/metrics/mod.rs new file mode 100644 index 0000000..581cea6 --- /dev/null +++ b/graph/src/components/metrics/mod.rs @@ -0,0 +1,301 @@ +pub use prometheus::core::Collector; +pub use prometheus::{ + labels, Counter, CounterVec, Error as PrometheusError, Gauge, GaugeVec, Histogram, + HistogramOpts, HistogramVec, Opts, Registry, +}; +pub mod subgraph; + +use std::collections::HashMap; + +/// Metrics for measuring where time is spent during indexing. +pub mod stopwatch; + +/// Aggregates over individual values. +pub mod aggregate; + +fn deployment_labels(subgraph: &str) -> HashMap { + labels! { String::from("deployment") => String::from(subgraph), } +} + +/// Create an unregistered counter with labels +pub fn counter_with_labels( + name: &str, + help: &str, + const_labels: HashMap, +) -> Result { + let opts = Opts::new(name, help).const_labels(const_labels); + Counter::with_opts(opts) +} + +/// Create an unregistered gauge with labels +pub fn gauge_with_labels( + name: &str, + help: &str, + const_labels: HashMap, +) -> Result { + let opts = Opts::new(name, help).const_labels(const_labels); + Gauge::with_opts(opts) +} + +pub trait MetricsRegistry: Send + Sync + 'static { + fn register(&self, name: &str, c: Box); + + fn unregister(&self, metric: Box); + + fn global_counter( + &self, + name: &str, + help: &str, + const_labels: HashMap, + ) -> Result; + + fn global_counter_vec( + &self, + name: &str, + help: &str, + variable_labels: &[&str], + ) -> Result; + + fn global_deployment_counter( + &self, + name: &str, + help: &str, + subgraph: &str, + ) -> Result { + self.global_counter(name, help, deployment_labels(subgraph)) + } + + fn global_deployment_counter_vec( + &self, + name: &str, + help: &str, + subgraph: &str, + variable_labels: &[&str], + ) -> Result; + + fn global_gauge( + &self, + name: &str, + help: &str, + const_labels: HashMap, + ) -> Result; + + fn global_gauge_vec( + &self, + name: &str, + help: &str, + variable_labels: &[&str], + ) -> Result; + + fn new_gauge( + &self, + name: &str, + help: &str, + const_labels: HashMap, + ) -> Result, PrometheusError> { + let opts = Opts::new(name, help).const_labels(const_labels); + let gauge = Box::new(Gauge::with_opts(opts)?); + self.register(name, gauge.clone()); + Ok(gauge) + } + + fn new_deployment_gauge( + &self, + name: &str, + help: &str, + subgraph: &str, + ) -> Result { + let opts = Opts::new(name, help).const_labels(deployment_labels(subgraph)); + let gauge = Gauge::with_opts(opts)?; + self.register(name, Box::new(gauge.clone())); + Ok(gauge) + } + + fn new_gauge_vec( + &self, + name: &str, + help: &str, + variable_labels: Vec, + ) -> Result, PrometheusError> { + let opts = Opts::new(name, help); + let gauges = Box::new(GaugeVec::new( + opts, + variable_labels + .iter() + .map(String::as_str) + .collect::>() + .as_slice(), + )?); + self.register(name, gauges.clone()); + Ok(gauges) + } + + fn new_deployment_gauge_vec( + &self, + name: &str, + help: &str, + subgraph: &str, + variable_labels: Vec, + ) -> Result, PrometheusError> { + let opts = Opts::new(name, help).const_labels(deployment_labels(subgraph)); + let gauges = Box::new(GaugeVec::new( + opts, + variable_labels + .iter() + .map(String::as_str) + .collect::>() + .as_slice(), + )?); + self.register(name, gauges.clone()); + Ok(gauges) + } + + fn new_counter(&self, name: &str, help: &str) -> Result, PrometheusError> { + let opts = Opts::new(name, help); + let counter = Box::new(Counter::with_opts(opts)?); + self.register(name, counter.clone()); + Ok(counter) + } + + fn new_counter_with_labels( + &self, + name: &str, + help: &str, + const_labels: HashMap, + ) -> Result, PrometheusError> { + let counter = Box::new(counter_with_labels(name, help, const_labels)?); + self.register(name, counter.clone()); + Ok(counter) + } + + fn new_deployment_counter( + &self, + name: &str, + help: &str, + subgraph: &str, + ) -> Result { + let counter = counter_with_labels(name, help, deployment_labels(subgraph))?; + self.register(name, Box::new(counter.clone())); + Ok(counter) + } + + fn new_counter_vec( + &self, + name: &str, + help: &str, + variable_labels: Vec, + ) -> Result, PrometheusError> { + let opts = Opts::new(name, help); + let counters = Box::new(CounterVec::new( + opts, + variable_labels + .iter() + .map(String::as_str) + .collect::>() + .as_slice(), + )?); + self.register(name, counters.clone()); + Ok(counters) + } + + fn new_deployment_counter_vec( + &self, + name: &str, + help: &str, + subgraph: &str, + variable_labels: Vec, + ) -> Result, PrometheusError> { + let opts = Opts::new(name, help).const_labels(deployment_labels(subgraph)); + let counters = Box::new(CounterVec::new( + opts, + variable_labels + .iter() + .map(String::as_str) + .collect::>() + .as_slice(), + )?); + self.register(name, counters.clone()); + Ok(counters) + } + + fn new_deployment_histogram( + &self, + name: &str, + help: &str, + subgraph: &str, + buckets: Vec, + ) -> Result, PrometheusError> { + let opts = HistogramOpts::new(name, help) + .const_labels(deployment_labels(subgraph)) + .buckets(buckets); + let histogram = Box::new(Histogram::with_opts(opts)?); + self.register(name, histogram.clone()); + Ok(histogram) + } + + fn new_histogram( + &self, + name: &str, + help: &str, + buckets: Vec, + ) -> Result, PrometheusError> { + let opts = HistogramOpts::new(name, help).buckets(buckets); + let histogram = Box::new(Histogram::with_opts(opts)?); + self.register(name, histogram.clone()); + Ok(histogram) + } + + fn new_histogram_vec( + &self, + name: &str, + help: &str, + variable_labels: Vec, + buckets: Vec, + ) -> Result, PrometheusError> { + let opts = Opts::new(name, help); + let histograms = Box::new(HistogramVec::new( + HistogramOpts { + common_opts: opts, + buckets, + }, + variable_labels + .iter() + .map(String::as_str) + .collect::>() + .as_slice(), + )?); + self.register(name, histograms.clone()); + Ok(histograms) + } + + fn new_deployment_histogram_vec( + &self, + name: &str, + help: &str, + subgraph: &str, + variable_labels: Vec, + buckets: Vec, + ) -> Result, PrometheusError> { + let opts = Opts::new(name, help).const_labels(deployment_labels(subgraph)); + let histograms = Box::new(HistogramVec::new( + HistogramOpts { + common_opts: opts, + buckets, + }, + variable_labels + .iter() + .map(String::as_str) + .collect::>() + .as_slice(), + )?); + self.register(name, histograms.clone()); + Ok(histograms) + } + + fn global_histogram_vec( + &self, + name: &str, + help: &str, + variable_labels: &[&str], + ) -> Result; +} diff --git a/graph/src/components/metrics/stopwatch.rs b/graph/src/components/metrics/stopwatch.rs new file mode 100644 index 0000000..fe56cdb --- /dev/null +++ b/graph/src/components/metrics/stopwatch.rs @@ -0,0 +1,163 @@ +use crate::prelude::*; +use std::sync::{atomic::AtomicBool, atomic::Ordering, Mutex}; +use std::time::Instant; + +/// This is a "section guard", that closes the section on drop. +pub struct Section { + id: String, + stopwatch: StopwatchMetrics, +} + +impl Section { + /// A more readable `drop`. + pub fn end(self) {} +} + +impl Drop for Section { + fn drop(&mut self) { + self.stopwatch.end_section(std::mem::take(&mut self.id)) + } +} + +/// Usage example: +/// ```ignore +/// // Start counting time for the "main_section". +/// let _main_section = stopwatch.start_section("main_section"); +/// // do stuff... +/// // Pause timer for "main_section", start for "child_section". +/// let child_section = stopwatch.start_section("child_section"); +/// // do stuff... +/// // Register time spent in "child_section", implicitly going back to "main_section". +/// section.end(); +/// // do stuff... +/// // At the end of the scope `_main_section` is dropped, which is equivalent to calling +/// // `_main_section.end()`. +#[derive(Clone)] +pub struct StopwatchMetrics { + disabled: Arc, + inner: Arc>, +} + +impl CheapClone for StopwatchMetrics {} + +impl StopwatchMetrics { + pub fn new( + logger: Logger, + subgraph_id: DeploymentHash, + stage: &str, + registry: Arc, + ) -> Self { + let stage = stage.to_owned(); + let mut inner = StopwatchInner { + counter: registry + .global_deployment_counter_vec( + "deployment_sync_secs", + "total time spent syncing", + subgraph_id.as_str(), + &["section", "stage"], + ) + .unwrap_or_else(|_| { + panic!( + "failed to register subgraph_sync_total_secs prometheus counter for {}", + subgraph_id + ) + }), + logger, + stage, + section_stack: Vec::new(), + timer: Instant::now(), + }; + + // Start a base section so that all time is accounted for. + inner.start_section("unknown".to_owned()); + + StopwatchMetrics { + disabled: Arc::new(AtomicBool::new(false)), + inner: Arc::new(Mutex::new(inner)), + } + } + + pub fn start_section(&self, id: &str) -> Section { + let id = id.to_owned(); + if !self.disabled.load(Ordering::SeqCst) { + self.inner.lock().unwrap().start_section(id.clone()) + } + + // If disabled, this will do nothing on drop. + Section { + id, + stopwatch: self.clone(), + } + } + + /// Turns `start_section` and `end_section` into no-ops, no more metrics will be updated. + pub fn disable(&self) { + self.disabled.store(true, Ordering::SeqCst) + } + + fn end_section(&self, id: String) { + if !self.disabled.load(Ordering::SeqCst) { + self.inner.lock().unwrap().end_section(id) + } + } +} + +/// We want to account for all subgraph indexing time, based on "wall clock" time. To do this we +/// break down indexing into _sequential_ sections, and register the total time spent in each. So +/// that there is no double counting, time spent in child sections doesn't count for the parent. +struct StopwatchInner { + logger: Logger, + + // Counter for the total time the subgraph spent syncing in various sections. + counter: CounterVec, + + // The top section (last item) is the one that's currently executing. + section_stack: Vec, + + // The timer is reset whenever a section starts or ends. + timer: Instant, + + // The processing stage the metrics belong to; for pipelined uses, the + // pipeline stage + stage: String, +} + +impl StopwatchInner { + fn record_and_reset(&mut self) { + if let Some(section) = self.section_stack.last() { + // Register the current timer. + let elapsed = self.timer.elapsed().as_secs_f64(); + self.counter + .get_metric_with_label_values(&[section, &self.stage]) + .map(|counter| counter.inc_by(elapsed)) + .unwrap_or_else(|e| { + error!(self.logger, "failed to find counter for section"; + "id" => section, + "error" => e.to_string()); + }); + } + + // Reset the timer. + self.timer = Instant::now(); + } + + fn start_section(&mut self, id: String) { + self.record_and_reset(); + self.section_stack.push(id); + } + + fn end_section(&mut self, id: String) { + // Validate that the expected section is running. + match self.section_stack.last() { + Some(current_section) if current_section == &id => { + self.record_and_reset(); + self.section_stack.pop(); + } + Some(current_section) => error!(self.logger, "`end_section` with mismatched section"; + "current" => current_section, + "received" => id), + None => error!(self.logger, "`end_section` with no current section"; + "received" => id), + } + } +} diff --git a/graph/src/components/metrics/subgraph.rs b/graph/src/components/metrics/subgraph.rs new file mode 100644 index 0000000..5fff2f3 --- /dev/null +++ b/graph/src/components/metrics/subgraph.rs @@ -0,0 +1,101 @@ +use crate::blockchain::block_stream::BlockStreamMetrics; +use crate::prelude::{Gauge, Histogram, HostMetrics, MetricsRegistry}; +use std::collections::HashMap; +use std::sync::Arc; + +use super::stopwatch::StopwatchMetrics; + +pub struct SubgraphInstanceMetrics { + pub block_trigger_count: Box, + pub block_processing_duration: Box, + pub block_ops_transaction_duration: Box, + + pub stopwatch: StopwatchMetrics, + trigger_processing_duration: Box, +} + +impl SubgraphInstanceMetrics { + pub fn new( + registry: Arc, + subgraph_hash: &str, + stopwatch: StopwatchMetrics, + ) -> Self { + let block_trigger_count = registry + .new_deployment_histogram( + "deployment_block_trigger_count", + "Measures the number of triggers in each block for a subgraph deployment", + subgraph_hash, + vec![1.0, 5.0, 10.0, 20.0, 50.0], + ) + .expect("failed to create `deployment_block_trigger_count` histogram"); + let trigger_processing_duration = registry + .new_deployment_histogram( + "deployment_trigger_processing_duration", + "Measures duration of trigger processing for a subgraph deployment", + subgraph_hash, + vec![0.01, 0.05, 0.1, 0.5, 1.5, 5.0, 10.0, 30.0, 120.0], + ) + .expect("failed to create `deployment_trigger_processing_duration` histogram"); + let block_processing_duration = registry + .new_deployment_histogram( + "deployment_block_processing_duration", + "Measures duration of block processing for a subgraph deployment", + subgraph_hash, + vec![0.05, 0.2, 0.7, 1.5, 4.0, 10.0, 60.0, 120.0, 240.0], + ) + .expect("failed to create `deployment_block_processing_duration` histogram"); + let block_ops_transaction_duration = registry + .new_deployment_histogram( + "deployment_transact_block_operations_duration", + "Measures duration of commiting all the entity operations in a block and updating the subgraph pointer", + subgraph_hash, + vec![0.01, 0.05, 0.1, 0.3, 0.7, 2.0], + ) + .expect("failed to create `deployment_transact_block_operations_duration_{}"); + + Self { + block_trigger_count, + block_processing_duration, + trigger_processing_duration, + block_ops_transaction_duration, + stopwatch, + } + } + + pub fn observe_trigger_processing_duration(&self, duration: f64) { + self.trigger_processing_duration.observe(duration); + } + + pub fn unregister(&self, registry: Arc) { + registry.unregister(self.block_processing_duration.clone()); + registry.unregister(self.block_trigger_count.clone()); + registry.unregister(self.trigger_processing_duration.clone()); + registry.unregister(self.block_ops_transaction_duration.clone()); + } +} + +pub struct SubgraphInstanceManagerMetrics { + pub subgraph_count: Box, +} + +impl SubgraphInstanceManagerMetrics { + pub fn new(registry: Arc) -> Self { + let subgraph_count = registry + .new_gauge( + "deployment_count", + "Counts the number of deployments currently being indexed by the graph-node.", + HashMap::new(), + ) + .expect("failed to create `deployment_count` gauge"); + Self { subgraph_count } + } +} + +pub struct RunnerMetrics { + /// Sensors to measure the execution of the subgraph instance + pub subgraph: Arc, + /// Sensors to measure the execution of the subgraph's runtime hosts + pub host: Arc, + /// Sensors to measure the BlockStream metrics + pub stream: Arc, +} diff --git a/graph/src/components/mod.rs b/graph/src/components/mod.rs new file mode 100644 index 0000000..79d698f --- /dev/null +++ b/graph/src/components/mod.rs @@ -0,0 +1,81 @@ +//! The Graph network nodes are internally structured as a layers of reusable +//! components with non-blocking communication, with each component having a +//! corresponding trait defining it's interface. +//! +//! As examples of components, at the top layer component there is the GraphQL +//! server which interacts with clients, and at the lowest layer we have data +//! sources that interact with storage backends. +//! +//! The layers are not well defined, but it's expected that a higher-level +//! component will make requests for a lower-level component to respond, and +//! that a lower-level component will send events to interested higher-level +//! components when it's state changes. +//! +//! A request/response interaction between C1 and C2 is made by C1 requiring an +//! `Arc` in it's constructor and then calling the functions defined on C2. +//! +//! Event-based interactions propagate changes in the underlining data upwards +//! in the component graph, with low level components generating event streams +//! based on changes in external systems, mid level components transforming +//! these streams and high level components finally consuming the received +//! events. +//! +//! These events are communicated through sinks and streams (typically senders +//! and receivers of channels), which are managed by long-running Tokio tasks. +//! Each component may have an internal task for handling input events and +//! sending out output events, and the "dumb pipes" that plug together components +//! are tasks that send out events in the order that they are received. +//! +//! A component declares it's inputs and outputs by having `EventConsumer` and +//! `EventProducer` traits as supertraits. +//! +//! Components should use the helper functions in this module (e.g. `forward`) +//! that define common operations on event streams, facilitating the +//! configuration of component graphs. + +use futures::prelude::*; + +/// Components dealing with subgraphs. +pub mod subgraph; + +/// Components dealing with Ethereum. +pub mod ethereum; + +/// Components dealing with processing GraphQL. +pub mod graphql; + +/// Components powering GraphQL, JSON-RPC, WebSocket APIs, Metrics. +pub mod server; + +/// Components dealing with storing entities. +pub mod store; + +pub mod link_resolver; + +pub mod trigger_processor; + +/// Components dealing with collecting metrics +pub mod metrics; + +/// Components dealing with versioning +pub mod versions; + +/// A component that receives events of type `T`. +pub trait EventConsumer { + /// Get the event sink. + /// + /// Avoid calling directly, prefer helpers such as `forward`. + fn event_sink(&self) -> Box + Send>; +} + +/// A component that outputs events of type `T`. +pub trait EventProducer { + /// Get the event stream. Because we use single-consumer semantics, the + /// first caller will take the output stream and any further calls will + /// return `None`. + /// + /// Avoid calling directly, prefer helpers such as `forward`. + fn take_event_stream(&mut self) -> Option + Send>>; +} + +pub mod transaction_receipt; diff --git a/graph/src/components/server/index_node.rs b/graph/src/components/server/index_node.rs new file mode 100644 index 0000000..eddf1fa --- /dev/null +++ b/graph/src/components/server/index_node.rs @@ -0,0 +1,31 @@ +use std::sync::Arc; + +use futures::prelude::*; + +use crate::prelude::{BlockNumber, Schema}; + +/// This is only needed to support the explorer API. +#[derive(Debug)] +pub struct VersionInfo { + pub created_at: String, + pub deployment_id: String, + pub latest_ethereum_block_number: Option, + pub total_ethereum_blocks_count: Option, + pub synced: bool, + pub failed: bool, + pub description: Option, + pub repository: Option, + pub schema: Arc, + pub network: String, +} + +/// Common trait for index node server implementations. +pub trait IndexNodeServer { + type ServeError; + + /// Creates a new Tokio task that, when spawned, brings up the index node server. + fn serve( + &mut self, + port: u16, + ) -> Result + Send>, Self::ServeError>; +} diff --git a/graph/src/components/server/metrics.rs b/graph/src/components/server/metrics.rs new file mode 100644 index 0000000..1bd9f4e --- /dev/null +++ b/graph/src/components/server/metrics.rs @@ -0,0 +1,12 @@ +use futures::prelude::*; + +/// Common trait for index node server implementations. +pub trait MetricsServer { + type ServeError; + + /// Creates a new Tokio task that, when spawned, brings up the index node server. + fn serve( + &mut self, + port: u16, + ) -> Result + Send>, Self::ServeError>; +} diff --git a/graph/src/components/server/mod.rs b/graph/src/components/server/mod.rs new file mode 100644 index 0000000..b0a510c --- /dev/null +++ b/graph/src/components/server/mod.rs @@ -0,0 +1,11 @@ +/// Component for running GraphQL queries over HTTP. +pub mod query; + +/// Component for running GraphQL subscriptions over WebSockets. +pub mod subscription; + +/// Component for the index node server. +pub mod index_node; + +/// Components for the Prometheus metrics server. +pub mod metrics; diff --git a/graph/src/components/server/query.rs b/graph/src/components/server/query.rs new file mode 100644 index 0000000..9fca8ea --- /dev/null +++ b/graph/src/components/server/query.rs @@ -0,0 +1,71 @@ +use crate::data::query::QueryError; +use futures::prelude::*; +use std::error::Error; +use std::fmt; + +use crate::components::store::StoreError; + +/// Errors that can occur while processing incoming requests. +#[derive(Debug)] +pub enum GraphQLServerError { + ClientError(String), + QueryError(QueryError), + InternalError(String), +} + +impl From for GraphQLServerError { + fn from(e: QueryError) -> Self { + GraphQLServerError::QueryError(e) + } +} + +impl From for GraphQLServerError { + fn from(e: StoreError) -> Self { + match e { + StoreError::ConstraintViolation(s) => GraphQLServerError::InternalError(s), + _ => GraphQLServerError::ClientError(e.to_string()), + } + } +} + +impl fmt::Display for GraphQLServerError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + GraphQLServerError::ClientError(ref s) => { + write!(f, "GraphQL server error (client error): {}", s) + } + GraphQLServerError::QueryError(ref e) => { + write!(f, "GraphQL server error (query error): {}", e) + } + GraphQLServerError::InternalError(ref s) => { + write!(f, "GraphQL server error (internal error): {}", s) + } + } + } +} + +impl Error for GraphQLServerError { + fn description(&self) -> &str { + "Failed to process the GraphQL request" + } + + fn cause(&self) -> Option<&dyn Error> { + match *self { + GraphQLServerError::ClientError(_) => None, + GraphQLServerError::QueryError(ref e) => Some(e), + GraphQLServerError::InternalError(_) => None, + } + } +} + +/// Common trait for GraphQL server implementations. +pub trait GraphQLServer { + type ServeError; + + /// Creates a new Tokio task that, when spawned, brings up the GraphQL server. + fn serve( + &mut self, + port: u16, + ws_port: u16, + ) -> Result + Send>, Self::ServeError>; +} diff --git a/graph/src/components/server/subscription.rs b/graph/src/components/server/subscription.rs new file mode 100644 index 0000000..dae6193 --- /dev/null +++ b/graph/src/components/server/subscription.rs @@ -0,0 +1,8 @@ +use async_trait::async_trait; + +/// Common trait for GraphQL subscription servers. +#[async_trait] +pub trait SubscriptionServer { + /// Returns a Future that, when spawned, brings up the GraphQL subscription server. + async fn serve(self, port: u16); +} diff --git a/graph/src/components/store/cache.rs b/graph/src/components/store/cache.rs new file mode 100644 index 0000000..e5e9903 --- /dev/null +++ b/graph/src/components/store/cache.rs @@ -0,0 +1,360 @@ +use anyhow::anyhow; +use std::collections::{BTreeMap, HashMap}; +use std::fmt::{self, Debug}; +use std::sync::Arc; + +use crate::blockchain::BlockPtr; +use crate::components::store::{ + self as s, Entity, EntityKey, EntityOp, EntityOperation, EntityType, +}; +use crate::data_source::DataSource; +use crate::prelude::{Schema, ENV_VARS}; +use crate::util::lfu_cache::LfuCache; + +/// A cache for entities from the store that provides the basic functionality +/// needed for the store interactions in the host exports. This struct tracks +/// how entities are modified, and caches all entities looked up from the +/// store. The cache makes sure that +/// (1) no entity appears in more than one operation +/// (2) only entities that will actually be changed from what they +/// are in the store are changed +pub struct EntityCache { + /// The state of entities in the store. An entry of `None` + /// means that the entity is not present in the store + current: LfuCache>, + + /// The accumulated changes to an entity. + updates: HashMap, + + // Updates for a currently executing handler. + handler_updates: HashMap, + + // Marks whether updates should go in `handler_updates`. + in_handler: bool, + + data_sources: Vec, + + /// The store is only used to read entities. + pub store: Arc, + + schema: Arc, +} + +impl Debug for EntityCache { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("EntityCache") + .field("current", &self.current) + .field("updates", &self.updates) + .finish() + } +} + +pub struct ModificationsAndCache { + pub modifications: Vec, + pub data_sources: Vec, + pub entity_lfu_cache: LfuCache>, +} + +impl EntityCache { + pub fn new(store: Arc) -> Self { + Self { + current: LfuCache::new(), + updates: HashMap::new(), + handler_updates: HashMap::new(), + in_handler: false, + data_sources: vec![], + schema: store.input_schema(), + store, + } + } + + pub fn with_current( + store: Arc, + current: LfuCache>, + ) -> EntityCache { + EntityCache { + current, + updates: HashMap::new(), + handler_updates: HashMap::new(), + in_handler: false, + data_sources: vec![], + schema: store.input_schema(), + store, + } + } + + pub(crate) fn enter_handler(&mut self) { + assert!(!self.in_handler); + self.in_handler = true; + } + + pub(crate) fn exit_handler(&mut self) { + assert!(self.in_handler); + self.in_handler = false; + + // Apply all handler updates to the main `updates`. + let handler_updates = Vec::from_iter(self.handler_updates.drain()); + for (key, op) in handler_updates { + self.entity_op(key, op) + } + } + + pub(crate) fn exit_handler_and_discard_changes(&mut self) { + assert!(self.in_handler); + self.in_handler = false; + self.handler_updates.clear(); + } + + pub fn get(&mut self, eref: &EntityKey) -> Result, s::QueryExecutionError> { + // Get the current entity, apply any updates from `updates`, then + // from `handler_updates`. + let mut entity = self.current.get_entity(&*self.store, eref)?; + if let Some(op) = self.updates.get(eref).cloned() { + entity = op.apply_to(entity) + } + if let Some(op) = self.handler_updates.get(eref).cloned() { + entity = op.apply_to(entity) + } + Ok(entity) + } + + pub fn remove(&mut self, key: EntityKey) { + self.entity_op(key, EntityOp::Remove); + } + + /// Store the `entity` under the given `key`. The `entity` may be only a + /// partial entity; the cache will ensure partial updates get merged + /// with existing data. The entity will be validated against the + /// subgraph schema, and any errors will result in an `Err` being + /// returned. + pub fn set(&mut self, key: EntityKey, mut entity: Entity) -> Result<(), anyhow::Error> { + fn check_id(key: &EntityKey, prev_id: &str) -> Result<(), anyhow::Error> { + if prev_id != key.entity_id.as_str() { + return Err(anyhow!( + "Value of {} attribute 'id' conflicts with ID passed to `store.set()`: \ + {} != {}", + key.entity_type, + prev_id, + key.entity_id, + )); + } else { + Ok(()) + } + } + + // Set the id if there isn't one yet, and make sure that a + // previously set id agrees with the one in the `key` + match entity.get("id") { + Some(s::Value::String(s)) => check_id(&key, s)?, + Some(s::Value::Bytes(b)) => check_id(&key, &b.to_string())?, + Some(_) => { + // The validation will catch the type mismatch + } + None => { + let value = self.schema.id_value(&key)?; + entity.set("id", value); + } + } + + let is_valid = entity.validate(&self.schema, &key).is_ok(); + + self.entity_op(key.clone(), EntityOp::Update(entity)); + + // The updates we were given are not valid by themselves; force a + // lookup in the database and check again with an entity that merges + // the existing entity with the changes + if !is_valid { + let entity = self.get(&key)?.ok_or_else(|| { + anyhow!( + "Failed to read entity {}[{}] back from cache", + key.entity_type, + key.entity_id + ) + })?; + entity.validate(&self.schema, &key)?; + } + + Ok(()) + } + + pub fn append(&mut self, operations: Vec) { + assert!(!self.in_handler); + + for operation in operations { + match operation { + EntityOperation::Set { key, data } => { + self.entity_op(key, EntityOp::Update(data)); + } + EntityOperation::Remove { key } => { + self.entity_op(key, EntityOp::Remove); + } + } + } + } + + /// Add a dynamic data source + pub fn add_data_source(&mut self, data_source: &DataSource) { + self.data_sources + .push(data_source.as_stored_dynamic_data_source()); + } + + fn entity_op(&mut self, key: EntityKey, op: EntityOp) { + use std::collections::hash_map::Entry; + let updates = match self.in_handler { + true => &mut self.handler_updates, + false => &mut self.updates, + }; + + match updates.entry(key) { + Entry::Vacant(entry) => { + entry.insert(op); + } + Entry::Occupied(mut entry) => entry.get_mut().accumulate(op), + } + } + + pub(crate) fn extend(&mut self, other: EntityCache) { + assert!(!other.in_handler); + + self.current.extend(other.current); + for (key, op) in other.updates { + self.entity_op(key, op); + } + } + + /// Return the changes that have been made via `set` and `remove` as + /// `EntityModification`, making sure to only produce one when a change + /// to the current state is actually needed. + /// + /// Also returns the updated `LfuCache`. + pub fn as_modifications(mut self) -> Result { + assert!(!self.in_handler); + + // The first step is to make sure all entities being set are in `self.current`. + // For each subgraph, we need a map of entity type to missing entity ids. + let missing = self + .updates + .keys() + .filter(|key| !self.current.contains_key(key)); + + // For immutable types, we assume that the subgraph is well-behaved, + // and all updated immutable entities are in fact new, and skip + // looking them up in the store. That ultimately always leads to an + // `Insert` modification for immutable entities; if the assumption + // is wrong and the store already has a version of the entity from a + // previous block, the attempt to insert will trigger a constraint + // violation in the database, ensuring correctness + let missing = missing.filter(|key| !self.schema.is_immutable(&key.entity_type)); + + let mut missing_by_type: BTreeMap<&EntityType, Vec<&str>> = BTreeMap::new(); + for key in missing { + missing_by_type + .entry(&key.entity_type) + .or_default() + .push(&key.entity_id); + } + + for (entity_type, entities) in self.store.get_many(missing_by_type)? { + for entity in entities { + let key = EntityKey { + entity_type: entity_type.clone(), + entity_id: entity.id().unwrap().into(), + }; + self.current.insert(key, Some(entity)); + } + } + + let mut mods = Vec::new(); + for (key, update) in self.updates { + use s::EntityModification::*; + + let current = self.current.remove(&key).and_then(|entity| entity); + let modification = match (current, update) { + // Entity was created + (None, EntityOp::Update(updates)) | (None, EntityOp::Overwrite(updates)) => { + // Merging with an empty entity removes null fields. + let mut data = Entity::new(); + data.merge_remove_null_fields(updates); + self.current.insert(key.clone(), Some(data.clone())); + Some(Insert { key, data }) + } + // Entity may have been changed + (Some(current), EntityOp::Update(updates)) => { + let mut data = current.clone(); + data.merge_remove_null_fields(updates); + self.current.insert(key.clone(), Some(data.clone())); + if current != data { + Some(Overwrite { key, data }) + } else { + None + } + } + // Entity was removed and then updated, so it will be overwritten + (Some(current), EntityOp::Overwrite(data)) => { + self.current.insert(key.clone(), Some(data.clone())); + if current != data { + Some(Overwrite { key, data }) + } else { + None + } + } + // Existing entity was deleted + (Some(_), EntityOp::Remove) => { + self.current.insert(key.clone(), None); + Some(Remove { key }) + } + // Entity was deleted, but it doesn't exist in the store + (None, EntityOp::Remove) => None, + }; + if let Some(modification) = modification { + mods.push(modification) + } + } + self.current.evict(ENV_VARS.mappings.entity_cache_size); + + Ok(ModificationsAndCache { + modifications: mods, + data_sources: self.data_sources, + entity_lfu_cache: self.current, + }) + } +} + +impl LfuCache> { + // Helper for cached lookup of an entity. + fn get_entity( + &mut self, + store: &(impl s::ReadStore + ?Sized), + key: &EntityKey, + ) -> Result, s::QueryExecutionError> { + match self.get(key) { + None => { + let mut entity = store.get(key)?; + if let Some(entity) = &mut entity { + // `__typename` is for queries not for mappings. + entity.remove("__typename"); + } + self.insert(key.clone(), entity.clone()); + Ok(entity) + } + Some(data) => Ok(data.to_owned()), + } + } +} + +/// Represents an item retrieved from an +/// [`EthereumCallCache`](super::EthereumCallCache) implementor. +pub struct CachedEthereumCall { + /// The BLAKE3 hash that uniquely represents this cache item. The way this + /// hash is constructed is an implementation detail. + pub blake3_id: Vec, + + /// Block details related to this Ethereum call. + pub block_ptr: BlockPtr, + + /// The address to the called contract. + pub contract_address: ethabi::Address, + + /// The encoded return value of this call. + pub return_value: Vec, +} diff --git a/graph/src/components/store/err.rs b/graph/src/components/store/err.rs new file mode 100644 index 0000000..7187e6a --- /dev/null +++ b/graph/src/components/store/err.rs @@ -0,0 +1,120 @@ +use super::{BlockNumber, DeploymentHash, DeploymentSchemaVersion}; +use crate::prelude::QueryExecutionError; +use anyhow::{anyhow, Error}; +use diesel::result::Error as DieselError; +use thiserror::Error; +use tokio::task::JoinError; + +#[derive(Error, Debug)] +pub enum StoreError { + #[error("store error: {0:#}")] + Unknown(Error), + #[error( + "tried to set entity of type `{0}` with ID \"{1}\" but an entity of type `{2}`, \ + which has an interface in common with `{0}`, exists with the same ID" + )] + ConflictingId(String, String, String), // (entity, id, conflicting_entity) + #[error("unknown field '{0}'")] + UnknownField(String), + #[error("unknown table '{0}'")] + UnknownTable(String), + #[error("malformed directive '{0}'")] + MalformedDirective(String), + #[error("query execution failed: {0}")] + QueryExecutionError(String), + #[error("invalid identifier: {0}")] + InvalidIdentifier(String), + #[error( + "subgraph `{0}` has already processed block `{1}`; \ + there are most likely two (or more) nodes indexing this subgraph" + )] + DuplicateBlockProcessing(DeploymentHash, BlockNumber), + /// An internal error where we expected the application logic to enforce + /// some constraint, e.g., that subgraph names are unique, but found that + /// constraint to not hold + #[error("internal constraint violated: {0}")] + ConstraintViolation(String), + #[error("deployment not found: {0}")] + DeploymentNotFound(String), + #[error("shard not found: {0} (this usually indicates a misconfiguration)")] + UnknownShard(String), + #[error("Fulltext search not yet deterministic")] + FulltextSearchNonDeterministic, + #[error("operation was canceled")] + Canceled, + #[error("database unavailable")] + DatabaseUnavailable, + #[error("database disabled")] + DatabaseDisabled, + #[error("subgraph forking failed: {0}")] + ForkFailure(String), + #[error("subgraph writer poisoned by previous error")] + Poisoned, + #[error("panic in subgraph writer: {0}")] + WriterPanic(JoinError), + #[error( + "found schema version {0} but this graph node only supports versions up to {}. \ + Did you downgrade Graph Node?", + DeploymentSchemaVersion::LATEST + )] + UnsupportedDeploymentSchemaVersion(i32), +} + +// Convenience to report a constraint violation +#[macro_export] +macro_rules! constraint_violation { + ($msg:expr) => {{ + StoreError::ConstraintViolation(format!("{}", $msg)) + }}; + ($fmt:expr, $($arg:tt)*) => {{ + StoreError::ConstraintViolation(format!($fmt, $($arg)*)) + }} +} + +impl From for StoreError { + fn from(e: DieselError) -> Self { + // When the error is caused by a closed connection, treat the error + // as 'database unavailable'. When this happens during indexing, the + // indexing machinery will retry in that case rather than fail the + // subgraph + if let DieselError::DatabaseError(_, info) = &e { + if info + .message() + .contains("server closed the connection unexpectedly") + { + return StoreError::DatabaseUnavailable; + } + } + StoreError::Unknown(e.into()) + } +} + +impl From<::diesel::r2d2::PoolError> for StoreError { + fn from(e: ::diesel::r2d2::PoolError) -> Self { + StoreError::Unknown(e.into()) + } +} + +impl From for StoreError { + fn from(e: Error) -> Self { + StoreError::Unknown(e) + } +} + +impl From for StoreError { + fn from(e: serde_json::Error) -> Self { + StoreError::Unknown(e.into()) + } +} + +impl From for StoreError { + fn from(e: QueryExecutionError) -> Self { + StoreError::QueryExecutionError(e.to_string()) + } +} + +impl From for StoreError { + fn from(e: std::fmt::Error) -> Self { + StoreError::Unknown(anyhow!("{}", e.to_string())) + } +} diff --git a/graph/src/components/store/mod.rs b/graph/src/components/store/mod.rs new file mode 100644 index 0000000..77cb363 --- /dev/null +++ b/graph/src/components/store/mod.rs @@ -0,0 +1,1110 @@ +mod cache; +mod err; +mod traits; + +pub use cache::{CachedEthereumCall, EntityCache, ModificationsAndCache}; +pub use err::StoreError; +use itertools::Itertools; +pub use traits::*; + +use futures::stream::poll_fn; +use futures::{Async, Poll, Stream}; +use graphql_parser::schema as s; +use serde::{Deserialize, Serialize}; +use std::collections::btree_map::Entry; +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::fmt; +use std::fmt::Display; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, RwLock}; +use std::time::Duration; + +use crate::blockchain::{Block, Blockchain}; +use crate::data::store::scalar::Bytes; +use crate::data::store::*; +use crate::data::value::Word; +use crate::prelude::*; + +/// The type name of an entity. This is the string that is used in the +/// subgraph's GraphQL schema as `type NAME @entity { .. }` +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct EntityType(Word); + +impl EntityType { + /// Construct a new entity type. Ideally, this is only called when + /// `entity_type` either comes from the GraphQL schema, or from + /// the database from fields that are known to contain a valid entity type + pub fn new(entity_type: String) -> Self { + Self(entity_type.into()) + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } + + pub fn into_string(self) -> String { + self.0.to_string() + } + + pub fn is_poi(&self) -> bool { + self.0.as_str() == "Poi$" + } +} + +impl fmt::Display for EntityType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl<'a> From<&s::ObjectType<'a, String>> for EntityType { + fn from(object_type: &s::ObjectType<'a, String>) -> Self { + EntityType::new(object_type.name.to_owned()) + } +} + +impl<'a> From<&s::InterfaceType<'a, String>> for EntityType { + fn from(interface_type: &s::InterfaceType<'a, String>) -> Self { + EntityType::new(interface_type.name.to_owned()) + } +} + +// This conversion should only be used in tests since it makes it too +// easy to convert random strings into entity types +#[cfg(debug_assertions)] +impl From<&str> for EntityType { + fn from(s: &str) -> Self { + EntityType::new(s.to_owned()) + } +} + +impl CheapClone for EntityType {} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct EntityFilterDerivative(bool); + +impl EntityFilterDerivative { + pub fn new(derived: bool) -> Self { + Self(derived) + } + + pub fn is_derived(&self) -> bool { + self.0 + } +} + +/// Key by which an individual entity in the store can be accessed. Stores +/// only the entity type and id. The deployment must be known from context. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct EntityKey { + /// Name of the entity type. + pub entity_type: EntityType, + + /// ID of the individual entity. + pub entity_id: Word, +} + +impl EntityKey { + pub fn data(entity_type: String, entity_id: String) -> Self { + Self { + entity_type: EntityType::new(entity_type), + entity_id: entity_id.into(), + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Child { + pub attr: Attribute, + pub entity_type: EntityType, + pub filter: Box, + pub derived: bool, +} + +/// Supported types of store filters. +#[derive(Clone, Debug, PartialEq)] +pub enum EntityFilter { + And(Vec), + Or(Vec), + Equal(Attribute, Value), + Not(Attribute, Value), + GreaterThan(Attribute, Value), + LessThan(Attribute, Value), + GreaterOrEqual(Attribute, Value), + LessOrEqual(Attribute, Value), + In(Attribute, Vec), + NotIn(Attribute, Vec), + Contains(Attribute, Value), + ContainsNoCase(Attribute, Value), + NotContains(Attribute, Value), + NotContainsNoCase(Attribute, Value), + StartsWith(Attribute, Value), + StartsWithNoCase(Attribute, Value), + NotStartsWith(Attribute, Value), + NotStartsWithNoCase(Attribute, Value), + EndsWith(Attribute, Value), + EndsWithNoCase(Attribute, Value), + NotEndsWith(Attribute, Value), + NotEndsWithNoCase(Attribute, Value), + ChangeBlockGte(BlockNumber), + Child(Child), +} + +// A somewhat concise string representation of a filter +impl fmt::Display for EntityFilter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use EntityFilter::*; + + match self { + And(fs) => { + write!(f, "{}", fs.iter().map(|f| f.to_string()).join(" and ")) + } + Or(fs) => { + write!(f, "{}", fs.iter().map(|f| f.to_string()).join(" or ")) + } + Equal(a, v) => write!(f, "{a} = {v}"), + Not(a, v) => write!(f, "{a} != {v}"), + GreaterThan(a, v) => write!(f, "{a} > {v}"), + LessThan(a, v) => write!(f, "{a} < {v}"), + GreaterOrEqual(a, v) => write!(f, "{a} >= {v}"), + LessOrEqual(a, v) => write!(f, "{a} <= {v}"), + In(a, vs) => write!( + f, + "{a} in ({})", + vs.into_iter().map(|v| v.to_string()).join(",") + ), + NotIn(a, vs) => write!( + f, + "{a} not in ({})", + vs.into_iter().map(|v| v.to_string()).join(",") + ), + Contains(a, v) => write!(f, "{a} ~ *{v}*"), + ContainsNoCase(a, v) => write!(f, "{a} ~ *{v}*i"), + NotContains(a, v) => write!(f, "{a} !~ *{v}*"), + NotContainsNoCase(a, v) => write!(f, "{a} !~ *{v}*i"), + StartsWith(a, v) => write!(f, "{a} ~ ^{v}*"), + StartsWithNoCase(a, v) => write!(f, "{a} ~ ^{v}*i"), + NotStartsWith(a, v) => write!(f, "{a} !~ ^{v}*"), + NotStartsWithNoCase(a, v) => write!(f, "{a} !~ ^{v}*i"), + EndsWith(a, v) => write!(f, "{a} ~ *{v}$"), + EndsWithNoCase(a, v) => write!(f, "{a} ~ *{v}$i"), + NotEndsWith(a, v) => write!(f, "{a} !~ *{v}$"), + NotEndsWithNoCase(a, v) => write!(f, "{a} !~ *{v}$i"), + ChangeBlockGte(b) => write!(f, "block >= {b}"), + Child(child /* a, et, cf, _ */) => write!( + f, + "join on {} with {}({})", + child.attr, + child.entity_type, + child.filter.to_string() + ), + } + } +} + +// Define some convenience methods +impl EntityFilter { + pub fn new_equal( + attribute_name: impl Into, + attribute_value: impl Into, + ) -> Self { + EntityFilter::Equal(attribute_name.into(), attribute_value.into()) + } + + pub fn new_in( + attribute_name: impl Into, + attribute_values: Vec>, + ) -> Self { + EntityFilter::In( + attribute_name.into(), + attribute_values.into_iter().map(Into::into).collect(), + ) + } + + pub fn and_maybe(self, other: Option) -> Self { + use EntityFilter as f; + match other { + Some(other) => match (self, other) { + (f::And(mut fs1), f::And(mut fs2)) => { + fs1.append(&mut fs2); + f::And(fs1) + } + (f::And(mut fs1), f2) => { + fs1.push(f2); + f::And(fs1) + } + (f1, f::And(mut fs2)) => { + fs2.push(f1); + f::And(fs2) + } + (f1, f2) => f::And(vec![f1, f2]), + }, + None => self, + } + } +} + +/// The order in which entities should be restored from a store. +#[derive(Clone, Debug, PartialEq)] +pub enum EntityOrder { + /// Order ascending by the given attribute. Use `id` as a tie-breaker + Ascending(String, ValueType), + /// Order descending by the given attribute. Use `id` as a tie-breaker + Descending(String, ValueType), + /// Order by the `id` of the entities + Default, + /// Do not order at all. This speeds up queries where we know that + /// order does not matter + Unordered, +} + +/// How many entities to return, how many to skip etc. +#[derive(Clone, Debug, PartialEq)] +pub struct EntityRange { + /// Limit on how many entities to return. + pub first: Option, + + /// How many entities to skip. + pub skip: u32, +} + +impl EntityRange { + /// Query for the first `n` entities. + pub fn first(n: u32) -> Self { + Self { + first: Some(n), + skip: 0, + } + } +} + +/// The attribute we want to window by in an `EntityWindow`. We have to +/// distinguish between scalar and list attributes since we need to use +/// different queries for them, and the JSONB storage scheme can not +/// determine that by itself +#[derive(Clone, Debug, PartialEq)] +pub enum WindowAttribute { + Scalar(String), + List(String), +} + +impl WindowAttribute { + pub fn name(&self) -> &str { + match self { + WindowAttribute::Scalar(name) => name, + WindowAttribute::List(name) => name, + } + } +} + +/// How to connect children to their parent when the child table does not +/// store parent id's +#[derive(Clone, Debug, PartialEq)] +pub enum ParentLink { + /// The parent stores a list of child ids. The ith entry in the outer + /// vector contains the id of the children for `EntityWindow.ids[i]` + List(Vec>), + /// The parent stores the id of one child. The ith entry in the + /// vector contains the id of the child of the parent with id + /// `EntityWindow.ids[i]` + Scalar(Vec), +} + +/// How many children a parent can have when the child stores +/// the id of the parent +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ChildMultiplicity { + Single, + Many, +} + +/// How to select children for their parents depending on whether the +/// child stores parent ids (`Direct`) or the parent +/// stores child ids (`Parent`) +#[derive(Clone, Debug, PartialEq)] +pub enum EntityLink { + /// The parent id is stored in this child attribute + Direct(WindowAttribute, ChildMultiplicity), + /// Join with the parents table to get at the parent id + Parent(EntityType, ParentLink), +} + +/// Window results of an `EntityQuery` query along the parent's id: +/// the `order_by`, `order_direction`, and `range` of the query apply to +/// entities that belong to the same parent. Only entities that belong to +/// one of the parents listed in `ids` will be included in the query result. +/// +/// Note that different windows can vary both by the entity type and id of +/// the children, but also by how to get from a child to its parent, i.e., +/// it is possible that two windows access the same entity type, but look +/// at different attributes to connect to parent entities +#[derive(Clone, Debug, PartialEq)] +pub struct EntityWindow { + /// The entity type for this window + pub child_type: EntityType, + /// The ids of parents that should be considered for this window + pub ids: Vec, + /// How to get the parent id + pub link: EntityLink, + pub column_names: AttributeNames, +} + +/// The base collections from which we are going to get entities for use in +/// `EntityQuery`; the result of the query comes from applying the query's +/// filter and order etc. to the entities described in this collection. For +/// a windowed collection order and range are applied to each individual +/// window +#[derive(Clone, Debug, PartialEq)] +pub enum EntityCollection { + /// Use all entities of the given types + All(Vec<(EntityType, AttributeNames)>), + /// Use entities according to the windows. The set of entities that we + /// apply order and range to is formed by taking all entities matching + /// the window, and grouping them by the attribute of the window. Entities + /// that have the same value in the `attribute` field of their window are + /// grouped together. Note that it is possible to have one window for + /// entity type `A` and attribute `a`, and another for entity type `B` and + /// column `b`; they will be grouped by using `A.a` and `B.b` as the keys + Window(Vec), +} + +impl EntityCollection { + pub fn entity_types_and_column_names(&self) -> BTreeMap { + let mut map = BTreeMap::new(); + match self { + EntityCollection::All(pairs) => pairs.iter().for_each(|(entity_type, column_names)| { + map.insert(entity_type.clone(), column_names.clone()); + }), + EntityCollection::Window(windows) => windows.iter().for_each( + |EntityWindow { + child_type, + column_names, + .. + }| match map.entry(child_type.clone()) { + Entry::Occupied(mut entry) => entry.get_mut().extend(column_names.clone()), + Entry::Vacant(entry) => { + entry.insert(column_names.clone()); + } + }, + ), + } + map + } +} + +/// The type we use for block numbers. This has to be a signed integer type +/// since Postgres does not support unsigned integer types. But 2G ought to +/// be enough for everybody +pub type BlockNumber = i32; + +pub const BLOCK_NUMBER_MAX: BlockNumber = std::i32::MAX; + +/// A query for entities in a store. +/// +/// Details of how query generation for `EntityQuery` works can be found +/// at https://github.com/graphprotocol/rfcs/blob/master/engineering-plans/0001-graphql-query-prefetching.md +#[derive(Clone, Debug)] +pub struct EntityQuery { + /// ID of the subgraph. + pub subgraph_id: DeploymentHash, + + /// The block height at which to execute the query. Set this to + /// `BLOCK_NUMBER_MAX` to run the query at the latest available block. + /// If the subgraph uses JSONB storage, anything but `BLOCK_NUMBER_MAX` + /// will cause an error as JSONB storage does not support querying anything + /// but the latest block + pub block: BlockNumber, + + /// The names of the entity types being queried. The result is the union + /// (with repetition) of the query for each entity. + pub collection: EntityCollection, + + /// Filter to filter entities by. + pub filter: Option, + + /// How to order the entities + pub order: EntityOrder, + + /// A range to limit the size of the result. + pub range: EntityRange, + + /// Optional logger for anything related to this query + pub logger: Option, + + pub query_id: Option, + + _force_use_of_new: (), +} + +impl EntityQuery { + pub fn new( + subgraph_id: DeploymentHash, + block: BlockNumber, + collection: EntityCollection, + ) -> Self { + EntityQuery { + subgraph_id, + block, + collection, + filter: None, + order: EntityOrder::Default, + range: EntityRange::first(100), + logger: None, + query_id: None, + _force_use_of_new: (), + } + } + + pub fn filter(mut self, filter: EntityFilter) -> Self { + self.filter = Some(filter); + self + } + + pub fn order(mut self, order: EntityOrder) -> Self { + self.order = order; + self + } + + pub fn range(mut self, range: EntityRange) -> Self { + self.range = range; + self + } + + pub fn first(mut self, first: u32) -> Self { + self.range.first = Some(first); + self + } + + pub fn skip(mut self, skip: u32) -> Self { + self.range.skip = skip; + self + } + + pub fn simplify(mut self) -> Self { + // If there is one window, with one id, in a direct relation to the + // entities, we can simplify the query by changing the filter and + // getting rid of the window + if let EntityCollection::Window(windows) = &self.collection { + if windows.len() == 1 { + let window = windows.first().expect("we just checked"); + if window.ids.len() == 1 { + let id = window.ids.first().expect("we just checked"); + if let EntityLink::Direct(attribute, _) = &window.link { + let filter = match attribute { + WindowAttribute::Scalar(name) => { + EntityFilter::Equal(name.to_owned(), id.into()) + } + WindowAttribute::List(name) => { + EntityFilter::Contains(name.to_owned(), Value::from(vec![id])) + } + }; + self.filter = Some(filter.and_maybe(self.filter)); + self.collection = EntityCollection::All(vec![( + window.child_type.to_owned(), + window.column_names.clone(), + )]); + } + } + } + } + self + } +} + +/// Operation types that lead to entity changes. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[serde(rename_all = "lowercase")] +pub enum EntityChangeOperation { + /// An entity was added or updated + Set, + /// An existing entity was removed. + Removed, +} + +/// Entity change events emitted by [Store](trait.Store.html) implementations. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub enum EntityChange { + Data { + subgraph_id: DeploymentHash, + /// Entity type name of the changed entity. + entity_type: EntityType, + }, + Assignment { + deployment: DeploymentLocator, + operation: EntityChangeOperation, + }, +} + +impl EntityChange { + pub fn for_data(subgraph_id: DeploymentHash, key: EntityKey) -> Self { + Self::Data { + subgraph_id: subgraph_id, + entity_type: key.entity_type, + } + } + + pub fn for_assignment(deployment: DeploymentLocator, operation: EntityChangeOperation) -> Self { + Self::Assignment { + deployment, + operation, + } + } + + pub fn as_filter(&self) -> SubscriptionFilter { + use EntityChange::*; + match self { + Data { + subgraph_id, + entity_type, + .. + } => SubscriptionFilter::Entities(subgraph_id.clone(), entity_type.clone()), + Assignment { .. } => SubscriptionFilter::Assignment, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +/// The store emits `StoreEvents` to indicate that some entities have changed. +/// For block-related data, at most one `StoreEvent` is emitted for each block +/// that is processed. The `changes` vector contains the details of what changes +/// were made, and to which entity. +/// +/// Since the 'subgraph of subgraphs' is special, and not directly related to +/// any specific blocks, `StoreEvents` for it are generated as soon as they are +/// written to the store. +pub struct StoreEvent { + // The tag is only there to make it easier to track StoreEvents in the + // logs as they flow through the system + pub tag: usize, + pub changes: HashSet, +} + +impl StoreEvent { + pub fn new(changes: Vec) -> StoreEvent { + static NEXT_TAG: AtomicUsize = AtomicUsize::new(0); + + let tag = NEXT_TAG.fetch_add(1, Ordering::Relaxed); + let changes = changes.into_iter().collect(); + StoreEvent { tag, changes } + } + + pub fn from_mods<'a, I: IntoIterator>( + subgraph_id: &DeploymentHash, + mods: I, + ) -> Self { + let changes: Vec<_> = mods + .into_iter() + .map(|op| { + use self::EntityModification::*; + match op { + Insert { key, .. } | Overwrite { key, .. } | Remove { key } => { + EntityChange::for_data(subgraph_id.clone(), key.clone()) + } + } + }) + .collect(); + StoreEvent::new(changes) + } + + /// Extend `ev1` with `ev2`. If `ev1` is `None`, just set it to `ev2` + fn accumulate(logger: &Logger, ev1: &mut Option, ev2: StoreEvent) { + if let Some(e) = ev1 { + trace!(logger, "Adding changes to event"; + "from" => ev2.tag, "to" => e.tag); + e.changes.extend(ev2.changes); + } else { + *ev1 = Some(ev2); + } + } + + pub fn extend(mut self, other: StoreEvent) -> Self { + self.changes.extend(other.changes); + self + } + + pub fn matches(&self, filters: &BTreeSet) -> bool { + self.changes + .iter() + .any(|change| filters.iter().any(|filter| filter.matches(change))) + } +} + +impl fmt::Display for StoreEvent { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "StoreEvent[{}](changes: {})", + self.tag, + self.changes.len() + ) + } +} + +impl PartialEq for StoreEvent { + fn eq(&self, other: &StoreEvent) -> bool { + // Ignore tag for equality + self.changes == other.changes + } +} + +/// A `StoreEventStream` produces the `StoreEvents`. Various filters can be applied +/// to it to reduce which and how many events are delivered by the stream. +pub struct StoreEventStream { + source: S, +} + +/// A boxed `StoreEventStream` +pub type StoreEventStreamBox = + StoreEventStream, Error = ()> + Send>>; + +pub type UnitStream = Box + Unpin + Send + Sync>; + +impl Stream for StoreEventStream +where + S: Stream, Error = ()> + Send, +{ + type Item = Arc; + type Error = (); + + fn poll(&mut self) -> Result>, Self::Error> { + self.source.poll() + } +} + +impl StoreEventStream +where + S: Stream, Error = ()> + Send + 'static, +{ + // Create a new `StoreEventStream` from another such stream + pub fn new(source: S) -> Self { + StoreEventStream { source } + } + + /// Filter a `StoreEventStream` by subgraph and entity. Only events that have + /// at least one change to one of the given (subgraph, entity) combinations + /// will be delivered by the filtered stream. + pub fn filter_by_entities(self, filters: BTreeSet) -> StoreEventStreamBox { + let source = self.source.filter(move |event| event.matches(&filters)); + + StoreEventStream::new(Box::new(source)) + } + + /// Reduce the frequency with which events are generated while a + /// subgraph deployment is syncing. While the given `deployment` is not + /// synced yet, events from `source` are reported at most every + /// `interval`. At the same time, no event is held for longer than + /// `interval`. The `StoreEvents` that arrive during an interval appear + /// on the returned stream as a single `StoreEvent`; the events are + /// combined by using the maximum of all sources and the concatenation + /// of the changes of the `StoreEvents` received during the interval. + // + // Currently unused, needs to be made compatible with `subscribe_no_payload`. + pub async fn throttle_while_syncing( + self, + logger: &Logger, + store: Arc, + interval: Duration, + ) -> StoreEventStreamBox { + // Check whether a deployment is marked as synced in the store. Note that in the moment a + // subgraph becomes synced any existing subscriptions will continue to be throttled since + // this is not re-checked. + let synced = store.is_deployment_synced().await.unwrap_or(false); + + let mut pending_event: Option = None; + let mut source = self.source.fuse(); + let mut had_err = false; + let mut delay = tokio::time::sleep(interval).unit_error().boxed().compat(); + let logger = logger.clone(); + + let source = Box::new(poll_fn(move || -> Poll>, ()> { + if had_err { + // We had an error the last time through, but returned the pending + // event first. Indicate the error now + had_err = false; + return Err(()); + } + + if synced { + return source.poll(); + } + + // Check if interval has passed since the last time we sent something. + // If it has, start a new delay timer + let should_send = match futures::future::Future::poll(&mut delay) { + Ok(Async::NotReady) => false, + // Timer errors are harmless. Treat them as if the timer had + // become ready. + Ok(Async::Ready(())) | Err(_) => { + delay = tokio::time::sleep(interval).unit_error().boxed().compat(); + true + } + }; + + // Get as many events as we can off of the source stream + loop { + match source.poll() { + Ok(Async::NotReady) => { + if should_send && pending_event.is_some() { + let event = pending_event.take().map(Arc::new); + return Ok(Async::Ready(event)); + } else { + return Ok(Async::NotReady); + } + } + Ok(Async::Ready(None)) => { + let event = pending_event.take().map(Arc::new); + return Ok(Async::Ready(event)); + } + Ok(Async::Ready(Some(event))) => { + StoreEvent::accumulate(&logger, &mut pending_event, (*event).clone()); + } + Err(()) => { + // Before we report the error, deliver what we have accumulated so far. + // We will report the error the next time poll() is called + if pending_event.is_some() { + had_err = true; + let event = pending_event.take().map(Arc::new); + return Ok(Async::Ready(event)); + } else { + return Err(()); + } + } + }; + } + })); + StoreEventStream::new(source) + } +} + +/// An entity operation that can be transacted into the store. +#[derive(Clone, Debug, PartialEq)] +pub enum EntityOperation { + /// Locates the entity specified by `key` and sets its attributes according to the contents of + /// `data`. If no entity exists with this key, creates a new entity. + Set { key: EntityKey, data: Entity }, + + /// Removes an entity with the specified key, if one exists. + Remove { key: EntityKey }, +} + +#[derive(Debug, PartialEq)] +pub enum UnfailOutcome { + Noop, + Unfailed, +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct StoredDynamicDataSource { + pub manifest_idx: u32, + pub param: Option, + pub context: Option, + pub creation_block: Option, + pub is_offchain: bool, +} + +/// An internal identifer for the specific instance of a deployment. The +/// identifier only has meaning in the context of a specific instance of +/// graph-node. Only store code should ever construct or consume it; all +/// other code passes it around as an opaque token. +#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub struct DeploymentId(pub i32); + +impl Display for DeploymentId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + write!(f, "{}", self.0) + } +} + +impl DeploymentId { + pub fn new(id: i32) -> Self { + Self(id) + } +} + +/// A unique identifier for a deployment that specifies both its external +/// identifier (`hash`) and its unique internal identifier (`id`) which +/// ensures we are talking about a unique location for the deployment's data +/// in the store +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub struct DeploymentLocator { + pub id: DeploymentId, + pub hash: DeploymentHash, +} + +impl CheapClone for DeploymentLocator {} + +impl slog::Value for DeploymentLocator { + fn serialize( + &self, + record: &slog::Record, + key: slog::Key, + serializer: &mut dyn slog::Serializer, + ) -> slog::Result { + slog::Value::serialize(&self.to_string(), record, key, serializer) + } +} + +impl DeploymentLocator { + pub fn new(id: DeploymentId, hash: DeploymentHash) -> Self { + Self { id, hash } + } +} + +impl Display for DeploymentLocator { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}[{}]", self.hash, self.id) + } +} + +// The type that the connection pool uses to track wait times for +// connection checkouts +pub type PoolWaitStats = Arc>; + +/// An entity operation that can be transacted into the store; as opposed to +/// `EntityOperation`, we already know whether a `Set` should be an `Insert` +/// or `Update` +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EntityModification { + /// Insert the entity + Insert { key: EntityKey, data: Entity }, + /// Update the entity by overwriting it + Overwrite { key: EntityKey, data: Entity }, + /// Remove the entity + Remove { key: EntityKey }, +} + +impl EntityModification { + pub fn entity_ref(&self) -> &EntityKey { + use EntityModification::*; + match self { + Insert { key, .. } | Overwrite { key, .. } | Remove { key } => key, + } + } + + pub fn entity(&self) -> Option<&Entity> { + match self { + EntityModification::Insert { data, .. } + | EntityModification::Overwrite { data, .. } => Some(data), + EntityModification::Remove { .. } => None, + } + } + + pub fn is_remove(&self) -> bool { + match self { + EntityModification::Remove { .. } => true, + _ => false, + } + } +} + +/// A representation of entity operations that can be accumulated. +#[derive(Debug, Clone)] +enum EntityOp { + Remove, + Update(Entity), + Overwrite(Entity), +} + +impl EntityOp { + fn apply_to(self, entity: Option) -> Option { + use EntityOp::*; + match (self, entity) { + (Remove, _) => None, + (Overwrite(new), _) | (Update(new), None) => Some(new), + (Update(updates), Some(mut entity)) => { + entity.merge_remove_null_fields(updates); + Some(entity) + } + } + } + + fn accumulate(&mut self, next: EntityOp) { + use EntityOp::*; + let update = match next { + // Remove and Overwrite ignore the current value. + Remove | Overwrite(_) => { + *self = next; + return; + } + Update(update) => update, + }; + + // We have an update, apply it. + match self { + // This is how `Overwrite` is constructed, by accumulating `Update` onto `Remove`. + Remove => *self = Overwrite(update), + Update(current) | Overwrite(current) => current.merge(update), + } + } +} + +/// Determines which columns should be selected in a table. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum AttributeNames { + /// Select all columns. Equivalent to a `"SELECT *"`. + All, + /// Individual column names to be selected. + Select(BTreeSet), +} + +impl AttributeNames { + fn insert(&mut self, column_name: &str) { + match self { + AttributeNames::All => { + let mut set = BTreeSet::new(); + set.insert(column_name.to_string()); + *self = AttributeNames::Select(set) + } + AttributeNames::Select(set) => { + set.insert(column_name.to_string()); + } + } + } + + pub fn update(&mut self, field_name: &str) { + if Self::is_meta_field(field_name) { + return; + } + self.insert(field_name) + } + + /// Adds a attribute name. Ignores meta fields. + pub fn add_str(&mut self, field_name: &str) { + if Self::is_meta_field(field_name) { + return; + } + self.insert(field_name); + } + + /// Returns `true` for meta field names, `false` otherwise. + fn is_meta_field(field_name: &str) -> bool { + field_name.starts_with("__") + } + + pub fn extend(&mut self, other: Self) { + use AttributeNames::*; + match (self, other) { + (All, All) => {} + (self_ @ All, other @ Select(_)) => *self_ = other, + (Select(_), All) => { + unreachable!() + } + (Select(a), Select(b)) => a.extend(b), + } + } +} + +#[derive(Debug, Clone)] +pub struct PartialBlockPtr { + pub number: BlockNumber, + pub hash: Option, +} + +impl From for PartialBlockPtr { + fn from(number: BlockNumber) -> Self { + Self { number, hash: None } + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum DeploymentSchemaVersion { + /// V0, baseline version, in which: + /// - A relational schema is used. + /// - Each deployment has its own namespace for entity tables. + /// - Dynamic data sources are stored in `subgraphs.dynamic_ethereum_contract_data_source`. + V0 = 0, + + /// V1: Dynamic data sources moved to `sgd*.data_sources$`. + V1 = 1, +} + +impl DeploymentSchemaVersion { + // Latest schema version supported by this version of graph node. + pub const LATEST: Self = Self::V1; + + pub fn private_data_sources(self) -> bool { + use DeploymentSchemaVersion::*; + match self { + V0 => false, + V1 => true, + } + } +} + +impl TryFrom for DeploymentSchemaVersion { + type Error = StoreError; + + fn try_from(value: i32) -> Result { + match value { + 0 => Ok(Self::V0), + 1 => Ok(Self::V1), + _ => Err(StoreError::UnsupportedDeploymentSchemaVersion(value)), + } + } +} + +impl fmt::Display for DeploymentSchemaVersion { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&(*self as i32), f) + } +} + +/// A `ReadStore` that is always empty. +pub struct EmptyStore { + schema: Arc, +} + +impl EmptyStore { + pub fn new(schema: Arc) -> Self { + EmptyStore { schema } + } +} + +impl ReadStore for EmptyStore { + fn get(&self, _key: &EntityKey) -> Result, StoreError> { + Ok(None) + } + + fn get_many( + &self, + _ids_for_type: BTreeMap<&EntityType, Vec<&str>>, + ) -> Result>, StoreError> { + Ok(BTreeMap::new()) + } + + fn input_schema(&self) -> Arc { + self.schema.cheap_clone() + } +} + +/// An estimate of the number of entities and the number of entity versions +/// in a database table +#[derive(Clone, Debug)] +pub struct VersionStats { + pub entities: i32, + pub versions: i32, + pub tablename: String, + /// The ratio `entities / versions` + pub ratio: f64, +} + +/// Callbacks for `SubgraphStore.prune` so that callers can report progress +/// of the pruning procedure to users +#[allow(unused_variables)] +pub trait PruneReporter: Send + 'static { + fn start_analyze(&mut self) {} + fn start_analyze_table(&mut self, table: &str) {} + fn finish_analyze_table(&mut self, table: &str) {} + fn finish_analyze(&mut self, stats: &[VersionStats]) {} + + fn copy_final_start(&mut self, earliest_block: BlockNumber, final_block: BlockNumber) {} + fn copy_final_batch(&mut self, table: &str, rows: usize, total_rows: usize, finished: bool) {} + fn copy_final_finish(&mut self) {} + + fn start_switch(&mut self) {} + fn copy_nonfinal_start(&mut self, table: &str) {} + fn copy_nonfinal_finish(&mut self, table: &str, rows: usize) {} + fn finish_switch(&mut self) {} + + fn finish_prune(&mut self) {} +} diff --git a/graph/src/components/store/traits.rs b/graph/src/components/store/traits.rs new file mode 100644 index 0000000..1749b8b --- /dev/null +++ b/graph/src/components/store/traits.rs @@ -0,0 +1,517 @@ +use web3::types::{Address, H256}; + +use super::*; +use crate::blockchain::block_stream::FirehoseCursor; +use crate::components::server::index_node::VersionInfo; +use crate::components::transaction_receipt; +use crate::components::versions::ApiVersion; +use crate::data::query::Trace; +use crate::data::subgraph::status; +use crate::data::value::Word; +use crate::data::{query::QueryTarget, subgraph::schema::*}; + +pub trait SubscriptionManager: Send + Sync + 'static { + /// Subscribe to changes for specific subgraphs and entities. + /// + /// Returns a stream of store events that match the input arguments. + fn subscribe(&self, entities: BTreeSet) -> StoreEventStreamBox; + + /// If the payload is not required, use for a more efficient subscription mechanism backed by a watcher. + fn subscribe_no_payload(&self, entities: BTreeSet) -> UnitStream; +} + +/// Subgraph forking is the process of lazily fetching entities +/// from another subgraph's store (usually a remote one). +pub trait SubgraphFork: Send + Sync + 'static { + fn fetch(&self, entity_type: String, id: String) -> Result, StoreError>; +} + +/// A special trait to handle looking up ENS names from special rainbow +/// tables that need to be manually loaded into the system +pub trait EnsLookup: Send + Sync + 'static { + /// Find the reverse of keccak256 for `hash` through looking it up in the + /// rainbow table. + fn find_name(&self, hash: &str) -> Result, StoreError>; +} + +/// An entry point for all operations that require access to the node's storage +/// layer. It provides access to a [`BlockStore`] and a [`SubgraphStore`]. +pub trait Store: Clone + StatusStore + Send + Sync + 'static { + /// The [`BlockStore`] implementor used by this [`Store`]. + type BlockStore: BlockStore; + + /// The [`SubgraphStore`] implementor used by this [`Store`]. + type SubgraphStore: SubgraphStore; + + fn block_store(&self) -> Arc; + + fn subgraph_store(&self) -> Arc; +} + +/// Common trait for store implementations. +#[async_trait] +pub trait SubgraphStore: Send + Sync + 'static { + fn ens_lookup(&self) -> Arc; + + /// Check if the store is accepting queries for the specified subgraph. + /// May return true even if the specified subgraph is not currently assigned to an indexing + /// node, as the store will still accept queries. + fn is_deployed(&self, id: &DeploymentHash) -> Result; + + /// Create a new deployment for the subgraph `name`. If the deployment + /// already exists (as identified by the `schema.id`), reuse that, otherwise + /// create a new deployment, and point the current or pending version of + /// `name` at it, depending on the `mode` + fn create_subgraph_deployment( + &self, + name: SubgraphName, + schema: &Schema, + deployment: DeploymentCreate, + node_id: NodeId, + network: String, + mode: SubgraphVersionSwitchingMode, + ) -> Result; + + /// Create a new subgraph with the given name. If one already exists, use + /// the existing one. Return the `id` of the newly created or existing + /// subgraph + fn create_subgraph(&self, name: SubgraphName) -> Result; + + /// Remove a subgraph and all its versions; if deployments that were used + /// by this subgraph do not need to be indexed anymore, also remove + /// their assignment, but keep the deployments themselves around + fn remove_subgraph(&self, name: SubgraphName) -> Result<(), StoreError>; + + /// Assign the subgraph with `id` to the node `node_id`. If there is no + /// assignment for the given deployment, report an error. + fn reassign_subgraph( + &self, + deployment: &DeploymentLocator, + node_id: &NodeId, + ) -> Result<(), StoreError>; + + fn assigned_node(&self, deployment: &DeploymentLocator) -> Result, StoreError>; + + fn assignments(&self, node: &NodeId) -> Result, StoreError>; + + /// Return `true` if a subgraph `name` exists, regardless of whether the + /// subgraph has any deployments attached to it + fn subgraph_exists(&self, name: &SubgraphName) -> Result; + + /// Returns a collection of all [`EntityModification`] items in relation to + /// the given [`BlockNumber`]. No distinction is made between inserts and + /// updates, which may be returned as either [`EntityModification::Insert`] + /// or [`EntityModification::Overwrite`]. + fn entity_changes_in_block( + &self, + subgraph_id: &DeploymentHash, + block_number: BlockNumber, + ) -> Result, StoreError>; + + /// Return the GraphQL schema supplied by the user + fn input_schema(&self, subgraph_id: &DeploymentHash) -> Result, StoreError>; + + /// Return the GraphQL schema that was derived from the user's schema by + /// adding a root query type etc. to it + fn api_schema( + &self, + subgraph_id: &DeploymentHash, + api_version: &ApiVersion, + ) -> Result, StoreError>; + + /// Return a `SubgraphFork`, derived from the user's `debug-fork` deployment argument, + /// that is used for debugging purposes only. + fn debug_fork( + &self, + subgraph_id: &DeploymentHash, + logger: Logger, + ) -> Result>, StoreError>; + + /// Return a `WritableStore` that is used for indexing subgraphs. Only + /// code that is part of indexing a subgraph should ever use this. The + /// `logger` will be used to log important messages related to the + /// subgraph. + /// + /// This function should only be called in situations where no + /// assumptions about the in-memory state of writing has been made; in + /// particular, no assumptions about whether previous writes have + /// actually been committed or not. + async fn writable( + self: Arc, + logger: Logger, + deployment: DeploymentId, + ) -> Result, StoreError>; + + /// Return the minimum block pointer of all deployments with this `id` + /// that we would use to query or copy from; in particular, this will + /// ignore any instances of this deployment that are in the process of + /// being set up + async fn least_block_ptr(&self, id: &DeploymentHash) -> Result, StoreError>; + + async fn is_healthy(&self, id: &DeploymentHash) -> Result; + + /// Find the deployment locators for the subgraph with the given hash + fn locators(&self, hash: &str) -> Result, StoreError>; +} + +pub trait ReadStore: Send + Sync + 'static { + /// Looks up an entity using the given store key at the latest block. + fn get(&self, key: &EntityKey) -> Result, StoreError>; + + /// Look up multiple entities as of the latest block. Returns a map of + /// entities by type. + fn get_many( + &self, + ids_for_type: BTreeMap<&EntityType, Vec<&str>>, + ) -> Result>, StoreError>; + + fn input_schema(&self) -> Arc; +} + +// This silly impl is needed until https://github.com/rust-lang/rust/issues/65991 is stable. +impl ReadStore for Arc { + fn get(&self, key: &EntityKey) -> Result, StoreError> { + (**self).get(key) + } + + fn get_many( + &self, + ids_for_type: BTreeMap<&EntityType, Vec<&str>>, + ) -> Result>, StoreError> { + (**self).get_many(ids_for_type) + } + + fn input_schema(&self) -> Arc { + (**self).input_schema() + } +} + +/// A view of the store for indexing. All indexing-related operations need +/// to go through this trait. Methods in this trait will never return a +/// `StoreError::DatabaseUnavailable`. Instead, they will retry the +/// operation indefinitely until it succeeds. +#[async_trait] +pub trait WritableStore: ReadStore { + /// Get a pointer to the most recently processed block in the subgraph. + fn block_ptr(&self) -> Option; + + /// Returns the Firehose `cursor` this deployment is currently at in the block stream of events. This + /// is used when re-connecting a Firehose stream to start back exactly where we left off. + fn block_cursor(&self) -> FirehoseCursor; + + /// Start an existing subgraph deployment. + async fn start_subgraph_deployment(&self, logger: &Logger) -> Result<(), StoreError>; + + /// Revert the entity changes from a single block atomically in the store, and update the + /// subgraph block pointer to `block_ptr_to`. + /// + /// `block_ptr_to` must point to the parent block of the subgraph block pointer. + async fn revert_block_operations( + &self, + block_ptr_to: BlockPtr, + firehose_cursor: FirehoseCursor, + ) -> Result<(), StoreError>; + + /// If a deterministic error happened, this function reverts the block operations from the + /// current block to the previous block. + async fn unfail_deterministic_error( + &self, + current_ptr: &BlockPtr, + parent_ptr: &BlockPtr, + ) -> Result; + + /// If a non-deterministic error happened and the current deployment head is past the error + /// block range, this function unfails the subgraph and deletes the error. + fn unfail_non_deterministic_error( + &self, + current_ptr: &BlockPtr, + ) -> Result; + + /// Set subgraph status to failed with the given error as the cause. + async fn fail_subgraph(&self, error: SubgraphError) -> Result<(), StoreError>; + + async fn supports_proof_of_indexing(&self) -> Result; + + /// Transact the entity changes from a single block atomically into the store, and update the + /// subgraph block pointer to `block_ptr_to`, and update the firehose cursor to `firehose_cursor` + /// + /// `block_ptr_to` must point to a child block of the current subgraph block pointer. + async fn transact_block_operations( + &self, + block_ptr_to: BlockPtr, + firehose_cursor: FirehoseCursor, + mods: Vec, + stopwatch: &StopwatchMetrics, + data_sources: Vec, + deterministic_errors: Vec, + manifest_idx_and_name: Vec<(u32, String)>, + offchain_to_remove: Vec, + ) -> Result<(), StoreError>; + + /// The deployment `id` finished syncing, mark it as synced in the database + /// and promote it to the current version in the subgraphs where it was the + /// pending version so far + fn deployment_synced(&self) -> Result<(), StoreError>; + + /// Return true if the deployment with the given id is fully synced, + /// and return false otherwise. Errors from the store are passed back up + async fn is_deployment_synced(&self) -> Result; + + fn unassign_subgraph(&self) -> Result<(), StoreError>; + + /// Load the dynamic data sources for the given deployment + async fn load_dynamic_data_sources( + &self, + manifest_idx_and_name: Vec<(u32, String)>, + ) -> Result, StoreError>; + + /// Report the name of the shard in which the subgraph is stored. This + /// should only be used for reporting and monitoring + fn shard(&self) -> &str; + + async fn health(&self) -> Result; + + /// Wait for the background writer to finish processing its queue + async fn flush(&self) -> Result<(), StoreError>; +} + +#[async_trait] +pub trait QueryStoreManager: Send + Sync + 'static { + /// Get a new `QueryStore`. A `QueryStore` is tied to a DB replica, so if Graph Node is + /// configured to use secondary DB servers the queries will be distributed between servers. + /// + /// The query store is specific to a deployment, and `id` must indicate + /// which deployment will be queried. It is not possible to use the id of the + /// metadata subgraph, though the resulting store can be used to query + /// metadata about the deployment `id` (but not metadata about other deployments). + /// + /// If `for_subscription` is true, the main replica will always be used. + async fn query_store( + &self, + target: QueryTarget, + for_subscription: bool, + ) -> Result, QueryExecutionError>; +} + +pub trait BlockStore: Send + Sync + 'static { + type ChainStore: ChainStore; + + fn chain_store(&self, network: &str) -> Option>; +} + +/// Common trait for blockchain store implementations. +#[async_trait] +pub trait ChainStore: Send + Sync + 'static { + /// Get a pointer to this blockchain's genesis block. + fn genesis_block_ptr(&self) -> Result; + + /// Insert a block into the store (or update if they are already present). + async fn upsert_block(&self, block: Arc) -> Result<(), Error>; + + fn upsert_light_blocks(&self, blocks: &[&dyn Block]) -> Result<(), Error>; + + /// Try to update the head block pointer to the block with the highest block number. + /// + /// Only updates pointer if there is a block with a higher block number than the current head + /// block, and the `ancestor_count` most recent ancestors of that block are in the store. + /// Note that this means if the Ethereum node returns a different "latest block" with a + /// different hash but same number, we do not update the chain head pointer. + /// This situation can happen on e.g. Infura where requests are load balanced across many + /// Ethereum nodes, in which case it's better not to continuously revert and reapply the latest + /// blocks. + /// + /// If the pointer was updated, returns `Ok(vec![])`, and fires a HeadUpdateEvent. + /// + /// If no block has a number higher than the current head block, returns `Ok(vec![])`. + /// + /// If the candidate new head block had one or more missing ancestors, returns + /// `Ok(missing_blocks)`, where `missing_blocks` is a nonexhaustive list of missing blocks. + async fn attempt_chain_head_update( + self: Arc, + ancestor_count: BlockNumber, + ) -> Result, Error>; + + /// Get the current head block pointer for this chain. + /// Any changes to the head block pointer will be to a block with a larger block number, never + /// to a block with a smaller or equal block number. + /// + /// The head block pointer will be None on initial set up. + async fn chain_head_ptr(self: Arc) -> Result, Error>; + + /// In-memory time cached version of `chain_head_ptr`. + async fn cached_head_ptr(self: Arc) -> Result, Error>; + + /// Get the current head block cursor for this chain. + /// + /// The head block cursor will be None on initial set up. + fn chain_head_cursor(&self) -> Result, Error>; + + /// This method does actually three operations: + /// - Upserts received block into blocks table + /// - Update chain head block into networks table + /// - Update chain head cursor into networks table + async fn set_chain_head( + self: Arc, + block: Arc, + cursor: String, + ) -> Result<(), Error>; + + /// Returns the blocks present in the store. + fn blocks(&self, hashes: &[BlockHash]) -> Result, Error>; + + /// Get the `offset`th ancestor of `block_hash`, where offset=0 means the block matching + /// `block_hash` and offset=1 means its parent. Returns None if unable to complete due to + /// missing blocks in the chain store. + /// + /// Returns an error if the offset would reach past the genesis block. + async fn ancestor_block( + self: Arc, + block_ptr: BlockPtr, + offset: BlockNumber, + ) -> Result, Error>; + + /// Remove old blocks from the cache we maintain in the database and + /// return a pair containing the number of the oldest block retained + /// and the number of blocks deleted. + /// We will never remove blocks that are within `ancestor_count` of + /// the chain head. + fn cleanup_cached_blocks( + &self, + ancestor_count: BlockNumber, + ) -> Result, Error>; + + /// Return the hashes of all blocks with the given number + fn block_hashes_by_block_number(&self, number: BlockNumber) -> Result, Error>; + + /// Confirm that block number `number` has hash `hash` and that the store + /// may purge any other blocks with that number + fn confirm_block_hash(&self, number: BlockNumber, hash: &BlockHash) -> Result; + + /// Find the block with `block_hash` and return the network name, number and timestamp if present. + /// Currently, the timestamp is only returned if it's present in the top level block. This format is + /// depends on the chain and the implementation of Blockchain::Block for the specific chain. + /// eg: {"block": { "timestamp": 123123123 } } + async fn block_number( + &self, + hash: &BlockHash, + ) -> Result)>, StoreError>; + + /// Tries to retrieve all transactions receipts for a given block. + async fn transaction_receipts_in_block( + &self, + block_ptr: &H256, + ) -> Result, StoreError>; +} + +pub trait EthereumCallCache: Send + Sync + 'static { + /// Returns the return value of the provided Ethereum call, if present in + /// the cache. + fn get_call( + &self, + contract_address: ethabi::Address, + encoded_call: &[u8], + block: BlockPtr, + ) -> Result>, Error>; + + /// Returns all cached calls for a given `block`. This method does *not* + /// update the last access time of the returned cached calls. + fn get_calls_in_block(&self, block: BlockPtr) -> Result, Error>; + + /// Stores the provided Ethereum call in the cache. + fn set_call( + &self, + contract_address: ethabi::Address, + encoded_call: &[u8], + block: BlockPtr, + return_value: &[u8], + ) -> Result<(), Error>; +} + +/// Store operations used when serving queries for a specific deployment +#[async_trait] +pub trait QueryStore: Send + Sync { + fn find_query_values( + &self, + query: EntityQuery, + ) -> Result<(Vec>, Trace), QueryExecutionError>; + + async fn is_deployment_synced(&self) -> Result; + + async fn block_ptr(&self) -> Result, StoreError>; + + async fn block_number(&self, block_hash: &BlockHash) + -> Result, StoreError>; + + /// Returns the blocknumber as well as the timestamp. Timestamp depends on the chain block type + /// and can have multiple formats, it can also not be prevent. For now this is only available + /// for EVM chains both firehose and rpc. + async fn block_number_with_timestamp( + &self, + block_hash: &BlockHash, + ) -> Result)>, StoreError>; + + fn wait_stats(&self) -> Result; + + async fn has_deterministic_errors(&self, block: BlockNumber) -> Result; + + /// Find the current state for the subgraph deployment `id` and + /// return details about it needed for executing queries + async fn deployment_state(&self) -> Result; + + fn api_schema(&self) -> Result, QueryExecutionError>; + + fn network_name(&self) -> &str; + + /// A permit should be acquired before starting query execution. + async fn query_permit(&self) -> Result; +} + +/// A view of the store that can provide information about the indexing status +/// of any subgraph and any deployment +#[async_trait] +pub trait StatusStore: Send + Sync + 'static { + /// A permit should be acquired before starting query execution. + async fn query_permit(&self) -> Result; + + fn status(&self, filter: status::Filter) -> Result, StoreError>; + + /// Support for the explorer-specific API + fn version_info(&self, version_id: &str) -> Result; + + /// Support for the explorer-specific API; note that `subgraph_id` must be + /// the id of an entry in `subgraphs.subgraph`, not that of a deployment. + /// The return values are the ids of the `subgraphs.subgraph_version` for + /// the current and pending versions of the subgraph + fn versions_for_subgraph_id( + &self, + subgraph_id: &str, + ) -> Result<(Option, Option), StoreError>; + + /// Support for the explorer-specific API. Returns a vector of (name, version) of all + /// subgraphs for a given deployment hash. + fn subgraphs_for_deployment_hash( + &self, + deployment_hash: &str, + ) -> Result, StoreError>; + + /// A value of None indicates that the table is not available. Re-deploying + /// the subgraph fixes this. It is undesirable to force everything to + /// re-sync from scratch, so existing deployments will continue without a + /// Proof of Indexing. Once all subgraphs have been re-deployed the Option + /// can be removed. + async fn get_proof_of_indexing( + &self, + subgraph_id: &DeploymentHash, + indexer: &Option
, + block: BlockPtr, + ) -> Result, StoreError>; + + /// Like `get_proof_of_indexing` but returns a Proof of Indexing signed by + /// address `0x00...0`, which allows it to be shared in public without + /// revealing the indexers _real_ Proof of Indexing. + async fn get_public_proof_of_indexing( + &self, + subgraph_id: &DeploymentHash, + block_number: BlockNumber, + ) -> Result, StoreError>; +} diff --git a/graph/src/components/subgraph/host.rs b/graph/src/components/subgraph/host.rs new file mode 100644 index 0000000..27d7d4c --- /dev/null +++ b/graph/src/components/subgraph/host.rs @@ -0,0 +1,172 @@ +use std::cmp::PartialEq; +use std::sync::Arc; +use std::time::Instant; + +use anyhow::Error; +use async_trait::async_trait; +use futures::sync::mpsc; + +use crate::components::store::SubgraphFork; +use crate::data_source::{ + DataSource, DataSourceTemplate, MappingTrigger, TriggerData, TriggerWithHandler, +}; +use crate::prelude::*; +use crate::{blockchain::Blockchain, components::subgraph::SharedProofOfIndexing}; +use crate::{components::metrics::HistogramVec, runtime::DeterministicHostError}; + +#[derive(Debug)] +pub enum MappingError { + /// A possible reorg was detected while running the mapping. + PossibleReorg(anyhow::Error), + Unknown(anyhow::Error), +} + +impl From for MappingError { + fn from(e: anyhow::Error) -> Self { + MappingError::Unknown(e) + } +} + +impl From for MappingError { + fn from(value: DeterministicHostError) -> MappingError { + MappingError::Unknown(value.inner()) + } +} + +impl MappingError { + pub fn context(self, s: String) -> Self { + use MappingError::*; + match self { + PossibleReorg(e) => PossibleReorg(e.context(s)), + Unknown(e) => Unknown(e.context(s)), + } + } +} + +/// Common trait for runtime host implementations. +#[async_trait] +pub trait RuntimeHost: Send + Sync + 'static { + fn data_source(&self) -> &DataSource; + + fn match_and_decode( + &self, + trigger: &TriggerData, + block: &Arc, + logger: &Logger, + ) -> Result>>, Error>; + + async fn process_mapping_trigger( + &self, + logger: &Logger, + block_ptr: BlockPtr, + trigger: TriggerWithHandler>, + state: BlockState, + proof_of_indexing: SharedProofOfIndexing, + debug_fork: &Option>, + ) -> Result, MappingError>; + + /// Block number in which this host was created. + /// Returns `None` for static data sources. + fn creation_block_number(&self) -> Option; +} + +pub struct HostMetrics { + handler_execution_time: Box, + host_fn_execution_time: Box, + pub stopwatch: StopwatchMetrics, +} + +impl HostMetrics { + pub fn new( + registry: Arc, + subgraph: &str, + stopwatch: StopwatchMetrics, + ) -> Self { + let handler_execution_time = registry + .new_deployment_histogram_vec( + "deployment_handler_execution_time", + "Measures the execution time for handlers", + subgraph, + vec![String::from("handler")], + vec![0.1, 0.5, 1.0, 10.0, 100.0], + ) + .expect("failed to create `deployment_handler_execution_time` histogram"); + let host_fn_execution_time = registry + .new_deployment_histogram_vec( + "deployment_host_fn_execution_time", + "Measures the execution time for host functions", + subgraph, + vec![String::from("host_fn_name")], + vec![0.025, 0.05, 0.2, 2.0, 8.0, 20.0], + ) + .expect("failed to create `deployment_host_fn_execution_time` histogram"); + Self { + handler_execution_time, + host_fn_execution_time, + stopwatch, + } + } + + pub fn observe_handler_execution_time(&self, duration: f64, handler: &str) { + self.handler_execution_time + .with_label_values(&[handler][..]) + .observe(duration); + } + + pub fn observe_host_fn_execution_time(&self, duration: f64, fn_name: &str) { + self.host_fn_execution_time + .with_label_values(&[fn_name][..]) + .observe(duration); + } + + pub fn time_host_fn_execution_region( + self: Arc, + fn_name: &'static str, + ) -> HostFnExecutionTimer { + HostFnExecutionTimer { + start: Instant::now(), + metrics: self, + fn_name, + } + } +} + +#[must_use] +pub struct HostFnExecutionTimer { + start: Instant, + metrics: Arc, + fn_name: &'static str, +} + +impl Drop for HostFnExecutionTimer { + fn drop(&mut self) { + let elapsed = (Instant::now() - self.start).as_secs_f64(); + self.metrics + .observe_host_fn_execution_time(elapsed, self.fn_name) + } +} + +pub trait RuntimeHostBuilder: Clone + Send + Sync + 'static { + type Host: RuntimeHost + PartialEq; + type Req: 'static + Send; + + /// Build a new runtime host for a subgraph data source. + fn build( + &self, + network_name: String, + subgraph_id: DeploymentHash, + data_source: DataSource, + top_level_templates: Arc>>, + mapping_request_sender: mpsc::Sender, + metrics: Arc, + ) -> Result; + + /// Spawn a mapping and return a channel for mapping requests. The sender should be able to be + /// cached and shared among mappings that use the same wasm file. + fn spawn_mapping( + raw_module: &[u8], + logger: Logger, + subgraph_id: DeploymentHash, + metrics: Arc, + ) -> Result, anyhow::Error>; +} diff --git a/graph/src/components/subgraph/instance.rs b/graph/src/components/subgraph/instance.rs new file mode 100644 index 0000000..ec35c82 --- /dev/null +++ b/graph/src/components/subgraph/instance.rs @@ -0,0 +1,107 @@ +use crate::{ + blockchain::Blockchain, + components::store::{EntityKey, ReadStore, StoredDynamicDataSource}, + data::subgraph::schema::SubgraphError, + data_source::DataSourceTemplate, + prelude::*, + util::lfu_cache::LfuCache, +}; + +#[derive(Clone, Debug)] +pub struct DataSourceTemplateInfo { + pub template: DataSourceTemplate, + pub params: Vec, + pub context: Option, + pub creation_block: BlockNumber, +} + +#[derive(Debug)] +pub struct BlockState { + pub entity_cache: EntityCache, + pub deterministic_errors: Vec, + created_data_sources: Vec>, + + // Data sources created in the current handler. + handler_created_data_sources: Vec>, + + // offchain data sources to be removed because they've been processed. + pub offchain_to_remove: Vec, + + // Marks whether a handler is currently executing. + in_handler: bool, +} + +impl BlockState { + pub fn new(store: impl ReadStore, lfu_cache: LfuCache>) -> Self { + BlockState { + entity_cache: EntityCache::with_current(Arc::new(store), lfu_cache), + deterministic_errors: Vec::new(), + created_data_sources: Vec::new(), + handler_created_data_sources: Vec::new(), + offchain_to_remove: Vec::new(), + in_handler: false, + } + } + + pub fn extend(&mut self, other: BlockState) { + assert!(!other.in_handler); + + let BlockState { + entity_cache, + deterministic_errors, + created_data_sources, + handler_created_data_sources, + offchain_to_remove, + in_handler, + } = self; + + match in_handler { + true => handler_created_data_sources.extend(other.created_data_sources), + false => created_data_sources.extend(other.created_data_sources), + } + deterministic_errors.extend(other.deterministic_errors); + entity_cache.extend(other.entity_cache); + offchain_to_remove.extend(other.offchain_to_remove); + } + + pub fn has_errors(&self) -> bool { + !self.deterministic_errors.is_empty() + } + + pub fn has_created_data_sources(&self) -> bool { + assert!(!self.in_handler); + !self.created_data_sources.is_empty() + } + + pub fn drain_created_data_sources(&mut self) -> Vec> { + assert!(!self.in_handler); + std::mem::take(&mut self.created_data_sources) + } + + pub fn enter_handler(&mut self) { + assert!(!self.in_handler); + self.in_handler = true; + self.entity_cache.enter_handler() + } + + pub fn exit_handler(&mut self) { + assert!(self.in_handler); + self.in_handler = false; + self.created_data_sources + .append(&mut self.handler_created_data_sources); + self.entity_cache.exit_handler() + } + + pub fn exit_handler_and_discard_changes_due_to_error(&mut self, e: SubgraphError) { + assert!(self.in_handler); + self.in_handler = false; + self.handler_created_data_sources.clear(); + self.entity_cache.exit_handler_and_discard_changes(); + self.deterministic_errors.push(e); + } + + pub fn push_created_data_source(&mut self, ds: DataSourceTemplateInfo) { + assert!(self.in_handler); + self.handler_created_data_sources.push(ds); + } +} diff --git a/graph/src/components/subgraph/instance_manager.rs b/graph/src/components/subgraph/instance_manager.rs new file mode 100644 index 0000000..3b1777e --- /dev/null +++ b/graph/src/components/subgraph/instance_manager.rs @@ -0,0 +1,20 @@ +use crate::prelude::BlockNumber; +use std::sync::Arc; + +use crate::components::store::DeploymentLocator; + +/// A `SubgraphInstanceManager` loads and manages subgraph instances. +/// +/// When a subgraph is added, the subgraph instance manager creates and starts +/// a subgraph instances for the subgraph. When a subgraph is removed, the +/// subgraph instance manager stops and removes the corresponding instance. +#[async_trait::async_trait] +pub trait SubgraphInstanceManager: Send + Sync + 'static { + async fn start_subgraph( + self: Arc, + deployment: DeploymentLocator, + manifest: serde_yaml::Mapping, + stop_block: Option, + ); + fn stop_subgraph(&self, deployment: DeploymentLocator); +} diff --git a/graph/src/components/subgraph/mod.rs b/graph/src/components/subgraph/mod.rs new file mode 100644 index 0000000..cd930c8 --- /dev/null +++ b/graph/src/components/subgraph/mod.rs @@ -0,0 +1,18 @@ +mod host; +mod instance; +mod instance_manager; +mod proof_of_indexing; +mod provider; +mod registrar; + +pub use crate::prelude::Entity; + +pub use self::host::{HostMetrics, MappingError, RuntimeHost, RuntimeHostBuilder}; +pub use self::instance::{BlockState, DataSourceTemplateInfo}; +pub use self::instance_manager::SubgraphInstanceManager; +pub use self::proof_of_indexing::{ + CausalityRegion, ProofOfIndexing, ProofOfIndexingEvent, ProofOfIndexingFinisher, + ProofOfIndexingVersion, SharedProofOfIndexing, +}; +pub use self::provider::SubgraphAssignmentProvider; +pub use self::registrar::{SubgraphRegistrar, SubgraphVersionSwitchingMode}; diff --git a/graph/src/components/subgraph/proof_of_indexing/event.rs b/graph/src/components/subgraph/proof_of_indexing/event.rs new file mode 100644 index 0000000..06d4e9f --- /dev/null +++ b/graph/src/components/subgraph/proof_of_indexing/event.rs @@ -0,0 +1,148 @@ +use crate::prelude::{impl_slog_value, Value}; +use stable_hash_legacy::StableHasher; +use std::collections::{BTreeMap, HashMap}; +use std::fmt; +use strum::AsStaticRef as _; +use strum_macros::AsStaticStr; + +#[derive(AsStaticStr)] +pub enum ProofOfIndexingEvent<'a> { + /// For when an entity is removed from the store. + RemoveEntity { entity_type: &'a str, id: &'a str }, + /// For when an entity is set into the store. + SetEntity { + entity_type: &'a str, + id: &'a str, + data: &'a HashMap, + }, + /// For when a deterministic error has happened. + /// + /// The number of redacted events covers the events previous to this one + /// which are no longer transacted to the database. The property that we + /// want to maintain is that no two distinct databases share the same PoI. + /// Since there is no event for the beginning of a handler the + /// non-fatal-errors feature creates an ambiguity without this field. This + /// is best illustrated by example. Consider: + /// 1. Start handler + /// 1. Save Entity A + /// 2. Start handler + /// 2. Save Entity B + /// 3. Save Entity C + /// 4. Deterministic Error + /// + /// The Deterministic Error redacts the effect of 2.1 and 2.2 since entity B + /// and C are not saved to the database. + /// + /// Without the redacted events field, this results in the following event + /// stream for the PoI: [Save(A), Save(B), Save(C), DeterministicError] + /// + /// But, an equivalent PoI would be generated with this sequence of events: + /// 1. Start handler + /// 1. Save Entity A + /// 2. Save Entity B + /// 2. Start handler + /// 1. Save Entity C + /// 2. Deterministic Error + /// + /// The databases would be different even though the PoI is the same. (The + /// first database in [A] and the second is [A, B]) + /// + /// By emitting the number of redacted events we get a different PoI for + /// different databases because the PoIs become: + /// + /// [Save(A), Save(B), Save(C), DeterministicError(2)] + /// + /// [Save(A), Save(B), Save(C), DeterministicError(1)] + /// + /// for the first and second cases respectively. + DeterministicError { redacted_events: u64 }, +} + +impl stable_hash_legacy::StableHash for ProofOfIndexingEvent<'_> { + fn stable_hash(&self, mut sequence_number: H::Seq, state: &mut H) { + use stable_hash_legacy::prelude::*; + use ProofOfIndexingEvent::*; + + self.as_static() + .stable_hash(sequence_number.next_child(), state); + match self { + RemoveEntity { entity_type, id } => { + entity_type.stable_hash(sequence_number.next_child(), state); + id.stable_hash(sequence_number.next_child(), state); + } + SetEntity { + entity_type, + id, + data, + } => { + entity_type.stable_hash(sequence_number.next_child(), state); + id.stable_hash(sequence_number.next_child(), state); + data.stable_hash(sequence_number.next_child(), state); + } + DeterministicError { redacted_events } => { + redacted_events.stable_hash(sequence_number.next_child(), state) + } + } + } +} + +impl stable_hash::StableHash for ProofOfIndexingEvent<'_> { + fn stable_hash(&self, field_address: H::Addr, state: &mut H) { + use stable_hash::prelude::*; + + let variant = match self { + Self::RemoveEntity { entity_type, id } => { + entity_type.stable_hash(field_address.child(0), state); + id.stable_hash(field_address.child(1), state); + 1 + } + Self::SetEntity { + entity_type, + id, + data, + } => { + entity_type.stable_hash(field_address.child(0), state); + id.stable_hash(field_address.child(1), state); + data.stable_hash(field_address.child(2), state); + 2 + } + Self::DeterministicError { redacted_events } => { + redacted_events.stable_hash(field_address.child(0), state); + 3 + } + }; + + state.write(field_address, &[variant]); + } +} + +/// Different than #[derive(Debug)] in order to be deterministic so logs can be +/// diffed easily. In particular, we swap out the HashMap for a BTreeMap when +/// printing the data field of the SetEntity variant so that the keys are +/// sorted. +impl fmt::Debug for ProofOfIndexingEvent<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut builder = f.debug_struct(self.as_static()); + match self { + Self::RemoveEntity { entity_type, id } => { + builder.field("entity_type", entity_type); + builder.field("id", id); + } + Self::SetEntity { + entity_type, + id, + data, + } => { + builder.field("entity_type", entity_type); + builder.field("id", id); + builder.field("data", &data.iter().collect::>()); + } + Self::DeterministicError { redacted_events } => { + builder.field("redacted_events", redacted_events); + } + } + builder.finish() + } +} + +impl_slog_value!(ProofOfIndexingEvent<'_>, "{:?}"); diff --git a/graph/src/components/subgraph/proof_of_indexing/mod.rs b/graph/src/components/subgraph/proof_of_indexing/mod.rs new file mode 100644 index 0000000..712b84d --- /dev/null +++ b/graph/src/components/subgraph/proof_of_indexing/mod.rs @@ -0,0 +1,318 @@ +mod event; +mod online; +mod reference; + +pub use event::ProofOfIndexingEvent; +pub use online::{ProofOfIndexing, ProofOfIndexingFinisher}; +pub use reference::CausalityRegion; + +use atomic_refcell::AtomicRefCell; +use std::sync::Arc; + +#[derive(Copy, Clone, Debug)] +pub enum ProofOfIndexingVersion { + Fast, + Legacy, +} + +/// This concoction of types is to allow MappingContext to be static, yet still +/// have shared mutable data for derive_with_empty_block_state. The static +/// requirement is so that host exports can be static for wasmtime. +/// AtomicRefCell is chosen over Mutex because concurrent access is +/// intentionally disallowed - PoI requires sequential access to the hash +/// function within a given causality region even if ownership is shared across +/// multiple mapping contexts. +/// +/// The Option<_> is because not all subgraphs support PoI until re-deployed. +/// Eventually this can be removed. +/// +/// This is not a great place to define this type, since the ProofOfIndexing +/// shouldn't "know" these details about wasmtime and subgraph re-deployments, +/// but the APIs that would make use of this are in graph/components so this +/// lives here for lack of a better choice. +pub type SharedProofOfIndexing = Option>>; + +#[cfg(test)] +mod tests { + use super::*; + use crate::prelude::{BlockPtr, DeploymentHash, Value}; + use maplit::hashmap; + use online::ProofOfIndexingFinisher; + use reference::*; + use slog::{o, Discard, Logger}; + use stable_hash::{fast_stable_hash, utils::check_for_child_errors}; + use stable_hash_legacy::crypto::SetHasher; + use stable_hash_legacy::utils::stable_hash as stable_hash_legacy; + use std::collections::HashMap; + use std::convert::TryInto; + use web3::types::{Address, H256}; + + /// Verify that the stable hash of a reference and online implementation match + fn check(case: Case, cache: &mut HashMap) { + let logger = Logger::root(Discard, o!()); + + // Does a sanity check to ensure that the schema itself is correct, + // which is separate to verifying that the online/offline version + // return the same result. + check_for_child_errors(&case.data).expect("Found child errors"); + + let offline_fast = tiny_keccak::keccak256(&fast_stable_hash(&case.data).to_le_bytes()); + let offline_legacy = stable_hash_legacy::(&case.data); + + for (version, offline, hardcoded) in [ + (ProofOfIndexingVersion::Legacy, offline_legacy, case.legacy), + (ProofOfIndexingVersion::Fast, offline_fast, case.fast), + ] { + // The code is meant to approximate what happens during indexing as + // close as possible. The API for the online PoI is meant to be + // pretty foolproof so that the actual usage will also match. + + // Create a database which stores intermediate PoIs + let mut db = HashMap::>::new(); + + let mut block_count = 1; + for causality_region in case.data.causality_regions.values() { + block_count = causality_region.blocks.len(); + break; + } + + for block_i in 0..block_count { + let mut stream = ProofOfIndexing::new(block_i.try_into().unwrap(), version); + + for (name, region) in case.data.causality_regions.iter() { + let block = ®ion.blocks[block_i]; + + for evt in block.events.iter() { + stream.write(&logger, name, evt); + } + } + + for (name, region) in stream.take() { + let prev = db.get(&name); + let update = region.pause(prev.map(|v| &v[..])); + db.insert(name, update); + } + } + + let block_number = (block_count - 1) as u64; + let block_ptr = BlockPtr::from((case.data.block_hash, block_number)); + + // This region emulates the request + let mut finisher = ProofOfIndexingFinisher::new( + &block_ptr, + &case.data.subgraph_id, + &case.data.indexer, + version, + ); + for (name, region) in db.iter() { + finisher.add_causality_region(name, region); + } + + let online = hex::encode(finisher.finish()); + let offline = hex::encode(&offline); + assert_eq!(&online, &offline); + assert_eq!(&online, hardcoded); + + if let Some(prev) = cache.insert(offline, case.name) { + panic!("Found conflict for case: {} == {}", case.name, prev); + } + } + } + + struct Case<'a> { + name: &'static str, + legacy: &'static str, + fast: &'static str, + data: PoI<'a>, + } + + /// This test checks that each case resolves to a unique hash, and that + /// in each case the reference and online versions match + #[test] + fn online_vs_reference() { + let data = hashmap! { + "val".to_owned() => Value::Int(1) + }; + let data_empty = hashmap! {}; + let data2 = hashmap! { + "key".to_owned() => Value::String("s".to_owned()), + "null".to_owned() => Value::Null, + }; + + let mut cases = vec![ + // Simple case of basically nothing + Case { + name: "genesis", + legacy: "401e5bef572bc3a56b0ced0eb6cb4619d2ca748db6af8855828d16ff3446cfdd", + fast: "dced49c45eac68e8b3d8f857928e7be6c270f2db8b56b0d7f27ce725100bae01", + data: PoI { + subgraph_id: DeploymentHash::new("test").unwrap(), + block_hash: H256::repeat_byte(1), + causality_regions: HashMap::new(), + indexer: None, + }, + }, + // Add an event + Case { + name: "one_event", + legacy: "9241634bfc8a9a12c796a0de6f326326a74967cd477ee7ce78fbac20a9e9c303", + fast: "bb3c37659d4bc799b9dcf3d17b1b1e93847f5fc0b2c50ee6a27f13b5c07f7e97", + data: PoI { + subgraph_id: DeploymentHash::new("test").unwrap(), + block_hash: H256::repeat_byte(1), + causality_regions: hashmap! { + "eth".to_owned() => CausalityRegion { + blocks: vec! [ + Block::default(), + Block { + events: vec![ + ProofOfIndexingEvent::SetEntity { + entity_type: "t", + id: "id", + data: &data_empty, + } + ] + } + ], + }, + }, + indexer: Some(Address::repeat_byte(1)), + }, + }, + // Try adding a couple more blocks, including an empty block on the end + Case { + name: "multiple_blocks", + legacy: "775fa30bbaef2a8659456a317923a36f46e3715e6c9cf43203dd3486af4e361f", + fast: "3bb882049e8f4a11cd4a7a005c6ce3b3c779a0e90057a9556c595660e626268d", + data: PoI { + subgraph_id: DeploymentHash::new("b").unwrap(), + block_hash: H256::repeat_byte(3), + causality_regions: hashmap! { + "eth".to_owned() => CausalityRegion { + blocks: vec! [ + Block::default(), + Block { + events: vec![ + ProofOfIndexingEvent::SetEntity { + entity_type: "type", + id: "id", + data: &data, + } + ] + }, + Block::default(), + Block { + events: vec![ + ProofOfIndexingEvent::SetEntity { + entity_type: "type", + id: "id", + data: &data_empty, + } + ] + }, + Block::default(), + ], + }, + }, + indexer: Some(Address::repeat_byte(1)), + }, + }, + // Try adding another causality region + Case { + name: "causality_regions", + legacy: "13e6fd2b581911c80d935d4f098b40ef3d87cbc564b5a635c81b06091a381e54", + fast: "b2cb70acd4a1337a67df810fe4c5c2fb3d3a3b2b8eb137dbb592bd6014869362", + data: PoI { + subgraph_id: DeploymentHash::new("b").unwrap(), + block_hash: H256::repeat_byte(3), + causality_regions: hashmap! { + "eth".to_owned() => CausalityRegion { + blocks: vec! [ + Block::default(), + Block { + events: vec![ + ProofOfIndexingEvent::SetEntity { + entity_type: "type", + id: "id", + data: &data2, + } + ] + }, + Block::default(), + Block { + events: vec![ + ProofOfIndexingEvent::RemoveEntity { + entity_type: "type", + id: "id", + } + ] + }, + Block::default(), + ], + }, + "ipfs".to_owned() => CausalityRegion { + blocks: vec! [ + Block::default(), + Block { + events: vec![ + ProofOfIndexingEvent::SetEntity { + entity_type: "type", + id: "id", + data: &data, + } + ] + }, + Block::default(), + Block { + events: vec![ + ProofOfIndexingEvent::SetEntity { + entity_type: "type", + id: "id", + data: &data, + } + ] + }, + Block::default(), + ], + }, + }, + indexer: Some(Address::repeat_byte(1)), + }, + }, + // Back to the one event case, but try adding some data. + Case { + name: "data", + legacy: "cd3020511cf4c88dd2be542aca4f95bb2a67b06e29f444bcdf44009933b8ff31", + fast: "a992ba24702615a3f591014f7351acf85a35b75e1f8646fc8d77509c4b5d31ed", + data: PoI { + subgraph_id: DeploymentHash::new("test").unwrap(), + block_hash: H256::repeat_byte(1), + causality_regions: hashmap! { + "eth".to_owned() => CausalityRegion { + blocks: vec! [ + Block::default(), + Block { + events: vec![ + ProofOfIndexingEvent::SetEntity { + entity_type: "type", + id: "id", + data: &data, + } + ] + } + ], + }, + }, + indexer: Some(Address::repeat_byte(4)), + }, + }, + ]; + + // Lots of data up there ⬆️ to test. Finally, loop over each case, comparing the reference and + // online version, then checking that there are no conflicts for the reference versions. + let mut results = HashMap::new(); + for case in cases.drain(..) { + check(case, &mut results); + } + } +} diff --git a/graph/src/components/subgraph/proof_of_indexing/online.rs b/graph/src/components/subgraph/proof_of_indexing/online.rs new file mode 100644 index 0000000..013f2a8 --- /dev/null +++ b/graph/src/components/subgraph/proof_of_indexing/online.rs @@ -0,0 +1,316 @@ +//! This is an online (streaming) implementation of the reference implementation +//! Any hash constructed from here should be the same as if the same data was given +//! to the reference implementation, but this is updated incrementally + +use super::{ProofOfIndexingEvent, ProofOfIndexingVersion}; +use crate::{ + blockchain::BlockPtr, + prelude::{debug, BlockNumber, DeploymentHash, Logger, ENV_VARS}, + util::stable_hash_glue::AsBytes, +}; +use stable_hash::{fast::FastStableHasher, FieldAddress, StableHash, StableHasher}; +use stable_hash_legacy::crypto::{Blake3SeqNo, SetHasher}; +use stable_hash_legacy::prelude::{ + StableHash as StableHashLegacy, StableHasher as StableHasherLegacy, *, +}; +use std::collections::HashMap; +use std::convert::TryInto; +use std::fmt; +use web3::types::Address; + +pub struct BlockEventStream { + vec_length: u64, + handler_start: u64, + block_index: u64, + hasher: Hashers, +} + +enum Hashers { + Fast(FastStableHasher), + Legacy(SetHasher), +} + +impl Hashers { + fn new(version: ProofOfIndexingVersion) -> Self { + match version { + ProofOfIndexingVersion::Legacy => Hashers::Legacy(SetHasher::new()), + ProofOfIndexingVersion::Fast => Hashers::Fast(FastStableHasher::new()), + } + } + + fn from_bytes(bytes: &[u8]) -> Self { + match bytes.try_into() { + Ok(bytes) => Hashers::Fast(FastStableHasher::from_bytes(bytes)), + Err(_) => Hashers::Legacy(SetHasher::from_bytes(bytes)), + } + } + + fn write(&mut self, value: &T, children: &[u64]) + where + T: StableHash + StableHashLegacy, + { + match self { + Hashers::Fast(fast) => { + let addr = children.iter().fold(u128::root(), |s, i| s.child(*i)); + StableHash::stable_hash(value, addr, fast); + } + Hashers::Legacy(legacy) => { + let seq_no = traverse_seq_no(children); + StableHashLegacy::stable_hash(value, seq_no, legacy); + } + } + } +} + +/// Go directly to a SequenceNumber identifying a field within a struct. +/// This is best understood by example. Consider the struct: +/// +/// struct Outer { +/// inners: Vec, +/// outer_num: i32 +/// } +/// struct Inner { +/// inner_num: i32, +/// inner_str: String, +/// } +/// +/// Let's say that we have the following data: +/// Outer { +/// inners: vec![ +/// Inner { +/// inner_num: 10, +/// inner_str: "THIS", +/// }, +/// ], +/// outer_num: 0, +/// } +/// +/// And we need to identify the string "THIS", at outer.inners[0].inner_str; +/// This would require the following: +/// traverse_seq_no(&[ +/// 0, // Outer.inners +/// 0, // Vec[0] +/// 1, // Inner.inner_str +///]) +// Performance: Could write a specialized function for this, avoiding a bunch of clones of Blake3SeqNo +fn traverse_seq_no(counts: &[u64]) -> Blake3SeqNo { + counts.iter().fold(Blake3SeqNo::root(), |mut s, i| { + s.skip(*i as usize); + s.next_child() + }) +} + +impl BlockEventStream { + fn new(block_number: BlockNumber, version: ProofOfIndexingVersion) -> Self { + let block_index: u64 = block_number.try_into().unwrap(); + + Self { + vec_length: 0, + handler_start: 0, + block_index, + hasher: Hashers::new(version), + } + } + + /// Finishes the current block and returns the serialized hash function to + /// be resumed later. Cases in which the hash function is resumed include + /// when asking for the final PoI, or when combining with the next modified + /// block via the argument `prev` + pub fn pause(mut self, prev: Option<&[u8]>) -> Vec { + self.hasher + .write(&self.vec_length, &[1, 0, self.block_index, 0]); + match self.hasher { + Hashers::Legacy(mut digest) => { + if let Some(prev) = prev { + let prev = SetHasher::from_bytes(prev); + // SequenceNumber::root() is misleading here since the parameter + // is unused. + digest.finish_unordered(prev, SequenceNumber::root()); + } + digest.to_bytes() + } + Hashers::Fast(mut digest) => { + if let Some(prev) = prev { + let prev = prev + .try_into() + .expect("Expected valid fast stable hash representation"); + let prev = FastStableHasher::from_bytes(prev); + digest.mixin(&prev); + } + digest.to_bytes().to_vec() + } + } + } + + fn write(&mut self, event: &ProofOfIndexingEvent<'_>) { + let children = &[ + 1, // kvp -> v + 0, // CausalityRegion.blocks: Vec + self.block_index, // Vec -> [i] + 0, // Block.events -> Vec + self.vec_length, + ]; + self.hasher.write(&event, children); + self.vec_length += 1; + } + + fn start_handler(&mut self) { + self.handler_start = self.vec_length; + } +} + +pub struct ProofOfIndexing { + version: ProofOfIndexingVersion, + block_number: BlockNumber, + /// The POI is updated for each data source independently. This is necessary because + /// some data sources (eg: IPFS files) may be unreliable and therefore cannot mix + /// state with other data sources. This may also give us some freedom to change + /// the order of triggers in the future. + per_causality_region: HashMap, +} + +impl fmt::Debug for ProofOfIndexing { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("ProofOfIndexing").field(&"...").finish() + } +} + +impl ProofOfIndexing { + pub fn new(block_number: BlockNumber, version: ProofOfIndexingVersion) -> Self { + Self { + version, + block_number, + per_causality_region: HashMap::new(), + } + } +} + +impl ProofOfIndexing { + pub fn write_deterministic_error(&mut self, logger: &Logger, causality_region: &str) { + let redacted_events = self.with_causality_region(causality_region, |entry| { + entry.vec_length - entry.handler_start + }); + + self.write( + logger, + causality_region, + &ProofOfIndexingEvent::DeterministicError { redacted_events }, + ) + } + + /// Adds an event to the digest of the ProofOfIndexingStream local to the causality region + pub fn write( + &mut self, + logger: &Logger, + causality_region: &str, + event: &ProofOfIndexingEvent<'_>, + ) { + if ENV_VARS.log_poi_events { + debug!( + logger, + "Proof of indexing event"; + "event" => &event, + "causality_region" => causality_region + ); + } + + self.with_causality_region(causality_region, |entry| entry.write(event)) + } + + pub fn start_handler(&mut self, causality_region: &str) { + self.with_causality_region(causality_region, |entry| entry.start_handler()) + } + + // This is just here because the raw_entry API is not stabilized. + fn with_causality_region(&mut self, causality_region: &str, f: F) -> T + where + F: FnOnce(&mut BlockEventStream) -> T, + { + if let Some(causality_region) = self.per_causality_region.get_mut(causality_region) { + f(causality_region) + } else { + let mut entry = BlockEventStream::new(self.block_number, self.version); + let result = f(&mut entry); + self.per_causality_region + .insert(causality_region.to_owned(), entry); + result + } + } + + pub fn take(self) -> HashMap { + self.per_causality_region + } +} + +pub struct ProofOfIndexingFinisher { + block_number: BlockNumber, + state: Hashers, + causality_count: usize, +} + +impl ProofOfIndexingFinisher { + pub fn new( + block: &BlockPtr, + subgraph_id: &DeploymentHash, + indexer: &Option
, + version: ProofOfIndexingVersion, + ) -> Self { + let mut state = Hashers::new(version); + + // Add PoI.subgraph_id + state.write(&subgraph_id, &[1]); + + // Add PoI.block_hash + state.write(&AsBytes(block.hash_slice()), &[2]); + + // Add PoI.indexer + state.write(&indexer.as_ref().map(|i| AsBytes(i.as_bytes())), &[3]); + + ProofOfIndexingFinisher { + block_number: block.number, + state, + causality_count: 0, + } + } + + pub fn add_causality_region(&mut self, name: &str, region: &[u8]) { + let mut state = Hashers::from_bytes(region); + + // Finish the blocks vec by writing kvp[v], CausalityRegion.blocks.len() + // + 1 is to account that the length of the blocks array for the genesis block is 1, not 0. + state.write(&(self.block_number + 1), &[1, 0]); + + // Add the name (kvp[k]). + state.write(&name, &[0]); + + // Mixin the region into PoI.causality_regions. + match state { + Hashers::Legacy(legacy) => { + let state = legacy.finish(); + self.state.write(&AsBytes(&state), &[0, 1]); + } + Hashers::Fast(fast) => { + let state = fast.to_bytes(); + self.state.write(&AsBytes(&state), &[0]); + } + } + + self.causality_count += 1; + } + + pub fn finish(mut self) -> [u8; 32] { + if let Hashers::Legacy(_) = self.state { + // Add PoI.causality_regions.len() + // Note that technically to get the same sequence number one would need + // to call causality_regions_count_seq_no.skip(self.causality_count); + // but it turns out that the result happens to be the same for + // non-negative numbers. + self.state.write(&self.causality_count, &[0, 2]); + } + + match self.state { + Hashers::Legacy(legacy) => legacy.finish(), + Hashers::Fast(fast) => tiny_keccak::keccak256(&fast.finish().to_le_bytes()), + } + } +} diff --git a/graph/src/components/subgraph/proof_of_indexing/reference.rs b/graph/src/components/subgraph/proof_of_indexing/reference.rs new file mode 100644 index 0000000..63d9703 --- /dev/null +++ b/graph/src/components/subgraph/proof_of_indexing/reference.rs @@ -0,0 +1,51 @@ +use super::ProofOfIndexingEvent; +use crate::prelude::DeploymentHash; +use crate::util::stable_hash_glue::{impl_stable_hash, AsBytes}; +use std::collections::HashMap; +use web3::types::{Address, H256}; + +/// The PoI is the StableHash of this struct. This reference implementation is +/// mostly here just to make sure that the online implementation is +/// well-implemented (without conflicting sequence numbers, or other oddities). +/// It's just way easier to check that this works, and serves as a kind of +/// documentation as a side-benefit. +pub struct PoI<'a> { + pub causality_regions: HashMap>, + pub subgraph_id: DeploymentHash, + pub block_hash: H256, + pub indexer: Option
, +} + +fn h256_as_bytes(val: &H256) -> AsBytes<&[u8]> { + AsBytes(val.as_bytes()) +} + +fn indexer_opt_as_bytes(val: &Option
) -> Option> { + val.as_ref().map(|v| AsBytes(v.as_bytes())) +} + +impl_stable_hash!(PoI<'_> { + causality_regions, + subgraph_id, + block_hash: h256_as_bytes, + indexer: indexer_opt_as_bytes +}); + +pub struct CausalityRegion<'a> { + pub blocks: Vec>, +} + +impl_stable_hash!(CausalityRegion<'_> {blocks}); + +impl CausalityRegion<'_> { + pub fn from_network(network: &str) -> String { + format!("ethereum/{}", network) + } +} + +#[derive(Default)] +pub struct Block<'a> { + pub events: Vec>, +} + +impl_stable_hash!(Block<'_> {events}); diff --git a/graph/src/components/subgraph/provider.rs b/graph/src/components/subgraph/provider.rs new file mode 100644 index 0000000..5edc223 --- /dev/null +++ b/graph/src/components/subgraph/provider.rs @@ -0,0 +1,17 @@ +use async_trait::async_trait; + +use crate::{components::store::DeploymentLocator, prelude::*}; + +/// Common trait for subgraph providers. +#[async_trait] +pub trait SubgraphAssignmentProvider: Send + Sync + 'static { + async fn start( + &self, + deployment: DeploymentLocator, + stop_block: Option, + ) -> Result<(), SubgraphAssignmentProviderError>; + async fn stop( + &self, + deployment: DeploymentLocator, + ) -> Result<(), SubgraphAssignmentProviderError>; +} diff --git a/graph/src/components/subgraph/registrar.rs b/graph/src/components/subgraph/registrar.rs new file mode 100644 index 0000000..cfb2c2f --- /dev/null +++ b/graph/src/components/subgraph/registrar.rs @@ -0,0 +1,56 @@ +use std::str::FromStr; + +use async_trait::async_trait; + +use crate::{components::store::DeploymentLocator, prelude::*}; + +#[derive(Clone, Copy, Debug)] +pub enum SubgraphVersionSwitchingMode { + Instant, + Synced, +} + +impl SubgraphVersionSwitchingMode { + pub fn parse(mode: &str) -> Self { + Self::from_str(mode).unwrap() + } +} + +impl FromStr for SubgraphVersionSwitchingMode { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_ascii_lowercase().as_str() { + "instant" => Ok(SubgraphVersionSwitchingMode::Instant), + "synced" => Ok(SubgraphVersionSwitchingMode::Synced), + _ => Err(format!("invalid version switching mode: {:?}", s)), + } + } +} + +/// Common trait for subgraph registrars. +#[async_trait] +pub trait SubgraphRegistrar: Send + Sync + 'static { + async fn create_subgraph( + &self, + name: SubgraphName, + ) -> Result; + + async fn create_subgraph_version( + &self, + name: SubgraphName, + hash: DeploymentHash, + assignment_node_id: NodeId, + debug_fork: Option, + start_block_block: Option, + graft_block_override: Option, + ) -> Result; + + async fn remove_subgraph(&self, name: SubgraphName) -> Result<(), SubgraphRegistrarError>; + + async fn reassign_subgraph( + &self, + hash: &DeploymentHash, + node_id: &NodeId, + ) -> Result<(), SubgraphRegistrarError>; +} diff --git a/graph/src/components/transaction_receipt.rs b/graph/src/components/transaction_receipt.rs new file mode 100644 index 0000000..30b2bb2 --- /dev/null +++ b/graph/src/components/transaction_receipt.rs @@ -0,0 +1,39 @@ +//! Code for retrieving transaction receipts from the database. +//! +//! This module exposes the [`LightTransactionReceipt`] type, which holds basic information about +//! the retrieved transaction receipts. + +use web3::types::{TransactionReceipt, H256, U256, U64}; + +/// Like web3::types::Receipt, but with fewer fields. +#[derive(Debug, PartialEq)] +pub struct LightTransactionReceipt { + pub transaction_hash: H256, + pub transaction_index: U64, + pub block_hash: Option, + pub block_number: Option, + pub gas_used: Option, + pub status: Option, +} + +impl From for LightTransactionReceipt { + fn from(receipt: TransactionReceipt) -> Self { + let TransactionReceipt { + transaction_hash, + transaction_index, + block_hash, + block_number, + gas_used, + status, + .. + } = receipt; + LightTransactionReceipt { + transaction_hash, + transaction_index, + block_hash, + block_number, + gas_used, + status, + } + } +} diff --git a/graph/src/components/trigger_processor.rs b/graph/src/components/trigger_processor.rs new file mode 100644 index 0000000..ce02a21 --- /dev/null +++ b/graph/src/components/trigger_processor.rs @@ -0,0 +1,31 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use slog::Logger; + +use crate::{blockchain::Blockchain, data_source::TriggerData, prelude::SubgraphInstanceMetrics}; + +use super::{ + store::SubgraphFork, + subgraph::{BlockState, MappingError, RuntimeHostBuilder, SharedProofOfIndexing}, +}; + +#[async_trait] +pub trait TriggerProcessor: Sync + Send +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + async fn process_trigger( + &self, + logger: &Logger, + hosts: &[Arc], + block: &Arc, + trigger: &TriggerData, + mut state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &str, + debug_fork: &Option>, + subgraph_metrics: &Arc, + ) -> Result, MappingError>; +} diff --git a/graph/src/components/versions/features.rs b/graph/src/components/versions/features.rs new file mode 100644 index 0000000..5d3b377 --- /dev/null +++ b/graph/src/components/versions/features.rs @@ -0,0 +1,2 @@ +#[derive(Clone, PartialEq, Eq, Debug, Ord, PartialOrd, Hash)] +pub enum FeatureFlag {} diff --git a/graph/src/components/versions/mod.rs b/graph/src/components/versions/mod.rs new file mode 100644 index 0000000..675e12e --- /dev/null +++ b/graph/src/components/versions/mod.rs @@ -0,0 +1,5 @@ +mod features; +mod registry; + +pub use features::FeatureFlag; +pub use registry::{ApiVersion, VERSIONS}; diff --git a/graph/src/components/versions/registry.rs b/graph/src/components/versions/registry.rs new file mode 100644 index 0000000..d365e90 --- /dev/null +++ b/graph/src/components/versions/registry.rs @@ -0,0 +1,71 @@ +use crate::prelude::FeatureFlag; +use itertools::Itertools; +use lazy_static::lazy_static; +use semver::{Version, VersionReq}; +use std::collections::HashMap; + +lazy_static! { + static ref VERSION_COLLECTION: HashMap> = { + vec![ + // baseline version + (Version::new(1, 0, 0), vec![]), + ].into_iter().collect() + }; + + // Sorted vector of versions. From higher to lower. + pub static ref VERSIONS: Vec<&'static Version> = { + let mut versions = VERSION_COLLECTION.keys().collect_vec().clone(); + versions.sort_by(|a, b| b.partial_cmp(a).unwrap()); + versions + }; +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ApiVersion { + pub version: Version, + features: Vec, +} + +impl ApiVersion { + pub fn new(version_requirement: &VersionReq) -> Result { + let version = Self::resolve(&version_requirement)?; + + Ok(Self { + version: version.clone(), + features: VERSION_COLLECTION + .get(&version) + .expect(format!("Version {:?} is not supported", version).as_str()) + .to_vec(), + }) + } + + pub fn from_version(version: &Version) -> Result { + ApiVersion::new( + &VersionReq::parse(version.to_string().as_str()) + .map_err(|error| format!("Invalid version requirement: {}", error))?, + ) + } + + pub fn supports(&self, feature: FeatureFlag) -> bool { + self.features.contains(&feature) + } + + fn resolve(version_requirement: &VersionReq) -> Result<&Version, String> { + for version in VERSIONS.iter() { + if version_requirement.matches(version) { + return Ok(version.clone()); + } + } + + Err("Could not resolve the version".to_string()) + } +} + +impl Default for ApiVersion { + fn default() -> Self { + // Default to the latest version. + // The `VersionReq::default()` returns `*` which means "any version". + // The first matching version is the latest version. + ApiVersion::new(&VersionReq::default()).unwrap() + } +} diff --git a/graph/src/data/graphql/effort.rs b/graph/src/data/graphql/effort.rs new file mode 100644 index 0000000..23753a2 --- /dev/null +++ b/graph/src/data/graphql/effort.rs @@ -0,0 +1,504 @@ +//! Utilities to keep moving statistics about queries + +use prometheus::core::GenericCounter; +use rand::{prelude::Rng, thread_rng}; +use std::collections::{HashMap, HashSet}; +use std::iter::FromIterator; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; + +use crate::components::metrics::{Counter, Gauge, MetricsRegistry}; +use crate::components::store::PoolWaitStats; +use crate::data::graphql::shape_hash::shape_hash; +use crate::data::query::{CacheStatus, QueryExecutionError}; +use crate::prelude::q; +use crate::prelude::{async_trait, debug, info, o, warn, Logger, QueryLoadManager, ENV_VARS}; +use crate::util::stats::MovingStats; + +struct QueryEffort { + inner: Arc>, +} + +/// Track the effort for queries (identified by their ShapeHash) over a +/// time window. +struct QueryEffortInner { + window_size: Duration, + bin_size: Duration, + effort: HashMap, + total: MovingStats, +} + +/// Create a `QueryEffort` that uses the window and bin sizes configured in +/// the environment +impl Default for QueryEffort { + fn default() -> Self { + Self::new(ENV_VARS.load_window_size, ENV_VARS.load_bin_size) + } +} + +impl QueryEffort { + pub fn new(window_size: Duration, bin_size: Duration) -> Self { + Self { + inner: Arc::new(RwLock::new(QueryEffortInner::new(window_size, bin_size))), + } + } + + pub fn add(&self, shape_hash: u64, duration: Duration, gauge: &Gauge) { + let mut inner = self.inner.write().unwrap(); + inner.add(shape_hash, duration); + gauge.set(inner.total.average().unwrap_or(Duration::ZERO).as_millis() as f64); + } + + /// Return what we know right now about the effort for the query + /// `shape_hash`, and about the total effort. If we have no measurements + /// at all, return `ZERO_DURATION` as the total effort. If we have no + /// data for the particular query, return `None` as the effort + /// for the query + pub fn current_effort(&self, shape_hash: u64) -> (Option, Duration) { + let inner = self.inner.read().unwrap(); + let total_effort = inner.total.duration(); + let query_effort = inner.effort.get(&shape_hash).map(|stats| stats.duration()); + (query_effort, total_effort) + } +} + +impl QueryEffortInner { + fn new(window_size: Duration, bin_size: Duration) -> Self { + Self { + window_size, + bin_size, + effort: HashMap::default(), + total: MovingStats::new(window_size, bin_size), + } + } + + fn add(&mut self, shape_hash: u64, duration: Duration) { + let window_size = self.window_size; + let bin_size = self.bin_size; + let now = Instant::now(); + self.effort + .entry(shape_hash) + .or_insert_with(|| MovingStats::new(window_size, bin_size)) + .add_at(now, duration); + self.total.add_at(now, duration); + } +} + +/// What to log about the state we are currently in +enum KillStateLogEvent { + /// Overload is starting right now + Start, + /// Overload has been going on for the duration + Ongoing(Duration), + /// No longer overloaded, reducing the kill_rate + Settling, + /// Overload was resolved after duration time + Resolved(Duration), + /// Don't log anything right now + Skip, +} + +struct KillState { + // A value between 0 and 1, where 0 means 'respond to all queries' + // and 1 means 'do not respond to any queries' + kill_rate: f64, + // We adjust the `kill_rate` at most every `KILL_RATE_UPDATE_INTERVAL` + last_update: Instant, + // When the current overload situation started + overload_start: Option, + // Throttle logging while we are overloaded to no more often than + // once every 30s + last_overload_log: Instant, +} + +impl KillState { + fn new() -> Self { + // Set before to an instant long enough ago so that we don't + // immediately log or adjust the kill rate if the node is already + // under load. Unfortunately, on OSX, `Instant` measures time from + // the last boot, and if that was less than 60s ago, we can't + // subtract 60s from `now`. Since the worst that can happen if + // we set `before` to `now` is that we might log more than strictly + // necessary, and adjust the kill rate one time too often right after + // node start, it is acceptable to fall back to `now` + let before = { + let long_ago = Duration::from_secs(60); + let now = Instant::now(); + now.checked_sub(long_ago).unwrap_or(now) + }; + Self { + kill_rate: 0.0, + last_update: before, + overload_start: None, + last_overload_log: before, + } + } + + fn log_event(&mut self, now: Instant, kill_rate: f64, overloaded: bool) -> KillStateLogEvent { + use KillStateLogEvent::*; + + if let Some(overload_start) = self.overload_start { + if !overloaded { + if kill_rate == 0.0 { + self.overload_start = None; + Resolved(overload_start.elapsed()) + } else { + Settling + } + } else if now.saturating_duration_since(self.last_overload_log) + > Duration::from_secs(30) + { + self.last_overload_log = now; + Ongoing(overload_start.elapsed()) + } else { + Skip + } + } else if overloaded { + self.overload_start = Some(now); + self.last_overload_log = now; + Start + } else { + Skip + } + } +} + +/// Indicate what the load manager wants query execution to do with a query +#[derive(Debug, Clone, Copy)] +pub enum Decision { + /// Proceed with executing the query + Proceed, + /// The query is too expensive and should not be executed + TooExpensive, + /// The service is overloaded, and we should not execute the query + /// right now + Throttle, +} + +impl Decision { + pub fn to_result(self) -> Result<(), QueryExecutionError> { + use Decision::*; + match self { + Proceed => Ok(()), + TooExpensive => Err(QueryExecutionError::TooExpensive), + Throttle => Err(QueryExecutionError::Throttled), + } + } +} + +pub struct LoadManager { + logger: Logger, + effort: QueryEffort, + /// List of query shapes that have been statically blocked through + /// configuration + blocked_queries: HashSet, + /// List of query shapes that have caused more than `JAIL_THRESHOLD` + /// proportion of the work while the system was overloaded. Currently, + /// there is no way for a query to get out of jail other than + /// restarting the process + jailed_queries: RwLock>, + kill_state: RwLock, + effort_gauge: Box, + query_counters: HashMap, + kill_rate_gauge: Box, +} + +impl LoadManager { + pub fn new( + logger: &Logger, + blocked_queries: Vec>, + registry: Arc, + ) -> Self { + let logger = logger.new(o!("component" => "LoadManager")); + let blocked_queries = blocked_queries + .into_iter() + .map(|doc| shape_hash(&doc)) + .collect::>(); + + let mode = if ENV_VARS.load_management_is_disabled() { + "disabled" + } else if ENV_VARS.load_simulate { + "simulation" + } else { + "enabled" + }; + info!(logger, "Creating LoadManager in {} mode", mode,); + + let effort_gauge = registry + .new_gauge( + "query_effort_ms", + "Moving average of time spent running queries", + HashMap::new(), + ) + .expect("failed to create `query_effort_ms` counter"); + let kill_rate_gauge = registry + .new_gauge( + "query_kill_rate", + "The rate at which the load manager kills queries", + HashMap::new(), + ) + .expect("failed to create `query_kill_rate` counter"); + let query_counters = CacheStatus::iter() + .map(|s| { + let labels = HashMap::from_iter(vec![("cache_status".to_owned(), s.to_string())]); + let counter = registry + .global_counter( + "query_cache_status_count", + "Count toplevel GraphQL fields executed and their cache status", + labels, + ) + .expect("Failed to register query_counter metric"); + (*s, counter) + }) + .collect::>(); + + Self { + logger, + effort: QueryEffort::default(), + blocked_queries, + jailed_queries: RwLock::new(HashSet::new()), + kill_state: RwLock::new(KillState::new()), + effort_gauge, + query_counters, + kill_rate_gauge, + } + } + + /// Record that we spent `duration` amount of work for the query + /// `shape_hash`, where `cache_status` indicates whether the query + /// was cached or had to actually run + pub fn record_work(&self, shape_hash: u64, duration: Duration, cache_status: CacheStatus) { + self.query_counters + .get(&cache_status) + .map(GenericCounter::inc); + if !ENV_VARS.load_management_is_disabled() { + self.effort.add(shape_hash, duration, &self.effort_gauge); + } + } + + /// Decide whether we should decline to run the query with this + /// `ShapeHash`. This is the heart of reacting to overload situations. + /// + /// The decision to decline a query is geared towards mitigating two + /// different ways in which the system comes under high load: + /// 1) A relatively small number of queries causes a large fraction + /// of the overall work that goes into responding to queries. That + /// is usually inadvertent, and the result of a dApp using a new query, + /// or the data for a subgraph changing in a way that makes a query + /// that was previously fast take a long time + /// 2) A large number of queries that by themselves are reasonably fast + /// cause so much work that the system gets bogged down. When none + /// of them by themselves is expensive, it becomes impossible to + /// name a culprit for an overload, and we therefore shed + /// increasing amounts of traffic by declining to run queries + /// in proportion to the work they cause + /// + /// Note that any mitigation for (2) is prone to flip-flopping in and + /// out of overload situations, as we will oscillate between being + /// overloaded and not being overloaded, though we'd expect the amount + /// of traffic we shed to settle on something that stays close to the + /// point where we alternate between the two states. + /// + /// We detect whether we are in an overloaded situation by looking at + /// the average wait time for connection checkouts. If that exceeds + /// [`ENV_VARS.load_threshold`], we consider ourselves to be in an overload + /// situation. + /// + /// There are several criteria that will lead to us declining to run + /// a query with a certain `ShapeHash`: + /// 1) If the query is one of the configured `blocked_queries`, we will + /// always decline + /// 2) If a query, during an overload situation, causes more than + /// `JAIL_THRESHOLD` fraction of the total query effort, we will + /// refuse to run this query again for the lifetime of the process + /// 3) During an overload situation, we step a `kill_rate` from 0 to 1, + /// roughly in steps of `KILL_RATE_STEP`, though with an eye towards + /// not hitting a `kill_rate` of 1 too soon. We will decline to run + /// queries randomly with a probability of + /// kill_rate * query_effort / total_effort + /// + /// If [`ENV_VARS.load_threshold`] is set to 0, we bypass all this logic, + /// and only ever decline to run statically configured queries (1). In that + /// case, we also do not take any locks when asked to update statistics, + /// or to check whether we are overloaded; these operations amount to + /// noops. + pub fn decide(&self, wait_stats: &PoolWaitStats, shape_hash: u64, query: &str) -> Decision { + use Decision::*; + + if self.blocked_queries.contains(&shape_hash) { + return TooExpensive; + } + if ENV_VARS.load_management_is_disabled() { + return Proceed; + } + + if self.jailed_queries.read().unwrap().contains(&shape_hash) { + return if ENV_VARS.load_simulate { + Proceed + } else { + TooExpensive + }; + } + + let (overloaded, wait_ms) = self.overloaded(wait_stats); + let (kill_rate, last_update) = self.kill_state(); + if !overloaded && kill_rate == 0.0 { + return Proceed; + } + + let (query_effort, total_effort) = self.effort.current_effort(shape_hash); + // When `total_effort` is `Duratino::ZERO`, we haven't done any work. All are + // welcome + if total_effort.is_zero() { + return Proceed; + } + + // If `query_effort` is `None`, we haven't seen the query. Since we + // are in an overload situation, we are very suspicious of new things + // and assume the worst. This ensures that even if we only ever see + // new queries, we drop `kill_rate` amount of traffic + let known_query = query_effort.is_some(); + let query_effort = query_effort.unwrap_or(total_effort).as_millis() as f64; + let total_effort = total_effort.as_millis() as f64; + + // When this variable is not set, we never jail any queries. + if let Some(jail_threshold) = ENV_VARS.load_jail_threshold { + if known_query && query_effort / total_effort > jail_threshold { + // Any single query that causes at least JAIL_THRESHOLD of the + // effort in an overload situation gets killed + warn!(self.logger, "Jailing query"; + "query" => query, + "wait_ms" => wait_ms.as_millis(), + "query_effort_ms" => query_effort, + "total_effort_ms" => total_effort, + "ratio" => format!("{:.4}", query_effort/total_effort)); + self.jailed_queries.write().unwrap().insert(shape_hash); + return if ENV_VARS.load_simulate { + Proceed + } else { + TooExpensive + }; + } + } + + // Kill random queries in case we have no queries, or not enough queries + // that cause at least 20% of the effort + let kill_rate = self.update_kill_rate(kill_rate, last_update, overloaded, wait_ms); + let decline = + thread_rng().gen_bool((kill_rate * query_effort / total_effort).min(1.0).max(0.0)); + if decline { + if ENV_VARS.load_simulate { + debug!(self.logger, "Declining query"; + "query" => query, + "wait_ms" => wait_ms.as_millis(), + "query_weight" => format!("{:.2}", query_effort / total_effort), + "kill_rate" => format!("{:.4}", kill_rate), + ); + return Proceed; + } else { + return Throttle; + } + } + Proceed + } + + fn overloaded(&self, wait_stats: &PoolWaitStats) -> (bool, Duration) { + let store_avg = wait_stats.read().unwrap().average(); + let overloaded = store_avg + .map(|average| average > ENV_VARS.load_threshold) + .unwrap_or(false); + (overloaded, store_avg.unwrap_or(Duration::ZERO)) + } + + fn kill_state(&self) -> (f64, Instant) { + let state = self.kill_state.read().unwrap(); + (state.kill_rate, state.last_update) + } + + fn update_kill_rate( + &self, + mut kill_rate: f64, + last_update: Instant, + overloaded: bool, + wait_ms: Duration, + ) -> f64 { + // The rates by which we increase and decrease the `kill_rate`; when + // we increase the `kill_rate`, we do that in a way so that we do drop + // fewer queries as the `kill_rate` approaches 1.0. After `n` + // consecutive steps of increasing the `kill_rate`, it will + // be `1 - (1-KILL_RATE_STEP_UP)^n` + // + // When we step down, we do that in fixed size steps to move away from + // dropping queries fairly quickly so that after `n` steps of reducing + // the `kill_rate`, it is at most `1 - n * KILL_RATE_STEP_DOWN` + // + // The idea behind this is that we want to be conservative when we drop + // queries, but aggressive when we reduce the amount of queries we drop + // to disrupt traffic for as little as possible. + const KILL_RATE_STEP_UP: f64 = 0.1; + const KILL_RATE_STEP_DOWN: f64 = 2.0 * KILL_RATE_STEP_UP; + const KILL_RATE_UPDATE_INTERVAL: Duration = Duration::from_millis(1000); + + assert!(overloaded || kill_rate > 0.0); + + let now = Instant::now(); + if now.saturating_duration_since(last_update) > KILL_RATE_UPDATE_INTERVAL { + // Update the kill_rate + if overloaded { + kill_rate = (kill_rate + KILL_RATE_STEP_UP * (1.0 - kill_rate)).min(1.0); + } else { + kill_rate = (kill_rate - KILL_RATE_STEP_DOWN).max(0.0); + } + let event = { + let mut state = self.kill_state.write().unwrap(); + state.kill_rate = kill_rate; + state.last_update = now; + state.log_event(now, kill_rate, overloaded) + }; + // Log information about what's happening after we've released the + // lock on self.kill_state + use KillStateLogEvent::*; + match event { + Settling => { + info!(self.logger, "Query overload improving"; + "wait_ms" => wait_ms.as_millis(), + "kill_rate" => format!("{:.4}", kill_rate), + "event" => "settling"); + } + Resolved(duration) => { + info!(self.logger, "Query overload resolved"; + "duration_ms" => duration.as_millis(), + "wait_ms" => wait_ms.as_millis(), + "event" => "resolved"); + } + Ongoing(duration) => { + info!(self.logger, "Query overload still happening"; + "duration_ms" => duration.as_millis(), + "wait_ms" => wait_ms.as_millis(), + "kill_rate" => format!("{:.4}", kill_rate), + "event" => "ongoing"); + } + Start => { + warn!(self.logger, "Query overload"; + "wait_ms" => wait_ms.as_millis(), + "event" => "start"); + } + Skip => { /* do nothing */ } + } + } + self.kill_rate_gauge.set(kill_rate); + kill_rate + } +} + +#[async_trait] +impl QueryLoadManager for LoadManager { + fn record_work(&self, shape_hash: u64, duration: Duration, cache_status: CacheStatus) { + self.query_counters + .get(&cache_status) + .map(|counter| counter.inc()); + if !ENV_VARS.load_management_is_disabled() { + self.effort.add(shape_hash, duration, &self.effort_gauge); + } + } +} diff --git a/graph/src/data/graphql/ext.rs b/graph/src/data/graphql/ext.rs new file mode 100644 index 0000000..c25cdbc --- /dev/null +++ b/graph/src/data/graphql/ext.rs @@ -0,0 +1,444 @@ +use super::ObjectOrInterface; +use crate::data::schema::{META_FIELD_TYPE, SCHEMA_TYPE_NAME}; +use crate::prelude::s::{ + Definition, Directive, Document, EnumType, Field, InterfaceType, ObjectType, Type, + TypeDefinition, Value, +}; +use crate::prelude::ENV_VARS; +use std::collections::{BTreeMap, HashMap}; + +pub trait ObjectTypeExt { + fn field(&self, name: &str) -> Option<&Field>; + fn is_meta(&self) -> bool; + fn is_immutable(&self) -> bool; +} + +impl ObjectTypeExt for ObjectType { + fn field(&self, name: &str) -> Option<&Field> { + self.fields.iter().find(|field| field.name == name) + } + + fn is_meta(&self) -> bool { + self.name == META_FIELD_TYPE + } + + fn is_immutable(&self) -> bool { + self.find_directive("entity") + .and_then(|dir| dir.argument("immutable")) + .map(|value| match value { + Value::Boolean(b) => *b, + _ => false, + }) + .unwrap_or(false) + } +} + +impl ObjectTypeExt for InterfaceType { + fn field(&self, name: &str) -> Option<&Field> { + self.fields.iter().find(|field| field.name == name) + } + + fn is_meta(&self) -> bool { + false + } + + fn is_immutable(&self) -> bool { + false + } +} + +pub trait DocumentExt { + fn get_object_type_definitions(&self) -> Vec<&ObjectType>; + + fn get_interface_type_definitions(&self) -> Vec<&InterfaceType>; + + fn get_object_type_definition(&self, name: &str) -> Option<&ObjectType>; + + fn get_object_and_interface_type_fields(&self) -> HashMap<&str, &Vec>; + + fn get_enum_definitions(&self) -> Vec<&EnumType>; + + fn find_interface(&self, name: &str) -> Option<&InterfaceType>; + + fn get_fulltext_directives(&self) -> Result, anyhow::Error>; + + fn get_root_query_type(&self) -> Option<&ObjectType>; + + fn get_root_subscription_type(&self) -> Option<&ObjectType>; + + fn object_or_interface(&self, name: &str) -> Option>; + + fn get_named_type(&self, name: &str) -> Option<&TypeDefinition>; + + /// Return `true` if the type does not allow selection of child fields. + /// + /// # Panics + /// + /// If `field_type` names an unknown type + fn is_leaf_type(&self, field_type: &Type) -> bool; +} + +impl DocumentExt for Document { + /// Returns all object type definitions in the schema. + fn get_object_type_definitions(&self) -> Vec<&ObjectType> { + self.definitions + .iter() + .filter_map(|d| match d { + Definition::TypeDefinition(TypeDefinition::Object(t)) => Some(t), + _ => None, + }) + .collect() + } + + /// Returns all interface definitions in the schema. + fn get_interface_type_definitions(&self) -> Vec<&InterfaceType> { + self.definitions + .iter() + .filter_map(|d| match d { + Definition::TypeDefinition(TypeDefinition::Interface(t)) => Some(t), + _ => None, + }) + .collect() + } + + fn get_object_type_definition(&self, name: &str) -> Option<&ObjectType> { + self.get_object_type_definitions() + .into_iter() + .find(|object_type| object_type.name.eq(name)) + } + + fn get_object_and_interface_type_fields(&self) -> HashMap<&str, &Vec> { + self.definitions + .iter() + .filter_map(|d| match d { + Definition::TypeDefinition(TypeDefinition::Object(t)) => { + Some((t.name.as_str(), &t.fields)) + } + Definition::TypeDefinition(TypeDefinition::Interface(t)) => { + Some((&t.name, &t.fields)) + } + _ => None, + }) + .collect() + } + + fn get_enum_definitions(&self) -> Vec<&EnumType> { + self.definitions + .iter() + .filter_map(|d| match d { + Definition::TypeDefinition(TypeDefinition::Enum(e)) => Some(e), + _ => None, + }) + .collect() + } + + fn find_interface(&self, name: &str) -> Option<&InterfaceType> { + self.definitions.iter().find_map(|d| match d { + Definition::TypeDefinition(TypeDefinition::Interface(t)) if t.name == name => Some(t), + _ => None, + }) + } + + fn get_fulltext_directives(&self) -> Result, anyhow::Error> { + let directives = self.get_object_type_definition(SCHEMA_TYPE_NAME).map_or( + vec![], + |subgraph_schema_type| { + subgraph_schema_type + .directives + .iter() + .filter(|directives| directives.name.eq("fulltext")) + .collect() + }, + ); + if !ENV_VARS.allow_non_deterministic_fulltext_search && !directives.is_empty() { + Err(anyhow::anyhow!("Fulltext search is not yet deterministic")) + } else { + Ok(directives) + } + } + + /// Returns the root query type (if there is one). + fn get_root_query_type(&self) -> Option<&ObjectType> { + self.definitions + .iter() + .filter_map(|d| match d { + Definition::TypeDefinition(TypeDefinition::Object(t)) if t.name == "Query" => { + Some(t) + } + _ => None, + }) + .peekable() + .next() + } + + fn get_root_subscription_type(&self) -> Option<&ObjectType> { + self.definitions + .iter() + .filter_map(|d| match d { + Definition::TypeDefinition(TypeDefinition::Object(t)) + if t.name == "Subscription" => + { + Some(t) + } + _ => None, + }) + .peekable() + .next() + } + + fn object_or_interface(&self, name: &str) -> Option> { + match self.get_named_type(name) { + Some(TypeDefinition::Object(t)) => Some(t.into()), + Some(TypeDefinition::Interface(t)) => Some(t.into()), + _ => None, + } + } + + fn get_named_type(&self, name: &str) -> Option<&TypeDefinition> { + self.definitions + .iter() + .filter_map(|def| match def { + Definition::TypeDefinition(typedef) => Some(typedef), + _ => None, + }) + .find(|typedef| match typedef { + TypeDefinition::Object(t) => t.name == name, + TypeDefinition::Enum(t) => t.name == name, + TypeDefinition::InputObject(t) => t.name == name, + TypeDefinition::Interface(t) => t.name == name, + TypeDefinition::Scalar(t) => t.name == name, + TypeDefinition::Union(t) => t.name == name, + }) + } + + fn is_leaf_type(&self, field_type: &Type) -> bool { + match self + .get_named_type(field_type.get_base_type()) + .expect("names of field types have been validated") + { + TypeDefinition::Enum(_) | TypeDefinition::Scalar(_) => true, + TypeDefinition::Object(_) + | TypeDefinition::Interface(_) + | TypeDefinition::Union(_) + | TypeDefinition::InputObject(_) => false, + } + } +} + +pub trait TypeExt { + fn get_base_type(&self) -> &str; + fn is_list(&self) -> bool; + fn is_non_null(&self) -> bool; +} + +impl TypeExt for Type { + fn get_base_type(&self) -> &str { + match self { + Type::NamedType(name) => name, + Type::NonNullType(inner) => Self::get_base_type(inner), + Type::ListType(inner) => Self::get_base_type(inner), + } + } + + fn is_list(&self) -> bool { + match self { + Type::NamedType(_) => false, + Type::NonNullType(inner) => inner.is_list(), + Type::ListType(_) => true, + } + } + + // Returns true if the given type is a non-null type. + fn is_non_null(&self) -> bool { + match self { + Type::NonNullType(_) => true, + _ => false, + } + } +} + +pub trait DirectiveExt { + fn argument(&self, name: &str) -> Option<&Value>; +} + +impl DirectiveExt for Directive { + fn argument(&self, name: &str) -> Option<&Value> { + self.arguments + .iter() + .find(|(key, _value)| key == name) + .map(|(_argument, value)| value) + } +} + +pub trait ValueExt { + fn as_object(&self) -> Option<&BTreeMap>; + fn as_list(&self) -> Option<&Vec>; + fn as_str(&self) -> Option<&str>; + fn as_enum(&self) -> Option<&str>; +} + +impl ValueExt for Value { + fn as_object(&self) -> Option<&BTreeMap> { + match self { + Value::Object(object) => Some(object), + _ => None, + } + } + + fn as_list(&self) -> Option<&Vec> { + match self { + Value::List(list) => Some(list), + _ => None, + } + } + + fn as_str(&self) -> Option<&str> { + match self { + Value::String(string) => Some(string), + _ => None, + } + } + + fn as_enum(&self) -> Option<&str> { + match self { + Value::Enum(e) => Some(e), + _ => None, + } + } +} + +pub trait DirectiveFinder { + fn find_directive(&self, name: &str) -> Option<&Directive>; + fn is_derived(&self) -> bool; +} + +impl DirectiveFinder for ObjectType { + fn find_directive(&self, name: &str) -> Option<&Directive> { + self.directives + .iter() + .find(|directive| directive.name.eq(&name)) + } + + fn is_derived(&self) -> bool { + let is_derived = |directive: &Directive| directive.name.eq("derivedFrom"); + + self.directives.iter().any(is_derived) + } +} + +impl DirectiveFinder for Field { + fn find_directive(&self, name: &str) -> Option<&Directive> { + self.directives + .iter() + .find(|directive| directive.name.eq(name)) + } + + fn is_derived(&self) -> bool { + let is_derived = |directive: &Directive| directive.name.eq("derivedFrom"); + + self.directives.iter().any(is_derived) + } +} + +impl DirectiveFinder for Vec { + fn find_directive(&self, name: &str) -> Option<&Directive> { + self.iter().find(|directive| directive.name.eq(&name)) + } + + fn is_derived(&self) -> bool { + let is_derived = |directive: &Directive| directive.name.eq("derivedFrom"); + + self.iter().any(is_derived) + } +} + +pub trait TypeDefinitionExt { + fn name(&self) -> &str; + + // Return `true` if this is the definition of a type from the + // introspection schema + fn is_introspection(&self) -> bool { + self.name().starts_with("__") + } +} + +impl TypeDefinitionExt for TypeDefinition { + fn name(&self) -> &str { + match self { + TypeDefinition::Scalar(t) => &t.name, + TypeDefinition::Object(t) => &t.name, + TypeDefinition::Interface(t) => &t.name, + TypeDefinition::Union(t) => &t.name, + TypeDefinition::Enum(t) => &t.name, + TypeDefinition::InputObject(t) => &t.name, + } + } +} + +pub trait FieldExt { + // Return `true` if this is the name of one of the query fields from the + // introspection schema + fn is_introspection(&self) -> bool; +} + +impl FieldExt for Field { + fn is_introspection(&self) -> bool { + &self.name == "__schema" || &self.name == "__type" + } +} + +#[cfg(test)] +mod directive_finder_tests { + use graphql_parser::parse_schema; + + use super::*; + + const SCHEMA: &str = " + type BuyEvent implements Event @derivedFrom(field: \"buyEvent\") { + id: ID!, + transaction: Transaction! @derivedFrom(field: \"buyEvent\") + }"; + + /// Makes sure that the DirectiveFinder::find_directive implementation for ObjectiveType and Field works + #[test] + fn find_directive_impls() { + let ast = parse_schema::(SCHEMA).unwrap(); + let object_types = ast.get_object_type_definitions(); + assert_eq!(object_types.len(), 1); + let object_type = object_types[0]; + + // The object type BuyEvent has a @derivedFrom directive + assert!(object_type.find_directive("derivedFrom").is_some()); + + // BuyEvent has no deprecated directive + assert!(object_type.find_directive("deprecated").is_none()); + + let fields = &object_type.fields; + assert_eq!(fields.len(), 2); + + // Field 1 `id` is not derived + assert!(fields[0].find_directive("derivedFrom").is_none()); + // Field 2 `transaction` is derived + assert!(fields[1].find_directive("derivedFrom").is_some()); + } + + /// Makes sure that the DirectiveFinder::is_derived implementation for ObjectiveType and Field works + #[test] + fn is_derived_impls() { + let ast = parse_schema::(SCHEMA).unwrap(); + let object_types = ast.get_object_type_definitions(); + assert_eq!(object_types.len(), 1); + let object_type = object_types[0]; + + // The object type BuyEvent is derived + assert!(object_type.is_derived()); + + let fields = &object_type.fields; + assert_eq!(fields.len(), 2); + + // Field 1 `id` is not derived + assert!(!fields[0].is_derived()); + // Field 2 `transaction` is derived + assert!(fields[1].is_derived()); + } +} diff --git a/graph/src/data/graphql/mod.rs b/graph/src/data/graphql/mod.rs new file mode 100644 index 0000000..b41df57 --- /dev/null +++ b/graph/src/data/graphql/mod.rs @@ -0,0 +1,33 @@ +mod serialization; + +/// Traits to navigate the GraphQL AST +pub mod ext; +pub use ext::{DirectiveExt, DocumentExt, ObjectTypeExt, TypeExt, ValueExt}; + +/// Utilities for working with GraphQL values. +mod values; + +/// Serializable wrapper around a GraphQL value. +pub use self::serialization::SerializableValue; + +pub use self::values::{ + // Trait for converting from GraphQL values into other types. + TryFromValue, + + // Trait for plucking typed values from a GraphQL list. + ValueList, + + // Trait for plucking typed values out of a GraphQL value maps. + ValueMap, +}; + +pub mod shape_hash; + +pub mod effort; + +pub mod object_or_interface; +pub use object_or_interface::ObjectOrInterface; + +pub mod object_macro; +pub use crate::object; +pub use object_macro::{object_value, IntoValue}; diff --git a/graph/src/data/graphql/object_macro.rs b/graph/src/data/graphql/object_macro.rs new file mode 100644 index 0000000..8af3bbc --- /dev/null +++ b/graph/src/data/graphql/object_macro.rs @@ -0,0 +1,116 @@ +use crate::data::value::Object; +use crate::prelude::q; +use crate::prelude::r; +use std::iter::FromIterator; + +/// Creates a `graphql_parser::query::Value::Object` from key/value pairs. +/// If you don't need to determine which keys are included dynamically at runtime +/// consider using the `object! {}` macro instead. +pub fn object_value(data: Vec<(&str, r::Value)>) -> r::Value { + r::Value::Object(Object::from_iter( + data.into_iter().map(|(k, v)| (k.to_string(), v)), + )) +} + +pub trait IntoValue { + fn into_value(self) -> r::Value; +} + +impl IntoValue for r::Value { + #[inline] + fn into_value(self) -> r::Value { + self + } +} + +impl IntoValue for &'_ str { + #[inline] + fn into_value(self) -> r::Value { + self.to_owned().into_value() + } +} + +impl IntoValue for i32 { + #[inline] + fn into_value(self) -> r::Value { + r::Value::Int(self as i64) + } +} + +impl IntoValue for q::Number { + #[inline] + fn into_value(self) -> r::Value { + r::Value::Int(self.as_i64().unwrap()) + } +} + +impl IntoValue for u64 { + #[inline] + fn into_value(self) -> r::Value { + r::Value::String(self.to_string()) + } +} + +impl IntoValue for Option { + #[inline] + fn into_value(self) -> r::Value { + match self { + Some(v) => v.into_value(), + None => r::Value::Null, + } + } +} + +impl IntoValue for Vec { + #[inline] + fn into_value(self) -> r::Value { + r::Value::List(self.into_iter().map(|e| e.into_value()).collect::>()) + } +} + +impl IntoValue for &[u8] { + #[inline] + fn into_value(self) -> r::Value { + r::Value::String(format!("0x{}", hex::encode(self))) + } +} + +impl IntoValue for chrono::NaiveDate { + #[inline] + fn into_value(self) -> r::Value { + r::Value::String(self.format("%Y-%m-%d").to_string()) + } +} + +macro_rules! impl_into_values { + ($(($T:ty, $V:ident)),*) => { + $( + impl IntoValue for $T { + #[inline] + fn into_value(self) -> r::Value { + r::Value::$V(self) + } + } + )+ + }; +} + +impl_into_values![(String, String), (f64, Float), (bool, Boolean)]; + +/// Creates a `data::value::Value::Object` from key/value pairs. +#[macro_export] +macro_rules! object { + ($($name:ident: $value:expr,)*) => { + { + let mut result = Vec::new(); + $( + let value = $crate::data::graphql::object_macro::IntoValue::into_value($value); + result.push((stringify!($name).to_string(), value)); + )* + $crate::prelude::r::Value::Object($crate::data::value::Object::from_iter(result)) + } + }; + ($($name:ident: $value:expr),*) => { + object! {$($name: $value,)*} + }; +} diff --git a/graph/src/data/graphql/object_or_interface.rs b/graph/src/data/graphql/object_or_interface.rs new file mode 100644 index 0000000..4805355 --- /dev/null +++ b/graph/src/data/graphql/object_or_interface.rs @@ -0,0 +1,146 @@ +use crate::prelude::Schema; +use crate::{components::store::EntityType, prelude::s}; +use std::cmp::Ordering; +use std::collections::BTreeMap; +use std::hash::{Hash, Hasher}; +use std::mem; + +use super::ObjectTypeExt; + +#[derive(Copy, Clone, Debug)] +pub enum ObjectOrInterface<'a> { + Object(&'a s::ObjectType), + Interface(&'a s::InterfaceType), +} + +impl<'a> PartialEq for ObjectOrInterface<'a> { + fn eq(&self, other: &Self) -> bool { + use ObjectOrInterface::*; + match (self, other) { + (Object(a), Object(b)) => a.name == b.name, + (Interface(a), Interface(b)) => a.name == b.name, + (Interface(_), Object(_)) | (Object(_), Interface(_)) => false, + } + } +} + +impl<'a> Eq for ObjectOrInterface<'a> {} + +impl<'a> Hash for ObjectOrInterface<'a> { + fn hash(&self, state: &mut H) { + mem::discriminant(self).hash(state); + self.name().hash(state) + } +} + +impl<'a> PartialOrd for ObjectOrInterface<'a> { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl<'a> Ord for ObjectOrInterface<'a> { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + use ObjectOrInterface::*; + match (self, other) { + (Object(a), Object(b)) => a.name.cmp(&b.name), + (Interface(a), Interface(b)) => a.name.cmp(&b.name), + (Interface(_), Object(_)) => Ordering::Less, + (Object(_), Interface(_)) => Ordering::Greater, + } + } +} + +impl<'a> From<&'a s::ObjectType> for ObjectOrInterface<'a> { + fn from(object: &'a s::ObjectType) -> Self { + ObjectOrInterface::Object(object) + } +} + +impl<'a> From<&'a s::InterfaceType> for ObjectOrInterface<'a> { + fn from(interface: &'a s::InterfaceType) -> Self { + ObjectOrInterface::Interface(interface) + } +} + +impl<'a> From> for EntityType { + fn from(ooi: ObjectOrInterface) -> Self { + match ooi { + ObjectOrInterface::Object(ty) => EntityType::from(ty), + ObjectOrInterface::Interface(ty) => EntityType::from(ty), + } + } +} + +impl<'a> ObjectOrInterface<'a> { + pub fn is_object(self) -> bool { + match self { + ObjectOrInterface::Object(_) => true, + ObjectOrInterface::Interface(_) => false, + } + } + + pub fn is_interface(self) -> bool { + match self { + ObjectOrInterface::Object(_) => false, + ObjectOrInterface::Interface(_) => true, + } + } + + pub fn name(self) -> &'a str { + match self { + ObjectOrInterface::Object(object) => &object.name, + ObjectOrInterface::Interface(interface) => &interface.name, + } + } + + pub fn directives(self) -> &'a Vec { + match self { + ObjectOrInterface::Object(object) => &object.directives, + ObjectOrInterface::Interface(interface) => &interface.directives, + } + } + + pub fn fields(self) -> &'a Vec { + match self { + ObjectOrInterface::Object(object) => &object.fields, + ObjectOrInterface::Interface(interface) => &interface.fields, + } + } + + pub fn field(&self, name: &str) -> Option<&s::Field> { + self.fields().iter().find(|field| &field.name == name) + } + + pub fn object_types(self, schema: &'a Schema) -> Option> { + match self { + ObjectOrInterface::Object(object) => Some(vec![object]), + ObjectOrInterface::Interface(interface) => schema + .types_for_interface() + .get(&interface.into()) + .map(|object_types| object_types.iter().collect()), + } + } + + /// `typename` is the name of an object type. Matches if `self` is an object and has the same + /// name, or if self is an interface implemented by `typename`. + pub fn matches( + self, + typename: &str, + types_for_interface: &BTreeMap>, + ) -> bool { + match self { + ObjectOrInterface::Object(o) => o.name == typename, + ObjectOrInterface::Interface(i) => types_for_interface[&i.into()] + .iter() + .any(|o| o.name == typename), + } + } + + pub fn is_meta(&self) -> bool { + match self { + ObjectOrInterface::Object(o) => o.is_meta(), + ObjectOrInterface::Interface(i) => i.is_meta(), + } + } +} diff --git a/graph/src/data/graphql/serialization.rs b/graph/src/data/graphql/serialization.rs new file mode 100644 index 0000000..b9820e7 --- /dev/null +++ b/graph/src/data/graphql/serialization.rs @@ -0,0 +1,36 @@ +use crate::prelude::q::*; +use serde::ser::*; + +/// Serializable wrapper around a GraphQL value. +pub struct SerializableValue<'a>(pub &'a Value); + +impl<'a> Serialize for SerializableValue<'a> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self.0 { + Value::Boolean(v) => serializer.serialize_bool(*v), + Value::Enum(v) => serializer.serialize_str(v), + Value::Float(v) => serializer.serialize_f64(*v), + Value::Int(v) => serializer.serialize_newtype_struct("Number", &v.as_i64().unwrap()), + Value::List(l) => { + let mut seq = serializer.serialize_seq(Some(l.len()))?; + for v in l { + seq.serialize_element(&SerializableValue(v))?; + } + seq.end() + } + Value::Null => serializer.serialize_none(), + Value::String(s) => serializer.serialize_str(s), + Value::Object(o) => { + let mut map = serializer.serialize_map(Some(o.len()))?; + for (k, v) in o { + map.serialize_entry(k, &SerializableValue(v))?; + } + map.end() + } + Value::Variable(_) => unreachable!("output cannot contain variables"), + } + } +} diff --git a/graph/src/data/graphql/shape_hash.rs b/graph/src/data/graphql/shape_hash.rs new file mode 100644 index 0000000..00ab647 --- /dev/null +++ b/graph/src/data/graphql/shape_hash.rs @@ -0,0 +1,179 @@ +//! Calculate a hash for a GraphQL query that reflects the shape of +//! the query. The shape hash will be the same for two instances of a query +//! that are deemed identical except for unimportant details. Those details +//! are any values used with filters, and any differences in the query +//! name or response keys + +use crate::prelude::{q, s}; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; + +type ShapeHasher = DefaultHasher; + +pub trait ShapeHash { + fn shape_hash(&self, hasher: &mut ShapeHasher); +} + +pub fn shape_hash(query: &q::Document) -> u64 { + let mut hasher = DefaultHasher::new(); + query.shape_hash(&mut hasher); + hasher.finish() +} + +// In all ShapeHash implementations, we never include anything to do with +// the position of the element in the query, i.e., fields that involve +// `Pos` + +impl ShapeHash for q::Document { + fn shape_hash(&self, hasher: &mut ShapeHasher) { + for defn in &self.definitions { + use q::Definition::*; + match defn { + Operation(op) => op.shape_hash(hasher), + Fragment(frag) => frag.shape_hash(hasher), + } + } + } +} + +impl ShapeHash for q::OperationDefinition { + fn shape_hash(&self, hasher: &mut ShapeHasher) { + use graphql_parser::query::OperationDefinition::*; + // We want `[query|subscription|mutation] things { BODY }` to hash + // to the same thing as just `things { BODY }` + match self { + SelectionSet(set) => set.shape_hash(hasher), + Query(query) => query.selection_set.shape_hash(hasher), + Mutation(mutation) => mutation.selection_set.shape_hash(hasher), + Subscription(subscription) => subscription.selection_set.shape_hash(hasher), + } + } +} + +impl ShapeHash for q::FragmentDefinition { + fn shape_hash(&self, hasher: &mut ShapeHasher) { + // Omit directives + self.name.hash(hasher); + self.type_condition.shape_hash(hasher); + self.selection_set.shape_hash(hasher); + } +} + +impl ShapeHash for q::SelectionSet { + fn shape_hash(&self, hasher: &mut ShapeHasher) { + for item in &self.items { + item.shape_hash(hasher); + } + } +} + +impl ShapeHash for q::Selection { + fn shape_hash(&self, hasher: &mut ShapeHasher) { + use graphql_parser::query::Selection::*; + match self { + Field(field) => field.shape_hash(hasher), + FragmentSpread(spread) => spread.shape_hash(hasher), + InlineFragment(frag) => frag.shape_hash(hasher), + } + } +} + +impl ShapeHash for q::Field { + fn shape_hash(&self, hasher: &mut ShapeHasher) { + // Omit alias, directives + self.name.hash(hasher); + self.selection_set.shape_hash(hasher); + for (name, value) in &self.arguments { + name.hash(hasher); + value.shape_hash(hasher); + } + } +} + +impl ShapeHash for s::Value { + fn shape_hash(&self, hasher: &mut ShapeHasher) { + use graphql_parser::schema::Value::*; + + match self { + Variable(_) | Int(_) | Float(_) | String(_) | Boolean(_) | Null | Enum(_) => { + /* ignore */ + } + List(values) => { + for value in values { + value.shape_hash(hasher); + } + } + Object(map) => { + for (name, value) in map { + name.hash(hasher); + value.shape_hash(hasher); + } + } + } + } +} + +impl ShapeHash for q::FragmentSpread { + fn shape_hash(&self, hasher: &mut ShapeHasher) { + // Omit directives + self.fragment_name.hash(hasher) + } +} + +impl ShapeHash for q::InlineFragment { + fn shape_hash(&self, hasher: &mut ShapeHasher) { + // Omit directives + self.type_condition.shape_hash(hasher); + self.selection_set.shape_hash(hasher); + } +} + +impl ShapeHash for Option { + fn shape_hash(&self, hasher: &mut ShapeHasher) { + match self { + None => false.hash(hasher), + Some(t) => { + Some(true).hash(hasher); + t.shape_hash(hasher); + } + } + } +} + +impl ShapeHash for q::TypeCondition { + fn shape_hash(&self, hasher: &mut ShapeHasher) { + match self { + q::TypeCondition::On(value) => value.hash(hasher), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use graphql_parser::parse_query; + + #[test] + fn identical_and_different() { + const Q1: &str = "query things($stuff: Int) { things(where: { stuff_gt: $stuff }) { id } }"; + const Q2: &str = "{ things(where: { stuff_gt: 42 }) { id } }"; + const Q3: &str = "{ things(where: { stuff_lte: 42 }) { id } }"; + const Q4: &str = "{ things(where: { stuff_gt: 42 }) { id name } }"; + let q1 = parse_query(Q1) + .expect("q1 is syntactically valid") + .into_static(); + let q2 = parse_query(Q2) + .expect("q2 is syntactically valid") + .into_static(); + let q3 = parse_query(Q3) + .expect("q3 is syntactically valid") + .into_static(); + let q4 = parse_query(Q4) + .expect("q4 is syntactically valid") + .into_static(); + + assert_eq!(shape_hash(&q1), shape_hash(&q2)); + assert_ne!(shape_hash(&q1), shape_hash(&q3)); + assert_ne!(shape_hash(&q2), shape_hash(&q4)); + } +} diff --git a/graph/src/data/graphql/values.rs b/graph/src/data/graphql/values.rs new file mode 100644 index 0000000..7db68d7 --- /dev/null +++ b/graph/src/data/graphql/values.rs @@ -0,0 +1,237 @@ +use anyhow::{anyhow, Error}; +use std::collections::HashMap; +use std::convert::TryFrom; +use std::str::FromStr; + +use crate::blockchain::BlockHash; +use crate::data::value::Object; +use crate::prelude::{r, BigInt, Entity}; +use web3::types::H160; + +pub trait TryFromValue: Sized { + fn try_from_value(value: &r::Value) -> Result; +} + +impl TryFromValue for r::Value { + fn try_from_value(value: &r::Value) -> Result { + Ok(value.clone()) + } +} + +impl TryFromValue for bool { + fn try_from_value(value: &r::Value) -> Result { + match value { + r::Value::Boolean(b) => Ok(*b), + _ => Err(anyhow!("Cannot parse value into a boolean: {:?}", value)), + } + } +} + +impl TryFromValue for String { + fn try_from_value(value: &r::Value) -> Result { + match value { + r::Value::String(s) => Ok(s.clone()), + r::Value::Enum(s) => Ok(s.clone()), + _ => Err(anyhow!("Cannot parse value into a string: {:?}", value)), + } + } +} + +impl TryFromValue for u64 { + fn try_from_value(value: &r::Value) -> Result { + match value { + r::Value::Int(n) => { + if *n >= 0 { + Ok(*n as u64) + } else { + Err(anyhow!("Cannot parse value into an integer/u64: {:?}", n)) + } + } + // `BigInt`s are represented as `String`s. + r::Value::String(s) => u64::from_str(s).map_err(Into::into), + _ => Err(anyhow!( + "Cannot parse value into an integer/u64: {:?}", + value + )), + } + } +} + +impl TryFromValue for i32 { + fn try_from_value(value: &r::Value) -> Result { + match value { + r::Value::Int(n) => { + let n = *n; + i32::try_from(n).map_err(Error::from) + } + // `BigInt`s are represented as `String`s. + r::Value::String(s) => i32::from_str(s).map_err(Into::into), + _ => Err(anyhow!( + "Cannot parse value into an integer/u64: {:?}", + value + )), + } + } +} + +impl TryFromValue for H160 { + fn try_from_value(value: &r::Value) -> Result { + match value { + r::Value::String(s) => { + // `H160::from_str` takes a hex string with no leading `0x`. + let string = s.trim_start_matches("0x"); + H160::from_str(string).map_err(|e| { + anyhow!("Cannot parse Address/H160 value from string `{}`: {}", s, e) + }) + } + _ => Err(anyhow!( + "Cannot parse value into an Address/H160: {:?}", + value + )), + } + } +} + +impl TryFromValue for BlockHash { + fn try_from_value(value: &r::Value) -> Result { + match value { + r::Value::String(s) => BlockHash::from_str(s) + .map_err(|e| anyhow!("Cannot parse hex value from string `{}`: {}", s, e)), + _ => Err(anyhow!("Cannot parse non-string value: {:?}", value)), + } + } +} + +impl TryFromValue for BigInt { + fn try_from_value(value: &r::Value) -> Result { + match value { + r::Value::String(s) => BigInt::from_str(s) + .map_err(|e| anyhow!("Cannot parse BigInt value from string `{}`: {}", s, e)), + _ => Err(anyhow!("Cannot parse value into an BigInt: {:?}", value)), + } + } +} + +impl TryFromValue for Vec +where + T: TryFromValue, +{ + fn try_from_value(value: &r::Value) -> Result { + match value { + r::Value::List(values) => values.iter().try_fold(vec![], |mut values, value| { + values.push(T::try_from_value(value)?); + Ok(values) + }), + _ => Err(anyhow!("Cannot parse value into a vector: {:?}", value)), + } + } +} + +/// Assumes the entity is stored as a JSON string. +impl TryFromValue for Entity { + fn try_from_value(value: &r::Value) -> Result { + match value { + r::Value::String(s) => serde_json::from_str(s).map_err(Into::into), + _ => Err(anyhow!( + "Cannot parse entity, value is not a string: {:?}", + value + )), + } + } +} + +pub trait ValueMap { + fn get_required(&self, key: &str) -> Result; + fn get_optional(&self, key: &str) -> Result, Error>; +} + +impl ValueMap for r::Value { + fn get_required(&self, key: &str) -> Result { + match self { + r::Value::Object(map) => map.get_required(key), + _ => Err(anyhow!("value is not a map: {:?}", self)), + } + } + + fn get_optional(&self, key: &str) -> Result, Error> + where + T: TryFromValue, + { + match self { + r::Value::Object(map) => map.get_optional(key), + _ => Err(anyhow!("value is not a map: {:?}", self)), + } + } +} + +impl ValueMap for &Object { + fn get_required(&self, key: &str) -> Result + where + T: TryFromValue, + { + self.get(key) + .ok_or_else(|| anyhow!("Required field `{}` not set", key)) + .and_then(T::try_from_value) + } + + fn get_optional(&self, key: &str) -> Result, Error> + where + T: TryFromValue, + { + self.get(key).map_or(Ok(None), |value| match value { + r::Value::Null => Ok(None), + _ => T::try_from_value(value).map(Some), + }) + } +} + +impl ValueMap for &HashMap<&str, r::Value> { + fn get_required(&self, key: &str) -> Result + where + T: TryFromValue, + { + self.get(key) + .ok_or_else(|| anyhow!("Required field `{}` not set", key)) + .and_then(T::try_from_value) + } + + fn get_optional(&self, key: &str) -> Result, Error> + where + T: TryFromValue, + { + self.get(key).map_or(Ok(None), |value| match value { + r::Value::Null => Ok(None), + _ => T::try_from_value(value).map(Some), + }) + } +} + +pub trait ValueList { + fn get_values(&self) -> Result, Error> + where + T: TryFromValue; +} + +impl ValueList for r::Value { + fn get_values(&self) -> Result, Error> + where + T: TryFromValue, + { + match self { + r::Value::List(values) => values.get_values(), + _ => Err(anyhow!("value is not a list: {:?}", self)), + } + } +} + +impl ValueList for Vec { + fn get_values(&self) -> Result, Error> + where + T: TryFromValue, + { + self.iter().try_fold(vec![], |mut acc, value| { + acc.push(T::try_from_value(value)?); + Ok(acc) + }) + } +} diff --git a/graph/src/data/graphql/visitor.rs b/graph/src/data/graphql/visitor.rs new file mode 100644 index 0000000..94d26c0 --- /dev/null +++ b/graph/src/data/graphql/visitor.rs @@ -0,0 +1,62 @@ +use crate::prelude::q; + +pub trait Visitor { + fn enter_field(&mut self, _: &q::Field) -> Result<(), E> { + Ok(()) + } + fn leave_field(&mut self, _: &mut q::Field) -> Result<(), E> { + Ok(()) + } + + fn enter_query(&mut self, _: &q::Query) -> Result<(), E> { + Ok(()) + } + fn leave_query(&mut self, _: &mut q::Query) -> Result<(), E> { + Ok(()) + } + + fn visit_fragment_spread(&mut self, _: &q::FragmentSpread) -> Result<(), E> { + Ok(()) + } +} + +pub fn visit(visitor: &mut dyn Visitor, doc: &mut q::Document) -> Result<(), E> { + for def in &mut doc.definitions { + match def { + q::Definition::Operation(op) => match op { + q::OperationDefinition::SelectionSet(set) => { + visit_selection_set(visitor, set)?; + } + q::OperationDefinition::Query(query) => { + visitor.enter_query(query)?; + visit_selection_set(visitor, &mut query.selection_set)?; + visitor.leave_query(query)?; + } + q::OperationDefinition::Mutation(_) => todo!(), + q::OperationDefinition::Subscription(_) => todo!(), + }, + q::Definition::Fragment(frag) => {} + } + } + Ok(()) +} + +fn visit_selection_set( + visitor: &mut dyn Visitor, + set: &mut q::SelectionSet, +) -> Result<(), E> { + for sel in &mut set.items { + match sel { + q::Selection::Field(field) => { + visitor.enter_field(field)?; + visit_selection_set(visitor, &mut field.selection_set)?; + visitor.leave_field(field)?; + } + q::Selection::FragmentSpread(frag) => { + visitor.visit_fragment_spread(frag)?; + } + q::Selection::InlineFragment(frag) => {} + } + } + Ok(()) +} diff --git a/graph/src/data/introspection.graphql b/graph/src/data/introspection.graphql new file mode 100644 index 0000000..c3d2c1b --- /dev/null +++ b/graph/src/data/introspection.graphql @@ -0,0 +1,98 @@ +# A GraphQL introspection schema for inclusion in a subgraph's API schema. +# The schema differs from the 'standard' introspection schema in that it +# doesn't have a Query type nor scalar declarations as they come from the +# API schema. + +type __Schema { + types: [__Type!]! + queryType: __Type! + mutationType: __Type + subscriptionType: __Type + directives: [__Directive!]! +} + +type __Type { + kind: __TypeKind! + name: String + description: String + + # OBJECT and INTERFACE only + fields(includeDeprecated: Boolean = false): [__Field!] + + # OBJECT only + interfaces: [__Type!] + + # INTERFACE and UNION only + possibleTypes: [__Type!] + + # ENUM only + enumValues(includeDeprecated: Boolean = false): [__EnumValue!] + + # INPUT_OBJECT only + inputFields: [__InputValue!] + + # NON_NULL and LIST only + ofType: __Type +} + +type __Field { + name: String! + description: String + args: [__InputValue!]! + type: __Type! + isDeprecated: Boolean! + deprecationReason: String +} + +type __InputValue { + name: String! + description: String + type: __Type! + defaultValue: String +} + +type __EnumValue { + name: String! + description: String + isDeprecated: Boolean! + deprecationReason: String +} + +enum __TypeKind { + SCALAR + OBJECT + INTERFACE + UNION + ENUM + INPUT_OBJECT + LIST + NON_NULL +} + +type __Directive { + name: String! + description: String + locations: [__DirectiveLocation!]! + args: [__InputValue!]! +} + +enum __DirectiveLocation { + QUERY + MUTATION + SUBSCRIPTION + FIELD + FRAGMENT_DEFINITION + FRAGMENT_SPREAD + INLINE_FRAGMENT + SCHEMA + SCALAR + OBJECT + FIELD_DEFINITION + ARGUMENT_DEFINITION + INTERFACE + UNION + ENUM + ENUM_VALUE + INPUT_OBJECT + INPUT_FIELD_DEFINITION +} \ No newline at end of file diff --git a/graph/src/data/mod.rs b/graph/src/data/mod.rs new file mode 100644 index 0000000..b308c75 --- /dev/null +++ b/graph/src/data/mod.rs @@ -0,0 +1,20 @@ +/// Data types for dealing with subgraphs. +pub mod subgraph; + +/// Data types for dealing with GraphQL queries. +pub mod query; + +/// Data types for dealing with GraphQL schemas. +pub mod schema; + +/// Data types for dealing with storing entities. +pub mod store; + +/// Data types for dealing with GraphQL subscriptions. +pub mod subscription; + +/// Data types for dealing with GraphQL values. +pub mod graphql; + +/// Our representation of values for query results and the like +pub mod value; diff --git a/graph/src/data/query/cache_status.rs b/graph/src/data/query/cache_status.rs new file mode 100644 index 0000000..93d7a96 --- /dev/null +++ b/graph/src/data/query/cache_status.rs @@ -0,0 +1,43 @@ +use std::fmt; +use std::slice::Iter; + +/// Used for checking if a response hit the cache. +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub enum CacheStatus { + /// Hit is a hit in the generational cache. + Hit, + + /// Shared is a hit in the herd cache. + Shared, + + /// Insert is a miss that inserted in the generational cache. + Insert, + + /// A miss is none of the above. + Miss, +} + +impl Default for CacheStatus { + fn default() -> Self { + CacheStatus::Miss + } +} + +impl fmt::Display for CacheStatus { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + CacheStatus::Hit => f.write_str("hit"), + CacheStatus::Shared => f.write_str("shared"), + CacheStatus::Insert => f.write_str("insert"), + CacheStatus::Miss => f.write_str("miss"), + } + } +} + +impl CacheStatus { + pub fn iter() -> Iter<'static, CacheStatus> { + use CacheStatus::*; + static STATUSES: [CacheStatus; 4] = [Hit, Shared, Insert, Miss]; + STATUSES.iter() + } +} diff --git a/graph/src/data/query/error.rs b/graph/src/data/query/error.rs new file mode 100644 index 0000000..602162d --- /dev/null +++ b/graph/src/data/query/error.rs @@ -0,0 +1,458 @@ +use graphql_parser::Pos; +use hex::FromHexError; +use num_bigint; +use serde::ser::*; +use std::collections::HashMap; +use std::error::Error; +use std::fmt; +use std::string::FromUtf8Error; +use std::sync::Arc; + +use crate::data::subgraph::*; +use crate::prelude::q; +use crate::{components::store::StoreError, prelude::CacheWeight}; + +#[derive(Debug, Clone)] +pub struct CloneableAnyhowError(Arc); + +impl From for CloneableAnyhowError { + fn from(f: anyhow::Error) -> Self { + Self(Arc::new(f)) + } +} + +/// Error caused while executing a [Query](struct.Query.html). +#[derive(Debug, Clone)] +pub enum QueryExecutionError { + OperationNameRequired, + OperationNotFound(String), + NotSupported(String), + NoRootSubscriptionObjectType, + NonNullError(Pos, String), + ListValueError(Pos, String), + NamedTypeError(String), + AbstractTypeError(String), + InvalidArgumentError(Pos, String, q::Value), + MissingArgumentError(Pos, String), + ValidationError(Option, String), + InvalidVariableTypeError(Pos, String), + MissingVariableError(Pos, String), + ResolveEntitiesError(String), + OrderByNotSupportedError(String, String), + OrderByNotSupportedForType(String), + FilterNotSupportedError(String, String), + UnknownField(Pos, String, String), + EmptyQuery, + MultipleSubscriptionFields, + SubgraphDeploymentIdError(String), + RangeArgumentsError(&'static str, u32, i64), + InvalidFilterError, + EntityFieldError(String, String), + ListTypesError(String, Vec), + ListFilterError(String), + ValueParseError(String, String), + AttributeTypeError(String, String), + EntityParseError(String), + StoreError(CloneableAnyhowError), + Timeout, + EmptySelectionSet(String), + AmbiguousDerivedFromResult(Pos, String, String, String), + Unimplemented(String), + EnumCoercionError(Pos, String, q::Value, String, Vec), + ScalarCoercionError(Pos, String, q::Value, String), + TooComplex(u64, u64), // (complexity, max_complexity) + TooDeep(u8), // max_depth + CyclicalFragment(String), + TooExpensive, + Throttled, + UndefinedFragment(String), + Panic(String), + EventStreamError, + FulltextQueryRequiresFilter, + FulltextQueryInvalidSyntax(String), + DeploymentReverted, + SubgraphManifestResolveError(Arc), + InvalidSubgraphManifest, + ResultTooBig(usize, usize), + DeploymentNotFound(String), +} + +impl QueryExecutionError { + pub fn is_attestable(&self) -> bool { + use self::QueryExecutionError::*; + match self { + OperationNameRequired + | OperationNotFound(_) + | NotSupported(_) + | NoRootSubscriptionObjectType + | NonNullError(_, _) + | NamedTypeError(_) + | AbstractTypeError(_) + | InvalidArgumentError(_, _, _) + | MissingArgumentError(_, _) + | InvalidVariableTypeError(_, _) + | MissingVariableError(_, _) + | OrderByNotSupportedError(_, _) + | OrderByNotSupportedForType(_) + | FilterNotSupportedError(_, _) + | UnknownField(_, _, _) + | EmptyQuery + | MultipleSubscriptionFields + | SubgraphDeploymentIdError(_) + | InvalidFilterError + | EntityFieldError(_, _) + | ListTypesError(_, _) + | ListFilterError(_) + | AttributeTypeError(_, _) + | EmptySelectionSet(_) + | Unimplemented(_) + | CyclicalFragment(_) + | UndefinedFragment(_) + | FulltextQueryInvalidSyntax(_) + | FulltextQueryRequiresFilter => true, + ListValueError(_, _) + | ResolveEntitiesError(_) + | RangeArgumentsError(_, _, _) + | ValueParseError(_, _) + | EntityParseError(_) + | StoreError(_) + | Timeout + | EnumCoercionError(_, _, _, _, _) + | ScalarCoercionError(_, _, _, _) + | AmbiguousDerivedFromResult(_, _, _, _) + | TooComplex(_, _) + | TooDeep(_) + | Panic(_) + | EventStreamError + | TooExpensive + | Throttled + | DeploymentReverted + | SubgraphManifestResolveError(_) + | InvalidSubgraphManifest + | ValidationError(_, _) + | ResultTooBig(_, _) + | DeploymentNotFound(_) => false, + } + } +} + +impl Error for QueryExecutionError { + fn description(&self) -> &str { + "Query execution error" + } + + fn cause(&self) -> Option<&dyn Error> { + None + } +} + +impl fmt::Display for QueryExecutionError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use self::QueryExecutionError::*; + + match self { + OperationNameRequired => write!(f, "Operation name required"), + OperationNotFound(s) => { + write!(f, "Operation name not found `{}`", s) + } + ValidationError(_pos, message) => { + write!(f, "{}", message) + } + NotSupported(s) => write!(f, "Not supported: {}", s), + NoRootSubscriptionObjectType => { + write!(f, "No root Subscription type defined in the schema") + } + NonNullError(_, s) => { + write!(f, "Null value resolved for non-null field `{}`", s) + } + ListValueError(_, s) => { + write!(f, "Non-list value resolved for list field `{}`", s) + } + NamedTypeError(s) => { + write!(f, "Failed to resolve named type `{}`", s) + } + AbstractTypeError(s) => { + write!(f, "Failed to resolve abstract type `{}`", s) + } + InvalidArgumentError(_, s, v) => { + write!(f, "Invalid value provided for argument `{}`: {:?}", s, v) + } + MissingArgumentError(_, s) => { + write!(f, "No value provided for required argument: `{}`", s) + } + InvalidVariableTypeError(_, s) => { + write!(f, "Variable `{}` must have an input type", s) + } + MissingVariableError(_, s) => { + write!(f, "No value provided for required variable `{}`", s) + } + ResolveEntitiesError(e) => { + write!(f, "Failed to get entities from store: {}", e) + } + OrderByNotSupportedError(entity, field) => { + write!(f, "Ordering by `{}` is not supported for type `{}`", field, entity) + } + OrderByNotSupportedForType(field_type) => { + write!(f, "Ordering by `{}` fields is not supported", field_type) + } + FilterNotSupportedError(value, filter) => { + write!(f, "Filter not supported by value `{}`: `{}`", value, filter) + } + UnknownField(_, t, s) => { + write!(f, "Type `{}` has no field `{}`", t, s) + } + EmptyQuery => write!(f, "The query is empty"), + MultipleSubscriptionFields => write!( + f, + "Only a single top-level field is allowed in subscriptions" + ), + SubgraphDeploymentIdError(s) => { + write!(f, "Failed to get subgraph ID from type: `{}`", s) + } + RangeArgumentsError(arg, max, actual) => { + write!(f, "The `{}` argument must be between 0 and {}, but is {}", arg, max, actual) + } + InvalidFilterError => write!(f, "Filter must by an object"), + EntityFieldError(e, a) => { + write!(f, "Entity `{}` has no attribute `{}`", e, a) + } + + ListTypesError(s, v) => write!( + f, + "Values passed to filter `{}` must be of the same type but are of different types: {}", + s, + v.join(", ") + ), + ListFilterError(s) => { + write!(f, "Non-list value passed to `{}` filter", s) + } + ValueParseError(t, e) => { + write!(f, "Failed to decode `{}` value: `{}`", t, e) + } + AttributeTypeError(value, ty) => { + write!(f, "Query contains value with invalid type `{}`: `{}`", ty, value) + } + EntityParseError(s) => { + write!(f, "Broken entity found in store: {}", s) + } + StoreError(e) => { + write!(f, "Store error: {}", e.0) + } + Timeout => write!(f, "Query timed out"), + EmptySelectionSet(entity_type) => { + write!(f, "Selection set for type `{}` is empty", entity_type) + } + AmbiguousDerivedFromResult(_, field, target_type, target_field) => { + write!(f, "Ambiguous result for derived field `{}`: \ + Multiple `{}` entities refer back via `{}`", + field, target_type, target_field) + } + Unimplemented(feature) => { + write!(f, "Feature `{}` is not yet implemented", feature) + } + EnumCoercionError(_, field, value, enum_type, values) => { + write!(f, "Failed to coerce value `{}` of field `{}` to enum type `{}`. Possible values are: {}", value, field, enum_type, values.join(", ")) + } + ScalarCoercionError(_, field, value, scalar_type) => { + write!(f, "Failed to coerce value `{}` of field `{}` to scalar type `{}`", value, field, scalar_type) + } + TooComplex(complexity, max_complexity) => { + write!(f, "query potentially returns `{}` entities or more and thereby exceeds \ + the limit of `{}` entities. Possible solutions are reducing the depth \ + of the query, querying fewer relationships or using `first` to \ + return smaller collections", complexity, max_complexity) + } + TooDeep(max_depth) => write!(f, "query has a depth that exceeds the limit of `{}`", max_depth), + CyclicalFragment(name) =>write!(f, "query has fragment cycle including `{}`", name), + UndefinedFragment(frag_name) => write!(f, "fragment `{}` is not defined", frag_name), + Panic(msg) => write!(f, "panic processing query: {}", msg), + EventStreamError => write!(f, "error in the subscription event stream"), + FulltextQueryRequiresFilter => write!(f, "fulltext search queries can only use EntityFilter::Equal"), + FulltextQueryInvalidSyntax(msg) => write!(f, "Invalid fulltext search query syntax. Error: {}. Hint: Search terms with spaces need to be enclosed in single quotes", msg), + TooExpensive => write!(f, "query is too expensive"), + Throttled => write!(f, "service is overloaded and can not run the query right now. Please try again in a few minutes"), + DeploymentReverted => write!(f, "the chain was reorganized while executing the query"), + SubgraphManifestResolveError(e) => write!(f, "failed to resolve subgraph manifest: {}", e), + InvalidSubgraphManifest => write!(f, "invalid subgraph manifest file"), + ResultTooBig(actual, limit) => write!(f, "the result size of {} is larger than the allowed limit of {}", actual, limit), + DeploymentNotFound(id_or_name) => write!(f, "deployment `{}` does not exist", id_or_name) + } + } +} + +impl From for Vec { + fn from(e: QueryExecutionError) -> Self { + vec![e] + } +} + +impl From for QueryExecutionError { + fn from(e: FromHexError) -> Self { + QueryExecutionError::ValueParseError("Bytes".to_string(), e.to_string()) + } +} + +impl From for QueryExecutionError { + fn from(e: num_bigint::ParseBigIntError) -> Self { + QueryExecutionError::ValueParseError("BigInt".to_string(), format!("{}", e)) + } +} + +impl From for QueryExecutionError { + fn from(e: bigdecimal::ParseBigDecimalError) -> Self { + QueryExecutionError::ValueParseError("BigDecimal".to_string(), format!("{}", e)) + } +} + +impl From for QueryExecutionError { + fn from(e: StoreError) -> Self { + match e { + StoreError::DeploymentNotFound(id_or_name) => { + QueryExecutionError::DeploymentNotFound(id_or_name) + } + _ => QueryExecutionError::StoreError(CloneableAnyhowError(Arc::new(e.into()))), + } + } +} + +impl From for QueryExecutionError { + fn from(e: SubgraphManifestResolveError) -> Self { + QueryExecutionError::SubgraphManifestResolveError(Arc::new(e)) + } +} + +/// Error caused while processing a [Query](struct.Query.html) request. +#[derive(Clone, Debug)] +pub enum QueryError { + EncodingError(FromUtf8Error), + ParseError(Arc), + ExecutionError(QueryExecutionError), + IndexingError, +} + +impl QueryError { + pub fn is_attestable(&self) -> bool { + match self { + QueryError::EncodingError(_) | QueryError::ParseError(_) => true, + QueryError::ExecutionError(err) => err.is_attestable(), + QueryError::IndexingError => false, + } + } +} + +impl From for QueryError { + fn from(e: FromUtf8Error) -> Self { + QueryError::EncodingError(e) + } +} + +impl From for QueryError { + fn from(e: QueryExecutionError) -> Self { + QueryError::ExecutionError(e) + } +} + +impl Error for QueryError { + fn description(&self) -> &str { + "Query error" + } + + fn cause(&self) -> Option<&dyn Error> { + match *self { + QueryError::EncodingError(ref e) => Some(e), + QueryError::ExecutionError(ref e) => Some(e), + _ => None, + } + } +} + +impl fmt::Display for QueryError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + QueryError::EncodingError(ref e) => write!(f, "{}", e), + QueryError::ExecutionError(ref e) => write!(f, "{}", e), + QueryError::ParseError(ref e) => write!(f, "{}", e), + + // This error message is part of attestable responses. + QueryError::IndexingError => write!(f, "indexing_error"), + } + } +} + +impl Serialize for QueryError { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + use self::QueryExecutionError::*; + + let mut map = serializer.serialize_map(Some(1))?; + + let msg = match self { + // Serialize parse errors with their location (line, column) to make it easier + // for users to find where the errors are; this is likely to change as the + // graphql_parser team makes improvements to their error reporting + QueryError::ParseError(_) => { + // Split the inner message into (first line, rest) + let msg = format!("{}", self); + let inner_msg = msg.replace("query parse error:", ""); + let inner_msg = inner_msg.trim(); + let parts: Vec<&str> = inner_msg.splitn(2, '\n').collect(); + + // Find the colon in the first line and split there + let colon_pos = parts[0].rfind(':').unwrap(); + let (a, b) = parts[0].split_at(colon_pos); + + // Find the line and column numbers and convert them to u32 + let line: u32 = a + .matches(char::is_numeric) + .collect::() + .parse() + .unwrap(); + let column: u32 = b + .matches(char::is_numeric) + .collect::() + .parse() + .unwrap(); + + // Generate the list of locations + let mut location = HashMap::new(); + location.insert("line", line); + location.insert("column", column); + map.serialize_entry("locations", &vec![location])?; + + // Only use the remainder after the location as the error message + parts[1].to_string() + } + + // Serialize entity resolution errors using their position + QueryError::ExecutionError(NonNullError(pos, _)) + | QueryError::ExecutionError(ListValueError(pos, _)) + | QueryError::ExecutionError(InvalidArgumentError(pos, _, _)) + | QueryError::ExecutionError(MissingArgumentError(pos, _)) + | QueryError::ExecutionError(InvalidVariableTypeError(pos, _)) + | QueryError::ExecutionError(MissingVariableError(pos, _)) + | QueryError::ExecutionError(AmbiguousDerivedFromResult(pos, _, _, _)) + | QueryError::ExecutionError(EnumCoercionError(pos, _, _, _, _)) + | QueryError::ExecutionError(ScalarCoercionError(pos, _, _, _)) + | QueryError::ExecutionError(UnknownField(pos, _, _)) => { + let mut location = HashMap::new(); + location.insert("line", pos.line); + location.insert("column", pos.column); + map.serialize_entry("locations", &vec![location])?; + format!("{}", self) + } + _ => format!("{}", self), + }; + + map.serialize_entry("message", msg.as_str())?; + map.end() + } +} + +impl CacheWeight for QueryError { + fn indirect_weight(&self) -> usize { + // Errors don't have a weight since they are never cached + 0 + } +} diff --git a/graph/src/data/query/mod.rs b/graph/src/data/query/mod.rs new file mode 100644 index 0000000..7b5a901 --- /dev/null +++ b/graph/src/data/query/mod.rs @@ -0,0 +1,11 @@ +mod cache_status; +mod error; +mod query; +mod result; +mod trace; + +pub use self::cache_status::CacheStatus; +pub use self::error::{QueryError, QueryExecutionError}; +pub use self::query::{Query, QueryTarget, QueryVariables}; +pub use self::result::{QueryResult, QueryResults}; +pub use self::trace::Trace; diff --git a/graph/src/data/query/query.rs b/graph/src/data/query/query.rs new file mode 100644 index 0000000..2eaf351 --- /dev/null +++ b/graph/src/data/query/query.rs @@ -0,0 +1,164 @@ +use serde::de::Deserializer; +use serde::Deserialize; +use std::collections::{BTreeMap, HashMap}; +use std::convert::TryFrom; +use std::ops::{Deref, DerefMut}; +use std::sync::Arc; + +use crate::{ + data::graphql::shape_hash::shape_hash, + prelude::{q, r, ApiVersion, DeploymentHash, SubgraphName, ENV_VARS}, +}; + +fn deserialize_number<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let i: i32 = Deserialize::deserialize(deserializer)?; + Ok(q::Number::from(i)) +} + +fn deserialize_list<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let values: Vec = Deserialize::deserialize(deserializer)?; + Ok(values.into_iter().map(|v| v.0).collect()) +} + +fn deserialize_object<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let pairs: BTreeMap = + Deserialize::deserialize(deserializer)?; + Ok(pairs.into_iter().map(|(k, v)| (k, v.0)).collect()) +} + +#[derive(Deserialize)] +#[serde(untagged, remote = "q::Value")] +enum GraphQLValue { + #[serde(deserialize_with = "deserialize_number")] + Int(q::Number), + Float(f64), + String(String), + Boolean(bool), + Null, + Enum(String), + #[serde(deserialize_with = "deserialize_list")] + List(Vec), + #[serde(deserialize_with = "deserialize_object")] + Object(BTreeMap), +} + +/// Variable value for a GraphQL query. +#[derive(Clone, Debug, Deserialize)] +pub struct DeserializableGraphQlValue(#[serde(with = "GraphQLValue")] q::Value); + +fn deserialize_variables<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + use serde::de::Error; + let pairs: BTreeMap = + Deserialize::deserialize(deserializer)?; + pairs + .into_iter() + .map(|(k, DeserializableGraphQlValue(v))| r::Value::try_from(v).map(|v| (k, v))) + .collect::>() + .map_err(|v| D::Error::custom(format!("failed to convert to r::Value: {:?}", v))) +} + +/// Variable values for a GraphQL query. +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct QueryVariables( + #[serde(deserialize_with = "deserialize_variables")] HashMap, +); + +impl QueryVariables { + pub fn new(variables: HashMap) -> Self { + QueryVariables(variables) + } +} + +impl Deref for QueryVariables { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for QueryVariables { + fn deref_mut(&mut self) -> &mut HashMap { + &mut self.0 + } +} + +impl serde::ser::Serialize for QueryVariables { + fn serialize(&self, serializer: S) -> Result + where + S: serde::ser::Serializer, + { + use serde::ser::SerializeMap; + + let mut map = serializer.serialize_map(Some(self.0.len()))?; + for (k, v) in &self.0 { + map.serialize_entry(k, &v)?; + } + map.end() + } +} + +#[derive(Clone, Debug)] +pub enum QueryTarget { + Name(SubgraphName, ApiVersion), + Deployment(DeploymentHash, ApiVersion), +} + +impl QueryTarget { + pub fn get_version(&self) -> &ApiVersion { + match self { + Self::Deployment(_, version) | Self::Name(_, version) => version, + } + } +} + +/// A GraphQL query as submitted by a client, either directly or through a subscription. +#[derive(Clone, Debug)] +pub struct Query { + pub document: q::Document, + pub variables: Option, + pub shape_hash: u64, + pub query_text: Arc, + pub variables_text: Arc, + _force_use_of_new: (), +} + +impl Query { + pub fn new(document: q::Document, variables: Option) -> Self { + let shape_hash = shape_hash(&document); + + let (query_text, variables_text) = if ENV_VARS.log_gql_timing() + || (ENV_VARS.graphql.enable_validations && ENV_VARS.graphql.silent_graphql_validations) + { + ( + document + .format(graphql_parser::Style::default().indent(0)) + .replace('\n', " "), + serde_json::to_string(&variables).unwrap_or_default(), + ) + } else { + ("(gql logging turned off)".to_owned(), "".to_owned()) + }; + + Query { + document, + variables, + shape_hash, + query_text: Arc::new(query_text), + variables_text: Arc::new(variables_text), + _force_use_of_new: (), + } + } +} diff --git a/graph/src/data/query/result.rs b/graph/src/data/query/result.rs new file mode 100644 index 0000000..6b8bebd --- /dev/null +++ b/graph/src/data/query/result.rs @@ -0,0 +1,375 @@ +use super::error::{QueryError, QueryExecutionError}; +use crate::data::value::Object; +use crate::prelude::{r, CacheWeight, DeploymentHash}; +use http::header::{ + ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, + CONTENT_TYPE, +}; +use serde::ser::*; +use serde::Serialize; +use std::convert::TryFrom; +use std::sync::Arc; + +use super::Trace; + +fn serialize_data(data: &Option, serializer: S) -> Result +where + S: Serializer, +{ + let mut ser = serializer.serialize_map(None)?; + + // Unwrap: data is only serialized if it is `Some`. + for (k, v) in data.as_ref().unwrap() { + ser.serialize_entry(k, v)?; + } + ser.end() +} + +fn serialize_value_map<'a, S>( + data: impl Iterator, + serializer: S, +) -> Result +where + S: Serializer, +{ + let mut ser = serializer.serialize_map(None)?; + for map in data { + for (k, v) in map { + ser.serialize_entry(k, v)?; + } + } + ser.end() +} + +pub type Data = Object; + +#[derive(Debug)] +/// A collection of query results that is serialized as a single result. +pub struct QueryResults { + results: Vec>, +} + +impl QueryResults { + pub fn empty() -> Self { + QueryResults { + results: Vec::new(), + } + } + + pub fn first(&self) -> Option<&Arc> { + self.results.first() + } + + pub fn has_errors(&self) -> bool { + self.results.iter().any(|result| result.has_errors()) + } + + pub fn not_found(&self) -> bool { + self.results.iter().any(|result| result.not_found()) + } + + pub fn deployment_hash(&self) -> Option<&DeploymentHash> { + self.results + .iter() + .filter_map(|result| result.deployment.as_ref()) + .next() + } + + pub fn traces(&self) -> Vec<&Trace> { + self.results.iter().map(|res| &res.trace).collect() + } +} + +impl Serialize for QueryResults { + fn serialize(&self, serializer: S) -> Result { + let mut len = 0; + let has_data = self.results.iter().any(|r| r.has_data()); + if has_data { + len += 1; + } + let has_errors = self.results.iter().any(|r| r.has_errors()); + if has_errors { + len += 1; + } + + let mut state = serializer.serialize_struct("QueryResults", len)?; + + // Serialize data. + if has_data { + struct SerData<'a>(&'a QueryResults); + + impl Serialize for SerData<'_> { + fn serialize(&self, serializer: S) -> Result { + serialize_value_map( + self.0.results.iter().filter_map(|r| r.data.as_ref()), + serializer, + ) + } + } + + state.serialize_field("data", &SerData(self))?; + } + + // Serialize errors. + if has_errors { + struct SerError<'a>(&'a QueryResults); + + impl Serialize for SerError<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut seq = serializer.serialize_seq(None)?; + for err in self.0.results.iter().map(|r| &r.errors).flatten() { + seq.serialize_element(err)?; + } + seq.end() + } + } + + state.serialize_field("errors", &SerError(self))?; + } + + state.end() + } +} + +impl From for QueryResults { + fn from(x: Data) -> Self { + QueryResults { + results: vec![Arc::new(x.into())], + } + } +} + +impl From for QueryResults { + fn from(x: QueryResult) -> Self { + QueryResults { + results: vec![Arc::new(x)], + } + } +} + +impl From> for QueryResults { + fn from(x: Arc) -> Self { + QueryResults { results: vec![x] } + } +} + +impl From for QueryResults { + fn from(x: QueryExecutionError) -> Self { + QueryResults { + results: vec![Arc::new(x.into())], + } + } +} + +impl From> for QueryResults { + fn from(x: Vec) -> Self { + QueryResults { + results: vec![Arc::new(x.into())], + } + } +} + +impl QueryResults { + pub fn append(&mut self, other: Arc) { + self.results.push(other); + } + + pub fn as_http_response>(&self) -> http::Response { + let status_code = http::StatusCode::OK; + let json = + serde_json::to_string(self).expect("Failed to serialize GraphQL response to JSON"); + http::Response::builder() + .status(status_code) + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .header(ACCESS_CONTROL_ALLOW_HEADERS, "Content-Type, User-Agent") + .header(ACCESS_CONTROL_ALLOW_METHODS, "GET, OPTIONS, POST") + .header(CONTENT_TYPE, "application/json") + .header( + "Graph-Attestable", + self.results.iter().all(|r| r.is_attestable()).to_string(), + ) + .body(T::from(json)) + .unwrap() + } +} + +/// The result of running a query, if successful. +#[derive(Debug, Default, Serialize)] +pub struct QueryResult { + #[serde( + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_data" + )] + data: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + errors: Vec, + #[serde(skip_serializing)] + pub deployment: Option, + #[serde(skip_serializing)] + pub trace: Trace, +} + +impl QueryResult { + pub fn new(data: Data) -> Self { + QueryResult { + data: Some(data), + errors: Vec::new(), + deployment: None, + trace: Trace::None, + } + } + + /// This is really `clone`, but we do not want to implement `Clone`; + /// this is only meant for test purposes and should not be used in production + /// code since cloning query results can be very expensive + pub fn duplicate(&self) -> Self { + Self { + data: self.data.clone(), + errors: self.errors.clone(), + deployment: self.deployment.clone(), + trace: Trace::None, + } + } + + pub fn has_errors(&self) -> bool { + !self.errors.is_empty() + } + + pub fn not_found(&self) -> bool { + self.errors.iter().any(|e| { + matches!( + e, + QueryError::ExecutionError(QueryExecutionError::DeploymentNotFound(_)) + ) + }) + } + + pub fn has_data(&self) -> bool { + self.data.is_some() + } + + pub fn is_attestable(&self) -> bool { + self.errors.iter().all(|err| err.is_attestable()) + } + + pub fn to_result(self) -> Result, Vec> { + if self.has_errors() { + Err(self.errors) + } else { + Ok(self.data.map(r::Value::Object)) + } + } + + pub fn take_data(&mut self) -> Option { + self.data.take() + } + + pub fn set_data(&mut self, data: Option) { + self.data = data + } + + pub fn errors_mut(&mut self) -> &mut Vec { + &mut self.errors + } + + pub fn data(&self) -> Option<&Data> { + self.data.as_ref() + } +} + +impl From for QueryResult { + fn from(e: QueryExecutionError) -> Self { + QueryResult { + data: None, + errors: vec![e.into()], + deployment: None, + trace: Trace::None, + } + } +} + +impl From for QueryResult { + fn from(e: QueryError) -> Self { + QueryResult { + data: None, + errors: vec![e], + deployment: None, + trace: Trace::None, + } + } +} + +impl From> for QueryResult { + fn from(e: Vec) -> Self { + QueryResult { + data: None, + errors: e.into_iter().map(QueryError::from).collect(), + deployment: None, + trace: Trace::None, + } + } +} + +impl From for QueryResult { + fn from(val: Object) -> Self { + QueryResult::new(val) + } +} + +impl From<(Object, Trace)> for QueryResult { + fn from((val, trace): (Object, Trace)) -> Self { + let mut res = QueryResult::new(val); + res.trace = trace; + res + } +} + +impl TryFrom for QueryResult { + type Error = &'static str; + + fn try_from(value: r::Value) -> Result { + match value { + r::Value::Object(map) => Ok(QueryResult::from(map)), + _ => Err("only objects can be turned into a QueryResult"), + } + } +} + +impl, E: Into> From> for QueryResult { + fn from(result: Result) -> Self { + match result { + Ok(v) => v.into(), + Err(e) => e.into(), + } + } +} + +impl CacheWeight for QueryResult { + fn indirect_weight(&self) -> usize { + self.data.indirect_weight() + self.errors.indirect_weight() + } +} + +// Check that when we serialize a `QueryResult` with multiple entries +// in `data` it appears as if we serialized one big map +#[test] +fn multiple_data_items() { + use serde_json::json; + + fn make_obj(key: &str, value: &str) -> Arc { + let obj = Object::from_iter([(key.to_owned(), r::Value::String(value.to_owned()))]); + Arc::new(obj.into()) + } + + let obj1 = make_obj("key1", "value1"); + let obj2 = make_obj("key2", "value2"); + + let mut res = QueryResults::empty(); + res.append(obj1); + res.append(obj2); + + let expected = + serde_json::to_string(&json!({"data":{"key1": "value1", "key2": "value2"}})).unwrap(); + let actual = serde_json::to_string(&res).unwrap(); + assert_eq!(expected, actual) +} diff --git a/graph/src/data/query/trace.rs b/graph/src/data/query/trace.rs new file mode 100644 index 0000000..11a26e0 --- /dev/null +++ b/graph/src/data/query/trace.rs @@ -0,0 +1,77 @@ +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; + +use serde::Serialize; + +use crate::env::ENV_VARS; + +#[derive(Debug, Serialize)] +pub enum Trace { + None, + Root { + query: Arc, + elapsed: Mutex, + children: Vec<(String, Trace)>, + }, + Query { + query: String, + elapsed: Duration, + entity_count: usize, + + children: Vec<(String, Trace)>, + }, +} + +impl Default for Trace { + fn default() -> Self { + Self::None + } +} + +impl Trace { + pub fn root(query: Arc) -> Trace { + if ENV_VARS.log_sql_timing() || ENV_VARS.log_gql_timing() { + return Trace::Root { + query, + elapsed: Mutex::new(Duration::from_millis(0)), + children: Vec::new(), + }; + } else { + Trace::None + } + } + + pub fn finish(&self, dur: Duration) { + match self { + Trace::None | Trace::Query { .. } => { /* nothing to do */ } + Trace::Root { elapsed, .. } => *elapsed.lock().unwrap() = dur, + } + } + + pub fn query(query: &str, elapsed: Duration, entity_count: usize) -> Trace { + Trace::Query { + query: query.to_string(), + elapsed, + entity_count, + children: Vec::new(), + } + } + + pub fn push(&mut self, name: &str, trace: Trace) { + match (self, &trace) { + (Self::Root { children, .. }, Self::Query { .. }) => { + children.push((name.to_string(), trace)) + } + (Self::Query { children, .. }, Self::Query { .. }) => { + children.push((name.to_string(), trace)) + } + (Self::None, Self::None) | (Self::Root { .. }, Self::None) => { /* tracing is turned off */ + } + (s, t) => { + unreachable!("can not add child self: {:#?} trace: {:#?}", s, t) + } + } + } +} diff --git a/graph/src/data/schema.rs b/graph/src/data/schema.rs new file mode 100644 index 0000000..20de5cd --- /dev/null +++ b/graph/src/data/schema.rs @@ -0,0 +1,1954 @@ +use crate::cheap_clone::CheapClone; +use crate::components::store::{EntityKey, EntityType, SubgraphStore}; +use crate::data::graphql::ext::{DirectiveExt, DirectiveFinder, DocumentExt, TypeExt, ValueExt}; +use crate::data::graphql::ObjectTypeExt; +use crate::data::store::{self, ValueType}; +use crate::data::subgraph::{DeploymentHash, SubgraphName}; +use crate::prelude::{ + anyhow, lazy_static, + q::Value, + s::{self, Definition, InterfaceType, ObjectType, TypeDefinition, *}, +}; + +use anyhow::{Context, Error}; +use graphql_parser::{self, Pos}; +use inflector::Inflector; +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::convert::TryFrom; +use std::fmt; +use std::hash::Hash; +use std::iter::FromIterator; +use std::str::FromStr; +use std::sync::Arc; + +use super::graphql::ObjectOrInterface; +use super::store::scalar; + +pub const SCHEMA_TYPE_NAME: &str = "_Schema_"; + +pub const META_FIELD_TYPE: &str = "_Meta_"; +pub const META_FIELD_NAME: &str = "_meta"; + +pub const BLOCK_FIELD_TYPE: &str = "_Block_"; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Strings(Vec); + +impl fmt::Display for Strings { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + let s = (&self.0).join(", "); + write!(f, "{}", s) + } +} + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum SchemaValidationError { + #[error("Interface `{0}` not defined")] + InterfaceUndefined(String), + + #[error("@entity directive missing on the following types: `{0}`")] + EntityDirectivesMissing(Strings), + + #[error( + "Entity type `{0}` does not satisfy interface `{1}` because it is missing \ + the following fields: {2}" + )] + InterfaceFieldsMissing(String, String, Strings), // (type, interface, missing_fields) + #[error("Implementors of interface `{0}` use different id types `{1}`. They must all use the same type")] + InterfaceImplementorsMixId(String, String), + #[error("Field `{1}` in type `{0}` has invalid @derivedFrom: {2}")] + InvalidDerivedFrom(String, String, String), // (type, field, reason) + #[error("The following type names are reserved: `{0}`")] + UsageOfReservedTypes(Strings), + #[error("_Schema_ type is only for @imports and must not have any fields")] + SchemaTypeWithFields, + #[error("Imported subgraph name `{0}` is invalid")] + ImportedSubgraphNameInvalid(String), + #[error("Imported subgraph id `{0}` is invalid")] + ImportedSubgraphIdInvalid(String), + #[error("The _Schema_ type only allows @import directives")] + InvalidSchemaTypeDirectives, + #[error( + r#"@import directives must have the form \ +@import(types: ["A", {{ name: "B", as: "C"}}], from: {{ name: "org/subgraph"}}) or \ +@import(types: ["A", {{ name: "B", as: "C"}}], from: {{ id: "Qm..."}})"# + )] + ImportDirectiveInvalid, + #[error("Type `{0}`, field `{1}`: type `{2}` is neither defined nor imported")] + FieldTypeUnknown(String, String, String), // (type_name, field_name, field_type) + #[error("Imported type `{0}` does not exist in the `{1}` schema")] + ImportedTypeUndefined(String, String), // (type_name, schema) + #[error("Fulltext directive name undefined")] + FulltextNameUndefined, + #[error("Fulltext directive name overlaps with type: {0}")] + FulltextNameConflict(String), + #[error("Fulltext directive name overlaps with an existing entity field or a top-level query field: {0}")] + FulltextNameCollision(String), + #[error("Fulltext language is undefined")] + FulltextLanguageUndefined, + #[error("Fulltext language is invalid: {0}")] + FulltextLanguageInvalid(String), + #[error("Fulltext algorithm is undefined")] + FulltextAlgorithmUndefined, + #[error("Fulltext algorithm is invalid: {0}")] + FulltextAlgorithmInvalid(String), + #[error("Fulltext include is invalid")] + FulltextIncludeInvalid, + #[error("Fulltext directive requires an 'include' list")] + FulltextIncludeUndefined, + #[error("Fulltext 'include' list must contain an object")] + FulltextIncludeObjectMissing, + #[error( + "Fulltext 'include' object must contain 'entity' (String) and 'fields' (List) attributes" + )] + FulltextIncludeEntityMissingOrIncorrectAttributes, + #[error("Fulltext directive includes an entity not found on the subgraph schema")] + FulltextIncludedEntityNotFound, + #[error("Fulltext include field must have a 'name' attribute")] + FulltextIncludedFieldMissingRequiredProperty, + #[error("Fulltext entity field, {0}, not found or not a string")] + FulltextIncludedFieldInvalid(String), +} + +#[derive(Clone, Debug, PartialEq)] +pub enum FulltextLanguage { + Simple, + Danish, + Dutch, + English, + Finnish, + French, + German, + Hungarian, + Italian, + Norwegian, + Portugese, + Romanian, + Russian, + Spanish, + Swedish, + Turkish, +} + +impl TryFrom<&str> for FulltextLanguage { + type Error = String; + fn try_from(language: &str) -> Result { + match &language[..] { + "simple" => Ok(FulltextLanguage::Simple), + "da" => Ok(FulltextLanguage::Danish), + "nl" => Ok(FulltextLanguage::Dutch), + "en" => Ok(FulltextLanguage::English), + "fi" => Ok(FulltextLanguage::Finnish), + "fr" => Ok(FulltextLanguage::French), + "de" => Ok(FulltextLanguage::German), + "hu" => Ok(FulltextLanguage::Hungarian), + "it" => Ok(FulltextLanguage::Italian), + "no" => Ok(FulltextLanguage::Norwegian), + "pt" => Ok(FulltextLanguage::Portugese), + "ro" => Ok(FulltextLanguage::Romanian), + "ru" => Ok(FulltextLanguage::Russian), + "es" => Ok(FulltextLanguage::Spanish), + "sv" => Ok(FulltextLanguage::Swedish), + "tr" => Ok(FulltextLanguage::Turkish), + invalid => Err(format!( + "Provided language for fulltext search is invalid: {}", + invalid + )), + } + } +} + +impl FulltextLanguage { + pub fn as_str(&self) -> &'static str { + match self { + Self::Simple => "simple", + Self::Danish => "danish", + Self::Dutch => "dutch", + Self::English => "english", + Self::Finnish => "finnish", + Self::French => "french", + Self::German => "german", + Self::Hungarian => "hungarian", + Self::Italian => "italian", + Self::Norwegian => "norwegian", + Self::Portugese => "portugese", + Self::Romanian => "romanian", + Self::Russian => "russian", + Self::Spanish => "spanish", + Self::Swedish => "swedish", + Self::Turkish => "turkish", + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum FulltextAlgorithm { + Rank, + ProximityRank, +} + +impl TryFrom<&str> for FulltextAlgorithm { + type Error = String; + fn try_from(algorithm: &str) -> Result { + match algorithm { + "rank" => Ok(FulltextAlgorithm::Rank), + "proximityRank" => Ok(FulltextAlgorithm::ProximityRank), + invalid => Err(format!( + "The provided fulltext search algorithm {} is invalid. It must be one of: rank, proximityRank", + invalid, + )), + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct FulltextConfig { + pub language: FulltextLanguage, + pub algorithm: FulltextAlgorithm, +} + +pub struct FulltextDefinition { + pub config: FulltextConfig, + pub included_fields: HashSet, + pub name: String, +} + +impl From<&s::Directive> for FulltextDefinition { + // Assumes the input is a Fulltext Directive that has already been validated because it makes + // liberal use of unwrap() where specific types are expected + fn from(directive: &Directive) -> Self { + let name = directive.argument("name").unwrap().as_str().unwrap(); + + let algorithm = FulltextAlgorithm::try_from( + directive.argument("algorithm").unwrap().as_enum().unwrap(), + ) + .unwrap(); + + let language = + FulltextLanguage::try_from(directive.argument("language").unwrap().as_enum().unwrap()) + .unwrap(); + + let included_entity_list = directive.argument("include").unwrap().as_list().unwrap(); + // Currently fulltext query fields are limited to 1 entity, so we just take the first (and only) included Entity + let included_entity = included_entity_list.first().unwrap().as_object().unwrap(); + let included_field_values = included_entity.get("fields").unwrap().as_list().unwrap(); + let included_fields: HashSet = included_field_values + .iter() + .map(|field| { + field + .as_object() + .unwrap() + .get("name") + .unwrap() + .as_str() + .unwrap() + .into() + }) + .collect(); + + FulltextDefinition { + config: FulltextConfig { + language, + algorithm, + }, + included_fields, + name: name.into(), + } + } +} +#[derive(Debug, Error, PartialEq, Eq, Clone)] +pub enum SchemaImportError { + #[error("Schema for imported subgraph `{0}` was not found")] + ImportedSchemaNotFound(SchemaReference), + #[error("Subgraph for imported schema `{0}` is not deployed")] + ImportedSubgraphNotFound(SchemaReference), +} + +/// The representation of a single type from an import statement. This +/// corresponds either to a string `"Thing"` or an object +/// `{name: "Thing", as: "Stuff"}`. The first form is equivalent to +/// `{name: "Thing", as: "Thing"}` +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ImportedType { + /// The 'name' + name: String, + /// The 'as' alias or a copy of `name` if the user did not specify an alias + alias: String, + /// Whether the alias was explicitly given or is just a copy of the name + explicit: bool, +} + +impl fmt::Display for ImportedType { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + if self.explicit { + write!(f, "name: {}, as: {}", self.name, self.alias) + } else { + write!(f, "{}", self.name) + } + } +} + +impl ImportedType { + fn parse(type_import: &Value) -> Option { + match type_import { + Value::String(type_name) => Some(ImportedType { + name: type_name.to_string(), + alias: type_name.to_string(), + explicit: false, + }), + Value::Object(type_name_as) => { + match (type_name_as.get("name"), type_name_as.get("as")) { + (Some(name), Some(az)) => Some(ImportedType { + name: name.to_string(), + alias: az.to_string(), + explicit: true, + }), + _ => None, + } + } + _ => None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct SchemaReference { + subgraph: DeploymentHash, +} + +impl fmt::Display for SchemaReference { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "{}", self.subgraph) + } +} + +impl SchemaReference { + fn new(subgraph: DeploymentHash) -> Self { + SchemaReference { subgraph } + } + + pub fn resolve( + &self, + store: Arc, + ) -> Result, SchemaImportError> { + store + .input_schema(&self.subgraph) + .map_err(|_| SchemaImportError::ImportedSchemaNotFound(self.clone())) + } + + fn parse(value: &Value) -> Option { + match value { + Value::Object(map) => match map.get("id") { + Some(Value::String(id)) => match DeploymentHash::new(id) { + Ok(id) => Some(SchemaReference::new(id)), + _ => None, + }, + _ => None, + }, + _ => None, + } + } +} + +#[derive(Debug)] +pub struct ApiSchema { + schema: Schema, + + // Root types for the api schema. + pub query_type: Arc, + pub subscription_type: Option>, + object_types: HashMap>, +} + +impl ApiSchema { + /// `api_schema` will typically come from `fn api_schema` in the graphql + /// crate. + /// + /// In addition, the API schema has an introspection schema mixed into + /// `api_schema`. In particular, the `Query` type has fields called + /// `__schema` and `__type` + pub fn from_api_schema(mut api_schema: Schema) -> Result { + add_introspection_schema(&mut api_schema.document); + + let query_type = api_schema + .document + .get_root_query_type() + .context("no root `Query` in the schema")? + .clone(); + let subscription_type = api_schema + .document + .get_root_subscription_type() + .cloned() + .map(Arc::new); + + let object_types = HashMap::from_iter( + api_schema + .document + .get_object_type_definitions() + .into_iter() + .map(|obj_type| (obj_type.name.clone(), Arc::new(obj_type.clone()))), + ); + + Ok(Self { + schema: api_schema, + query_type: Arc::new(query_type), + subscription_type, + object_types, + }) + } + + pub fn document(&self) -> &s::Document { + &self.schema.document + } + + pub fn id(&self) -> &DeploymentHash { + &self.schema.id + } + + pub fn schema(&self) -> &Schema { + &self.schema + } + + pub fn types_for_interface(&self) -> &BTreeMap> { + &self.schema.types_for_interface + } + + /// Returns `None` if the type implements no interfaces. + pub fn interfaces_for_type(&self, type_name: &EntityType) -> Option<&Vec> { + self.schema.interfaces_for_type(type_name) + } + + /// Return an `Arc` around the `ObjectType` from our internal cache + /// + /// # Panics + /// If `obj_type` is not part of this schema, this function panics + pub fn object_type(&self, obj_type: &ObjectType) -> Arc { + self.object_types + .get(&obj_type.name) + .expect("ApiSchema.object_type is only used with existing types") + .cheap_clone() + } + + pub fn get_named_type(&self, name: &str) -> Option<&TypeDefinition> { + self.schema.document.get_named_type(name) + } + + /// Returns true if the given type is an input type. + /// + /// Uses the algorithm outlined on + /// https://facebook.github.io/graphql/draft/#IsInputType(). + pub fn is_input_type(&self, t: &s::Type) -> bool { + match t { + s::Type::NamedType(name) => { + let named_type = self.get_named_type(name); + named_type.map_or(false, |type_def| match type_def { + s::TypeDefinition::Scalar(_) + | s::TypeDefinition::Enum(_) + | s::TypeDefinition::InputObject(_) => true, + _ => false, + }) + } + s::Type::ListType(inner) => self.is_input_type(inner), + s::Type::NonNullType(inner) => self.is_input_type(inner), + } + } + + pub fn get_root_query_type_def(&self) -> Option<&s::TypeDefinition> { + self.schema + .document + .definitions + .iter() + .find_map(|d| match d { + s::Definition::TypeDefinition(def @ s::TypeDefinition::Object(_)) => match def { + s::TypeDefinition::Object(t) if t.name == "Query" => Some(def), + _ => None, + }, + _ => None, + }) + } + + pub fn object_or_interface(&self, name: &str) -> Option> { + if name.starts_with("__") { + INTROSPECTION_SCHEMA.object_or_interface(name) + } else { + self.schema.document.object_or_interface(name) + } + } + + /// Returns the type definition that a field type corresponds to. + pub fn get_type_definition_from_field<'a>( + &'a self, + field: &s::Field, + ) -> Option<&'a s::TypeDefinition> { + self.get_type_definition_from_type(&field.field_type) + } + + /// Returns the type definition for a type. + pub fn get_type_definition_from_type<'a>( + &'a self, + t: &s::Type, + ) -> Option<&'a s::TypeDefinition> { + match t { + s::Type::NamedType(name) => self.get_named_type(name), + s::Type::ListType(inner) => self.get_type_definition_from_type(inner), + s::Type::NonNullType(inner) => self.get_type_definition_from_type(inner), + } + } + + #[cfg(debug_assertions)] + pub fn definitions(&self) -> impl Iterator> { + self.schema.document.definitions.iter() + } +} + +lazy_static! { + static ref INTROSPECTION_SCHEMA: Document = { + let schema = include_str!("introspection.graphql"); + parse_schema(schema).expect("the schema `introspection.graphql` is invalid") + }; +} + +fn add_introspection_schema(schema: &mut Document) { + fn introspection_fields() -> Vec { + // Generate fields for the root query fields in an introspection schema, + // the equivalent of the fields of the `Query` type: + // + // type Query { + // __schema: __Schema! + // __type(name: String!): __Type + // } + + let type_args = vec![InputValue { + position: Pos::default(), + description: None, + name: "name".to_string(), + value_type: Type::NonNullType(Box::new(Type::NamedType("String".to_string()))), + default_value: None, + directives: vec![], + }]; + + vec![ + Field { + position: Pos::default(), + description: None, + name: "__schema".to_string(), + arguments: vec![], + field_type: Type::NonNullType(Box::new(Type::NamedType("__Schema".to_string()))), + directives: vec![], + }, + Field { + position: Pos::default(), + description: None, + name: "__type".to_string(), + arguments: type_args, + field_type: Type::NamedType("__Type".to_string()), + directives: vec![], + }, + ] + } + + schema + .definitions + .extend(INTROSPECTION_SCHEMA.definitions.iter().cloned()); + + let query_type = schema + .definitions + .iter_mut() + .filter_map(|d| match d { + Definition::TypeDefinition(TypeDefinition::Object(t)) if t.name == "Query" => Some(t), + _ => None, + }) + .peekable() + .next() + .expect("no root `Query` in the schema"); + query_type.fields.append(&mut introspection_fields()); +} + +/// A validated and preprocessed GraphQL schema for a subgraph. +#[derive(Clone, Debug, PartialEq)] +pub struct Schema { + pub id: DeploymentHash, + pub document: s::Document, + + // Maps type name to implemented interfaces. + pub interfaces_for_type: BTreeMap>, + + // Maps an interface name to the list of entities that implement it. + pub types_for_interface: BTreeMap>, + + immutable_types: HashSet, +} + +impl Schema { + /// Create a new schema. The document must already have been validated + // + // TODO: The way some validation is expected to be done beforehand, and + // some is done here makes it incredibly murky whether a `Schema` is + // fully validated. The code should be changed to make sure that a + // `Schema` is always fully valid + pub fn new(id: DeploymentHash, document: s::Document) -> Result { + let (interfaces_for_type, types_for_interface) = Self::collect_interfaces(&document)?; + let immutable_types = Self::collect_immutable_types(&document); + + let mut schema = Schema { + id: id.clone(), + document, + interfaces_for_type, + types_for_interface, + immutable_types, + }; + + schema.add_subgraph_id_directives(id); + + Ok(schema) + } + + /// Construct a value for the entity type's id attribute + pub fn id_value(&self, key: &EntityKey) -> Result { + let base_type = self + .document + .get_object_type_definition(key.entity_type.as_str()) + .ok_or_else(|| { + anyhow!( + "Entity {}[{}]: unknown entity type `{}`", + key.entity_type, + key.entity_id, + key.entity_type + ) + })? + .field("id") + .unwrap() + .field_type + .get_base_type(); + + match base_type { + "ID" | "String" => Ok(store::Value::String(key.entity_id.to_string())), + "Bytes" => Ok(store::Value::Bytes(scalar::Bytes::from_str( + &key.entity_id, + )?)), + s => { + return Err(anyhow!( + "Entity type {} uses illegal type {} for id column", + key.entity_type, + s + )) + } + } + } + + pub fn is_immutable(&self, entity_type: &EntityType) -> bool { + self.immutable_types.contains(entity_type) + } + + pub fn resolve_schema_references( + &self, + store: Arc, + ) -> ( + HashMap>, + Vec, + ) { + let mut schemas = HashMap::new(); + let mut visit_log = HashSet::new(); + let import_errors = self.resolve_import_graph(store, &mut schemas, &mut visit_log); + (schemas, import_errors) + } + + fn resolve_import_graph( + &self, + store: Arc, + schemas: &mut HashMap>, + visit_log: &mut HashSet, + ) -> Vec { + // Use the visit log to detect cycles in the import graph + self.imported_schemas() + .into_iter() + .fold(vec![], |mut errors, schema_ref| { + match schema_ref.resolve(store.clone()) { + Ok(schema) => { + schemas.insert(schema_ref, schema.clone()); + // If this node in the graph has already been visited stop traversing + if !visit_log.contains(&schema.id) { + visit_log.insert(schema.id.clone()); + errors.extend(schema.resolve_import_graph( + store.clone(), + schemas, + visit_log, + )); + } + } + Err(err) => { + errors.push(err); + } + } + errors + }) + } + + fn collect_interfaces( + document: &s::Document, + ) -> Result< + ( + BTreeMap>, + BTreeMap>, + ), + SchemaValidationError, + > { + // Initialize with an empty vec for each interface, so we don't + // miss interfaces that have no implementors. + let mut types_for_interface = + BTreeMap::from_iter(document.definitions.iter().filter_map(|d| match d { + Definition::TypeDefinition(TypeDefinition::Interface(t)) => { + Some((EntityType::from(t), vec![])) + } + _ => None, + })); + let mut interfaces_for_type = BTreeMap::<_, Vec<_>>::new(); + + for object_type in document.get_object_type_definitions() { + for implemented_interface in object_type.implements_interfaces.clone() { + let interface_type = document + .definitions + .iter() + .find_map(|def| match def { + Definition::TypeDefinition(TypeDefinition::Interface(i)) + if i.name.eq(&implemented_interface) => + { + Some(i.clone()) + } + _ => None, + }) + .ok_or_else(|| { + SchemaValidationError::InterfaceUndefined(implemented_interface.clone()) + })?; + + Self::validate_interface_implementation(object_type, &interface_type)?; + + interfaces_for_type + .entry(EntityType::from(object_type)) + .or_default() + .push(interface_type); + types_for_interface + .get_mut(&EntityType::new(implemented_interface)) + .unwrap() + .push(object_type.clone()); + } + } + + Ok((interfaces_for_type, types_for_interface)) + } + + fn collect_immutable_types(document: &s::Document) -> HashSet { + HashSet::from_iter( + document + .get_object_type_definitions() + .into_iter() + .filter(|obj_type| obj_type.is_immutable()) + .map(Into::into), + ) + } + + pub fn parse(raw: &str, id: DeploymentHash) -> Result { + let document = graphql_parser::parse_schema(raw)?.into_static(); + + Schema::new(id, document).map_err(Into::into) + } + + fn imported_types(&self) -> HashMap { + fn parse_types(import: &Directive) -> Vec { + import + .argument("types") + .map_or(vec![], |value| match value { + Value::List(types) => types.iter().filter_map(ImportedType::parse).collect(), + _ => vec![], + }) + } + + self.subgraph_schema_object_type() + .map_or(HashMap::new(), |object| { + object + .directives + .iter() + .filter(|directive| directive.name.eq("import")) + .map(|import| { + import.argument("from").map_or(vec![], |from| { + SchemaReference::parse(from).map_or(vec![], |schema_ref| { + parse_types(import) + .into_iter() + .map(|imported_type| (imported_type, schema_ref.clone())) + .collect() + }) + }) + }) + .flatten() + .collect::>() + }) + } + + pub fn imported_schemas(&self) -> Vec { + self.subgraph_schema_object_type().map_or(vec![], |object| { + object + .directives + .iter() + .filter(|directive| directive.name.eq("import")) + .filter_map(|directive| directive.argument("from")) + .filter_map(SchemaReference::parse) + .collect() + }) + } + + pub fn name_argument_value_from_directive(directive: &Directive) -> Value { + directive + .argument("name") + .expect("fulltext directive must have name argument") + .clone() + } + + /// Returned map has one an entry for each interface in the schema. + pub fn types_for_interface(&self) -> &BTreeMap> { + &self.types_for_interface + } + + /// Returns `None` if the type implements no interfaces. + pub fn interfaces_for_type(&self, type_name: &EntityType) -> Option<&Vec> { + self.interfaces_for_type.get(type_name) + } + + // Adds a @subgraphId(id: ...) directive to object/interface/enum types in the schema. + pub fn add_subgraph_id_directives(&mut self, id: DeploymentHash) { + for definition in self.document.definitions.iter_mut() { + let subgraph_id_argument = (String::from("id"), s::Value::String(id.to_string())); + + let subgraph_id_directive = s::Directive { + name: "subgraphId".to_string(), + position: Pos::default(), + arguments: vec![subgraph_id_argument], + }; + + if let Definition::TypeDefinition(ref mut type_definition) = definition { + let (name, directives) = match type_definition { + TypeDefinition::Object(object_type) => { + (&object_type.name, &mut object_type.directives) + } + TypeDefinition::Interface(interface_type) => { + (&interface_type.name, &mut interface_type.directives) + } + TypeDefinition::Enum(enum_type) => (&enum_type.name, &mut enum_type.directives), + TypeDefinition::Scalar(scalar_type) => { + (&scalar_type.name, &mut scalar_type.directives) + } + TypeDefinition::InputObject(input_object_type) => { + (&input_object_type.name, &mut input_object_type.directives) + } + TypeDefinition::Union(union_type) => { + (&union_type.name, &mut union_type.directives) + } + }; + + if !name.eq(SCHEMA_TYPE_NAME) + && !directives + .iter() + .any(|directive| directive.name.eq("subgraphId")) + { + directives.push(subgraph_id_directive); + } + }; + } + } + + pub fn validate( + &self, + schemas: &HashMap>, + ) -> Result<(), Vec> { + let mut errors: Vec = [ + self.validate_schema_types(), + self.validate_derived_from(), + self.validate_schema_type_has_no_fields(), + self.validate_directives_on_schema_type(), + self.validate_reserved_types_usage(), + self.validate_interface_id_type(), + ] + .into_iter() + .filter(Result::is_err) + // Safe unwrap due to the filter above + .map(Result::unwrap_err) + .collect(); + + errors.append(&mut self.validate_fields()); + errors.append(&mut self.validate_import_directives()); + errors.append(&mut self.validate_fulltext_directives()); + errors.append(&mut self.validate_imported_types(schemas)); + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } + + fn validate_schema_type_has_no_fields(&self) -> Result<(), SchemaValidationError> { + match self + .subgraph_schema_object_type() + .and_then(|subgraph_schema_type| { + if !subgraph_schema_type.fields.is_empty() { + Some(SchemaValidationError::SchemaTypeWithFields) + } else { + None + } + }) { + Some(err) => Err(err), + None => Ok(()), + } + } + + fn validate_directives_on_schema_type(&self) -> Result<(), SchemaValidationError> { + match self + .subgraph_schema_object_type() + .and_then(|subgraph_schema_type| { + if !subgraph_schema_type + .directives + .iter() + .filter(|directive| { + !directive.name.eq("import") && !directive.name.eq("fulltext") + }) + .next() + .is_none() + { + Some(SchemaValidationError::InvalidSchemaTypeDirectives) + } else { + None + } + }) { + Some(err) => Err(err), + None => Ok(()), + } + } + + /// Check the syntax of a single `@import` directive + fn validate_import_directive_arguments(import: &Directive) -> Option { + fn validate_import_type(typ: &Value) -> Result<(), ()> { + match typ { + Value::String(_) => Ok(()), + Value::Object(typ) => match (typ.get("name"), typ.get("as")) { + (Some(Value::String(_)), Some(Value::String(_))) => Ok(()), + _ => Err(()), + }, + _ => Err(()), + } + } + + fn types_are_valid(types: Option<&Value>) -> bool { + // All of the elements in the `types` field are valid: either + // a string or an object with keys `name` and `as` which are strings + if let Some(Value::List(types)) = types { + types + .iter() + .try_for_each(validate_import_type) + .err() + .is_none() + } else { + false + } + } + + fn from_is_valid(from: Option<&Value>) -> bool { + if let Some(Value::Object(from)) = from { + let has_id = matches!(from.get("id"), Some(Value::String(_))); + + let has_name = matches!(from.get("name"), Some(Value::String(_))); + has_id ^ has_name + } else { + false + } + } + + if from_is_valid(import.argument("from")) && types_are_valid(import.argument("types")) { + None + } else { + Some(SchemaValidationError::ImportDirectiveInvalid) + } + } + + fn validate_import_directive_schema_reference_parses( + directive: &Directive, + ) -> Option { + directive.argument("from").and_then(|from| match from { + Value::Object(from) => { + let id_parse_error = match from.get("id") { + Some(Value::String(id)) => match DeploymentHash::new(id) { + Err(_) => { + Some(SchemaValidationError::ImportedSubgraphIdInvalid(id.clone())) + } + _ => None, + }, + _ => None, + }; + let name_parse_error = match from.get("name") { + Some(Value::String(name)) => match SubgraphName::new(name) { + Err(_) => Some(SchemaValidationError::ImportedSubgraphNameInvalid( + name.clone(), + )), + _ => None, + }, + _ => None, + }; + id_parse_error.or(name_parse_error) + } + _ => None, + }) + } + + fn validate_fulltext_directives(&self) -> Vec { + self.subgraph_schema_object_type() + .map_or(vec![], |subgraph_schema_type| { + subgraph_schema_type + .directives + .iter() + .filter(|directives| directives.name.eq("fulltext")) + .fold(vec![], |mut errors, fulltext| { + errors.extend(self.validate_fulltext_directive_name(fulltext).into_iter()); + errors.extend( + self.validate_fulltext_directive_language(fulltext) + .into_iter(), + ); + errors.extend( + self.validate_fulltext_directive_algorithm(fulltext) + .into_iter(), + ); + errors.extend( + self.validate_fulltext_directive_includes(fulltext) + .into_iter(), + ); + errors + }) + }) + } + + fn validate_fulltext_directive_name(&self, fulltext: &Directive) -> Vec { + let name = match fulltext.argument("name") { + Some(Value::String(name)) => name, + _ => return vec![SchemaValidationError::FulltextNameUndefined], + }; + + let local_types: Vec<&ObjectType> = self + .document + .get_object_type_definitions() + .into_iter() + .collect(); + + // Validate that the fulltext field doesn't collide with any top-level Query fields + // generated for entity types. The field name conversions should always align with those used + // to create the field names in `graphql::schema::api::query_fields_for_type()`. + if local_types.iter().any(|typ| { + typ.fields.iter().any(|field| { + name == &field.name.as_str().to_camel_case() + || name == &field.name.to_plural().to_camel_case() + || field.name.eq(name) + }) + }) { + return vec![SchemaValidationError::FulltextNameCollision( + name.to_string(), + )]; + } + + // Validate that each fulltext directive has a distinct name + if self + .subgraph_schema_object_type() + .unwrap() + .directives + .iter() + .filter(|directive| directive.name.eq("fulltext")) + .filter_map(|fulltext| { + // Collect all @fulltext directives with the same name + match fulltext.argument("name") { + Some(Value::String(n)) if name.eq(n) => Some(n.as_str()), + _ => None, + } + }) + .count() + > 1 + { + return vec![SchemaValidationError::FulltextNameConflict( + name.to_string(), + )]; + } else { + return vec![]; + } + } + + fn validate_fulltext_directive_language( + &self, + fulltext: &Directive, + ) -> Vec { + let language = match fulltext.argument("language") { + Some(Value::Enum(language)) => language, + _ => return vec![SchemaValidationError::FulltextLanguageUndefined], + }; + match FulltextLanguage::try_from(language.as_str()) { + Ok(_) => vec![], + Err(_) => vec![SchemaValidationError::FulltextLanguageInvalid( + language.to_string(), + )], + } + } + + fn validate_fulltext_directive_algorithm( + &self, + fulltext: &Directive, + ) -> Vec { + let algorithm = match fulltext.argument("algorithm") { + Some(Value::Enum(algorithm)) => algorithm, + _ => return vec![SchemaValidationError::FulltextAlgorithmUndefined], + }; + match FulltextAlgorithm::try_from(algorithm.as_str()) { + Ok(_) => vec![], + Err(_) => vec![SchemaValidationError::FulltextAlgorithmInvalid( + algorithm.to_string(), + )], + } + } + + fn validate_fulltext_directive_includes( + &self, + fulltext: &Directive, + ) -> Vec { + // Only allow fulltext directive on local types + let local_types: Vec<&ObjectType> = self + .document + .get_object_type_definitions() + .into_iter() + .collect(); + + // Validate that each entity in fulltext.include exists + let includes = match fulltext.argument("include") { + Some(Value::List(includes)) if !includes.is_empty() => includes, + _ => return vec![SchemaValidationError::FulltextIncludeUndefined], + }; + + for include in includes { + match include.as_object() { + None => return vec![SchemaValidationError::FulltextIncludeObjectMissing], + Some(include_entity) => { + let (entity, fields) = + match (include_entity.get("entity"), include_entity.get("fields")) { + (Some(Value::String(entity)), Some(Value::List(fields))) => { + (entity, fields) + } + _ => return vec![SchemaValidationError::FulltextIncludeEntityMissingOrIncorrectAttributes], + }; + + // Validate the included entity type is one of the local types + let entity_type = match local_types + .iter() + .cloned() + .find(|typ| typ.name[..].eq(entity)) + { + None => return vec![SchemaValidationError::FulltextIncludedEntityNotFound], + Some(t) => t.clone(), + }; + + for field_value in fields { + let field_name = match field_value { + Value::Object(field_map) => match field_map.get("name") { + Some(Value::String(name)) => name, + _ => return vec![SchemaValidationError::FulltextIncludedFieldMissingRequiredProperty], + }, + _ => return vec![SchemaValidationError::FulltextIncludeEntityMissingOrIncorrectAttributes], + }; + + // Validate the included field is a String field on the local entity types specified + if !&entity_type + .fields + .iter() + .any(|field| { + let base_type: &str = field.field_type.get_base_type(); + matches!(ValueType::from_str(base_type), Ok(ValueType::String) if field.name.eq(field_name)) + }) + { + return vec![SchemaValidationError::FulltextIncludedFieldInvalid( + field_name.clone(), + )]; + }; + } + } + } + } + // Fulltext include validations all passed, so we return an empty vector + return vec![]; + } + + fn validate_import_directives(&self) -> Vec { + self.subgraph_schema_object_type() + .map_or(vec![], |subgraph_schema_type| { + subgraph_schema_type + .directives + .iter() + .filter(|directives| directives.name.eq("import")) + .fold(vec![], |mut errors, import| { + Self::validate_import_directive_arguments(import) + .into_iter() + .for_each(|err| errors.push(err)); + Self::validate_import_directive_schema_reference_parses(import) + .into_iter() + .for_each(|err| errors.push(err)); + errors + }) + }) + } + + fn validate_imported_types( + &self, + schemas: &HashMap>, + ) -> Vec { + self.imported_types() + .iter() + .fold(vec![], |mut errors, (imported_type, schema_ref)| { + schemas + .get(schema_ref) + .and_then(|schema| { + let local_types = schema.document.get_object_type_definitions(); + let imported_types = schema.imported_types(); + + // Ensure that the imported type is either local to + // the respective schema or is itself imported + // If the imported type is itself imported, do not + // recursively check the schema + let schema_handle = schema_ref.subgraph.to_string(); + let name = imported_type.name.as_str(); + + let is_local = local_types.iter().any(|object| object.name == name); + let is_imported = imported_types + .iter() + .any(|(import, _)| name == import.alias); + if !is_local && !is_imported { + Some(SchemaValidationError::ImportedTypeUndefined( + name.to_string(), + schema_handle, + )) + } else { + None + } + }) + .into_iter() + .for_each(|err| errors.push(err)); + errors + }) + } + + fn validate_fields(&self) -> Vec { + let local_types = self.document.get_object_and_interface_type_fields(); + let local_enums = self + .document + .get_enum_definitions() + .iter() + .map(|enu| enu.name.clone()) + .collect::>(); + let imported_types = self.imported_types(); + local_types + .iter() + .fold(vec![], |errors, (type_name, fields)| { + fields.iter().fold(errors, |mut errors, field| { + let base = field.field_type.get_base_type(); + if ValueType::is_scalar(base) { + return errors; + } + if local_types.contains_key(base) { + return errors; + } + if imported_types + .iter() + .any(|(imported_type, _)| &imported_type.alias == base) + { + return errors; + } + if local_enums.iter().any(|enu| enu.eq(base)) { + return errors; + } + errors.push(SchemaValidationError::FieldTypeUnknown( + type_name.to_string(), + field.name.to_string(), + base.to_string(), + )); + errors + }) + }) + } + + /// Checks if the schema is using types that are reserved + /// by `graph-node` + fn validate_reserved_types_usage(&self) -> Result<(), SchemaValidationError> { + let document = &self.document; + let object_types: Vec<_> = document + .get_object_type_definitions() + .into_iter() + .map(|obj_type| &obj_type.name) + .collect(); + + let interface_types: Vec<_> = document + .get_interface_type_definitions() + .into_iter() + .map(|iface_type| &iface_type.name) + .collect(); + + // TYPE_NAME_filter types for all object and interface types + let mut filter_types: Vec = object_types + .iter() + .chain(interface_types.iter()) + .map(|type_name| format!("{}_filter", type_name)) + .collect(); + + // TYPE_NAME_orderBy types for all object and interface types + let mut order_by_types: Vec<_> = object_types + .iter() + .chain(interface_types.iter()) + .map(|type_name| format!("{}_orderBy", type_name)) + .collect(); + + let mut reserved_types: Vec = vec![ + // The built-in scalar types + "Boolean".into(), + "ID".into(), + "Int".into(), + "BigDecimal".into(), + "String".into(), + "Bytes".into(), + "BigInt".into(), + // Reserved Query and Subscription types + "Query".into(), + "Subscription".into(), + ]; + + reserved_types.append(&mut filter_types); + reserved_types.append(&mut order_by_types); + + // `reserved_types` will now only contain + // the reserved types that the given schema *is* using. + // + // That is, if the schema is compliant and not using any reserved + // types, then it'll become an empty vector + reserved_types.retain(|reserved_type| document.get_named_type(reserved_type).is_some()); + + if reserved_types.is_empty() { + Ok(()) + } else { + Err(SchemaValidationError::UsageOfReservedTypes(Strings( + reserved_types, + ))) + } + } + + fn validate_schema_types(&self) -> Result<(), SchemaValidationError> { + let types_without_entity_directive = self + .document + .get_object_type_definitions() + .iter() + .filter(|t| t.find_directive("entity").is_none() && !t.name.eq(SCHEMA_TYPE_NAME)) + .map(|t| t.name.to_owned()) + .collect::>(); + if types_without_entity_directive.is_empty() { + Ok(()) + } else { + Err(SchemaValidationError::EntityDirectivesMissing(Strings( + types_without_entity_directive, + ))) + } + } + + fn validate_derived_from(&self) -> Result<(), SchemaValidationError> { + // Helper to construct a DerivedFromInvalid + fn invalid( + object_type: &ObjectType, + field_name: &str, + reason: &str, + ) -> SchemaValidationError { + SchemaValidationError::InvalidDerivedFrom( + object_type.name.to_owned(), + field_name.to_owned(), + reason.to_owned(), + ) + } + + let type_definitions = self.document.get_object_type_definitions(); + let object_and_interface_type_fields = self.document.get_object_and_interface_type_fields(); + + // Iterate over all derived fields in all entity types; include the + // interface types that the entity with the `@derivedFrom` implements + // and the `field` argument of @derivedFrom directive + for (object_type, interface_types, field, target_field) in type_definitions + .clone() + .iter() + .flat_map(|object_type| { + object_type + .fields + .iter() + .map(move |field| (object_type, field)) + }) + .filter_map(|(object_type, field)| { + field.find_directive("derivedFrom").map(|directive| { + ( + object_type, + object_type + .implements_interfaces + .iter() + .filter(|iface| { + // Any interface that has `field` can be used + // as the type of the field + self.document + .find_interface(iface) + .map(|iface| { + iface + .fields + .iter() + .any(|ifield| ifield.name.eq(&field.name)) + }) + .unwrap_or(false) + }) + .collect::>(), + field, + directive.argument("field"), + ) + }) + }) + { + // Turn `target_field` into the string name of the field + let target_field = target_field.ok_or_else(|| { + invalid( + object_type, + &field.name, + "the @derivedFrom directive must have a `field` argument", + ) + })?; + let target_field = match target_field { + Value::String(s) => s, + _ => { + return Err(invalid( + object_type, + &field.name, + "the @derivedFrom `field` argument must be a string", + )) + } + }; + + // Check that the type we are deriving from exists + let target_type_name = field.field_type.get_base_type(); + let target_fields = object_and_interface_type_fields + .get(target_type_name) + .ok_or_else(|| { + invalid( + object_type, + &field.name, + "type must be an existing entity or interface", + ) + })?; + + // Check that the type we are deriving from has a field with the + // right name and type + let target_field = target_fields + .iter() + .find(|field| field.name.eq(target_field)) + .ok_or_else(|| { + let msg = format!( + "field `{}` does not exist on type `{}`", + target_field, target_type_name + ); + invalid(object_type, &field.name, &msg) + })?; + + // The field we are deriving from has to point back to us; as an + // exception, we allow deriving from the `id` of another type. + // For that, we will wind up comparing the `id`s of the two types + // when we query, and just assume that that's ok. + let target_field_type = target_field.field_type.get_base_type(); + if target_field_type != object_type.name + && target_field_type != "ID" + && !interface_types + .iter() + .any(|iface| target_field_type.eq(iface.as_str())) + { + fn type_signatures(name: &str) -> Vec { + vec![ + format!("{}", name), + format!("{}!", name), + format!("[{}!]", name), + format!("[{}!]!", name), + ] + } + + let mut valid_types = type_signatures(&object_type.name); + valid_types.extend( + interface_types + .iter() + .flat_map(|iface| type_signatures(iface)), + ); + let valid_types = valid_types.join(", "); + + let msg = format!( + "field `{tf}` on type `{tt}` must have one of the following types: {valid_types}", + tf = target_field.name, + tt = target_type_name, + valid_types = valid_types, + ); + return Err(invalid(object_type, &field.name, &msg)); + } + } + Ok(()) + } + + /// Validate that `object` implements `interface`. + fn validate_interface_implementation( + object: &ObjectType, + interface: &InterfaceType, + ) -> Result<(), SchemaValidationError> { + // Check that all fields in the interface exist in the object with same name and type. + let mut missing_fields = vec![]; + for i in &interface.fields { + if !object + .fields + .iter() + .any(|o| o.name.eq(&i.name) && o.field_type.eq(&i.field_type)) + { + missing_fields.push(i.to_string().trim().to_owned()); + } + } + if !missing_fields.is_empty() { + Err(SchemaValidationError::InterfaceFieldsMissing( + object.name.clone(), + interface.name.clone(), + Strings(missing_fields), + )) + } else { + Ok(()) + } + } + + fn validate_interface_id_type(&self) -> Result<(), SchemaValidationError> { + for (intf, obj_types) in &self.types_for_interface { + let id_types: HashSet<&str> = HashSet::from_iter( + obj_types + .iter() + .filter_map(|obj_type| obj_type.field("id")) + .map(|f| f.field_type.get_base_type()) + .map(|name| if name == "ID" { "String" } else { name }), + ); + if id_types.len() > 1 { + return Err(SchemaValidationError::InterfaceImplementorsMixId( + intf.to_string(), + id_types.iter().join(", "), + )); + } + } + Ok(()) + } + + fn subgraph_schema_object_type(&self) -> Option<&ObjectType> { + self.document + .get_object_type_definitions() + .into_iter() + .find(|object_type| object_type.name.eq(SCHEMA_TYPE_NAME)) + } + + pub fn entity_fulltext_definitions( + entity: &str, + document: &Document, + ) -> Result, anyhow::Error> { + Ok(document + .get_fulltext_directives()? + .into_iter() + .filter(|directive| match directive.argument("include") { + Some(Value::List(includes)) if !includes.is_empty() => { + includes.iter().any(|include| match include { + Value::Object(include) => match include.get("entity") { + Some(Value::String(fulltext_entity)) if fulltext_entity == entity => { + true + } + _ => false, + }, + _ => false, + }) + } + _ => false, + }) + .map(FulltextDefinition::from) + .collect()) + } +} + +#[test] +fn non_existing_interface() { + let schema = "type Foo implements Bar @entity { foo: Int }"; + let res = Schema::parse(schema, DeploymentHash::new("dummy").unwrap()); + let error = res + .unwrap_err() + .downcast::() + .unwrap(); + assert_eq!( + error, + SchemaValidationError::InterfaceUndefined("Bar".to_owned()) + ); +} + +#[test] +fn invalid_interface_implementation() { + let schema = " + interface Foo { + x: Int, + y: Int + } + + type Bar implements Foo @entity { + x: Boolean + } + "; + let res = Schema::parse(schema, DeploymentHash::new("dummy").unwrap()); + assert_eq!( + res.unwrap_err().to_string(), + "Entity type `Bar` does not satisfy interface `Foo` because it is missing \ + the following fields: x: Int, y: Int", + ); +} + +#[test] +fn interface_implementations_id_type() { + fn check_schema(bar_id: &str, baz_id: &str, ok: bool) { + let schema = format!( + "interface Foo {{ x: Int }} + type Bar implements Foo @entity {{ + id: {bar_id}! + x: Int + }} + + type Baz implements Foo @entity {{ + id: {baz_id}! + x: Int + }}" + ); + let schema = Schema::parse(&schema, DeploymentHash::new("dummy").unwrap()).unwrap(); + let res = schema.validate(&HashMap::new()); + if ok { + assert!(matches!(res, Ok(_))); + } else { + assert!(matches!(res, Err(_))); + assert!(matches!( + res.unwrap_err()[0], + SchemaValidationError::InterfaceImplementorsMixId(_, _) + )); + } + } + check_schema("ID", "ID", true); + check_schema("ID", "String", true); + check_schema("ID", "Bytes", false); + check_schema("Bytes", "String", false); +} + +#[test] +fn test_derived_from_validation() { + const OTHER_TYPES: &str = " +type B @entity { id: ID! } +type C @entity { id: ID! } +type D @entity { id: ID! } +type E @entity { id: ID! } +type F @entity { id: ID! } +type G @entity { id: ID! a: BigInt } +type H @entity { id: ID! a: A! } +# This sets up a situation where we need to allow `Transaction.from` to +# point to an interface because of `Account.txn` +type Transaction @entity { from: Address! } +interface Address { txn: Transaction! @derivedFrom(field: \"from\") } +type Account implements Address @entity { id: ID!, txn: Transaction! @derivedFrom(field: \"from\") }"; + + fn validate(field: &str, errmsg: &str) { + let raw = format!("type A @entity {{ id: ID!\n {} }}\n{}", field, OTHER_TYPES); + + let document = graphql_parser::parse_schema(&raw) + .expect("Failed to parse raw schema") + .into_static(); + let schema = Schema::new(DeploymentHash::new("id").unwrap(), document).unwrap(); + match schema.validate_derived_from() { + Err(ref e) => match e { + SchemaValidationError::InvalidDerivedFrom(_, _, msg) => assert_eq!(errmsg, msg), + _ => panic!("expected variant SchemaValidationError::DerivedFromInvalid"), + }, + Ok(_) => { + if errmsg != "ok" { + panic!("expected validation for `{}` to fail", field) + } + } + } + } + + validate( + "b: B @derivedFrom(field: \"a\")", + "field `a` does not exist on type `B`", + ); + validate( + "c: [C!]! @derivedFrom(field: \"a\")", + "field `a` does not exist on type `C`", + ); + validate( + "d: D @derivedFrom", + "the @derivedFrom directive must have a `field` argument", + ); + validate( + "e: E @derivedFrom(attr: \"a\")", + "the @derivedFrom directive must have a `field` argument", + ); + validate( + "f: F @derivedFrom(field: 123)", + "the @derivedFrom `field` argument must be a string", + ); + validate( + "g: G @derivedFrom(field: \"a\")", + "field `a` on type `G` must have one of the following types: A, A!, [A!], [A!]!", + ); + validate("h: H @derivedFrom(field: \"a\")", "ok"); + validate( + "i: NotAType @derivedFrom(field: \"a\")", + "type must be an existing entity or interface", + ); + validate("j: B @derivedFrom(field: \"id\")", "ok"); +} + +#[test] +fn test_reserved_type_with_fields() { + const ROOT_SCHEMA: &str = " +type _Schema_ { id: ID! }"; + + let document = graphql_parser::parse_schema(ROOT_SCHEMA).expect("Failed to parse root schema"); + let schema = Schema::new(DeploymentHash::new("id").unwrap(), document).unwrap(); + assert_eq!( + schema + .validate_schema_type_has_no_fields() + .expect_err("Expected validation to fail due to fields defined on the reserved type"), + SchemaValidationError::SchemaTypeWithFields + ) +} + +#[test] +fn test_reserved_type_directives() { + const ROOT_SCHEMA: &str = " +type _Schema_ @illegal"; + + let document = graphql_parser::parse_schema(ROOT_SCHEMA).expect("Failed to parse root schema"); + let schema = Schema::new(DeploymentHash::new("id").unwrap(), document).unwrap(); + assert_eq!( + schema.validate_directives_on_schema_type().expect_err( + "Expected validation to fail due to extra imports defined on the reserved type" + ), + SchemaValidationError::InvalidSchemaTypeDirectives + ) +} + +#[test] +fn test_imports_directive_from_argument() { + const ROOT_SCHEMA: &str = r#" +type _Schema_ @import(types: ["T", "A", "C"])"#; + + let document = graphql_parser::parse_schema(ROOT_SCHEMA).expect("Failed to parse root schema"); + let schema = Schema::new(DeploymentHash::new("id").unwrap(), document).unwrap(); + match schema + .validate_import_directives() + .into_iter() + .find(|err| *err == SchemaValidationError::ImportDirectiveInvalid) { + None => panic!( + "Expected validation for `{}` to fail due to an @imports directive without a `from` argument", + ROOT_SCHEMA, + ), + _ => (), + } +} + +#[test] +fn test_enums_pass_field_validation() { + const ROOT_SCHEMA: &str = r#" +enum Color { + RED + GREEN +} + +type A @entity { + id: ID! + color: Color +}"#; + + let document = graphql_parser::parse_schema(ROOT_SCHEMA).expect("Failed to parse root schema"); + let schema = Schema::new(DeploymentHash::new("id").unwrap(), document).unwrap(); + assert_eq!(schema.validate_fields().len(), 0); +} + +#[test] +fn test_recursively_imported_type_validates() { + const ROOT_SCHEMA: &str = r#" +type _Schema_ @import(types: ["T"], from: { id: "c1id" })"#; + const CHILD_1_SCHEMA: &str = r#" +type _Schema_ @import(types: ["T"], from: { id: "c2id" })"#; + const CHILD_2_SCHEMA: &str = r#" +type T @entity { id: ID! } +"#; + + let root_document = + graphql_parser::parse_schema(ROOT_SCHEMA).expect("Failed to parse root schema"); + let child_1_document = + graphql_parser::parse_schema(CHILD_1_SCHEMA).expect("Failed to parse child 1 schema"); + let child_2_document = + graphql_parser::parse_schema(CHILD_2_SCHEMA).expect("Failed to parse child 2 schema"); + + let c1id = DeploymentHash::new("c1id").unwrap(); + let c2id = DeploymentHash::new("c2id").unwrap(); + let root_schema = Schema::new(DeploymentHash::new("rid").unwrap(), root_document).unwrap(); + let child_1_schema = Schema::new(c1id.clone(), child_1_document).unwrap(); + let child_2_schema = Schema::new(c2id.clone(), child_2_document).unwrap(); + + let mut schemas = HashMap::new(); + schemas.insert(SchemaReference::new(c1id), Arc::new(child_1_schema)); + schemas.insert(SchemaReference::new(c2id), Arc::new(child_2_schema)); + + match root_schema.validate_imported_types(&schemas).is_empty() { + false => panic!( + "Expected imported types validation for `{}` to suceed", + ROOT_SCHEMA, + ), + true => (), + } +} + +#[test] +fn test_recursively_imported_type_which_dne_fails_validation() { + const ROOT_SCHEMA: &str = r#" +type _Schema_ @import(types: ["T"], from: { id:"c1id"})"#; + const CHILD_1_SCHEMA: &str = r#" +type _Schema_ @import(types: [{name: "T", as: "A"}], from: { id:"c2id"})"#; + const CHILD_2_SCHEMA: &str = r#" +type T @entity { id: ID! } +"#; + let root_document = + graphql_parser::parse_schema(ROOT_SCHEMA).expect("Failed to parse root schema"); + let child_1_document = + graphql_parser::parse_schema(CHILD_1_SCHEMA).expect("Failed to parse child 1 schema"); + let child_2_document = + graphql_parser::parse_schema(CHILD_2_SCHEMA).expect("Failed to parse child 2 schema"); + + let c1id = DeploymentHash::new("c1id").unwrap(); + let c2id = DeploymentHash::new("c2id").unwrap(); + let root_schema = Schema::new(DeploymentHash::new("rid").unwrap(), root_document).unwrap(); + let child_1_schema = Schema::new(c1id.clone(), child_1_document).unwrap(); + let child_2_schema = Schema::new(c2id.clone(), child_2_document).unwrap(); + + let mut schemas = HashMap::new(); + schemas.insert(SchemaReference::new(c1id), Arc::new(child_1_schema)); + schemas.insert(SchemaReference::new(c2id), Arc::new(child_2_schema)); + + match root_schema.validate_imported_types(&schemas).into_iter().find(|err| match err { + SchemaValidationError::ImportedTypeUndefined(_, _) => true, + _ => false, + }) { + None => panic!( + "Expected imported types validation to fail because an imported type was missing in the target schema", + ), + _ => (), + } +} + +#[test] +fn test_reserved_types_validation() { + let reserved_types = [ + // Built-in scalars + "Boolean", + "ID", + "Int", + "BigDecimal", + "String", + "Bytes", + "BigInt", + // Reserved keywords + "Query", + "Subscription", + ]; + + let dummy_hash = DeploymentHash::new("dummy").unwrap(); + + for reserved_type in reserved_types { + let schema = format!("type {} @entity {{ _: Boolean }}\n", reserved_type); + + let schema = Schema::parse(&schema, dummy_hash.clone()).unwrap(); + + let errors = schema.validate(&HashMap::new()).unwrap_err(); + for error in errors { + assert!(matches!( + error, + SchemaValidationError::UsageOfReservedTypes(_) + )) + } + } +} + +#[test] +fn test_reserved_filter_and_group_by_types_validation() { + const SCHEMA: &str = r#" + type Gravatar @entity { + _: Boolean + } + type Gravatar_filter @entity { + _: Boolean + } + type Gravatar_orderBy @entity { + _: Boolean + } + "#; + + let dummy_hash = DeploymentHash::new("dummy").unwrap(); + + let schema = Schema::parse(SCHEMA, dummy_hash).unwrap(); + + let errors = schema.validate(&HashMap::new()).unwrap_err(); + + // The only problem in the schema is the usage of reserved types + assert_eq!(errors.len(), 1); + + assert!(matches!( + &errors[0], + SchemaValidationError::UsageOfReservedTypes(Strings(_)) + )); + + // We know this will match due to the assertion above + match &errors[0] { + SchemaValidationError::UsageOfReservedTypes(Strings(reserved_types)) => { + let expected_types: Vec = + vec!["Gravatar_filter".into(), "Gravatar_orderBy".into()]; + assert_eq!(reserved_types, &expected_types); + } + _ => unreachable!(), + } +} + +#[test] +fn test_fulltext_directive_validation() { + const SCHEMA: &str = r#" +type _Schema_ @fulltext( + name: "metadata" + language: en + algorithm: rank + include: [ + { + entity: "Gravatar", + fields: [ + { name: "displayName"}, + { name: "imageUrl"}, + ] + } + ] +) +type Gravatar @entity { + id: ID! + owner: Bytes! + displayName: String! + imageUrl: String! +}"#; + + let document = graphql_parser::parse_schema(SCHEMA).expect("Failed to parse schema"); + let schema = Schema::new(DeploymentHash::new("id1").unwrap(), document).unwrap(); + + assert_eq!(schema.validate_fulltext_directives(), vec![]); +} diff --git a/graph/src/data/store/ethereum.rs b/graph/src/data/store/ethereum.rs new file mode 100644 index 0000000..ada156e --- /dev/null +++ b/graph/src/data/store/ethereum.rs @@ -0,0 +1,51 @@ +use super::scalar; +use crate::prelude::*; +use web3::types::{Address, Bytes, H2048, H256, H64, U128, U256, U64}; + +impl From for Value { + fn from(n: U128) -> Value { + Value::BigInt(scalar::BigInt::from_signed_u256(&n.into())) + } +} + +impl From
for Value { + fn from(address: Address) -> Value { + Value::Bytes(scalar::Bytes::from(address.as_ref())) + } +} + +impl From for Value { + fn from(hash: H64) -> Value { + Value::Bytes(scalar::Bytes::from(hash.as_ref())) + } +} + +impl From for Value { + fn from(hash: H256) -> Value { + Value::Bytes(scalar::Bytes::from(hash.as_ref())) + } +} + +impl From for Value { + fn from(hash: H2048) -> Value { + Value::Bytes(scalar::Bytes::from(hash.as_ref())) + } +} + +impl From for Value { + fn from(bytes: Bytes) -> Value { + Value::Bytes(scalar::Bytes::from(bytes.0.as_slice())) + } +} + +impl From for Value { + fn from(n: U64) -> Value { + Value::BigInt(BigInt::from(n)) + } +} + +impl From for Value { + fn from(n: U256) -> Value { + Value::BigInt(BigInt::from_unsigned_u256(&n)) + } +} diff --git a/graph/src/data/store/mod.rs b/graph/src/data/store/mod.rs new file mode 100644 index 0000000..e08385e --- /dev/null +++ b/graph/src/data/store/mod.rs @@ -0,0 +1,1022 @@ +use crate::{ + components::store::{DeploymentLocator, EntityKey, EntityType}, + data::graphql::ObjectTypeExt, + prelude::{anyhow::Context, q, r, s, CacheWeight, QueryExecutionError, Schema}, + runtime::gas::{Gas, GasSizeOf}, +}; +use crate::{data::subgraph::DeploymentHash, prelude::EntityChange}; +use anyhow::{anyhow, Error}; +use itertools::Itertools; +use serde::de; +use serde::{Deserialize, Serialize}; +use stable_hash::{FieldAddress, StableHash, StableHasher}; +use std::convert::TryFrom; +use std::fmt; +use std::iter::FromIterator; +use std::str::FromStr; +use std::{ + borrow::Cow, + collections::{BTreeMap, HashMap}, +}; +use strum::AsStaticRef as _; +use strum_macros::AsStaticStr; + +use super::graphql::{ext::DirectiveFinder, DocumentExt as _, TypeExt as _}; + +/// Custom scalars in GraphQL. +pub mod scalar; + +// Ethereum compatibility. +pub mod ethereum; + +/// Filter subscriptions +#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum SubscriptionFilter { + /// Receive updates about all entities from the given deployment of the + /// given type + Entities(DeploymentHash, EntityType), + /// Subscripe to changes in deployment assignments + Assignment, +} + +impl SubscriptionFilter { + pub fn matches(&self, change: &EntityChange) -> bool { + match (self, change) { + ( + Self::Entities(eid, etype), + EntityChange::Data { + subgraph_id, + entity_type, + .. + }, + ) => subgraph_id == eid && entity_type == etype, + (Self::Assignment, EntityChange::Assignment { .. }) => true, + _ => false, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct NodeId(String); + +impl NodeId { + pub fn new(s: impl Into) -> Result { + let s = s.into(); + + // Enforce minimum and maximum length limit + if s.len() > 63 || s.len() < 1 { + return Err(()); + } + + Ok(NodeId(s)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for NodeId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl slog::Value for NodeId { + fn serialize( + &self, + _record: &slog::Record, + key: slog::Key, + serializer: &mut dyn slog::Serializer, + ) -> slog::Result { + serializer.emit_str(key, self.0.as_str()) + } +} + +impl<'de> de::Deserialize<'de> for NodeId { + fn deserialize(deserializer: D) -> Result + where + D: de::Deserializer<'de>, + { + let s: String = de::Deserialize::deserialize(deserializer)?; + NodeId::new(s.clone()) + .map_err(|()| de::Error::invalid_value(de::Unexpected::Str(&s), &"valid node ID")) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(tag = "type")] +pub enum AssignmentEvent { + Add { + deployment: DeploymentLocator, + node_id: NodeId, + }, + Remove { + deployment: DeploymentLocator, + node_id: NodeId, + }, +} + +impl AssignmentEvent { + pub fn node_id(&self) -> &NodeId { + match self { + AssignmentEvent::Add { node_id, .. } => node_id, + AssignmentEvent::Remove { node_id, .. } => node_id, + } + } +} + +/// An entity attribute name is represented as a string. +pub type Attribute = String; + +pub const ID: &str = "ID"; +pub const BYTES_SCALAR: &str = "Bytes"; +pub const BIG_INT_SCALAR: &str = "BigInt"; +pub const BIG_DECIMAL_SCALAR: &str = "BigDecimal"; + +#[derive(Clone, Debug, PartialEq)] +pub enum ValueType { + Boolean, + BigInt, + Bytes, + BigDecimal, + Int, + String, +} + +impl FromStr for ValueType { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s { + "Boolean" => Ok(ValueType::Boolean), + "BigInt" => Ok(ValueType::BigInt), + "Bytes" => Ok(ValueType::Bytes), + "BigDecimal" => Ok(ValueType::BigDecimal), + "Int" => Ok(ValueType::Int), + "String" | "ID" => Ok(ValueType::String), + s => Err(anyhow!("Type not available in this context: {}", s)), + } + } +} + +impl ValueType { + /// Return `true` if `s` is the name of a builtin scalar type + pub fn is_scalar(s: &str) -> bool { + Self::from_str(s).is_ok() + } +} + +// Note: Do not modify fields without also making a backward compatible change to the StableHash impl (below) +/// An attribute value is represented as an enum with variants for all supported value types. +#[derive(Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(tag = "type", content = "data")] +#[derive(AsStaticStr)] +pub enum Value { + String(String), + Int(i32), + BigDecimal(scalar::BigDecimal), + Bool(bool), + List(Vec), + Null, + Bytes(scalar::Bytes), + BigInt(scalar::BigInt), +} + +impl stable_hash_legacy::StableHash for Value { + fn stable_hash( + &self, + mut sequence_number: H::Seq, + state: &mut H, + ) { + use stable_hash_legacy::prelude::*; + use Value::*; + + // This is the default, so write nothing. + if self == &Null { + return; + } + stable_hash_legacy::StableHash::stable_hash( + &self.as_static().to_string(), + sequence_number.next_child(), + state, + ); + + match self { + Null => unreachable!(), + String(inner) => { + stable_hash_legacy::StableHash::stable_hash(inner, sequence_number, state) + } + Int(inner) => { + stable_hash_legacy::StableHash::stable_hash(inner, sequence_number, state) + } + BigDecimal(inner) => { + stable_hash_legacy::StableHash::stable_hash(inner, sequence_number, state) + } + Bool(inner) => { + stable_hash_legacy::StableHash::stable_hash(inner, sequence_number, state) + } + List(inner) => { + stable_hash_legacy::StableHash::stable_hash(inner, sequence_number, state) + } + Bytes(inner) => { + stable_hash_legacy::StableHash::stable_hash(inner, sequence_number, state) + } + BigInt(inner) => { + stable_hash_legacy::StableHash::stable_hash(inner, sequence_number, state) + } + } + } +} + +impl StableHash for Value { + fn stable_hash(&self, field_address: H::Addr, state: &mut H) { + use Value::*; + + // This is the default, so write nothing. + if self == &Null { + return; + } + + let variant = match self { + Null => unreachable!(), + String(inner) => { + inner.stable_hash(field_address.child(0), state); + 1 + } + Int(inner) => { + inner.stable_hash(field_address.child(0), state); + 2 + } + BigDecimal(inner) => { + inner.stable_hash(field_address.child(0), state); + 3 + } + Bool(inner) => { + inner.stable_hash(field_address.child(0), state); + 4 + } + List(inner) => { + inner.stable_hash(field_address.child(0), state); + 5 + } + Bytes(inner) => { + inner.stable_hash(field_address.child(0), state); + 6 + } + BigInt(inner) => { + inner.stable_hash(field_address.child(0), state); + 7 + } + }; + + state.write(field_address, &[variant]) + } +} + +impl Value { + pub fn from_query_value(value: &r::Value, ty: &s::Type) -> Result { + use graphql_parser::schema::Type::{ListType, NamedType, NonNullType}; + + Ok(match (value, ty) { + // When dealing with non-null types, use the inner type to convert the value + (value, NonNullType(t)) => Value::from_query_value(value, t)?, + + (r::Value::List(values), ListType(ty)) => Value::List( + values + .iter() + .map(|value| Self::from_query_value(value, ty)) + .collect::, _>>()?, + ), + + (r::Value::List(values), NamedType(n)) => Value::List( + values + .iter() + .map(|value| Self::from_query_value(value, &NamedType(n.to_string()))) + .collect::, _>>()?, + ), + (r::Value::Enum(e), NamedType(_)) => Value::String(e.clone()), + (r::Value::String(s), NamedType(n)) => { + // Check if `ty` is a custom scalar type, otherwise assume it's + // just a string. + match n.as_str() { + BYTES_SCALAR => Value::Bytes(scalar::Bytes::from_str(s)?), + BIG_INT_SCALAR => Value::BigInt(scalar::BigInt::from_str(s)?), + BIG_DECIMAL_SCALAR => Value::BigDecimal(scalar::BigDecimal::from_str(s)?), + _ => Value::String(s.clone()), + } + } + (r::Value::Int(i), _) => Value::Int(*i as i32), + (r::Value::Boolean(b), _) => Value::Bool(b.to_owned()), + (r::Value::Null, _) => Value::Null, + _ => { + return Err(QueryExecutionError::AttributeTypeError( + value.to_string(), + ty.to_string(), + )); + } + }) + } + + pub fn as_string(self) -> Option { + if let Value::String(s) = self { + Some(s) + } else { + None + } + } + + pub fn as_str(&self) -> Option<&str> { + if let Value::String(s) = self { + Some(s.as_str()) + } else { + None + } + } + + pub fn is_string(&self) -> bool { + matches!(self, Value::String(_)) + } + + pub fn as_int(&self) -> Option { + if let Value::Int(i) = self { + Some(*i) + } else { + None + } + } + + pub fn as_big_decimal(self) -> Option { + if let Value::BigDecimal(d) = self { + Some(d) + } else { + None + } + } + + pub fn as_bool(self) -> Option { + if let Value::Bool(b) = self { + Some(b) + } else { + None + } + } + + pub fn as_list(self) -> Option> { + if let Value::List(v) = self { + Some(v) + } else { + None + } + } + + pub fn as_bytes(self) -> Option { + if let Value::Bytes(b) = self { + Some(b) + } else { + None + } + } + + pub fn as_bigint(self) -> Option { + if let Value::BigInt(b) = self { + Some(b) + } else { + None + } + } + + /// Return the name of the type of this value for display to the user + pub fn type_name(&self) -> String { + match self { + Value::BigDecimal(_) => "BigDecimal".to_owned(), + Value::BigInt(_) => "BigInt".to_owned(), + Value::Bool(_) => "Boolean".to_owned(), + Value::Bytes(_) => "Bytes".to_owned(), + Value::Int(_) => "Int".to_owned(), + Value::List(values) => { + if let Some(v) = values.first() { + format!("[{}]", v.type_name()) + } else { + "[Any]".to_owned() + } + } + Value::Null => "Null".to_owned(), + Value::String(_) => "String".to_owned(), + } + } + + pub fn is_assignable(&self, scalar_type: &ValueType, is_list: bool) -> bool { + match (self, scalar_type) { + (Value::String(_), ValueType::String) + | (Value::BigDecimal(_), ValueType::BigDecimal) + | (Value::BigInt(_), ValueType::BigInt) + | (Value::Bool(_), ValueType::Boolean) + | (Value::Bytes(_), ValueType::Bytes) + | (Value::Int(_), ValueType::Int) + | (Value::Null, _) => true, + (Value::List(values), _) if is_list => values + .iter() + .all(|value| value.is_assignable(scalar_type, false)), + _ => false, + } + } +} + +impl fmt::Display for Value { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "{}", + match self { + Value::String(s) => s.to_string(), + Value::Int(i) => i.to_string(), + Value::BigDecimal(d) => d.to_string(), + Value::Bool(b) => b.to_string(), + Value::Null => "null".to_string(), + Value::List(ref values) => + format!("[{}]", values.iter().map(ToString::to_string).join(", ")), + Value::Bytes(ref bytes) => bytes.to_string(), + Value::BigInt(ref number) => number.to_string(), + } + ) + } +} + +impl fmt::Debug for Value { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::String(s) => f.debug_tuple("String").field(s).finish(), + Self::Int(i) => f.debug_tuple("Int").field(i).finish(), + Self::BigDecimal(d) => d.fmt(f), + Self::Bool(arg0) => f.debug_tuple("Bool").field(arg0).finish(), + Self::List(arg0) => f.debug_tuple("List").field(arg0).finish(), + Self::Null => write!(f, "Null"), + Self::Bytes(bytes) => bytes.fmt(f), + Self::BigInt(number) => number.fmt(f), + } + } +} + +impl From for q::Value { + fn from(value: Value) -> Self { + match value { + Value::String(s) => q::Value::String(s), + Value::Int(i) => q::Value::Int(q::Number::from(i)), + Value::BigDecimal(d) => q::Value::String(d.to_string()), + Value::Bool(b) => q::Value::Boolean(b), + Value::Null => q::Value::Null, + Value::List(values) => { + q::Value::List(values.into_iter().map(|value| value.into()).collect()) + } + Value::Bytes(bytes) => q::Value::String(bytes.to_string()), + Value::BigInt(number) => q::Value::String(number.to_string()), + } + } +} + +impl From for r::Value { + fn from(value: Value) -> Self { + match value { + Value::String(s) => r::Value::String(s), + Value::Int(i) => r::Value::Int(i as i64), + Value::BigDecimal(d) => r::Value::String(d.to_string()), + Value::Bool(b) => r::Value::Boolean(b), + Value::Null => r::Value::Null, + Value::List(values) => { + r::Value::List(values.into_iter().map(|value| value.into()).collect()) + } + Value::Bytes(bytes) => r::Value::String(bytes.to_string()), + Value::BigInt(number) => r::Value::String(number.to_string()), + } + } +} + +impl<'a> From<&'a str> for Value { + fn from(value: &'a str) -> Value { + Value::String(value.to_owned()) + } +} + +impl From for Value { + fn from(value: String) -> Value { + Value::String(value) + } +} + +impl<'a> From<&'a String> for Value { + fn from(value: &'a String) -> Value { + Value::String(value.clone()) + } +} + +impl From for Value { + fn from(value: scalar::Bytes) -> Value { + Value::Bytes(value) + } +} + +impl From for Value { + fn from(value: bool) -> Value { + Value::Bool(value) + } +} + +impl From for Value { + fn from(value: i32) -> Value { + Value::Int(value) + } +} + +impl From for Value { + fn from(value: scalar::BigDecimal) -> Value { + Value::BigDecimal(value) + } +} + +impl From for Value { + fn from(value: scalar::BigInt) -> Value { + Value::BigInt(value) + } +} + +impl From for Value { + fn from(value: u64) -> Value { + Value::BigInt(value.into()) + } +} + +impl TryFrom for Option { + type Error = Error; + + fn try_from(value: Value) -> Result { + match value { + Value::BigInt(n) => Ok(Some(n)), + Value::Null => Ok(None), + _ => Err(anyhow!("Value is not an BigInt")), + } + } +} + +impl From> for Value +where + T: Into, +{ + fn from(values: Vec) -> Value { + Value::List(values.into_iter().map(Into::into).collect()) + } +} + +impl From> for Value +where + Value: From, +{ + fn from(x: Option) -> Value { + match x { + Some(x) => x.into(), + None => Value::Null, + } + } +} + +/// An entity is represented as a map of attribute names to values. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +pub struct Entity(HashMap); + +impl stable_hash_legacy::StableHash for Entity { + #[inline] + fn stable_hash( + &self, + mut sequence_number: H::Seq, + state: &mut H, + ) { + use stable_hash_legacy::SequenceNumber; + let Self(inner) = self; + stable_hash_legacy::StableHash::stable_hash(inner, sequence_number.next_child(), state); + } +} + +impl StableHash for Entity { + fn stable_hash(&self, field_address: H::Addr, state: &mut H) { + let Self(inner) = self; + StableHash::stable_hash(inner, field_address.child(0), state); + } +} + +#[macro_export] +macro_rules! entity { + ($($name:ident: $value:expr,)*) => { + { + let mut result = $crate::data::store::Entity::new(); + $( + result.set(stringify!($name), $crate::data::store::Value::from($value)); + )* + result + } + }; + ($($name:ident: $value:expr),*) => { + entity! {$($name: $value,)*} + }; +} + +impl Entity { + /// Creates a new entity with no attributes set. + pub fn new() -> Self { + Default::default() + } + + pub fn get(&self, key: &str) -> Option<&Value> { + self.0.get(key) + } + + pub fn insert(&mut self, key: String, value: Value) -> Option { + self.0.insert(key, value) + } + + pub fn remove(&mut self, key: &str) -> Option { + self.0.remove(key) + } + + pub fn contains_key(&self, key: &str) -> bool { + self.0.contains_key(key) + } + + // This collects the entity into an ordered vector so that it can be iterated deterministically. + pub fn sorted(self) -> Vec<(String, Value)> { + let mut v: Vec<_> = self.0.into_iter().collect(); + v.sort_by(|(k1, _), (k2, _)| k1.cmp(k2)); + v + } + + /// Return the ID of this entity. If the ID is a string, return the + /// string. If it is `Bytes`, return it as a hex string with a `0x` + /// prefix. If the ID is not set or anything but a `String` or `Bytes`, + /// return an error + pub fn id(&self) -> Result { + match self.get("id") { + None => Err(anyhow!("Entity is missing an `id` attribute")), + Some(Value::String(s)) => Ok(s.to_owned()), + Some(Value::Bytes(b)) => Ok(b.to_string()), + _ => Err(anyhow!("Entity has non-string `id` attribute")), + } + } + + /// Convenience method to save having to `.into()` the arguments. + pub fn set(&mut self, name: impl Into, value: impl Into) -> Option { + self.0.insert(name.into(), value.into()) + } + + /// Merges an entity update `update` into this entity. + /// + /// If a key exists in both entities, the value from `update` is chosen. + /// If a key only exists on one entity, the value from that entity is chosen. + /// If a key is set to `Value::Null` in `update`, the key/value pair is set to `Value::Null`. + pub fn merge(&mut self, update: Entity) { + for (key, value) in update.0.into_iter() { + self.insert(key, value); + } + } + + /// Merges an entity update `update` into this entity, removing `Value::Null` values. + /// + /// If a key exists in both entities, the value from `update` is chosen. + /// If a key only exists on one entity, the value from that entity is chosen. + /// If a key is set to `Value::Null` in `update`, the key/value pair is removed. + pub fn merge_remove_null_fields(&mut self, update: Entity) { + for (key, value) in update.0.into_iter() { + match value { + Value::Null => self.remove(&key), + _ => self.insert(key, value), + }; + } + } + + /// Validate that this entity matches the object type definition in the + /// schema. An entity that passes these checks can be stored + /// successfully in the subgraph's database schema + pub fn validate(&self, schema: &Schema, key: &EntityKey) -> Result<(), anyhow::Error> { + fn scalar_value_type(schema: &Schema, field_type: &s::Type) -> ValueType { + use s::TypeDefinition as t; + match field_type { + s::Type::NamedType(name) => ValueType::from_str(name).unwrap_or_else(|_| { + match schema.document.get_named_type(name) { + Some(t::Object(obj_type)) => { + let id = obj_type.field("id").expect("all object types have an id"); + scalar_value_type(schema, &id.field_type) + } + Some(t::Interface(intf)) => { + // Validation checks that all implementors of an + // interface use the same type for `id`. It is + // therefore enough to use the id type of one of + // the implementors + match schema + .types_for_interface() + .get(&EntityType::new(intf.name.clone())) + .expect("interface type names are known") + .first() + { + None => { + // Nothing is implementing this interface; we assume it's of type string + // see also: id-type-for-unimplemented-interfaces + ValueType::String + } + Some(obj_type) => { + let id = + obj_type.field("id").expect("all object types have an id"); + scalar_value_type(schema, &id.field_type) + } + } + } + Some(t::Enum(_)) => ValueType::String, + Some(t::Scalar(_)) => unreachable!("user-defined scalars are not used"), + Some(t::Union(_)) => unreachable!("unions are not used"), + Some(t::InputObject(_)) => unreachable!("inputObjects are not used"), + None => unreachable!("names of field types have been validated"), + } + }), + s::Type::NonNullType(inner) => scalar_value_type(schema, inner), + s::Type::ListType(inner) => scalar_value_type(schema, inner), + } + } + + if key.entity_type.is_poi() { + // Users can't modify Poi entities, and therefore they do not + // need to be validated. In addition, the schema has no object + // type for them, and validation would therefore fail + return Ok(()); + } + let object_type_definitions = schema.document.get_object_type_definitions(); + let object_type = object_type_definitions + .iter() + .find(|object_type| key.entity_type.as_str() == object_type.name) + .with_context(|| { + format!( + "Entity {}[{}]: unknown entity type `{}`", + key.entity_type, key.entity_id, key.entity_type + ) + })?; + + for field in &object_type.fields { + let is_derived = field.is_derived(); + match (self.get(&field.name), is_derived) { + (Some(value), false) => { + let scalar_type = scalar_value_type(schema, &field.field_type); + if field.field_type.is_list() { + // Check for inhomgeneous lists to produce a better + // error message for them; other problems, like + // assigning a scalar to a list will be caught below + if let Value::List(elts) = value { + for (index, elt) in elts.iter().enumerate() { + if !elt.is_assignable(&scalar_type, false) { + anyhow::bail!( + "Entity {}[{}]: field `{}` is of type {}, but the value `{}` \ + contains a {} at index {}", + key.entity_type, + key.entity_id, + field.name, + &field.field_type, + value, + elt.type_name(), + index + ); + } + } + } + } + if !value.is_assignable(&scalar_type, field.field_type.is_list()) { + anyhow::bail!( + "Entity {}[{}]: the value `{}` for field `{}` must have type {} but has type {}", + key.entity_type, + key.entity_id, + value, + field.name, + &field.field_type, + value.type_name() + ); + } + } + (None, false) => { + if field.field_type.is_non_null() { + anyhow::bail!( + "Entity {}[{}]: missing value for non-nullable field `{}`", + key.entity_type, + key.entity_id, + field.name, + ); + } + } + (Some(_), true) => { + anyhow::bail!( + "Entity {}[{}]: field `{}` is derived and can not be set", + key.entity_type, + key.entity_id, + field.name, + ); + } + (None, true) => { + // derived fields should not be set + } + } + } + Ok(()) + } +} + +impl From for BTreeMap { + fn from(entity: Entity) -> BTreeMap { + entity.0.into_iter().map(|(k, v)| (k, v.into())).collect() + } +} + +impl From for q::Value { + fn from(entity: Entity) -> q::Value { + q::Value::Object(entity.into()) + } +} + +impl From> for Entity { + fn from(m: HashMap) -> Entity { + Entity(m) + } +} + +impl<'a> From<&'a Entity> for Cow<'a, Entity> { + fn from(entity: &'a Entity) -> Self { + Cow::Borrowed(entity) + } +} + +impl<'a> From> for Entity { + fn from(entries: Vec<(&'a str, Value)>) -> Entity { + Entity::from(HashMap::from_iter( + entries.into_iter().map(|(k, v)| (String::from(k), v)), + )) + } +} + +impl CacheWeight for Entity { + fn indirect_weight(&self) -> usize { + self.0.indirect_weight() + } +} + +impl GasSizeOf for Entity { + fn gas_size_of(&self) -> Gas { + self.0.gas_size_of() + } +} + +/// A value that can (maybe) be converted to an `Entity`. +pub trait TryIntoEntity { + fn try_into_entity(self) -> Result; +} + +#[test] +fn value_bytes() { + let graphql_value = r::Value::String("0x8f494c66afc1d3f8ac1b45df21f02a46".to_owned()); + let ty = q::Type::NamedType(BYTES_SCALAR.to_owned()); + let from_query = Value::from_query_value(&graphql_value, &ty).unwrap(); + assert_eq!( + from_query, + Value::Bytes(scalar::Bytes::from( + &[143, 73, 76, 102, 175, 193, 211, 248, 172, 27, 69, 223, 33, 240, 42, 70][..] + )) + ); + assert_eq!(r::Value::from(from_query), graphql_value); +} + +#[test] +fn value_bigint() { + let big_num = "340282366920938463463374607431768211456"; + let graphql_value = r::Value::String(big_num.to_owned()); + let ty = q::Type::NamedType(BIG_INT_SCALAR.to_owned()); + let from_query = Value::from_query_value(&graphql_value, &ty).unwrap(); + assert_eq!( + from_query, + Value::BigInt(FromStr::from_str(big_num).unwrap()) + ); + assert_eq!(r::Value::from(from_query), graphql_value); +} + +#[test] +fn entity_validation() { + fn make_thing(name: &str) -> Entity { + let mut thing = Entity::new(); + thing.set("id", name); + thing.set("name", name); + thing.set("stuff", "less"); + thing.set("favorite_color", "red"); + thing.set("things", Value::List(vec![])); + thing + } + + fn check(thing: Entity, errmsg: &str) { + const DOCUMENT: &str = " + enum Color { red, yellow, blue } + interface Stuff { id: ID!, name: String! } + type Cruft @entity { + id: ID!, + thing: Thing! + } + type Thing @entity { + id: ID!, + name: String!, + favorite_color: Color, + stuff: Stuff, + things: [Thing!]! + # Make sure we do not validate derived fields; it's ok + # to store a thing with a null Cruft + cruft: Cruft! @derivedFrom(field: \"thing\") + }"; + let subgraph = DeploymentHash::new("doesntmatter").unwrap(); + let schema = + crate::prelude::Schema::parse(DOCUMENT, subgraph).expect("Failed to parse test schema"); + let id = thing.id().unwrap_or("none".to_owned()); + let key = EntityKey::data("Thing".to_owned(), id.clone()); + + let err = thing.validate(&schema, &key); + if errmsg == "" { + assert!( + err.is_ok(), + "checking entity {}: expected ok but got {}", + id, + err.unwrap_err() + ); + } else { + if let Err(e) = err { + assert_eq!(errmsg, e.to_string(), "checking entity {}", id); + } else { + panic!( + "Expected error `{}` but got ok when checking entity {}", + errmsg, id + ); + } + } + } + + let mut thing = make_thing("t1"); + thing.set("things", Value::from(vec!["thing1", "thing2"])); + check(thing, ""); + + let thing = make_thing("t2"); + check(thing, ""); + + let mut thing = make_thing("t3"); + thing.remove("name"); + check( + thing, + "Entity Thing[t3]: missing value for non-nullable field `name`", + ); + + let mut thing = make_thing("t4"); + thing.remove("things"); + check( + thing, + "Entity Thing[t4]: missing value for non-nullable field `things`", + ); + + let mut thing = make_thing("t5"); + thing.set("name", Value::Int(32)); + check( + thing, + "Entity Thing[t5]: the value `32` for field `name` must \ + have type String! but has type Int", + ); + + let mut thing = make_thing("t6"); + thing.set("things", Value::List(vec!["thing1".into(), 17.into()])); + check( + thing, + "Entity Thing[t6]: field `things` is of type [Thing!]!, \ + but the value `[thing1, 17]` contains a Int at index 1", + ); + + let mut thing = make_thing("t7"); + thing.remove("favorite_color"); + thing.remove("stuff"); + check(thing, ""); + + let mut thing = make_thing("t8"); + thing.set("cruft", "wat"); + check( + thing, + "Entity Thing[t8]: field `cruft` is derived and can not be set", + ); +} + +#[test] +fn fmt_debug() { + assert_eq!("String(\"hello\")", format!("{:?}", Value::from("hello"))); + assert_eq!("Int(17)", format!("{:?}", Value::Int(17))); + assert_eq!("Bool(false)", format!("{:?}", Value::Bool(false))); + assert_eq!("Null", format!("{:?}", Value::Null)); + + let bd = Value::BigDecimal(scalar::BigDecimal::from(-0.17)); + assert_eq!("BigDecimal(-0.17)", format!("{:?}", bd)); + + let bytes = Value::Bytes(scalar::Bytes::from([222, 173, 190, 239].as_slice())); + assert_eq!("Bytes(0xdeadbeef)", format!("{:?}", bytes)); + + let bi = Value::BigInt(scalar::BigInt::from(-17i32)); + assert_eq!("BigInt(-17)", format!("{:?}", bi)); +} diff --git a/graph/src/data/store/scalar.rs b/graph/src/data/store/scalar.rs new file mode 100644 index 0000000..fb43cd7 --- /dev/null +++ b/graph/src/data/store/scalar.rs @@ -0,0 +1,739 @@ +use diesel::deserialize::FromSql; +use diesel::serialize::ToSql; +use diesel_derives::{AsExpression, FromSqlRow}; +use hex; +use num_bigint; +use serde::{self, Deserialize, Serialize}; +use stable_hash::utils::AsInt; +use stable_hash::{FieldAddress, StableHash}; +use stable_hash_legacy::SequenceNumber; +use thiserror::Error; +use web3::types::*; + +use std::convert::{TryFrom, TryInto}; +use std::fmt::{self, Display, Formatter}; +use std::io::Write; +use std::ops::{Add, BitAnd, BitOr, Deref, Div, Mul, Rem, Shl, Shr, Sub}; +use std::str::FromStr; + +pub use num_bigint::Sign as BigIntSign; + +use crate::blockchain::BlockHash; +use crate::util::stable_hash_glue::{impl_stable_hash, AsBytes}; + +/// All operations on `BigDecimal` return a normalized value. +// Caveat: The exponent is currently an i64 and may overflow. See +// https://github.com/akubera/bigdecimal-rs/issues/54. +// Using `#[serde(from = "BigDecimal"]` makes sure deserialization calls `BigDecimal::new()`. +#[derive( + Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, AsExpression, FromSqlRow, +)] +#[serde(from = "bigdecimal::BigDecimal")] +#[sql_type = "diesel::sql_types::Numeric"] +pub struct BigDecimal(bigdecimal::BigDecimal); + +impl From for BigDecimal { + fn from(big_decimal: bigdecimal::BigDecimal) -> Self { + BigDecimal(big_decimal).normalized() + } +} + +impl BigDecimal { + /// These are the limits of IEEE-754 decimal128, a format we may want to switch to. See + /// https://en.wikipedia.org/wiki/Decimal128_floating-point_format. + pub const MIN_EXP: i32 = -6143; + pub const MAX_EXP: i32 = 6144; + pub const MAX_SIGNFICANT_DIGITS: i32 = 34; + + pub fn new(digits: BigInt, exp: i64) -> Self { + // bigdecimal uses `scale` as the opposite of the power of ten, so negate `exp`. + Self::from(bigdecimal::BigDecimal::new(digits.0, -exp)) + } + + pub fn parse_bytes(bytes: &[u8]) -> Option { + bigdecimal::BigDecimal::parse_bytes(bytes, 10).map(Self) + } + + pub fn zero() -> BigDecimal { + use bigdecimal::Zero; + + BigDecimal(bigdecimal::BigDecimal::zero()) + } + + pub fn as_bigint_and_exponent(&self) -> (num_bigint::BigInt, i64) { + self.0.as_bigint_and_exponent() + } + + pub fn digits(&self) -> u64 { + self.0.digits() + } + + // Copy-pasted from `bigdecimal::BigDecimal::normalize`. We can use the upstream version once it + // is included in a released version supported by Diesel. + #[must_use] + pub fn normalized(&self) -> BigDecimal { + if self == &BigDecimal::zero() { + return BigDecimal::zero(); + } + + // Round to the maximum significant digits. + let big_decimal = self.0.with_prec(Self::MAX_SIGNFICANT_DIGITS as u64); + + let (bigint, exp) = big_decimal.as_bigint_and_exponent(); + let (sign, mut digits) = bigint.to_radix_be(10); + let trailing_count = digits.iter().rev().take_while(|i| **i == 0).count(); + digits.truncate(digits.len() - trailing_count); + let int_val = num_bigint::BigInt::from_radix_be(sign, &digits, 10).unwrap(); + let scale = exp - trailing_count as i64; + + BigDecimal(bigdecimal::BigDecimal::new(int_val, scale)) + } +} + +impl Display for BigDecimal { + fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> { + self.0.fmt(f) + } +} + +impl fmt::Debug for BigDecimal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "BigDecimal({})", self.0) + } +} + +impl FromStr for BigDecimal { + type Err = ::Err; + + fn from_str(s: &str) -> Result { + Ok(Self::from(bigdecimal::BigDecimal::from_str(s)?)) + } +} + +impl From for BigDecimal { + fn from(n: i32) -> Self { + Self::from(bigdecimal::BigDecimal::from(n)) + } +} + +impl From for BigDecimal { + fn from(n: i64) -> Self { + Self::from(bigdecimal::BigDecimal::from(n)) + } +} + +impl From for BigDecimal { + fn from(n: u64) -> Self { + Self::from(bigdecimal::BigDecimal::from(n)) + } +} + +impl From for BigDecimal { + fn from(n: f64) -> Self { + Self::from(bigdecimal::BigDecimal::from(n)) + } +} + +impl Add for BigDecimal { + type Output = Self; + + fn add(self, other: Self) -> Self { + Self::from(self.0.add(other.0)) + } +} + +impl Sub for BigDecimal { + type Output = Self; + + fn sub(self, other: Self) -> Self { + Self::from(self.0.sub(other.0)) + } +} + +impl Mul for BigDecimal { + type Output = Self; + + fn mul(self, other: Self) -> Self { + Self::from(self.0.mul(other.0)) + } +} + +impl Div for BigDecimal { + type Output = Self; + + fn div(self, other: Self) -> Self { + if other == BigDecimal::from(0) { + panic!("Cannot divide by zero-valued `BigDecimal`!") + } + + Self::from(self.0.div(other.0)) + } +} + +// Used only for JSONB support +impl ToSql for BigDecimal { + fn to_sql( + &self, + out: &mut diesel::serialize::Output, + ) -> diesel::serialize::Result { + <_ as ToSql>::to_sql(&self.0, out) + } +} + +impl FromSql for BigDecimal { + fn from_sql( + bytes: Option<&::RawValue>, + ) -> diesel::deserialize::Result { + Ok(Self::from(bigdecimal::BigDecimal::from_sql(bytes)?)) + } +} + +impl bigdecimal::ToPrimitive for BigDecimal { + fn to_i64(&self) -> Option { + self.0.to_i64() + } + fn to_u64(&self) -> Option { + self.0.to_u64() + } +} + +impl stable_hash_legacy::StableHash for BigDecimal { + fn stable_hash( + &self, + mut sequence_number: H::Seq, + state: &mut H, + ) { + let (int, exp) = self.as_bigint_and_exponent(); + // This only allows for backward compatible changes between + // BigDecimal and unsigned ints + stable_hash_legacy::StableHash::stable_hash(&exp, sequence_number.next_child(), state); + stable_hash_legacy::StableHash::stable_hash(&BigInt(int), sequence_number, state); + } +} + +impl StableHash for BigDecimal { + fn stable_hash(&self, field_address: H::Addr, state: &mut H) { + // This implementation allows for backward compatible changes from integers (signed or unsigned) + // when the exponent is zero. + let (int, exp) = self.as_bigint_and_exponent(); + StableHash::stable_hash(&exp, field_address.child(1), state); + // Normally it would be a red flag to pass field_address in after having used a child slot. + // But, we know the implementation of StableHash for BigInt will not use child(1) and that + // it will not in the future due to having no forward schema evolutions for ints and the + // stability guarantee. + // + // For reference, ints use child(0) for the sign and write the little endian bytes to the parent slot. + BigInt(int).stable_hash(field_address, state); + } +} + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct BigInt(num_bigint::BigInt); + +impl stable_hash_legacy::StableHash for BigInt { + #[inline] + fn stable_hash( + &self, + sequence_number: H::Seq, + state: &mut H, + ) { + stable_hash_legacy::utils::AsInt { + is_negative: self.0.sign() == BigIntSign::Minus, + little_endian: &self.to_bytes_le().1, + } + .stable_hash(sequence_number, state) + } +} + +impl StableHash for BigInt { + fn stable_hash(&self, field_address: H::Addr, state: &mut H) { + AsInt { + is_negative: self.0.sign() == BigIntSign::Minus, + little_endian: &self.to_bytes_le().1, + } + .stable_hash(field_address, state) + } +} + +#[derive(Error, Debug)] +pub enum BigIntOutOfRangeError { + #[error("Cannot convert negative BigInt into type")] + Negative, + #[error("BigInt value is too large for type")] + Overflow, +} + +impl<'a> TryFrom<&'a BigInt> for u64 { + type Error = BigIntOutOfRangeError; + fn try_from(value: &'a BigInt) -> Result { + let (sign, bytes) = value.to_bytes_le(); + + if sign == num_bigint::Sign::Minus { + return Err(BigIntOutOfRangeError::Negative); + } + + if bytes.len() > 8 { + return Err(BigIntOutOfRangeError::Overflow); + } + + // Replace this with u64::from_le_bytes when stabilized + let mut n = 0u64; + let mut shift_dist = 0; + for b in bytes { + n |= (b as u64) << shift_dist; + shift_dist += 8; + } + Ok(n) + } +} + +impl TryFrom for u64 { + type Error = BigIntOutOfRangeError; + fn try_from(value: BigInt) -> Result { + (&value).try_into() + } +} + +impl fmt::Debug for BigInt { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "BigInt({})", self) + } +} + +impl BigInt { + pub fn from_unsigned_bytes_le(bytes: &[u8]) -> Self { + BigInt(num_bigint::BigInt::from_bytes_le( + num_bigint::Sign::Plus, + bytes, + )) + } + + pub fn from_signed_bytes_le(bytes: &[u8]) -> Self { + BigInt(num_bigint::BigInt::from_signed_bytes_le(bytes)) + } + + pub fn from_signed_bytes_be(bytes: &[u8]) -> Self { + BigInt(num_bigint::BigInt::from_signed_bytes_be(bytes)) + } + + pub fn to_bytes_le(&self) -> (BigIntSign, Vec) { + self.0.to_bytes_le() + } + + pub fn to_bytes_be(&self) -> (BigIntSign, Vec) { + self.0.to_bytes_be() + } + + pub fn to_signed_bytes_le(&self) -> Vec { + self.0.to_signed_bytes_le() + } + + /// Deprecated. Use try_into instead + pub fn to_u64(&self) -> u64 { + self.try_into().unwrap() + } + + pub fn from_unsigned_u256(n: &U256) -> Self { + let mut bytes: [u8; 32] = [0; 32]; + n.to_little_endian(&mut bytes); + BigInt::from_unsigned_bytes_le(&bytes) + } + + pub fn from_signed_u256(n: &U256) -> Self { + let mut bytes: [u8; 32] = [0; 32]; + n.to_little_endian(&mut bytes); + BigInt::from_signed_bytes_le(&bytes) + } + + pub fn to_signed_u256(&self) -> U256 { + let bytes = self.to_signed_bytes_le(); + if self < &BigInt::from(0) { + assert!( + bytes.len() <= 32, + "BigInt value does not fit into signed U256" + ); + let mut i_bytes: [u8; 32] = [255; 32]; + i_bytes[..bytes.len()].copy_from_slice(&bytes); + U256::from_little_endian(&i_bytes) + } else { + U256::from_little_endian(&bytes) + } + } + + pub fn to_unsigned_u256(&self) -> U256 { + let (sign, bytes) = self.to_bytes_le(); + assert!( + sign == BigIntSign::NoSign || sign == BigIntSign::Plus, + "negative value encountered for U256: {}", + self + ); + U256::from_little_endian(&bytes) + } + + pub fn pow(self, exponent: u8) -> Self { + use num_traits::pow::Pow; + + BigInt(self.0.pow(&exponent)) + } + + pub fn bits(&self) -> usize { + self.0.bits() + } +} + +impl Display for BigInt { + fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> { + self.0.fmt(f) + } +} + +impl From for BigInt { + fn from(big_int: num_bigint::BigInt) -> BigInt { + BigInt(big_int) + } +} + +impl From for BigInt { + fn from(i: i32) -> BigInt { + BigInt(i.into()) + } +} + +impl From for BigInt { + fn from(i: u64) -> BigInt { + BigInt(i.into()) + } +} + +impl From for BigInt { + fn from(i: i64) -> BigInt { + BigInt(i.into()) + } +} + +impl From for BigInt { + /// This implementation assumes that U64 represents an unsigned U64, + /// and not a signed U64 (aka int64 in Solidity). Right now, this is + /// all we need (for block numbers). If it ever becomes necessary to + /// handle signed U64s, we should add the same + /// `{to,from}_{signed,unsigned}_u64` methods that we have for U64. + fn from(n: U64) -> BigInt { + BigInt::from(n.as_u64()) + } +} + +impl From for BigInt { + /// This implementation assumes that U128 represents an unsigned U128, + /// and not a signed U128 (aka int128 in Solidity). Right now, this is + /// all we need (for block numbers). If it ever becomes necessary to + /// handle signed U128s, we should add the same + /// `{to,from}_{signed,unsigned}_u128` methods that we have for U256. + fn from(n: U128) -> BigInt { + let mut bytes: [u8; 16] = [0; 16]; + n.to_little_endian(&mut bytes); + BigInt::from_unsigned_bytes_le(&bytes) + } +} + +impl FromStr for BigInt { + type Err = ::Err; + + fn from_str(s: &str) -> Result { + num_bigint::BigInt::from_str(s).map(BigInt) + } +} + +impl Serialize for BigInt { + fn serialize(&self, serializer: S) -> Result { + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for BigInt { + fn deserialize>(deserializer: D) -> Result { + use serde::de::Error; + + let decimal_string = ::deserialize(deserializer)?; + BigInt::from_str(&decimal_string).map_err(D::Error::custom) + } +} + +impl Add for BigInt { + type Output = BigInt; + + fn add(self, other: BigInt) -> BigInt { + BigInt(self.0.add(other.0)) + } +} + +impl Sub for BigInt { + type Output = BigInt; + + fn sub(self, other: BigInt) -> BigInt { + BigInt(self.0.sub(other.0)) + } +} + +impl Mul for BigInt { + type Output = BigInt; + + fn mul(self, other: BigInt) -> BigInt { + BigInt(self.0.mul(other.0)) + } +} + +impl Div for BigInt { + type Output = BigInt; + + fn div(self, other: BigInt) -> BigInt { + if other == BigInt::from(0) { + panic!("Cannot divide by zero-valued `BigInt`!") + } + + BigInt(self.0.div(other.0)) + } +} + +impl Rem for BigInt { + type Output = BigInt; + + fn rem(self, other: BigInt) -> BigInt { + BigInt(self.0.rem(other.0)) + } +} + +impl BitOr for BigInt { + type Output = Self; + + fn bitor(self, other: Self) -> Self { + Self::from(self.0.bitor(other.0)) + } +} + +impl BitAnd for BigInt { + type Output = Self; + + fn bitand(self, other: Self) -> Self { + Self::from(self.0.bitand(other.0)) + } +} + +impl Shl for BigInt { + type Output = Self; + + fn shl(self, bits: u8) -> Self { + Self::from(self.0.shl(bits.into())) + } +} + +impl Shr for BigInt { + type Output = Self; + + fn shr(self, bits: u8) -> Self { + Self::from(self.0.shr(bits.into())) + } +} + +/// A byte array that's serialized as a hex string prefixed by `0x`. +#[derive(Clone, PartialEq, Eq)] +pub struct Bytes(Box<[u8]>); + +impl Deref for Bytes { + type Target = [u8]; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl fmt::Debug for Bytes { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "Bytes(0x{})", hex::encode(&self.0)) + } +} + +impl_stable_hash!(Bytes(transparent: AsBytes)); + +impl Bytes { + pub fn as_slice(&self) -> &[u8] { + &self.0 + } +} + +impl Display for Bytes { + fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> { + write!(f, "0x{}", hex::encode(&self.0)) + } +} + +impl FromStr for Bytes { + type Err = hex::FromHexError; + + fn from_str(s: &str) -> Result { + hex::decode(s.trim_start_matches("0x")).map(|x| Bytes(x.into())) + } +} + +impl<'a> From<&'a [u8]> for Bytes { + fn from(array: &[u8]) -> Self { + Bytes(array.into()) + } +} + +impl From
for Bytes { + fn from(address: Address) -> Bytes { + Bytes::from(address.as_ref()) + } +} + +impl From for Bytes { + fn from(bytes: web3::types::Bytes) -> Bytes { + Bytes::from(bytes.0.as_slice()) + } +} + +impl From for Bytes { + fn from(hash: BlockHash) -> Self { + Bytes(hash.0) + } +} + +impl Serialize for Bytes { + fn serialize(&self, serializer: S) -> Result { + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Bytes { + fn deserialize>(deserializer: D) -> Result { + use serde::de::Error; + + let hex_string = ::deserialize(deserializer)?; + Bytes::from_str(&hex_string).map_err(D::Error::custom) + } +} + +impl From<[u8; N]> for Bytes { + fn from(array: [u8; N]) -> Bytes { + Bytes(array.into()) + } +} + +impl From> for Bytes { + fn from(vec: Vec) -> Self { + Bytes(vec.into()) + } +} + +#[cfg(test)] +mod test { + use super::{BigDecimal, BigInt, Bytes}; + use stable_hash_legacy::crypto::SetHasher; + use stable_hash_legacy::prelude::*; + use stable_hash_legacy::utils::stable_hash; + use std::str::FromStr; + use web3::types::U64; + + #[test] + fn bigint_to_from_u64() { + for n in 0..100 { + let u = U64::from(n as u64); + let bn = BigInt::from(u); + assert_eq!(n, bn.to_u64()); + } + } + + fn crypto_stable_hash(value: impl StableHash) -> ::Out { + stable_hash::(&value) + } + + fn same_stable_hash(left: impl StableHash, right: impl StableHash) { + let left = crypto_stable_hash(left); + let right = crypto_stable_hash(right); + assert_eq!(left, right); + } + + #[test] + fn big_int_stable_hash_same_as_int() { + same_stable_hash(0, BigInt::from(0u64)); + same_stable_hash(1, BigInt::from(1u64)); + same_stable_hash(1u64 << 20, BigInt::from(1u64 << 20)); + + same_stable_hash(-1, BigInt::from_signed_bytes_le(&(-1i32).to_le_bytes())); + } + + #[test] + fn big_decimal_stable_hash_same_as_uint() { + same_stable_hash(0, BigDecimal::from(0u64)); + same_stable_hash(4, BigDecimal::from(4i64)); + same_stable_hash(1u64 << 21, BigDecimal::from(1u64 << 21)); + } + + #[test] + fn big_decimal_stable() { + let cases = vec![ + ( + "28b09c9c3f3e2fe037631b7fbccdf65c37594073016d8bf4bb0708b3fda8066a", + "0.1", + ), + ( + "74fb39f038d2f1c8975740bf2651a5ac0403330ee7e9367f9563cbd7d21086bd", + "-0.1", + ), + ( + "1d79e0476bc5d6fe6074fb54636b04fd3bc207053c767d9cb5e710ba5f002441", + "198.98765544", + ), + ( + "e63f6ad2c65f193aa9eba18dd7e1043faa2d6183597ba84c67765aaa95c95351", + "0.00000093937698", + ), + ( + "6b06b34cc714810072988dc46c493c66a6b6c2c2dd0030271aa3adf3b3f21c20", + "98765587998098786876.0", + ), + ]; + for (hash, s) in cases.iter() { + let dec = BigDecimal::from_str(s).unwrap(); + assert_eq!(*hash, hex::encode(crypto_stable_hash(dec))); + } + } + + #[test] + fn test_normalize() { + let vals = vec![ + ( + BigDecimal::new(BigInt::from(10), -2), + BigDecimal(bigdecimal::BigDecimal::new(1.into(), 1)), + "0.1", + ), + ( + BigDecimal::new(BigInt::from(132400), 4), + BigDecimal(bigdecimal::BigDecimal::new(1324.into(), -6)), + "1324000000", + ), + ( + BigDecimal::new(BigInt::from(1_900_000), -3), + BigDecimal(bigdecimal::BigDecimal::new(19.into(), -2)), + "1900", + ), + (BigDecimal::new(0.into(), 3), BigDecimal::zero(), "0"), + (BigDecimal::new(0.into(), -5), BigDecimal::zero(), "0"), + ]; + + for (not_normalized, normalized, string) in vals { + assert_eq!(not_normalized.normalized(), normalized); + assert_eq!(not_normalized.normalized().to_string(), string); + assert_eq!(normalized.to_string(), string); + } + } + + #[test] + fn fmt_debug() { + let bi = BigInt::from(-17); + let bd = BigDecimal::new(bi.clone(), -2); + let bytes = Bytes::from([222, 173, 190, 239].as_slice()); + assert_eq!("BigInt(-17)", format!("{:?}", bi)); + assert_eq!("BigDecimal(-0.17)", format!("{:?}", bd)); + assert_eq!("Bytes(0xdeadbeef)", format!("{:?}", bytes)); + } +} diff --git a/graph/src/data/subgraph/api_version.rs b/graph/src/data/subgraph/api_version.rs new file mode 100644 index 0000000..7975639 --- /dev/null +++ b/graph/src/data/subgraph/api_version.rs @@ -0,0 +1,108 @@ +use itertools::Itertools; +use semver::Version; +use std::collections::BTreeSet; +use thiserror::Error; + +pub const API_VERSION_0_0_2: Version = Version::new(0, 0, 2); + +/// This version adds a new subgraph validation step that rejects manifests whose mappings have +/// different API versions if at least one of them is equal to or higher than `0.0.5`. +pub const API_VERSION_0_0_5: Version = Version::new(0, 0, 5); + +// Adds two new fields to the Transaction object: nonce and input +pub const API_VERSION_0_0_6: Version = Version::new(0, 0, 6); + +/// Enables event handlers to require transaction receipts in the runtime. +pub const API_VERSION_0_0_7: Version = Version::new(0, 0, 7); + +/// Before this check was introduced, there were already subgraphs in the wild with spec version +/// 0.0.3, due to confusion with the api version. To avoid breaking those, we accept 0.0.3 though it +/// doesn't exist. +pub const SPEC_VERSION_0_0_3: Version = Version::new(0, 0, 3); + +/// This version supports subgraph feature management. +pub const SPEC_VERSION_0_0_4: Version = Version::new(0, 0, 4); + +/// This version supports event handlers having access to transaction receipts. +pub const SPEC_VERSION_0_0_5: Version = Version::new(0, 0, 5); + +/// Enables the Fast POI calculation variant. +pub const SPEC_VERSION_0_0_6: Version = Version::new(0, 0, 6); + +/// Enables offchain data sources. +pub const SPEC_VERSION_0_0_7: Version = Version::new(0, 0, 7); + +pub const MIN_SPEC_VERSION: Version = Version::new(0, 0, 2); + +#[derive(Clone, PartialEq, Debug)] +pub struct UnifiedMappingApiVersion(Option); + +impl UnifiedMappingApiVersion { + pub fn equal_or_greater_than(&self, other_version: &Version) -> bool { + assert!( + other_version >= &API_VERSION_0_0_5, + "api versions before 0.0.5 should not be used for comparison" + ); + match &self.0 { + Some(version) => version >= other_version, + None => false, + } + } + + pub(super) fn try_from_versions( + versions: impl Iterator, + ) -> Result { + let unique_versions: BTreeSet = versions.collect(); + + let all_below_referential_version = unique_versions.iter().all(|v| *v < API_VERSION_0_0_5); + let all_the_same = unique_versions.len() == 1; + + let unified_version: Option = match (all_below_referential_version, all_the_same) { + (false, false) => return Err(DifferentMappingApiVersions(unique_versions)), + (false, true) => Some(unique_versions.iter().nth(0).unwrap().clone()), + (true, _) => None, + }; + + Ok(UnifiedMappingApiVersion(unified_version)) + } +} + +pub(super) fn format_versions(versions: &BTreeSet) -> String { + versions.iter().map(ToString::to_string).join(", ") +} + +#[derive(Error, Debug, PartialEq)] +#[error("Expected a single apiVersion for mappings. Found: {}.", format_versions(.0))] +pub struct DifferentMappingApiVersions(pub BTreeSet); + +#[test] +fn unified_mapping_api_version_from_iterator() { + let input = [ + vec![Version::new(0, 0, 5), Version::new(0, 0, 5)], // Ok(Some(0.0.5)) + vec![Version::new(0, 0, 6), Version::new(0, 0, 6)], // Ok(Some(0.0.6)) + vec![Version::new(0, 0, 3), Version::new(0, 0, 4)], // Ok(None) + vec![Version::new(0, 0, 4), Version::new(0, 0, 4)], // Ok(None) + vec![Version::new(0, 0, 3), Version::new(0, 0, 5)], // Err({0.0.3, 0.0.5}) + vec![Version::new(0, 0, 6), Version::new(0, 0, 5)], // Err({0.0.5, 0.0.6}) + ]; + let output: [Result; 6] = [ + Ok(UnifiedMappingApiVersion(Some(Version::new(0, 0, 5)))), + Ok(UnifiedMappingApiVersion(Some(Version::new(0, 0, 6)))), + Ok(UnifiedMappingApiVersion(None)), + Ok(UnifiedMappingApiVersion(None)), + Err(DifferentMappingApiVersions( + input[4].iter().cloned().collect::>(), + )), + Err(DifferentMappingApiVersions( + input[5].iter().cloned().collect::>(), + )), + ]; + for (version_vec, expected_unified_version) in input.iter().zip(output.iter()) { + let unified = UnifiedMappingApiVersion::try_from_versions(version_vec.iter().cloned()); + match (unified, expected_unified_version) { + (Ok(a), Ok(b)) => assert_eq!(a, *b), + (Err(a), Err(b)) => assert_eq!(a, *b), + _ => panic!(), + } + } +} diff --git a/graph/src/data/subgraph/features.rs b/graph/src/data/subgraph/features.rs new file mode 100644 index 0000000..29c769f --- /dev/null +++ b/graph/src/data/subgraph/features.rs @@ -0,0 +1,176 @@ +//! Functions to detect subgraph features. +//! +//! The rationale of this module revolves around the concept of feature declaration and detection. +//! +//! Features are declared in the `subgraph.yml` file, also known as the subgraph's manifest, and are +//! validated by a graph-node instance during the deploy phase or by direct request. +//! +//! A feature validation error will be triggered if a subgraph use any feature without declaring it +//! in the `features` section of the manifest file. +//! +//! Feature validation is performed by the [`validate_subgraph_features`] function. + +use crate::{ + blockchain::Blockchain, + data::{graphql::DocumentExt, schema::Schema, subgraph::SubgraphManifest}, + prelude::{Deserialize, Serialize}, +}; +use itertools::Itertools; +use std::{collections::BTreeSet, fmt, str::FromStr}; + +use super::calls_host_fn; + +/// This array must contain all IPFS-related functions that are exported by the host WASM runtime. +/// +/// For reference, search this codebase for: ff652476-e6ad-40e4-85b8-e815d6c6e5e2 +const IPFS_ON_ETHEREUM_CONTRACTS_FUNCTION_NAMES: [&'static str; 3] = + ["ipfs.cat", "ipfs.getBlock", "ipfs.map"]; + +#[derive(Debug, Deserialize, Serialize, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "camelCase")] +pub enum SubgraphFeature { + NonFatalErrors, + Grafting, + FullTextSearch, + #[serde(alias = "nonDeterministicIpfs")] + IpfsOnEthereumContracts, +} + +impl fmt::Display for SubgraphFeature { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + serde_plain::to_string(self) + .map_err(|_| fmt::Error) + .and_then(|x| write!(f, "{}", x)) + } +} + +impl FromStr for SubgraphFeature { + type Err = anyhow::Error; + + fn from_str(value: &str) -> anyhow::Result { + serde_plain::from_str(value) + .map_err(|_error| anyhow::anyhow!("Invalid subgraph feature: {}", value)) + } +} + +#[derive(PartialEq, Eq, PartialOrd, Ord, Serialize, thiserror::Error, Debug)] +pub enum SubgraphFeatureValidationError { + /// A feature is used by the subgraph but it is not declared in the `features` section of the manifest file. + #[error("The feature `{}` is used by the subgraph but it is not declared in the manifest.", fmt_subgraph_features(.0))] + Undeclared(BTreeSet), + + /// The provided compiled mapping is not a valid WASM module. + #[error("Failed to parse the provided mapping WASM module")] + InvalidMapping, +} + +fn fmt_subgraph_features(subgraph_features: &BTreeSet) -> String { + subgraph_features.iter().join(", ") +} + +pub fn validate_subgraph_features( + manifest: &SubgraphManifest, +) -> Result, SubgraphFeatureValidationError> { + let declared: &BTreeSet = &manifest.features; + let used = detect_features(manifest)?; + let undeclared: BTreeSet = used.difference(declared).cloned().collect(); + if !undeclared.is_empty() { + Err(SubgraphFeatureValidationError::Undeclared(undeclared)) + } else { + Ok(used) + } +} + +pub fn detect_features( + manifest: &SubgraphManifest, +) -> Result, InvalidMapping> { + let features = vec![ + detect_non_fatal_errors(manifest), + detect_grafting(manifest), + detect_full_text_search(&manifest.schema), + detect_ipfs_on_ethereum_contracts(manifest)?, + ] + .into_iter() + .flatten() + .collect(); + Ok(features) +} + +fn detect_non_fatal_errors( + manifest: &SubgraphManifest, +) -> Option { + if manifest.features.contains(&SubgraphFeature::NonFatalErrors) { + Some(SubgraphFeature::NonFatalErrors) + } else { + None + } +} + +fn detect_grafting(manifest: &SubgraphManifest) -> Option { + manifest.graft.as_ref().map(|_| SubgraphFeature::Grafting) +} + +fn detect_full_text_search(schema: &Schema) -> Option { + match schema.document.get_fulltext_directives() { + Ok(directives) => (!directives.is_empty()).then(|| SubgraphFeature::FullTextSearch), + + Err(_) => { + // Currently we return an error from `get_fulltext_directives` function if the + // fullTextSearch directive is found. + Some(SubgraphFeature::FullTextSearch) + } + } +} + +pub struct InvalidMapping; + +impl From for SubgraphFeatureValidationError { + fn from(_: InvalidMapping) -> Self { + SubgraphFeatureValidationError::InvalidMapping + } +} + +fn detect_ipfs_on_ethereum_contracts( + manifest: &SubgraphManifest, +) -> Result, InvalidMapping> { + for runtime in manifest.runtimes() { + for function_name in IPFS_ON_ETHEREUM_CONTRACTS_FUNCTION_NAMES { + if calls_host_fn(&runtime, function_name).map_err(|_| InvalidMapping)? { + return Ok(Some(SubgraphFeature::IpfsOnEthereumContracts)); + } + } + } + Ok(None) +} + +#[cfg(test)] +mod tests { + use super::*; + use SubgraphFeature::*; + const VARIANTS: [SubgraphFeature; 4] = [ + NonFatalErrors, + Grafting, + FullTextSearch, + IpfsOnEthereumContracts, + ]; + const STRING: [&str; 4] = [ + "nonFatalErrors", + "grafting", + "fullTextSearch", + "ipfsOnEthereumContracts", + ]; + + #[test] + fn subgraph_feature_display() { + for (variant, string) in VARIANTS.iter().zip(STRING.iter()) { + assert_eq!(variant.to_string(), *string) + } + } + + #[test] + fn subgraph_feature_from_str() { + for (variant, string) in VARIANTS.iter().zip(STRING.iter()) { + assert_eq!(SubgraphFeature::from_str(string).unwrap(), *variant) + } + } +} diff --git a/graph/src/data/subgraph/mod.rs b/graph/src/data/subgraph/mod.rs new file mode 100644 index 0000000..3e451eb --- /dev/null +++ b/graph/src/data/subgraph/mod.rs @@ -0,0 +1,873 @@ +/// Rust representation of the GraphQL schema for a `SubgraphManifest`. +pub mod schema; + +/// API version and spec version. +pub mod api_version; +pub use api_version::*; + +pub mod features; +pub mod status; + +pub use features::{SubgraphFeature, SubgraphFeatureValidationError}; + +use anyhow::ensure; +use anyhow::{anyhow, Error}; +use futures03::{future::try_join3, stream::FuturesOrdered, TryStreamExt as _}; +use semver::Version; +use serde::{de, ser}; +use serde_yaml; +use slog::{debug, info, Logger}; +use stable_hash::{FieldAddress, StableHash}; +use stable_hash_legacy::SequenceNumber; +use std::{collections::BTreeSet, marker::PhantomData}; +use thiserror::Error; +use wasmparser; +use web3::types::Address; + +use crate::{ + blockchain::{BlockPtr, Blockchain, DataSource as _}, + components::{ + link_resolver::LinkResolver, + store::{DeploymentLocator, StoreError, SubgraphStore}, + }, + data::{ + graphql::TryFromValue, + query::QueryExecutionError, + schema::{Schema, SchemaImportError, SchemaValidationError}, + store::Entity, + subgraph::features::validate_subgraph_features, + }, + data_source::{ + offchain::OFFCHAIN_KINDS, DataSource, DataSourceTemplate, UnresolvedDataSource, + UnresolvedDataSourceTemplate, + }, + prelude::{r, CheapClone, ENV_VARS}, +}; + +use crate::prelude::{impl_slog_value, BlockNumber, Deserialize, Serialize}; + +use std::fmt; +use std::ops::Deref; +use std::str::FromStr; +use std::sync::Arc; + +/// Deserialize an Address (with or without '0x' prefix). +fn deserialize_address<'de, D>(deserializer: D) -> Result, D::Error> +where + D: de::Deserializer<'de>, +{ + use serde::de::Error; + + let s: String = de::Deserialize::deserialize(deserializer)?; + let address = s.trim_start_matches("0x"); + Address::from_str(address) + .map_err(D::Error::custom) + .map(Some) +} + +/// The IPFS hash used to identifiy a deployment externally, i.e., the +/// `Qm..` string that `graph-cli` prints when deploying to a subgraph +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct DeploymentHash(String); + +impl stable_hash_legacy::StableHash for DeploymentHash { + #[inline] + fn stable_hash( + &self, + mut sequence_number: H::Seq, + state: &mut H, + ) { + let Self(inner) = self; + stable_hash_legacy::StableHash::stable_hash(inner, sequence_number.next_child(), state); + } +} + +impl StableHash for DeploymentHash { + fn stable_hash(&self, field_address: H::Addr, state: &mut H) { + let Self(inner) = self; + stable_hash::StableHash::stable_hash(inner, field_address.child(0), state); + } +} + +impl_slog_value!(DeploymentHash); + +/// `DeploymentHash` is fixed-length so cheap to clone. +impl CheapClone for DeploymentHash {} + +impl DeploymentHash { + /// Check that `s` is a valid `SubgraphDeploymentId` and create a new one. + /// If `s` is longer than 46 characters, or contains characters other than + /// alphanumeric characters or `_`, return s (as a `String`) as the error + pub fn new(s: impl Into) -> Result { + let s = s.into(); + + // Enforce length limit + if s.len() > 46 { + return Err(s); + } + + // Check that the ID contains only allowed characters. + if !s.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return Err(s); + } + + // Allow only deployment id's for 'real' subgraphs, not the old + // metadata subgraph. + if s == "subgraphs" { + return Err(s); + } + + Ok(DeploymentHash(s)) + } + + pub fn to_ipfs_link(&self) -> Link { + Link { + link: format!("/ipfs/{}", self), + } + } +} + +impl Deref for DeploymentHash { + type Target = String; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl fmt::Display for DeploymentHash { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl ser::Serialize for DeploymentHash { + fn serialize(&self, serializer: S) -> Result + where + S: ser::Serializer, + { + serializer.serialize_str(&self.0) + } +} + +impl<'de> de::Deserialize<'de> for DeploymentHash { + fn deserialize(deserializer: D) -> Result + where + D: de::Deserializer<'de>, + { + let s: String = de::Deserialize::deserialize(deserializer)?; + DeploymentHash::new(s) + .map_err(|s| de::Error::invalid_value(de::Unexpected::Str(&s), &"valid subgraph name")) + } +} + +impl TryFromValue for DeploymentHash { + fn try_from_value(value: &r::Value) -> Result { + Self::new(String::try_from_value(value)?) + .map_err(|s| anyhow!("Invalid subgraph ID `{}`", s)) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct SubgraphName(String); + +impl SubgraphName { + pub fn new(s: impl Into) -> Result { + let s = s.into(); + + // Note: these validation rules must be kept consistent with the validation rules + // implemented in any other components that rely on subgraph names. + + // Enforce length limits + if s.is_empty() || s.len() > 255 { + return Err(()); + } + + // Check that the name contains only allowed characters. + if !s + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '/') + { + return Err(()); + } + + // Parse into components and validate each + for part in s.split('/') { + // Each part must be non-empty + if part.is_empty() { + return Err(()); + } + + // To keep URLs unambiguous, reserve the token "graphql" + if part == "graphql" { + return Err(()); + } + + // Part should not start or end with a special character. + let first_char = part.chars().next().unwrap(); + let last_char = part.chars().last().unwrap(); + if !first_char.is_ascii_alphanumeric() + || !last_char.is_ascii_alphanumeric() + || !part.chars().any(|c| c.is_ascii_alphabetic()) + { + return Err(()); + } + } + + Ok(SubgraphName(s)) + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +impl fmt::Display for SubgraphName { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl ser::Serialize for SubgraphName { + fn serialize(&self, serializer: S) -> Result + where + S: ser::Serializer, + { + serializer.serialize_str(&self.0) + } +} + +impl<'de> de::Deserialize<'de> for SubgraphName { + fn deserialize(deserializer: D) -> Result + where + D: de::Deserializer<'de>, + { + let s: String = de::Deserialize::deserialize(deserializer)?; + SubgraphName::new(s.clone()) + .map_err(|()| de::Error::invalid_value(de::Unexpected::Str(&s), &"valid subgraph name")) + } +} + +/// Result of a creating a subgraph in the registar. +#[derive(Serialize)] +pub struct CreateSubgraphResult { + /// The ID of the subgraph that was created. + pub id: String, +} + +#[derive(Error, Debug)] +pub enum SubgraphRegistrarError { + #[error("subgraph resolve error: {0}")] + ResolveError(SubgraphManifestResolveError), + #[error("subgraph already exists: {0}")] + NameExists(String), + #[error("subgraph name not found: {0}")] + NameNotFound(String), + #[error("network not supported by registrar: {0}")] + NetworkNotSupported(Error), + #[error("deployment not found: {0}")] + DeploymentNotFound(String), + #[error("deployment assignment unchanged: {0}")] + DeploymentAssignmentUnchanged(String), + #[error("subgraph registrar internal query error: {0}")] + QueryExecutionError(#[from] QueryExecutionError), + #[error("subgraph registrar error with store: {0}")] + StoreError(StoreError), + #[error("subgraph validation error: {}", display_vector(.0))] + ManifestValidationError(Vec), + #[error("subgraph deployment error: {0}")] + SubgraphDeploymentError(StoreError), + #[error("subgraph registrar error: {0}")] + Unknown(#[from] anyhow::Error), +} + +impl From for SubgraphRegistrarError { + fn from(e: StoreError) -> Self { + match e { + StoreError::DeploymentNotFound(id) => SubgraphRegistrarError::DeploymentNotFound(id), + e => SubgraphRegistrarError::StoreError(e), + } + } +} + +impl From for SubgraphRegistrarError { + fn from(e: SubgraphManifestValidationError) -> Self { + SubgraphRegistrarError::ManifestValidationError(vec![e]) + } +} + +#[derive(Error, Debug)] +pub enum SubgraphAssignmentProviderError { + #[error("Subgraph resolve error: {0}")] + ResolveError(Error), + /// Occurs when attempting to remove a subgraph that's not hosted. + #[error("Subgraph with ID {0} already running")] + AlreadyRunning(DeploymentHash), + #[error("Subgraph with ID {0} is not running")] + NotRunning(DeploymentLocator), + #[error("Subgraph provider error: {0}")] + Unknown(#[from] anyhow::Error), +} + +impl From<::diesel::result::Error> for SubgraphAssignmentProviderError { + fn from(e: ::diesel::result::Error) -> Self { + SubgraphAssignmentProviderError::Unknown(e.into()) + } +} + +#[derive(Error, Debug)] +pub enum SubgraphManifestValidationWarning { + #[error("schema validation produced warnings: {0:?}")] + SchemaValidationWarning(SchemaImportError), +} + +#[derive(Error, Debug)] +pub enum SubgraphManifestValidationError { + #[error("subgraph has no data sources")] + NoDataSources, + #[error("subgraph source address is required")] + SourceAddressRequired, + #[error("subgraph cannot index data from different Ethereum networks")] + MultipleEthereumNetworks, + #[error("subgraph must have at least one Ethereum network data source")] + EthereumNetworkRequired, + #[error("the specified block must exist on the Ethereum network")] + BlockNotFound(String), + #[error("imported schema(s) are invalid: {0:?}")] + SchemaImportError(Vec), + #[error("schema validation failed: {0:?}")] + SchemaValidationError(Vec), + #[error("the graft base is invalid: {0}")] + GraftBaseInvalid(String), + #[error("subgraph must use a single apiVersion across its data sources. Found: {}", format_versions(&(.0).0))] + DifferentApiVersions(#[from] DifferentMappingApiVersions), + #[error(transparent)] + FeatureValidationError(#[from] SubgraphFeatureValidationError), + #[error("data source {0} is invalid: {1}")] + DataSourceValidation(String, Error), +} + +#[derive(Error, Debug)] +pub enum SubgraphManifestResolveError { + #[error("parse error: {0}")] + ParseError(#[from] serde_yaml::Error), + #[error("subgraph is not UTF-8")] + NonUtf8, + #[error("subgraph is not valid YAML")] + InvalidFormat, + #[error("resolve error: {0}")] + ResolveError(anyhow::Error), +} + +/// Data source contexts are conveniently represented as entities. +pub type DataSourceContext = Entity; + +/// IPLD link. +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +pub struct Link { + #[serde(rename = "/")] + pub link: String, +} + +impl From for Link { + fn from(s: S) -> Self { + Self { + link: s.to_string(), + } + } +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct UnresolvedSchema { + pub file: Link, +} + +impl UnresolvedSchema { + pub async fn resolve( + self, + id: DeploymentHash, + resolver: &Arc, + logger: &Logger, + ) -> Result { + info!(logger, "Resolve schema"; "link" => &self.file.link); + + let schema_bytes = resolver.cat(logger, &self.file).await?; + Schema::parse(&String::from_utf8(schema_bytes)?, id) + } +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct Source { + /// The contract address for the data source. We allow data sources + /// without an address for 'wildcard' triggers that catch all possible + /// events with the given `abi` + #[serde(default, deserialize_with = "deserialize_address")] + pub address: Option
, + pub abi: String, + #[serde(rename = "startBlock", default)] + pub start_block: BlockNumber, +} + +pub fn calls_host_fn(runtime: &[u8], host_fn: &str) -> anyhow::Result { + use wasmparser::Payload; + + for payload in wasmparser::Parser::new(0).parse_all(runtime) { + if let Payload::ImportSection(s) = payload? { + for import in s { + let import = import?; + if import.field == Some(host_fn) { + return Ok(true); + } + } + } + } + + Ok(false) +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Graft { + pub base: DeploymentHash, + pub block: BlockNumber, +} + +impl Graft { + async fn validate( + &self, + store: Arc, + ) -> Result<(), SubgraphManifestValidationError> { + use SubgraphManifestValidationError::*; + + let last_processed_block = store + .least_block_ptr(&self.base) + .await + .map_err(|e| GraftBaseInvalid(e.to_string()))?; + let is_base_healthy = store + .is_healthy(&self.base) + .await + .map_err(|e| GraftBaseInvalid(e.to_string()))?; + + // We are being defensive here: we don't know which specific + // instance of a subgraph we will use as the base for the graft, + // since the notion of which of these instances is active can change + // between this check and when the graft actually happens when the + // subgraph is started. We therefore check that any instance of the + // base subgraph is suitable. + match (last_processed_block, is_base_healthy) { + (None, _) => Err(GraftBaseInvalid(format!( + "failed to graft onto `{}` since it has not processed any blocks", + self.base + ))), + (Some(ptr), true) if ptr.number < self.block => Err(GraftBaseInvalid(format!( + "failed to graft onto `{}` at block {} since it has only processed block {}", + self.base, self.block, ptr.number + ))), + // If the base deployment is failed *and* the `graft.block` is not + // less than the `base.block`, the graft shouldn't be permitted. + // + // The developer should change their `graft.block` in the manifest + // to `base.block - 1` or less. + (Some(ptr), false) if !(self.block < ptr.number) => Err(GraftBaseInvalid(format!( + "failed to graft onto `{}` at block {} since it's not healthy. You can graft it starting at block {} backwards", + self.base, self.block, ptr.number - 1 + ))), + (Some(_), _) => Ok(()), + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BaseSubgraphManifest { + pub id: DeploymentHash, + pub spec_version: Version, + #[serde(default)] + pub features: BTreeSet, + pub description: Option, + pub repository: Option, + pub schema: S, + pub data_sources: Vec, + pub graft: Option, + #[serde(default)] + pub templates: Vec, + #[serde(skip_serializing, default)] + pub chain: PhantomData, +} + +/// SubgraphManifest with IPFS links unresolved +type UnresolvedSubgraphManifest = BaseSubgraphManifest< + C, + UnresolvedSchema, + UnresolvedDataSource, + UnresolvedDataSourceTemplate, +>; + +/// SubgraphManifest validated with IPFS links resolved +pub type SubgraphManifest = + BaseSubgraphManifest, DataSourceTemplate>; + +/// Unvalidated SubgraphManifest +pub struct UnvalidatedSubgraphManifest(SubgraphManifest); + +impl UnvalidatedSubgraphManifest { + /// Entry point for resolving a subgraph definition. + /// Right now the only supported links are of the form: + /// `/ipfs/QmUmg7BZC1YP1ca66rRtWKxpXp77WgVHrnv263JtDuvs2k` + pub async fn resolve( + id: DeploymentHash, + raw: serde_yaml::Mapping, + resolver: &Arc, + logger: &Logger, + max_spec_version: semver::Version, + ) -> Result { + Ok(Self( + SubgraphManifest::resolve_from_raw(id, raw, resolver, logger, max_spec_version).await?, + )) + } + + /// Validates the subgraph manifest file. + /// + /// Graft base validation will be skipped if the parameter `validate_graft_base` is false. + pub async fn validate( + self, + store: Arc, + validate_graft_base: bool, + ) -> Result, Vec> { + let (schemas, _) = self.0.schema.resolve_schema_references(store.clone()); + + let mut errors: Vec = vec![]; + + // Validate that the manifest has at least one data source + if self.0.data_sources.is_empty() { + errors.push(SubgraphManifestValidationError::NoDataSources); + } + + for ds in &self.0.data_sources { + errors.extend(ds.validate().into_iter().map(|e| { + SubgraphManifestValidationError::DataSourceValidation(ds.name().to_owned(), e) + })); + } + + // For API versions newer than 0.0.5, validate that all mappings uses the same api_version + if let Err(different_api_versions) = self.0.unified_mapping_api_version() { + errors.push(different_api_versions.into()); + }; + + let mut networks = self + .0 + .data_sources + .iter() + .filter_map(|d| Some(d.as_onchain()?.network()?.to_string())) + .collect::>(); + networks.sort(); + networks.dedup(); + match networks.len() { + 0 => errors.push(SubgraphManifestValidationError::EthereumNetworkRequired), + 1 => (), + _ => errors.push(SubgraphManifestValidationError::MultipleEthereumNetworks), + } + + self.0 + .schema + .validate(&schemas) + .err() + .into_iter() + .for_each(|schema_errors| { + errors.push(SubgraphManifestValidationError::SchemaValidationError( + schema_errors, + )); + }); + + if let Some(graft) = &self.0.graft { + if ENV_VARS.disable_grafts { + errors.push(SubgraphManifestValidationError::GraftBaseInvalid( + "Grafting of subgraphs is currently disabled".to_owned(), + )); + } + if validate_graft_base { + if let Err(graft_err) = graft.validate(store).await { + errors.push(graft_err); + } + } + } + + // Validate subgraph feature usage and declaration. + if self.0.spec_version >= SPEC_VERSION_0_0_4 { + if let Err(feature_validation_error) = validate_subgraph_features(&self.0) { + errors.push(feature_validation_error.into()) + } + } + + match errors.is_empty() { + true => Ok(self.0), + false => Err(errors), + } + } + + pub fn spec_version(&self) -> &Version { + &self.0.spec_version + } +} + +impl SubgraphManifest { + /// Entry point for resolving a subgraph definition. + pub async fn resolve_from_raw( + id: DeploymentHash, + mut raw: serde_yaml::Mapping, + resolver: &Arc, + logger: &Logger, + max_spec_version: semver::Version, + ) -> Result { + // Inject the IPFS hash as the ID of the subgraph into the definition. + raw.insert("id".into(), id.to_string().into()); + + // Parse the YAML data into an UnresolvedSubgraphManifest + let unresolved: UnresolvedSubgraphManifest = serde_yaml::from_value(raw.into())?; + + debug!(logger, "Features {:?}", unresolved.features); + + let resolved = unresolved + .resolve(resolver, logger, max_spec_version) + .await + .map_err(SubgraphManifestResolveError::ResolveError)?; + + if (resolved.spec_version < SPEC_VERSION_0_0_7) + && resolved + .data_sources + .iter() + .any(|ds| OFFCHAIN_KINDS.contains(&ds.kind())) + { + return Err(SubgraphManifestResolveError::ResolveError(anyhow!( + "Offchain data sources not supported prior to {}", + SPEC_VERSION_0_0_7 + ))); + } + + Ok(resolved) + } + + pub fn network_name(&self) -> String { + // Assume the manifest has been validated, ensuring network names are homogenous + self.data_sources + .iter() + .find_map(|d| Some(d.as_onchain()?.network()?.to_string())) + .expect("Validated manifest does not have a network defined on any datasource") + } + + pub fn start_blocks(&self) -> Vec { + self.data_sources + .iter() + .filter_map(|d| Some(d.as_onchain()?.start_block())) + .collect() + } + + pub fn api_versions(&self) -> impl Iterator + '_ { + self.templates + .iter() + .map(|template| template.api_version()) + .chain(self.data_sources.iter().map(|source| source.api_version())) + } + + pub fn runtimes(&self) -> impl Iterator>> + '_ { + self.templates + .iter() + .filter_map(|template| template.runtime()) + .chain( + self.data_sources + .iter() + .filter_map(|source| source.runtime()), + ) + } + + pub fn unified_mapping_api_version( + &self, + ) -> Result { + UnifiedMappingApiVersion::try_from_versions(self.api_versions()) + } +} + +impl UnresolvedSubgraphManifest { + pub async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + max_spec_version: semver::Version, + ) -> Result, anyhow::Error> { + let UnresolvedSubgraphManifest { + id, + spec_version, + features, + description, + repository, + schema, + data_sources, + graft, + templates, + chain, + } = self; + + if !(MIN_SPEC_VERSION..=max_spec_version.clone()).contains(&spec_version) { + return Err(anyhow!( + "This Graph Node only supports manifest spec versions between {} and {}, but subgraph `{}` uses `{}`", + MIN_SPEC_VERSION, + max_spec_version, + id, + spec_version + )); + } + + let ds_count = data_sources.len(); + if ds_count as u64 + templates.len() as u64 > u32::MAX as u64 { + return Err(anyhow!( + "Subgraph has too many declared data sources and templates", + )); + } + + let (schema, data_sources, templates) = try_join3( + schema.resolve(id.clone(), &resolver, logger), + data_sources + .into_iter() + .enumerate() + .map(|(idx, ds)| ds.resolve(&resolver, logger, idx as u32)) + .collect::>() + .try_collect::>(), + templates + .into_iter() + .enumerate() + .map(|(idx, template)| { + template.resolve(&resolver, logger, ds_count as u32 + idx as u32) + }) + .collect::>() + .try_collect::>(), + ) + .await?; + + for ds in &data_sources { + ensure!( + semver::VersionReq::parse(&format!("<= {}", ENV_VARS.mappings.max_api_version)) + .unwrap() + .matches(&ds.api_version()), + "The maximum supported mapping API version of this indexer is {}, but `{}` was found", + ENV_VARS.mappings.max_api_version, + ds.api_version() + ); + } + + Ok(SubgraphManifest { + id, + spec_version, + features, + description, + repository, + schema, + data_sources, + graft, + templates, + chain, + }) + } +} + +/// Important details about the current state of a subgraph deployment +/// used while executing queries against a deployment +/// +/// The `reorg_count` and `max_reorg_depth` fields are maintained (in the +/// database) by `store::metadata::forward_block_ptr` and +/// `store::metadata::revert_block_ptr` which get called as part of transacting +/// new entities into the store or reverting blocks. +#[derive(Debug, Clone)] +pub struct DeploymentState { + pub id: DeploymentHash, + /// The number of blocks that were ever reverted in this subgraph. This + /// number increases monotonically every time a block is reverted + pub reorg_count: u32, + /// The maximum number of blocks we ever reorged without moving a block + /// forward in between + pub max_reorg_depth: u32, + /// The last block that the subgraph has processed + pub latest_block: BlockPtr, + /// The earliest block that the subgraph has processed + pub earliest_block_number: BlockNumber, +} + +impl DeploymentState { + /// Is this subgraph deployed and has it processed any blocks? + pub fn is_deployed(&self) -> bool { + self.latest_block.number > 0 + } + + pub fn block_queryable(&self, block: BlockNumber) -> Result<(), String> { + if block > self.latest_block.number { + return Err(format!( + "subgraph {} has only indexed up to block number {} \ + and data for block number {} is therefore not yet available", + self.id, self.latest_block.number, block + )); + } + if block < self.earliest_block_number { + return Err(format!( + "subgraph {} only has data starting at block number {} \ + and data for block number {} is therefore not available", + self.id, self.earliest_block_number, block + )); + } + Ok(()) + } +} + +fn display_vector(input: &[impl std::fmt::Display]) -> impl std::fmt::Display { + let formatted_errors = input + .iter() + .map(ToString::to_string) + .collect::>() + .join("; "); + format!("[{}]", formatted_errors) +} + +#[test] +fn test_subgraph_name_validation() { + assert!(SubgraphName::new("a").is_ok()); + assert!(SubgraphName::new("a/a").is_ok()); + assert!(SubgraphName::new("a-lOng-name_with_0ne-component").is_ok()); + assert!(SubgraphName::new("a-long-name_with_one-3omponent").is_ok()); + assert!(SubgraphName::new("a/b_c").is_ok()); + assert!(SubgraphName::new("A/Z-Z").is_ok()); + assert!(SubgraphName::new("a1/A-A").is_ok()); + assert!(SubgraphName::new("aaa/a1").is_ok()); + assert!(SubgraphName::new("1a/aaaa").is_ok()); + assert!(SubgraphName::new("aaaa/1a").is_ok()); + assert!(SubgraphName::new("2nena4test/lala").is_ok()); + + assert!(SubgraphName::new("").is_err()); + assert!(SubgraphName::new("/a").is_err()); + assert!(SubgraphName::new("a/").is_err()); + assert!(SubgraphName::new("a//a").is_err()); + assert!(SubgraphName::new("a/0").is_err()); + assert!(SubgraphName::new("a/_").is_err()); + assert!(SubgraphName::new("a/a_").is_err()); + assert!(SubgraphName::new("a/_a").is_err()); + assert!(SubgraphName::new("aaaa aaaaa").is_err()); + assert!(SubgraphName::new("aaaa!aaaaa").is_err()); + assert!(SubgraphName::new("aaaa+aaaaa").is_err()); + assert!(SubgraphName::new("a/graphql").is_err()); + assert!(SubgraphName::new("graphql/a").is_err()); + assert!(SubgraphName::new("this-component-is-very-long-but-we-dont-care").is_ok()); +} + +#[test] +fn test_display_vector() { + let manifest_validation_error = SubgraphRegistrarError::ManifestValidationError(vec![ + SubgraphManifestValidationError::NoDataSources, + SubgraphManifestValidationError::SourceAddressRequired, + ]); + + let expected_display_message = + "subgraph validation error: [subgraph has no data sources; subgraph source address is required]" + .to_string(); + + assert_eq!( + expected_display_message, + format!("{}", manifest_validation_error) + ) +} diff --git a/graph/src/data/subgraph/schema.rs b/graph/src/data/subgraph/schema.rs new file mode 100644 index 0000000..c8e5d7f --- /dev/null +++ b/graph/src/data/subgraph/schema.rs @@ -0,0 +1,223 @@ +//! Entity types that contain the graph-node state. + +use anyhow::{anyhow, Error}; +use hex; +use lazy_static::lazy_static; +use rand::rngs::OsRng; +use rand::Rng; +use std::str::FromStr; +use std::{fmt, fmt::Display}; + +use super::DeploymentHash; +use crate::data::graphql::TryFromValue; +use crate::data::store::Value; +use crate::data::subgraph::SubgraphManifest; +use crate::prelude::*; +use crate::util::stable_hash_glue::impl_stable_hash; +use crate::{blockchain::Blockchain, components::store::EntityType}; + +pub const POI_TABLE: &str = "poi2$"; +lazy_static! { + pub static ref POI_OBJECT: EntityType = EntityType::new("Poi$".to_string()); +} + +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum SubgraphHealth { + /// Syncing without errors. + Healthy, + + /// Syncing but has errors. + Unhealthy, + + /// No longer syncing due to fatal error. + Failed, +} + +impl SubgraphHealth { + pub fn as_str(&self) -> &'static str { + match self { + SubgraphHealth::Healthy => "healthy", + SubgraphHealth::Unhealthy => "unhealthy", + SubgraphHealth::Failed => "failed", + } + } + + pub fn is_failed(&self) -> bool { + match self { + SubgraphHealth::Failed => true, + SubgraphHealth::Healthy | SubgraphHealth::Unhealthy => false, + } + } +} + +impl FromStr for SubgraphHealth { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s { + "healthy" => Ok(SubgraphHealth::Healthy), + "unhealthy" => Ok(SubgraphHealth::Unhealthy), + "failed" => Ok(SubgraphHealth::Failed), + _ => Err(anyhow!("failed to parse `{}` as SubgraphHealth", s)), + } + } +} + +impl From for String { + fn from(health: SubgraphHealth) -> String { + health.as_str().to_string() + } +} + +impl From for Value { + fn from(health: SubgraphHealth) -> Value { + String::from(health).into() + } +} + +impl From for q::Value { + fn from(health: SubgraphHealth) -> q::Value { + q::Value::Enum(health.into()) + } +} + +impl From for r::Value { + fn from(health: SubgraphHealth) -> r::Value { + r::Value::Enum(health.into()) + } +} + +impl TryFromValue for SubgraphHealth { + fn try_from_value(value: &r::Value) -> Result { + match value { + r::Value::Enum(health) => SubgraphHealth::from_str(health), + _ => Err(anyhow!( + "cannot parse value as SubgraphHealth: `{:?}`", + value + )), + } + } +} + +/// The deployment data that is needed to create a deployment +pub struct DeploymentCreate { + pub manifest: SubgraphManifestEntity, + pub earliest_block: Option, + pub graft_base: Option, + pub graft_block: Option, + pub debug_fork: Option, +} + +impl DeploymentCreate { + pub fn new( + source_manifest: &SubgraphManifest, + earliest_block: Option, + ) -> Self { + Self { + manifest: SubgraphManifestEntity::from(source_manifest), + earliest_block: earliest_block.cheap_clone(), + graft_base: None, + graft_block: None, + debug_fork: None, + } + } + + pub fn graft(mut self, base: Option<(DeploymentHash, BlockPtr)>) -> Self { + if let Some((subgraph, ptr)) = base { + self.graft_base = Some(subgraph); + self.graft_block = Some(ptr); + } + self + } + + pub fn debug(mut self, fork: Option) -> Self { + self.debug_fork = fork; + self + } +} + +/// The representation of a subgraph deployment when reading an existing +/// deployment +#[derive(Debug)] +pub struct SubgraphDeploymentEntity { + pub manifest: SubgraphManifestEntity, + pub failed: bool, + pub health: SubgraphHealth, + pub synced: bool, + pub fatal_error: Option, + pub non_fatal_errors: Vec, + pub earliest_block: Option, + pub latest_block: Option, + pub graft_base: Option, + pub graft_block: Option, + pub debug_fork: Option, + pub reorg_count: i32, + pub current_reorg_depth: i32, + pub max_reorg_depth: i32, +} + +#[derive(Debug)] +pub struct SubgraphManifestEntity { + pub spec_version: String, + pub description: Option, + pub repository: Option, + pub features: Vec, + pub schema: String, +} + +impl<'a, C: Blockchain> From<&'a super::SubgraphManifest> for SubgraphManifestEntity { + fn from(manifest: &'a super::SubgraphManifest) -> Self { + Self { + spec_version: manifest.spec_version.to_string(), + description: manifest.description.clone(), + repository: manifest.repository.clone(), + features: manifest.features.iter().map(|f| f.to_string()).collect(), + schema: manifest.schema.document.clone().to_string(), + } + } +} + +#[derive(Clone, Debug)] +pub struct SubgraphError { + pub subgraph_id: DeploymentHash, + pub message: String, + pub block_ptr: Option, + pub handler: Option, + + // `true` if we are certain the error is deterministic. If in doubt, this is `false`. + pub deterministic: bool, +} + +impl Display for SubgraphError { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "{}", self.message)?; + if let Some(handler) = &self.handler { + write!(f, " in handler `{}`", handler)?; + } + if let Some(block_ptr) = &self.block_ptr { + write!(f, " at block {}", block_ptr)?; + } + Ok(()) + } +} + +impl_stable_hash!(SubgraphError { + subgraph_id, + message, + block_ptr, + handler, + deterministic +}); + +pub fn generate_entity_id() -> String { + // Fast crypto RNG from operating system + let mut rng = OsRng::default(); + + // 128 random bits + let id_bytes: [u8; 16] = rng.gen(); + + // 32 hex chars + // Comparable to uuidv4, but without the hyphens, + // and without spending bits on a version identifier. + hex::encode(id_bytes) +} diff --git a/graph/src/data/subgraph/status.rs b/graph/src/data/subgraph/status.rs new file mode 100644 index 0000000..0813221 --- /dev/null +++ b/graph/src/data/subgraph/status.rs @@ -0,0 +1,171 @@ +//! Support for the indexing status API + +use super::schema::{SubgraphError, SubgraphHealth}; +use crate::blockchain::BlockHash; +use crate::components::store::{BlockNumber, DeploymentId}; +use crate::data::graphql::{object, IntoValue}; +use crate::prelude::{r, BlockPtr, Value}; + +pub enum Filter { + /// Get all versions for the named subgraph + SubgraphName(String), + /// Get the current (`true`) or pending (`false`) version of the named + /// subgraph + SubgraphVersion(String, bool), + /// Get the status of all deployments whose the given given IPFS hashes + Deployments(Vec), + /// Get the status of all deployments with the given ids + DeploymentIds(Vec), +} + +/// Light wrapper around `EthereumBlockPointer` that is compatible with GraphQL values. +#[derive(Debug)] +pub struct EthereumBlock(BlockPtr); + +impl EthereumBlock { + pub fn new(hash: BlockHash, number: BlockNumber) -> Self { + EthereumBlock(BlockPtr::new(hash, number)) + } + + pub fn to_ptr(self) -> BlockPtr { + self.0 + } + + pub fn number(&self) -> i32 { + self.0.number + } +} + +impl IntoValue for EthereumBlock { + fn into_value(self) -> r::Value { + object! { + __typename: "EthereumBlock", + hash: self.0.hash_hex(), + number: format!("{}", self.0.number), + } + } +} + +impl From for EthereumBlock { + fn from(ptr: BlockPtr) -> Self { + Self(ptr) + } +} + +/// Indexing status information related to the chain. Right now, we only +/// support Ethereum, but once we support more chains, we'll have to turn this into +/// an enum +#[derive(Debug)] +pub struct ChainInfo { + /// The network name (e.g. `mainnet`, `ropsten`, `rinkeby`, `kovan` or `goerli`). + pub network: String, + /// The current head block of the chain. + pub chain_head_block: Option, + /// The earliest block available for this subgraph (only the number). + pub earliest_block_number: BlockNumber, + /// The latest block that the subgraph has synced to. + pub latest_block: Option, +} + +impl IntoValue for ChainInfo { + fn into_value(self) -> r::Value { + let ChainInfo { + network, + chain_head_block, + earliest_block_number, + latest_block, + } = self; + object! { + // `__typename` is needed for the `ChainIndexingStatus` interface + // in GraphQL to work. + __typename: "EthereumIndexingStatus", + network: network, + chainHeadBlock: chain_head_block, + earliestBlock: object! { + __typename: "EarliestBlock", + number: earliest_block_number, + hash: "0x0" + }, + latestBlock: latest_block, + } + } +} + +#[derive(Debug)] +pub struct Info { + pub id: DeploymentId, + + /// The deployment hash + pub subgraph: String, + + /// Whether or not the subgraph has synced all the way to the current chain head. + pub synced: bool, + pub health: SubgraphHealth, + pub fatal_error: Option, + pub non_fatal_errors: Vec, + + /// Indexing status on different chains involved in the subgraph's data sources. + pub chains: Vec, + + pub entity_count: u64, + + /// ID of the Graph Node that the subgraph is indexed by. + pub node: Option, +} + +impl IntoValue for Info { + fn into_value(self) -> r::Value { + let Info { + id: _, + subgraph, + chains, + entity_count, + fatal_error, + health, + node, + non_fatal_errors, + synced, + } = self; + + fn subgraph_error_to_value(subgraph_error: SubgraphError) -> r::Value { + let SubgraphError { + subgraph_id, + message, + block_ptr, + handler, + deterministic, + } = subgraph_error; + + object! { + __typename: "SubgraphError", + subgraphId: subgraph_id.to_string(), + message: message, + handler: handler, + block: object! { + __typename: "Block", + number: block_ptr.as_ref().map(|x| x.number), + hash: block_ptr.map(|x| r::Value::from(Value::Bytes(x.hash.into()))), + }, + deterministic: deterministic, + } + } + + let non_fatal_errors: Vec<_> = non_fatal_errors + .into_iter() + .map(subgraph_error_to_value) + .collect(); + let fatal_error_val = fatal_error.map_or(r::Value::Null, subgraph_error_to_value); + + object! { + __typename: "SubgraphIndexingStatus", + subgraph: subgraph, + synced: synced, + health: r::Value::from(health), + fatalError: fatal_error_val, + nonFatalErrors: non_fatal_errors, + chains: chains.into_iter().map(|chain| chain.into_value()).collect::>(), + entityCount: format!("{}", entity_count), + node: node, + } + } +} diff --git a/graph/src/data/subscription/error.rs b/graph/src/data/subscription/error.rs new file mode 100644 index 0000000..20cf3f3 --- /dev/null +++ b/graph/src/data/subscription/error.rs @@ -0,0 +1,34 @@ +use serde::ser::*; + +use crate::prelude::QueryExecutionError; +use thiserror::Error; + +/// Error caused while processing a [Subscription](struct.Subscription.html) request. +#[derive(Debug, Error)] +pub enum SubscriptionError { + #[error("GraphQL error: {0:?}")] + GraphQLError(Vec), +} + +impl From for SubscriptionError { + fn from(e: QueryExecutionError) -> Self { + SubscriptionError::GraphQLError(vec![e]) + } +} + +impl From> for SubscriptionError { + fn from(e: Vec) -> Self { + SubscriptionError::GraphQLError(e) + } +} +impl Serialize for SubscriptionError { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut map = serializer.serialize_map(Some(1))?; + let msg = format!("{}", self); + map.serialize_entry("message", msg.as_str())?; + map.end() + } +} diff --git a/graph/src/data/subscription/mod.rs b/graph/src/data/subscription/mod.rs new file mode 100644 index 0000000..093c000 --- /dev/null +++ b/graph/src/data/subscription/mod.rs @@ -0,0 +1,7 @@ +mod error; +mod result; +mod subscription; + +pub use self::error::SubscriptionError; +pub use self::result::{QueryResultStream, SubscriptionResult}; +pub use self::subscription::Subscription; diff --git a/graph/src/data/subscription/result.rs b/graph/src/data/subscription/result.rs new file mode 100644 index 0000000..648ce79 --- /dev/null +++ b/graph/src/data/subscription/result.rs @@ -0,0 +1,10 @@ +use crate::prelude::QueryResult; +use std::pin::Pin; +use std::sync::Arc; + +/// A stream of query results for a subscription. +pub type QueryResultStream = + Pin> + Send>>; + +/// The result of running a subscription, if successful. +pub type SubscriptionResult = QueryResultStream; diff --git a/graph/src/data/subscription/subscription.rs b/graph/src/data/subscription/subscription.rs new file mode 100644 index 0000000..8ae6b87 --- /dev/null +++ b/graph/src/data/subscription/subscription.rs @@ -0,0 +1,11 @@ +use crate::prelude::Query; + +/// A GraphQL subscription made by a client. +/// +/// At the moment, this only contains the GraphQL query submitted as the +/// subscription payload. +#[derive(Clone, Debug)] +pub struct Subscription { + /// The GraphQL subscription query. + pub query: Query, +} diff --git a/graph/src/data/value.rs b/graph/src/data/value.rs new file mode 100644 index 0000000..fc62ef9 --- /dev/null +++ b/graph/src/data/value.rs @@ -0,0 +1,435 @@ +use crate::prelude::{q, s, CacheWeight}; +use serde::ser::{SerializeMap, SerializeSeq, Serializer}; +use serde::Serialize; +use std::collections::BTreeMap; +use std::convert::TryFrom; +use std::iter::FromIterator; + +/// An immutable string that is more memory-efficient since it only has an +/// overhead of 16 bytes for storing a string vs the 24 bytes that `String` +/// requires +#[derive(Clone, Default, Debug, PartialOrd, Ord, PartialEq, Eq, Hash)] +pub struct Word(Box); + +impl Word { + pub fn as_str(&self) -> &str { + &*self.0 + } +} + +impl std::fmt::Display for Word { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::ops::Deref for Word { + type Target = str; + + fn deref(&self) -> &Self::Target { + &*self.0 + } +} + +impl From<&str> for Word { + fn from(s: &str) -> Self { + Word(s.into()) + } +} + +impl From for Word { + fn from(s: String) -> Self { + Word(s.into_boxed_str()) + } +} + +impl Serialize for Word { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.0.serialize(serializer) + } +} + +impl<'de> serde::Deserialize<'de> for Word { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + String::deserialize(deserializer).map(Into::into) + } +} + +#[derive(Clone, Debug, PartialEq)] +struct Entry { + key: Option, + value: Value, +} + +impl Entry { + fn new(key: Word, value: Value) -> Self { + Entry { + key: Some(key), + value, + } + } + + fn has_key(&self, key: &str) -> bool { + match &self.key { + None => false, + Some(k) => k.as_str() == key, + } + } +} + +#[derive(Clone, PartialEq, Default)] +pub struct Object(Box<[Entry]>); + +impl Object { + pub fn get(&self, key: &str) -> Option<&Value> { + self.0 + .iter() + .find(|entry| entry.has_key(key)) + .map(|entry| &entry.value) + } + + pub fn remove(&mut self, key: &str) -> Option { + self.0 + .iter_mut() + .find(|entry| entry.has_key(key)) + .map(|entry| { + entry.key = None; + std::mem::replace(&mut entry.value, Value::Null) + }) + } + + pub fn iter(&self) -> impl Iterator { + ObjectIter::new(self) + } + + fn len(&self) -> usize { + self.0.len() + } + + pub fn extend(&mut self, other: Object) { + let mut entries = std::mem::replace(&mut self.0, Box::new([])).into_vec(); + entries.extend(other.0.into_vec()); + self.0 = entries.into_boxed_slice(); + } +} + +impl FromIterator<(String, Value)> for Object { + fn from_iter>(iter: T) -> Self { + let mut items: Vec<_> = Vec::new(); + for (key, value) in iter { + items.push(Entry::new(key.into(), value)) + } + Object(items.into_boxed_slice()) + } +} + +pub struct ObjectOwningIter { + iter: std::vec::IntoIter, +} + +impl Iterator for ObjectOwningIter { + type Item = (Word, Value); + + fn next(&mut self) -> Option { + while let Some(entry) = self.iter.next() { + if let Some(key) = entry.key { + return Some((key, entry.value)); + } + } + None + } +} + +impl IntoIterator for Object { + type Item = (Word, Value); + + type IntoIter = ObjectOwningIter; + + fn into_iter(self) -> Self::IntoIter { + ObjectOwningIter { + iter: self.0.into_vec().into_iter(), + } + } +} + +pub struct ObjectIter<'a> { + iter: std::slice::Iter<'a, Entry>, +} + +impl<'a> ObjectIter<'a> { + fn new(object: &'a Object) -> Self { + Self { + iter: object.0.iter(), + } + } +} +impl<'a> Iterator for ObjectIter<'a> { + type Item = (&'a str, &'a Value); + + fn next(&mut self) -> Option { + while let Some(entry) = self.iter.next() { + if let Some(key) = &entry.key { + return Some((key.as_str(), &entry.value)); + } + } + None + } +} + +impl<'a> IntoIterator for &'a Object { + type Item = as Iterator>::Item; + + type IntoIter = ObjectIter<'a>; + + fn into_iter(self) -> Self::IntoIter { + ObjectIter::new(self) + } +} + +impl CacheWeight for Entry { + fn indirect_weight(&self) -> usize { + self.key.indirect_weight() + self.value.indirect_weight() + } +} + +impl CacheWeight for Object { + fn indirect_weight(&self) -> usize { + self.0.iter().map(CacheWeight::indirect_weight).sum() + } +} + +impl std::fmt::Debug for Object { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Value { + Int(i64), + Float(f64), + String(String), + Boolean(bool), + Null, + Enum(String), + List(Vec), + Object(Object), +} + +impl Value { + pub fn object(map: BTreeMap) -> Self { + let items = map + .into_iter() + .map(|(key, value)| Entry::new(key, value)) + .collect(); + Value::Object(Object(items)) + } + + pub fn is_null(&self) -> bool { + matches!(self, Value::Null) + } + + pub fn coerce_enum(self, using_type: &s::EnumType) -> Result { + match self { + Value::Null => Ok(Value::Null), + Value::String(name) | Value::Enum(name) + if using_type.values.iter().any(|value| value.name == name) => + { + Ok(Value::Enum(name)) + } + _ => Err(self), + } + } + + pub fn coerce_scalar(self, using_type: &s::ScalarType) -> Result { + match (using_type.name.as_str(), self) { + (_, Value::Null) => Ok(Value::Null), + ("Boolean", Value::Boolean(b)) => Ok(Value::Boolean(b)), + ("BigDecimal", Value::Float(f)) => Ok(Value::String(f.to_string())), + ("BigDecimal", Value::Int(i)) => Ok(Value::String(i.to_string())), + ("BigDecimal", Value::String(s)) => Ok(Value::String(s)), + ("Int", Value::Int(num)) => { + if i32::min_value() as i64 <= num && num <= i32::max_value() as i64 { + Ok(Value::Int(num)) + } else { + Err(Value::Int(num)) + } + } + ("String", Value::String(s)) => Ok(Value::String(s)), + ("ID", Value::String(s)) => Ok(Value::String(s)), + ("ID", Value::Int(n)) => Ok(Value::String(n.to_string())), + ("Bytes", Value::String(s)) => Ok(Value::String(s)), + ("BigInt", Value::String(s)) => Ok(Value::String(s)), + ("BigInt", Value::Int(n)) => Ok(Value::String(n.to_string())), + ("JSONObject", Value::Object(obj)) => Ok(Value::Object(obj)), + ("Date", Value::String(obj)) => Ok(Value::String(obj)), + (_, v) => Err(v), + } + } +} + +impl std::fmt::Display for Value { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match *self { + Value::Int(ref num) => write!(f, "{}", num), + Value::Float(val) => write!(f, "{}", val), + Value::String(ref val) => write!(f, "\"{}\"", val.replace('"', "\\\"")), + Value::Boolean(true) => write!(f, "true"), + Value::Boolean(false) => write!(f, "false"), + Value::Null => write!(f, "null"), + Value::Enum(ref name) => write!(f, "{}", name), + Value::List(ref items) => { + write!(f, "[")?; + if !items.is_empty() { + write!(f, "{}", items[0])?; + for item in &items[1..] { + write!(f, ", {}", item)?; + } + } + write!(f, "]") + } + Value::Object(ref items) => { + write!(f, "{{")?; + let mut first = true; + for (name, value) in items.iter() { + if first { + first = false; + } else { + write!(f, ", ")?; + } + write!(f, "{}: {}", name, value)?; + } + write!(f, "}}") + } + } + } +} + +impl CacheWeight for Value { + fn indirect_weight(&self) -> usize { + match self { + Value::Boolean(_) | Value::Int(_) | Value::Null | Value::Float(_) => 0, + Value::Enum(s) | Value::String(s) => s.indirect_weight(), + Value::List(l) => l.indirect_weight(), + Value::Object(o) => o.indirect_weight(), + } + } +} + +impl Serialize for Value { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + Value::Boolean(v) => serializer.serialize_bool(*v), + Value::Enum(v) => serializer.serialize_str(v), + Value::Float(v) => serializer.serialize_f64(*v), + Value::Int(v) => serializer.serialize_i64(*v), + Value::List(l) => { + let mut seq = serializer.serialize_seq(Some(l.len()))?; + for v in l { + seq.serialize_element(v)?; + } + seq.end() + } + Value::Null => serializer.serialize_none(), + Value::String(s) => serializer.serialize_str(s), + Value::Object(o) => { + let mut map = serializer.serialize_map(Some(o.len()))?; + for (k, v) in o { + map.serialize_entry(k, v)?; + } + map.end() + } + } + } +} + +impl TryFrom for Value { + type Error = q::Value; + + fn try_from(value: q::Value) -> Result { + match value { + q::Value::Variable(_) => Err(value), + q::Value::Int(ref num) => match num.as_i64() { + Some(i) => Ok(Value::Int(i)), + None => Err(value), + }, + q::Value::Float(f) => Ok(Value::Float(f)), + q::Value::String(s) => Ok(Value::String(s)), + q::Value::Boolean(b) => Ok(Value::Boolean(b)), + q::Value::Null => Ok(Value::Null), + q::Value::Enum(s) => Ok(Value::Enum(s)), + q::Value::List(vals) => { + let vals: Vec<_> = vals + .into_iter() + .map(Value::try_from) + .collect::, _>>()?; + Ok(Value::List(vals)) + } + q::Value::Object(map) => { + let mut rmap = BTreeMap::new(); + for (key, value) in map.into_iter() { + let value = Value::try_from(value)?; + rmap.insert(key.into(), value); + } + Ok(Value::object(rmap)) + } + } + } +} + +impl From for Value { + fn from(value: serde_json::Value) -> Self { + match value { + serde_json::Value::Null => Value::Null, + serde_json::Value::Bool(b) => Value::Boolean(b), + serde_json::Value::Number(n) => match n.as_i64() { + Some(i) => Value::Int(i), + None => Value::Float(n.as_f64().unwrap()), + }, + serde_json::Value::String(s) => Value::String(s), + serde_json::Value::Array(vals) => { + let vals: Vec<_> = vals.into_iter().map(Value::from).collect::>(); + Value::List(vals) + } + serde_json::Value::Object(map) => { + let obj = + Object::from_iter(map.into_iter().map(|(key, val)| (key, Value::from(val)))); + Value::Object(obj) + } + } + } +} + +impl From for q::Value { + fn from(value: Value) -> Self { + match value { + Value::Int(i) => q::Value::Int((i as i32).into()), + Value::Float(f) => q::Value::Float(f), + Value::String(s) => q::Value::String(s), + Value::Boolean(b) => q::Value::Boolean(b), + Value::Null => q::Value::Null, + Value::Enum(s) => q::Value::Enum(s), + Value::List(vals) => { + let vals: Vec = vals.into_iter().map(q::Value::from).collect(); + q::Value::List(vals) + } + Value::Object(map) => { + let mut rmap = BTreeMap::new(); + for (key, value) in map.into_iter() { + let value = q::Value::from(value); + rmap.insert(key.to_string(), value); + } + q::Value::Object(rmap) + } + } + } +} diff --git a/graph/src/data_source/mod.rs b/graph/src/data_source/mod.rs new file mode 100644 index 0000000..2d92a27 --- /dev/null +++ b/graph/src/data_source/mod.rs @@ -0,0 +1,407 @@ +pub mod offchain; + +use crate::{ + blockchain::{ + BlockPtr, Blockchain, DataSource as _, DataSourceTemplate as _, TriggerData as _, + UnresolvedDataSource as _, UnresolvedDataSourceTemplate as _, + }, + components::{ + link_resolver::LinkResolver, + store::{BlockNumber, StoredDynamicDataSource}, + subgraph::DataSourceTemplateInfo, + }, + data_source::offchain::OFFCHAIN_KINDS, + prelude::{CheapClone as _, DataSourceContext}, +}; +use anyhow::Error; +use semver::Version; +use serde::{de::IntoDeserializer as _, Deserialize, Deserializer}; +use slog::{Logger, SendSyncRefUnwindSafeKV}; +use std::{collections::BTreeMap, fmt, sync::Arc}; + +#[derive(Debug)] +pub enum DataSource { + Onchain(C::DataSource), + Offchain(offchain::DataSource), +} + +impl TryFrom> for DataSource { + type Error = Error; + + fn try_from(info: DataSourceTemplateInfo) -> Result { + match &info.template { + DataSourceTemplate::Onchain(_) => { + C::DataSource::try_from(info).map(DataSource::Onchain) + } + DataSourceTemplate::Offchain(_) => { + offchain::DataSource::try_from(info).map(DataSource::Offchain) + } + } + } +} + +impl DataSource { + pub fn as_onchain(&self) -> Option<&C::DataSource> { + match self { + Self::Onchain(ds) => Some(&ds), + Self::Offchain(_) => None, + } + } + + pub fn as_offchain(&self) -> Option<&offchain::DataSource> { + match self { + Self::Onchain(_) => None, + Self::Offchain(ds) => Some(&ds), + } + } + + pub fn address(&self) -> Option> { + match self { + Self::Onchain(ds) => ds.address().map(ToOwned::to_owned), + Self::Offchain(ds) => ds.address(), + } + } + + pub fn name(&self) -> &str { + match self { + Self::Onchain(ds) => ds.name(), + Self::Offchain(ds) => &ds.name, + } + } + + pub fn kind(&self) -> &str { + match self { + Self::Onchain(ds) => ds.kind(), + Self::Offchain(ds) => &ds.kind, + } + } + + pub fn creation_block(&self) -> Option { + match self { + Self::Onchain(ds) => ds.creation_block(), + Self::Offchain(ds) => ds.creation_block, + } + } + + pub fn context(&self) -> Arc> { + match self { + Self::Onchain(ds) => ds.context(), + Self::Offchain(ds) => ds.context.clone(), + } + } + + pub fn api_version(&self) -> Version { + match self { + Self::Onchain(ds) => ds.api_version(), + Self::Offchain(ds) => ds.mapping.api_version.clone(), + } + } + + pub fn runtime(&self) -> Option>> { + match self { + Self::Onchain(ds) => ds.runtime(), + Self::Offchain(ds) => Some(ds.mapping.runtime.cheap_clone()), + } + } + + pub fn match_and_decode( + &self, + trigger: &TriggerData, + block: &Arc, + logger: &Logger, + ) -> Result>>, Error> { + match (self, trigger) { + (Self::Onchain(ds), TriggerData::Onchain(trigger)) => ds + .match_and_decode(trigger, block, logger) + .map(|t| t.map(|t| t.map(MappingTrigger::Onchain))), + (Self::Offchain(ds), TriggerData::Offchain(trigger)) => { + Ok(ds.match_and_decode(trigger)) + } + (Self::Onchain(_), TriggerData::Offchain(_)) + | (Self::Offchain(_), TriggerData::Onchain(_)) => Ok(None), + } + } + + pub fn is_duplicate_of(&self, other: &Self) -> bool { + match (self, other) { + (Self::Onchain(a), Self::Onchain(b)) => a.is_duplicate_of(b), + (Self::Offchain(a), Self::Offchain(b)) => { + // See also: data-source-is-duplicate-of + a.manifest_idx == b.manifest_idx && a.source == b.source && a.context == b.context + } + _ => false, + } + } + + pub fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { + match self { + Self::Onchain(ds) => ds.as_stored_dynamic_data_source(), + Self::Offchain(ds) => ds.as_stored_dynamic_data_source(), + } + } + + pub fn from_stored_dynamic_data_source( + template: &DataSourceTemplate, + stored: StoredDynamicDataSource, + ) -> Result { + match template { + DataSourceTemplate::Onchain(template) => { + C::DataSource::from_stored_dynamic_data_source(template, stored) + .map(DataSource::Onchain) + } + DataSourceTemplate::Offchain(template) => { + offchain::DataSource::from_stored_dynamic_data_source(template, stored) + .map(DataSource::Offchain) + } + } + } + + pub fn validate(&self) -> Vec { + match self { + Self::Onchain(ds) => ds.validate(), + Self::Offchain(_) => vec![], + } + } +} + +#[derive(Debug)] +pub enum UnresolvedDataSource { + Onchain(C::UnresolvedDataSource), + Offchain(offchain::UnresolvedDataSource), +} + +impl UnresolvedDataSource { + pub async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + manifest_idx: u32, + ) -> Result, anyhow::Error> { + match self { + Self::Onchain(unresolved) => unresolved + .resolve(resolver, logger, manifest_idx) + .await + .map(DataSource::Onchain), + Self::Offchain(unresolved) => unresolved + .resolve(resolver, logger, manifest_idx) + .await + .map(DataSource::Offchain), + } + } +} + +#[derive(Debug)] +pub enum DataSourceTemplate { + Onchain(C::DataSourceTemplate), + Offchain(offchain::DataSourceTemplate), +} + +impl DataSourceTemplate { + pub fn as_onchain(&self) -> Option<&C::DataSourceTemplate> { + match self { + Self::Onchain(ds) => Some(ds), + Self::Offchain(_) => None, + } + } + + pub fn into_onchain(self) -> Option { + match self { + Self::Onchain(ds) => Some(ds), + Self::Offchain(_) => None, + } + } + + pub fn name(&self) -> &str { + match self { + Self::Onchain(ds) => ds.name(), + Self::Offchain(ds) => &ds.name, + } + } + + pub fn api_version(&self) -> semver::Version { + match self { + Self::Onchain(ds) => ds.api_version(), + Self::Offchain(ds) => ds.mapping.api_version.clone(), + } + } + + pub fn runtime(&self) -> Option>> { + match self { + Self::Onchain(ds) => ds.runtime(), + Self::Offchain(ds) => Some(ds.mapping.runtime.clone()), + } + } + + pub fn manifest_idx(&self) -> u32 { + match self { + Self::Onchain(ds) => ds.manifest_idx(), + Self::Offchain(ds) => ds.manifest_idx, + } + } +} + +#[derive(Clone, Debug)] +pub enum UnresolvedDataSourceTemplate { + Onchain(C::UnresolvedDataSourceTemplate), + Offchain(offchain::UnresolvedDataSourceTemplate), +} + +impl Default for UnresolvedDataSourceTemplate { + fn default() -> Self { + Self::Onchain(C::UnresolvedDataSourceTemplate::default()) + } +} + +impl UnresolvedDataSourceTemplate { + pub async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + manifest_idx: u32, + ) -> Result, Error> { + match self { + Self::Onchain(ds) => ds + .resolve(resolver, logger, manifest_idx) + .await + .map(DataSourceTemplate::Onchain), + Self::Offchain(ds) => ds + .resolve(resolver, logger, manifest_idx) + .await + .map(DataSourceTemplate::Offchain), + } + } +} + +pub struct TriggerWithHandler { + pub trigger: T, + handler: String, + block_ptr: BlockPtr, + logging_extras: Arc, +} + +impl fmt::Debug for TriggerWithHandler { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut builder = f.debug_struct("TriggerWithHandler"); + builder.field("trigger", &self.trigger); + builder.field("handler", &self.handler); + builder.finish() + } +} + +impl TriggerWithHandler { + pub fn new(trigger: T, handler: String, block_ptr: BlockPtr) -> Self { + Self { + trigger, + handler, + block_ptr, + logging_extras: Arc::new(slog::o! {}), + } + } + + pub fn new_with_logging_extras( + trigger: T, + handler: String, + block_ptr: BlockPtr, + logging_extras: Arc, + ) -> Self { + TriggerWithHandler { + trigger, + handler, + block_ptr, + logging_extras, + } + } + + /// Additional key-value pairs to be logged with the "Done processing trigger" message. + pub fn logging_extras(&self) -> Arc { + self.logging_extras.cheap_clone() + } + + pub fn handler_name(&self) -> &str { + &self.handler + } + + fn map(self, f: impl FnOnce(T) -> T_) -> TriggerWithHandler { + TriggerWithHandler { + trigger: f(self.trigger), + handler: self.handler, + block_ptr: self.block_ptr, + logging_extras: self.logging_extras, + } + } + + pub fn block_ptr(&self) -> BlockPtr { + self.block_ptr.clone() + } +} + +pub enum TriggerData { + Onchain(C::TriggerData), + Offchain(offchain::TriggerData), +} + +impl TriggerData { + pub fn error_context(&self) -> String { + match self { + Self::Onchain(trigger) => trigger.error_context(), + Self::Offchain(trigger) => format!("{:?}", trigger.source), + } + } +} + +#[derive(Debug)] +pub enum MappingTrigger { + Onchain(C::MappingTrigger), + Offchain(offchain::TriggerData), +} + +macro_rules! clone_data_source { + ($t:ident) => { + impl Clone for $t { + fn clone(&self) -> Self { + match self { + Self::Onchain(ds) => Self::Onchain(ds.clone()), + Self::Offchain(ds) => Self::Offchain(ds.clone()), + } + } + } + }; +} + +clone_data_source!(DataSource); +clone_data_source!(DataSourceTemplate); + +macro_rules! deserialize_data_source { + ($t:ident) => { + impl<'de, C: Blockchain> Deserialize<'de> for $t { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let map: BTreeMap = BTreeMap::deserialize(deserializer)?; + let kind = map + .get("kind") + .ok_or(serde::de::Error::missing_field("kind"))? + .as_str() + .unwrap_or("?"); + if OFFCHAIN_KINDS.contains(&kind) { + offchain::$t::deserialize(map.into_deserializer()) + .map_err(serde::de::Error::custom) + .map($t::Offchain) + } else if (&C::KIND.to_string() == kind) || C::ALIASES.contains(&kind) { + C::$t::deserialize(map.into_deserializer()) + .map_err(serde::de::Error::custom) + .map($t::Onchain) + } else { + Err(serde::de::Error::custom(format!( + "data source has invalid `kind`; expected {}, file/ipfs", + C::KIND, + ))) + } + } + } + }; +} + +deserialize_data_source!(UnresolvedDataSource); +deserialize_data_source!(UnresolvedDataSourceTemplate); diff --git a/graph/src/data_source/offchain.rs b/graph/src/data_source/offchain.rs new file mode 100644 index 0000000..a36ca1f --- /dev/null +++ b/graph/src/data_source/offchain.rs @@ -0,0 +1,272 @@ +use crate::{ + blockchain::{BlockPtr, Blockchain}, + components::{ + link_resolver::LinkResolver, + store::{BlockNumber, StoredDynamicDataSource}, + subgraph::DataSourceTemplateInfo, + }, + data::store::scalar::Bytes, + data_source, + prelude::{DataSourceContext, Link}, +}; +use anyhow::{self, Context, Error}; +use cid::Cid; +use serde::Deserialize; +use slog::{info, Logger}; +use std::{fmt, sync::Arc}; + +use super::TriggerWithHandler; + +pub const OFFCHAIN_KINDS: &'static [&'static str] = &["file/ipfs"]; + +#[derive(Clone, Debug)] +pub struct DataSource { + pub kind: String, + pub name: String, + pub manifest_idx: u32, + pub source: Source, + pub mapping: Mapping, + pub context: Arc>, + pub creation_block: Option, +} + +impl TryFrom> for DataSource { + type Error = Error; + + fn try_from(info: DataSourceTemplateInfo) -> Result { + let template = match info.template { + data_source::DataSourceTemplate::Offchain(template) => template, + data_source::DataSourceTemplate::Onchain(_) => { + anyhow::bail!("Cannot create offchain data source from onchain template") + } + }; + let source = info.params.get(0).ok_or(anyhow::anyhow!( + "Failed to create data source from template `{}`: source parameter is missing", + template.name + ))?; + Ok(Self { + kind: template.kind.clone(), + name: template.name.clone(), + manifest_idx: template.manifest_idx, + source: Source::Ipfs(source.parse()?), + mapping: template.mapping.clone(), + context: Arc::new(info.context), + creation_block: Some(info.creation_block), + }) + } +} + +impl DataSource { + pub fn match_and_decode( + &self, + trigger: &TriggerData, + ) -> Option>> { + if self.source != trigger.source { + return None; + } + + Some(TriggerWithHandler::new( + data_source::MappingTrigger::Offchain(trigger.clone()), + self.mapping.handler.clone(), + BlockPtr::new(Default::default(), self.creation_block.unwrap_or(0)), + )) + } + + pub fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { + let param = match self.source { + Source::Ipfs(link) => Bytes::from(link.to_bytes()), + }; + let context = self + .context + .as_ref() + .as_ref() + .map(|ctx| serde_json::to_value(&ctx).unwrap()); + StoredDynamicDataSource { + manifest_idx: self.manifest_idx, + param: Some(param), + context, + creation_block: self.creation_block, + is_offchain: true, + } + } + + pub fn from_stored_dynamic_data_source( + template: &DataSourceTemplate, + stored: StoredDynamicDataSource, + ) -> Result { + let param = stored.param.context("no param on stored data source")?; + let source = Source::Ipfs(Cid::try_from(param.as_slice().to_vec())?); + let context = Arc::new(stored.context.map(serde_json::from_value).transpose()?); + Ok(Self { + kind: template.kind.clone(), + name: template.name.clone(), + manifest_idx: stored.manifest_idx, + source, + mapping: template.mapping.clone(), + context, + creation_block: stored.creation_block, + }) + } + + /// The concept of an address may or not make sense for an offchain data source, but this is + /// used as the value to be returned to mappings from the `dataSource.address()` host function. + pub fn address(&self) -> Option> { + match self.source { + Source::Ipfs(cid) => Some(cid.to_bytes()), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Source { + Ipfs(Cid), +} + +#[derive(Clone, Debug)] +pub struct Mapping { + pub language: String, + pub api_version: semver::Version, + pub entities: Vec, + pub handler: String, + pub runtime: Arc>, + pub link: Link, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize)] +pub struct UnresolvedDataSource { + pub kind: String, + pub name: String, + pub source: UnresolvedSource, + pub mapping: UnresolvedMapping, +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +pub struct UnresolvedSource { + file: Link, +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UnresolvedMapping { + pub api_version: String, + pub language: String, + pub file: Link, + pub handler: String, + pub entities: Vec, +} + +impl UnresolvedDataSource { + #[allow(unreachable_code)] + #[allow(unused_variables)] + pub async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + manifest_idx: u32, + ) -> Result { + anyhow::bail!( + "static file data sources are not yet supported, \\ + for details see https://github.com/graphprotocol/graph-node/issues/3864" + ); + + info!(logger, "Resolve offchain data source"; + "name" => &self.name, + "kind" => &self.kind, + "source" => format_args!("{:?}", &self.source), + ); + let source = match self.kind.as_str() { + "file/ipfs" => Source::Ipfs(self.source.file.link.parse()?), + _ => { + anyhow::bail!( + "offchain data source has invalid `kind`, expected `file/ipfs` but found {}", + self.kind + ); + } + }; + Ok(DataSource { + manifest_idx, + kind: self.kind, + name: self.name, + source, + mapping: self.mapping.resolve(&*resolver, logger).await?, + context: Arc::new(None), + creation_block: None, + }) + } +} + +impl UnresolvedMapping { + pub async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + ) -> Result { + info!(logger, "Resolve offchain mapping"; "link" => &self.file.link); + Ok(Mapping { + language: self.language, + api_version: semver::Version::parse(&self.api_version)?, + entities: self.entities, + handler: self.handler, + runtime: Arc::new(resolver.cat(logger, &self.file).await?), + link: self.file, + }) + } +} + +#[derive(Clone, Debug, Deserialize)] +pub struct UnresolvedDataSourceTemplate { + pub kind: String, + pub network: Option, + pub name: String, + pub mapping: UnresolvedMapping, +} + +#[derive(Clone, Debug)] +pub struct DataSourceTemplate { + pub kind: String, + pub network: Option, + pub name: String, + pub manifest_idx: u32, + pub mapping: Mapping, +} + +impl UnresolvedDataSourceTemplate { + pub async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + manifest_idx: u32, + ) -> Result { + info!(logger, "Resolve data source template"; "name" => &self.name); + + Ok(DataSourceTemplate { + kind: self.kind, + network: self.network, + name: self.name, + manifest_idx, + mapping: self.mapping.resolve(resolver, logger).await?, + }) + } +} + +#[derive(Clone)] +pub struct TriggerData { + pub source: Source, + pub data: Arc, +} + +impl fmt::Debug for TriggerData { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + #[derive(Debug)] + struct TriggerDataWithoutData<'a> { + _source: &'a Source, + } + write!( + f, + "{:?}", + TriggerDataWithoutData { + _source: &self.source + } + ) + } +} diff --git a/graph/src/env/graphql.rs b/graph/src/env/graphql.rs new file mode 100644 index 0000000..6c06b53 --- /dev/null +++ b/graph/src/env/graphql.rs @@ -0,0 +1,176 @@ +use std::fmt; + +use super::*; + +#[derive(Clone)] +pub struct EnvVarsGraphQl { + /// Set by the flag `ENABLE_GRAPHQL_VALIDATIONS`. On by default. + pub enable_validations: bool, + /// Set by the flag `SILENT_GRAPHQL_VALIDATIONS`. On by default. + pub silent_graphql_validations: bool, + pub subscription_throttle_interval: Duration, + /// This is the timeout duration for SQL queries. + /// + /// If it is not set, no statement timeout will be enforced. The statement + /// timeout is local, i.e., can only be used within a transaction and + /// will be cleared at the end of the transaction. + /// + /// Set by the environment variable `GRAPH_SQL_STATEMENT_TIMEOUT` (expressed + /// in seconds). No default value is provided. + pub sql_statement_timeout: Option, + + /// Set by the environment variable `GRAPH_CACHED_SUBGRAPH_IDS` (comma + /// separated). When the value of the variable is `*`, queries are cached + /// for all subgraphs, which is the default + /// behavior. + pub cached_subgraph_ids: CachedSubgraphIds, + /// In how many shards (mutexes) the query block cache is split. + /// Ideally this should divide 256 so that the distribution of queries to + /// shards is even. + /// + /// Set by the environment variable `GRAPH_QUERY_BLOCK_CACHE_SHARDS`. The + /// default value is 128. + pub query_block_cache_shards: u8, + /// Set by the environment variable `GRAPH_QUERY_LFU_CACHE_SHARDS`. The + /// default value is set to whatever `GRAPH_QUERY_BLOCK_CACHE_SHARDS` is set + /// to. Set to 0 to disable this cache. + pub query_lfu_cache_shards: u8, + /// How many blocks per network should be kept in the query cache. When the + /// limit is reached, older blocks are evicted. This should be kept small + /// since a lookup to the cache is O(n) on this value, and the cache memory + /// usage also increases with larger number. Set to 0 to disable + /// the cache. + /// + /// Set by the environment variable `GRAPH_QUERY_CACHE_BLOCKS`. The default + /// value is 2. + pub query_cache_blocks: usize, + /// Maximum total memory to be used by the cache. Each block has a max size of + /// `QUERY_CACHE_MAX_MEM` / (`QUERY_CACHE_BLOCKS` * + /// `GRAPH_QUERY_BLOCK_CACHE_SHARDS`). + /// + /// Set by the environment variable `GRAPH_QUERY_CACHE_MAX_MEM` (expressed + /// in MB). The default value is 1GB. + pub query_cache_max_mem: usize, + /// Set by the environment variable `GRAPH_QUERY_CACHE_STALE_PERIOD`. The + /// default value is 100. + pub query_cache_stale_period: u64, + /// Set by the environment variable `GRAPH_GRAPHQL_QUERY_TIMEOUT` (expressed in + /// seconds). No default value is provided. + pub query_timeout: Option, + /// Set by the environment variable `GRAPH_GRAPHQL_MAX_COMPLEXITY`. No + /// default value is provided. + pub max_complexity: Option, + /// Set by the environment variable `GRAPH_GRAPHQL_MAX_DEPTH`. The default + /// value is 255. + pub max_depth: u8, + /// Set by the environment variable `GRAPH_GRAPHQL_MAX_FIRST`. The default + /// value is 1000. + pub max_first: u32, + /// Set by the environment variable `GRAPH_GRAPHQL_MAX_SKIP`. The default + /// value is 4294967295 ([`u32::MAX`]). + pub max_skip: u32, + /// Allow skipping the check whether a deployment has changed while + /// we were running a query. Once we are sure that the check mechanism + /// is reliable, this variable should be removed. + /// + /// Set by the flag `GRAPHQL_ALLOW_DEPLOYMENT_CHANGE`. Off by default. + pub allow_deployment_change: bool, + /// Set by the environment variable `GRAPH_GRAPHQL_WARN_RESULT_SIZE`. The + /// default value is [`usize::MAX`]. + pub warn_result_size: usize, + /// Set by the environment variable `GRAPH_GRAPHQL_ERROR_RESULT_SIZE`. The + /// default value is [`usize::MAX`]. + pub error_result_size: usize, + /// Set by the flag `GRAPH_GRAPHQL_MAX_OPERATIONS_PER_CONNECTION`. + /// Defaults to 1000. + pub max_operations_per_connection: usize, +} + +// This does not print any values avoid accidentally leaking any sensitive env vars +impl fmt::Debug for EnvVarsGraphQl { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "env vars") + } +} + +impl From for EnvVarsGraphQl { + fn from(x: InnerGraphQl) -> Self { + Self { + enable_validations: x.enable_validations.0, + silent_graphql_validations: x.silent_graphql_validations.0, + subscription_throttle_interval: Duration::from_millis( + x.subscription_throttle_interval_in_ms, + ), + sql_statement_timeout: x.sql_statement_timeout_in_secs.map(Duration::from_secs), + cached_subgraph_ids: if x.cached_subgraph_ids == "*" { + CachedSubgraphIds::All + } else { + CachedSubgraphIds::Only( + x.cached_subgraph_ids + .split(',') + .map(str::to_string) + .collect(), + ) + }, + query_block_cache_shards: x.query_block_cache_shards, + query_lfu_cache_shards: x + .query_lfu_cache_shards + .unwrap_or(x.query_block_cache_shards), + query_cache_blocks: x.query_cache_blocks, + query_cache_max_mem: x.query_cache_max_mem_in_mb.0 * 1000 * 1000, + query_cache_stale_period: x.query_cache_stale_period, + query_timeout: x.query_timeout_in_secs.map(Duration::from_secs), + max_complexity: x.max_complexity.map(|x| x.0), + max_depth: x.max_depth.0, + max_first: x.max_first, + max_skip: x.max_skip.0, + allow_deployment_change: x.allow_deployment_change.0, + warn_result_size: x.warn_result_size.0 .0, + error_result_size: x.error_result_size.0 .0, + max_operations_per_connection: x.max_operations_per_connection, + } + } +} + +#[derive(Clone, Debug, Envconfig)] +pub struct InnerGraphQl { + #[envconfig(from = "ENABLE_GRAPHQL_VALIDATIONS", default = "false")] + enable_validations: EnvVarBoolean, + #[envconfig(from = "SILENT_GRAPHQL_VALIDATIONS", default = "true")] + silent_graphql_validations: EnvVarBoolean, + #[envconfig(from = "SUBSCRIPTION_THROTTLE_INTERVAL", default = "1000")] + subscription_throttle_interval_in_ms: u64, + #[envconfig(from = "GRAPH_SQL_STATEMENT_TIMEOUT")] + sql_statement_timeout_in_secs: Option, + + #[envconfig(from = "GRAPH_CACHED_SUBGRAPH_IDS", default = "*")] + cached_subgraph_ids: String, + #[envconfig(from = "GRAPH_QUERY_BLOCK_CACHE_SHARDS", default = "128")] + query_block_cache_shards: u8, + #[envconfig(from = "GRAPH_QUERY_LFU_CACHE_SHARDS")] + query_lfu_cache_shards: Option, + #[envconfig(from = "GRAPH_QUERY_CACHE_BLOCKS", default = "2")] + query_cache_blocks: usize, + #[envconfig(from = "GRAPH_QUERY_CACHE_MAX_MEM", default = "1000")] + query_cache_max_mem_in_mb: NoUnderscores, + #[envconfig(from = "GRAPH_QUERY_CACHE_STALE_PERIOD", default = "100")] + query_cache_stale_period: u64, + #[envconfig(from = "GRAPH_GRAPHQL_QUERY_TIMEOUT")] + query_timeout_in_secs: Option, + #[envconfig(from = "GRAPH_GRAPHQL_MAX_COMPLEXITY")] + max_complexity: Option>, + #[envconfig(from = "GRAPH_GRAPHQL_MAX_DEPTH", default = "")] + max_depth: WithDefaultUsize, + #[envconfig(from = "GRAPH_GRAPHQL_MAX_FIRST", default = "1000")] + max_first: u32, + #[envconfig(from = "GRAPH_GRAPHQL_MAX_SKIP", default = "")] + max_skip: WithDefaultUsize, + #[envconfig(from = "GRAPHQL_ALLOW_DEPLOYMENT_CHANGE", default = "false")] + allow_deployment_change: EnvVarBoolean, + #[envconfig(from = "GRAPH_GRAPHQL_WARN_RESULT_SIZE", default = "")] + warn_result_size: WithDefaultUsize, { usize::MAX }>, + #[envconfig(from = "GRAPH_GRAPHQL_ERROR_RESULT_SIZE", default = "")] + error_result_size: WithDefaultUsize, { usize::MAX }>, + #[envconfig(from = "GRAPH_GRAPHQL_MAX_OPERATIONS_PER_CONNECTION", default = "1000")] + max_operations_per_connection: usize, +} diff --git a/graph/src/env/mappings.rs b/graph/src/env/mappings.rs new file mode 100644 index 0000000..82507da --- /dev/null +++ b/graph/src/env/mappings.rs @@ -0,0 +1,113 @@ +use std::fmt; + +use super::*; + +#[derive(Clone)] +pub struct EnvVarsMapping { + /// Forces the cache eviction policy to take its own memory overhead into account. + /// + /// Set by the flag `DEAD_WEIGHT`. Setting `DEAD_WEIGHT` is dangerous since it can lead to a + /// situation where an empty cache is bigger than the max_weight, + /// which leads to a panic. Off by default. + pub entity_cache_dead_weight: bool, + /// Size limit of the entity LFU cache. + /// + /// Set by the environment variable `GRAPH_ENTITY_CACHE_SIZE` (expressed in + /// kilobytes). The default value is 10 megabytes. + pub entity_cache_size: usize, + /// Set by the environment variable `GRAPH_MAX_API_VERSION`. The default + /// value is `0.0.7`. + pub max_api_version: Version, + /// Set by the environment variable `GRAPH_MAPPING_HANDLER_TIMEOUT` + /// (expressed in seconds). No default is provided. + pub timeout: Option, + /// Maximum stack size for the WASM runtime. + /// + /// Set by the environment variable `GRAPH_RUNTIME_MAX_STACK_SIZE` + /// (expressed in bytes). The default value is 512KiB. + pub max_stack_size: usize, + + /// Set by the environment variable `GRAPH_MAX_IPFS_CACHE_FILE_SIZE` + /// (expressed in bytes). The default value is 1MiB. + pub max_ipfs_cache_file_size: usize, + /// Set by the environment variable `GRAPH_MAX_IPFS_CACHE_SIZE`. The default + /// value is 50 items. + pub max_ipfs_cache_size: u64, + /// The timeout for all IPFS requests. + /// + /// Set by the environment variable `GRAPH_IPFS_TIMEOUT` (expressed in + /// seconds). The default value is 30s. + pub ipfs_timeout: Duration, + /// Sets the `ipfs.map` file size limit. + /// + /// Set by the environment variable `GRAPH_MAX_IPFS_MAP_FILE_SIZE_LIMIT` + /// (expressed in bytes). The default value is 256MiB. + pub max_ipfs_map_file_size: usize, + /// Sets the `ipfs.cat` file size limit. + /// + /// Set by the environment variable `GRAPH_MAX_IPFS_FILE_BYTES` (expressed in + /// bytes). Defaults to 256 MiB. + pub max_ipfs_file_bytes: usize, + pub max_ipfs_concurrent_requests: u16, + /// Set by the flag `GRAPH_ALLOW_NON_DETERMINISTIC_IPFS`. Off by + /// default. + pub allow_non_deterministic_ipfs: bool, +} + +// This does not print any values avoid accidentally leaking any sensitive env vars +impl fmt::Debug for EnvVarsMapping { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "env vars") + } +} + +impl From for EnvVarsMapping { + fn from(x: InnerMappingHandlers) -> Self { + Self { + entity_cache_dead_weight: x.entity_cache_dead_weight.0, + entity_cache_size: x.entity_cache_size_in_kb * 1000, + + max_api_version: x.max_api_version, + timeout: x.mapping_handler_timeout_in_secs.map(Duration::from_secs), + max_stack_size: x.runtime_max_stack_size.0 .0, + + max_ipfs_cache_file_size: x.max_ipfs_cache_file_size.0, + max_ipfs_cache_size: x.max_ipfs_cache_size, + ipfs_timeout: Duration::from_secs(x.ipfs_timeout_in_secs), + max_ipfs_map_file_size: x.max_ipfs_map_file_size.0, + max_ipfs_file_bytes: x.max_ipfs_file_bytes.0, + max_ipfs_concurrent_requests: x.max_ipfs_concurrent_requests, + allow_non_deterministic_ipfs: x.allow_non_deterministic_ipfs.0, + } + } +} + +#[derive(Clone, Debug, Envconfig)] +pub struct InnerMappingHandlers { + #[envconfig(from = "DEAD_WEIGHT", default = "false")] + entity_cache_dead_weight: EnvVarBoolean, + #[envconfig(from = "GRAPH_ENTITY_CACHE_SIZE", default = "10000")] + entity_cache_size_in_kb: usize, + #[envconfig(from = "GRAPH_MAX_API_VERSION", default = "0.0.7")] + max_api_version: Version, + #[envconfig(from = "GRAPH_MAPPING_HANDLER_TIMEOUT")] + mapping_handler_timeout_in_secs: Option, + #[envconfig(from = "GRAPH_RUNTIME_MAX_STACK_SIZE", default = "")] + runtime_max_stack_size: WithDefaultUsize, { 512 * 1024 }>, + + // IPFS. + #[envconfig(from = "GRAPH_MAX_IPFS_CACHE_FILE_SIZE", default = "")] + max_ipfs_cache_file_size: WithDefaultUsize, + #[envconfig(from = "GRAPH_MAX_IPFS_CACHE_SIZE", default = "50")] + max_ipfs_cache_size: u64, + #[envconfig(from = "GRAPH_IPFS_TIMEOUT", default = "30")] + ipfs_timeout_in_secs: u64, + #[envconfig(from = "GRAPH_MAX_IPFS_MAP_FILE_SIZE", default = "")] + max_ipfs_map_file_size: WithDefaultUsize, + #[envconfig(from = "GRAPH_MAX_IPFS_FILE_BYTES", default = "")] + max_ipfs_file_bytes: WithDefaultUsize, + #[envconfig(from = "GRAPH_MAX_IPFS_CONCURRENT_REQUESTS", default = "100")] + max_ipfs_concurrent_requests: u16, + #[envconfig(from = "GRAPH_ALLOW_NON_DETERMINISTIC_IPFS", default = "false")] + allow_non_deterministic_ipfs: EnvVarBoolean, +} diff --git a/graph/src/env/mod.rs b/graph/src/env/mod.rs new file mode 100644 index 0000000..b7e1fc7 --- /dev/null +++ b/graph/src/env/mod.rs @@ -0,0 +1,445 @@ +mod graphql; +mod mappings; +mod store; + +use envconfig::Envconfig; +use lazy_static::lazy_static; +use semver::Version; +use std::{ + collections::HashSet, + env::VarError, + fmt, + str::FromStr, + sync::atomic::{AtomicBool, Ordering}, + time::Duration, +}; + +use self::graphql::*; +use self::mappings::*; +use self::store::*; +use crate::{ + components::subgraph::SubgraphVersionSwitchingMode, runtime::gas::CONST_MAX_GAS_PER_HANDLER, +}; + +pub static UNSAFE_CONFIG: AtomicBool = AtomicBool::new(false); + +lazy_static! { + pub static ref ENV_VARS: EnvVars = EnvVars::from_env().unwrap(); +} + +// This is currently unused but is kept as a potentially useful mechanism. +/// Panics if: +/// - The value is not UTF8. +/// - The value cannot be parsed as T. +/// - The value differs from the default, and `--unsafe-config` flag is not set. +pub fn unsafe_env_var + Eq>( + name: &'static str, + default_value: T, +) -> T { + let var = match std::env::var(name) { + Ok(var) => var, + Err(VarError::NotPresent) => return default_value, + Err(VarError::NotUnicode(_)) => panic!("environment variable {} is not UTF8", name), + }; + + let value = var + .parse::() + .unwrap_or_else(|e| panic!("failed to parse environment variable {}: {}", name, e)); + + if !UNSAFE_CONFIG.load(Ordering::SeqCst) && value != default_value { + panic!( + "unsafe environment variable {} is set. The recommended action is to unset it. \ + If this is not an indexer on the network, \ + you may provide the `--unsafe-config` to allow setting this variable.", + name + ) + } + + value +} + +/// Panics if: +/// - The value is not UTF8. +/// - The value cannot be parsed as T.. +pub fn env_var + Eq>( + name: &'static str, + default_value: T, +) -> T { + let var = match std::env::var(name) { + Ok(var) => var, + Err(VarError::NotPresent) => return default_value, + Err(VarError::NotUnicode(_)) => panic!("environment variable {} is not UTF8", name), + }; + + var.parse::() + .unwrap_or_else(|e| panic!("failed to parse environment variable {}: {}", name, e)) +} + +#[derive(Clone)] +#[non_exhaustive] +pub struct EnvVars { + pub graphql: EnvVarsGraphQl, + pub mappings: EnvVarsMapping, + pub store: EnvVarsStore, + + /// Enables query throttling when getting database connections goes over this value. + /// Load management can be disabled by setting this to 0. + /// + /// Set by the environment variable `GRAPH_LOAD_THRESHOLD` (expressed in + /// milliseconds). The default value is 0. + pub load_threshold: Duration, + /// When the system is overloaded, any query that causes more than this + /// fraction of the effort will be rejected for as long as the process is + /// running (i.e. even after the overload situation is resolved). + /// + /// Set by the environment variable `GRAPH_LOAD_THRESHOLD` + /// (expressed as a number). No default value is provided. When *not* set, + /// no queries will ever be jailed, even though they will still be subject + /// to normal load management when the system is overloaded. + pub load_jail_threshold: Option, + /// When this is active, the system will trigger all the steps that the load + /// manager would given the other load management configuration settings, + /// but never actually decline to run a query; instead, log about load + /// management decisions. + /// + /// Set by the flag `GRAPH_LOAD_SIMULATE`. + pub load_simulate: bool, + /// Set by the flag `GRAPH_ALLOW_NON_DETERMINISTIC_FULLTEXT_SEARCH`, but + /// enabled anyway (overridden) if [debug + /// assertions](https://doc.rust-lang.org/reference/conditional-compilation.html#debug_assertions) + /// are enabled. + pub allow_non_deterministic_fulltext_search: bool, + /// Set by the environment variable `GRAPH_MAX_SPEC_VERSION`. The default + /// value is `0.0.7`. + pub max_spec_version: Version, + /// Set by the flag `GRAPH_DISABLE_GRAFTS`. + pub disable_grafts: bool, + /// Set by the environment variable `GRAPH_LOAD_WINDOW_SIZE` (expressed in + /// seconds). The default value is 300 seconds. + pub load_window_size: Duration, + /// Set by the environment variable `GRAPH_LOAD_BIN_SIZE` (expressed in + /// seconds). The default value is 1 second. + pub load_bin_size: Duration, + /// Set by the environment variable + /// `GRAPH_ELASTIC_SEARCH_FLUSH_INTERVAL_SECS` (expressed in seconds). The + /// default value is 5 seconds. + pub elastic_search_flush_interval: Duration, + /// Set by the environment variable + /// `GRAPH_ELASTIC_SEARCH_MAX_RETRIES`. The default value is 5. + pub elastic_search_max_retries: usize, + /// If an instrumented lock is contended for longer than the specified + /// duration, a warning will be logged. + /// + /// Set by the environment variable `GRAPH_LOCK_CONTENTION_LOG_THRESHOLD_MS` + /// (expressed in milliseconds). The default value is 100ms. + pub lock_contention_log_threshold: Duration, + /// This is configurable only for debugging purposes. This value is set by + /// the protocol, so indexers running in the network should never set this + /// config. + /// + /// Set by the environment variable `GRAPH_MAX_GAS_PER_HANDLER`. + pub max_gas_per_handler: u64, + /// Set by the environment variable `GRAPH_LOG_QUERY_TIMING`. + pub log_query_timing: HashSet, + /// A + /// [`chrono`](https://docs.rs/chrono/latest/chrono/#formatting-and-parsing) + /// -like format string for logs. + /// + /// Set by the environment variable `GRAPH_LOG_TIME_FORMAT`. The default + /// value is `%b %d %H:%M:%S%.3f`. + pub log_time_format: String, + /// Set by the flag `GRAPH_LOG_POI_EVENTS`. + pub log_poi_events: bool, + /// Set by the environment variable `GRAPH_LOG`. + pub log_levels: Option, + /// Set by the flag `EXPERIMENTAL_STATIC_FILTERS`. Off by default. + pub experimental_static_filters: bool, + /// Set by the environment variable + /// `EXPERIMENTAL_SUBGRAPH_VERSION_SWITCHING_MODE`. The default value is + /// `"instant"`. + pub subgraph_version_switching_mode: SubgraphVersionSwitchingMode, + /// Set by the flag `GRAPH_KILL_IF_UNRESPONSIVE`. Off by default. + pub kill_if_unresponsive: bool, + /// Guards public access to POIs in the `index-node`. + /// + /// Set by the environment variable `GRAPH_POI_ACCESS_TOKEN`. No default + /// value is provided. + pub poi_access_token: Option, + /// Set by the environment variable `GRAPH_SUBGRAPH_MAX_DATA_SOURCES`. No + /// default value is provided. + pub subgraph_max_data_sources: Option, + /// Keep deterministic errors non-fatal even if the subgraph is pending. + /// Used for testing Graph Node itself. + /// + /// Set by the flag `GRAPH_DISABLE_FAIL_FAST`. Off by default. + pub disable_fail_fast: bool, + /// Ceiling for the backoff retry of non-deterministic errors. + /// + /// Set by the environment variable `GRAPH_SUBGRAPH_ERROR_RETRY_CEIL_SECS` + /// (expressed in seconds). The default value is 1800s (30 minutes). + pub subgraph_error_retry_ceil: Duration, + /// Experimental feature. + /// + /// Set by the flag `GRAPH_ENABLE_SELECT_BY_SPECIFIC_ATTRIBUTES`. Off by + /// default. + pub enable_select_by_specific_attributes: bool, + /// Verbose logging of mapping inputs. + /// + /// Set by the flag `GRAPH_LOG_TRIGGER_DATA`. Off by + /// default. + pub log_trigger_data: bool, + /// Set by the environment variable `GRAPH_EXPLORER_TTL` + /// (expressed in seconds). The default value is 10s. + pub explorer_ttl: Duration, + /// Set by the environment variable `GRAPH_EXPLORER_LOCK_THRESHOLD` + /// (expressed in milliseconds). The default value is 100ms. + pub explorer_lock_threshold: Duration, + /// Set by the environment variable `GRAPH_EXPLORER_QUERY_THRESHOLD` + /// (expressed in milliseconds). The default value is 500ms. + pub explorer_query_threshold: Duration, + /// Set by the environment variable `EXTERNAL_HTTP_BASE_URL`. No default + /// value is provided. + pub external_http_base_url: Option, + /// Set by the environment variable `EXTERNAL_WS_BASE_URL`. No default + /// value is provided. + pub external_ws_base_url: Option, +} + +impl EnvVars { + pub fn from_env() -> Result { + let inner = Inner::init_from_env()?; + let graphql = InnerGraphQl::init_from_env()?.into(); + let mapping_handlers = InnerMappingHandlers::init_from_env()?.into(); + let store = InnerStore::init_from_env()?.into(); + + Ok(Self { + graphql, + mappings: mapping_handlers, + store, + + load_threshold: Duration::from_millis(inner.load_threshold_in_ms), + load_jail_threshold: inner.load_jail_threshold, + load_simulate: inner.load_simulate.0, + allow_non_deterministic_fulltext_search: inner + .allow_non_deterministic_fulltext_search + .0 + || cfg!(debug_assertions), + max_spec_version: inner.max_spec_version, + disable_grafts: inner.disable_grafts.0, + load_window_size: Duration::from_secs(inner.load_window_size_in_secs), + load_bin_size: Duration::from_secs(inner.load_bin_size_in_secs), + elastic_search_flush_interval: Duration::from_secs( + inner.elastic_search_flush_interval_in_secs, + ), + elastic_search_max_retries: inner.elastic_search_max_retries, + lock_contention_log_threshold: Duration::from_millis( + inner.lock_contention_log_threshold_in_ms, + ), + max_gas_per_handler: inner.max_gas_per_handler.0 .0, + log_query_timing: inner + .log_query_timing + .split(',') + .map(str::to_string) + .collect(), + log_time_format: inner.log_time_format, + log_poi_events: inner.log_poi_events.0, + log_levels: inner.log_levels, + experimental_static_filters: inner.experimental_static_filters.0, + subgraph_version_switching_mode: inner.subgraph_version_switching_mode, + kill_if_unresponsive: inner.kill_if_unresponsive.0, + poi_access_token: inner.poi_access_token, + subgraph_max_data_sources: inner.subgraph_max_data_sources, + disable_fail_fast: inner.disable_fail_fast.0, + subgraph_error_retry_ceil: Duration::from_secs(inner.subgraph_error_retry_ceil_in_secs), + enable_select_by_specific_attributes: inner.enable_select_by_specific_attributes.0, + log_trigger_data: inner.log_trigger_data.0, + explorer_ttl: Duration::from_secs(inner.explorer_ttl_in_secs), + explorer_lock_threshold: Duration::from_millis(inner.explorer_lock_threshold_in_msec), + explorer_query_threshold: Duration::from_millis(inner.explorer_query_threshold_in_msec), + external_http_base_url: inner.external_http_base_url, + external_ws_base_url: inner.external_ws_base_url, + }) + } + + /// Equivalent to checking if [`EnvVar::load_threshold`] is set to + /// [`Duration::ZERO`]. + pub fn load_management_is_disabled(&self) -> bool { + self.load_threshold.is_zero() + } + + fn log_query_timing_contains(&self, kind: &str) -> bool { + self.log_query_timing.iter().any(|s| s == kind) + } + + pub fn log_sql_timing(&self) -> bool { + self.log_query_timing_contains("sql") + } + + pub fn log_gql_timing(&self) -> bool { + self.log_query_timing_contains("gql") + } + + pub fn log_gql_cache_timing(&self) -> bool { + self.log_query_timing_contains("cache") && self.log_gql_timing() + } +} + +impl Default for EnvVars { + fn default() -> Self { + ENV_VARS.clone() + } +} + +// This does not print any values avoid accidentally leaking any sensitive env vars +impl fmt::Debug for EnvVars { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "env vars") + } +} + +#[derive(Clone, Debug, Envconfig)] +struct Inner { + #[envconfig(from = "GRAPH_LOAD_THRESHOLD", default = "0")] + load_threshold_in_ms: u64, + #[envconfig(from = "GRAPH_LOAD_JAIL_THRESHOLD")] + load_jail_threshold: Option, + #[envconfig(from = "GRAPH_LOAD_SIMULATE", default = "false")] + load_simulate: EnvVarBoolean, + #[envconfig( + from = "GRAPH_ALLOW_NON_DETERMINISTIC_FULLTEXT_SEARCH", + default = "false" + )] + allow_non_deterministic_fulltext_search: EnvVarBoolean, + #[envconfig(from = "GRAPH_MAX_SPEC_VERSION", default = "0.0.7")] + max_spec_version: Version, + #[envconfig(from = "GRAPH_DISABLE_GRAFTS", default = "false")] + disable_grafts: EnvVarBoolean, + #[envconfig(from = "GRAPH_LOAD_WINDOW_SIZE", default = "300")] + load_window_size_in_secs: u64, + #[envconfig(from = "GRAPH_LOAD_BIN_SIZE", default = "1")] + load_bin_size_in_secs: u64, + #[envconfig(from = "GRAPH_ELASTIC_SEARCH_FLUSH_INTERVAL_SECS", default = "5")] + elastic_search_flush_interval_in_secs: u64, + #[envconfig(from = "GRAPH_ELASTIC_SEARCH_MAX_RETRIES", default = "5")] + elastic_search_max_retries: usize, + #[envconfig(from = "GRAPH_LOCK_CONTENTION_LOG_THRESHOLD_MS", default = "100")] + lock_contention_log_threshold_in_ms: u64, + + // For now this is set absurdly high by default because we've seen many cases of gas being + // overestimated and failing otherwise legit subgraphs. Once gas costs have been better + // benchmarked and adjusted, and out of gas has been made a deterministic error, this default + // should be removed and this should somehow be gated on `UNSAFE_CONFIG`. + #[envconfig(from = "GRAPH_MAX_GAS_PER_HANDLER", default = "1_000_000_000_000_000")] + max_gas_per_handler: + WithDefaultUsize, { CONST_MAX_GAS_PER_HANDLER as usize }>, + #[envconfig(from = "GRAPH_LOG_QUERY_TIMING", default = "")] + log_query_timing: String, + #[envconfig(from = "GRAPH_LOG_TIME_FORMAT", default = "%b %d %H:%M:%S%.3f")] + log_time_format: String, + #[envconfig(from = "GRAPH_LOG_POI_EVENTS", default = "false")] + log_poi_events: EnvVarBoolean, + #[envconfig(from = "GRAPH_LOG")] + log_levels: Option, + #[envconfig(from = "EXPERIMENTAL_STATIC_FILTERS", default = "false")] + experimental_static_filters: EnvVarBoolean, + #[envconfig( + from = "EXPERIMENTAL_SUBGRAPH_VERSION_SWITCHING_MODE", + default = "instant" + )] + subgraph_version_switching_mode: SubgraphVersionSwitchingMode, + #[envconfig(from = "GRAPH_KILL_IF_UNRESPONSIVE", default = "false")] + kill_if_unresponsive: EnvVarBoolean, + #[envconfig(from = "GRAPH_POI_ACCESS_TOKEN")] + poi_access_token: Option, + #[envconfig(from = "GRAPH_SUBGRAPH_MAX_DATA_SOURCES")] + subgraph_max_data_sources: Option, + #[envconfig(from = "GRAPH_DISABLE_FAIL_FAST", default = "false")] + disable_fail_fast: EnvVarBoolean, + #[envconfig(from = "GRAPH_SUBGRAPH_ERROR_RETRY_CEIL_SECS", default = "1800")] + subgraph_error_retry_ceil_in_secs: u64, + #[envconfig(from = "GRAPH_ENABLE_SELECT_BY_SPECIFIC_ATTRIBUTES", default = "false")] + enable_select_by_specific_attributes: EnvVarBoolean, + #[envconfig(from = "GRAPH_LOG_TRIGGER_DATA", default = "false")] + log_trigger_data: EnvVarBoolean, + #[envconfig(from = "GRAPH_EXPLORER_TTL", default = "10")] + explorer_ttl_in_secs: u64, + #[envconfig(from = "GRAPH_EXPLORER_LOCK_THRESHOLD", default = "100")] + explorer_lock_threshold_in_msec: u64, + #[envconfig(from = "GRAPH_EXPLORER_QUERY_THRESHOLD", default = "500")] + explorer_query_threshold_in_msec: u64, + #[envconfig(from = "EXTERNAL_HTTP_BASE_URL")] + external_http_base_url: Option, + #[envconfig(from = "EXTERNAL_WS_BASE_URL")] + external_ws_base_url: Option, +} + +#[derive(Clone, Debug)] +pub enum CachedSubgraphIds { + All, + Only(Vec), +} + +/// When reading [`bool`] values from environment variables, we must be able to +/// parse many different ways to specify booleans: +/// +/// - Empty strings, i.e. as a flag. +/// - `true` or `false`. +/// - `1` or `0`. +#[derive(Copy, Clone, Debug)] +pub struct EnvVarBoolean(pub bool); + +impl FromStr for EnvVarBoolean { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "" | "true" | "1" => Ok(Self(true)), + "false" | "0" => Ok(Self(false)), + _ => Err("Invalid env. var. flag, expected true / false / 1 / 0".to_string()), + } + } +} + +/// Allows us to parse stuff ignoring underscores, notably big numbers. +#[derive(Copy, Clone, Debug)] +pub struct NoUnderscores(T); + +impl FromStr for NoUnderscores +where + T: FromStr, + T::Err: ToString, +{ + type Err = String; + + fn from_str(s: &str) -> Result { + match T::from_str(s.replace('_', "").as_str()) { + Ok(x) => Ok(Self(x)), + Err(e) => Err(e.to_string()), + } + } +} + +/// Provide a numeric ([`usize`]) default value if the environment flag is +/// empty. +#[derive(Copy, Clone, Debug)] +pub struct WithDefaultUsize(T); + +impl FromStr for WithDefaultUsize +where + T: FromStr, + T::Err: ToString, +{ + type Err = String; + + fn from_str(s: &str) -> Result { + let x = if s.is_empty() { + T::from_str(N.to_string().as_str()) + } else { + T::from_str(s) + }; + match x { + Ok(x) => Ok(Self(x)), + Err(e) => Err(e.to_string()), + } + } +} diff --git a/graph/src/env/store.rs b/graph/src/env/store.rs new file mode 100644 index 0000000..077088b --- /dev/null +++ b/graph/src/env/store.rs @@ -0,0 +1,176 @@ +use std::fmt; + +use super::*; + +#[derive(Clone)] +pub struct EnvVarsStore { + /// Set by the environment variable `GRAPH_CHAIN_HEAD_WATCHER_TIMEOUT` + /// (expressed in seconds). The default value is 30 seconds. + pub chain_head_watcher_timeout: Duration, + /// This is how long statistics that influence query execution are cached in + /// memory before they are reloaded from the database. + /// + /// Set by the environment variable `GRAPH_QUERY_STATS_REFRESH_INTERVAL` + /// (expressed in seconds). The default value is 300 seconds. + pub query_stats_refresh_interval: Duration, + /// This can be used to effectively disable the query semaphore by setting + /// it to a high number, but there's typically no need to configure this. + /// + /// Set by the environment variable `GRAPH_EXTRA_QUERY_PERMITS`. The default + /// value is 0. + pub extra_query_permits: usize, + /// Set by the environment variable `LARGE_NOTIFICATION_CLEANUP_INTERVAL` + /// (expressed in seconds). The default value is 300 seconds. + pub large_notification_cleanup_interval: Duration, + /// Set by the environment variable `GRAPH_NOTIFICATION_BROADCAST_TIMEOUT` + /// (expressed in seconds). The default value is 60 seconds. + pub notification_broadcast_timeout: Duration, + /// This variable is only here temporarily until we can settle on the right + /// batch size through experimentation, and should then just become an + /// ordinary constant. + /// + /// Set by the environment variable `TYPEA_BATCH_SIZE`. + pub typea_batch_size: usize, + /// Allows for some optimizations when running relational queries. Set this + /// to 0 to turn off this optimization. + /// + /// Set by the environment variable `TYPED_CHILDREN_SET_SIZE`. + pub typed_children_set_size: usize, + /// When enabled, turns `ORDER BY id` into `ORDER BY id, block_range` in + /// some relational queries. + /// + /// Set by the flag `ORDER_BY_BLOCK_RANGE`. Not meant as a user-tunable, + /// only as an emergency setting for the hosted service. Remove after + /// 2022-07-01 if hosted service had no issues with it being `true` + pub order_by_block_range: bool, + /// When the flag is present, `ORDER BY` clauses are changed so that `asc` + /// and `desc` ordering produces reverse orders. Setting the flag turns the + /// new, correct behavior off. + /// + /// Set by the flag `REVERSIBLE_ORDER_BY_OFF`. + pub reversible_order_by_off: bool, + /// Whether to disable the notifications that feed GraphQL + /// subscriptions. When the flag is set, no updates + /// about entity changes will be sent to query nodes. + /// + /// Set by the flag `GRAPH_DISABLE_SUBSCRIPTION_NOTIFICATIONS`. Not set + /// by default. + pub disable_subscription_notifications: bool, + /// A fallback in case the logic to remember database availability goes + /// wrong; when this is set, we always try to get a connection and never + /// use the availability state we remembered. + /// + /// Set by the flag `GRAPH_STORE_CONNECTION_TRY_ALWAYS`. Disabled by + /// default. + pub connection_try_always: bool, + /// Set by the environment variable `GRAPH_REMOVE_UNUSED_INTERVAL` + /// (expressed in minutes). The default value is 360 minutes. + pub remove_unused_interval: chrono::Duration, + + // These should really be set through the configuration file, especially for + // `GRAPH_STORE_CONNECTION_MIN_IDLE` and + // `GRAPH_STORE_CONNECTION_IDLE_TIMEOUT`. It's likely that they should be + // configured differently for each pool. + /// Set by the environment variable `GRAPH_STORE_CONNECTION_TIMEOUT` (expressed + /// in milliseconds). The default value is 5000ms. + pub connection_timeout: Duration, + /// Set by the environment variable `GRAPH_STORE_CONNECTION_MIN_IDLE`. No + /// default value is provided. + pub connection_min_idle: Option, + /// Set by the environment variable `GRAPH_STORE_CONNECTION_IDLE_TIMEOUT` + /// (expressed in seconds). The default value is 600s. + pub connection_idle_timeout: Duration, + + /// The size of the write queue; this many blocks can be buffered for + /// writing before calls to transact block operations will block. + /// Setting this to `0` disables pipelined writes, and writes will be + /// done synchronously. + pub write_queue_size: usize, + + /// This is just in case new behavior causes issues. This can be removed + /// once the new behavior has run in the hosted service for a few days + /// without issues. + pub disable_error_for_toplevel_parents: bool, +} + +// This does not print any values avoid accidentally leaking any sensitive env vars +impl fmt::Debug for EnvVarsStore { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "env vars") + } +} + +impl From for EnvVarsStore { + fn from(x: InnerStore) -> Self { + Self { + chain_head_watcher_timeout: Duration::from_secs(x.chain_head_watcher_timeout_in_secs), + query_stats_refresh_interval: Duration::from_secs( + x.query_stats_refresh_interval_in_secs, + ), + extra_query_permits: x.extra_query_permits, + large_notification_cleanup_interval: Duration::from_secs( + x.large_notification_cleanup_interval_in_secs, + ), + notification_broadcast_timeout: Duration::from_secs( + x.notification_broadcast_timeout_in_secs, + ), + typea_batch_size: x.typea_batch_size, + typed_children_set_size: x.typed_children_set_size, + order_by_block_range: x.order_by_block_range.0, + reversible_order_by_off: x.reversible_order_by_off.0, + disable_subscription_notifications: x.disable_subscription_notifications.0, + connection_try_always: x.connection_try_always.0, + remove_unused_interval: chrono::Duration::minutes( + x.remove_unused_interval_in_minutes as i64, + ), + connection_timeout: Duration::from_millis(x.connection_timeout_in_millis), + connection_min_idle: x.connection_min_idle, + connection_idle_timeout: Duration::from_secs(x.connection_idle_timeout_in_secs), + write_queue_size: x.write_queue_size, + disable_error_for_toplevel_parents: x.disable_error_for_toplevel_parents.0, + } + } +} + +#[derive(Clone, Debug, Envconfig)] +pub struct InnerStore { + #[envconfig(from = "GRAPH_CHAIN_HEAD_WATCHER_TIMEOUT", default = "30")] + chain_head_watcher_timeout_in_secs: u64, + #[envconfig(from = "GRAPH_QUERY_STATS_REFRESH_INTERVAL", default = "300")] + query_stats_refresh_interval_in_secs: u64, + #[envconfig(from = "GRAPH_EXTRA_QUERY_PERMITS", default = "0")] + extra_query_permits: usize, + #[envconfig(from = "LARGE_NOTIFICATION_CLEANUP_INTERVAL", default = "300")] + large_notification_cleanup_interval_in_secs: u64, + #[envconfig(from = "GRAPH_NOTIFICATION_BROADCAST_TIMEOUT", default = "60")] + notification_broadcast_timeout_in_secs: u64, + #[envconfig(from = "TYPEA_BATCH_SIZE", default = "150")] + typea_batch_size: usize, + #[envconfig(from = "TYPED_CHILDREN_SET_SIZE", default = "150")] + typed_children_set_size: usize, + #[envconfig(from = "ORDER_BY_BLOCK_RANGE", default = "true")] + order_by_block_range: EnvVarBoolean, + #[envconfig(from = "REVERSIBLE_ORDER_BY_OFF", default = "false")] + reversible_order_by_off: EnvVarBoolean, + #[envconfig(from = "GRAPH_DISABLE_SUBSCRIPTION_NOTIFICATIONS", default = "false")] + disable_subscription_notifications: EnvVarBoolean, + #[envconfig(from = "GRAPH_STORE_CONNECTION_TRY_ALWAYS", default = "false")] + connection_try_always: EnvVarBoolean, + #[envconfig(from = "GRAPH_REMOVE_UNUSED_INTERVAL", default = "360")] + remove_unused_interval_in_minutes: u64, + + // These should really be set through the configuration file, especially for + // `GRAPH_STORE_CONNECTION_MIN_IDLE` and + // `GRAPH_STORE_CONNECTION_IDLE_TIMEOUT`. It's likely that they should be + // configured differently for each pool. + #[envconfig(from = "GRAPH_STORE_CONNECTION_TIMEOUT", default = "5000")] + connection_timeout_in_millis: u64, + #[envconfig(from = "GRAPH_STORE_CONNECTION_MIN_IDLE")] + connection_min_idle: Option, + #[envconfig(from = "GRAPH_STORE_CONNECTION_IDLE_TIMEOUT", default = "600")] + connection_idle_timeout_in_secs: u64, + #[envconfig(from = "GRAPH_STORE_WRITE_QUEUE", default = "5")] + write_queue_size: usize, + #[envconfig(from = "GRAPH_DISABLE_ERROR_FOR_TOPLEVEL_PARENTS", default = "false")] + disable_error_for_toplevel_parents: EnvVarBoolean, +} diff --git a/graph/src/ext/futures.rs b/graph/src/ext/futures.rs new file mode 100644 index 0000000..57eae69 --- /dev/null +++ b/graph/src/ext/futures.rs @@ -0,0 +1,330 @@ +use crate::prelude::tokio::macros::support::Poll; +use crate::prelude::{Pin, StoreError}; +use futures03::channel::oneshot; +use futures03::{future::Fuse, Future, FutureExt, Stream}; +use std::fmt::{Debug, Display}; +use std::sync::{Arc, Mutex, Weak}; +use std::task::Context; +use std::time::Duration; + +/// A cancelable stream or future. +/// +/// Created by calling `cancelable` extension method. +/// Can be canceled through the corresponding `CancelGuard`. +pub struct Cancelable { + inner: T, + cancel_receiver: Fuse>, + on_cancel: C, +} + +impl Cancelable { + pub fn get_mut(&mut self) -> &mut T { + &mut self.inner + } +} + +/// It's not viable to use `select` directly, so we do a custom implementation. +impl S::Item + Unpin> Stream for Cancelable { + type Item = S::Item; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + // Error if the stream was canceled by dropping the sender. + match self.cancel_receiver.poll_unpin(cx) { + Poll::Ready(Ok(_)) => unreachable!(), + Poll::Ready(Err(_)) => Poll::Ready(Some((self.on_cancel)())), + Poll::Pending => Pin::new(&mut self.inner).poll_next(cx), + } + } +} + +impl F::Output + Unpin> Future for Cancelable { + type Output = F::Output; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // Error if the future was canceled by dropping the sender. + // `canceled` is fused so we may ignore `Ok`s. + match self.cancel_receiver.poll_unpin(cx) { + Poll::Ready(Ok(_)) => unreachable!(), + Poll::Ready(Err(_)) => Poll::Ready((self.on_cancel)()), + Poll::Pending => Pin::new(&mut self.inner).poll(cx), + } + } +} + +/// A `CancelGuard` or `SharedCancelGuard`. +pub trait Canceler { + /// Adds `cancel_sender` to the set being guarded. + /// Avoid calling directly and prefer using `cancelable`. + fn add_cancel_sender(&self, cancel_sender: oneshot::Sender<()>); +} + +/// Cancels any guarded futures and streams when dropped. +#[derive(Debug, Default)] +pub struct CancelGuard { + /// This is the only non-temporary strong reference to this `Arc`, therefore + /// the `Vec` should be dropped shortly after `self` is dropped. + cancel_senders: Arc>>>, +} + +impl CancelGuard { + /// Creates a guard that initially guards nothing. + pub fn new() -> Self { + Self::default() + } + + /// A more readable `drop`. + pub fn cancel(self) {} + + pub fn handle(&self) -> CancelHandle { + CancelHandle { + guard: Arc::downgrade(&self.cancel_senders), + } + } +} + +impl Canceler for CancelGuard { + fn add_cancel_sender(&self, cancel_sender: oneshot::Sender<()>) { + self.cancel_senders.lock().unwrap().push(cancel_sender); + } +} + +/// A shared handle to a guard, used to add more cancelables. The handle +/// may outlive the guard, if `cancelable` is called with a handle to a +/// dropped guard, then the future or stream it is immediately canceled. +/// +/// Dropping a handle has no effect. +#[derive(Clone, Debug)] +pub struct CancelHandle { + guard: Weak>>>, +} + +pub trait CancelToken { + fn is_canceled(&self) -> bool; + fn check_cancel(&self) -> Result<(), Canceled> { + if self.is_canceled() { + Err(Canceled) + } else { + Ok(()) + } + } +} + +pub struct NeverCancel; + +impl CancelToken for NeverCancel { + #[inline] + fn is_canceled(&self) -> bool { + false + } +} + +pub struct Canceled; + +impl CancelToken for CancelHandle { + fn is_canceled(&self) -> bool { + // Has been canceled if and only if the guard is gone. + self.guard.upgrade().is_none() + } +} + +impl Canceler for CancelHandle { + fn add_cancel_sender(&self, cancel_sender: oneshot::Sender<()>) { + if let Some(guard) = self.guard.upgrade() { + // If the guard exists, register the canceler. + guard.lock().unwrap().push(cancel_sender); + } else { + // Otherwise cancel immediately. + drop(cancel_sender) + } + } +} + +/// A cancelation guard that can be canceled through a shared reference such as +/// an `Arc`. +/// +/// To cancel guarded streams or futures, call `cancel` or drop the guard. +#[derive(Debug)] +pub struct SharedCancelGuard { + guard: Mutex>, +} + +impl SharedCancelGuard { + /// Creates a guard that initially guards nothing. + pub fn new() -> Self { + Self::default() + } + + /// Cancels the stream, a noop if already canceled. + pub fn cancel(&self) { + *self.guard.lock().unwrap() = None + } + + pub fn is_canceled(&self) -> bool { + self.guard.lock().unwrap().is_none() + } + + pub fn handle(&self) -> CancelHandle { + if let Some(ref guard) = *self.guard.lock().unwrap() { + guard.handle() + } else { + // A handle that is always canceled. + CancelHandle { guard: Weak::new() } + } + } +} + +impl Default for SharedCancelGuard { + fn default() -> Self { + Self { + guard: Mutex::new(Some(CancelGuard::new())), + } + } +} + +impl Canceler for SharedCancelGuard { + /// Cancels immediately if `self` has already been canceled. + fn add_cancel_sender(&self, cancel_sender: oneshot::Sender<()>) { + if let Some(ref mut guard) = *self.guard.lock().unwrap() { + guard.add_cancel_sender(cancel_sender); + } else { + drop(cancel_sender) + } + } +} + +/// An implementor of `Canceler` that never cancels, +/// making `cancelable` a noop. +#[derive(Debug, Default)] +pub struct DummyCancelGuard; + +impl Canceler for DummyCancelGuard { + fn add_cancel_sender(&self, cancel_sender: oneshot::Sender<()>) { + // Send to the channel, preventing cancelation. + let _ = cancel_sender.send(()); + } +} + +pub trait StreamExtension: Stream + Sized { + /// When `cancel` is called on a `CancelGuard` or it is dropped, + /// `Cancelable` receives an error. + /// + fn cancelable Self::Item>( + self, + guard: &impl Canceler, + on_cancel: C, + ) -> Cancelable; +} + +impl StreamExtension for S { + fn cancelable S::Item>( + self, + guard: &impl Canceler, + on_cancel: C, + ) -> Cancelable { + let (canceler, cancel_receiver) = oneshot::channel(); + guard.add_cancel_sender(canceler); + Cancelable { + inner: self, + cancel_receiver: cancel_receiver.fuse(), + on_cancel, + } + } +} + +pub trait FutureExtension: Future + Sized { + /// When `cancel` is called on a `CancelGuard` or it is dropped, + /// `Cancelable` receives an error. + /// + /// `on_cancel` is called to make an error value upon cancelation. + fn cancelable Self::Output>( + self, + guard: &impl Canceler, + on_cancel: C, + ) -> Cancelable; + + fn timeout(self, dur: Duration) -> tokio::time::Timeout; +} + +impl FutureExtension for F { + fn cancelable F::Output>( + self, + guard: &impl Canceler, + on_cancel: C, + ) -> Cancelable { + let (canceler, cancel_receiver) = oneshot::channel(); + guard.add_cancel_sender(canceler); + Cancelable { + inner: self, + cancel_receiver: cancel_receiver.fuse(), + on_cancel, + } + } + + fn timeout(self, dur: Duration) -> tokio::time::Timeout { + tokio::time::timeout(dur, self) + } +} + +#[derive(thiserror::Error, Debug)] +pub enum CancelableError { + #[error("operation canceled")] + Cancel, + + #[error("{0:}")] + Error(E), +} + +impl From for CancelableError { + fn from(e: StoreError) -> Self { + Self::Error(anyhow::Error::from(e)) + } +} + +impl From> for CancelableError { + fn from(e: CancelableError) -> Self { + match e { + CancelableError::Error(e) => CancelableError::Error(e.into()), + CancelableError::Cancel => CancelableError::Cancel, + } + } +} + +impl From for CancelableError { + fn from(e: StoreError) -> Self { + Self::Error(e) + } +} + +impl From for CancelableError { + fn from(_: Canceled) -> Self { + Self::Cancel + } +} + +impl From for CancelableError { + fn from(e: diesel::result::Error) -> Self { + Self::Error(e.into()) + } +} + +impl From for CancelableError { + fn from(e: diesel::result::Error) -> Self { + Self::Error(e.into()) + } +} + +impl From for CancelableError { + fn from(e: anyhow::Error) -> Self { + Self::Error(e) + } +} + +impl From> for StoreError { + fn from(err: CancelableError) -> StoreError { + use CancelableError::*; + match err { + Cancel => StoreError::Canceled, + Error(e) => e, + } + } +} diff --git a/graph/src/ext/mod.rs b/graph/src/ext/mod.rs new file mode 100644 index 0000000..4e9773f --- /dev/null +++ b/graph/src/ext/mod.rs @@ -0,0 +1,2 @@ +///! Extension traits for external types. +pub mod futures; diff --git a/graph/src/firehose/.gitignore b/graph/src/firehose/.gitignore new file mode 100644 index 0000000..0a1cbe2 --- /dev/null +++ b/graph/src/firehose/.gitignore @@ -0,0 +1,3 @@ +# For an unknown reason, the build script generates this file but it should not. +# See https://github.com/hyperium/tonic/issues/757 +google.protobuf.rs \ No newline at end of file diff --git a/graph/src/firehose/codec.rs b/graph/src/firehose/codec.rs new file mode 100644 index 0000000..c3b81ce --- /dev/null +++ b/graph/src/firehose/codec.rs @@ -0,0 +1,20 @@ +#[rustfmt::skip] +#[path = "sf.firehose.v1.rs"] +mod pbfirehose; + +#[rustfmt::skip] +#[path = "sf.ethereum.transform.v1.rs"] +mod pbethereum; + +#[rustfmt::skip] +#[path = "sf.near.transform.v1.rs"] +mod pbnear; + +#[rustfmt::skip] +#[path = "sf.cosmos.transform.v1.rs"] +mod pbcosmos; + +pub use pbcosmos::*; +pub use pbethereum::*; +pub use pbfirehose::*; +pub use pbnear::*; diff --git a/graph/src/firehose/endpoints.rs b/graph/src/firehose/endpoints.rs new file mode 100644 index 0000000..344506a --- /dev/null +++ b/graph/src/firehose/endpoints.rs @@ -0,0 +1,332 @@ +use crate::{ + blockchain::Block as BlockchainBlock, + blockchain::BlockPtr, + cheap_clone::CheapClone, + components::store::BlockNumber, + firehose::{decode_firehose_block, ForkStep}, + prelude::{debug, info}, + substreams, +}; +use futures03::StreamExt; +use http::uri::{Scheme, Uri}; +use rand::prelude::IteratorRandom; +use slog::Logger; +use std::{collections::BTreeMap, fmt::Display, iter, sync::Arc, time::Duration}; +use tonic::{ + metadata::MetadataValue, + transport::{Channel, ClientTlsConfig}, + Request, +}; + +use super::codec as firehose; + +#[derive(Clone, Debug)] +pub struct FirehoseEndpoint { + pub provider: String, + pub token: Option, + pub filters_enabled: bool, + pub compression_enabled: bool, + channel: Channel, +} + +impl Display for FirehoseEndpoint { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Display::fmt(self.provider.as_str(), f) + } +} + +impl FirehoseEndpoint { + pub fn new>( + provider: S, + url: S, + token: Option, + filters_enabled: bool, + compression_enabled: bool, + conn_pool_size: u16, + ) -> Self { + let uri = url + .as_ref() + .parse::() + .expect("the url should have been validated by now, so it is a valid Uri"); + + let endpoint_builder = match uri.scheme().unwrap_or(&Scheme::HTTP).as_str() { + "http" => Channel::builder(uri), + "https" => Channel::builder(uri) + .tls_config(ClientTlsConfig::new()) + .expect("TLS config on this host is invalid"), + _ => panic!("invalid uri scheme for firehose endpoint"), + }; + + // Note on the connection window size: We run multiple block streams on a same connection, + // and a problematic subgraph with a stalled block stream might consume the entire window + // capacity for its http2 stream and never release it. If there are enough stalled block + // streams to consume all the capacity on the http2 connection, then _all_ subgraphs using + // this same http2 connection will stall. At a default stream window size of 2^16, setting + // the connection window size to the maximum of 2^31 allows for 2^15 streams without any + // contention, which is effectively unlimited for normal graph node operation. + // + // Note: Do not set `http2_keep_alive_interval` or `http2_adaptive_window`, as these will + // send ping frames, and many cloud load balancers will drop connections that frequently + // send pings. + let endpoint = endpoint_builder + .initial_connection_window_size(Some((1 << 31) - 1)) + .connect_timeout(Duration::from_secs(10)) + .tcp_keepalive(Some(Duration::from_secs(15))) + // Timeout on each request, so the timeout to estabilish each 'Blocks' stream. + .timeout(Duration::from_secs(120)); + + // Load balancing on a same endpoint is useful because it creates a connection pool. + let channel = Channel::balance_list(iter::repeat(endpoint).take(conn_pool_size as usize)); + + FirehoseEndpoint { + provider: provider.as_ref().to_string(), + channel, + token, + filters_enabled, + compression_enabled, + } + } + + pub async fn genesis_block_ptr(&self, logger: &Logger) -> Result + where + M: prost::Message + BlockchainBlock + Default + 'static, + { + info!(logger, "Requesting genesis block from firehose"); + + // We use 0 here to mean the genesis block of the chain. Firehose + // when seeing start block number 0 will always return the genesis + // block of the chain, even if the chain's start block number is + // not starting at block #0. + self.block_ptr_for_number::(logger, 0).await + } + + pub async fn block_ptr_for_number( + &self, + logger: &Logger, + number: BlockNumber, + ) -> Result + where + M: prost::Message + BlockchainBlock + Default + 'static, + { + let token_metadata = match self.token.clone() { + Some(token) => Some(MetadataValue::from_str(token.as_str())?), + None => None, + }; + + let mut client = firehose::stream_client::StreamClient::with_interceptor( + self.channel.cheap_clone(), + move |mut r: Request<()>| { + if let Some(ref t) = token_metadata { + r.metadata_mut().insert("authorization", t.clone()); + } + + Ok(r) + }, + ) + .accept_gzip(); + + if self.compression_enabled { + client = client.send_gzip(); + } + + debug!( + logger, + "Connecting to firehose to retrieve block for number {}", number + ); + + // The trick is the following. + // + // Firehose `start_block_num` and `stop_block_num` are both inclusive, so we specify + // the block we are looking for in both. + // + // Now, the remaining question is how the block from the canonical chain is picked. We + // leverage the fact that Firehose will always send the block in the longuest chain as the + // last message of this request. + // + // That way, we either get the final block if the block is now in a final segment of the + // chain (or probabilisticly if not finality concept exists for the chain). Or we get the + // block that is in the longuest chain according to Firehose. + let response_stream = client + .blocks(firehose::Request { + start_block_num: number as i64, + stop_block_num: number as u64, + fork_steps: vec![ForkStep::StepNew as i32, ForkStep::StepUndo as i32], + ..Default::default() + }) + .await?; + + let mut block_stream = response_stream.into_inner(); + + debug!(logger, "Retrieving block(s) from firehose"); + + let mut latest_received_block: Option = None; + while let Some(message) = block_stream.next().await { + match message { + Ok(v) => { + let block = decode_firehose_block::(&v)?.ptr(); + + match latest_received_block { + None => { + latest_received_block = Some(block); + } + Some(ref actual_ptr) => { + // We want to receive all events related to a specific block number, + // however, in some circumstances, it seems Firehose would not stop sending + // blocks (`start_block_num: 0 and stop_block_num: 0` on NEAR seems to trigger + // this). + // + // To prevent looping infinitely, we stop as soon as a new received block's + // number is higher than the latest received block's number, in which case it + // means it's an event for a block we are not interested in. + if block.number > actual_ptr.number { + break; + } + + latest_received_block = Some(block); + } + } + } + Err(e) => return Err(anyhow::format_err!("firehose error {}", e)), + }; + } + + match latest_received_block { + Some(block_ptr) => Ok(block_ptr), + None => Err(anyhow::format_err!( + "Firehose should have returned at least one block for request" + )), + } + } + + pub async fn stream_blocks( + self: Arc, + request: firehose::Request, + ) -> Result, anyhow::Error> { + let token_metadata = match self.token.clone() { + Some(token) => Some(MetadataValue::from_str(token.as_str())?), + None => None, + }; + + let mut client = firehose::stream_client::StreamClient::with_interceptor( + self.channel.cheap_clone(), + move |mut r: Request<()>| { + if let Some(ref t) = token_metadata { + r.metadata_mut().insert("authorization", t.clone()); + } + + Ok(r) + }, + ) + .accept_gzip(); + if self.compression_enabled { + client = client.send_gzip(); + } + + let response_stream = client.blocks(request).await?; + let block_stream = response_stream.into_inner(); + + Ok(block_stream) + } + + pub async fn substreams( + self: Arc, + request: substreams::Request, + ) -> Result, anyhow::Error> { + let token_metadata = match self.token.clone() { + Some(token) => Some(MetadataValue::from_str(token.as_str())?), + None => None, + }; + + let mut client = substreams::stream_client::StreamClient::with_interceptor( + self.channel.cheap_clone(), + move |mut r: Request<()>| { + if let Some(ref t) = token_metadata { + r.metadata_mut().insert("authorization", t.clone()); + } + + Ok(r) + }, + ); + + let response_stream = client.blocks(request).await?; + let block_stream = response_stream.into_inner(); + + Ok(block_stream) + } +} + +#[derive(Clone, Debug)] +pub struct FirehoseEndpoints(Vec>); + +impl FirehoseEndpoints { + pub fn new() -> Self { + Self(vec![]) + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn random(&self) -> Option<&Arc> { + // Select from the matching adapters randomly + let mut rng = rand::thread_rng(); + self.0.iter().choose(&mut rng) + } + + pub fn remove(&mut self, provider: &str) { + self.0 + .retain(|network_endpoint| network_endpoint.provider != provider); + } +} + +impl From>> for FirehoseEndpoints { + fn from(val: Vec>) -> Self { + FirehoseEndpoints(val) + } +} + +#[derive(Clone, Debug)] +pub struct FirehoseNetworks { + /// networks contains a map from chain id (`near-mainnet`, `near-testnet`, `solana-mainnet`, etc.) + /// to a list of FirehoseEndpoint (type wrapper around `Arc>`). + pub networks: BTreeMap, +} + +impl FirehoseNetworks { + pub fn new() -> FirehoseNetworks { + FirehoseNetworks { + networks: BTreeMap::new(), + } + } + + pub fn insert(&mut self, chain_id: String, endpoint: Arc) { + let endpoints = self + .networks + .entry(chain_id) + .or_insert_with(FirehoseEndpoints::new); + + endpoints.0.push(endpoint); + } + + pub fn remove(&mut self, chain_id: &str, provider: &str) { + if let Some(endpoints) = self.networks.get_mut(chain_id) { + endpoints.remove(provider); + } + } + + /// Returns a `Vec` of tuples where the first element of the tuple is + /// the chain's id and the second one is an endpoint for this chain. + /// There can be mulitple tuple with the same chain id but with different + /// endpoint where multiple providers exist for a single chain id. + pub fn flatten(&self) -> Vec<(String, Arc)> { + self.networks + .iter() + .flat_map(|(chain_id, firehose_endpoints)| { + firehose_endpoints + .0 + .iter() + .map(move |endpoint| (chain_id.clone(), endpoint.clone())) + }) + .collect() + } +} diff --git a/graph/src/firehose/helpers.rs b/graph/src/firehose/helpers.rs new file mode 100644 index 0000000..b66052b --- /dev/null +++ b/graph/src/firehose/helpers.rs @@ -0,0 +1,19 @@ +use std::sync::Arc; + +use crate::blockchain::Block as BlockchainBlock; +use crate::firehose; +use anyhow::Error; + +pub fn decode_firehose_block( + block_response: &firehose::Response, +) -> Result, Error> +where + M: prost::Message + BlockchainBlock + Default + 'static, +{ + let any_block = block_response + .block + .as_ref() + .expect("block payload information should always be present"); + + Ok(Arc::new(M::decode(any_block.value.as_ref())?)) +} diff --git a/graph/src/firehose/mod.rs b/graph/src/firehose/mod.rs new file mode 100644 index 0000000..8dd12b0 --- /dev/null +++ b/graph/src/firehose/mod.rs @@ -0,0 +1,7 @@ +mod codec; +mod endpoints; +mod helpers; + +pub use codec::*; +pub use endpoints::*; +pub use helpers::decode_firehose_block; diff --git a/graph/src/firehose/sf.cosmos.transform.v1.rs b/graph/src/firehose/sf.cosmos.transform.v1.rs new file mode 100644 index 0000000..f93fef1 --- /dev/null +++ b/graph/src/firehose/sf.cosmos.transform.v1.rs @@ -0,0 +1,5 @@ +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EventTypeFilter { + #[prost(string, repeated, tag="1")] + pub event_types: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} diff --git a/graph/src/firehose/sf.ethereum.transform.v1.rs b/graph/src/firehose/sf.ethereum.transform.v1.rs new file mode 100644 index 0000000..b677e17 --- /dev/null +++ b/graph/src/firehose/sf.ethereum.transform.v1.rs @@ -0,0 +1,69 @@ +/// CombinedFilter is a combination of "LogFilters" and "CallToFilters" +/// +/// It transforms the requested stream in two ways: +/// 1. STRIPPING +/// The block data is stripped from all transactions that don't +/// match any of the filters. +/// +/// 2. SKIPPING +/// If an "block index" covers a range containing a +/// block that does NOT match any of the filters, the block will be +/// skipped altogether, UNLESS send_all_block_headers is enabled +/// In that case, the block would still be sent, but without any +/// transactionTrace +/// +/// The SKIPPING feature only applies to historical blocks, because +/// the "block index" is always produced after the merged-blocks files +/// are produced. Therefore, the "live" blocks are never filtered out. +/// +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CombinedFilter { + #[prost(message, repeated, tag="1")] + pub log_filters: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="2")] + pub call_filters: ::prost::alloc::vec::Vec, + /// Always send all blocks. if they don't match any log_filters or call_filters, + /// all the transactions will be filtered out, sending only the header. + #[prost(bool, tag="3")] + pub send_all_block_headers: bool, +} +/// MultiLogFilter concatenates the results of each LogFilter (inclusive OR) +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MultiLogFilter { + #[prost(message, repeated, tag="1")] + pub log_filters: ::prost::alloc::vec::Vec, +} +/// LogFilter will match calls where *BOTH* +/// * the contract address that emits the log is one in the provided addresses -- OR addresses list is empty -- +/// * the event signature (topic.0) is one of the provided event_signatures -- OR event_signatures is empty -- +/// +/// a LogFilter with both empty addresses and event_signatures lists is invalid and will fail. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LogFilter { + #[prost(bytes="vec", repeated, tag="1")] + pub addresses: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + /// corresponds to the keccak of the event signature which is stores in topic.0 + #[prost(bytes="vec", repeated, tag="2")] + pub event_signatures: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, +} +/// MultiCallToFilter concatenates the results of each CallToFilter (inclusive OR) +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MultiCallToFilter { + #[prost(message, repeated, tag="1")] + pub call_filters: ::prost::alloc::vec::Vec, +} +/// CallToFilter will match calls where *BOTH* +/// * the contract address (TO) is one in the provided addresses -- OR addresses list is empty -- +/// * the method signature (in 4-bytes format) is one of the provided signatures -- OR signatures is empty -- +/// +/// a CallToFilter with both empty addresses and signatures lists is invalid and will fail. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CallToFilter { + #[prost(bytes="vec", repeated, tag="1")] + pub addresses: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + #[prost(bytes="vec", repeated, tag="2")] + pub signatures: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LightBlock { +} diff --git a/graph/src/firehose/sf.firehose.v1.rs b/graph/src/firehose/sf.firehose.v1.rs new file mode 100644 index 0000000..d57c793 --- /dev/null +++ b/graph/src/firehose/sf.firehose.v1.rs @@ -0,0 +1,332 @@ +/// For historical segments, forks are not passed +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Request { + /// Controls where the stream of blocks will start. + /// + /// The stream will start **inclusively** at the requested block num. + /// + /// When not provided, starts at first streamable block of the chain. Not all + /// chain starts at the same block number, so you might get an higher block than + /// requested when using default value of 0. + /// + /// Can be negative, will be resolved relative to the chain head block, assuming + /// a chain at head block #100, then using `-50` as the value will start at block + /// #50. If it resolves before first streamable block of chain, we assume start + /// of chain. + /// + /// If `start_cursor` is passed, this value is ignored and the stream instead starts + /// immediately after the Block pointed by the opaque `start_cursor` value. + #[prost(int64, tag="1")] + pub start_block_num: i64, + /// Controls where the stream of blocks will start which will be immediately after + /// the Block pointed by this opaque cursor. + /// + /// Obtain this value from a previously received from `Response.cursor`. + /// + /// This value takes precedence over `start_block_num`. + #[prost(string, tag="13")] + pub start_cursor: ::prost::alloc::string::String, + /// When non-zero, controls where the stream of blocks will stop. + /// + /// The stream will close **after** that block has passed so the boundary is + /// **inclusive**. + #[prost(uint64, tag="5")] + pub stop_block_num: u64, + /// Filter the steps you want to see. If not specified, defaults to all steps. + /// + /// Most common steps will be \[STEP_IRREVERSIBLE\], or [STEP_NEW, STEP_UNDO, STEP_IRREVERSIBLE]. + #[prost(enumeration="ForkStep", repeated, tag="8")] + pub fork_steps: ::prost::alloc::vec::Vec, + /// The CEL filter expression used to include transactions, specific to the target protocol, + /// works in combination with `exclude_filter_expr` value. + #[prost(string, tag="10")] + pub include_filter_expr: ::prost::alloc::string::String, + /// The CEL filter expression used to exclude transactions, specific to the target protocol, works + /// in combination with `include_filter_expr` value. + #[prost(string, tag="11")] + pub exclude_filter_expr: ::prost::alloc::string::String, + ///- EOS "handoffs:3" + ///- EOS "lib" + ///- EOS "confirms:3" + ///- ETH "confirms:200" + ///- ETH "confirms:7" + ///- SOL "commmitement:finalized" + ///- SOL "confirms:200" + #[prost(string, tag="17")] + pub irreversibility_condition: ::prost::alloc::string::String, + #[prost(message, repeated, tag="18")] + pub transforms: ::prost::alloc::vec::Vec<::prost_types::Any>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Response { + /// Chain specific block payload, one of: + /// - sf.eosio.codec.v1.Block + /// - sf.ethereum.codec.v1.Block + /// - sf.near.codec.v1.Block + /// - sf.solana.codec.v1.Block + #[prost(message, optional, tag="1")] + pub block: ::core::option::Option<::prost_types::Any>, + #[prost(enumeration="ForkStep", tag="6")] + pub step: i32, + #[prost(string, tag="10")] + pub cursor: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ForkStep { + StepUnknown = 0, + /// Block is new head block of the chain, that is linear with the previous block + StepNew = 1, + /// Block is now forked and should be undone, it's not the head block of the chain anymore + StepUndo = 2, + /// Block is now irreversible and can be committed to (finality is chain specific, see chain documentation for more details) + StepIrreversible = 4, +} +/// TODO: move to ethereum specific transforms +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum BlockDetails { + Full = 0, + Light = 1, +} +/// Generated client implementations. +pub mod stream_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + #[derive(Debug, Clone)] + pub struct StreamClient { + inner: tonic::client::Grpc, + } + impl StreamClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: std::convert::TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl StreamClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + Send + 'static, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> StreamClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + Send + Sync, + { + StreamClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with `gzip`. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_gzip(mut self) -> Self { + self.inner = self.inner.send_gzip(); + self + } + /// Enable decompressing responses with `gzip`. + #[must_use] + pub fn accept_gzip(mut self) -> Self { + self.inner = self.inner.accept_gzip(); + self + } + pub async fn blocks( + &mut self, + request: impl tonic::IntoRequest, + ) -> Result< + tonic::Response>, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sf.firehose.v1.Stream/Blocks", + ); + self.inner.server_streaming(request.into_request(), path, codec).await + } + } +} +/// Generated server implementations. +pub mod stream_server { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + ///Generated trait containing gRPC methods that should be implemented for use with StreamServer. + #[async_trait] + pub trait Stream: Send + Sync + 'static { + ///Server streaming response type for the Blocks method. + type BlocksStream: futures_core::Stream< + Item = Result, + > + + Send + + 'static; + async fn blocks( + &self, + request: tonic::Request, + ) -> Result, tonic::Status>; + } + #[derive(Debug)] + pub struct StreamServer { + inner: _Inner, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + } + struct _Inner(Arc); + impl StreamServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + let inner = _Inner(inner); + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with `gzip`. + #[must_use] + pub fn accept_gzip(mut self) -> Self { + self.accept_compression_encodings.enable_gzip(); + self + } + /// Compress responses with `gzip`, if the client supports it. + #[must_use] + pub fn send_gzip(mut self) -> Self { + self.send_compression_encodings.enable_gzip(); + self + } + } + impl tonic::codegen::Service> for StreamServer + where + T: Stream, + B: Body + Send + 'static, + B::Error: Into + Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + let inner = self.inner.clone(); + match req.uri().path() { + "/sf.firehose.v1.Stream/Blocks" => { + #[allow(non_camel_case_types)] + struct BlocksSvc(pub Arc); + impl tonic::server::ServerStreamingService + for BlocksSvc { + type Response = super::Response; + type ResponseStream = T::BlocksStream; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = self.0.clone(); + let fut = async move { (*inner).blocks(request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = BlocksSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + Ok( + http::Response::builder() + .status(200) + .header("grpc-status", "12") + .header("content-type", "application/grpc") + .body(empty_body()) + .unwrap(), + ) + }) + } + } + } + } + impl Clone for StreamServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + } + } + } + impl Clone for _Inner { + fn clone(&self) -> Self { + Self(self.0.clone()) + } + } + impl std::fmt::Debug for _Inner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } + } + impl tonic::transport::NamedService for StreamServer { + const NAME: &'static str = "sf.firehose.v1.Stream"; + } +} diff --git a/graph/src/firehose/sf.near.transform.v1.rs b/graph/src/firehose/sf.near.transform.v1.rs new file mode 100644 index 0000000..86972b6 --- /dev/null +++ b/graph/src/firehose/sf.near.transform.v1.rs @@ -0,0 +1,21 @@ +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BasicReceiptFilter { + #[prost(string, repeated, tag="1")] + pub accounts: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(message, repeated, tag="2")] + pub prefix_and_suffix_pairs: ::prost::alloc::vec::Vec, +} +/// PrefixSuffixPair applies a logical AND to prefix and suffix when both fields are non-empty. +/// * {prefix="hello",suffix="world"} will match "hello.world" but not "hello.friend" +/// * {prefix="hello",suffix=""} will match both "hello.world" and "hello.friend" +/// * {prefix="",suffix="world"} will match both "hello.world" and "good.day.world" +/// * {prefix="",suffix=""} is invalid +/// +/// Note that the suffix will usually have a TLD, ex: "mydomain.near" or "mydomain.testnet" +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PrefixSuffixPair { + #[prost(string, tag="1")] + pub prefix: ::prost::alloc::string::String, + #[prost(string, tag="2")] + pub suffix: ::prost::alloc::string::String, +} diff --git a/graph/src/ipfs_client.rs b/graph/src/ipfs_client.rs new file mode 100644 index 0000000..c99d83b --- /dev/null +++ b/graph/src/ipfs_client.rs @@ -0,0 +1,169 @@ +use crate::prelude::CheapClone; +use anyhow::Error; +use bytes::Bytes; +use futures03::Stream; +use http::header::CONTENT_LENGTH; +use http::Uri; +use reqwest::multipart; +use serde::Deserialize; +use std::time::Duration; +use std::{str::FromStr, sync::Arc}; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum StatApi { + Block, + Files, +} + +impl StatApi { + fn route(&self) -> &'static str { + match self { + Self::Block => "block", + Self::Files => "files", + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "PascalCase")] +struct BlockStatResponse { + size: u64, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "PascalCase")] +struct FilesStatResponse { + cumulative_size: u64, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct AddResponse { + pub name: String, + pub hash: String, + pub size: String, +} + +/// Reference type, clones will share the connection pool. +#[derive(Clone)] +pub struct IpfsClient { + base: Arc, + client: Arc, +} + +impl CheapClone for IpfsClient { + fn cheap_clone(&self) -> Self { + IpfsClient { + base: self.base.cheap_clone(), + client: self.client.cheap_clone(), + } + } +} + +impl IpfsClient { + pub fn new(base: &str) -> Result { + Ok(IpfsClient { + client: Arc::new(reqwest::Client::new()), + base: Arc::new(Uri::from_str(base)?), + }) + } + + pub fn localhost() -> Self { + IpfsClient { + client: Arc::new(reqwest::Client::new()), + base: Arc::new(Uri::from_str("http://localhost:5001").unwrap()), + } + } + + /// Calls stat for the given API route, and returns the total size of the object. + pub async fn stat_size( + &self, + api: StatApi, + mut cid: String, + timeout: Duration, + ) -> Result { + let route = format!("{}/stat", api.route()); + if api == StatApi::Files { + // files/stat requires a leading `/ipfs/`. + cid = format!("/ipfs/{}", cid); + } + let url = self.url(&route, &cid); + let res = self.call(url, None, Some(timeout)).await?; + match api { + StatApi::Files => Ok(res.json::().await?.cumulative_size), + StatApi::Block => Ok(res.json::().await?.size), + } + } + + /// Download the entire contents. + pub async fn cat_all(&self, cid: &str, timeout: Duration) -> Result { + self.call(self.url("cat", cid), None, Some(timeout)) + .await? + .bytes() + .await + } + + pub async fn cat( + &self, + cid: &str, + timeout: Option, + ) -> Result>, reqwest::Error> { + Ok(self + .call(self.url("cat", cid), None, timeout) + .await? + .bytes_stream()) + } + + pub async fn get_block(&self, cid: String) -> Result { + let form = multipart::Form::new().part("arg", multipart::Part::text(cid)); + self.call(format!("{}api/v0/block/get", self.base), Some(form), None) + .await? + .bytes() + .await + } + + pub async fn test(&self) -> Result<(), reqwest::Error> { + self.call(format!("{}api/v0/version", self.base), None, None) + .await + .map(|_| ()) + } + + pub async fn add(&self, data: Vec) -> Result { + let form = multipart::Form::new().part("path", multipart::Part::bytes(data)); + + self.call(format!("{}api/v0/add", self.base), Some(form), None) + .await? + .json() + .await + } + + fn url(&self, route: &str, arg: &str) -> String { + // URL security: We control the base and the route, user-supplied input goes only into the + // query parameters. + format!("{}api/v0/{}?arg={}", self.base, route, arg) + } + + async fn call( + &self, + url: String, + form: Option, + timeout: Option, + ) -> Result { + let mut req = self.client.post(&url); + if let Some(form) = form { + req = req.multipart(form); + } else { + // Some servers require `content-length` even for an empty body. + req = req.header(CONTENT_LENGTH, 0); + } + + if let Some(timeout) = timeout { + req = req.timeout(timeout) + } + + req.send() + .await + .map(|res| res.error_for_status()) + .and_then(|x| x) + } +} diff --git a/graph/src/lib.rs b/graph/src/lib.rs new file mode 100644 index 0000000..553f134 --- /dev/null +++ b/graph/src/lib.rs @@ -0,0 +1,207 @@ +/// Traits and types for all system components. +pub mod components; + +/// Common data types used throughout The Graph. +pub mod data; + +/// Utilities. +pub mod util; + +/// Extension traits for external types. +pub mod ext; + +/// Logging utilities +pub mod log; + +/// `CheapClone` trait. +pub mod cheap_clone; + +pub mod ipfs_client; + +pub mod data_source; + +pub mod blockchain; + +pub mod runtime; + +pub mod firehose; + +pub mod substreams; + +/// Helpers for parsing environment variables. +pub mod env; + +/// Wrapper for spawning tasks that abort on panic, which is our default. +mod task_spawn; +pub use task_spawn::{ + block_on, spawn, spawn_allow_panic, spawn_blocking, spawn_blocking_allow_panic, spawn_thread, +}; + +pub use anyhow; +pub use bytes; +pub use itertools; +pub use parking_lot; +pub use petgraph; +pub use prometheus; +pub use semver; +pub use slog; +pub use stable_hash_legacy; +pub use tokio; +pub use tokio_stream; +pub use url; + +/// A prelude that makes all system component traits and data types available. +/// +/// Add the following code to import all traits and data types listed below at once. +/// +/// ``` +/// use graph::prelude::*; +/// ``` +pub mod prelude { + pub use super::entity; + pub use ::anyhow; + pub use anyhow::{anyhow, Context as _, Error}; + pub use async_trait::async_trait; + pub use bigdecimal; + pub use chrono; + pub use envconfig; + pub use ethabi; + pub use futures::future; + pub use futures::prelude::*; + pub use futures::stream; + pub use futures03; + pub use futures03::compat::{Future01CompatExt, Sink01CompatExt, Stream01CompatExt}; + pub use futures03::future::{FutureExt as _, TryFutureExt}; + pub use futures03::sink::SinkExt as _; + pub use futures03::stream::{StreamExt as _, TryStreamExt}; + pub use hex; + pub use lazy_static::lazy_static; + pub use prost; + pub use rand; + pub use reqwest; + pub use serde; + pub use serde_derive::{Deserialize, Serialize}; + pub use serde_json; + pub use serde_yaml; + pub use slog::{self, crit, debug, error, info, o, trace, warn, Logger}; + pub use std::convert::TryFrom; + pub use std::fmt::Debug; + pub use std::iter::FromIterator; + pub use std::pin::Pin; + pub use std::sync::Arc; + pub use std::time::Duration; + pub use thiserror; + pub use tiny_keccak; + pub use tokio; + pub use tonic; + pub use web3; + + pub type DynTryFuture<'a, Ok = (), Err = Error> = + Pin> + Send + 'a>>; + + pub use crate::blockchain::{BlockHash, BlockPtr}; + + pub use crate::components::ethereum::{ + EthereumBlock, EthereumBlockWithCalls, EthereumCall, LightEthereumBlock, + LightEthereumBlockExt, + }; + pub use crate::components::graphql::{ + GraphQLMetrics, GraphQlRunner, QueryLoadManager, SubscriptionResultFuture, + }; + pub use crate::components::link_resolver::{JsonStreamValue, JsonValueStream, LinkResolver}; + pub use crate::components::metrics::{ + aggregate::Aggregate, stopwatch::StopwatchMetrics, subgraph::*, Collector, Counter, + CounterVec, Gauge, GaugeVec, Histogram, HistogramOpts, HistogramVec, MetricsRegistry, Opts, + PrometheusError, Registry, + }; + pub use crate::components::server::index_node::IndexNodeServer; + pub use crate::components::server::metrics::MetricsServer; + pub use crate::components::server::query::GraphQLServer; + pub use crate::components::server::subscription::SubscriptionServer; + pub use crate::components::store::{ + AttributeNames, BlockNumber, CachedEthereumCall, ChainStore, Child, ChildMultiplicity, + EntityCache, EntityChange, EntityChangeOperation, EntityCollection, EntityFilter, + EntityLink, EntityModification, EntityOperation, EntityOrder, EntityQuery, EntityRange, + EntityWindow, EthereumCallCache, ParentLink, PartialBlockPtr, PoolWaitStats, QueryStore, + QueryStoreManager, StoreError, StoreEvent, StoreEventStream, StoreEventStreamBox, + SubgraphStore, UnfailOutcome, WindowAttribute, BLOCK_NUMBER_MAX, + }; + pub use crate::components::subgraph::{ + BlockState, DataSourceTemplateInfo, HostMetrics, RuntimeHost, RuntimeHostBuilder, + SubgraphAssignmentProvider, SubgraphInstanceManager, SubgraphRegistrar, + SubgraphVersionSwitchingMode, + }; + pub use crate::components::trigger_processor::TriggerProcessor; + pub use crate::components::versions::{ApiVersion, FeatureFlag}; + pub use crate::components::{transaction_receipt, EventConsumer, EventProducer}; + pub use crate::env::ENV_VARS; + + pub use crate::cheap_clone::CheapClone; + pub use crate::data::graphql::{ + shape_hash::shape_hash, SerializableValue, TryFromValue, ValueMap, + }; + pub use crate::data::query::{ + Query, QueryError, QueryExecutionError, QueryResult, QueryTarget, QueryVariables, + }; + pub use crate::data::schema::{ApiSchema, Schema}; + pub use crate::data::store::ethereum::*; + pub use crate::data::store::scalar::{BigDecimal, BigInt, BigIntSign}; + pub use crate::data::store::{ + AssignmentEvent, Attribute, Entity, NodeId, SubscriptionFilter, TryIntoEntity, Value, + ValueType, + }; + pub use crate::data::subgraph::schema::SubgraphDeploymentEntity; + pub use crate::data::subgraph::{ + CreateSubgraphResult, DataSourceContext, DeploymentHash, DeploymentState, Link, + SubgraphAssignmentProviderError, SubgraphManifest, SubgraphManifestResolveError, + SubgraphManifestValidationError, SubgraphName, SubgraphRegistrarError, + UnvalidatedSubgraphManifest, + }; + pub use crate::data::subscription::{ + QueryResultStream, Subscription, SubscriptionError, SubscriptionResult, + }; + pub use crate::ext::futures::{ + CancelGuard, CancelHandle, CancelToken, CancelableError, FutureExtension, + SharedCancelGuard, StreamExtension, + }; + pub use crate::impl_slog_value; + pub use crate::log::codes::LogCode; + pub use crate::log::elastic::{elastic_logger, ElasticDrainConfig, ElasticLoggingConfig}; + pub use crate::log::factory::{ + ComponentLoggerConfig, ElasticComponentLoggerConfig, LoggerFactory, + }; + pub use crate::log::split::split_logger; + pub use crate::util::cache_weight::CacheWeight; + pub use crate::util::futures::{retry, TimeoutError}; + pub use crate::util::stats::MovingStats; + + macro_rules! static_graphql { + ($m:ident, $m2:ident, {$($n:ident,)*}) => { + pub mod $m { + use graphql_parser::$m2 as $m; + pub use $m::*; + $( + pub type $n = $m::$n<'static, String>; + )* + } + }; + } + + // Static graphql mods. These are to be phased out, with a preference + // toward making graphql generic over text. This helps to ease the + // transition by providing the old graphql-parse 0.2.x API + static_graphql!(q, query, { + Document, Value, OperationDefinition, InlineFragment, TypeCondition, + FragmentSpread, Field, Selection, SelectionSet, FragmentDefinition, + Directive, VariableDefinition, Type, Query, + }); + static_graphql!(s, schema, { + Field, Directive, InterfaceType, ObjectType, Value, TypeDefinition, + EnumType, Type, Document, ScalarType, InputValue, DirectiveDefinition, + UnionType, InputObjectType, EnumValue, + }); + + pub mod r { + pub use crate::data::value::Value; + } +} diff --git a/graph/src/log/codes.rs b/graph/src/log/codes.rs new file mode 100644 index 0000000..b12d52c --- /dev/null +++ b/graph/src/log/codes.rs @@ -0,0 +1,30 @@ +use std::fmt::{Display, Error, Formatter}; + +pub enum LogCode { + SubgraphStartFailure, + SubgraphSyncingFailure, + SubgraphSyncingFailureNotRecorded, + BlockIngestionStatus, + BlockIngestionLagging, + GraphQlQuerySuccess, + GraphQlQueryFailure, + TokioContention, +} + +impl Display for LogCode { + fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { + let value = match self { + LogCode::SubgraphStartFailure => "SubgraphStartFailure", + LogCode::SubgraphSyncingFailure => "SubgraphSyncingFailure", + LogCode::SubgraphSyncingFailureNotRecorded => "SubgraphSyncingFailureNotRecorded", + LogCode::BlockIngestionStatus => "BlockIngestionStatus", + LogCode::BlockIngestionLagging => "BlockIngestionLagging", + LogCode::GraphQlQuerySuccess => "GraphQLQuerySuccess", + LogCode::GraphQlQueryFailure => "GraphQLQueryFailure", + LogCode::TokioContention => "TokioContention", + }; + write!(f, "{}", value) + } +} + +impl_slog_value!(LogCode, "{}"); diff --git a/graph/src/log/elastic.rs b/graph/src/log/elastic.rs new file mode 100644 index 0000000..1a6294b --- /dev/null +++ b/graph/src/log/elastic.rs @@ -0,0 +1,390 @@ +use std::collections::HashMap; +use std::fmt; +use std::fmt::Write; +use std::result::Result; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use chrono::prelude::{SecondsFormat, Utc}; +use futures03::TryFutureExt; +use http::header::CONTENT_TYPE; +use reqwest; +use reqwest::Client; +use serde::ser::Serializer as SerdeSerializer; +use serde::Serialize; +use serde_json::json; +use slog::*; +use slog_async; + +use crate::util::futures::retry; + +/// General configuration parameters for Elasticsearch logging. +#[derive(Clone, Debug)] +pub struct ElasticLoggingConfig { + /// The Elasticsearch service to log to. + pub endpoint: String, + /// The Elasticsearch username. + pub username: Option, + /// The Elasticsearch password (optional). + pub password: Option, + /// A client to serve as a connection pool to the endpoint. + pub client: Client, +} + +/// Serializes an slog log level using a serde Serializer. +fn serialize_log_level(level: &Level, serializer: S) -> Result +where + S: SerdeSerializer, +{ + serializer.serialize_str(match level { + Level::Critical => "critical", + Level::Error => "error", + Level::Warning => "warning", + Level::Info => "info", + Level::Debug => "debug", + Level::Trace => "trace", + }) +} + +// Log message meta data. +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct ElasticLogMeta { + module: String, + line: i64, + column: i64, +} + +// Log message to be written to Elasticsearch. +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct ElasticLog { + id: String, + #[serde(flatten)] + custom_id: HashMap, + arguments: HashMap, + timestamp: String, + text: String, + #[serde(serialize_with = "serialize_log_level")] + level: Level, + meta: ElasticLogMeta, +} + +struct HashMapKVSerializer { + kvs: Vec<(String, String)>, +} + +impl HashMapKVSerializer { + fn new() -> Self { + HashMapKVSerializer { + kvs: Default::default(), + } + } + + fn finish(self) -> HashMap { + let mut map = HashMap::new(); + self.kvs.into_iter().for_each(|(k, v)| { + map.insert(k, v); + }); + map + } +} + +impl Serializer for HashMapKVSerializer { + fn emit_arguments(&mut self, key: Key, val: &fmt::Arguments) -> slog::Result { + Ok(self.kvs.push((key.into(), format!("{}", val)))) + } +} + +/// A super-simple slog Serializer for concatenating key/value arguments. +struct SimpleKVSerializer { + kvs: Vec<(String, String)>, +} + +impl SimpleKVSerializer { + /// Creates a new `SimpleKVSerializer`. + fn new() -> Self { + SimpleKVSerializer { + kvs: Default::default(), + } + } + + /// Collects all key/value arguments into a single, comma-separated string. + /// Returns the number of key/value pairs and the string itself. + fn finish(self) -> (usize, String) { + ( + self.kvs.len(), + self.kvs + .iter() + .map(|(k, v)| format!("{}: {}", k, v)) + .collect::>() + .join(", "), + ) + } +} + +impl Serializer for SimpleKVSerializer { + fn emit_arguments(&mut self, key: Key, val: &fmt::Arguments) -> slog::Result { + Ok(self.kvs.push((key.into(), format!("{}", val)))) + } +} + +/// Configuration for `ElasticDrain`. +#[derive(Clone, Debug)] +pub struct ElasticDrainConfig { + /// General Elasticsearch logging configuration. + pub general: ElasticLoggingConfig, + /// The Elasticsearch index to log to. + pub index: String, + /// The Elasticsearch type to use for logs. + pub document_type: String, + /// The name of the custom object id that the drain is for. + pub custom_id_key: String, + /// The custom id for the object that the drain is for. + pub custom_id_value: String, + /// The batching interval. + pub flush_interval: Duration, + /// Maximum retries in case of error. + pub max_retries: usize, +} + +/// An slog `Drain` for logging to Elasticsearch. +/// +/// Writes logs to Elasticsearch using the following format: +/// ```ignore +/// { +/// "_index": "subgraph-logs" +/// "_type": "log", +/// "_id": "Qmb31zcpzqga7ERaUTp83gVdYcuBasz4rXUHFufikFTJGU-2018-11-08T00:54:52.589258000Z", +/// "_source": { +/// "level": "debug", +/// "timestamp": "2018-11-08T00:54:52.589258000Z", +/// "subgraphId": "Qmb31zcpzqga7ERaUTp83gVdYcuBasz4rXUHFufikFTJGU", +/// "meta": { +/// "module": "graph_chain_ethereum::block_stream", +/// "line": 220, +/// "column": 9 +/// }, +/// "text": "Chain head pointer, number: 6661038, hash: 0xf089c457700a57798ced06bd3f18eef53bb8b46510bcefaf13615a8a26e4424a, component: BlockStream", +/// "id": "Qmb31zcpzqga7ERaUTp83gVdYcuBasz4rXUHFufikFTJGU-2018-11-08T00:54:52.589258000Z" +/// } +/// } +/// ``` +pub struct ElasticDrain { + config: ElasticDrainConfig, + error_logger: Logger, + logs: Arc>>, +} + +impl ElasticDrain { + /// Creates a new `ElasticDrain`. + pub fn new(config: ElasticDrainConfig, error_logger: Logger) -> Self { + let drain = ElasticDrain { + config, + error_logger, + logs: Arc::new(Mutex::new(vec![])), + }; + drain.periodically_flush_logs(); + drain + } + + fn periodically_flush_logs(&self) { + let flush_logger = self.error_logger.clone(); + let logs = self.logs.clone(); + let config = self.config.clone(); + let mut interval = tokio::time::interval(self.config.flush_interval); + let max_retries = self.config.max_retries; + + crate::task_spawn::spawn(async move { + loop { + interval.tick().await; + + let logs = logs.clone(); + let config = config.clone(); + let flush_logger = flush_logger.clone(); + let logs_to_send = { + let mut logs = logs.lock().unwrap(); + let logs_to_send = (*logs).clone(); + // Clear the logs, so the next batch can be recorded + logs.clear(); + logs_to_send + }; + + // Do nothing if there are no logs to flush + if logs_to_send.is_empty() { + continue; + } + + debug!( + flush_logger, + "Flushing {} logs to Elasticsearch", + logs_to_send.len() + ); + + // The Elasticsearch batch API takes requests with the following format: + // ```ignore + // action_and_meta_data\n + // optional_source\n + // action_and_meta_data\n + // optional_source\n + // ``` + // For more details, see: + // https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html + // + // We're assembly the request body in the same way below: + let batch_body = logs_to_send.iter().fold(String::from(""), |mut out, log| { + // Try to serialize the log itself to a JSON string + match serde_json::to_string(log) { + Ok(log_line) => { + // Serialize the action line to a string + let action_line = json!({ + "index": { + "_index": config.index, + "_type": config.document_type, + "_id": log.id, + } + }) + .to_string(); + + // Combine the two lines with newlines, make sure there is + // a newline at the end as well + out.push_str(format!("{}\n{}\n", action_line, log_line).as_str()); + } + Err(e) => { + error!( + flush_logger, + "Failed to serialize Elasticsearch log to JSON: {}", e + ); + } + }; + + out + }); + + // Build the batch API URL + let mut batch_url = reqwest::Url::parse(config.general.endpoint.as_str()) + .expect("invalid Elasticsearch URL"); + batch_url.set_path("_bulk"); + + // Send batch of logs to Elasticsearch + let header = match config.general.username { + Some(username) => config + .general + .client + .post(batch_url) + .header(CONTENT_TYPE, "application/json") + .basic_auth(username, config.general.password.clone()), + None => config + .general + .client + .post(batch_url) + .header(CONTENT_TYPE, "application/json"), + }; + + retry("send logs to elasticsearch", &flush_logger) + .limit(max_retries) + .timeout_secs(30) + .run(move || { + header + .try_clone() + .unwrap() // Unwrap: Request body not yet set + .body(batch_body.clone()) + .send() + .and_then(|response| async { response.error_for_status() }) + .map_ok(|_| ()) + }) + .await + .unwrap_or_else(|e| { + // Log if there was a problem sending the logs + error!(flush_logger, "Failed to send logs to Elasticsearch: {}", e); + }) + } + }); + } +} + +impl Drain for ElasticDrain { + type Ok = (); + type Err = (); + + fn log(&self, record: &Record, values: &OwnedKVList) -> Result { + // Don't sent `trace` logs to ElasticSearch. + if record.level() == Level::Trace { + return Ok(()); + } + let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let id = format!("{}-{}", self.config.custom_id_value, timestamp); + + // Serialize logger arguments + let mut serializer = SimpleKVSerializer::new(); + record + .kv() + .serialize(record, &mut serializer) + .expect("failed to serializer logger arguments"); + let (n_logger_kvs, logger_kvs) = serializer.finish(); + + // Serialize log message arguments + let mut serializer = SimpleKVSerializer::new(); + values + .serialize(record, &mut serializer) + .expect("failed to serialize log message arguments"); + let (n_value_kvs, value_kvs) = serializer.finish(); + + // Serialize log message arguments into hash map + let mut serializer = HashMapKVSerializer::new(); + record + .kv() + .serialize(record, &mut serializer) + .expect("failed to serialize log message arguments into hash map"); + let arguments = serializer.finish(); + + let mut text = format!("{}", record.msg()); + if n_logger_kvs > 0 { + write!(text, ", {}", logger_kvs).unwrap(); + } + if n_value_kvs > 0 { + write!(text, ", {}", value_kvs).unwrap(); + } + + // Prepare custom id for log document + let mut custom_id = HashMap::new(); + custom_id.insert( + self.config.custom_id_key.clone(), + self.config.custom_id_value.clone(), + ); + + // Prepare log document + let log = ElasticLog { + id, + custom_id, + arguments, + timestamp, + text, + level: record.level(), + meta: ElasticLogMeta { + module: record.module().into(), + line: record.line() as i64, + column: record.column() as i64, + }, + }; + + // Push the log into the queue + let mut logs = self.logs.lock().unwrap(); + logs.push(log); + + Ok(()) + } +} + +/// Creates a new asynchronous Elasticsearch logger. +/// +/// Uses `error_logger` to print any Elasticsearch logging errors, +/// so they don't go unnoticed. +pub fn elastic_logger(config: ElasticDrainConfig, error_logger: Logger) -> Logger { + let elastic_drain = ElasticDrain::new(config, error_logger).fuse(); + let async_drain = slog_async::Async::new(elastic_drain) + .chan_size(20000) + .build() + .fuse(); + Logger::root(async_drain, o!()) +} diff --git a/graph/src/log/factory.rs b/graph/src/log/factory.rs new file mode 100644 index 0000000..8565c56 --- /dev/null +++ b/graph/src/log/factory.rs @@ -0,0 +1,106 @@ +use slog::*; + +use crate::components::store::DeploymentLocator; +use crate::log::elastic::*; +use crate::log::split::*; +use crate::prelude::ENV_VARS; + +/// Configuration for component-specific logging to Elasticsearch. +pub struct ElasticComponentLoggerConfig { + pub index: String, +} + +/// Configuration for component-specific logging. +pub struct ComponentLoggerConfig { + pub elastic: Option, +} + +/// Factory for creating component and subgraph loggers. +#[derive(Clone)] +pub struct LoggerFactory { + parent: Logger, + elastic_config: Option, +} + +impl LoggerFactory { + /// Creates a new factory using a parent logger and optional Elasticsearch configuration. + pub fn new(logger: Logger, elastic_config: Option) -> Self { + Self { + parent: logger, + elastic_config, + } + } + + /// Creates a new factory with a new parent logger. + pub fn with_parent(&self, parent: Logger) -> Self { + Self { + parent, + elastic_config: self.elastic_config.clone(), + } + } + + /// Creates a component-specific logger with optional Elasticsearch support. + pub fn component_logger( + &self, + component: &str, + config: Option, + ) -> Logger { + let term_logger = self.parent.new(o!("component" => component.to_string())); + + match config { + None => term_logger, + Some(config) => match config.elastic { + None => term_logger, + Some(config) => self + .elastic_config + .clone() + .map(|elastic_config| { + split_logger( + term_logger.clone(), + elastic_logger( + ElasticDrainConfig { + general: elastic_config, + index: config.index, + document_type: String::from("log"), + custom_id_key: String::from("componentId"), + custom_id_value: component.to_string(), + flush_interval: ENV_VARS.elastic_search_flush_interval, + max_retries: ENV_VARS.elastic_search_max_retries, + }, + term_logger.clone(), + ), + ) + }) + .unwrap_or(term_logger), + }, + } + } + + /// Creates a subgraph logger with Elasticsearch support. + pub fn subgraph_logger(&self, loc: &DeploymentLocator) -> Logger { + let term_logger = self + .parent + .new(o!("subgraph_id" => loc.hash.to_string(), "sgd" => loc.id.to_string())); + + self.elastic_config + .clone() + .map(|elastic_config| { + split_logger( + term_logger.clone(), + elastic_logger( + ElasticDrainConfig { + general: elastic_config, + index: String::from("subgraph-logs"), + document_type: String::from("log"), + custom_id_key: String::from("subgraphId"), + custom_id_value: loc.hash.to_string(), + flush_interval: ENV_VARS.elastic_search_flush_interval, + max_retries: ENV_VARS.elastic_search_max_retries, + }, + term_logger.clone(), + ), + ) + }) + .unwrap_or(term_logger) + } +} diff --git a/graph/src/log/mod.rs b/graph/src/log/mod.rs new file mode 100644 index 0000000..6b28413 --- /dev/null +++ b/graph/src/log/mod.rs @@ -0,0 +1,383 @@ +#[macro_export] +macro_rules! impl_slog_value { + ($T:ty) => { + impl_slog_value!($T, "{}"); + }; + ($T:ty, $fmt:expr) => { + impl $crate::slog::Value for $T { + fn serialize( + &self, + record: &$crate::slog::Record, + key: $crate::slog::Key, + serializer: &mut dyn $crate::slog::Serializer, + ) -> $crate::slog::Result { + format!($fmt, self).serialize(record, key, serializer) + } + } + }; +} + +use isatty; +use slog::*; +use slog_async; +use slog_envlogger; +use slog_term::*; +use std::{fmt, io, result}; + +use crate::prelude::ENV_VARS; + +pub mod codes; +pub mod elastic; +pub mod factory; +pub mod split; + +pub fn logger(show_debug: bool) -> Logger { + let use_color = isatty::stdout_isatty(); + let decorator = slog_term::TermDecorator::new().build(); + let drain = CustomFormat::new(decorator, use_color).fuse(); + let drain = slog_envlogger::LogBuilder::new(drain) + .filter( + None, + if show_debug { + FilterLevel::Debug + } else { + FilterLevel::Info + }, + ) + .parse(ENV_VARS.log_levels.as_deref().unwrap_or("")) + .build(); + let drain = slog_async::Async::new(drain) + .chan_size(20000) + .build() + .fuse(); + Logger::root(drain, o!()) +} + +pub fn discard() -> Logger { + Logger::root(slog::Discard, o!()) +} + +pub struct CustomFormat +where + D: Decorator, +{ + decorator: D, + use_color: bool, +} + +impl Drain for CustomFormat +where + D: Decorator, +{ + type Ok = (); + type Err = io::Error; + + fn log(&self, record: &Record, values: &OwnedKVList) -> result::Result { + self.format_custom(record, values) + } +} + +impl CustomFormat +where + D: Decorator, +{ + pub fn new(decorator: D, use_color: bool) -> Self { + CustomFormat { + decorator, + use_color, + } + } + + fn format_custom(&self, record: &Record, values: &OwnedKVList) -> io::Result<()> { + self.decorator.with_record(record, values, |mut decorator| { + decorator.start_timestamp()?; + formatted_timestamp_local(&mut decorator)?; + decorator.start_whitespace()?; + write!(decorator, " ")?; + + decorator.start_level()?; + write!(decorator, "{}", record.level())?; + + decorator.start_whitespace()?; + write!(decorator, " ")?; + + decorator.start_msg()?; + write!(decorator, "{}", record.msg())?; + + // Collect key values from the record + let mut serializer = KeyValueSerializer::new(); + record.kv().serialize(record, &mut serializer)?; + let body_kvs = serializer.finish(); + + // Collect subgraph ID, components and extra key values from the record + let mut serializer = HeaderSerializer::new(); + values.serialize(record, &mut serializer)?; + let (subgraph_id, components, header_kvs) = serializer.finish(); + + // Regular key values first + for (k, v) in body_kvs.iter().chain(header_kvs.iter()) { + decorator.start_comma()?; + write!(decorator, ", ")?; + + decorator.start_key()?; + write!(decorator, "{}", k)?; + + decorator.start_separator()?; + write!(decorator, ": ")?; + + decorator.start_value()?; + write!(decorator, "{}", v)?; + } + + // Then log the subgraph ID (if present) + if let Some(subgraph_id) = subgraph_id.as_ref() { + decorator.start_comma()?; + write!(decorator, ", ")?; + decorator.start_key()?; + write!(decorator, "subgraph_id")?; + decorator.start_separator()?; + write!(decorator, ": ")?; + decorator.start_value()?; + if self.use_color { + write!(decorator, "\u{001b}[35m{}\u{001b}[0m", subgraph_id)?; + } else { + write!(decorator, "{}", subgraph_id)?; + } + } + + // Then log the component hierarchy + if !components.is_empty() { + decorator.start_comma()?; + write!(decorator, ", ")?; + decorator.start_key()?; + write!(decorator, "component")?; + decorator.start_separator()?; + write!(decorator, ": ")?; + decorator.start_value()?; + if self.use_color { + write!( + decorator, + "\u{001b}[36m{}\u{001b}[0m", + components.join(" > ") + )?; + } else { + write!(decorator, "{}", components.join(" > "))?; + } + } + + write!(decorator, "\n")?; + decorator.flush()?; + + Ok(()) + }) + } +} + +struct HeaderSerializer { + subgraph_id: Option, + components: Vec, + kvs: Vec<(String, String)>, +} + +impl HeaderSerializer { + pub fn new() -> Self { + Self { + subgraph_id: None, + components: vec![], + kvs: vec![], + } + } + + pub fn finish(mut self) -> (Option, Vec, Vec<(String, String)>) { + // Reverse components so the parent components come first + self.components.reverse(); + + (self.subgraph_id, self.components, self.kvs) + } +} + +macro_rules! s( + ($s:expr, $k:expr, $v:expr) => { + Ok(match $k { + "component" => $s.components.push(format!("{}", $v)), + "subgraph_id" => $s.subgraph_id = Some(format!("{}", $v)), + _ => $s.kvs.push(($k.into(), format!("{}", $v))), + }) + }; +); + +impl ser::Serializer for HeaderSerializer { + fn emit_none(&mut self, key: Key) -> slog::Result { + s!(self, key, "None") + } + + fn emit_unit(&mut self, key: Key) -> slog::Result { + s!(self, key, "()") + } + + fn emit_bool(&mut self, key: Key, val: bool) -> slog::Result { + s!(self, key, val) + } + + fn emit_char(&mut self, key: Key, val: char) -> slog::Result { + s!(self, key, val) + } + + fn emit_usize(&mut self, key: Key, val: usize) -> slog::Result { + s!(self, key, val) + } + + fn emit_isize(&mut self, key: Key, val: isize) -> slog::Result { + s!(self, key, val) + } + + fn emit_u8(&mut self, key: Key, val: u8) -> slog::Result { + s!(self, key, val) + } + + fn emit_i8(&mut self, key: Key, val: i8) -> slog::Result { + s!(self, key, val) + } + + fn emit_u16(&mut self, key: Key, val: u16) -> slog::Result { + s!(self, key, val) + } + + fn emit_i16(&mut self, key: Key, val: i16) -> slog::Result { + s!(self, key, val) + } + + fn emit_u32(&mut self, key: Key, val: u32) -> slog::Result { + s!(self, key, val) + } + + fn emit_i32(&mut self, key: Key, val: i32) -> slog::Result { + s!(self, key, val) + } + + fn emit_f32(&mut self, key: Key, val: f32) -> slog::Result { + s!(self, key, val) + } + + fn emit_u64(&mut self, key: Key, val: u64) -> slog::Result { + s!(self, key, val) + } + + fn emit_i64(&mut self, key: Key, val: i64) -> slog::Result { + s!(self, key, val) + } + + fn emit_f64(&mut self, key: Key, val: f64) -> slog::Result { + s!(self, key, val) + } + + fn emit_str(&mut self, key: Key, val: &str) -> slog::Result { + s!(self, key, val) + } + + fn emit_arguments(&mut self, key: Key, val: &fmt::Arguments) -> slog::Result { + s!(self, key, val) + } +} + +struct KeyValueSerializer { + kvs: Vec<(String, String)>, +} + +impl KeyValueSerializer { + pub fn new() -> Self { + Self { kvs: vec![] } + } + + pub fn finish(self) -> Vec<(String, String)> { + self.kvs + } +} + +macro_rules! s( + ($s:expr, $k:expr, $v:expr) => { + Ok($s.kvs.push(($k.into(), format!("{}", $v)))) + }; +); + +impl ser::Serializer for KeyValueSerializer { + fn emit_none(&mut self, key: Key) -> slog::Result { + s!(self, key, "None") + } + + fn emit_unit(&mut self, key: Key) -> slog::Result { + s!(self, key, "()") + } + + fn emit_bool(&mut self, key: Key, val: bool) -> slog::Result { + s!(self, key, val) + } + + fn emit_char(&mut self, key: Key, val: char) -> slog::Result { + s!(self, key, val) + } + + fn emit_usize(&mut self, key: Key, val: usize) -> slog::Result { + s!(self, key, val) + } + + fn emit_isize(&mut self, key: Key, val: isize) -> slog::Result { + s!(self, key, val) + } + + fn emit_u8(&mut self, key: Key, val: u8) -> slog::Result { + s!(self, key, val) + } + + fn emit_i8(&mut self, key: Key, val: i8) -> slog::Result { + s!(self, key, val) + } + + fn emit_u16(&mut self, key: Key, val: u16) -> slog::Result { + s!(self, key, val) + } + + fn emit_i16(&mut self, key: Key, val: i16) -> slog::Result { + s!(self, key, val) + } + + fn emit_u32(&mut self, key: Key, val: u32) -> slog::Result { + s!(self, key, val) + } + + fn emit_i32(&mut self, key: Key, val: i32) -> slog::Result { + s!(self, key, val) + } + + fn emit_f32(&mut self, key: Key, val: f32) -> slog::Result { + s!(self, key, val) + } + + fn emit_u64(&mut self, key: Key, val: u64) -> slog::Result { + s!(self, key, val) + } + + fn emit_i64(&mut self, key: Key, val: i64) -> slog::Result { + s!(self, key, val) + } + + fn emit_f64(&mut self, key: Key, val: f64) -> slog::Result { + s!(self, key, val) + } + + fn emit_str(&mut self, key: Key, val: &str) -> slog::Result { + s!(self, key, val) + } + + fn emit_arguments(&mut self, key: Key, val: &fmt::Arguments) -> slog::Result { + s!(self, key, val) + } +} + +fn formatted_timestamp_local(io: &mut impl io::Write) -> io::Result<()> { + write!( + io, + "{}", + chrono::Local::now().format(ENV_VARS.log_time_format.as_str()) + ) +} diff --git a/graph/src/log/split.rs b/graph/src/log/split.rs new file mode 100644 index 0000000..8e6711d --- /dev/null +++ b/graph/src/log/split.rs @@ -0,0 +1,74 @@ +use std::fmt::Debug; +use std::result::Result as StdResult; + +use slog::*; +use slog_async; + +/// An error that could come from either of two slog `Drain`s. +#[derive(Debug)] +enum SplitDrainError +where + D1: Drain, + D2: Drain, +{ + Drain1Error(D1::Err), + Drain2Error(D2::Err), +} + +/// An slog `Drain` that forwards log messages to two other drains. +struct SplitDrain +where + D1: Drain, + D2: Drain, +{ + drain1: D1, + drain2: D2, +} + +impl SplitDrain +where + D1: Drain, + D2: Drain, +{ + /// Creates a new split drain that forwards to the two provided drains. + fn new(drain1: D1, drain2: D2) -> Self { + SplitDrain { drain1, drain2 } + } +} + +impl Drain for SplitDrain +where + D1: Drain, + D2: Drain, +{ + type Ok = (); + type Err = SplitDrainError; + + fn log(&self, record: &Record, values: &OwnedKVList) -> StdResult { + self.drain1 + .log(record, values) + .map_err(SplitDrainError::Drain1Error) + .and( + self.drain2 + .log(record, values) + .map(|_| ()) + .map_err(SplitDrainError::Drain2Error), + ) + } +} + +/// Creates an async slog logger that writes to two drains in parallel. +pub fn split_logger(drain1: D1, drain2: D2) -> Logger +where + D1: Drain + Send + Debug + 'static, + D2: Drain + Send + Debug + 'static, + D1::Err: Debug, + D2::Err: Debug, +{ + let split_drain = SplitDrain::new(drain1.fuse(), drain2.fuse()).fuse(); + let async_drain = slog_async::Async::new(split_drain) + .chan_size(20000) + .build() + .fuse(); + Logger::root(async_drain, o!()) +} diff --git a/graph/src/runtime/asc_heap.rs b/graph/src/runtime/asc_heap.rs new file mode 100644 index 0000000..c391654 --- /dev/null +++ b/graph/src/runtime/asc_heap.rs @@ -0,0 +1,144 @@ +use std::mem::MaybeUninit; + +use semver::Version; + +use super::{ + gas::GasCounter, AscIndexId, AscPtr, AscType, DeterministicHostError, IndexForAscTypeId, +}; +/// A type that can read and write to the Asc heap. Call `asc_new` and `asc_get` +/// for reading and writing Rust structs from and to Asc. +/// +/// The implementor must provide the direct Asc interface with `raw_new` and `get`. +pub trait AscHeap { + /// Allocate new space and write `bytes`, return the allocated address. + fn raw_new(&mut self, bytes: &[u8], gas: &GasCounter) -> Result; + + fn read<'a>( + &self, + offset: u32, + buffer: &'a mut [MaybeUninit], + gas: &GasCounter, + ) -> Result<&'a mut [u8], DeterministicHostError>; + + fn read_u32(&self, offset: u32, gas: &GasCounter) -> Result; + + fn api_version(&self) -> Version; + + fn asc_type_id( + &mut self, + type_id_index: IndexForAscTypeId, + ) -> Result; +} + +/// Instantiate `rust_obj` as an Asc object of class `C`. +/// Returns a pointer to the Asc heap. +/// +/// This operation is expensive as it requires a call to `raw_new` for every +/// nested object. +pub fn asc_new( + heap: &mut H, + rust_obj: &T, + gas: &GasCounter, +) -> Result, DeterministicHostError> +where + C: AscType + AscIndexId, + T: ToAscObj, +{ + let obj = rust_obj.to_asc_obj(heap, gas)?; + AscPtr::alloc_obj(obj, heap, gas) +} + +/// Map an optional object to its Asc equivalent if Some, otherwise return a missing field error. +pub fn asc_new_or_missing( + heap: &mut H, + object: &Option, + gas: &GasCounter, + type_name: &str, + field_name: &str, +) -> Result, DeterministicHostError> +where + H: AscHeap + ?Sized, + O: ToAscObj, + A: AscType + AscIndexId, +{ + match object { + Some(o) => asc_new(heap, o, gas), + None => Err(missing_field_error(type_name, field_name)), + } +} + +/// Map an optional object to its Asc equivalent if Some, otherwise return null. +pub fn asc_new_or_null( + heap: &mut H, + object: &Option, + gas: &GasCounter, +) -> Result, DeterministicHostError> +where + H: AscHeap + ?Sized, + O: ToAscObj, + A: AscType + AscIndexId, +{ + match object { + Some(o) => asc_new(heap, o, gas), + None => Ok(AscPtr::null()), + } +} + +/// Create an error for a missing field in a type. +fn missing_field_error(type_name: &str, field_name: &str) -> DeterministicHostError { + DeterministicHostError::from(anyhow::anyhow!("{} missing {}", type_name, field_name)) +} + +/// Read the rust representation of an Asc object of class `C`. +/// +/// This operation is expensive as it requires a call to `get` for every +/// nested object. +pub fn asc_get( + heap: &H, + asc_ptr: AscPtr, + gas: &GasCounter, +) -> Result +where + C: AscType + AscIndexId, + T: FromAscObj, +{ + T::from_asc_obj(asc_ptr.read_ptr(heap, gas)?, heap, gas) +} + +/// Type that can be converted to an Asc object of class `C`. +pub trait ToAscObj { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result; +} + +impl> ToAscObj for &T { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + (*self).to_asc_obj(heap, gas) + } +} + +impl ToAscObj for bool { + fn to_asc_obj( + &self, + _heap: &mut H, + _gas: &GasCounter, + ) -> Result { + Ok(*self) + } +} + +/// Type that can be converted from an Asc object of class `C`. +pub trait FromAscObj: Sized { + fn from_asc_obj( + obj: C, + heap: &H, + gas: &GasCounter, + ) -> Result; +} diff --git a/graph/src/runtime/asc_ptr.rs b/graph/src/runtime/asc_ptr.rs new file mode 100644 index 0000000..aa7c1c6 --- /dev/null +++ b/graph/src/runtime/asc_ptr.rs @@ -0,0 +1,245 @@ +use super::gas::GasCounter; +use super::{padding_to_16, DeterministicHostError}; + +use super::{AscHeap, AscIndexId, AscType, IndexForAscTypeId}; +use semver::Version; +use std::fmt; +use std::marker::PhantomData; +use std::mem::MaybeUninit; + +/// The `rt_size` field contained in an AssemblyScript header has a size of 4 bytes. +const SIZE_OF_RT_SIZE: u32 = 4; + +/// A pointer to an object in the Asc heap. +pub struct AscPtr(u32, PhantomData); + +impl Copy for AscPtr {} + +impl Clone for AscPtr { + fn clone(&self) -> Self { + AscPtr(self.0, PhantomData) + } +} + +impl Default for AscPtr { + fn default() -> Self { + AscPtr(0, PhantomData) + } +} + +impl fmt::Debug for AscPtr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl AscPtr { + /// A raw pointer to be passed to Wasm. + pub fn wasm_ptr(self) -> u32 { + self.0 + } + + #[inline(always)] + pub fn new(heap_ptr: u32) -> Self { + Self(heap_ptr, PhantomData) + } +} + +impl AscPtr { + /// Create a pointer that is equivalent to AssemblyScript's `null`. + #[inline(always)] + pub fn null() -> Self { + AscPtr::new(0) + } + + /// Read from `self` into the Rust struct `C`. + pub fn read_ptr( + self, + heap: &H, + gas: &GasCounter, + ) -> Result { + let len = match heap.api_version() { + // TODO: The version check here conflicts with the comment on C::asc_size, + // which states "Only used for version <= 0.0.3." + version if version <= Version::new(0, 0, 4) => C::asc_size(self, heap, gas), + _ => self.read_len(heap, gas), + }?; + + let using_buffer = |buffer: &mut [MaybeUninit]| { + let buffer = heap.read(self.0, buffer, gas)?; + C::from_asc_bytes(buffer, &heap.api_version()) + }; + + let len = len as usize; + + if len <= 32 { + let mut buffer = [MaybeUninit::::uninit(); 32]; + using_buffer(&mut buffer[..len]) + } else { + let mut buffer = Vec::with_capacity(len); + using_buffer(buffer.spare_capacity_mut()) + } + } + + /// Allocate `asc_obj` as an Asc object of class `C`. + pub fn alloc_obj( + asc_obj: C, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> + where + C: AscIndexId, + { + match heap.api_version() { + version if version <= Version::new(0, 0, 4) => { + let heap_ptr = heap.raw_new(&asc_obj.to_asc_bytes()?, gas)?; + Ok(AscPtr::new(heap_ptr)) + } + _ => { + let mut bytes = asc_obj.to_asc_bytes()?; + + let aligned_len = padding_to_16(bytes.len()); + // Since AssemblyScript keeps all allocated objects with a 16 byte alignment, + // we need to do the same when we allocate ourselves. + bytes.extend(std::iter::repeat(0).take(aligned_len)); + + let header = Self::generate_header( + heap, + C::INDEX_ASC_TYPE_ID, + asc_obj.content_len(&bytes), + bytes.len(), + )?; + let header_len = header.len() as u32; + + let heap_ptr = heap.raw_new(&[header, bytes].concat(), gas)?; + + // Use header length as offset. so the AscPtr points directly at the content. + Ok(AscPtr::new(heap_ptr + header_len)) + } + } + } + + /// Helper used by arrays and strings to read their length. + /// Only used for version <= 0.0.4. + pub fn read_u32( + &self, + heap: &H, + gas: &GasCounter, + ) -> Result { + // Read the bytes pointed to by `self` as the bytes of a `u32`. + heap.read_u32(self.0, gas) + } + + /// Helper that generates an AssemblyScript header. + /// An AssemblyScript header has 20 bytes and it is composed of 5 values. + /// - mm_info: usize -> size of all header contents + payload contents + padding + /// - gc_info: usize -> first GC info (we don't free memory so it's irrelevant) + /// - gc_info2: usize -> second GC info (we don't free memory so it's irrelevant) + /// - rt_id: u32 -> identifier for the class being allocated + /// - rt_size: u32 -> content size + /// Only used for version >= 0.0.5. + fn generate_header( + heap: &mut H, + type_id_index: IndexForAscTypeId, + content_length: usize, + full_length: usize, + ) -> Result, DeterministicHostError> { + let mut header: Vec = Vec::with_capacity(20); + + let gc_info: [u8; 4] = (0u32).to_le_bytes(); + let gc_info2: [u8; 4] = (0u32).to_le_bytes(); + let asc_type_id = heap.asc_type_id(type_id_index)?; + let rt_id: [u8; 4] = asc_type_id.to_le_bytes(); + let rt_size: [u8; 4] = (content_length as u32).to_le_bytes(); + + let mm_info: [u8; 4] = + ((gc_info.len() + gc_info2.len() + rt_id.len() + rt_size.len() + full_length) as u32) + .to_le_bytes(); + + header.extend(&mm_info); + header.extend(&gc_info); + header.extend(&gc_info2); + header.extend(&rt_id); + header.extend(&rt_size); + + Ok(header) + } + + /// Helper to read the length from the header. + /// An AssemblyScript header has 20 bytes, and it's right before the content, and composed by: + /// - mm_info: usize + /// - gc_info: usize + /// - gc_info2: usize + /// - rt_id: u32 + /// - rt_size: u32 + /// This function returns the `rt_size`. + /// Only used for version >= 0.0.5. + pub fn read_len( + &self, + heap: &H, + gas: &GasCounter, + ) -> Result { + // We're trying to read the pointer below, we should check it's + // not null before using it. + self.check_is_not_null()?; + + let start_of_rt_size = self.0.checked_sub(SIZE_OF_RT_SIZE).ok_or_else(|| { + DeterministicHostError::from(anyhow::anyhow!( + "Subtract overflow on pointer: {}", + self.0 + )) + })?; + + heap.read_u32(start_of_rt_size, gas) + } + + /// Conversion to `u64` for use with `AscEnum`. + pub fn to_payload(&self) -> u64 { + self.0 as u64 + } + + /// We typically assume `AscPtr` is never null, but for types such as `string | null` it can be. + pub fn is_null(&self) -> bool { + self.0 == 0 + } + + /// There's no problem in an AscPtr being 'null' (see above AscPtr::is_null function). + /// However if one tries to read that pointer, it should fail with a helpful error message, + /// this function does this error handling. + /// + /// Summary: ALWAYS call this before reading an AscPtr. + pub fn check_is_not_null(&self) -> Result<(), DeterministicHostError> { + if self.is_null() { + return Err(DeterministicHostError::from(anyhow::anyhow!( + "Tried to read AssemblyScript value that is 'null'. Suggestion: look into the function that the error happened and add 'log' calls till you find where a 'null' value is being used as non-nullable. It's likely that you're calling a 'graph-ts' function (or operator) with a 'null' value when it doesn't support it." + ))); + } + + Ok(()) + } + + // Erase type information. + pub fn erase(self) -> AscPtr<()> { + AscPtr::new(self.0) + } +} + +impl From for AscPtr { + fn from(ptr: u32) -> Self { + AscPtr::new(ptr) + } +} + +impl AscType for AscPtr { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + let bytes = u32::from_asc_bytes(asc_obj, api_version)?; + Ok(AscPtr::new(bytes)) + } +} diff --git a/graph/src/runtime/gas/combinators.rs b/graph/src/runtime/gas/combinators.rs new file mode 100644 index 0000000..a6bc379 --- /dev/null +++ b/graph/src/runtime/gas/combinators.rs @@ -0,0 +1,136 @@ +use super::{Gas, GasSizeOf}; +use std::cmp::{max, min}; + +pub mod complexity { + use super::*; + + // Args have additive linear complexity + // Eg: O(N₁+N₂) + pub struct Linear; + // Args have multiplicative complexity + // Eg: O(N₁*N₂) + pub struct Mul; + + // Exponential complexity. + // Eg: O(N₁^N₂) + pub struct Exponential; + + // There is only one arg and it scales linearly with it's size + pub struct Size; + + // Complexity is captured by the lesser complexity of the two args + // Eg: O(min(N₁, N₂)) + pub struct Min; + + // Complexity is captured by the greater complexity of the two args + // Eg: O(max(N₁, N₂)) + pub struct Max; + + impl GasCombinator for Linear { + #[inline(always)] + fn combine(lhs: Gas, rhs: Gas) -> Gas { + lhs + rhs + } + } + + impl GasCombinator for Mul { + #[inline(always)] + fn combine(lhs: Gas, rhs: Gas) -> Gas { + Gas(lhs.0.saturating_mul(rhs.0)) + } + } + + impl GasCombinator for Min { + #[inline(always)] + fn combine(lhs: Gas, rhs: Gas) -> Gas { + min(lhs, rhs) + } + } + + impl GasCombinator for Max { + #[inline(always)] + fn combine(lhs: Gas, rhs: Gas) -> Gas { + max(lhs, rhs) + } + } + + impl GasSizeOf for Combine + where + T: GasSizeOf, + { + fn gas_size_of(&self) -> Gas { + self.0.gas_size_of() + } + } +} + +pub struct Combine(pub Tuple, pub Combinator); + +pub trait GasCombinator { + fn combine(lhs: Gas, rhs: Gas) -> Gas; +} + +impl GasSizeOf for Combine<(T0, T1), C> +where + T0: GasSizeOf, + T1: GasSizeOf, + C: GasCombinator, +{ + fn gas_size_of(&self) -> Gas { + let (a, b) = &self.0; + C::combine(a.gas_size_of(), b.gas_size_of()) + } + + #[inline] + fn const_gas_size_of() -> Option { + if let Some(t0) = T0::const_gas_size_of() { + if let Some(t1) = T1::const_gas_size_of() { + return Some(C::combine(t0, t1)); + } + } + None + } +} + +impl GasSizeOf for Combine<(T0, T1, T2), C> +where + T0: GasSizeOf, + T1: GasSizeOf, + T2: GasSizeOf, + C: GasCombinator, +{ + fn gas_size_of(&self) -> Gas { + let (a, b, c) = &self.0; + C::combine( + C::combine(a.gas_size_of(), b.gas_size_of()), + c.gas_size_of(), + ) + } + + #[inline] // Const propagation to the rescue. I hope. + fn const_gas_size_of() -> Option { + if let Some(t0) = T0::const_gas_size_of() { + if let Some(t1) = T1::const_gas_size_of() { + if let Some(t2) = T2::const_gas_size_of() { + return Some(C::combine(C::combine(t0, t1), t2)); + } + } + } + None + } +} + +impl GasSizeOf for Combine<(T0, u8), complexity::Exponential> +where + T0: GasSizeOf, +{ + fn gas_size_of(&self) -> Gas { + let (a, b) = &self.0; + Gas(a.gas_size_of().0.saturating_pow(*b as u32)) + } + + #[inline] + fn const_gas_size_of() -> Option { + None + } +} diff --git a/graph/src/runtime/gas/costs.rs b/graph/src/runtime/gas/costs.rs new file mode 100644 index 0000000..a4593a0 --- /dev/null +++ b/graph/src/runtime/gas/costs.rs @@ -0,0 +1,76 @@ +//! Stores all the gas costs is one place so they can be compared easily. +//! Determinism: Once deployed, none of these values can be changed without a version upgrade. + +use super::*; + +/// Using 10 gas = ~1ns for WASM instructions. +const GAS_PER_SECOND: u64 = 10_000_000_000; + +/// Set max gas to 1000 seconds worth of gas per handler. The intent here is to have the determinism +/// cutoff be very high, while still allowing more reasonable timer based cutoffs. Having a unit +/// like 10 gas for ~1ns allows us to be granular in instructions which are aggregated into metered +/// blocks via https://docs.rs/pwasm-utils/0.16.0/pwasm_utils/fn.inject_gas_counter.html But we can +/// still charge very high numbers for other things. +pub const CONST_MAX_GAS_PER_HANDLER: u64 = 1000 * GAS_PER_SECOND; + +/// Gas for instructions are aggregated into blocks, so hopefully gas calls each have relatively +/// large gas. But in the case they don't, we don't want the overhead of calling out into a host +/// export to be the dominant cost that causes unexpectedly high execution times. +/// +/// This value is based on the benchmark of an empty infinite loop, which does basically nothing +/// other than call the gas function. The benchmark result was closer to 5000 gas but use 10_000 to +/// be conservative. +pub const HOST_EXPORT_GAS: Gas = Gas(10_000); + +/// As a heuristic for the cost of host fns it makes sense to reason in terms of bandwidth and +/// calculate the cost from there. Because we don't have benchmarks for each host fn, we go with +/// pessimistic assumption of performance of 10 MB/s, which nonetheless allows for 10 GB to be +/// processed through host exports by a single handler at a 1000 seconds budget. +const DEFAULT_BYTE_PER_SECOND: u64 = 10_000_000; + +/// With the current parameters DEFAULT_GAS_PER_BYTE = 1_000. +const DEFAULT_GAS_PER_BYTE: u64 = GAS_PER_SECOND / DEFAULT_BYTE_PER_SECOND; + +/// Base gas cost for calling any host export. +/// Security: This must be non-zero. +pub const DEFAULT_BASE_COST: u64 = 100_000; + +pub const DEFAULT_GAS_OP: GasOp = GasOp { + base_cost: DEFAULT_BASE_COST, + size_mult: DEFAULT_GAS_PER_BYTE, +}; + +/// Because big math has a multiplicative complexity, that can result in high sizes, so assume a +/// bandwidth of 100 MB/s, faster than the default. +const BIG_MATH_BYTE_PER_SECOND: u64 = 100_000_000; +const BIG_MATH_GAS_PER_BYTE: u64 = GAS_PER_SECOND / BIG_MATH_BYTE_PER_SECOND; + +pub const BIG_MATH_GAS_OP: GasOp = GasOp { + base_cost: DEFAULT_BASE_COST, + size_mult: BIG_MATH_GAS_PER_BYTE, +}; + +// Allow up to 100,000 data sources to be created +pub const CREATE_DATA_SOURCE: Gas = Gas(CONST_MAX_GAS_PER_HANDLER / 100_000); + +pub const LOG_OP: GasOp = GasOp { + // Allow up to 100,000 logs + base_cost: CONST_MAX_GAS_PER_HANDLER / 100_000, + size_mult: DEFAULT_GAS_PER_BYTE, +}; + +// Saving to the store is one of the most expensive operations. +pub const STORE_SET: GasOp = GasOp { + // Allow up to 250k entities saved. + base_cost: CONST_MAX_GAS_PER_HANDLER / 250_000, + // If the size roughly corresponds to bytes, allow 1GB to be saved. + size_mult: CONST_MAX_GAS_PER_HANDLER / 1_000_000_000, +}; + +// Reading from the store is much cheaper than writing. +pub const STORE_GET: GasOp = GasOp { + base_cost: CONST_MAX_GAS_PER_HANDLER / 10_000_000, + size_mult: CONST_MAX_GAS_PER_HANDLER / 10_000_000_000, +}; + +pub const STORE_REMOVE: GasOp = STORE_SET; diff --git a/graph/src/runtime/gas/mod.rs b/graph/src/runtime/gas/mod.rs new file mode 100644 index 0000000..7d9bb38 --- /dev/null +++ b/graph/src/runtime/gas/mod.rs @@ -0,0 +1,110 @@ +mod combinators; +mod costs; +mod ops; +mod saturating; +mod size_of; +use crate::prelude::{CheapClone, ENV_VARS}; +use crate::runtime::DeterministicHostError; +pub use combinators::*; +pub use costs::DEFAULT_BASE_COST; +pub use costs::*; +pub use saturating::*; + +use std::sync::atomic::{AtomicU64, Ordering::SeqCst}; +use std::sync::Arc; +use std::{fmt, fmt::Display}; + +pub struct GasOp { + base_cost: u64, + size_mult: u64, +} + +impl GasOp { + pub fn with_args(&self, c: C, args: T) -> Gas + where + Combine: GasSizeOf, + { + Gas(self.base_cost) + Combine(args, c).gas_size_of() * self.size_mult + } +} + +/// Sort of a base unit for gas operations. For example, if one is operating +/// on a BigDecimal one might like to know how large that BigDecimal is compared +/// to other BigDecimals so that one could to (MultCost * gas_size_of(big_decimal)) +/// and re-use that logic for (WriteToDBCost or ReadFromDBCost) rather than having +/// one-offs for each use-case. +/// This is conceptually much like CacheWeight, but has some key differences. +/// First, this needs to be stable - like StableHash (same independent of +/// platform/compiler/run). Also this can be somewhat context dependent. An example +/// of context dependent costs might be if a value is being hex encoded or binary encoded +/// when serializing. +/// +/// Either implement gas_size_of or const_gas_size_of but never none or both. +pub trait GasSizeOf { + #[inline(always)] + fn gas_size_of(&self) -> Gas { + Self::const_gas_size_of().expect("GasSizeOf unimplemented") + } + /// Some when every member of the type has the same gas size. + #[inline(always)] + fn const_gas_size_of() -> Option { + None + } +} + +/// This wrapper ensures saturating arithmetic is used +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)] +pub struct Gas(u64); + +impl Gas { + pub const ZERO: Gas = Gas(0); + + pub const fn new(gas: u64) -> Self { + Gas(gas) + } + + #[cfg(debug_assertions)] + pub const fn value(&self) -> u64 { + self.0 + } +} + +impl Display for Gas { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + self.0.fmt(f) + } +} + +#[derive(Clone, Default)] +pub struct GasCounter(Arc); + +impl CheapClone for GasCounter {} + +impl GasCounter { + /// Alias of [`Default::default`]. + pub fn new() -> Self { + Self::default() + } + + /// This should be called once per host export + pub fn consume_host_fn(&self, mut amount: Gas) -> Result<(), DeterministicHostError> { + amount += costs::HOST_EXPORT_GAS; + let old = self + .0 + .fetch_update(SeqCst, SeqCst, |v| Some(v.saturating_add(amount.0))) + .unwrap(); + let new = old.saturating_add(amount.0); + if new >= ENV_VARS.max_gas_per_handler { + Err(DeterministicHostError::gas(anyhow::anyhow!( + "Gas limit exceeded. Used: {}", + new + ))) + } else { + Ok(()) + } + } + + pub fn get(&self) -> Gas { + Gas(self.0.load(SeqCst)) + } +} diff --git a/graph/src/runtime/gas/ops.rs b/graph/src/runtime/gas/ops.rs new file mode 100644 index 0000000..a7e5987 --- /dev/null +++ b/graph/src/runtime/gas/ops.rs @@ -0,0 +1,54 @@ +//! All the operators go here +//! Gas operations are all saturating and additive (never trending toward zero) + +use super::{Gas, SaturatingInto as _}; +use std::iter::Sum; +use std::ops::{Add, AddAssign, Mul, MulAssign}; + +impl Add for Gas { + type Output = Gas; + #[inline] + fn add(self, rhs: Gas) -> Self::Output { + Gas(self.0.saturating_add(rhs.0)) + } +} + +impl Mul for Gas { + type Output = Gas; + #[inline] + fn mul(self, rhs: u64) -> Self::Output { + Gas(self.0.saturating_mul(rhs)) + } +} + +impl Mul for Gas { + type Output = Gas; + #[inline] + fn mul(self, rhs: usize) -> Self::Output { + Gas(self.0.saturating_mul(rhs.saturating_into())) + } +} + +impl MulAssign for Gas { + #[inline] + fn mul_assign(&mut self, rhs: u64) { + self.0 = self.0.saturating_add(rhs); + } +} + +impl AddAssign for Gas { + #[inline] + fn add_assign(&mut self, rhs: Gas) { + self.0 = self.0.saturating_add(rhs.0); + } +} + +impl Sum for Gas { + fn sum>(iter: I) -> Self { + let mut sum = Gas::ZERO; + for elem in iter { + sum += elem; + } + sum + } +} diff --git a/graph/src/runtime/gas/saturating.rs b/graph/src/runtime/gas/saturating.rs new file mode 100644 index 0000000..de2a477 --- /dev/null +++ b/graph/src/runtime/gas/saturating.rs @@ -0,0 +1,51 @@ +use super::Gas; +use std::convert::TryInto as _; + +pub trait SaturatingFrom { + fn saturating_from(value: T) -> Self; +} + +// It would be good to put this trait into a new or existing crate. +// Tried conv but the owner seems to be away +// https://github.com/DanielKeep/rust-conv/issues/15 +pub trait SaturatingInto { + fn saturating_into(self) -> T; +} + +impl SaturatingInto for I +where + F: SaturatingFrom, +{ + #[inline(always)] + fn saturating_into(self) -> F { + F::saturating_from(self) + } +} + +impl SaturatingFrom for Gas { + #[inline] + fn saturating_from(value: usize) -> Gas { + Gas(value.try_into().unwrap_or(u64::MAX)) + } +} + +impl SaturatingFrom for u64 { + #[inline] + fn saturating_from(value: usize) -> Self { + value.try_into().unwrap_or(u64::MAX) + } +} + +impl SaturatingFrom for Gas { + #[inline] + fn saturating_from(value: f64) -> Self { + Gas(value as u64) + } +} + +impl SaturatingFrom for Gas { + #[inline] + fn saturating_from(value: u32) -> Self { + Gas(value as u64) + } +} diff --git a/graph/src/runtime/gas/size_of.rs b/graph/src/runtime/gas/size_of.rs new file mode 100644 index 0000000..49bb60b --- /dev/null +++ b/graph/src/runtime/gas/size_of.rs @@ -0,0 +1,175 @@ +//! Various implementations of GasSizeOf; + +use crate::{ + components::store::{EntityKey, EntityType}, + data::store::{scalar::Bytes, Value}, + prelude::{BigDecimal, BigInt}, +}; + +use super::{Gas, GasSizeOf, SaturatingInto as _}; + +impl GasSizeOf for Value { + fn gas_size_of(&self) -> Gas { + let inner = match self { + Value::BigDecimal(big_decimal) => big_decimal.gas_size_of(), + Value::String(string) => string.gas_size_of(), + Value::Null => Gas(1), + Value::List(list) => list.gas_size_of(), + Value::Int(int) => int.gas_size_of(), + Value::Bytes(bytes) => bytes.gas_size_of(), + Value::Bool(bool) => bool.gas_size_of(), + Value::BigInt(big_int) => big_int.gas_size_of(), + }; + Gas(4) + inner + } +} + +impl GasSizeOf for Bytes { + fn gas_size_of(&self) -> Gas { + (&self[..]).gas_size_of() + } +} + +impl GasSizeOf for bool { + #[inline(always)] + fn const_gas_size_of() -> Option { + Some(Gas(1)) + } +} + +impl GasSizeOf for Option +where + T: GasSizeOf, +{ + fn gas_size_of(&self) -> Gas { + if let Some(v) = self { + Gas(1) + v.gas_size_of() + } else { + Gas(1) + } + } +} + +impl GasSizeOf for BigInt { + fn gas_size_of(&self) -> Gas { + // Add one to always have an upper bound on the number of bytes required to represent the + // number, and so that `0` has a size of 1. + let n_bytes = self.bits() / 8 + 1; + n_bytes.saturating_into() + } +} + +impl GasSizeOf for BigDecimal { + fn gas_size_of(&self) -> Gas { + let (int, _) = self.as_bigint_and_exponent(); + BigInt::from(int).gas_size_of() + } +} + +impl GasSizeOf for str { + fn gas_size_of(&self) -> Gas { + self.len().saturating_into() + } +} + +impl GasSizeOf for String { + fn gas_size_of(&self) -> Gas { + self.as_str().gas_size_of() + } +} + +impl GasSizeOf for std::collections::HashMap +where + K: GasSizeOf, + V: GasSizeOf, +{ + fn gas_size_of(&self) -> Gas { + let members = match (K::const_gas_size_of(), V::const_gas_size_of()) { + (Some(k_gas), None) => { + self.values().map(|v| v.gas_size_of()).sum::() + k_gas * self.len() + } + (None, Some(v_gas)) => { + self.keys().map(|k| k.gas_size_of()).sum::() + v_gas * self.len() + } + (Some(k_gas), Some(v_gas)) => (k_gas + v_gas) * self.len(), + (None, None) => self + .iter() + .map(|(k, v)| k.gas_size_of() + v.gas_size_of()) + .sum(), + }; + members + Gas(32) + (Gas(8) * self.len()) + } +} + +impl GasSizeOf for &[T] +where + T: GasSizeOf, +{ + fn gas_size_of(&self) -> Gas { + if let Some(gas) = T::const_gas_size_of() { + gas * self.len() + } else { + self.iter().map(|e| e.gas_size_of()).sum() + } + } +} + +impl GasSizeOf for Vec +where + T: GasSizeOf, +{ + fn gas_size_of(&self) -> Gas { + let members = (&self[..]).gas_size_of(); + // Overhead for Vec so that Vec> is more expensive than Vec + members + Gas(16) + self.len().saturating_into() + } +} + +impl GasSizeOf for &T +where + T: GasSizeOf, +{ + #[inline(always)] + fn gas_size_of(&self) -> Gas { + ::gas_size_of(*self) + } + + #[inline(always)] + fn const_gas_size_of() -> Option { + T::const_gas_size_of() + } +} + +macro_rules! int_gas { + ($($name: ident),*) => { + $( + impl GasSizeOf for $name { + #[inline(always)] + fn const_gas_size_of() -> Option { + Some(std::mem::size_of::<$name>().saturating_into()) + } + } + )* + } +} + +int_gas!(u8, i8, u16, i16, u32, i32, u64, i64, u128, i128); + +impl GasSizeOf for usize { + fn const_gas_size_of() -> Option { + // Must be the same regardless of platform. + u64::const_gas_size_of() + } +} + +impl GasSizeOf for EntityKey { + fn gas_size_of(&self) -> Gas { + self.entity_type.gas_size_of() + self.entity_id.gas_size_of() + } +} + +impl GasSizeOf for EntityType { + fn gas_size_of(&self) -> Gas { + self.as_str().gas_size_of() + } +} diff --git a/graph/src/runtime/mod.rs b/graph/src/runtime/mod.rs new file mode 100644 index 0000000..14128e6 --- /dev/null +++ b/graph/src/runtime/mod.rs @@ -0,0 +1,452 @@ +//! Facilities for creating and reading objects on the memory of an AssemblyScript (Asc) WASM +//! module. Objects are passed through the `asc_new` and `asc_get` methods of an `AscHeap` +//! implementation. These methods take types that implement `To`/`FromAscObj` and are therefore +//! convertible to/from an `AscType`. + +pub mod gas; + +mod asc_heap; +mod asc_ptr; + +pub use asc_heap::{ + asc_get, asc_new, asc_new_or_missing, asc_new_or_null, AscHeap, FromAscObj, ToAscObj, +}; +pub use asc_ptr::AscPtr; + +use anyhow::Error; +use semver::Version; +use std::convert::TryInto; +use std::fmt; +use std::mem::size_of; + +use self::gas::GasCounter; + +/// Marker trait for AssemblyScript types that the id should +/// be in the header. +pub trait AscIndexId { + /// Constant string with the name of the type in AssemblyScript. + /// This is used to get the identifier for the type in memory layout. + /// Info about memory layout: + /// https://www.assemblyscript.org/memory.html#common-header-layout. + /// Info about identifier (`idof`): + /// https://www.assemblyscript.org/garbage-collection.html#runtime-interface + const INDEX_ASC_TYPE_ID: IndexForAscTypeId; +} + +/// A type that has a direct correspondence to an Asc type. +/// +/// This can be derived for structs that are `#[repr(C)]`, contain no padding +/// and whose fields are all `AscValue`. Enums can derive if they are `#[repr(u32)]`. +/// +/// Special classes like `ArrayBuffer` use custom impls. +/// +/// See https://github.com/graphprotocol/graph-node/issues/607 for more considerations. +pub trait AscType: Sized { + /// Transform the Rust representation of this instance into an sequence of + /// bytes that is precisely the memory layout of a corresponding Asc instance. + fn to_asc_bytes(&self) -> Result, DeterministicHostError>; + + /// The Rust representation of an Asc object as layed out in Asc memory. + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result; + + fn content_len(&self, asc_bytes: &[u8]) -> usize { + asc_bytes.len() + } + + /// Size of the corresponding Asc instance in bytes. + /// Only used for version <= 0.0.3. + fn asc_size( + _ptr: AscPtr, + _heap: &H, + _gas: &GasCounter, + ) -> Result { + Ok(std::mem::size_of::() as u32) + } +} + +// Only implemented because of structs that derive AscType and +// contain fields that are PhantomData. +impl AscType for std::marker::PhantomData { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + Ok(vec![]) + } + + fn from_asc_bytes( + asc_obj: &[u8], + _api_version: &Version, + ) -> Result { + assert!(asc_obj.is_empty()); + + Ok(Self) + } +} + +/// An Asc primitive or an `AscPtr` into the Asc heap. A type marked as +/// `AscValue` must have the same byte representation in Rust and Asc, including +/// same size, and size must be equal to alignment. +pub trait AscValue: AscType + Copy + Default {} + +impl AscType for bool { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + Ok(vec![*self as u8]) + } + + fn from_asc_bytes( + asc_obj: &[u8], + _api_version: &Version, + ) -> Result { + if asc_obj.len() != 1 { + Err(DeterministicHostError::from(anyhow::anyhow!( + "Incorrect size for bool. Expected 1, got {},", + asc_obj.len() + ))) + } else { + Ok(asc_obj[0] != 0) + } + } +} + +impl AscValue for bool {} +impl AscValue for AscPtr {} + +macro_rules! impl_asc_type { + ($($T:ty),*) => { + $( + impl AscType for $T { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + Ok(self.to_le_bytes().to_vec()) + } + + fn from_asc_bytes(asc_obj: &[u8], _api_version: &Version) -> Result { + let bytes = asc_obj.try_into().map_err(|_| { + DeterministicHostError::from(anyhow::anyhow!( + "Incorrect size for {}. Expected {}, got {},", + stringify!($T), + size_of::(), + asc_obj.len() + )) + })?; + + Ok(Self::from_le_bytes(bytes)) + } + } + + impl AscValue for $T {} + )* + }; +} + +impl_asc_type!(u8, u16, u32, u64, i8, i32, i64, f32, f64); + +/// Contains type IDs and their discriminants for every blockchain supported by Graph-Node. +/// +/// Each variant corresponds to the unique ID of an AssemblyScript concrete class used in the +/// [`runtime`]. +/// +/// # Rules for updating this enum +/// +/// 1 .The discriminants must have the same value as their counterparts in `TypeId` enum from +/// graph-ts' `global` module. If not, the runtime will fail to determine the correct class +/// during allocation. +/// 2. Each supported blockchain has a reserved space of 1,000 contiguous variants. +/// 3. Once defined, items and their discriminants cannot be changed, as this would break running +/// subgraphs compiled in previous versions of this representation. +#[repr(u32)] +#[derive(Copy, Clone, Debug)] +pub enum IndexForAscTypeId { + // Ethereum type IDs + String = 0, + ArrayBuffer = 1, + Int8Array = 2, + Int16Array = 3, + Int32Array = 4, + Int64Array = 5, + Uint8Array = 6, + Uint16Array = 7, + Uint32Array = 8, + Uint64Array = 9, + Float32Array = 10, + Float64Array = 11, + BigDecimal = 12, + ArrayBool = 13, + ArrayUint8Array = 14, + ArrayEthereumValue = 15, + ArrayStoreValue = 16, + ArrayJsonValue = 17, + ArrayString = 18, + ArrayEventParam = 19, + ArrayTypedMapEntryStringJsonValue = 20, + ArrayTypedMapEntryStringStoreValue = 21, + SmartContractCall = 22, + EventParam = 23, + EthereumTransaction = 24, + EthereumBlock = 25, + EthereumCall = 26, + WrappedTypedMapStringJsonValue = 27, + WrappedBool = 28, + WrappedJsonValue = 29, + EthereumValue = 30, + StoreValue = 31, + JsonValue = 32, + EthereumEvent = 33, + TypedMapEntryStringStoreValue = 34, + TypedMapEntryStringJsonValue = 35, + TypedMapStringStoreValue = 36, + TypedMapStringJsonValue = 37, + TypedMapStringTypedMapStringJsonValue = 38, + ResultTypedMapStringJsonValueBool = 39, + ResultJsonValueBool = 40, + ArrayU8 = 41, + ArrayU16 = 42, + ArrayU32 = 43, + ArrayU64 = 44, + ArrayI8 = 45, + ArrayI16 = 46, + ArrayI32 = 47, + ArrayI64 = 48, + ArrayF32 = 49, + ArrayF64 = 50, + ArrayBigDecimal = 51, + + // Near Type IDs + NearArrayDataReceiver = 52, + NearArrayCryptoHash = 53, + NearArrayActionEnum = 54, + NearArrayMerklePathItem = 55, + NearArrayValidatorStake = 56, + NearArraySlashedValidator = 57, + NearArraySignature = 58, + NearArrayChunkHeader = 59, + NearAccessKeyPermissionEnum = 60, + NearActionEnum = 61, + NearDirectionEnum = 62, + NearPublicKey = 63, + NearSignature = 64, + NearFunctionCallPermission = 65, + NearFullAccessPermission = 66, + NearAccessKey = 67, + NearDataReceiver = 68, + NearCreateAccountAction = 69, + NearDeployContractAction = 70, + NearFunctionCallAction = 71, + NearTransferAction = 72, + NearStakeAction = 73, + NearAddKeyAction = 74, + NearDeleteKeyAction = 75, + NearDeleteAccountAction = 76, + NearActionReceipt = 77, + NearSuccessStatusEnum = 78, + NearMerklePathItem = 79, + NearExecutionOutcome = 80, + NearSlashedValidator = 81, + NearBlockHeader = 82, + NearValidatorStake = 83, + NearChunkHeader = 84, + NearBlock = 85, + NearReceiptWithOutcome = 86, + // Reserved discriminant space for more Near type IDs: [87, 999]: + // Continue to add more Near type IDs here. + // e.g.: + // NextNearType = 87, + // AnotherNearType = 88, + // ... + // LastNearType = 999, + + // Reserved discriminant space for more Ethereum type IDs: [1000, 1499] + TransactionReceipt = 1000, + Log = 1001, + ArrayH256 = 1002, + ArrayLog = 1003, + // Continue to add more Ethereum type IDs here. + // e.g.: + // NextEthereumType = 1004, + // AnotherEthereumType = 1005, + // ... + // LastEthereumType = 1499, + + // Reserved discriminant space for Cosmos type IDs: [1,500, 2,499] + CosmosAny = 1500, + CosmosAnyArray = 1501, + CosmosBytesArray = 1502, + CosmosCoinArray = 1503, + CosmosCommitSigArray = 1504, + CosmosEventArray = 1505, + CosmosEventAttributeArray = 1506, + CosmosEvidenceArray = 1507, + CosmosModeInfoArray = 1508, + CosmosSignerInfoArray = 1509, + CosmosTxResultArray = 1510, + CosmosValidatorArray = 1511, + CosmosValidatorUpdateArray = 1512, + CosmosAuthInfo = 1513, + CosmosBlock = 1514, + CosmosBlockId = 1515, + CosmosBlockIdFlagEnum = 1516, + CosmosBlockParams = 1517, + CosmosCoin = 1518, + CosmosCommit = 1519, + CosmosCommitSig = 1520, + CosmosCompactBitArray = 1521, + CosmosConsensus = 1522, + CosmosConsensusParams = 1523, + CosmosDuplicateVoteEvidence = 1524, + CosmosDuration = 1525, + CosmosEvent = 1526, + CosmosEventAttribute = 1527, + CosmosEventData = 1528, + CosmosEventVote = 1529, + CosmosEvidence = 1530, + CosmosEvidenceList = 1531, + CosmosEvidenceParams = 1532, + CosmosFee = 1533, + CosmosHeader = 1534, + CosmosHeaderOnlyBlock = 1535, + CosmosLightBlock = 1536, + CosmosLightClientAttackEvidence = 1537, + CosmosModeInfo = 1538, + CosmosModeInfoMulti = 1539, + CosmosModeInfoSingle = 1540, + CosmosPartSetHeader = 1541, + CosmosPublicKey = 1542, + CosmosResponseBeginBlock = 1543, + CosmosResponseDeliverTx = 1544, + CosmosResponseEndBlock = 1545, + CosmosSignModeEnum = 1546, + CosmosSignedHeader = 1547, + CosmosSignedMsgTypeEnum = 1548, + CosmosSignerInfo = 1549, + CosmosTimestamp = 1550, + CosmosTip = 1551, + CosmosTransactionData = 1552, + CosmosTx = 1553, + CosmosTxBody = 1554, + CosmosTxResult = 1555, + CosmosValidator = 1556, + CosmosValidatorParams = 1557, + CosmosValidatorSet = 1558, + CosmosValidatorSetUpdates = 1559, + CosmosValidatorUpdate = 1560, + CosmosVersionParams = 1561, + + // Continue to add more Cosmos type IDs here. + // e.g.: + // NextCosmosType = 1562, + // AnotherCosmosType = 1563, + // ... + // LastCosmosType = 2499, + + // Arweave types + ArweaveBlock = 2500, + ArweaveProofOfAccess = 2501, + ArweaveTag = 2502, + ArweaveTagArray = 2503, + ArweaveTransaction = 2504, + ArweaveTransactionArray = 2505, + ArweaveTransactionWithBlockPtr = 2506, + // Continue to add more Arweave type IDs here. + // e.g.: + // NextArweaveType = 2507, + // AnotherArweaveType = 2508, + // ... + // LastArweaveType = 3499, + + // Reserved discriminant space for a future blockchain type IDs: [3,500, 4,499] + // + // Generated with the following shell script: + // + // ``` + // grep -Po "(?<=IndexForAscTypeId::)IDENDIFIER_PREFIX.*\b" SRC_FILE | sort |uniq | awk 'BEGIN{count=2500} {sub("$", " = "count",", $1); count++} 1' + // ``` + // + // INSTRUCTIONS: + // 1. Replace the IDENTIFIER_PREFIX and the SRC_FILE placeholders according to the blockchain + // name and implementation before running this script. + // 2. Replace `3500` part with the first number of that blockchain's reserved discriminant space. + // 3. Insert the output right before the end of this block. + UnitTestNetworkUnitTestTypeU32 = u32::MAX - 7, + UnitTestNetworkUnitTestTypeU32Array = u32::MAX - 6, + + UnitTestNetworkUnitTestTypeU16 = u32::MAX - 5, + UnitTestNetworkUnitTestTypeU16Array = u32::MAX - 4, + + UnitTestNetworkUnitTestTypeI8 = u32::MAX - 3, + UnitTestNetworkUnitTestTypeI8Array = u32::MAX - 2, + + UnitTestNetworkUnitTestTypeBool = u32::MAX - 1, + UnitTestNetworkUnitTestTypeBoolArray = u32::MAX, +} + +impl ToAscObj for IndexForAscTypeId { + fn to_asc_obj( + &self, + _heap: &mut H, + _gas: &GasCounter, + ) -> Result { + Ok(*self as u32) + } +} + +#[derive(Debug)] +pub enum DeterministicHostError { + Gas(Error), + Other(Error), +} + +impl DeterministicHostError { + pub fn gas(e: Error) -> Self { + DeterministicHostError::Gas(e) + } + + pub fn inner(self) -> Error { + match self { + DeterministicHostError::Gas(e) | DeterministicHostError::Other(e) => e, + } + } +} + +impl fmt::Display for DeterministicHostError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DeterministicHostError::Gas(e) | DeterministicHostError::Other(e) => e.fmt(f), + } + } +} + +impl From for DeterministicHostError { + fn from(e: Error) -> DeterministicHostError { + DeterministicHostError::Other(e) + } +} + +impl std::error::Error for DeterministicHostError {} + +#[derive(thiserror::Error, Debug)] +pub enum HostExportError { + #[error("{0:#}")] + Unknown(#[from] anyhow::Error), + + #[error("{0:#}")] + PossibleReorg(anyhow::Error), + + #[error("{0:#}")] + Deterministic(anyhow::Error), +} + +impl From for HostExportError { + fn from(value: DeterministicHostError) -> Self { + match value { + // Until we are confident on the gas numbers, gas errors are not deterministic + DeterministicHostError::Gas(e) => HostExportError::Unknown(e), + DeterministicHostError::Other(e) => HostExportError::Deterministic(e), + } + } +} + +pub const HEADER_SIZE: usize = 20; + +pub fn padding_to_16(content_length: usize) -> usize { + (16 - (HEADER_SIZE + content_length) % 16) % 16 +} diff --git a/graph/src/substreams/codec.rs b/graph/src/substreams/codec.rs new file mode 100644 index 0000000..23edcc3 --- /dev/null +++ b/graph/src/substreams/codec.rs @@ -0,0 +1,5 @@ +#[rustfmt::skip] +#[path = "sf.substreams.v1.rs"] +mod pbsubstreams; + +pub use pbsubstreams::*; diff --git a/graph/src/substreams/mod.rs b/graph/src/substreams/mod.rs new file mode 100644 index 0000000..38e96fd --- /dev/null +++ b/graph/src/substreams/mod.rs @@ -0,0 +1,3 @@ +mod codec; + +pub use codec::*; diff --git a/graph/src/substreams/sf.substreams.v1.rs b/graph/src/substreams/sf.substreams.v1.rs new file mode 100644 index 0000000..0f81ea7 --- /dev/null +++ b/graph/src/substreams/sf.substreams.v1.rs @@ -0,0 +1,616 @@ +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Request { + #[prost(int64, tag="1")] + pub start_block_num: i64, + #[prost(string, tag="2")] + pub start_cursor: ::prost::alloc::string::String, + #[prost(uint64, tag="3")] + pub stop_block_num: u64, + #[prost(enumeration="ForkStep", repeated, tag="4")] + pub fork_steps: ::prost::alloc::vec::Vec, + #[prost(string, tag="5")] + pub irreversibility_condition: ::prost::alloc::string::String, + #[prost(message, optional, tag="6")] + pub modules: ::core::option::Option, + #[prost(string, repeated, tag="7")] + pub output_modules: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, repeated, tag="8")] + pub initial_store_snapshot_for_modules: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Response { + #[prost(oneof="response::Message", tags="1, 2, 3, 4")] + pub message: ::core::option::Option, +} +/// Nested message and enum types in `Response`. +pub mod response { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Message { + /// Progress of data preparation, before sending in the stream of `data` events. + #[prost(message, tag="1")] + Progress(super::ModulesProgress), + #[prost(message, tag="2")] + SnapshotData(super::InitialSnapshotData), + #[prost(message, tag="3")] + SnapshotComplete(super::InitialSnapshotComplete), + #[prost(message, tag="4")] + Data(super::BlockScopedData), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct InitialSnapshotComplete { + #[prost(string, tag="1")] + pub cursor: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct InitialSnapshotData { + #[prost(string, tag="1")] + pub module_name: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub deltas: ::core::option::Option, + #[prost(uint64, tag="4")] + pub sent_keys: u64, + #[prost(uint64, tag="3")] + pub total_keys: u64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockScopedData { + #[prost(message, repeated, tag="1")] + pub outputs: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="3")] + pub clock: ::core::option::Option, + #[prost(enumeration="ForkStep", tag="6")] + pub step: i32, + #[prost(string, tag="10")] + pub cursor: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ModuleOutput { + #[prost(string, tag="1")] + pub name: ::prost::alloc::string::String, + #[prost(string, repeated, tag="4")] + pub logs: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// LogsTruncated is a flag that tells you if you received all the logs or if they + /// were truncated because you logged too much (fixed limit currently is set to 128 KiB). + #[prost(bool, tag="5")] + pub logs_truncated: bool, + #[prost(oneof="module_output::Data", tags="2, 3")] + pub data: ::core::option::Option, +} +/// Nested message and enum types in `ModuleOutput`. +pub mod module_output { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Data { + #[prost(message, tag="2")] + MapOutput(::prost_types::Any), + #[prost(message, tag="3")] + StoreDeltas(super::StoreDeltas), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ModulesProgress { + #[prost(message, repeated, tag="1")] + pub modules: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ModuleProgress { + #[prost(string, tag="1")] + pub name: ::prost::alloc::string::String, + #[prost(oneof="module_progress::Type", tags="2, 3, 4, 5")] + pub r#type: ::core::option::Option, +} +/// Nested message and enum types in `ModuleProgress`. +pub mod module_progress { + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct ProcessedRange { + #[prost(message, repeated, tag="1")] + pub processed_ranges: ::prost::alloc::vec::Vec, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct InitialState { + #[prost(uint64, tag="2")] + pub available_up_to_block: u64, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct ProcessedBytes { + #[prost(uint64, tag="1")] + pub total_bytes_read: u64, + #[prost(uint64, tag="2")] + pub total_bytes_written: u64, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Failed { + #[prost(string, tag="1")] + pub reason: ::prost::alloc::string::String, + #[prost(string, repeated, tag="2")] + pub logs: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// FailureLogsTruncated is a flag that tells you if you received all the logs or if they + /// were truncated because you logged too much (fixed limit currently is set to 128 KiB). + #[prost(bool, tag="3")] + pub logs_truncated: bool, + } + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Type { + #[prost(message, tag="2")] + ProcessedRanges(ProcessedRange), + #[prost(message, tag="3")] + InitialState(InitialState), + #[prost(message, tag="4")] + ProcessedBytes(ProcessedBytes), + #[prost(message, tag="5")] + Failed(Failed), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockRange { + #[prost(uint64, tag="1")] + pub start_block: u64, + #[prost(uint64, tag="2")] + pub end_block: u64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StoreDeltas { + #[prost(message, repeated, tag="1")] + pub deltas: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StoreDelta { + #[prost(enumeration="store_delta::Operation", tag="1")] + pub operation: i32, + #[prost(uint64, tag="2")] + pub ordinal: u64, + #[prost(string, tag="3")] + pub key: ::prost::alloc::string::String, + #[prost(bytes="vec", tag="4")] + pub old_value: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="5")] + pub new_value: ::prost::alloc::vec::Vec, +} +/// Nested message and enum types in `StoreDelta`. +pub mod store_delta { + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] + #[repr(i32)] + pub enum Operation { + Unset = 0, + Create = 1, + Update = 2, + Delete = 3, + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Output { + #[prost(uint64, tag="1")] + pub block_num: u64, + #[prost(string, tag="2")] + pub block_id: ::prost::alloc::string::String, + #[prost(message, optional, tag="4")] + pub timestamp: ::core::option::Option<::prost_types::Timestamp>, + #[prost(message, optional, tag="10")] + pub value: ::core::option::Option<::prost_types::Any>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Modules { + #[prost(message, repeated, tag="1")] + pub modules: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="2")] + pub binaries: ::prost::alloc::vec::Vec, +} +/// Binary represents some code compiled to its binary form. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Binary { + #[prost(string, tag="1")] + pub r#type: ::prost::alloc::string::String, + #[prost(bytes="vec", tag="2")] + pub content: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Module { + #[prost(string, tag="1")] + pub name: ::prost::alloc::string::String, + #[prost(uint32, tag="4")] + pub binary_index: u32, + #[prost(string, tag="5")] + pub binary_entrypoint: ::prost::alloc::string::String, + #[prost(message, repeated, tag="6")] + pub inputs: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag="7")] + pub output: ::core::option::Option, + #[prost(uint64, tag="8")] + pub initial_block: u64, + #[prost(oneof="module::Kind", tags="2, 3")] + pub kind: ::core::option::Option, +} +/// Nested message and enum types in `Module`. +pub mod module { + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct KindMap { + #[prost(string, tag="1")] + pub output_type: ::prost::alloc::string::String, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct KindStore { + /// The `update_policy` determines the functions available to mutate the store + /// (like `set()`, `set_if_not_exists()` or `sum()`, etc..) in + /// order to ensure that parallel operations are possible and deterministic + /// + /// Say a store cumulates keys from block 0 to 1M, and a second store + /// cumulates keys from block 1M to 2M. When we want to use this + /// store as a dependency for a downstream module, we will merge the + /// two stores according to this policy. + #[prost(enumeration="kind_store::UpdatePolicy", tag="1")] + pub update_policy: i32, + #[prost(string, tag="2")] + pub value_type: ::prost::alloc::string::String, + } + /// Nested message and enum types in `KindStore`. + pub mod kind_store { + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] + #[repr(i32)] + pub enum UpdatePolicy { + Unset = 0, + /// Provides a store where you can `set()` keys, and the latest key wins + Set = 1, + /// Provides a store where you can `set_if_not_exists()` keys, and the first key wins + SetIfNotExists = 2, + /// Provides a store where you can `add_*()` keys, where two stores merge by summing its values. + Add = 3, + /// Provides a store where you can `min_*()` keys, where two stores merge by leaving the minimum value. + Min = 4, + /// Provides a store where you can `max_*()` keys, where two stores merge by leaving the maximum value. + Max = 5, + } + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Input { + #[prost(oneof="input::Input", tags="1, 2, 3")] + pub input: ::core::option::Option, + } + /// Nested message and enum types in `Input`. + pub mod input { + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Source { + /// ex: "sf.ethereum.type.v1.Block" + #[prost(string, tag="1")] + pub r#type: ::prost::alloc::string::String, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Map { + /// ex: "block_to_pairs" + #[prost(string, tag="1")] + pub module_name: ::prost::alloc::string::String, + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Store { + #[prost(string, tag="1")] + pub module_name: ::prost::alloc::string::String, + #[prost(enumeration="store::Mode", tag="2")] + pub mode: i32, + } + /// Nested message and enum types in `Store`. + pub mod store { + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] + #[repr(i32)] + pub enum Mode { + Unset = 0, + Get = 1, + Deltas = 2, + } + } + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Input { + #[prost(message, tag="1")] + Source(Source), + #[prost(message, tag="2")] + Map(Map), + #[prost(message, tag="3")] + Store(Store), + } + } + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Output { + #[prost(string, tag="1")] + pub r#type: ::prost::alloc::string::String, + } + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Kind { + #[prost(message, tag="2")] + KindMap(KindMap), + #[prost(message, tag="3")] + KindStore(KindStore), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Clock { + #[prost(string, tag="1")] + pub id: ::prost::alloc::string::String, + #[prost(uint64, tag="2")] + pub number: u64, + #[prost(message, optional, tag="3")] + pub timestamp: ::core::option::Option<::prost_types::Timestamp>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Package { + /// Needs to be one so this file can be used _directly_ as a + /// buf `Image` andor a ProtoSet for grpcurl and other tools + #[prost(message, repeated, tag="1")] + pub proto_files: ::prost::alloc::vec::Vec<::prost_types::FileDescriptorProto>, + #[prost(uint64, tag="5")] + pub version: u64, + #[prost(message, optional, tag="6")] + pub modules: ::core::option::Option, + #[prost(message, repeated, tag="7")] + pub module_meta: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="8")] + pub package_meta: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PackageMetadata { + #[prost(string, tag="1")] + pub version: ::prost::alloc::string::String, + #[prost(string, tag="2")] + pub url: ::prost::alloc::string::String, + #[prost(string, tag="3")] + pub name: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub doc: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ModuleMetadata { + /// Corresponds to the index in `Package.metadata.package_meta` + #[prost(uint64, tag="1")] + pub package_index: u64, + #[prost(string, tag="2")] + pub doc: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ForkStep { + StepUnknown = 0, + /// Block is new head block of the chain, that is linear with the previous block + StepNew = 1, + /// Block is now forked and should be undone, it's not the head block of the chain anymore + StepUndo = 2, + /// Block is now irreversible and can be committed to (finality is chain specific, see chain documentation for more details) + StepIrreversible = 4, +} +/// Generated client implementations. +pub mod stream_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + #[derive(Debug, Clone)] + pub struct StreamClient { + inner: tonic::client::Grpc, + } + impl StreamClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: std::convert::TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl StreamClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + Send + 'static, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> StreamClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + Send + Sync, + { + StreamClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with `gzip`. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_gzip(mut self) -> Self { + self.inner = self.inner.send_gzip(); + self + } + /// Enable decompressing responses with `gzip`. + #[must_use] + pub fn accept_gzip(mut self) -> Self { + self.inner = self.inner.accept_gzip(); + self + } + pub async fn blocks( + &mut self, + request: impl tonic::IntoRequest, + ) -> Result< + tonic::Response>, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sf.substreams.v1.Stream/Blocks", + ); + self.inner.server_streaming(request.into_request(), path, codec).await + } + } +} +/// Generated server implementations. +pub mod stream_server { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + ///Generated trait containing gRPC methods that should be implemented for use with StreamServer. + #[async_trait] + pub trait Stream: Send + Sync + 'static { + ///Server streaming response type for the Blocks method. + type BlocksStream: futures_core::Stream< + Item = Result, + > + + Send + + 'static; + async fn blocks( + &self, + request: tonic::Request, + ) -> Result, tonic::Status>; + } + #[derive(Debug)] + pub struct StreamServer { + inner: _Inner, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + } + struct _Inner(Arc); + impl StreamServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + let inner = _Inner(inner); + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with `gzip`. + #[must_use] + pub fn accept_gzip(mut self) -> Self { + self.accept_compression_encodings.enable_gzip(); + self + } + /// Compress responses with `gzip`, if the client supports it. + #[must_use] + pub fn send_gzip(mut self) -> Self { + self.send_compression_encodings.enable_gzip(); + self + } + } + impl tonic::codegen::Service> for StreamServer + where + T: Stream, + B: Body + Send + 'static, + B::Error: Into + Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + let inner = self.inner.clone(); + match req.uri().path() { + "/sf.substreams.v1.Stream/Blocks" => { + #[allow(non_camel_case_types)] + struct BlocksSvc(pub Arc); + impl tonic::server::ServerStreamingService + for BlocksSvc { + type Response = super::Response; + type ResponseStream = T::BlocksStream; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = self.0.clone(); + let fut = async move { (*inner).blocks(request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = BlocksSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + Ok( + http::Response::builder() + .status(200) + .header("grpc-status", "12") + .header("content-type", "application/grpc") + .body(empty_body()) + .unwrap(), + ) + }) + } + } + } + } + impl Clone for StreamServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + } + } + } + impl Clone for _Inner { + fn clone(&self) -> Self { + Self(self.0.clone()) + } + } + impl std::fmt::Debug for _Inner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } + } + impl tonic::transport::NamedService for StreamServer { + const NAME: &'static str = "sf.substreams.v1.Stream"; + } +} diff --git a/graph/src/task_spawn.rs b/graph/src/task_spawn.rs new file mode 100644 index 0000000..c323d6d --- /dev/null +++ b/graph/src/task_spawn.rs @@ -0,0 +1,70 @@ +//! The functions in this module should be used to execute futures, serving as a facade to the +//! underlying executor implementation which currently is tokio. This serves a few purposes: +//! - Avoid depending directly on tokio APIs, making upgrades or a potential switch easier. +//! - Reflect our chosen default semantics of aborting on task panic, offering `*_allow_panic` +//! functions to opt out of that. +//! - Reflect that historically we've used blocking futures due to making DB calls directly within +//! futures. This point should go away once https://github.com/graphprotocol/graph-node/issues/905 +//! is resolved. Then the blocking flavors should no longer accept futures but closures. +//! +//! These should not be called from within executors other than tokio, particularly the blocking +//! functions will panic in that case. We should generally avoid mixing executors whenever possible. + +use futures03::future::{FutureExt, TryFutureExt}; +use std::future::Future as Future03; +use std::panic::AssertUnwindSafe; +use tokio::task::JoinHandle; + +fn abort_on_panic( + f: impl Future03 + Send + 'static, +) -> impl Future03 { + // We're crashing, unwind safety doesn't matter. + AssertUnwindSafe(f).catch_unwind().unwrap_or_else(|_| { + println!("Panic in tokio task, aborting!"); + std::process::abort() + }) +} + +/// Aborts on panic. +pub fn spawn(f: impl Future03 + Send + 'static) -> JoinHandle { + tokio::spawn(abort_on_panic(f)) +} + +pub fn spawn_allow_panic( + f: impl Future03 + Send + 'static, +) -> JoinHandle { + tokio::spawn(f) +} + +/// Aborts on panic. +pub fn spawn_blocking( + f: impl Future03 + Send + 'static, +) -> JoinHandle { + tokio::task::spawn_blocking(move || block_on(abort_on_panic(f))) +} + +/// Does not abort on panic, panics result in an `Err` in `JoinHandle`. +pub fn spawn_blocking_allow_panic( + f: impl 'static + FnOnce() -> R + Send, +) -> JoinHandle { + tokio::task::spawn_blocking(f) +} + +/// Runs the future on the current thread. Panics if not within a tokio runtime. +pub fn block_on(f: impl Future03) -> T { + tokio::runtime::Handle::current().block_on(f) +} + +/// Spawns a thread with access to the tokio runtime. Panics if the thread cannot be spawned. +pub fn spawn_thread( + name: impl Into, + f: impl 'static + FnOnce() + Send, +) -> std::thread::JoinHandle<()> { + let conf = std::thread::Builder::new().name(name.into()); + let runtime = tokio::runtime::Handle::current(); + conf.spawn(move || { + let _runtime_guard = runtime.enter(); + f() + }) + .unwrap() +} diff --git a/graph/src/util/backoff.rs b/graph/src/util/backoff.rs new file mode 100644 index 0000000..1a5d3b2 --- /dev/null +++ b/graph/src/util/backoff.rs @@ -0,0 +1,52 @@ +use std::time::Duration; + +/// Facilitate sleeping with an exponential backoff. Sleep durations will +/// increase by a factor of 2 from `base` until they reach `ceiling`, at +/// which point any call to `sleep` or `sleep_async` will sleep for +/// `ceiling` +pub struct ExponentialBackoff { + pub attempt: u64, + base: Duration, + ceiling: Duration, +} + +impl ExponentialBackoff { + pub fn new(base: Duration, ceiling: Duration) -> Self { + ExponentialBackoff { + attempt: 0, + base, + ceiling, + } + } + + /// Record that we made an attempt and sleep for the appropriate amount + /// of time. Do not use this from async contexts since it uses + /// `thread::sleep` + pub fn sleep(&mut self) { + std::thread::sleep(self.next_attempt()); + } + + /// Record that we made an attempt and sleep for the appropriate amount + /// of time + pub async fn sleep_async(&mut self) { + tokio::time::sleep(self.next_attempt()).await + } + + pub fn delay(&self) -> Duration { + let mut delay = self.base.saturating_mul(1 << self.attempt); + if delay > self.ceiling { + delay = self.ceiling; + } + delay + } + + fn next_attempt(&mut self) -> Duration { + let delay = self.delay(); + self.attempt += 1; + delay + } + + pub fn reset(&mut self) { + self.attempt = 0; + } +} diff --git a/graph/src/util/bounded_queue.rs b/graph/src/util/bounded_queue.rs new file mode 100644 index 0000000..5e0a666 --- /dev/null +++ b/graph/src/util/bounded_queue.rs @@ -0,0 +1,150 @@ +use std::{collections::VecDeque, sync::Mutex}; + +use crate::prelude::tokio::sync::Semaphore; + +/// An async-friendly queue of bounded size. In contrast to a bounded channel, +/// the queue makes it possible to modify and remove entries in it. +pub struct BoundedQueue { + /// The maximum number of entries allowed in the queue + capacity: usize, + /// The actual items in the queue. New items are appended at the back, and + /// popped off the front. + queue: Mutex>, + /// This semaphore has as many permits as there are empty spots in the + /// `queue`, i.e., `capacity - queue.len()` many permits + push_semaphore: Semaphore, + /// This semaphore has as many permits as there are entrie in the queue, + /// i.e., `queue.len()` many + pop_semaphore: Semaphore, +} + +impl std::fmt::Debug for BoundedQueue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let queue = self.queue.lock().unwrap(); + write!( + f, + "BoundedQueue[cap: {}, queue: {}/{}, push: {}, pop: {}]", + self.capacity, + queue.len(), + queue.capacity(), + self.push_semaphore.available_permits(), + self.pop_semaphore.available_permits(), + ) + } +} + +impl BoundedQueue { + pub fn with_capacity(capacity: usize) -> Self { + Self { + capacity, + queue: Mutex::new(VecDeque::with_capacity(capacity)), + push_semaphore: Semaphore::new(capacity), + pop_semaphore: Semaphore::new(0), + } + } + + /// Get an item from the queue. If the queue is currently empty + /// this method blocks until an item is available. + pub async fn pop(&self) -> T { + let permit = self.pop_semaphore.acquire().await.unwrap(); + let item = self + .queue + .lock() + .unwrap() + .pop_front() + .expect("the queue is not empty"); + permit.forget(); + self.push_semaphore.add_permits(1); + item + } + + /// Get an item from the queue without blocking; if the queue is empty, + /// return `None` + pub fn try_pop(&self) -> Option { + let permit = match self.pop_semaphore.try_acquire() { + Err(_) => return None, + Ok(permit) => permit, + }; + let item = self + .queue + .lock() + .unwrap() + .pop_front() + .expect("the queue is not empty"); + permit.forget(); + self.push_semaphore.add_permits(1); + Some(item) + } + + /// Take an item from the front of the queue and return a copy. If the + /// queue is currently empty this method blocks until an item is + /// available. + pub async fn peek(&self) -> T { + let _permit = self.pop_semaphore.acquire().await.unwrap(); + let queue = self.queue.lock().unwrap(); + let item = queue.front().expect("the queue is not empty"); + item.clone() + } + + /// Push an item into the queue. If the queue is currently full this method + /// blocks until an item is available + pub async fn push(&self, item: T) { + let permit = self.push_semaphore.acquire().await.unwrap(); + self.queue.lock().unwrap().push_back(item); + permit.forget(); + self.pop_semaphore.add_permits(1); + } + + pub async fn wait_empty(&self) { + self.push_semaphore + .acquire_many(self.capacity as u32) + .await + .map(|_| ()) + .expect("we never close the push_semaphore") + } + + pub fn len(&self) -> usize { + self.queue.lock().unwrap().len() + } + + pub fn is_empty(&self) -> bool { + self.queue.lock().unwrap().is_empty() + } + + pub fn capacity(&self) -> usize { + self.capacity + } + + /// Iterate over the entries in the queue from newest to oldest entry + /// atomically, applying `f` to each entry and returning the first + /// result that is not `None`. + /// + /// This method locks the queue while it is executing, and `f` should + /// therefore not do any slow work. + pub fn find_map(&self, f: F) -> Option + where + F: FnMut(&T) -> Option, + { + let queue = self.queue.lock().unwrap(); + queue.iter().rev().find_map(f) + } + + /// Iterate over the entries in the queue from newest to oldest entry + /// atomically, applying `f` to each entry and returning the result of + /// the last invocation of `f`. + /// + /// This method locks the queue while it is executing, and `f` should + /// therefore not do any slow work. + pub fn fold(&self, init: B, f: F) -> B + where + F: FnMut(B, &T) -> B, + { + let queue = self.queue.lock().unwrap(); + queue.iter().rev().fold(init, f) + } + + /// Clear the queue by popping entries until there are none left + pub fn clear(&self) { + while let Some(_) = self.try_pop() {} + } +} diff --git a/graph/src/util/cache_weight.rs b/graph/src/util/cache_weight.rs new file mode 100644 index 0000000..af15a82 --- /dev/null +++ b/graph/src/util/cache_weight.rs @@ -0,0 +1,246 @@ +use crate::{ + components::store::{EntityKey, EntityType}, + data::value::Word, + prelude::{q, BigDecimal, BigInt, Value}, +}; +use std::{ + collections::{BTreeMap, HashMap}, + mem, +}; + +/// Estimate of how much memory a value consumes. +/// Useful for measuring the size of caches. +pub trait CacheWeight { + /// Total weight of the value. + fn weight(&self) -> usize { + mem::size_of_val(self) + self.indirect_weight() + } + + /// The weight of values pointed to by this value but logically owned by it, which is not + /// accounted for by `size_of`. + fn indirect_weight(&self) -> usize; +} + +impl CacheWeight for Option { + fn indirect_weight(&self) -> usize { + match self { + Some(x) => x.indirect_weight(), + None => 0, + } + } +} + +impl CacheWeight for Vec { + fn indirect_weight(&self) -> usize { + self.iter().map(CacheWeight::indirect_weight).sum::() + + self.capacity() * mem::size_of::() + } +} + +impl CacheWeight for BTreeMap { + fn indirect_weight(&self) -> usize { + self.iter() + .map(|(key, value)| key.indirect_weight() + value.indirect_weight()) + .sum::() + + btree::node_size(self) + } +} + +impl CacheWeight for HashMap { + fn indirect_weight(&self) -> usize { + self.iter() + .map(|(key, value)| key.indirect_weight() + value.indirect_weight()) + .sum::() + + self.capacity() * mem::size_of::<(T, U, u64)>() + } +} + +impl CacheWeight for String { + fn indirect_weight(&self) -> usize { + self.capacity() + } +} + +impl CacheWeight for Word { + fn indirect_weight(&self) -> usize { + self.len() + } +} + +impl CacheWeight for BigDecimal { + fn indirect_weight(&self) -> usize { + ((self.digits() as f32 * std::f32::consts::LOG2_10) / 8.0).ceil() as usize + } +} + +impl CacheWeight for BigInt { + fn indirect_weight(&self) -> usize { + self.bits() / 8 + } +} + +impl CacheWeight for crate::data::store::scalar::Bytes { + fn indirect_weight(&self) -> usize { + self.as_slice().len() + } +} + +impl CacheWeight for Value { + fn indirect_weight(&self) -> usize { + match self { + Value::String(s) => s.indirect_weight(), + Value::BigDecimal(d) => d.indirect_weight(), + Value::List(values) => values.indirect_weight(), + Value::Bytes(bytes) => bytes.indirect_weight(), + Value::BigInt(n) => n.indirect_weight(), + Value::Int(_) | Value::Bool(_) | Value::Null => 0, + } + } +} + +impl CacheWeight for q::Value { + fn indirect_weight(&self) -> usize { + match self { + q::Value::Boolean(_) | q::Value::Int(_) | q::Value::Null | q::Value::Float(_) => 0, + q::Value::Enum(s) | q::Value::String(s) | q::Value::Variable(s) => s.indirect_weight(), + q::Value::List(l) => l.indirect_weight(), + q::Value::Object(o) => o.indirect_weight(), + } + } +} + +impl CacheWeight for usize { + fn indirect_weight(&self) -> usize { + 0 + } +} + +impl CacheWeight for EntityType { + fn indirect_weight(&self) -> usize { + 0 + } +} + +impl CacheWeight for EntityKey { + fn indirect_weight(&self) -> usize { + self.entity_id.indirect_weight() + self.entity_type.indirect_weight() + } +} + +impl CacheWeight for [u8; 32] { + fn indirect_weight(&self) -> usize { + 0 + } +} + +#[cfg(test)] +impl CacheWeight for &'static str { + fn indirect_weight(&self) -> usize { + 0 + } +} + +#[test] +fn big_decimal_cache_weight() { + use std::str::FromStr; + + // 22.4548 has 18 bits as binary, so 3 bytes. + let n = BigDecimal::from_str("22.454800000000").unwrap(); + assert_eq!(n.indirect_weight(), 3); +} + +/// Helpers to estimate the size of a `BTreeMap`. Everything in this module, +/// except for `node_size()` is copied from `std::collections::btree`. +/// +/// It is not possible to know how many nodes a BTree has, as +/// `BTreeMap` does not expose its depth or any other detail about +/// the true size of the BTree. We estimate that size, assuming the +/// average case, i.e., a BTree where every node has the average +/// between the minimum and maximum number of entries per node, i.e., +/// the average of (B-1) and (2*B-1) entries, which we call +/// `NODE_FILL`. The number of leaf nodes in the tree is then the +/// number of entries divided by `NODE_FILL`, and the number of +/// interior nodes can be determined by dividing the number of nodes +/// at the child level by `NODE_FILL` + +/// The other difficulty is that the structs with which `BTreeMap` +/// represents internal and leaf nodes are not public, so we can't +/// get their size with `std::mem::size_of`; instead, we base our +/// estimates of their size on the current `std` code, assuming that +/// these structs will not change + +pub mod btree { + use std::collections::BTreeMap; + use std::mem; + use std::{mem::MaybeUninit, ptr::NonNull}; + + const B: usize = 6; + const CAPACITY: usize = 2 * B - 1; + + /// Assume BTree nodes are this full (average of minimum and maximum fill) + const NODE_FILL: usize = ((B - 1) + (2 * B - 1)) / 2; + + type BoxedNode = NonNull>; + + struct InternalNode { + _data: LeafNode, + + /// The pointers to the children of this node. `len + 1` of these are considered + /// initialized and valid, except that near the end, while the tree is held + /// through borrow type `Dying`, some of these pointers are dangling. + _edges: [MaybeUninit>; 2 * B], + } + + struct LeafNode { + /// We want to be covariant in `K` and `V`. + _parent: Option>>, + + /// This node's index into the parent node's `edges` array. + /// `*node.parent.edges[node.parent_idx]` should be the same thing as `node`. + /// This is only guaranteed to be initialized when `parent` is non-null. + _parent_idx: MaybeUninit, + + /// The number of keys and values this node stores. + _len: u16, + + /// The arrays storing the actual data of the node. Only the first `len` elements of each + /// array are initialized and valid. + _keys: [MaybeUninit; CAPACITY], + _vals: [MaybeUninit; CAPACITY], + } + + /// Estimate the size of the BTreeMap `map` ignoring the size of any keys + /// and values + pub fn node_size(map: &BTreeMap) -> usize { + // Measure the size of internal and leaf nodes directly - that's why + // we copied all this code from `std` + let ln_sz = mem::size_of::>(); + let in_sz = mem::size_of::>(); + + // Estimate the number of internal and leaf nodes based on the only + // thing we can measure about a BTreeMap, the number of entries in + // it, and use our `NODE_FILL` assumption to estimate how the tree + // is structured. We try to be very good for small maps, since + // that's what we use most often in our code. This estimate is only + // for the indirect weight of the `BTreeMap` + let (leaves, int_nodes) = if map.is_empty() { + // An empty tree has no indirect weight + (0, 0) + } else if map.len() <= CAPACITY { + // We only have the root node + (1, 0) + } else { + // Estimate based on our `NODE_FILL` assumption + let leaves = map.len() / NODE_FILL + 1; + let mut prev_level = leaves / NODE_FILL + 1; + let mut int_nodes = prev_level; + while prev_level > 1 { + int_nodes += prev_level; + prev_level = prev_level / NODE_FILL + 1; + } + (leaves, int_nodes) + }; + + leaves * ln_sz + int_nodes * in_sz + } +} diff --git a/graph/src/util/error.rs b/graph/src/util/error.rs new file mode 100644 index 0000000..1b3cb88 --- /dev/null +++ b/graph/src/util/error.rs @@ -0,0 +1,19 @@ +// `ensure!` from `anyhow`, but calling `from`. +#[macro_export] +macro_rules! ensure { + ($cond:expr, $msg:literal $(,)?) => { + if !$cond { + return Err(From::from($crate::prelude::anyhow::anyhow!($msg))); + } + }; + ($cond:expr, $err:expr $(,)?) => { + if !$cond { + return Err(From::from($crate::prelude::anyhow::anyhow!($err))); + } + }; + ($cond:expr, $fmt:expr, $($arg:tt)*) => { + if !$cond { + return Err(From::from($crate::prelude::anyhow::anyhow!($fmt, $($arg)*))); + } + }; +} diff --git a/graph/src/util/futures.rs b/graph/src/util/futures.rs new file mode 100644 index 0000000..eba48ae --- /dev/null +++ b/graph/src/util/futures.rs @@ -0,0 +1,548 @@ +use crate::ext::futures::FutureExtension; +use futures03::{Future, FutureExt, TryFutureExt}; +use slog::{debug, trace, warn, Logger}; +use std::fmt::Debug; +use std::marker::PhantomData; +use std::sync::Arc; +use std::time::Duration; +use thiserror::Error; +use tokio_retry::strategy::{jitter, ExponentialBackoff}; +use tokio_retry::Retry; + +/// Generic helper function for retrying async operations with built-in logging. +/// +/// To use this helper, do the following: +/// +/// 1. Call this function with an operation name (used for logging) and a `Logger`. +/// 2. Optional: Chain a call to `.when(...)` to set a custom retry condition. +/// 3. Optional: call `.log_after(...)` or `.no_logging()`. +/// 4. Call either `.limit(...)` or `.no_limit()`. +/// 5. Call one of `.timeout_secs(...)`, `.timeout_millis(...)`, `.timeout(...)`, and +/// `.no_timeout()`. +/// 6. Call `.run(...)`. +/// +/// All steps are required, except Step 2 and Step 3. +/// +/// Example usage: +/// ``` +/// # extern crate graph; +/// # use graph::prelude::*; +/// # use tokio::time::timeout; +/// use std::future::Future; +/// use graph::prelude::{Logger, TimeoutError}; +/// # +/// # type Memes = (); // the memes are a lie :( +/// # +/// # async fn download_the_memes() -> Result { +/// # Ok(()) +/// # } +/// +/// fn async_function(logger: Logger) -> impl Future>> { +/// // Retry on error +/// retry("download memes", &logger) +/// .no_limit() // Retry forever +/// .timeout_secs(30) // Retry if an attempt takes > 30 seconds +/// .run(|| { +/// download_the_memes() // Return a Future +/// }) +/// } +/// ``` +pub fn retry(operation_name: impl ToString, logger: &Logger) -> RetryConfig { + RetryConfig { + operation_name: operation_name.to_string(), + logger: logger.to_owned(), + condition: RetryIf::Error, + log_after: 1, + warn_after: 10, + limit: RetryConfigProperty::Unknown, + phantom_item: PhantomData, + phantom_error: PhantomData, + } +} + +pub struct RetryConfig { + operation_name: String, + logger: Logger, + condition: RetryIf, + log_after: u64, + warn_after: u64, + limit: RetryConfigProperty, + phantom_item: PhantomData, + phantom_error: PhantomData, +} + +impl RetryConfig +where + I: Send, + E: Debug + Send + Send + Sync + 'static, +{ + /// Sets a function used to determine if a retry is needed. + /// Note: timeouts always trigger a retry. + /// + /// Overrides the default behaviour of retrying on any `Err`. + pub fn when

(mut self, predicate: P) -> Self + where + P: Fn(&Result) -> bool + Send + Sync + 'static, + { + self.condition = RetryIf::Predicate(Box::new(predicate)); + self + } + + /// Only log retries after `min_attempts` failed attempts. + pub fn log_after(mut self, min_attempts: u64) -> Self { + self.log_after = min_attempts; + self + } + + pub fn warn_after(mut self, min_attempts: u64) -> Self { + self.warn_after = min_attempts; + self + } + + /// Never log failed attempts. + /// May still log at `trace` logging level. + pub fn no_logging(mut self) -> Self { + self.log_after = u64::max_value(); + self.warn_after = u64::max_value(); + self + } + + /// Set a limit on how many retry attempts to make. + pub fn limit(mut self, limit: usize) -> Self { + self.limit.set(limit); + self + } + + /// Allow unlimited retry attempts. + pub fn no_limit(mut self) -> Self { + self.limit.clear(); + self + } + + /// Set how long (in seconds) to wait for an attempt to complete before giving up on that + /// attempt. + pub fn timeout_secs(self, timeout_secs: u64) -> RetryConfigWithTimeout { + self.timeout(Duration::from_secs(timeout_secs)) + } + + /// Set how long (in milliseconds) to wait for an attempt to complete before giving up on that + /// attempt. + pub fn timeout_millis(self, timeout_ms: u64) -> RetryConfigWithTimeout { + self.timeout(Duration::from_millis(timeout_ms)) + } + + /// Set how long to wait for an attempt to complete before giving up on that attempt. + pub fn timeout(self, timeout: Duration) -> RetryConfigWithTimeout { + RetryConfigWithTimeout { + inner: self, + timeout, + } + } + + /// Allow attempts to take as long as they need (or potentially hang forever). + pub fn no_timeout(self) -> RetryConfigNoTimeout { + RetryConfigNoTimeout { inner: self } + } +} + +pub struct RetryConfigWithTimeout { + inner: RetryConfig, + timeout: Duration, +} + +impl RetryConfigWithTimeout +where + I: Debug + Send + 'static, + E: Debug + Send + Send + Sync + 'static, +{ + /// Rerun the provided function as many times as needed. + pub fn run(self, mut try_it: F) -> impl Future>> + where + F: FnMut() -> R + Send + 'static, + R: Future> + Send + 'static, + { + let operation_name = self.inner.operation_name; + let logger = self.inner.logger.clone(); + let condition = self.inner.condition; + let log_after = self.inner.log_after; + let warn_after = self.inner.warn_after; + let limit_opt = self.inner.limit.unwrap(&operation_name, "limit"); + let timeout = self.timeout; + + trace!(logger, "Run with retry: {}", operation_name); + + run_retry( + operation_name, + logger, + condition, + log_after, + warn_after, + limit_opt, + move || { + try_it() + .timeout(timeout) + .map_err(|_| TimeoutError::Elapsed) + .and_then(|res| std::future::ready(res.map_err(TimeoutError::Inner))) + .boxed() + }, + ) + } +} + +pub struct RetryConfigNoTimeout { + inner: RetryConfig, +} + +impl RetryConfigNoTimeout { + /// Rerun the provided function as many times as needed. + pub fn run(self, try_it: F) -> impl Future> + where + I: Debug + Send + 'static, + E: Debug + Send + Sync + 'static, + F: Fn() -> R + Send + 'static, + R: Future> + Send, + { + let operation_name = self.inner.operation_name; + let logger = self.inner.logger.clone(); + let condition = self.inner.condition; + let log_after = self.inner.log_after; + let warn_after = self.inner.warn_after; + let limit_opt = self.inner.limit.unwrap(&operation_name, "limit"); + + trace!(logger, "Run with retry: {}", operation_name); + + run_retry( + operation_name, + logger, + condition, + log_after, + warn_after, + limit_opt, + // No timeout, so all errors are inner errors + move || try_it().map_err(TimeoutError::Inner), + ) + .map_err(|e| { + // No timeout, so all errors are inner errors + e.into_inner().unwrap() + }) + } +} + +#[derive(Error, Debug)] +pub enum TimeoutError { + #[error("{0:?}")] + Inner(T), + #[error("Timeout elapsed")] + Elapsed, +} + +impl TimeoutError { + pub fn is_elapsed(&self) -> bool { + match self { + TimeoutError::Inner(_) => false, + TimeoutError::Elapsed => true, + } + } + + pub fn into_inner(self) -> Option { + match self { + TimeoutError::Inner(x) => Some(x), + TimeoutError::Elapsed => None, + } + } +} + +fn run_retry( + operation_name: String, + logger: Logger, + condition: RetryIf, + log_after: u64, + warn_after: u64, + limit_opt: Option, + mut try_it_with_timeout: F, +) -> impl Future>> + Send +where + O: Debug + Send + 'static, + E: Debug + Send + Sync + 'static, + F: FnMut() -> R + Send + 'static, + R: Future>> + Send, +{ + let condition = Arc::new(condition); + + let mut attempt_count = 0; + + Retry::spawn(retry_strategy(limit_opt), move || { + let operation_name = operation_name.clone(); + let logger = logger.clone(); + let condition = condition.clone(); + + attempt_count += 1; + + try_it_with_timeout().then(move |result_with_timeout| { + let is_elapsed = result_with_timeout + .as_ref() + .err() + .map(TimeoutError::is_elapsed) + .unwrap_or(false); + + if is_elapsed { + if attempt_count >= log_after { + debug!( + logger, + "Trying again after {} timed out (attempt #{})", + &operation_name, + attempt_count, + ); + } + + // Wrap in Err to force retry + std::future::ready(Err(result_with_timeout)) + } else { + // Any error must now be an inner error. + // Unwrap the inner error so that the predicate doesn't need to think + // about timeout::Error. + let result = result_with_timeout.map_err(|e| e.into_inner().unwrap()); + + // If needs retry + if condition.check(&result) { + if attempt_count >= warn_after { + // This looks like it would be nice to de-duplicate, but if we try + // to use log! slog complains about requiring a const for the log level + // See also b05e1594-e408-4047-aefb-71fc60d70e8f + warn!( + logger, + "Trying again after {} failed (attempt #{}) with result {:?}", + &operation_name, + attempt_count, + result + ); + } else if attempt_count >= log_after { + // See also b05e1594-e408-4047-aefb-71fc60d70e8f + debug!( + logger, + "Trying again after {} failed (attempt #{}) with result {:?}", + &operation_name, + attempt_count, + result + ); + } + + // Wrap in Err to force retry + std::future::ready(Err(result.map_err(TimeoutError::Inner))) + } else { + // Wrap in Ok to prevent retry + std::future::ready(Ok(result.map_err(TimeoutError::Inner))) + } + } + }) + }) + .boxed() + .then(|retry_result| async { + // Unwrap the inner result. + // The outer Ok/Err is only used for retry control flow. + match retry_result { + Ok(r) => r, + Err(e) => e, + } + }) +} + +fn retry_strategy(limit_opt: Option) -> Box + Send> { + // Exponential backoff, but with a maximum + let max_delay_ms = 30_000; + let backoff = ExponentialBackoff::from_millis(2) + .max_delay(Duration::from_millis(max_delay_ms)) + .map(jitter); + + // Apply limit (maximum retry count) + match limit_opt { + Some(limit) => { + // Items are delays *between* attempts, + // so subtract 1 from limit. + Box::new(backoff.take(limit - 1)) + } + None => Box::new(backoff), + } +} + +enum RetryIf { + Error, + Predicate(Box) -> bool + Send + Sync>), +} + +impl RetryIf { + fn check(&self, result: &Result) -> bool { + match *self { + RetryIf::Error => result.is_err(), + RetryIf::Predicate(ref pred) => pred(result), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RetryConfigProperty { + /// Property was explicitly set + Set(V), + + /// Property was explicitly unset + Clear, + + /// Property was not explicitly set or unset + Unknown, +} + +impl RetryConfigProperty +where + V: PartialEq + Eq, +{ + fn set(&mut self, v: V) { + if *self != RetryConfigProperty::Unknown { + panic!("Retry config properties must be configured only once"); + } + + *self = RetryConfigProperty::Set(v); + } + + fn clear(&mut self) { + if *self != RetryConfigProperty::Unknown { + panic!("Retry config properties must be configured only once"); + } + + *self = RetryConfigProperty::Clear; + } + + fn unwrap(self, operation_name: &str, property_name: &str) -> Option { + match self { + RetryConfigProperty::Set(v) => Some(v), + RetryConfigProperty::Clear => None, + RetryConfigProperty::Unknown => panic!( + "Retry helper for {} must have {} parameter configured", + operation_name, property_name + ), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use futures::future; + use futures03::compat::Future01CompatExt; + use slog::o; + use std::sync::Mutex; + + #[test] + fn test() { + let logger = Logger::root(::slog::Discard, o!()); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let result = runtime.block_on(async { + let c = Mutex::new(0); + retry("test", &logger) + .no_logging() + .no_limit() + .no_timeout() + .run(move || { + let mut c_guard = c.lock().unwrap(); + *c_guard += 1; + + if *c_guard >= 10 { + future::ok(*c_guard).compat() + } else { + future::err(()).compat() + } + }) + .await + }); + assert_eq!(result, Ok(10)); + } + + #[test] + fn limit_reached() { + let logger = Logger::root(::slog::Discard, o!()); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let result = runtime.block_on({ + let c = Mutex::new(0); + retry("test", &logger) + .no_logging() + .limit(5) + .no_timeout() + .run(move || { + let mut c_guard = c.lock().unwrap(); + *c_guard += 1; + + if *c_guard >= 10 { + future::ok(*c_guard).compat() + } else { + future::err(*c_guard).compat() + } + }) + }); + assert_eq!(result, Err(5)); + } + + #[test] + fn limit_not_reached() { + let logger = Logger::root(::slog::Discard, o!()); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let result = runtime.block_on({ + let c = Mutex::new(0); + retry("test", &logger) + .no_logging() + .limit(20) + .no_timeout() + .run(move || { + let mut c_guard = c.lock().unwrap(); + *c_guard += 1; + + if *c_guard >= 10 { + future::ok(*c_guard).compat() + } else { + future::err(*c_guard).compat() + } + }) + }); + assert_eq!(result, Ok(10)); + } + + #[test] + fn custom_when() { + let logger = Logger::root(::slog::Discard, o!()); + let c = Mutex::new(0); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let result = runtime.block_on({ + retry("test", &logger) + .when(|result| result.unwrap() < 10) + .no_logging() + .limit(20) + .no_timeout() + .run(move || { + let mut c_guard = c.lock().unwrap(); + *c_guard += 1; + if *c_guard > 30 { + future::err(()).compat() + } else { + future::ok(*c_guard).compat() + } + }) + }); + + assert_eq!(result, Ok(10)); + } +} diff --git a/graph/src/util/jobs.rs b/graph/src/util/jobs.rs new file mode 100644 index 0000000..d366bcc --- /dev/null +++ b/graph/src/util/jobs.rs @@ -0,0 +1,148 @@ +//! A simple job running framework for predefined jobs running repeatedly +//! at fixed intervals. This facility is not meant for work that needs +//! to be done on a tight deadline, solely for work that needs to be done +//! at reasonably long intervals (like a few hours) + +use slog::{debug, info, o, trace, warn, Logger}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; + +/// An individual job to run. Each job should be written in a way that it +/// doesn't take more than a few minutes. +#[async_trait] +pub trait Job: Send + Sync { + fn name(&self) -> &str; + async fn run(&self, logger: &Logger); +} + +struct Task { + job: Arc, + logger: Logger, + interval: Duration, + next_run: Instant, +} + +pub struct Runner { + logger: Logger, + tasks: Vec, + pub stop: Arc, +} + +impl Runner { + pub fn new(logger: &Logger) -> Runner { + Runner { + logger: logger.new(o!("component" => "JobRunner")), + tasks: Vec::new(), + stop: Arc::new(AtomicBool::new(false)), + } + } + + pub fn register(&mut self, job: Arc, interval: Duration) { + let logger = self.logger.new(o!("job" => job.name().to_owned())); + // We want tasks to start running pretty soon after server start, but + // also want to avoid that they all need to run at the same time. We + // therefore run them a small fraction of their interval from now. For + // a job that runs daily, we'd do the first run in about 15 minutes + let next_run = Instant::now() + interval / 91; + let task = Task { + job, + interval, + logger, + next_run, + }; + self.tasks.push(task); + } + + pub async fn start(mut self) { + info!( + self.logger, + "Starting job runner with {} jobs", + self.tasks.len() + ); + + for task in &self.tasks { + let next = task.next_run.saturating_duration_since(Instant::now()); + debug!(self.logger, "Schedule for {}", task.job.name(); + "interval_s" => task.interval.as_secs(), + "first_run_in_s" => next.as_secs()); + } + + while !self.stop.load(Ordering::SeqCst) { + let now = Instant::now(); + let mut next = Instant::now() + Duration::from_secs(365 * 24 * 60 * 60); + for task in self.tasks.iter_mut() { + if task.next_run < now { + // This will become obnoxious pretty quickly + trace!(self.logger, "Running job"; "name" => task.job.name()); + // We only run one job at a time since we don't want to + // deal with the same job possibly starting twice. + task.job.run(&task.logger).await; + task.next_run = Instant::now() + task.interval; + } + next = next.min(task.next_run); + } + let wait = next.saturating_duration_since(Instant::now()); + tokio::time::sleep(wait).await; + } + self.stop.store(false, Ordering::SeqCst); + warn!(self.logger, "Received request to stop. Stopping runner"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{Arc, Mutex}; + use test_store::LOGGER; + + struct CounterJob { + count: Arc>, + } + + #[async_trait] + impl Job for CounterJob { + fn name(&self) -> &str { + "counter job" + } + + async fn run(&self, _: &Logger) { + let mut count = self.count.lock().expect("Failed to lock count"); + if *count < 10 { + *count += 1; + } + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn jobs_run() { + let count = Arc::new(Mutex::new(0)); + let job = CounterJob { + count: count.clone(), + }; + let mut runner = Runner::new(&*LOGGER); + runner.register(Arc::new(job), Duration::from_millis(10)); + let stop = runner.stop.clone(); + + crate::spawn_blocking(runner.start()); + + let start = Instant::now(); + loop { + let current = { *count.lock().unwrap() }; + if current >= 10 { + break; + } + if start.elapsed() > Duration::from_secs(2) { + assert!(false, "Counting to 10 took longer than 2 seconds"); + } + } + + stop.store(true, Ordering::SeqCst); + // Wait for the runner to shut down + while stop.load(Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + } +} diff --git a/graph/src/util/lfu_cache.rs b/graph/src/util/lfu_cache.rs new file mode 100644 index 0000000..55f252c --- /dev/null +++ b/graph/src/util/lfu_cache.rs @@ -0,0 +1,334 @@ +use crate::env::ENV_VARS; +use crate::prelude::CacheWeight; +use priority_queue::PriorityQueue; +use std::cmp::Reverse; +use std::fmt::Debug; +use std::hash::{Hash, Hasher}; +use std::time::{Duration, Instant}; + +// The number of `evict` calls without access after which an entry is considered stale. +const STALE_PERIOD: u64 = 100; + +/// `PartialEq` and `Hash` are delegated to the `key`. +#[derive(Clone, Debug)] +pub struct CacheEntry { + weight: usize, + key: K, + value: V, + will_stale: bool, +} + +impl PartialEq for CacheEntry { + fn eq(&self, other: &Self) -> bool { + self.key.eq(&other.key) + } +} + +impl Eq for CacheEntry {} + +impl Hash for CacheEntry { + fn hash(&self, state: &mut H) { + self.key.hash(state) + } +} + +impl CacheEntry { + fn cache_key(key: K) -> Self { + // Only the key matters for finding an entry in the cache. + CacheEntry { + key, + value: V::default(), + weight: 0, + will_stale: false, + } + } +} + +impl CacheEntry { + /// Estimate the size of a `CacheEntry` with the given key and value. Do + /// not count the size of `Self` since that is memory that is not freed + /// when the cache entry is dropped as its storage is embedded in the + /// `PriorityQueue` + fn weight(key: &K, value: &V) -> usize { + value.indirect_weight() + key.indirect_weight() + } +} + +// The priorities are `(stale, frequency)` tuples, first all stale entries will be popped and +// then non-stale entries by least frequency. +type Priority = (bool, Reverse); + +/// Statistics about what happened during cache eviction +pub struct EvictStats { + /// The weight of the cache after eviction + pub new_weight: usize, + /// The weight of the items that were evicted + pub evicted_weight: usize, + /// The number of entries after eviction + pub new_count: usize, + /// The number if entries that were evicted + pub evicted_count: usize, + /// Whether we updated the stale status of entries + pub stale_update: bool, + /// How long eviction took + pub evict_time: Duration, +} + +/// Each entry in the cache has a frequency, which is incremented by 1 on access. Entries also have +/// a weight, upon eviction first stale entries will be removed and then non-stale entries by order +/// of least frequency until the max weight is respected. This cache only removes entries on calls +/// to `evict`, so the max weight may be exceeded until `evict` is called. Every STALE_PERIOD +/// evictions entities are checked for staleness. +#[derive(Debug)] +pub struct LfuCache { + queue: PriorityQueue, Priority>, + total_weight: usize, + stale_counter: u64, + dead_weight: bool, +} + +impl Default for LfuCache { + fn default() -> Self { + LfuCache { + queue: PriorityQueue::new(), + total_weight: 0, + stale_counter: 0, + dead_weight: false, + } + } +} + +impl LfuCache { + pub fn new() -> Self { + LfuCache { + queue: PriorityQueue::new(), + total_weight: 0, + stale_counter: 0, + dead_weight: ENV_VARS.mappings.entity_cache_dead_weight, + } + } + + /// Updates and bumps freceny if already present. + pub fn insert(&mut self, key: K, value: V) { + let weight = CacheEntry::weight(&key, &value); + match self.get_mut(key.clone()) { + None => { + self.total_weight += weight; + self.queue.push( + CacheEntry { + weight, + key, + value, + will_stale: false, + }, + (false, Reverse(1)), + ); + } + Some(entry) => { + let old_weight = entry.weight; + entry.weight = weight; + entry.value = value; + self.total_weight -= old_weight; + self.total_weight += weight; + } + } + } + + #[cfg(test)] + fn weight(&self, key: K) -> usize { + let key_entry = CacheEntry::cache_key(key); + self.queue + .get(&key_entry) + .map(|(entry, _)| entry.weight) + .unwrap_or(0) + } + + fn get_mut(&mut self, key: K) -> Option<&mut CacheEntry> { + // Increment the frequency by 1 + let key_entry = CacheEntry::cache_key(key); + self.queue + .change_priority_by(&key_entry, |(s, Reverse(f))| (s, Reverse(f + 1))); + self.queue.get_mut(&key_entry).map(|x| { + x.0.will_stale = false; + x.0 + }) + } + + pub fn get(&mut self, key: &K) -> Option<&V> { + self.get_mut(key.clone()).map(|x| &x.value) + } + + pub fn remove(&mut self, key: &K) -> Option { + // `PriorityQueue` doesn't have a remove method, so emulate that by setting the priority to + // the absolute minimum and popping. + let key_entry = CacheEntry::cache_key(key.clone()); + self.queue + .change_priority(&key_entry, (true, Reverse(u64::min_value()))) + .and_then(|_| { + self.queue.pop().map(|(e, _)| { + assert_eq!(e.key, key_entry.key); + self.total_weight -= e.weight; + e.value + }) + }) + } + + pub fn contains_key(&self, key: &K) -> bool { + self.queue + .get(&CacheEntry::cache_key(key.clone())) + .is_some() + } + + pub fn is_empty(&self) -> bool { + self.queue.is_empty() + } + + pub fn len(&self) -> usize { + self.queue.len() + } + + /// Same as `evict_with_period(max_weight, STALE_PERIOD)` + pub fn evict(&mut self, max_weight: usize) -> Option { + self.evict_with_period(max_weight, STALE_PERIOD) + } + + /// Evict entries in the cache until the total weight of the cache is + /// equal to or smaller than `max_weight`. + /// + /// The return value is mostly useful for testing and diagnostics and can + /// safely ignored in normal use. It gives the sum of the weight of all + /// evicted entries, the weight before anything was evicted and the new + /// total weight of the cache, in that order, if anything was evicted + /// at all. If there was no reason to evict, `None` is returned. + pub fn evict_with_period( + &mut self, + max_weight: usize, + stale_period: u64, + ) -> Option { + if self.total_weight <= max_weight { + return None; + } + + let start = Instant::now(); + + self.stale_counter += 1; + if self.stale_counter == stale_period { + self.stale_counter = 0; + + // Entries marked `will_stale` were not accessed in this period. Properly mark them as + // stale in their priorities. Also mark all entities as `will_stale` for the _next_ + // period so that they will be marked stale next time unless they are updated or looked + // up between now and then. + for (e, p) in self.queue.iter_mut() { + p.0 = e.will_stale; + e.will_stale = true; + } + } + + let mut evicted = 0; + let old_len = self.len(); + let dead_weight = if self.dead_weight { + self.len() * (std::mem::size_of::>() + 40) + } else { + 0 + }; + while self.total_weight + dead_weight > max_weight { + let entry = self + .queue + .pop() + .expect("empty cache but total_weight > max_weight") + .0; + evicted += entry.weight; + self.total_weight -= entry.weight; + } + Some(EvictStats { + new_weight: self.total_weight, + evicted_weight: evicted, + new_count: self.len(), + evicted_count: old_len - self.len(), + stale_update: self.stale_counter == 0, + evict_time: start.elapsed(), + }) + } +} + +impl IntoIterator for LfuCache { + type Item = (CacheEntry, Priority); + type IntoIter = Box>; + + fn into_iter(self) -> Self::IntoIter { + Box::new(self.queue.into_iter()) + } +} + +impl Extend<(CacheEntry, Priority)> for LfuCache { + fn extend, Priority)>>(&mut self, iter: T) { + self.queue.extend(iter); + } +} + +#[test] +fn entity_lru_cache() { + #[derive(Default, Debug, PartialEq, Eq)] + struct Weight(usize); + + impl CacheWeight for Weight { + fn weight(&self) -> usize { + self.indirect_weight() + } + + fn indirect_weight(&self) -> usize { + self.0 + } + } + + let mut cache: LfuCache<&'static str, Weight> = LfuCache::new(); + cache.insert("panda", Weight(2)); + cache.insert("cow", Weight(1)); + let panda_weight = cache.weight("panda"); + let cow_weight = cache.weight("cow"); + + assert_eq!(cache.get(&"cow"), Some(&Weight(1))); + assert_eq!(cache.get(&"panda"), Some(&Weight(2))); + + // Nothing is evicted. + cache.evict(panda_weight + cow_weight); + assert_eq!(cache.len(), 2); + + // "cow" was accessed twice, so "panda" is evicted. + cache.get(&"cow"); + cache.evict(cow_weight); + assert!(cache.get(&"panda").is_none()); + + cache.insert("alligator", Weight(2)); + let alligator_weight = cache.weight("alligator"); + + // Give "cow" and "alligator" a high frequency. + for _ in 0..1000 { + cache.get(&"cow"); + cache.get(&"alligator"); + } + + // Insert a lion and make it weigh the same as the cow and the alligator + // together. + cache.insert("lion", Weight(0)); + let lion_weight = cache.weight("lion"); + let lion_inner_weight = cow_weight + alligator_weight - lion_weight; + cache.insert("lion", Weight(lion_inner_weight)); + let lion_weight = cache.weight("lion"); + + // Make "cow" and "alligator" stale and remove them. + for _ in 0..(2 * STALE_PERIOD) { + cache.get(&"lion"); + + // The "whale" is something to evict so the stale counter moves. + cache.insert("whale", Weight(100 * lion_weight)); + cache.evict(2 * lion_weight); + } + + // Either "cow" and "alligator" fit in the cache, or just "lion". + // "lion" will be kept, it had lower frequency but was not stale. + assert!(cache.get(&"cow").is_none()); + assert!(cache.get(&"alligator").is_none()); + assert_eq!(cache.get(&"lion"), Some(&Weight(lion_inner_weight))); +} diff --git a/graph/src/util/mem.rs b/graph/src/util/mem.rs new file mode 100644 index 0000000..b98b7d5 --- /dev/null +++ b/graph/src/util/mem.rs @@ -0,0 +1,13 @@ +use std::mem::{transmute, MaybeUninit}; + +/// Temporarily needed until MaybeUninit::write_slice is stabilized. +pub fn init_slice<'a, T>(src: &[T], dst: &'a mut [MaybeUninit]) -> &'a mut [T] +where + T: Copy, +{ + unsafe { + let uninit_src: &[MaybeUninit] = transmute(src); + dst.copy_from_slice(uninit_src); + &mut *(dst as *mut [MaybeUninit] as *mut [T]) + } +} diff --git a/graph/src/util/mod.rs b/graph/src/util/mod.rs new file mode 100644 index 0000000..8af2540 --- /dev/null +++ b/graph/src/util/mod.rs @@ -0,0 +1,31 @@ +/// Utilities for working with futures. +pub mod futures; + +/// Security utilities. +pub mod security; + +pub mod lfu_cache; + +pub mod timed_cache; + +pub mod error; + +pub mod stats; + +pub mod cache_weight; + +pub mod timed_rw_lock; + +pub mod jobs; + +/// Increasingly longer sleeps to back off some repeated operation +pub mod backoff; + +pub mod bounded_queue; + +pub mod stable_hash_glue; + +pub mod mem; + +/// Data structures instrumented with Prometheus metrics. +pub mod monitored; diff --git a/graph/src/util/monitored.rs b/graph/src/util/monitored.rs new file mode 100644 index 0000000..c0f7147 --- /dev/null +++ b/graph/src/util/monitored.rs @@ -0,0 +1,39 @@ +use prometheus::{Counter, Gauge}; +use std::collections::VecDeque; + +pub struct MonitoredVecDeque { + vec_deque: VecDeque, + depth: Gauge, + popped: Counter, +} + +impl MonitoredVecDeque { + pub fn new(depth: Gauge, popped: Counter) -> Self { + Self { + vec_deque: VecDeque::new(), + depth, + popped, + } + } + + pub fn push_back(&mut self, item: T) { + self.vec_deque.push_back(item); + self.depth.set(self.vec_deque.len() as f64); + } + + pub fn push_front(&mut self, item: T) { + self.vec_deque.push_front(item); + self.depth.set(self.vec_deque.len() as f64); + } + + pub fn pop_front(&mut self) -> Option { + let item = self.vec_deque.pop_front(); + self.depth.set(self.vec_deque.len() as f64); + self.popped.inc(); + item + } + + pub fn is_empty(&self) -> bool { + self.vec_deque.is_empty() + } +} diff --git a/graph/src/util/security.rs b/graph/src/util/security.rs new file mode 100644 index 0000000..4e19fb1 --- /dev/null +++ b/graph/src/util/security.rs @@ -0,0 +1,40 @@ +use std::fmt; +use url::Url; + +/// Helper function to redact passwords from URLs +fn display_url(url: &str) -> String { + let mut url = match Url::parse(url) { + Ok(url) => url, + Err(_) => return String::from(url), + }; + + if url.password().is_some() { + url.set_password(Some("HIDDEN_PASSWORD")) + .expect("failed to redact password"); + } + + String::from(url) +} + +pub struct SafeDisplay(pub T); + +impl fmt::Display for SafeDisplay { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // First, format the inner value + let inner = format!("{}", self.0); + + // Then, make sure to redact passwords from the inner string + write!(f, "{}", display_url(inner.as_str())) + } +} + +impl slog::Value for SafeDisplay { + fn serialize( + &self, + _rec: &slog::Record, + key: slog::Key, + serializer: &mut dyn slog::Serializer, + ) -> slog::Result { + serializer.emit_str(key, format!("{}", self).as_str()) + } +} diff --git a/graph/src/util/stable_hash_glue.rs b/graph/src/util/stable_hash_glue.rs new file mode 100644 index 0000000..8c872c4 --- /dev/null +++ b/graph/src/util/stable_hash_glue.rs @@ -0,0 +1,40 @@ +use stable_hash::{StableHash, StableHasher}; +use stable_hash_legacy::prelude::{ + StableHash as StableHashLegacy, StableHasher as StableHasherLegacy, +}; + +/// Implements StableHash and StableHashLegacy. This macro supports two forms: +/// Struct { field1, field2, ... } and Tuple(transparent). Each field supports +/// an optional modifier. For example: Tuple(transparent: AsBytes) +#[macro_export] +macro_rules! _impl_stable_hash { + ($T:ident$(<$lt:lifetime>)? {$($field:ident$(:$e:path)?),*}) => { + ::stable_hash::impl_stable_hash!($T$(<$lt>)? {$($field$(:$e)?),*}); + ::stable_hash_legacy::impl_stable_hash!($T$(<$lt>)? {$($field$(:$e)?),*}); + }; + ($T:ident$(<$lt:lifetime>)? (transparent$(:$e:path)?)) => { + ::stable_hash::impl_stable_hash!($T$(<$lt>)? (transparent$(:$e)?)); + ::stable_hash_legacy::impl_stable_hash!($T$(<$lt>)? (transparent$(:$e)?)); + }; +} +pub use crate::_impl_stable_hash as impl_stable_hash; + +pub struct AsBytes(pub T); + +impl StableHashLegacy for AsBytes +where + T: AsRef<[u8]>, +{ + fn stable_hash(&self, sequence_number: H::Seq, state: &mut H) { + stable_hash_legacy::utils::AsBytes(self.0.as_ref()).stable_hash(sequence_number, state); + } +} + +impl StableHash for AsBytes +where + T: AsRef<[u8]>, +{ + fn stable_hash(&self, field_address: H::Addr, state: &mut H) { + stable_hash::utils::AsBytes(self.0.as_ref()).stable_hash(field_address, state); + } +} diff --git a/graph/src/util/stats.rs b/graph/src/util/stats.rs new file mode 100644 index 0000000..b5e04c5 --- /dev/null +++ b/graph/src/util/stats.rs @@ -0,0 +1,222 @@ +use std::collections::VecDeque; +use std::time::{Duration, Instant}; + +use prometheus::Gauge; + +use crate::prelude::ENV_VARS; + +/// One bin of durations. The bin starts at time `start`, and we've added `count` +/// entries to it whose durations add up to `duration` +struct Bin { + start: Instant, + duration: Duration, + count: u32, +} + +impl Bin { + fn new(start: Instant) -> Self { + Self { + start, + duration: Duration::from_millis(0), + count: 0, + } + } + + /// Add a new measurement to the bin + fn add(&mut self, duration: Duration) { + self.count += 1; + self.duration += duration; + } + + /// Remove the measurements for `other` from this bin. Only used to + /// keep a running total of measurements in `MovingStats` + fn remove(&mut self, other: &Bin) { + self.count -= other.count; + self.duration -= other.duration; + } + + /// Return `true` if the average of measurements in this bin is above + /// `duration` + fn average_gt(&self, duration: Duration) -> bool { + // Compute self.duration / self.count > duration as + // self.duration > duration * self.count. If the RHS + // overflows, we assume the average would have been smaller + // than any duration + duration + .checked_mul(self.count) + .map(|rhs| self.duration > rhs) + .unwrap_or(false) + } +} + +/// Collect statistics over a moving window of size `window_size`. To keep +/// the amount of memory needed to store the values inside the window +/// constant, values are put into bins of size `bin_size`. For example, using +/// a `window_size` of 5 minutes and a bin size of one second would use +/// 300 bins. Each bin has constant size +pub struct MovingStats { + window_size: Duration, + bin_size: Duration, + /// The buffer with measurements. The back has the most recent entries, + /// and the front has the oldest entries + bins: VecDeque, + /// Sum over the values in `elements` The `start` of this bin + /// is meaningless + total: Bin, +} + +/// Create `MovingStats` that use the window and bin sizes configured in +/// the environment +impl Default for MovingStats { + fn default() -> Self { + Self::new(ENV_VARS.load_window_size, ENV_VARS.load_bin_size) + } +} + +impl MovingStats { + /// Track moving statistics over a window of `window_size` duration + /// and keep the measurements in bins of `bin_size` each. + /// + /// # Panics + /// + /// Panics if `window_size` or `bin_size` is `0`, or if `bin_size` >= + /// `window_size` + pub fn new(window_size: Duration, bin_size: Duration) -> Self { + assert!(window_size.as_millis() > 0); + assert!(bin_size.as_millis() > 0); + assert!(window_size > bin_size); + + let capacity = window_size.as_millis() as usize / bin_size.as_millis() as usize; + + MovingStats { + window_size, + bin_size, + bins: VecDeque::with_capacity(capacity), + total: Bin::new(Instant::now()), + } + } + + /// Return `true` if the average of measurements in within `window_size` + /// is above `duration` + pub fn average_gt(&self, duration: Duration) -> bool { + // Depending on how often add() is called, we should + // call expire_bins first, but that would require taking a + // `&mut self` + self.total.average_gt(duration) + } + + /// Return the average over the current window in milliseconds + pub fn average(&self) -> Option { + self.total.duration.checked_div(self.total.count) + } + + pub fn add(&mut self, duration: Duration) { + self.add_at(Instant::now(), duration); + } + + /// Add an entry with the given timestamp. Note that the entry will + /// still be added either to the current latest bin or a new + /// latest bin. It is expected that subsequent calls to `add_at` still + /// happen with monotonically increasing `now` values. If the `now` + /// values do not monotonically increase, the average calculation + /// becomes imprecise because values are expired later than they + /// should be. + pub fn add_at(&mut self, now: Instant, duration: Duration) { + let need_new_bin = self + .bins + .back() + .map(|bin| now.saturating_duration_since(bin.start) >= self.bin_size) + .unwrap_or(true); + if need_new_bin { + self.bins.push_back(Bin::new(now)); + } + self.expire_bins(now); + // unwrap is fine because we just added a bin if there wasn't one + // before + let bin = self.bins.back_mut().unwrap(); + bin.add(duration); + self.total.add(duration); + } + + fn expire_bins(&mut self, now: Instant) { + while self + .bins + .front() + .map(|existing| now.saturating_duration_since(existing.start) >= self.window_size) + .unwrap_or(false) + { + if let Some(existing) = self.bins.pop_front() { + self.total.remove(&existing); + } + } + } + + pub fn duration(&self) -> Duration { + self.total.duration + } + + /// Adds `duration` to the stats, and register the average ms to `avg_gauge`. + pub fn add_and_register(&mut self, duration: Duration, avg_gauge: &Gauge) { + let wait_avg = { + self.add(duration); + self.average() + }; + let wait_avg = wait_avg.map(|wait_avg| wait_avg.as_millis()).unwrap_or(0); + avg_gauge.set(wait_avg as f64); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{Duration, Instant}; + + #[allow(dead_code)] + fn dump_bin(msg: &str, bin: &Bin, start: Instant) { + println!( + "bin[{}]: age={}ms count={} duration={}ms", + msg, + bin.start.saturating_duration_since(start).as_millis(), + bin.count, + bin.duration.as_millis() + ); + } + + #[test] + fn add_one_const() { + let mut stats = MovingStats::new(Duration::from_secs(5), Duration::from_secs(1)); + let start = Instant::now(); + for i in 0..10 { + stats.add_at(start + Duration::from_secs(i), Duration::from_secs(1)); + } + assert_eq!(5, stats.bins.len()); + for (i, bin) in stats.bins.iter().enumerate() { + assert_eq!(1, bin.count); + assert_eq!(Duration::from_secs(1), bin.duration); + assert_eq!(Duration::from_secs(i as u64 + 5), (bin.start - start)); + } + assert_eq!(5, stats.total.count); + assert_eq!(Duration::from_secs(5), stats.total.duration); + assert!(stats.average_gt(Duration::from_millis(900))); + assert!(!stats.average_gt(Duration::from_secs(1))); + } + + #[test] + fn add_four_linear() { + let mut stats = MovingStats::new(Duration::from_secs(5), Duration::from_secs(1)); + let start = Instant::now(); + for i in 0..40 { + stats.add_at( + start + Duration::from_millis(250 * i), + Duration::from_secs(i), + ); + } + assert_eq!(5, stats.bins.len()); + for (b, bin) in stats.bins.iter().enumerate() { + assert_eq!(4, bin.count); + assert_eq!(Duration::from_secs(86 + 16 * b as u64), bin.duration); + } + assert_eq!(20, stats.total.count); + assert_eq!(Duration::from_secs(5 * 86 + 16 * 10), stats.total.duration); + } +} diff --git a/graph/src/util/timed_cache.rs b/graph/src/util/timed_cache.rs new file mode 100644 index 0000000..1d2e6c7 --- /dev/null +++ b/graph/src/util/timed_cache.rs @@ -0,0 +1,103 @@ +use std::{ + borrow::Borrow, + cmp::Eq, + collections::HashMap, + hash::Hash, + sync::{Arc, RwLock}, + time::{Duration, Instant}, +}; + +/// Caching of values for a specified amount of time +#[derive(Debug)] +struct CacheEntry { + value: Arc, + expires: Instant, +} + +/// A cache that keeps entries live for a fixed amount of time. It is assumed +/// that all that data that could possibly wind up in the cache is very small, +/// and that expired entries are replaced by an updated entry whenever expiry +/// is detected. In other words, the cache does not ever remove entries. +#[derive(Debug)] +pub struct TimedCache { + ttl: Duration, + entries: RwLock>>, +} + +impl TimedCache { + pub fn new(ttl: Duration) -> Self { + Self { + ttl, + entries: RwLock::new(HashMap::new()), + } + } + + /// Return the entry for `key` if it exists and is not expired yet, and + /// return `None` otherwise. Note that expired entries stay in the cache + /// as it is assumed that, after returning `None`, the caller will + /// immediately overwrite that entry with a call to `set` + pub fn get(&self, key: &Q) -> Option> + where + K: Borrow + Eq + Hash, + Q: Hash + Eq, + { + self.get_at(key, Instant::now()) + } + + fn get_at(&self, key: &Q, now: Instant) -> Option> + where + K: Borrow + Eq + Hash, + Q: Hash + Eq, + { + match self.entries.read().unwrap().get(key) { + Some(CacheEntry { value, expires }) if expires >= &now => Some(value.clone()), + _ => None, + } + } + + /// Associate `key` with `value` in the cache. The `value` will be + /// valid for `self.ttl` duration + pub fn set(&self, key: K, value: Arc) + where + K: Eq + Hash, + { + self.set_at(key, value, Instant::now()) + } + + fn set_at(&self, key: K, value: Arc, now: Instant) + where + K: Eq + Hash, + { + let entry = CacheEntry { + value, + expires: now + self.ttl, + }; + self.entries.write().unwrap().insert(key, entry); + } + + pub fn clear(&self) { + self.entries.write().unwrap().clear(); + } + + pub fn find(&self, pred: F) -> Option> + where + F: Fn(&V) -> bool, + { + self.entries + .read() + .unwrap() + .values() + .find(move |entry| pred(entry.value.as_ref())) + .map(|entry| entry.value.clone()) + } +} + +#[test] +fn cache() { + const KEY: &str = "one"; + let cache = TimedCache::::new(Duration::from_millis(10)); + let now = Instant::now(); + cache.set_at(KEY.to_string(), Arc::new("value".to_string()), now); + assert!(cache.get_at(KEY, now + Duration::from_millis(5)).is_some()); + assert!(cache.get_at(KEY, now + Duration::from_millis(15)).is_none()); +} diff --git a/graph/src/util/timed_rw_lock.rs b/graph/src/util/timed_rw_lock.rs new file mode 100644 index 0000000..6a9a486 --- /dev/null +++ b/graph/src/util/timed_rw_lock.rs @@ -0,0 +1,84 @@ +use parking_lot::{Mutex, RwLock}; +use slog::{warn, Logger}; +use std::time::{Duration, Instant}; + +use crate::prelude::ENV_VARS; + +/// Adds instrumentation for timing the performance of the lock. +pub struct TimedRwLock { + id: String, + lock: RwLock, + log_threshold: Duration, +} + +impl TimedRwLock { + pub fn new(x: T, id: impl Into) -> Self { + TimedRwLock { + id: id.into(), + lock: RwLock::new(x), + log_threshold: ENV_VARS.lock_contention_log_threshold, + } + } + + pub fn write(&self, logger: &Logger) -> parking_lot::RwLockWriteGuard { + loop { + let mut elapsed = Duration::from_secs(0); + match self.lock.try_write_for(self.log_threshold) { + Some(guard) => break guard, + None => { + elapsed += self.log_threshold; + warn!(logger, "Write lock taking a long time to acquire"; + "id" => &self.id, + "wait_ms" => elapsed.as_millis(), + ); + } + } + } + } + + pub fn read(&self, logger: &Logger) -> parking_lot::RwLockReadGuard { + loop { + let mut elapsed = Duration::from_secs(0); + match self.lock.try_read_for(self.log_threshold) { + Some(guard) => break guard, + None => { + elapsed += self.log_threshold; + warn!(logger, "Read lock taking a long time to acquire"; + "id" => &self.id, + "wait_ms" => elapsed.as_millis(), + ); + } + } + } + } +} + +/// Adds instrumentation for timing the performance of the lock. +pub struct TimedMutex { + id: String, + lock: Mutex, + log_threshold: Duration, +} + +impl TimedMutex { + pub fn new(x: T, id: impl Into) -> Self { + TimedMutex { + id: id.into(), + lock: Mutex::new(x), + log_threshold: ENV_VARS.lock_contention_log_threshold, + } + } + + pub fn lock(&self, logger: &Logger) -> parking_lot::MutexGuard { + let start = Instant::now(); + let guard = self.lock.lock(); + let elapsed = start.elapsed(); + if elapsed > self.log_threshold { + warn!(logger, "Mutex lock took a long time to acquire"; + "id" => &self.id, + "wait_ms" => elapsed.as_millis(), + ); + } + guard + } +} diff --git a/graph/tests/entity_cache.rs b/graph/tests/entity_cache.rs new file mode 100644 index 0000000..10935fd --- /dev/null +++ b/graph/tests/entity_cache.rs @@ -0,0 +1,350 @@ +use async_trait::async_trait; +use graph::blockchain::block_stream::FirehoseCursor; +use graph::blockchain::BlockPtr; +use graph::data::subgraph::schema::{SubgraphError, SubgraphHealth}; +use graph::prelude::{Schema, StopwatchMetrics, StoreError, UnfailOutcome}; +use lazy_static::lazy_static; +use slog::Logger; +use std::collections::BTreeMap; +use std::sync::Arc; + +use graph::components::store::{ + EntityKey, EntityType, ReadStore, StoredDynamicDataSource, WritableStore, +}; +use graph::{ + components::store::{DeploymentId, DeploymentLocator}, + prelude::{anyhow, DeploymentHash, Entity, EntityCache, EntityModification, Value}, +}; + +lazy_static! { + static ref SUBGRAPH_ID: DeploymentHash = DeploymentHash::new("entity_cache").unwrap(); + static ref DEPLOYMENT: DeploymentLocator = + DeploymentLocator::new(DeploymentId::new(-12), SUBGRAPH_ID.clone()); + static ref SCHEMA: Arc = Arc::new( + Schema::parse( + " + type Band @entity { + id: ID! + name: String! + founded: Int + label: String + } + ", + SUBGRAPH_ID.clone(), + ) + .expect("Test schema invalid") + ); +} + +struct MockStore { + get_many_res: BTreeMap>, +} + +impl MockStore { + fn new(get_many_res: BTreeMap>) -> Self { + Self { get_many_res } + } +} + +impl ReadStore for MockStore { + fn get(&self, key: &EntityKey) -> Result, StoreError> { + match self.get_many_res.get(&key.entity_type) { + Some(entities) => Ok(entities + .iter() + .find(|entity| entity.id().ok().as_deref() == Some(key.entity_id.as_str())) + .cloned()), + None => Err(StoreError::Unknown(anyhow!( + "nothing for type {}", + key.entity_type + ))), + } + } + + fn get_many( + &self, + _ids_for_type: BTreeMap<&EntityType, Vec<&str>>, + ) -> Result>, StoreError> { + Ok(self.get_many_res.clone()) + } + + fn input_schema(&self) -> Arc { + SCHEMA.clone() + } +} + +#[async_trait] +impl WritableStore for MockStore { + fn block_ptr(&self) -> Option { + unimplemented!() + } + + fn block_cursor(&self) -> FirehoseCursor { + unimplemented!() + } + + async fn start_subgraph_deployment(&self, _: &Logger) -> Result<(), StoreError> { + unimplemented!() + } + + async fn revert_block_operations( + &self, + _: BlockPtr, + _: FirehoseCursor, + ) -> Result<(), StoreError> { + unimplemented!() + } + + async fn unfail_deterministic_error( + &self, + _: &BlockPtr, + _: &BlockPtr, + ) -> Result { + unimplemented!() + } + + fn unfail_non_deterministic_error(&self, _: &BlockPtr) -> Result { + unimplemented!() + } + + async fn fail_subgraph(&self, _: SubgraphError) -> Result<(), StoreError> { + unimplemented!() + } + + async fn supports_proof_of_indexing(&self) -> Result { + unimplemented!() + } + + async fn transact_block_operations( + &self, + _: BlockPtr, + _: FirehoseCursor, + _: Vec, + _: &StopwatchMetrics, + _: Vec, + _: Vec, + _: Vec<(u32, String)>, + _: Vec, + ) -> Result<(), StoreError> { + unimplemented!() + } + + async fn is_deployment_synced(&self) -> Result { + unimplemented!() + } + + fn unassign_subgraph(&self) -> Result<(), StoreError> { + unimplemented!() + } + + async fn load_dynamic_data_sources( + &self, + _manifest_idx_and_name: Vec<(u32, String)>, + ) -> Result, StoreError> { + unimplemented!() + } + + fn deployment_synced(&self) -> Result<(), StoreError> { + unimplemented!() + } + + fn shard(&self) -> &str { + unimplemented!() + } + + async fn health(&self) -> Result { + unimplemented!() + } + + async fn flush(&self) -> Result<(), StoreError> { + unimplemented!() + } +} + +fn make_band(id: &'static str, data: Vec<(&str, Value)>) -> (EntityKey, Entity) { + ( + EntityKey { + entity_type: EntityType::new("Band".to_string()), + entity_id: id.into(), + }, + Entity::from(data), + ) +} + +fn sort_by_entity_key(mut mods: Vec) -> Vec { + mods.sort_by_key(|m| m.entity_ref().clone()); + mods +} + +#[tokio::test] +async fn empty_cache_modifications() { + let store = Arc::new(MockStore::new(BTreeMap::new())); + let cache = EntityCache::new(store.clone()); + let result = cache.as_modifications(); + assert_eq!(result.unwrap().modifications, vec![]); +} + +#[test] +fn insert_modifications() { + // Return no entities from the store, forcing the cache to treat any `set` + // operation as an insert. + let store = MockStore::new(BTreeMap::new()); + + let store = Arc::new(store); + let mut cache = EntityCache::new(store.clone()); + + let (mogwai_key, mogwai_data) = make_band( + "mogwai", + vec![("id", "mogwai".into()), ("name", "Mogwai".into())], + ); + cache.set(mogwai_key.clone(), mogwai_data.clone()).unwrap(); + + let (sigurros_key, sigurros_data) = make_band( + "sigurros", + vec![("id", "sigurros".into()), ("name", "Sigur Ros".into())], + ); + cache + .set(sigurros_key.clone(), sigurros_data.clone()) + .unwrap(); + + let result = cache.as_modifications(); + assert_eq!( + sort_by_entity_key(result.unwrap().modifications), + sort_by_entity_key(vec![ + EntityModification::Insert { + key: mogwai_key, + data: mogwai_data, + }, + EntityModification::Insert { + key: sigurros_key, + data: sigurros_data, + } + ]) + ); +} + +fn entity_version_map( + entity_type: &str, + entities: Vec, +) -> BTreeMap> { + let mut map = BTreeMap::new(); + map.insert(EntityType::from(entity_type), entities); + map +} + +#[test] +fn overwrite_modifications() { + // Pre-populate the store with entities so that the cache treats + // every set operation as an overwrite. + let store = { + let entities = vec![ + make_band( + "mogwai", + vec![("id", "mogwai".into()), ("name", "Mogwai".into())], + ) + .1, + make_band( + "sigurros", + vec![("id", "sigurros".into()), ("name", "Sigur Ros".into())], + ) + .1, + ]; + MockStore::new(entity_version_map("Band", entities)) + }; + + let store = Arc::new(store); + let mut cache = EntityCache::new(store.clone()); + + let (mogwai_key, mogwai_data) = make_band( + "mogwai", + vec![ + ("id", "mogwai".into()), + ("name", "Mogwai".into()), + ("founded", 1995.into()), + ], + ); + cache.set(mogwai_key.clone(), mogwai_data.clone()).unwrap(); + + let (sigurros_key, sigurros_data) = make_band( + "sigurros", + vec![ + ("id", "sigurros".into()), + ("name", "Sigur Ros".into()), + ("founded", 1994.into()), + ], + ); + cache + .set(sigurros_key.clone(), sigurros_data.clone()) + .unwrap(); + + let result = cache.as_modifications(); + assert_eq!( + sort_by_entity_key(result.unwrap().modifications), + sort_by_entity_key(vec![ + EntityModification::Overwrite { + key: mogwai_key, + data: mogwai_data, + }, + EntityModification::Overwrite { + key: sigurros_key, + data: sigurros_data, + } + ]) + ); +} + +#[test] +fn consecutive_modifications() { + // Pre-populate the store with data so that we can test setting a field to + // `Value::Null`. + let store = { + let entities = vec![ + make_band( + "mogwai", + vec![ + ("id", "mogwai".into()), + ("name", "Mogwai".into()), + ("label", "Chemikal Underground".into()), + ], + ) + .1, + ]; + + MockStore::new(entity_version_map("Band", entities)) + }; + + let store = Arc::new(store); + let mut cache = EntityCache::new(store.clone()); + + // First, add "founded" and change the "label". + let (update_key, update_data) = make_band( + "mogwai", + vec![ + ("id", "mogwai".into()), + ("founded", 1995.into()), + ("label", "Rock Action Records".into()), + ], + ); + cache.set(update_key.clone(), update_data.clone()).unwrap(); + + // Then, just reset the "label". + let (update_key, update_data) = make_band( + "mogwai", + vec![("id", "mogwai".into()), ("label", Value::Null)], + ); + cache.set(update_key.clone(), update_data.clone()).unwrap(); + + // We expect a single overwrite modification for the above that leaves "id" + // and "name" untouched, sets "founded" and removes the "label" field. + let result = cache.as_modifications(); + assert_eq!( + sort_by_entity_key(result.unwrap().modifications), + sort_by_entity_key(vec![EntityModification::Overwrite { + key: update_key, + data: Entity::from(vec![ + ("id", "mogwai".into()), + ("name", "Mogwai".into()), + ("founded", 1995.into()), + ]), + },]) + ); +} diff --git a/graphql/Cargo.toml b/graphql/Cargo.toml new file mode 100644 index 0000000..a973e20 --- /dev/null +++ b/graphql/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "graph-graphql" +version = "0.27.0" +edition = "2021" + +[dependencies] +crossbeam = "0.8" +graph = { path = "../graph" } +graphql-parser = "0.4.0" +graphql-tools = "0.2.0" +indexmap = "1.9" +Inflector = "0.11.3" +lazy_static = "1.2.0" +stable-hash_legacy = { version = "0.3.3", package = "stable-hash" } +stable-hash = { version = "0.4.2"} +defer = "0.1" +parking_lot = "0.12" +anyhow = "1.0" +async-recursion = "1.0.0" + +[dev-dependencies] +pretty_assertions = "1.3.0" +test-store = { path = "../store/test-store" } +graph-chain-ethereum = { path = "../chain/ethereum" } diff --git a/graphql/examples/schema.rs b/graphql/examples/schema.rs new file mode 100644 index 0000000..0bf77f7 --- /dev/null +++ b/graphql/examples/schema.rs @@ -0,0 +1,40 @@ +use graphql_parser::parse_schema; +use std::env; +use std::fs; +use std::process::exit; + +use graph_graphql::schema::api::api_schema; + +pub fn usage(msg: &str) -> ! { + println!("{}", msg); + println!("usage: schema schema.graphql"); + println!("\nPrint the API schema we derive from the given input schema"); + std::process::exit(1); +} + +pub fn ensure(res: Result, msg: &str) -> T { + match res { + Ok(ok) => ok, + Err(err) => { + eprintln!("{}:\n {}", msg, err); + exit(1) + } + } +} + +pub fn main() { + let args: Vec = env::args().collect(); + let schema = match args.len() { + 0 | 1 => usage("please provide a GraphQL schema"), + 2 => args[1].clone(), + _ => usage("too many arguments"), + }; + let schema = ensure(fs::read_to_string(schema), "Can not read schema file"); + let schema = ensure( + parse_schema(&schema).map(|v| v.into_static()), + "Failed to parse schema", + ); + let schema = ensure(api_schema(&schema), "Failed to convert to API schema"); + + println!("{}", schema); +} diff --git a/graphql/src/execution/ast.rs b/graphql/src/execution/ast.rs new file mode 100644 index 0000000..2a0c19e --- /dev/null +++ b/graphql/src/execution/ast.rs @@ -0,0 +1,376 @@ +use std::collections::HashSet; + +use graph::{ + components::store::EntityType, + data::graphql::ObjectOrInterface, + prelude::{anyhow, q, r, s, ApiSchema, QueryExecutionError, ValueMap}, +}; +use graphql_parser::Pos; + +use crate::schema::ast::ObjectType; + +/// A selection set is a table that maps object types to the fields that +/// should be selected for objects of that type. The types are always +/// concrete object types, never interface or union types. When a +/// `SelectionSet` is constructed, fragments must already have been resolved +/// as it only allows using fields. +/// +/// The set of types that a `SelectionSet` can accommodate must be set at +/// the time the `SelectionSet` is constructed. It is not possible to add +/// more types to it, but it is possible to add fields for all known types +/// or only some of them +#[derive(Debug, Clone, PartialEq)] +pub struct SelectionSet { + // Map object types to the list of fields that should be selected for + // them. In most cases, this will have a single entry. If the + // `SelectionSet` is attached to a field with an interface or union + // type, it will have an entry for each object type implementing that + // interface or being part of the union + items: Vec<(ObjectType, Vec)>, +} + +impl SelectionSet { + /// Create a new `SelectionSet` that can handle the given types + pub fn new(types: Vec) -> Self { + let items = types + .into_iter() + .map(|obj_type| (obj_type, Vec::new())) + .collect(); + SelectionSet { items } + } + + /// Create a new `SelectionSet` that can handle the same types as + /// `other`, but ignore all fields from `other` + pub fn empty_from(other: &SelectionSet) -> Self { + let items = other + .items + .iter() + .map(|(name, _)| (name.clone(), Vec::new())) + .collect(); + SelectionSet { items } + } + + /// Return `true` if this selection set does not select any fields for + /// its types + pub fn is_empty(&self) -> bool { + self.items.iter().all(|(_, fields)| fields.is_empty()) + } + + /// If the selection set contains a single field across all its types, + /// return it. Otherwise, return `None` + pub fn single_field(&self) -> Option<&Field> { + let mut iter = self.items.iter(); + let field = match iter.next() { + Some((_, fields)) => { + if fields.len() != 1 { + return None; + } else { + &fields[0] + } + } + None => return None, + }; + for (_, fields) in iter { + if fields.len() != 1 { + return None; + } + if &fields[0] != field { + return None; + } + } + Some(field) + } + + /// Iterate over all types and the fields for those types + pub fn fields(&self) -> impl Iterator)> { + self.items + .iter() + .map(|(obj_type, fields)| (obj_type, fields.iter())) + } + + /// Iterate over all types and the fields that are not leaf fields, i.e. + /// whose selection sets are not empty + pub fn interior_fields( + &self, + ) -> impl Iterator)> { + self.items + .iter() + .map(|(obj_type, fields)| (obj_type, fields.iter().filter(|field| !field.is_leaf()))) + } + + /// Iterate over all fields for the given object type + pub fn fields_for( + &self, + obj_type: &ObjectType, + ) -> Result, QueryExecutionError> { + let item = self + .items + .iter() + .find(|(our_type, _)| our_type == obj_type) + .ok_or_else(|| { + // see: graphql-bug-compat + // Once queries are validated, this can become a panic since + // users won't be able to trigger this any more + QueryExecutionError::ValidationError( + None, + format!("invalid query: no fields for type `{}`", obj_type.name), + ) + })?; + Ok(item.1.iter()) + } + + /// Append the field for all the sets' types + pub fn push(&mut self, new_field: &Field) -> Result<(), QueryExecutionError> { + for (_, fields) in &mut self.items { + Self::merge_field(fields, new_field.clone())?; + } + Ok(()) + } + + /// Append the fields for all the sets' types + pub fn push_fields(&mut self, fields: Vec<&Field>) -> Result<(), QueryExecutionError> { + for field in fields { + self.push(field)?; + } + Ok(()) + } + + /// Merge `self` with the fields from `other`, which must have the same, + /// or a subset of, the types of `self`. The `directives` are added to + /// `self`'s directives so that they take precedence over existing + /// directives with the same name + pub fn merge( + &mut self, + other: SelectionSet, + directives: Vec, + ) -> Result<(), QueryExecutionError> { + for (other_type, other_fields) in other.items { + let item = self + .items + .iter_mut() + .find(|(obj_type, _)| &other_type == obj_type) + .ok_or_else(|| { + // graphql-bug-compat: once queries are validated, this + // can become a panic since users won't be able to + // trigger this anymore + QueryExecutionError::ValidationError( + None, + format!( + "invalid query: can not merge fields because type `{}` showed up unexpectedly", + other_type.name + ), + ) + })?; + for mut other_field in other_fields { + other_field.prepend_directives(directives.clone()); + Self::merge_field(&mut item.1, other_field)?; + } + } + Ok(()) + } + + fn merge_field(fields: &mut Vec, new_field: Field) -> Result<(), QueryExecutionError> { + match fields + .iter_mut() + .find(|field| field.response_key() == new_field.response_key()) + { + Some(field) => { + // TODO: check that _field and new_field are mergeable, in + // particular that their name, directives and arguments are + // compatible + field.selection_set.merge(new_field.selection_set, vec![])?; + } + None => fields.push(new_field), + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Directive { + pub position: Pos, + pub name: String, + pub arguments: Vec<(String, r::Value)>, +} + +impl Directive { + /// Looks up the value of an argument of this directive + pub fn argument_value(&self, name: &str) -> Option<&r::Value> { + self.arguments + .iter() + .find(|(n, _)| n == name) + .map(|(_, v)| v) + } + + fn eval_if(&self) -> bool { + match self.argument_value("if") { + None => true, + Some(r::Value::Boolean(b)) => *b, + Some(_) => false, + } + } + + /// Return `true` if this directive says that we should not include the + /// field it is attached to. That is the case if the directive is + /// `include` and its `if` condition is `false`, or if it is `skip` and + /// its `if` condition is `true`. In all other cases, return `false` + pub fn skip(&self) -> bool { + match self.name.as_str() { + "include" => !self.eval_if(), + "skip" => self.eval_if(), + _ => false, + } + } +} + +/// A field to execute as part of a query. When the field is constructed by +/// `Query::new`, variables are interpolated, and argument values have +/// already been coerced to the appropriate types for the field argument +#[derive(Debug, Clone, PartialEq)] +pub struct Field { + pub position: Pos, + pub alias: Option, + pub name: String, + pub arguments: Vec<(String, r::Value)>, + pub directives: Vec, + pub selection_set: SelectionSet, +} + +impl Field { + /// Returns the response key of a field, which is either its name or its + /// alias (if there is one). + pub fn response_key(&self) -> &str { + self.alias.as_deref().unwrap_or(self.name.as_str()) + } + + /// Looks up the value of an argument for this field + pub fn argument_value(&self, name: &str) -> Option<&r::Value> { + self.arguments + .iter() + .find(|(n, _)| n == name) + .map(|(_, v)| v) + } + + fn prepend_directives(&mut self, mut directives: Vec) { + // TODO: check that the new directives don't conflict with existing + // directives + std::mem::swap(&mut self.directives, &mut directives); + self.directives.extend(directives); + } + + fn is_leaf(&self) -> bool { + self.selection_set.is_empty() + } +} + +impl ValueMap for Field { + fn get_required(&self, key: &str) -> Result { + self.argument_value(key) + .ok_or_else(|| anyhow!("Required field `{}` not set", key)) + .and_then(T::try_from_value) + } + + fn get_optional( + &self, + key: &str, + ) -> Result, anyhow::Error> { + self.argument_value(key) + .map_or(Ok(None), |value| match value { + r::Value::Null => Ok(None), + _ => T::try_from_value(value).map(Some), + }) + } +} + +/// A set of object types, generated from resolving interfaces into the +/// object types that implement them, and possibly narrowing further when +/// expanding fragments with type conditions +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum ObjectTypeSet { + Any, + Only(HashSet), +} + +impl ObjectTypeSet { + pub fn convert( + schema: &ApiSchema, + type_cond: Option<&q::TypeCondition>, + ) -> Result { + match type_cond { + Some(q::TypeCondition::On(name)) => Self::from_name(schema, name), + None => Ok(ObjectTypeSet::Any), + } + } + + pub fn from_name(schema: &ApiSchema, name: &str) -> Result { + let set = resolve_object_types(schema, name)?; + Ok(ObjectTypeSet::Only(set)) + } + + fn contains(&self, obj_type: &ObjectType) -> bool { + match self { + ObjectTypeSet::Any => true, + ObjectTypeSet::Only(set) => set.contains(obj_type), + } + } + + pub fn intersect(self, other: &ObjectTypeSet) -> ObjectTypeSet { + match self { + ObjectTypeSet::Any => other.clone(), + ObjectTypeSet::Only(set) => { + ObjectTypeSet::Only(set.into_iter().filter(|ty| other.contains(ty)).collect()) + } + } + } + + /// Return a list of the object type names that are in this type set and + /// are also implementations of `current_type` + pub fn type_names( + &self, + schema: &ApiSchema, + current_type: ObjectOrInterface<'_>, + ) -> Result, QueryExecutionError> { + Ok(resolve_object_types(schema, current_type.name())? + .into_iter() + .filter(|obj_type| match self { + ObjectTypeSet::Any => true, + ObjectTypeSet::Only(set) => set.contains(obj_type), + }) + .collect()) + } +} + +/// Look up the type `name` from the schema and resolve interfaces +/// and unions until we are left with a set of concrete object types +pub(crate) fn resolve_object_types( + schema: &ApiSchema, + name: &str, +) -> Result, QueryExecutionError> { + let mut set = HashSet::new(); + match schema + .get_named_type(name) + .ok_or_else(|| QueryExecutionError::AbstractTypeError(name.to_string()))? + { + s::TypeDefinition::Interface(intf) => { + for obj_ty in &schema.types_for_interface()[&EntityType::new(intf.name.to_string())] { + let obj_ty = schema.object_type(obj_ty); + set.insert(obj_ty.into()); + } + } + s::TypeDefinition::Union(tys) => { + for ty in &tys.types { + set.extend(resolve_object_types(schema, ty)?) + } + } + s::TypeDefinition::Object(ty) => { + let ty = schema.object_type(ty); + set.insert(ty.into()); + } + s::TypeDefinition::Scalar(_) + | s::TypeDefinition::Enum(_) + | s::TypeDefinition::InputObject(_) => { + return Err(QueryExecutionError::NamedTypeError(name.to_string())); + } + } + Ok(set) +} diff --git a/graphql/src/execution/cache.rs b/graphql/src/execution/cache.rs new file mode 100644 index 0000000..9343f5e --- /dev/null +++ b/graphql/src/execution/cache.rs @@ -0,0 +1,251 @@ +use futures03::future::FutureExt; +use futures03::future::Shared; +use graph::{ + prelude::{debug, futures03, BlockPtr, CheapClone, Logger, QueryResult}, + util::timed_rw_lock::TimedMutex, +}; +use stable_hash_legacy::crypto::SetHasher; +use stable_hash_legacy::prelude::*; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::{collections::HashMap, time::Duration}; +use std::{ + collections::{hash_map::Entry, VecDeque}, + time::Instant, +}; + +use super::QueryHash; + +type Hash = ::Out; + +type PinFut = Pin + 'static + Send>>; +/// Cache that keeps a result around as long as it is still being processed. +/// The cache ensures that the query is not re-entrant, so multiple consumers +/// of identical queries will not execute them in parallel. +/// +/// This has a lot in common with AsyncCache in the network-services repo, +/// but more specialized. +pub struct QueryCache { + cache: Arc>>>>, +} + +impl QueryCache { + pub fn new(id: impl Into) -> Self { + Self { + cache: Arc::new(TimedMutex::new(HashMap::new(), id)), + } + } + + /// Assumption: Whatever F is passed in consistently returns the same + /// value for any input - for all values of F used with this Cache. + /// + /// Returns `(value, cached)`, where `cached` is true if the value was + /// already in the cache and false otherwise. + pub async fn cached_query + Send + 'static>( + &self, + hash: Hash, + f: F, + logger: &Logger, + ) -> (R, bool) { + let f = f.boxed(); + + let (work, cached) = { + let mut cache = self.cache.lock(logger); + + match cache.entry(hash) { + Entry::Occupied(entry) => { + // This is already being worked on. + let entry = entry.get().cheap_clone(); + (entry, true) + } + Entry::Vacant(entry) => { + // New work, put it in the in-flight list. + let uncached = f.shared(); + entry.insert(uncached.clone()); + (uncached, false) + } + } + }; + + let _remove_guard = if !cached { + // Make sure to remove this from the in-flight list, even if `poll` panics. + Some(defer::defer(|| { + self.cache.lock(logger).remove(&hash); + })) + } else { + None + }; + + (work.await, cached) + } +} + +#[derive(Debug)] +struct CacheByBlock { + block: BlockPtr, + max_weight: usize, + weight: usize, + + // The value is `(result, n_hits)`. + cache: HashMap, AtomicU64)>, + total_insert_time: Duration, +} + +impl CacheByBlock { + fn new(block: BlockPtr, max_weight: usize) -> Self { + CacheByBlock { + block, + max_weight, + weight: 0, + cache: HashMap::new(), + total_insert_time: Duration::default(), + } + } + + fn get(&self, key: &QueryHash) -> Option<&Arc> { + let (value, hit_count) = self.cache.get(key)?; + hit_count.fetch_add(1, Ordering::SeqCst); + Some(value) + } + + /// Returns `true` if the insert was successful or `false` if the cache was full. + fn insert(&mut self, key: QueryHash, value: Arc, weight: usize) -> bool { + // We never try to insert errors into this cache, and always resolve some value. + assert!(!value.has_errors()); + let fits_in_cache = self.weight + weight <= self.max_weight; + if fits_in_cache { + let start = Instant::now(); + self.weight += weight; + self.cache.insert(key, (value, AtomicU64::new(0))); + self.total_insert_time += start.elapsed(); + } + fits_in_cache + } +} + +/// Organize block caches by network names. Since different networks +/// will be at different block heights, we need to keep their `CacheByBlock` +/// separate +pub struct QueryBlockCache { + shard: u8, + cache_by_network: Vec<(String, VecDeque)>, + max_weight: usize, + max_blocks: usize, +} + +impl QueryBlockCache { + pub fn new(max_blocks: usize, shard: u8, max_weight: usize) -> Self { + QueryBlockCache { + shard, + cache_by_network: Vec::new(), + max_weight, + max_blocks, + } + } + + pub fn insert( + &mut self, + network: &str, + block_ptr: BlockPtr, + key: QueryHash, + result: Arc, + weight: usize, + logger: Logger, + ) -> bool { + // Check if the cache is disabled + if self.max_blocks == 0 { + return false; + } + + // Get or insert the cache for this network. + let cache = match self + .cache_by_network + .iter_mut() + .find(|(n, _)| n == network) + .map(|(_, c)| c) + { + Some(c) => c, + None => { + self.cache_by_network + .push((network.to_owned(), VecDeque::new())); + &mut self.cache_by_network.last_mut().unwrap().1 + } + }; + + // If there is already a cache by the block of this query, just add it there. + if let Some(cache_by_block) = cache.iter_mut().find(|c| c.block == block_ptr) { + return cache_by_block.insert(key, result.cheap_clone(), weight); + } + + // We're creating a new `CacheByBlock` if: + // - There are none yet, this is the first query being cached, or + // - `block_ptr` is of higher or equal number than the most recent block in the cache. + // Otherwise this is a historical query that does not belong in the block cache. + if let Some(highest) = cache.iter().next() { + if highest.block.number > block_ptr.number { + return false; + } + }; + + if cache.len() == self.max_blocks { + // At capacity, so pop the oldest block. + // Stats are reported in a task since we don't need the lock for it. + let block = cache.pop_back().unwrap(); + let shard = self.shard; + let network = network.to_string(); + + graph::spawn(async move { + let insert_time_ms = block.total_insert_time.as_millis(); + let mut dead_inserts = 0; + let mut total_hits = 0; + for (_, hits) in block.cache.values() { + let hits = hits.load(Ordering::SeqCst); + total_hits += hits; + if hits == 0 { + dead_inserts += 1; + } + } + let n_entries = block.cache.len(); + debug!(logger, "Rotating query cache, stats for last block"; + "shard" => shard, + "network" => network, + "entries" => n_entries, + "avg_hits" => format!("{0:.2}", (total_hits as f64) / (n_entries as f64)), + "dead_inserts" => dead_inserts, + "fill_ratio" => format!("{0:.2}", (block.weight as f64) / (block.max_weight as f64)), + "avg_insert_time_ms" => format!("{0:.2}", insert_time_ms as f64 / (n_entries as f64)), + ) + }); + } + + // Create a new cache by block, insert this entry, and add it to the QUERY_CACHE. + let mut cache_by_block = CacheByBlock::new(block_ptr, self.max_weight); + let cache_insert = cache_by_block.insert(key, result, weight); + cache.push_front(cache_by_block); + cache_insert + } + + pub fn get( + &self, + network: &str, + block_ptr: &BlockPtr, + key: &QueryHash, + ) -> Option> { + if let Some(cache) = self + .cache_by_network + .iter() + .find(|(n, _)| n == network) + .map(|(_, c)| c) + { + // Iterate from the most recent block looking for a block that matches. + if let Some(cache_by_block) = cache.iter().find(|c| &c.block == block_ptr) { + if let Some(response) = cache_by_block.get(key) { + return Some(response.cheap_clone()); + } + } + } + None + } +} diff --git a/graphql/src/execution/execution.rs b/graphql/src/execution/execution.rs new file mode 100644 index 0000000..88e993a --- /dev/null +++ b/graphql/src/execution/execution.rs @@ -0,0 +1,890 @@ +use super::cache::{QueryBlockCache, QueryCache}; +use async_recursion::async_recursion; +use crossbeam::atomic::AtomicCell; +use graph::{ + data::{query::Trace, schema::META_FIELD_NAME, value::Object}, + prelude::{s, CheapClone}, + util::{lfu_cache::EvictStats, timed_rw_lock::TimedMutex}, +}; +use lazy_static::lazy_static; +use parking_lot::MutexGuard; +use std::time::Instant; +use std::{borrow::ToOwned, collections::HashSet}; + +use graph::data::graphql::*; +use graph::data::query::CacheStatus; +use graph::env::CachedSubgraphIds; +use graph::prelude::*; +use graph::util::{lfu_cache::LfuCache, stable_hash_glue::impl_stable_hash}; + +use super::QueryHash; +use crate::execution::ast as a; +use crate::introspection::{is_introspection_field, INTROSPECTION_QUERY_TYPE}; +use crate::prelude::*; +use crate::schema::ast as sast; + +lazy_static! { + // Sharded query results cache for recent blocks by network. + // The `VecDeque` works as a ring buffer with a capacity of `QUERY_CACHE_BLOCKS`. + static ref QUERY_BLOCK_CACHE: Vec> = { + let shards = ENV_VARS.graphql.query_block_cache_shards; + let blocks = ENV_VARS.graphql.query_cache_blocks; + + // The memory budget is evenly divided among blocks and their shards. + let max_weight = ENV_VARS.graphql.query_cache_max_mem / (blocks * shards as usize); + let mut caches = Vec::new(); + for i in 0..shards { + let id = format!("query_block_cache_{}", i); + caches.push(TimedMutex::new(QueryBlockCache::new(blocks, i, max_weight), id)) + } + caches + }; + static ref QUERY_HERD_CACHE: QueryCache> = QueryCache::new("query_herd_cache"); +} + +struct WeightedResult { + result: Arc, + weight: usize, +} + +impl CacheWeight for WeightedResult { + fn indirect_weight(&self) -> usize { + self.weight + } +} + +impl Default for WeightedResult { + fn default() -> Self { + WeightedResult { + result: Arc::new(QueryResult::new(Object::default())), + weight: 0, + } + } +} + +struct HashableQuery<'a> { + query_schema_id: &'a DeploymentHash, + selection_set: &'a a::SelectionSet, + block_ptr: &'a BlockPtr, +} + +// Note that the use of StableHash here is a little bit loose. In particular, +// we are converting items to a string inside here as a quick-and-dirty +// implementation. This precludes the ability to add new fields (unlikely +// anyway). So, this hash isn't really Stable in the way that the StableHash +// crate defines it. Since hashes are only persisted for this process, we don't +// need that property. The reason we are using StableHash is to get collision +// resistance and use it's foolproof API to prevent easy mistakes instead. +// +// This is also only as collision resistant insofar as the to_string impls are +// collision resistant. It is highly likely that this is ok, since these come +// from an ast. +// +// It is possible that multiple asts that are effectively the same query with +// different representations. This is considered not an issue. The worst +// possible outcome is that the same query will have multiple cache entries. +// But, the wrong result should not be served. +impl_stable_hash!(HashableQuery<'_> { + query_schema_id, + // Not stable! Uses to_string + // TODO: Performance: Save a cryptographic hash (Blake3) of the original query + // and pass it through, rather than formatting the selection set. + selection_set: format_selection_set, + block_ptr +}); + +fn format_selection_set(s: &a::SelectionSet) -> String { + format!("{:?}", s) +} + +// The key is: subgraph id + selection set + variables + fragment definitions +fn cache_key( + ctx: &ExecutionContext, + selection_set: &a::SelectionSet, + block_ptr: &BlockPtr, +) -> QueryHash { + // It is very important that all data used for the query is included. + // Otherwise, incorrect results may be returned. + let query = HashableQuery { + query_schema_id: ctx.query.schema.id(), + selection_set, + block_ptr, + }; + // Security: + // This uses the crypo stable hash because a collision would + // cause us to fetch the incorrect query response and possibly + // attest to it. A collision should be impossibly rare with the + // non-crypto version, but a determined attacker should be able + // to find one and cause disputes which we must avoid. + stable_hash::crypto_stable_hash(&query) +} + +fn lfu_cache( + logger: &Logger, + cache_key: &[u8; 32], +) -> Option>> { + lazy_static! { + static ref QUERY_LFU_CACHE: Vec>> = { + std::iter::repeat_with(|| TimedMutex::new(LfuCache::new(), "query_lfu_cache")) + .take(ENV_VARS.graphql.query_lfu_cache_shards as usize) + .collect() + }; + } + + match QUERY_LFU_CACHE.len() { + 0 => None, + n => { + let shard = (cache_key[0] as usize) % n; + Some(QUERY_LFU_CACHE[shard].lock(logger)) + } + } +} + +fn log_lfu_evict_stats( + logger: &Logger, + network: &str, + cache_key: &[u8; 32], + evict_stats: Option, +) { + let total_shards = ENV_VARS.graphql.query_lfu_cache_shards as usize; + + if total_shards > 0 { + if let Some(EvictStats { + new_weight, + evicted_weight, + new_count, + evicted_count, + stale_update, + evict_time, + }) = evict_stats + { + { + let shard = (cache_key[0] as usize) % total_shards; + let network = network.to_string(); + let logger = logger.clone(); + + graph::spawn(async move { + debug!(logger, "Evicted LFU cache"; + "shard" => shard, + "network" => network, + "entries" => new_count, + "entries_evicted" => evicted_count, + "weight" => new_weight, + "weight_evicted" => evicted_weight, + "stale_update" => stale_update, + "evict_time_ms" => evict_time.as_millis() + ) + }); + } + } + } +} +/// Contextual information passed around during query execution. +pub struct ExecutionContext +where + R: Resolver, +{ + /// The logger to use. + pub logger: Logger, + + /// The query to execute. + pub query: Arc, + + /// The resolver to use. + pub resolver: R, + + /// Time at which the query times out. + pub deadline: Option, + + /// Max value for `first`. + pub max_first: u32, + + /// Max value for `skip` + pub max_skip: u32, + + /// Records whether this was a cache hit, used for logging. + pub(crate) cache_status: AtomicCell, +} + +pub(crate) fn get_field<'a>( + object_type: impl Into>, + name: &str, +) -> Option { + if name == "__schema" || name == "__type" { + let object_type = &*INTROSPECTION_QUERY_TYPE; + sast::get_field(object_type, name).cloned() + } else { + sast::get_field(object_type, name).cloned() + } +} + +impl ExecutionContext +where + R: Resolver, +{ + pub fn as_introspection_context(&self) -> ExecutionContext { + let introspection_resolver = + IntrospectionResolver::new(&self.logger, self.query.schema.schema()); + + ExecutionContext { + logger: self.logger.cheap_clone(), + resolver: introspection_resolver, + query: self.query.cheap_clone(), + deadline: self.deadline, + max_first: std::u32::MAX, + max_skip: std::u32::MAX, + + // `cache_status` is a dead value for the introspection context. + cache_status: AtomicCell::new(CacheStatus::Miss), + } + } +} + +pub(crate) async fn execute_root_selection_set_uncached( + ctx: &ExecutionContext, + selection_set: &a::SelectionSet, + root_type: &sast::ObjectType, +) -> Result<(Object, Trace), Vec> { + // Split the top-level fields into introspection fields and + // regular data fields + let mut data_set = a::SelectionSet::empty_from(selection_set); + let mut intro_set = a::SelectionSet::empty_from(selection_set); + let mut meta_items = Vec::new(); + + for field in selection_set.fields_for(root_type)? { + // See if this is an introspection or data field. We don't worry about + // non-existent fields; those will cause an error later when we execute + // the data_set SelectionSet + if is_introspection_field(&field.name) { + intro_set.push(field)? + } else if field.name == META_FIELD_NAME || field.name == "__typename" { + meta_items.push(field) + } else { + data_set.push(field)? + } + } + + // If we are getting regular data, prefetch it from the database + let (mut values, trace) = if data_set.is_empty() && meta_items.is_empty() { + (Object::default(), Trace::None) + } else { + let (initial_data, trace) = ctx.resolver.prefetch(ctx, &data_set)?; + data_set.push_fields(meta_items)?; + ( + execute_selection_set_to_map(ctx, &data_set, root_type, initial_data).await?, + trace, + ) + }; + + // Resolve introspection fields, if there are any + if !intro_set.is_empty() { + let ictx = ctx.as_introspection_context(); + + values.extend( + execute_selection_set_to_map( + &ictx, + ctx.query.selection_set.as_ref(), + &*INTROSPECTION_QUERY_TYPE, + None, + ) + .await?, + ); + } + + Ok((values, trace)) +} + +/// Executes the root selection set of a query. +pub(crate) async fn execute_root_selection_set( + ctx: Arc>, + selection_set: Arc, + root_type: sast::ObjectType, + block_ptr: Option, +) -> Arc { + // Cache the cache key to not have to calculate it twice - once for lookup + // and once for insert. + let mut key: Option = None; + + let should_check_cache = R::CACHEABLE + && match ENV_VARS.graphql.cached_subgraph_ids { + CachedSubgraphIds::All => true, + CachedSubgraphIds::Only(ref subgraph_ids) => { + subgraph_ids.contains(ctx.query.schema.id()) + } + }; + + if should_check_cache { + if let (Some(block_ptr), Some(network)) = (block_ptr.as_ref(), &ctx.query.network) { + // JSONB and metadata queries use `BLOCK_NUMBER_MAX`. Ignore this case for two reasons: + // - Metadata queries are not cacheable. + // - Caching `BLOCK_NUMBER_MAX` would make this cache think all other blocks are old. + if block_ptr.number != BLOCK_NUMBER_MAX { + // Calculate the hash outside of the lock + let cache_key = cache_key(&ctx, &selection_set, block_ptr); + let shard = (cache_key[0] as usize) % QUERY_BLOCK_CACHE.len(); + + // Check if the response is cached, first in the recent blocks cache, + // and then in the LfuCache for historical queries + // The blocks are used to delimit how long locks need to be held + { + let cache = QUERY_BLOCK_CACHE[shard].lock(&ctx.logger); + if let Some(result) = cache.get(network, block_ptr, &cache_key) { + ctx.cache_status.store(CacheStatus::Hit); + return result; + } + } + if let Some(mut cache) = lfu_cache(&ctx.logger, &cache_key) { + if let Some(weighted) = cache.get(&cache_key) { + ctx.cache_status.store(CacheStatus::Hit); + return weighted.result.cheap_clone(); + } + } + key = Some(cache_key); + } + } + } + + let execute_ctx = ctx.cheap_clone(); + let execute_selection_set = selection_set.cheap_clone(); + let execute_root_type = root_type.cheap_clone(); + let run_query = async move { + let _permit = execute_ctx.resolver.query_permit().await; + + let logger = execute_ctx.logger.clone(); + let query_text = execute_ctx.query.query_text.cheap_clone(); + let variables_text = execute_ctx.query.variables_text.cheap_clone(); + match graph::spawn_blocking_allow_panic(move || { + let mut query_res = + QueryResult::from(graph::block_on(execute_root_selection_set_uncached( + &execute_ctx, + &execute_selection_set, + &execute_root_type, + ))); + + // Unwrap: In practice should never fail, but if it does we will catch the panic. + execute_ctx.resolver.post_process(&mut query_res).unwrap(); + query_res.deployment = Some(execute_ctx.query.schema.id().clone()); + Arc::new(query_res) + }) + .await + { + Ok(result) => result, + Err(e) => { + let e = e.into_panic(); + let e = match e + .downcast_ref::() + .map(String::as_str) + .or(e.downcast_ref::<&'static str>().copied()) + { + Some(e) => e.to_string(), + None => "panic is not a string".to_string(), + }; + error!( + logger, + "panic when processing graphql query"; + "panic" => e.to_string(), + "query" => query_text, + "variables" => variables_text, + ); + Arc::new(QueryResult::from(QueryExecutionError::Panic(e))) + } + } + }; + + let (result, herd_hit) = if let Some(key) = key { + QUERY_HERD_CACHE + .cached_query(key, run_query, &ctx.logger) + .await + } else { + (run_query.await, false) + }; + if herd_hit { + ctx.cache_status.store(CacheStatus::Shared); + } + + // Check if this query should be cached. + // Share errors from the herd cache, but don't store them in generational cache. + // In particular, there is a problem where asking for a block pointer beyond the chain + // head can cause the legitimate cache to be thrown out. + // It would be redundant to insert herd cache hits. + let no_cache = herd_hit || result.has_errors(); + if let (false, Some(key), Some(block_ptr), Some(network)) = + (no_cache, key, block_ptr, &ctx.query.network) + { + // Calculate the weight outside the lock. + let weight = result.weight(); + let shard = (key[0] as usize) % QUERY_BLOCK_CACHE.len(); + let inserted = QUERY_BLOCK_CACHE[shard].lock(&ctx.logger).insert( + network, + block_ptr, + key, + result.cheap_clone(), + weight, + ctx.logger.cheap_clone(), + ); + + if inserted { + ctx.cache_status.store(CacheStatus::Insert); + } else if let Some(mut cache) = lfu_cache(&ctx.logger, &key) { + // Results that are too old for the QUERY_BLOCK_CACHE go into the QUERY_LFU_CACHE + let max_mem = ENV_VARS.graphql.query_cache_max_mem + / ENV_VARS.graphql.query_lfu_cache_shards as usize; + + let evict_stats = + cache.evict_with_period(max_mem, ENV_VARS.graphql.query_cache_stale_period); + + log_lfu_evict_stats(&ctx.logger, network, &key, evict_stats); + + cache.insert( + key, + WeightedResult { + result: result.cheap_clone(), + weight, + }, + ); + ctx.cache_status.store(CacheStatus::Insert); + } + } + + result +} + +/// Executes a selection set, requiring the result to be of the given object type. +/// +/// Allows passing in a parent value during recursive processing of objects and their fields. +async fn execute_selection_set<'a>( + ctx: &'a ExecutionContext, + selection_set: &'a a::SelectionSet, + object_type: &sast::ObjectType, + prefetched_value: Option, +) -> Result> { + Ok(r::Value::Object( + execute_selection_set_to_map(ctx, selection_set, object_type, prefetched_value).await?, + )) +} + +async fn execute_selection_set_to_map<'a>( + ctx: &'a ExecutionContext, + selection_set: &'a a::SelectionSet, + object_type: &sast::ObjectType, + prefetched_value: Option, +) -> Result> { + let mut prefetched_object = match prefetched_value { + Some(r::Value::Object(object)) => Some(object), + Some(_) => unreachable!(), + None => None, + }; + let mut errors: Vec = Vec::new(); + let mut results = Vec::new(); + + // Gather fields that appear more than once with the same response key. + let multiple_response_keys = { + let mut multiple_response_keys = HashSet::new(); + let mut fields = HashSet::new(); + for field in selection_set.fields_for(object_type)? { + if !fields.insert(field.name.as_str()) { + multiple_response_keys.insert(field.name.as_str()); + } + } + multiple_response_keys + }; + + // Process all field groups in order + for field in selection_set.fields_for(object_type)? { + match ctx.deadline { + Some(deadline) if deadline < Instant::now() => { + errors.push(QueryExecutionError::Timeout); + break; + } + _ => (), + } + + let response_key = field.response_key(); + + // Unwrap: The query was validated to contain only valid fields. + let field_type = sast::get_field(object_type, &field.name).unwrap(); + + // Check if we have the value already. + let field_value = prefetched_object + .as_mut() + .map(|o| { + // Prefetched objects are associated to `prefetch:response_key`. + if let Some(val) = o.remove(&format!("prefetch:{}", response_key)) { + return Some(val); + } + + // Scalars and scalar lists are associated to the field name. + // If the field has more than one response key, we have to clone. + match multiple_response_keys.contains(field.name.as_str()) { + false => o.remove(&field.name), + true => o.get(&field.name).cloned(), + } + }) + .flatten(); + + if field.name.as_str() == "__typename" && field_value.is_none() { + results.push((response_key, r::Value::String(object_type.name.clone()))); + } else { + match execute_field(ctx, object_type, field_value, field, field_type).await { + Ok(v) => { + results.push((response_key, v)); + } + Err(mut e) => { + errors.append(&mut e); + } + } + } + } + + if errors.is_empty() { + let obj = Object::from_iter(results.into_iter().map(|(k, v)| (k.to_owned(), v))); + Ok(obj) + } else { + Err(errors) + } +} + +/// Executes a field. +async fn execute_field( + ctx: &ExecutionContext, + object_type: &s::ObjectType, + field_value: Option, + field: &a::Field, + field_definition: &s::Field, +) -> Result> { + resolve_field_value( + ctx, + object_type, + field_value, + field, + field_definition, + &field_definition.field_type, + ) + .and_then(|value| complete_value(ctx, field, &field_definition.field_type, value)) + .await +} + +/// Resolves the value of a field. +#[async_recursion] +async fn resolve_field_value( + ctx: &ExecutionContext, + object_type: &s::ObjectType, + field_value: Option, + field: &a::Field, + field_definition: &s::Field, + field_type: &s::Type, +) -> Result> { + match field_type { + s::Type::NonNullType(inner_type) => { + resolve_field_value( + ctx, + object_type, + field_value, + field, + field_definition, + inner_type.as_ref(), + ) + .await + } + + s::Type::NamedType(ref name) => { + resolve_field_value_for_named_type( + ctx, + object_type, + field_value, + field, + field_definition, + name, + ) + .await + } + + s::Type::ListType(inner_type) => { + resolve_field_value_for_list_type( + ctx, + object_type, + field_value, + field, + field_definition, + inner_type.as_ref(), + ) + .await + } + } +} + +/// Resolves the value of a field that corresponds to a named type. +async fn resolve_field_value_for_named_type( + ctx: &ExecutionContext, + object_type: &s::ObjectType, + field_value: Option, + field: &a::Field, + field_definition: &s::Field, + type_name: &str, +) -> Result> { + // Try to resolve the type name into the actual type + let named_type = ctx + .query + .schema + .get_named_type(type_name) + .ok_or_else(|| QueryExecutionError::NamedTypeError(type_name.to_string()))?; + match named_type { + // Let the resolver decide how the field (with the given object type) is resolved + s::TypeDefinition::Object(t) => { + ctx.resolver + .resolve_object(field_value, field, field_definition, t.into()) + .await + } + + // Let the resolver decide how values in the resolved object value + // map to values of GraphQL enums + s::TypeDefinition::Enum(t) => ctx.resolver.resolve_enum_value(field, t, field_value), + + // Let the resolver decide how values in the resolved object value + // map to values of GraphQL scalars + s::TypeDefinition::Scalar(t) => { + ctx.resolver + .resolve_scalar_value(object_type, field, t, field_value) + .await + } + + s::TypeDefinition::Interface(i) => { + ctx.resolver + .resolve_object(field_value, field, field_definition, i.into()) + .await + } + + s::TypeDefinition::Union(_) => Err(QueryExecutionError::Unimplemented("unions".to_owned())), + + s::TypeDefinition::InputObject(_) => unreachable!("input objects are never resolved"), + } + .map_err(|e| vec![e]) +} + +/// Resolves the value of a field that corresponds to a list type. +#[async_recursion] +async fn resolve_field_value_for_list_type( + ctx: &ExecutionContext, + object_type: &s::ObjectType, + field_value: Option, + field: &a::Field, + field_definition: &s::Field, + inner_type: &s::Type, +) -> Result> { + match inner_type { + s::Type::NonNullType(inner_type) => { + resolve_field_value_for_list_type( + ctx, + object_type, + field_value, + field, + field_definition, + inner_type, + ) + .await + } + + s::Type::NamedType(ref type_name) => { + let named_type = ctx + .query + .schema + .get_named_type(type_name) + .ok_or_else(|| QueryExecutionError::NamedTypeError(type_name.to_string()))?; + + match named_type { + // Let the resolver decide how the list field (with the given item object type) + // is resolved into a entities based on the (potential) parent object + s::TypeDefinition::Object(t) => ctx + .resolver + .resolve_objects(field_value, field, field_definition, t.into()) + .await + .map_err(|e| vec![e]), + + // Let the resolver decide how values in the resolved object value + // map to values of GraphQL enums + s::TypeDefinition::Enum(t) => { + ctx.resolver.resolve_enum_values(field, t, field_value) + } + + // Let the resolver decide how values in the resolved object value + // map to values of GraphQL scalars + s::TypeDefinition::Scalar(t) => { + ctx.resolver.resolve_scalar_values(field, t, field_value) + } + + s::TypeDefinition::Interface(t) => ctx + .resolver + .resolve_objects(field_value, field, field_definition, t.into()) + .await + .map_err(|e| vec![e]), + + s::TypeDefinition::Union(_) => Err(vec![QueryExecutionError::Unimplemented( + "unions".to_owned(), + )]), + + s::TypeDefinition::InputObject(_) => { + unreachable!("input objects are never resolved") + } + } + } + + // We don't support nested lists yet + s::Type::ListType(_) => Err(vec![QueryExecutionError::Unimplemented( + "nested list types".to_owned(), + )]), + } +} + +/// Ensures that a value matches the expected return type. +#[async_recursion] +async fn complete_value( + ctx: &ExecutionContext, + field: &a::Field, + field_type: &s::Type, + resolved_value: r::Value, +) -> Result> { + match field_type { + // Fail if the field type is non-null but the value is null + s::Type::NonNullType(inner_type) => { + match complete_value(ctx, field, inner_type, resolved_value).await? { + r::Value::Null => Err(vec![QueryExecutionError::NonNullError( + field.position, + field.name.to_string(), + )]), + + v => Ok(v), + } + } + + // If the resolved value is null, return null + _ if resolved_value.is_null() => Ok(resolved_value), + + // Complete list values + s::Type::ListType(inner_type) => { + match resolved_value { + // Complete list values individually + r::Value::List(mut values) => { + let mut errors = Vec::new(); + + // To avoid allocating a new vector this completes the values in place. + for value_place in &mut values { + // Put in a placeholder, complete the value, put the completed value back. + let value = std::mem::replace(value_place, r::Value::Null); + match complete_value(ctx, field, inner_type, value).await { + Ok(value) => { + *value_place = value; + } + Err(errs) => errors.extend(errs), + } + } + match errors.is_empty() { + true => Ok(r::Value::List(values)), + false => Err(errors), + } + } + + // Return field error if the resolved value for the list is not a list + _ => Err(vec![QueryExecutionError::ListValueError( + field.position, + field.name.to_string(), + )]), + } + } + + s::Type::NamedType(name) => { + let named_type = ctx.query.schema.get_named_type(name).unwrap(); + + match named_type { + // Complete scalar values + s::TypeDefinition::Scalar(scalar_type) => { + resolved_value.coerce_scalar(scalar_type).map_err(|value| { + vec![QueryExecutionError::ScalarCoercionError( + field.position, + field.name.to_owned(), + value.into(), + scalar_type.name.to_owned(), + )] + }) + } + + // Complete enum values + s::TypeDefinition::Enum(enum_type) => { + resolved_value.coerce_enum(enum_type).map_err(|value| { + vec![QueryExecutionError::EnumCoercionError( + field.position, + field.name.to_owned(), + value.into(), + enum_type.name.to_owned(), + enum_type + .values + .iter() + .map(|value| value.name.to_owned()) + .collect(), + )] + }) + } + + // Complete object types recursively + s::TypeDefinition::Object(object_type) => { + let object_type = ctx.query.schema.object_type(object_type).into(); + execute_selection_set( + ctx, + &field.selection_set, + &object_type, + Some(resolved_value), + ) + .await + } + + // Resolve interface types using the resolved value and complete the value recursively + s::TypeDefinition::Interface(_) => { + let object_type = resolve_abstract_type(ctx, named_type, &resolved_value)?; + + execute_selection_set( + ctx, + &field.selection_set, + &object_type, + Some(resolved_value), + ) + .await + } + + // Resolve union types using the resolved value and complete the value recursively + s::TypeDefinition::Union(_) => { + let object_type = resolve_abstract_type(ctx, named_type, &resolved_value)?; + + execute_selection_set( + ctx, + &field.selection_set, + &object_type, + Some(resolved_value), + ) + .await + } + + s::TypeDefinition::InputObject(_) => { + unreachable!("input objects are never resolved") + } + } + } + } +} + +/// Resolves an abstract type (interface, union) into an object type based on the given value. +fn resolve_abstract_type<'a>( + ctx: &'a ExecutionContext, + abstract_type: &s::TypeDefinition, + object_value: &r::Value, +) -> Result> { + // Let the resolver handle the type resolution, return an error if the resolution + // yields nothing + let obj_type = ctx + .resolver + .resolve_abstract_type(&ctx.query.schema, abstract_type, object_value) + .ok_or_else(|| { + vec![QueryExecutionError::AbstractTypeError( + sast::get_type_name(abstract_type).to_string(), + )] + })?; + Ok(ctx.query.schema.object_type(obj_type).into()) +} diff --git a/graphql/src/execution/mod.rs b/graphql/src/execution/mod.rs new file mode 100644 index 0000000..8e409d6 --- /dev/null +++ b/graphql/src/execution/mod.rs @@ -0,0 +1,17 @@ +mod cache; +/// Implementation of the GraphQL execution algorithm. +mod execution; +mod query; +/// Common trait for field resolvers used in the execution. +mod resolver; + +/// Our representation of a query AST +pub mod ast; + +use stable_hash_legacy::{crypto::SetHasher, StableHasher}; + +pub use self::execution::*; +pub use self::query::Query; +pub use self::resolver::Resolver; + +type QueryHash = ::Out; diff --git a/graphql/src/execution/query.rs b/graphql/src/execution/query.rs new file mode 100644 index 0000000..ddc0959 --- /dev/null +++ b/graphql/src/execution/query.rs @@ -0,0 +1,1050 @@ +use graph::data::graphql::DocumentExt as _; +use graph::data::value::Object; +use graphql_parser::Pos; +use graphql_tools::validation::rules::*; +use graphql_tools::validation::validate::{validate, ValidationPlan}; +use lazy_static::lazy_static; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::hash::{Hash, Hasher}; +use std::iter::FromIterator; +use std::sync::Arc; +use std::time::Instant; +use std::{collections::hash_map::DefaultHasher, convert::TryFrom}; + +use graph::data::graphql::{ext::TypeExt, ObjectOrInterface}; +use graph::data::query::QueryExecutionError; +use graph::data::query::{Query as GraphDataQuery, QueryVariables}; +use graph::data::schema::ApiSchema; +use graph::prelude::{ + info, o, q, r, s, warn, BlockNumber, CheapClone, GraphQLMetrics, Logger, TryFromValue, ENV_VARS, +}; + +use crate::execution::ast as a; +use crate::query::{ast as qast, ext::BlockConstraint}; +use crate::schema::ast::{self as sast}; +use crate::values::coercion; +use crate::{execution::get_field, schema::api::ErrorPolicy}; + +lazy_static! { + static ref GRAPHQL_VALIDATION_PLAN: ValidationPlan = + ValidationPlan::from(if !ENV_VARS.graphql.enable_validations { + vec![] + } else { + vec![ + Box::new(UniqueOperationNames::new()), + Box::new(LoneAnonymousOperation::new()), + Box::new(SingleFieldSubscriptions::new()), + Box::new(KnownTypeNames::new()), + Box::new(FragmentsOnCompositeTypes::new()), + Box::new(VariablesAreInputTypes::new()), + Box::new(LeafFieldSelections::new()), + Box::new(FieldsOnCorrectType::new()), + Box::new(UniqueFragmentNames::new()), + Box::new(KnownFragmentNames::new()), + Box::new(NoUnusedFragments::new()), + Box::new(OverlappingFieldsCanBeMerged::new()), + Box::new(NoFragmentsCycle::new()), + Box::new(PossibleFragmentSpreads::new()), + Box::new(NoUnusedVariables::new()), + Box::new(NoUndefinedVariables::new()), + Box::new(KnownArgumentNames::new()), + Box::new(UniqueArgumentNames::new()), + Box::new(UniqueVariableNames::new()), + Box::new(ProvidedRequiredArguments::new()), + Box::new(KnownDirectives::new()), + Box::new(VariablesInAllowedPosition::new()), + Box::new(ValuesOfCorrectType::new()), + Box::new(UniqueDirectivesPerLocation::new()), + ] + }); +} + +#[derive(Clone, Debug)] +pub enum ComplexityError { + TooDeep, + Overflow, + Invalid, + CyclicalFragment(String), +} + +#[derive(Copy, Clone)] +enum Kind { + Query, + Subscription, +} + +/// Helper to log the fields in a `SelectionSet` without cloning. Writes +/// a list of field names from the selection set separated by ';'. Using +/// ';' as a separator makes parsing the log a little easier since slog +/// uses ',' to separate key/value pairs. +/// If `SelectionSet` is `None`, log `*` to indicate that the query was +/// for the entire selection set of the query +struct SelectedFields<'a>(&'a a::SelectionSet); + +impl<'a> std::fmt::Display for SelectedFields<'a> { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { + let mut first = true; + for (obj_type, fields) in self.0.fields() { + write!(fmt, "{}:", obj_type.name)?; + for field in fields { + if first { + write!(fmt, "{}", field.response_key())?; + } else { + write!(fmt, ";{}", field.response_key())?; + } + first = false; + } + } + if first { + // There wasn't a single `q::Selection::Field` in the set. That + // seems impossible, but log '-' to be on the safe side + write!(fmt, "-")?; + } + + Ok(()) + } +} + +/// A GraphQL query that has been preprocessed and checked and is ready +/// for execution. Checking includes validating all query fields and, if +/// desired, checking the query's complexity +// +// The implementation contains various workarounds to make it compatible +// with the previous implementation when it comes to queries that are not +// fully spec compliant and should be rejected through rigorous validation +// against the GraphQL spec. Once we do validate queries, code that is +// marked with `graphql-bug-compat` can be deleted. +pub struct Query { + /// The schema against which to execute the query + pub schema: Arc, + /// The root selection set of the query. All variable references have already been resolved + pub selection_set: Arc, + /// The ShapeHash of the original query + pub shape_hash: u64, + + pub network: Option, + + pub logger: Logger, + + start: Instant, + + kind: Kind, + + /// Used only for logging; if logging is configured off, these will + /// have dummy values + pub query_text: Arc, + pub variables_text: Arc, + pub query_id: String, +} + +fn validate_query( + logger: &Logger, + query: &GraphDataQuery, + document: &s::Document, +) -> Result<(), Vec> { + let validation_errors = validate(&document, &query.document, &GRAPHQL_VALIDATION_PLAN); + + if !validation_errors.is_empty() { + if !ENV_VARS.graphql.silent_graphql_validations { + return Err(validation_errors + .into_iter() + .map(|e| { + QueryExecutionError::ValidationError( + e.locations.first().cloned(), + e.message.clone(), + ) + }) + .collect()); + } else { + warn!( + &logger, + "GraphQL Validation failure"; + "query" => &query.query_text, + "variables" => &query.variables_text, + "errors" => format!("[{:?}]", validation_errors.iter().map(|e| e.message.clone()).collect::>().join(", ")) + ); + } + } + + Ok(()) +} + +impl Query { + /// Process the raw GraphQL query `query` and prepare for executing it. + /// The returned `Query` has already been validated and, if `max_complexity` + /// is given, also checked whether it is too complex. If validation fails, + /// or the query is too complex, errors are returned + pub fn new( + logger: &Logger, + schema: Arc, + network: Option, + query: GraphDataQuery, + max_complexity: Option, + max_depth: u8, + metrics: Arc, + ) -> Result, Vec> { + let validation_phase_start = Instant::now(); + validate_query(logger, &query, &schema.document())?; + metrics.observe_query_validation(validation_phase_start.elapsed(), schema.id()); + + let mut operation = None; + let mut fragments = HashMap::new(); + for defn in query.document.definitions.into_iter() { + match defn { + q::Definition::Operation(op) => match operation { + None => operation = Some(op), + Some(_) => return Err(vec![QueryExecutionError::OperationNameRequired]), + }, + q::Definition::Fragment(frag) => { + fragments.insert(frag.name.clone(), frag); + } + } + } + let operation = operation.ok_or(QueryExecutionError::OperationNameRequired)?; + + let variables = coerce_variables(schema.as_ref(), &operation, query.variables)?; + let (kind, selection_set) = match operation { + q::OperationDefinition::Query(q::Query { selection_set, .. }) => { + (Kind::Query, selection_set) + } + // Queries can be run by just sending a selection set + q::OperationDefinition::SelectionSet(selection_set) => (Kind::Query, selection_set), + q::OperationDefinition::Subscription(q::Subscription { selection_set, .. }) => { + (Kind::Subscription, selection_set) + } + q::OperationDefinition::Mutation(_) => { + return Err(vec![QueryExecutionError::NotSupported( + "Mutations are not supported".to_owned(), + )]) + } + }; + + let query_hash = { + let mut hasher = DefaultHasher::new(); + query.query_text.hash(&mut hasher); + query.variables_text.hash(&mut hasher); + hasher.finish() + }; + let query_id = format!("{:x}-{:x}", query.shape_hash, query_hash); + let logger = logger.new(o!( + "subgraph_id" => schema.id().clone(), + "query_id" => query_id.clone() + )); + + let start = Instant::now(); + let root_type = match kind { + Kind::Query => schema.query_type.as_ref(), + Kind::Subscription => schema.subscription_type.as_ref().unwrap(), + }; + // Use an intermediate struct so we can modify the query before + // enclosing it in an Arc + let raw_query = RawQuery { + schema: schema.cheap_clone(), + variables, + selection_set, + fragments, + root_type, + }; + + // It's important to check complexity first, so `validate_fields` + // doesn't risk a stack overflow from invalid queries. We don't + // really care about the resulting complexity, only that all the + // checks that `check_complexity` performs pass successfully + let _ = raw_query.check_complexity(max_complexity, max_depth)?; + raw_query.validate_fields()?; + let selection_set = raw_query.convert()?; + + let query = Self { + schema, + selection_set: Arc::new(selection_set), + shape_hash: query.shape_hash, + kind, + network, + logger, + start, + query_text: query.query_text.cheap_clone(), + variables_text: query.variables_text.cheap_clone(), + query_id, + }; + + Ok(Arc::new(query)) + } + + /// Return the block constraint for the toplevel query field(s), merging + /// consecutive fields that have the same block constraint, while making + /// sure that the fields appear in the same order as they did in the + /// query + /// + /// Also returns the combined error policy for those fields, which is + /// `Deny` if any field is `Deny` and `Allow` otherwise. + pub fn block_constraint( + &self, + ) -> Result, Vec> + { + let mut bcs: Vec<(BlockConstraint, (a::SelectionSet, ErrorPolicy))> = Vec::new(); + + let root_type = sast::ObjectType::from(self.schema.query_type.cheap_clone()); + let mut prev_bc: Option = None; + for field in self.selection_set.fields_for(&root_type)? { + let bc = match field.argument_value("block") { + Some(bc) => BlockConstraint::try_from_value(bc).map_err(|_| { + vec![QueryExecutionError::InvalidArgumentError( + Pos::default(), + "block".to_string(), + bc.clone().into(), + )] + })?, + None => BlockConstraint::Latest, + }; + + let field_error_policy = match field.argument_value("subgraphError") { + Some(value) => ErrorPolicy::try_from(value).map_err(|_| { + vec![QueryExecutionError::InvalidArgumentError( + Pos::default(), + "subgraphError".to_string(), + value.clone().into(), + )] + })?, + None => ErrorPolicy::Deny, + }; + + let next_bc = Some(bc.clone()); + if prev_bc == next_bc { + let (selection_set, error_policy) = &mut bcs.last_mut().unwrap().1; + selection_set.push(field)?; + if field_error_policy == ErrorPolicy::Deny { + *error_policy = ErrorPolicy::Deny; + } + } else { + let mut selection_set = a::SelectionSet::empty_from(&self.selection_set); + selection_set.push(field)?; + bcs.push((bc, (selection_set, field_error_policy))) + } + prev_bc = next_bc; + } + Ok(bcs) + } + + /// Return `true` if this is a query, and not a subscription or + /// mutation + pub fn is_query(&self) -> bool { + match self.kind { + Kind::Query => true, + Kind::Subscription => false, + } + } + + /// Return `true` if this is a subscription, not a query or a mutation + pub fn is_subscription(&self) -> bool { + match self.kind { + Kind::Subscription => true, + Kind::Query => false, + } + } + + /// Log details about the overall execution of the query + pub fn log_execution(&self, block: BlockNumber) { + if ENV_VARS.log_gql_timing() { + info!( + &self.logger, + "Query timing (GraphQL)"; + "query" => &self.query_text, + "variables" => &self.variables_text, + "query_time_ms" => self.start.elapsed().as_millis(), + "block" => block, + ); + } + } + + /// Log details about how the part of the query corresponding to + /// `selection_set` was cached + pub fn log_cache_status( + &self, + selection_set: &a::SelectionSet, + block: BlockNumber, + start: Instant, + cache_status: String, + ) { + if ENV_VARS.log_gql_cache_timing() { + info!( + &self.logger, + "Query caching"; + "query_time_ms" => start.elapsed().as_millis(), + "cached" => cache_status, + "selection" => %SelectedFields(selection_set), + "block" => block, + ); + } + } +} + +/// Coerces variable values for an operation. +pub fn coerce_variables( + schema: &ApiSchema, + operation: &q::OperationDefinition, + mut variables: Option, +) -> Result, Vec> { + let mut coerced_values = HashMap::new(); + let mut errors = vec![]; + + for variable_def in qast::get_variable_definitions(operation) + .into_iter() + .flatten() + { + // Skip variable if it has an invalid type + if !schema.is_input_type(&variable_def.var_type) { + errors.push(QueryExecutionError::InvalidVariableTypeError( + variable_def.position, + variable_def.name.to_owned(), + )); + continue; + } + + let value = variables + .as_mut() + .and_then(|vars| vars.remove(&variable_def.name)); + + let value = match value.or_else(|| { + variable_def + .default_value + .clone() + .map(r::Value::try_from) + .transpose() + .unwrap() + }) { + // No variable value provided and no default for non-null type, fail + None => { + if sast::is_non_null_type(&variable_def.var_type) { + errors.push(QueryExecutionError::MissingVariableError( + variable_def.position, + variable_def.name.to_owned(), + )); + }; + continue; + } + Some(value) => value, + }; + + // We have a variable value, attempt to coerce it to the value type + // of the variable definition + coerced_values.insert( + variable_def.name.to_owned(), + coerce_variable(schema, variable_def, value)?, + ); + } + + if errors.is_empty() { + Ok(coerced_values) + } else { + Err(errors) + } +} + +fn coerce_variable( + schema: &ApiSchema, + variable_def: &q::VariableDefinition, + value: r::Value, +) -> Result> { + use crate::values::coercion::coerce_value; + + let resolver = |name: &str| schema.get_named_type(name); + + coerce_value(value, &variable_def.var_type, &resolver).map_err(|value| { + vec![QueryExecutionError::InvalidArgumentError( + variable_def.position, + variable_def.name.to_owned(), + value.into(), + )] + }) +} + +struct RawQuery<'s> { + /// The schema against which to execute the query + schema: Arc, + /// The variables for the query, coerced into proper values + variables: HashMap, + /// The root selection set of the query + selection_set: q::SelectionSet, + + fragments: HashMap, + root_type: &'s s::ObjectType, +} + +impl<'s> RawQuery<'s> { + fn check_complexity( + &self, + max_complexity: Option, + max_depth: u8, + ) -> Result> { + let complexity = self.complexity(max_depth).map_err(|e| vec![e])?; + if let Some(max_complexity) = max_complexity { + if complexity > max_complexity { + return Err(vec![QueryExecutionError::TooComplex( + complexity, + max_complexity, + )]); + } + } + Ok(complexity) + } + + fn complexity_inner<'a>( + &'a self, + ty: &s::TypeDefinition, + selection_set: &'a q::SelectionSet, + max_depth: u8, + depth: u8, + visited_fragments: &'a HashSet<&'a str>, + ) -> Result { + use ComplexityError::*; + + if depth >= max_depth { + return Err(TooDeep); + } + + selection_set + .items + .iter() + .try_fold(0, |total_complexity, selection| { + match selection { + q::Selection::Field(field) => { + // Empty selection sets are the base case. + if field.selection_set.items.is_empty() { + return Ok(total_complexity); + } + + // Get field type to determine if this is a collection query. + let s_field = match ty { + s::TypeDefinition::Object(t) => get_field(t, &field.name), + s::TypeDefinition::Interface(t) => get_field(t, &field.name), + + // `Scalar` and `Enum` cannot have selection sets. + // `InputObject` can't appear in a selection. + // `Union` is not yet supported. + s::TypeDefinition::Scalar(_) + | s::TypeDefinition::Enum(_) + | s::TypeDefinition::InputObject(_) + | s::TypeDefinition::Union(_) => None, + } + .ok_or(Invalid)?; + + let field_complexity = self.complexity_inner( + self.schema + .get_named_type(s_field.field_type.get_base_type()) + .ok_or(Invalid)?, + &field.selection_set, + max_depth, + depth + 1, + visited_fragments, + )?; + + // Non-collection queries pass through. + if !sast::is_list_or_non_null_list_field(&s_field) { + return Ok(total_complexity + field_complexity); + } + + // For collection queries, check the `first` argument. + let max_entities = qast::get_argument_value(&field.arguments, "first") + .and_then(|arg| match arg { + q::Value::Int(n) => Some(n.as_i64()? as u64), + _ => None, + }) + .unwrap_or(100); + max_entities + .checked_add( + max_entities.checked_mul(field_complexity).ok_or(Overflow)?, + ) + .ok_or(Overflow) + } + q::Selection::FragmentSpread(fragment) => { + let def = self.fragments.get(&fragment.fragment_name).unwrap(); + let q::TypeCondition::On(type_name) = &def.type_condition; + let ty = self.schema.get_named_type(type_name).ok_or(Invalid)?; + + // Copy `visited_fragments` on write. + let mut visited_fragments = visited_fragments.clone(); + if !visited_fragments.insert(&fragment.fragment_name) { + return Err(CyclicalFragment(fragment.fragment_name.clone())); + } + self.complexity_inner( + ty, + &def.selection_set, + max_depth, + depth + 1, + &visited_fragments, + ) + } + q::Selection::InlineFragment(fragment) => { + let ty = match &fragment.type_condition { + Some(q::TypeCondition::On(type_name)) => { + self.schema.get_named_type(type_name).ok_or(Invalid)? + } + _ => ty, + }; + self.complexity_inner( + ty, + &fragment.selection_set, + max_depth, + depth + 1, + visited_fragments, + ) + } + } + .and_then(|complexity| total_complexity.checked_add(complexity).ok_or(Overflow)) + }) + } + + /// See https://developer.github.com/v4/guides/resource-limitations/. + /// + /// If the query is invalid, returns `Ok(0)` so that execution proceeds and + /// gives a proper error. + fn complexity(&self, max_depth: u8) -> Result { + let root_type = self.schema.get_root_query_type_def().unwrap(); + + match self.complexity_inner( + root_type, + &self.selection_set, + max_depth, + 0, + &HashSet::new(), + ) { + Ok(complexity) => Ok(complexity), + Err(ComplexityError::Invalid) => Ok(0), + Err(ComplexityError::TooDeep) => Err(QueryExecutionError::TooDeep(max_depth)), + Err(ComplexityError::Overflow) => { + Err(QueryExecutionError::TooComplex(u64::max_value(), 0)) + } + Err(ComplexityError::CyclicalFragment(name)) => { + Err(QueryExecutionError::CyclicalFragment(name)) + } + } + } + + fn validate_fields(&self) -> Result<(), Vec> { + let root_type = self.schema.query_type.as_ref(); + + let errors = + self.validate_fields_inner(&"Query".to_owned(), root_type.into(), &self.selection_set); + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } + + // Checks for invalid selections. + fn validate_fields_inner( + &self, + type_name: &str, + ty: ObjectOrInterface<'_>, + selection_set: &q::SelectionSet, + ) -> Vec { + selection_set + .items + .iter() + .fold(vec![], |mut errors, selection| { + match selection { + q::Selection::Field(field) => match get_field(ty, &field.name) { + Some(s_field) => { + let base_type = s_field.field_type.get_base_type(); + if self.schema.get_named_type(base_type).is_none() { + errors.push(QueryExecutionError::NamedTypeError(base_type.into())); + } else if let Some(ty) = self.schema.object_or_interface(base_type) { + errors.extend(self.validate_fields_inner( + base_type, + ty, + &field.selection_set, + )) + } + } + None => errors.push(QueryExecutionError::UnknownField( + field.position, + type_name.into(), + field.name.clone(), + )), + }, + q::Selection::FragmentSpread(fragment) => { + match self.fragments.get(&fragment.fragment_name) { + Some(frag) => { + let q::TypeCondition::On(type_name) = &frag.type_condition; + match self.schema.object_or_interface(type_name) { + Some(ty) => errors.extend(self.validate_fields_inner( + type_name, + ty, + &frag.selection_set, + )), + None => errors.push(QueryExecutionError::NamedTypeError( + type_name.clone(), + )), + } + } + None => errors.push(QueryExecutionError::UndefinedFragment( + fragment.fragment_name.clone(), + )), + } + } + q::Selection::InlineFragment(fragment) => match &fragment.type_condition { + Some(q::TypeCondition::On(type_name)) => { + match self.schema.object_or_interface(type_name) { + Some(ty) => errors.extend(self.validate_fields_inner( + type_name, + ty, + &fragment.selection_set, + )), + None => errors + .push(QueryExecutionError::NamedTypeError(type_name.clone())), + } + } + _ => errors.extend(self.validate_fields_inner( + type_name, + ty, + &fragment.selection_set, + )), + }, + } + errors + }) + } + + fn convert(self) -> Result> { + let RawQuery { + schema, + variables, + selection_set, + fragments, + root_type, + } = self; + + let transform = Transform { + schema, + variables, + fragments, + }; + transform.expand_selection_set(selection_set, &a::ObjectTypeSet::Any, root_type.into()) + } +} + +struct Transform { + schema: Arc, + variables: HashMap, + fragments: HashMap, +} + +impl Transform { + /// Look up the value of the variable `name`. If the variable is not + /// defined, return `r::Value::Null` + // graphql-bug-compat: Once queries are fully validated, all variables + // will be defined + fn variable(&self, name: &str) -> r::Value { + self.variables.get(name).cloned().unwrap_or(r::Value::Null) + } + + /// Interpolate variable references in the arguments `args` + fn interpolate_arguments( + &self, + args: Vec<(String, q::Value)>, + pos: &Pos, + ) -> Vec<(String, r::Value)> { + args.into_iter() + .map(|(name, val)| { + let val = self.interpolate_value(val, pos); + (name, val) + }) + .collect() + } + + /// Turn `value` into an `r::Value` by resolving variable references + fn interpolate_value(&self, value: q::Value, pos: &Pos) -> r::Value { + match value { + q::Value::Variable(var) => self.variable(&var), + q::Value::Int(ref num) => { + r::Value::Int(num.as_i64().expect("q::Value::Int contains an i64")) + } + q::Value::Float(f) => r::Value::Float(f), + q::Value::String(s) => r::Value::String(s), + q::Value::Boolean(b) => r::Value::Boolean(b), + q::Value::Null => r::Value::Null, + q::Value::Enum(s) => r::Value::Enum(s), + q::Value::List(vals) => { + let vals = vals + .into_iter() + .map(|val| self.interpolate_value(val, pos)) + .collect(); + r::Value::List(vals) + } + q::Value::Object(map) => { + let mut rmap = BTreeMap::new(); + for (key, value) in map.into_iter() { + let value = self.interpolate_value(value, pos); + rmap.insert(key.into(), value); + } + r::Value::object(rmap) + } + } + } + + /// Interpolate variable references in directives. Return the directives + /// and a boolean indicating whether the element these directives are + /// attached to should be skipped + fn interpolate_directives( + &self, + dirs: Vec, + ) -> Result<(Vec, bool), QueryExecutionError> { + let dirs: Vec<_> = dirs + .into_iter() + .map(|dir| { + let q::Directive { + name, + position, + arguments, + } = dir; + let arguments = self.interpolate_arguments(arguments, &position); + a::Directive { + name, + position, + arguments, + } + }) + .collect(); + let skip = dirs.iter().any(|dir| dir.skip()); + Ok((dirs, skip)) + } + + /// Coerces argument values into GraphQL values. + pub fn coerce_argument_values<'a>( + &self, + arguments: &mut Vec<(String, r::Value)>, + ty: ObjectOrInterface<'a>, + field_name: &str, + ) -> Result<(), Vec> { + let mut errors = vec![]; + + let resolver = |name: &str| self.schema.get_named_type(name); + + let mut defined_args: usize = 0; + for argument_def in sast::get_argument_definitions(ty, field_name) + .into_iter() + .flatten() + { + let arg_value = arguments + .iter_mut() + .find(|arg| arg.0 == argument_def.name) + .map(|arg| &mut arg.1); + if arg_value.is_some() { + defined_args += 1; + } + match coercion::coerce_input_value( + arg_value.as_deref().cloned(), + argument_def, + &resolver, + ) { + Ok(Some(value)) => { + let value = if argument_def.name == *"text" { + r::Value::Object(Object::from_iter(vec![(field_name.to_string(), value)])) + } else { + value + }; + match arg_value { + Some(arg_value) => *arg_value = value, + None => arguments.push((argument_def.name.clone(), value)), + } + } + Ok(None) => {} + Err(e) => errors.push(e), + } + } + + // see: graphql-bug-compat + // avoids error 'unknown argument on field' + if defined_args < arguments.len() { + // `arguments` contains undefined arguments, remove them + match sast::get_argument_definitions(ty, field_name) { + None => arguments.clear(), + Some(arg_defs) => { + arguments.retain(|(name, _)| arg_defs.iter().any(|def| &def.name == name)) + } + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } + + /// Expand fragments and interpolate variables in a field. Return `None` + /// if the field should be skipped + fn expand_field( + &self, + field: q::Field, + parent_type: ObjectOrInterface<'_>, + ) -> Result, Vec> { + let q::Field { + position, + alias, + name, + arguments, + directives, + selection_set, + } = field; + + // Short-circuit '__typename' since it is not a real field + if name == "__typename" { + return Ok(Some(a::Field { + position, + alias, + name, + arguments: vec![], + directives: vec![], + selection_set: a::SelectionSet::new(vec![]), + })); + } + + let field_type = parent_type.field(&name).ok_or_else(|| { + vec![QueryExecutionError::UnknownField( + position, + parent_type.name().to_string(), + name.clone(), + )] + })?; + + let (directives, skip) = self.interpolate_directives(directives)?; + if skip { + return Ok(None); + } + + let mut arguments = self.interpolate_arguments(arguments, &position); + self.coerce_argument_values(&mut arguments, parent_type, &name)?; + + let is_leaf_type = self.schema.document().is_leaf_type(&field_type.field_type); + let selection_set = if selection_set.items.is_empty() { + if !is_leaf_type { + // see: graphql-bug-compat + // Field requires selection, ignore this field + return Ok(None); + } + a::SelectionSet::new(vec![]) + } else { + if is_leaf_type { + // see: graphql-bug-compat + // Field does not allow selections, ignore selections + a::SelectionSet::new(vec![]) + } else { + let ty = field_type.field_type.get_base_type(); + let type_set = a::ObjectTypeSet::from_name(&self.schema, ty)?; + let ty = self.schema.object_or_interface(ty).unwrap(); + self.expand_selection_set(selection_set, &type_set, ty)? + } + }; + + Ok(Some(a::Field { + position, + alias, + name, + arguments, + directives, + selection_set, + })) + } + + /// Expand fragments and interpolate variables in a selection set + fn expand_selection_set( + &self, + set: q::SelectionSet, + type_set: &a::ObjectTypeSet, + ty: ObjectOrInterface<'_>, + ) -> Result> { + let q::SelectionSet { span: _, items } = set; + // check_complexity already checked for cycles in fragment + // expansion, i.e. situations where a named fragment includes itself + // recursively. We still want to guard against spreading the same + // fragment twice at the same level in the query + let mut visited_fragments = HashSet::new(); + + // All the types that could possibly be returned by this selection set + let types = type_set.type_names(&self.schema, ty)?; + let mut newset = a::SelectionSet::new(types); + + for sel in items { + match sel { + q::Selection::Field(field) => { + if let Some(field) = self.expand_field(field, ty)? { + newset.push(&field)?; + } + } + q::Selection::FragmentSpread(spread) => { + // TODO: we ignore the directives here (and so did the + // old implementation), but that seems wrong + let q::FragmentSpread { + position: _, + fragment_name, + directives: _, + } = spread; + let frag = self.fragments.get(&fragment_name).unwrap(); + if visited_fragments.insert(fragment_name) { + let q::FragmentDefinition { + position: _, + name: _, + type_condition, + directives, + selection_set, + } = frag; + self.expand_fragment( + directives.clone(), + Some(type_condition), + type_set, + selection_set.clone(), + ty, + &mut newset, + )?; + } + } + q::Selection::InlineFragment(frag) => { + let q::InlineFragment { + position: _, + type_condition, + directives, + selection_set, + } = frag; + self.expand_fragment( + directives, + type_condition.as_ref(), + type_set, + selection_set, + ty, + &mut newset, + )?; + } + } + } + Ok(newset) + } + + fn expand_fragment( + &self, + directives: Vec, + frag_cond: Option<&q::TypeCondition>, + type_set: &a::ObjectTypeSet, + selection_set: q::SelectionSet, + ty: ObjectOrInterface, + newset: &mut a::SelectionSet, + ) -> Result<(), Vec> { + let (directives, skip) = self.interpolate_directives(directives)?; + // Field names in fragment spreads refer to this type, which will + // usually be different from the outer type + let ty = match frag_cond { + Some(q::TypeCondition::On(name)) => self + .schema + .object_or_interface(name) + .expect("type names on fragment spreads are valid"), + None => ty, + }; + if !skip { + let type_set = a::ObjectTypeSet::convert(&self.schema, frag_cond)?.intersect(type_set); + let selection_set = self.expand_selection_set(selection_set, &type_set, ty)?; + newset.merge(selection_set, directives)?; + } + Ok(()) + } +} diff --git a/graphql/src/execution/resolver.rs b/graphql/src/execution/resolver.rs new file mode 100644 index 0000000..f601b6f --- /dev/null +++ b/graphql/src/execution/resolver.rs @@ -0,0 +1,124 @@ +use graph::components::store::UnitStream; +use graph::data::query::Trace; +use graph::prelude::{async_trait, s, tokio, ApiSchema, Error, QueryExecutionError}; +use graph::{ + data::graphql::ObjectOrInterface, + prelude::{r, QueryResult}, +}; + +use crate::execution::{ast as a, ExecutionContext}; + +/// A GraphQL resolver that can resolve entities, enum values, scalar types and interfaces/unions. +#[async_trait] +pub trait Resolver: Sized + Send + Sync + 'static { + const CACHEABLE: bool; + + async fn query_permit(&self) -> Result; + + /// Prepare for executing a query by prefetching as much data as possible + fn prefetch( + &self, + ctx: &ExecutionContext, + selection_set: &a::SelectionSet, + ) -> Result<(Option, Trace), Vec>; + + /// Resolves list of objects, `prefetched_objects` is `Some` if the parent already calculated the value. + async fn resolve_objects( + &self, + prefetched_objects: Option, + field: &a::Field, + field_definition: &s::Field, + object_type: ObjectOrInterface<'_>, + ) -> Result; + + /// Resolves an object, `prefetched_object` is `Some` if the parent already calculated the value. + async fn resolve_object( + &self, + prefetched_object: Option, + field: &a::Field, + field_definition: &s::Field, + object_type: ObjectOrInterface<'_>, + ) -> Result; + + /// Resolves an enum value for a given enum type. + fn resolve_enum_value( + &self, + _field: &a::Field, + _enum_type: &s::EnumType, + value: Option, + ) -> Result { + Ok(value.unwrap_or(r::Value::Null)) + } + + /// Resolves a scalar value for a given scalar type. + async fn resolve_scalar_value( + &self, + _parent_object_type: &s::ObjectType, + _field: &a::Field, + _scalar_type: &s::ScalarType, + value: Option, + ) -> Result { + // This code is duplicated. + // See also c2112309-44fd-4a84-92a0-5a651e6ed548 + Ok(value.unwrap_or(r::Value::Null)) + } + + /// Resolves a list of enum values for a given enum type. + fn resolve_enum_values( + &self, + _field: &a::Field, + _enum_type: &s::EnumType, + value: Option, + ) -> Result> { + Ok(value.unwrap_or(r::Value::Null)) + } + + /// Resolves a list of scalar values for a given list type. + fn resolve_scalar_values( + &self, + _field: &a::Field, + _scalar_type: &s::ScalarType, + value: Option, + ) -> Result> { + Ok(value.unwrap_or(r::Value::Null)) + } + + // Resolves an abstract type into the specific type of an object. + fn resolve_abstract_type<'a>( + &self, + schema: &'a ApiSchema, + _abstract_type: &s::TypeDefinition, + object_value: &r::Value, + ) -> Option<&'a s::ObjectType> { + let concrete_type_name = match object_value { + // All objects contain `__typename` + r::Value::Object(data) => match &data.get("__typename").unwrap() { + r::Value::String(name) => name.clone(), + _ => unreachable!("__typename must be a string"), + }, + _ => unreachable!("abstract type value must be an object"), + }; + + // A name returned in a `__typename` must exist in the schema. + match schema.get_named_type(&concrete_type_name).unwrap() { + s::TypeDefinition::Object(object) => Some(object), + _ => unreachable!("only objects may implement interfaces"), + } + } + + // Resolves a change stream for a given field. + fn resolve_field_stream( + &self, + _schema: &ApiSchema, + _object_type: &s::ObjectType, + _field: &a::Field, + ) -> Result { + Err(QueryExecutionError::NotSupported(String::from( + "Resolving field streams is not supported by this resolver", + ))) + } + + fn post_process(&self, _result: &mut QueryResult) -> Result<(), Error> { + Ok(()) + } +} diff --git a/graphql/src/introspection/mod.rs b/graphql/src/introspection/mod.rs new file mode 100644 index 0000000..16b7512 --- /dev/null +++ b/graphql/src/introspection/mod.rs @@ -0,0 +1,5 @@ +mod resolver; +mod schema; + +pub use self::resolver::IntrospectionResolver; +pub use self::schema::{is_introspection_field, INTROSPECTION_DOCUMENT, INTROSPECTION_QUERY_TYPE}; diff --git a/graphql/src/introspection/resolver.rs b/graphql/src/introspection/resolver.rs new file mode 100644 index 0000000..ed95e00 --- /dev/null +++ b/graphql/src/introspection/resolver.rs @@ -0,0 +1,446 @@ +use graph::data::graphql::ext::{FieldExt, TypeDefinitionExt}; +use graph::data::query::Trace; +use graphql_parser::Pos; +use std::collections::BTreeMap; + +use graph::data::graphql::{object, DocumentExt, ObjectOrInterface}; +use graph::prelude::*; + +use crate::execution::ast as a; +use crate::prelude::*; +use crate::schema::ast as sast; + +type TypeObjectsMap = BTreeMap; + +/// Our Schema has the introspection schema mixed in. When we build the +/// `TypeObjectsMap`, suppress types and fields that belong to the +/// introspection schema +fn schema_type_objects(schema: &Schema) -> TypeObjectsMap { + sast::get_type_definitions(&schema.document) + .iter() + .filter(|def| !def.is_introspection()) + .fold(BTreeMap::new(), |mut type_objects, typedef| { + let type_name = sast::get_type_name(typedef); + if !type_objects.contains_key(type_name) { + let type_object = type_definition_object(schema, &mut type_objects, typedef); + type_objects.insert(type_name.to_owned(), type_object); + } + type_objects + }) +} + +fn type_object(schema: &Schema, type_objects: &mut TypeObjectsMap, t: &s::Type) -> r::Value { + match t { + // We store the name of the named type here to be able to resolve it dynamically later + s::Type::NamedType(s) => r::Value::String(s.to_owned()), + s::Type::ListType(ref inner) => list_type_object(schema, type_objects, inner), + s::Type::NonNullType(ref inner) => non_null_type_object(schema, type_objects, inner), + } +} + +fn list_type_object( + schema: &Schema, + type_objects: &mut TypeObjectsMap, + inner_type: &s::Type, +) -> r::Value { + object! { + kind: r::Value::Enum(String::from("LIST")), + ofType: type_object(schema, type_objects, inner_type), + } +} + +fn non_null_type_object( + schema: &Schema, + type_objects: &mut TypeObjectsMap, + inner_type: &s::Type, +) -> r::Value { + object! { + kind: r::Value::Enum(String::from("NON_NULL")), + ofType: type_object(schema, type_objects, inner_type), + } +} + +fn type_definition_object( + schema: &Schema, + type_objects: &mut TypeObjectsMap, + typedef: &s::TypeDefinition, +) -> r::Value { + let type_name = sast::get_type_name(typedef); + + type_objects.get(type_name).cloned().unwrap_or_else(|| { + let type_object = match typedef { + s::TypeDefinition::Enum(enum_type) => enum_type_object(enum_type), + s::TypeDefinition::InputObject(input_object_type) => { + input_object_type_object(schema, type_objects, input_object_type) + } + s::TypeDefinition::Interface(interface_type) => { + interface_type_object(schema, type_objects, interface_type) + } + s::TypeDefinition::Object(object_type) => { + object_type_object(schema, type_objects, object_type) + } + s::TypeDefinition::Scalar(scalar_type) => scalar_type_object(scalar_type), + s::TypeDefinition::Union(union_type) => union_type_object(schema, union_type), + }; + + type_objects.insert(type_name.to_owned(), type_object.clone()); + type_object + }) +} + +fn enum_type_object(enum_type: &s::EnumType) -> r::Value { + object! { + kind: r::Value::Enum(String::from("ENUM")), + name: enum_type.name.to_owned(), + description: enum_type.description.clone(), + enumValues: enum_values(enum_type), + } +} + +fn enum_values(enum_type: &s::EnumType) -> r::Value { + r::Value::List(enum_type.values.iter().map(enum_value).collect()) +} + +fn enum_value(enum_value: &s::EnumValue) -> r::Value { + object! { + name: enum_value.name.to_owned(), + description: enum_value.description.clone(), + isDeprecated: false, + deprecationReason: r::Value::Null, + } +} + +fn input_object_type_object( + schema: &Schema, + type_objects: &mut TypeObjectsMap, + input_object_type: &s::InputObjectType, +) -> r::Value { + let input_values = input_values(schema, type_objects, &input_object_type.fields); + object! { + name: input_object_type.name.to_owned(), + kind: r::Value::Enum(String::from("INPUT_OBJECT")), + description: input_object_type.description.clone(), + inputFields: input_values, + } +} + +fn interface_type_object( + schema: &Schema, + type_objects: &mut TypeObjectsMap, + interface_type: &s::InterfaceType, +) -> r::Value { + object! { + name: interface_type.name.to_owned(), + kind: r::Value::Enum(String::from("INTERFACE")), + description: interface_type.description.clone(), + fields: + field_objects(schema, type_objects, &interface_type.fields), + possibleTypes: schema.types_for_interface()[&interface_type.into()] + .iter() + .map(|object_type| r::Value::String(object_type.name.to_owned())) + .collect::>(), + } +} + +fn object_type_object( + schema: &Schema, + type_objects: &mut TypeObjectsMap, + object_type: &s::ObjectType, +) -> r::Value { + type_objects + .get(&object_type.name) + .cloned() + .unwrap_or_else(|| { + let type_object = object! { + kind: r::Value::Enum(String::from("OBJECT")), + name: object_type.name.to_owned(), + description: object_type.description.clone(), + fields: field_objects(schema, type_objects, &object_type.fields), + interfaces: object_interfaces(schema, type_objects, object_type), + }; + + type_objects.insert(object_type.name.to_owned(), type_object.clone()); + type_object + }) +} + +fn field_objects( + schema: &Schema, + type_objects: &mut TypeObjectsMap, + fields: &[s::Field], +) -> r::Value { + r::Value::List( + fields + .iter() + .filter(|field| !field.is_introspection()) + .map(|field| field_object(schema, type_objects, field)) + .collect(), + ) +} + +fn field_object(schema: &Schema, type_objects: &mut TypeObjectsMap, field: &s::Field) -> r::Value { + object! { + name: field.name.to_owned(), + description: field.description.clone(), + args: input_values(schema, type_objects, &field.arguments), + type: type_object(schema, type_objects, &field.field_type), + isDeprecated: false, + deprecationReason: r::Value::Null, + } +} + +fn object_interfaces( + schema: &Schema, + type_objects: &mut TypeObjectsMap, + object_type: &s::ObjectType, +) -> r::Value { + r::Value::List( + schema + .interfaces_for_type(&object_type.into()) + .unwrap_or(&vec![]) + .iter() + .map(|typedef| interface_type_object(schema, type_objects, typedef)) + .collect(), + ) +} + +fn scalar_type_object(scalar_type: &s::ScalarType) -> r::Value { + object! { + name: scalar_type.name.to_owned(), + kind: r::Value::Enum(String::from("SCALAR")), + description: scalar_type.description.clone(), + isDeprecated: false, + deprecationReason: r::Value::Null, + } +} + +fn union_type_object(schema: &Schema, union_type: &s::UnionType) -> r::Value { + object! { + name: union_type.name.to_owned(), + kind: r::Value::Enum(String::from("UNION")), + description: union_type.description.clone(), + possibleTypes: + schema.document.get_object_type_definitions() + .iter() + .filter(|object_type| { + object_type + .implements_interfaces + .iter() + .any(|implemented_name| implemented_name == &union_type.name) + }) + .map(|object_type| r::Value::String(object_type.name.to_owned())) + .collect::>(), + } +} + +fn schema_directive_objects(schema: &Schema, type_objects: &mut TypeObjectsMap) -> r::Value { + r::Value::List( + schema + .document + .definitions + .iter() + .filter_map(|d| match d { + s::Definition::DirectiveDefinition(dd) => Some(dd), + _ => None, + }) + .map(|dd| directive_object(schema, type_objects, dd)) + .collect(), + ) +} + +fn directive_object( + schema: &Schema, + type_objects: &mut TypeObjectsMap, + directive: &s::DirectiveDefinition, +) -> r::Value { + object! { + name: directive.name.to_owned(), + description: directive.description.clone(), + locations: directive_locations(directive), + args: input_values(schema, type_objects, &directive.arguments), + } +} + +fn directive_locations(directive: &s::DirectiveDefinition) -> r::Value { + r::Value::List( + directive + .locations + .iter() + .map(|location| location.as_str()) + .map(|name| r::Value::Enum(name.to_owned())) + .collect(), + ) +} + +fn input_values( + schema: &Schema, + type_objects: &mut TypeObjectsMap, + input_values: &[s::InputValue], +) -> Vec { + input_values + .iter() + .map(|value| input_value(schema, type_objects, value)) + .collect() +} + +fn input_value( + schema: &Schema, + type_objects: &mut TypeObjectsMap, + input_value: &s::InputValue, +) -> r::Value { + object! { + name: input_value.name.to_owned(), + description: input_value.description.clone(), + type: type_object(schema, type_objects, &input_value.value_type), + defaultValue: + input_value + .default_value + .as_ref() + .map_or(r::Value::Null, |value| { + r::Value::String(format!("{}", value)) + }), + } +} + +#[derive(Clone)] +pub struct IntrospectionResolver { + _logger: Logger, + type_objects: TypeObjectsMap, + directives: r::Value, +} + +impl IntrospectionResolver { + pub fn new(logger: &Logger, schema: &Schema) -> Self { + let logger = logger.new(o!("component" => "IntrospectionResolver")); + + // Generate queryable objects for all types in the schema + let mut type_objects = schema_type_objects(schema); + + // Generate queryable objects for all directives in the schema + let directives = schema_directive_objects(schema, &mut type_objects); + + IntrospectionResolver { + _logger: logger, + type_objects, + directives, + } + } + + fn schema_object(&self) -> r::Value { + object! { + queryType: + self.type_objects + .get(&String::from("Query")) + .cloned(), + subscriptionType: + self.type_objects + .get(&String::from("Subscription")) + .cloned(), + mutationType: r::Value::Null, + types: self.type_objects.values().cloned().collect::>(), + directives: self.directives.clone(), + } + } + + fn type_object(&self, name: &r::Value) -> r::Value { + match name { + r::Value::String(s) => Some(s), + _ => None, + } + .and_then(|name| self.type_objects.get(name).cloned()) + .unwrap_or(r::Value::Null) + } +} + +/// A GraphQL resolver that can resolve entities, enum values, scalar types and interfaces/unions. +#[async_trait] +impl Resolver for IntrospectionResolver { + // `IntrospectionResolver` is not used as a "top level" resolver, + // see `fn as_introspection_context`, so this value is irrelevant. + const CACHEABLE: bool = false; + + async fn query_permit(&self) -> Result { + unreachable!() + } + + fn prefetch( + &self, + _: &ExecutionContext, + _: &a::SelectionSet, + ) -> Result<(Option, Trace), Vec> { + Ok((None, Trace::None)) + } + + async fn resolve_objects( + &self, + prefetched_objects: Option, + field: &a::Field, + _field_definition: &s::Field, + _object_type: ObjectOrInterface<'_>, + ) -> Result { + match field.name.as_str() { + "possibleTypes" => { + let type_names = match prefetched_objects { + Some(r::Value::List(type_names)) => Some(type_names), + _ => None, + } + .unwrap_or_default(); + + if !type_names.is_empty() { + Ok(r::Value::List( + type_names + .iter() + .filter_map(|type_name| match type_name { + r::Value::String(ref type_name) => Some(type_name), + _ => None, + }) + .filter_map(|type_name| self.type_objects.get(type_name).cloned()) + .map(r::Value::try_from) + .collect::>() + .map_err(|v| { + QueryExecutionError::ValueParseError( + "internal error resolving type name".to_string(), + v.to_string(), + ) + })?, + )) + } else { + Ok(r::Value::Null) + } + } + _ => Ok(prefetched_objects.unwrap_or(r::Value::Null)), + } + } + + async fn resolve_object( + &self, + prefetched_object: Option, + field: &a::Field, + _field_definition: &s::Field, + _object_type: ObjectOrInterface<'_>, + ) -> Result { + let object = match field.name.as_str() { + "__schema" => self.schema_object(), + "__type" => { + let name = field.argument_value("name").ok_or_else(|| { + QueryExecutionError::MissingArgumentError( + Pos::default(), + "missing argument `name` in `__type(name: String!)`".to_owned(), + ) + })?; + self.type_object(name) + } + "type" | "ofType" => match prefetched_object { + Some(r::Value::String(type_name)) => self + .type_objects + .get(&type_name) + .cloned() + .unwrap_or(r::Value::Null), + Some(v) => v, + None => r::Value::Null, + }, + _ => prefetched_object.unwrap_or(r::Value::Null), + }; + Ok(object) + } +} diff --git a/graphql/src/introspection/schema.rs b/graphql/src/introspection/schema.rs new file mode 100644 index 0000000..97379af --- /dev/null +++ b/graphql/src/introspection/schema.rs @@ -0,0 +1,132 @@ +use std::sync::Arc; + +use graphql_parser; + +use graph::data::graphql::ext::DocumentExt; +use graph::data::graphql::ext::ObjectTypeExt; +use graph::prelude::s::Document; + +use lazy_static::lazy_static; + +use crate::schema::ast as sast; + +const INTROSPECTION_SCHEMA: &str = " +scalar Boolean +scalar Float +scalar Int +scalar ID +scalar String + +type Query { + __schema: __Schema! + __type(name: String!): __Type +} + +type __Schema { + types: [__Type!]! + queryType: __Type! + mutationType: __Type + subscriptionType: __Type + directives: [__Directive!]! +} + +type __Type { + kind: __TypeKind! + name: String + description: String + + # OBJECT and INTERFACE only + fields(includeDeprecated: Boolean = false): [__Field!] + + # OBJECT only + interfaces: [__Type!] + + # INTERFACE and UNION only + possibleTypes: [__Type!] + + # ENUM only + enumValues(includeDeprecated: Boolean = false): [__EnumValue!] + + # INPUT_OBJECT only + inputFields: [__InputValue!] + + # NON_NULL and LIST only + ofType: __Type +} + +type __Field { + name: String! + description: String + args: [__InputValue!]! + type: __Type! + isDeprecated: Boolean! + deprecationReason: String +} + +type __InputValue { + name: String! + description: String + type: __Type! + defaultValue: String +} + +type __EnumValue { + name: String! + description: String + isDeprecated: Boolean! + deprecationReason: String +} + +enum __TypeKind { + SCALAR + OBJECT + INTERFACE + UNION + ENUM + INPUT_OBJECT + LIST + NON_NULL +} + +type __Directive { + name: String! + description: String + locations: [__DirectiveLocation!]! + args: [__InputValue!]! +} + +enum __DirectiveLocation { + QUERY + MUTATION + SUBSCRIPTION + FIELD + FRAGMENT_DEFINITION + FRAGMENT_SPREAD + INLINE_FRAGMENT + SCHEMA + SCALAR + OBJECT + FIELD_DEFINITION + ARGUMENT_DEFINITION + INTERFACE + UNION + ENUM + ENUM_VALUE + INPUT_OBJECT + INPUT_FIELD_DEFINITION +}"; + +lazy_static! { + pub static ref INTROSPECTION_DOCUMENT: Document = + graphql_parser::parse_schema(INTROSPECTION_SCHEMA).unwrap(); + pub static ref INTROSPECTION_QUERY_TYPE: sast::ObjectType = sast::ObjectType::from(Arc::new( + INTROSPECTION_DOCUMENT + .get_root_query_type() + .unwrap() + .clone() + )); +} + +pub fn is_introspection_field(name: &str) -> bool { + INTROSPECTION_QUERY_TYPE.field(name).is_some() +} diff --git a/graphql/src/lib.rs b/graphql/src/lib.rs new file mode 100644 index 0000000..310f58a --- /dev/null +++ b/graphql/src/lib.rs @@ -0,0 +1,49 @@ +pub extern crate graphql_parser; + +/// Utilities for working with GraphQL schemas. +pub mod schema; + +/// Utilities for schema introspection. +pub mod introspection; + +/// Utilities for executing GraphQL. +mod execution; + +/// Utilities for executing GraphQL queries and working with query ASTs. +pub mod query; + +/// Utilities for executing GraphQL subscriptions. +pub mod subscription; + +/// Utilities for working with GraphQL values. +mod values; + +/// Utilities for querying `Store` components. +mod store; + +/// The external interface for actually running queries +mod runner; + +/// Utilities for working with Prometheus. +mod metrics; + +/// Prelude that exports the most important traits and types. +pub mod prelude { + pub use super::execution::{ast as a, ExecutionContext, Query, Resolver}; + pub use super::introspection::IntrospectionResolver; + pub use super::query::{execute_query, ext::BlockConstraint, QueryExecutionOptions}; + pub use super::schema::{api_schema, APISchemaError}; + pub use super::store::StoreResolver; + pub use super::subscription::SubscriptionExecutionOptions; + pub use super::values::MaybeCoercible; + + pub use super::metrics::GraphQLMetrics; + pub use super::runner::GraphQlRunner; + pub use graph::prelude::s::ObjectType; +} + +#[cfg(debug_assertions)] +pub mod test_support { + pub use super::metrics::GraphQLMetrics; + pub use super::runner::INITIAL_DEPLOYMENT_STATE_FOR_TESTS; +} diff --git a/graphql/src/metrics.rs b/graphql/src/metrics.rs new file mode 100644 index 0000000..5163e8c --- /dev/null +++ b/graphql/src/metrics.rs @@ -0,0 +1,136 @@ +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; +use std::time::Duration; + +use graph::data::query::QueryResults; +use graph::prelude::{DeploymentHash, GraphQLMetrics as GraphQLMetricsTrait, MetricsRegistry}; +use graph::prometheus::{Gauge, Histogram, HistogramVec}; + +pub struct GraphQLMetrics { + query_execution_time: Box, + query_parsing_time: Box, + query_validation_time: Box, + query_result_size: Box, + query_result_size_max: Box, +} + +impl fmt::Debug for GraphQLMetrics { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "GraphQLMetrics {{ }}") + } +} + +impl GraphQLMetricsTrait for GraphQLMetrics { + fn observe_query_execution(&self, duration: Duration, results: &QueryResults) { + let id = results + .deployment_hash() + .map(|h| h.as_str()) + .unwrap_or_else(|| { + if results.not_found() { + "notfound" + } else { + "unknown" + } + }); + let status = if results.has_errors() { + "failed" + } else { + "success" + }; + self.query_execution_time + .with_label_values(&[id, status]) + .observe(duration.as_secs_f64()); + } + + fn observe_query_parsing(&self, duration: Duration, results: &QueryResults) { + let id = results + .deployment_hash() + .map(|h| h.as_str()) + .unwrap_or_else(|| { + if results.not_found() { + "notfound" + } else { + "unknown" + } + }); + self.query_parsing_time + .with_label_values(&[id]) + .observe(duration.as_secs_f64()); + } + + fn observe_query_validation(&self, duration: Duration, id: &DeploymentHash) { + self.query_validation_time + .with_label_values(&[id.as_str()]) + .observe(duration.as_secs_f64()); + } +} + +impl GraphQLMetrics { + pub fn new(registry: Arc) -> Self { + let query_execution_time = registry + .new_histogram_vec( + "query_execution_time", + "Execution time for successful GraphQL queries", + vec![String::from("deployment"), String::from("status")], + vec![0.1, 0.5, 1.0, 10.0, 100.0], + ) + .expect("failed to create `query_execution_time` histogram"); + let query_parsing_time = registry + .new_histogram_vec( + "query_parsing_time", + "Parsing time for GraphQL queries", + vec![String::from("deployment")], + vec![0.1, 0.5, 1.0, 10.0, 100.0], + ) + .expect("failed to create `query_parsing_time` histogram"); + + let query_validation_time = registry + .new_histogram_vec( + "query_validation_time", + "Validation time for GraphQL queries", + vec![String::from("deployment")], + vec![0.1, 0.5, 1.0, 10.0, 100.0], + ) + .expect("failed to create `query_validation_time` histogram"); + + let bins = (10..32).map(|n| 2u64.pow(n) as f64).collect::>(); + let query_result_size = registry + .new_histogram( + "query_result_size", + "the size of the result of successful GraphQL queries (in CacheWeight)", + bins, + ) + .unwrap(); + + let query_result_size_max = registry + .new_gauge( + "query_result_max", + "the maximum size of a query result (in CacheWeight)", + HashMap::new(), + ) + .unwrap(); + + Self { + query_execution_time, + query_parsing_time, + query_validation_time, + query_result_size, + query_result_size_max, + } + } + + // Tests need to construct one of these, but normal code doesn't + #[cfg(debug_assertions)] + pub fn make(registry: Arc) -> Self { + Self::new(registry) + } + + pub fn observe_query_result_size(&self, size: usize) { + let size = size as f64; + self.query_result_size.observe(size); + if self.query_result_size_max.get() < size { + self.query_result_size_max.set(size); + } + } +} diff --git a/graphql/src/query/ast.rs b/graphql/src/query/ast.rs new file mode 100644 index 0000000..431c2b6 --- /dev/null +++ b/graphql/src/query/ast.rs @@ -0,0 +1,74 @@ +use graph::prelude::q::*; + +use graph::prelude::QueryExecutionError; + +/// Returns the operation for the given name (or the only operation if no name is defined). +pub fn get_operation<'a>( + document: &'a Document, + name: Option<&str>, +) -> Result<&'a OperationDefinition, QueryExecutionError> { + let operations = get_operations(document); + + match (name, operations.len()) { + (None, 1) => Ok(operations[0]), + (None, _) => Err(QueryExecutionError::OperationNameRequired), + (Some(s), n) if n > 0 => operations + .into_iter() + .find(|op| match get_operation_name(op) { + Some(n) => s == n, + None => false, + }) + .ok_or_else(|| QueryExecutionError::OperationNotFound(s.to_string())), + _ => Err(QueryExecutionError::OperationNameRequired), + } +} + +/// Returns all operation definitions in the document. +pub fn get_operations(document: &Document) -> Vec<&OperationDefinition> { + document + .definitions + .iter() + .filter_map(|d| match d { + Definition::Operation(op) => Some(op), + _ => None, + }) + .collect() +} + +/// Returns the name of the given operation (if it has one). +pub fn get_operation_name(operation: &OperationDefinition) -> Option<&str> { + match operation { + OperationDefinition::Mutation(m) => m.name.as_deref(), + OperationDefinition::Query(q) => q.name.as_deref(), + OperationDefinition::SelectionSet(_) => None, + OperationDefinition::Subscription(s) => s.name.as_deref(), + } +} + +/// Looks up a directive in a selection, if it is provided. +pub fn get_directive(selection: &Selection, name: String) -> Option<&Directive> { + match selection { + Selection::Field(field) => field + .directives + .iter() + .find(|directive| directive.name == name), + _ => None, + } +} + +/// Looks up the value of an argument in a vector of (name, value) tuples. +pub fn get_argument_value<'a>(arguments: &'a [(String, Value)], name: &str) -> Option<&'a Value> { + arguments.iter().find(|(n, _)| n == name).map(|(_, v)| v) +} + +/// Returns the variable definitions for an operation. +pub fn get_variable_definitions( + operation: &OperationDefinition, +) -> Option<&Vec> { + match operation { + OperationDefinition::Query(q) => Some(&q.variable_definitions), + OperationDefinition::Subscription(s) => Some(&s.variable_definitions), + OperationDefinition::Mutation(m) => Some(&m.variable_definitions), + OperationDefinition::SelectionSet(_) => None, + } +} diff --git a/graphql/src/query/ext.rs b/graphql/src/query/ext.rs new file mode 100644 index 0000000..a285a4b --- /dev/null +++ b/graphql/src/query/ext.rs @@ -0,0 +1,97 @@ +//! Extension traits for graphql_parser::query structs + +use graph::blockchain::BlockHash; +use graph::prelude::TryFromValue; +use graphql_parser::Pos; + +use std::collections::{BTreeMap, HashMap}; + +use anyhow::anyhow; +use graph::data::query::QueryExecutionError; +use graph::prelude::{q, r, BlockNumber, Error}; + +pub trait ValueExt: Sized { + fn as_object(&self) -> &BTreeMap; + fn as_string(&self) -> &str; + + /// If `self` is a variable reference, look it up in `vars` and return + /// that. Otherwise, just return `self`. + /// + /// If `self` is a variable reference, but has no entry in `vars` return + /// an error + fn lookup<'a>( + &'a self, + vars: &'a HashMap, + pos: Pos, + ) -> Result<&'a Self, QueryExecutionError>; +} + +impl ValueExt for q::Value { + fn as_object(&self) -> &BTreeMap { + match self { + q::Value::Object(object) => object, + _ => panic!("expected a Value::Object"), + } + } + + fn as_string(&self) -> &str { + match self { + q::Value::String(string) => string, + _ => panic!("expected a Value::String"), + } + } + + fn lookup<'a>( + &'a self, + vars: &'a HashMap, + pos: Pos, + ) -> Result<&'a q::Value, QueryExecutionError> { + match self { + q::Value::Variable(name) => vars + .get(name) + .ok_or_else(|| QueryExecutionError::MissingVariableError(pos, name.to_owned())), + _ => Ok(self), + } + } +} + +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub enum BlockConstraint { + Hash(BlockHash), + Number(BlockNumber), + /// Execute the query on the latest block only if the the subgraph has progressed to or past the + /// given block number. + Min(BlockNumber), + Latest, +} + +impl Default for BlockConstraint { + fn default() -> Self { + BlockConstraint::Latest + } +} + +impl TryFromValue for BlockConstraint { + /// `value` should be the output of input object coercion. + fn try_from_value(value: &r::Value) -> Result { + let map = match value { + r::Value::Object(map) => map, + r::Value::Null => return Ok(Self::default()), + _ => return Err(anyhow!("invalid `BlockConstraint`")), + }; + + if let Some(hash) = map.get("hash") { + Ok(BlockConstraint::Hash(TryFromValue::try_from_value(hash)?)) + } else if let Some(number_value) = map.get("number") { + Ok(BlockConstraint::Number(BlockNumber::try_from_value( + number_value, + )?)) + } else if let Some(number_value) = map.get("number_gte") { + Ok(BlockConstraint::Min(BlockNumber::try_from_value( + number_value, + )?)) + } else { + Err(anyhow!("invalid `BlockConstraint`")) + } + } +} diff --git a/graphql/src/query/mod.rs b/graphql/src/query/mod.rs new file mode 100644 index 0000000..b8f9b3f --- /dev/null +++ b/graphql/src/query/mod.rs @@ -0,0 +1,85 @@ +use graph::prelude::{BlockPtr, CheapClone, QueryExecutionError, QueryResult}; +use std::sync::Arc; +use std::time::Instant; + +use graph::data::graphql::effort::LoadManager; + +use crate::execution::{ast as a, *}; + +/// Utilities for working with GraphQL query ASTs. +pub mod ast; + +/// Extension traits +pub mod ext; + +/// Options available for query execution. +pub struct QueryExecutionOptions { + /// The resolver to use. + pub resolver: R, + + /// Time at which the query times out. + pub deadline: Option, + + /// Maximum value for the `first` argument. + pub max_first: u32, + + /// Maximum value for the `skip` argument + pub max_skip: u32, + + pub load_manager: Arc, +} + +/// Executes a query and returns a result. +/// If the query is not cacheable, the `Arc` may be unwrapped. +pub async fn execute_query( + query: Arc, + selection_set: Option, + block_ptr: Option, + options: QueryExecutionOptions, +) -> Arc +where + R: Resolver, +{ + // Create a fresh execution context + let ctx = Arc::new(ExecutionContext { + logger: query.logger.clone(), + resolver: options.resolver, + query: query.clone(), + deadline: options.deadline, + max_first: options.max_first, + max_skip: options.max_skip, + cache_status: Default::default(), + }); + + if !query.is_query() { + return Arc::new( + QueryExecutionError::NotSupported("Only queries are supported".to_string()).into(), + ); + } + let selection_set = selection_set + .map(Arc::new) + .unwrap_or_else(|| query.selection_set.cheap_clone()); + + // Execute top-level `query { ... }` and `{ ... }` expressions. + let query_type = ctx.query.schema.query_type.cheap_clone().into(); + let start = Instant::now(); + let result = execute_root_selection_set( + ctx.cheap_clone(), + selection_set.cheap_clone(), + query_type, + block_ptr.clone(), + ) + .await; + let elapsed = start.elapsed(); + let cache_status = ctx.cache_status.load(); + options + .load_manager + .record_work(query.shape_hash, elapsed, cache_status); + query.log_cache_status( + &selection_set, + block_ptr.map(|b| b.number).unwrap_or(0), + start, + cache_status.to_string(), + ); + result +} diff --git a/graphql/src/runner.rs b/graphql/src/runner.rs new file mode 100644 index 0000000..968c426 --- /dev/null +++ b/graphql/src/runner.rs @@ -0,0 +1,280 @@ +use std::sync::Arc; +use std::time::Instant; + +use crate::metrics::GraphQLMetrics; +use crate::prelude::{QueryExecutionOptions, StoreResolver, SubscriptionExecutionOptions}; +use crate::query::execute_query; +use crate::subscription::execute_prepared_subscription; +use graph::prelude::MetricsRegistry; +use graph::{ + components::store::SubscriptionManager, + prelude::{ + async_trait, o, CheapClone, DeploymentState, GraphQLMetrics as GraphQLMetricsTrait, + GraphQlRunner as GraphQlRunnerTrait, Logger, Query, QueryExecutionError, Subscription, + SubscriptionError, SubscriptionResult, ENV_VARS, + }, +}; +use graph::{data::graphql::effort::LoadManager, prelude::QueryStoreManager}; +use graph::{ + data::query::{QueryResults, QueryTarget}, + prelude::QueryStore, +}; + +/// GraphQL runner implementation for The Graph. +pub struct GraphQlRunner { + logger: Logger, + store: Arc, + subscription_manager: Arc, + load_manager: Arc, + graphql_metrics: Arc, +} + +#[cfg(debug_assertions)] +lazy_static::lazy_static! { + // Test only, see c435c25decbc4ad7bbbadf8e0ced0ff2 + pub static ref INITIAL_DEPLOYMENT_STATE_FOR_TESTS: std::sync::Mutex> = std::sync::Mutex::new(None); +} + +impl GraphQlRunner +where + S: QueryStoreManager, + SM: SubscriptionManager, +{ + /// Creates a new query runner. + pub fn new( + logger: &Logger, + store: Arc, + subscription_manager: Arc, + load_manager: Arc, + registry: Arc, + ) -> Self { + let logger = logger.new(o!("component" => "GraphQlRunner")); + let graphql_metrics = Arc::new(GraphQLMetrics::new(registry)); + GraphQlRunner { + logger, + store, + subscription_manager, + load_manager, + graphql_metrics, + } + } + + /// Check if the subgraph state differs from `state` now in a way that + /// would affect a query that looked at data as fresh as `latest_block`. + /// If the subgraph did change, return the `Err` that should be sent back + /// to clients to indicate that condition + async fn deployment_changed( + &self, + store: &dyn QueryStore, + state: DeploymentState, + latest_block: u64, + ) -> Result<(), QueryExecutionError> { + if ENV_VARS.graphql.allow_deployment_change { + return Ok(()); + } + let new_state = store.deployment_state().await?; + assert!(new_state.reorg_count >= state.reorg_count); + if new_state.reorg_count > state.reorg_count { + // One or more reorgs happened; each reorg can't have gone back + // farther than `max_reorg_depth`, so that querying at blocks + // far enough away from the previous latest block is fine. Taking + // this into consideration is important, since most of the time + // there is only one reorg of one block, and we therefore avoid + // flagging a lot of queries a bit behind the head + let n_blocks = new_state.max_reorg_depth * (new_state.reorg_count - state.reorg_count); + if latest_block + n_blocks as u64 > state.latest_block.number as u64 { + return Err(QueryExecutionError::DeploymentReverted); + } + } + Ok(()) + } + + async fn execute( + &self, + query: Query, + target: QueryTarget, + max_complexity: Option, + max_depth: Option, + max_first: Option, + max_skip: Option, + metrics: Arc, + ) -> Result { + // We need to use the same `QueryStore` for the entire query to ensure + // we have a consistent view if the world, even when replicas, which + // are eventually consistent, are in use. If we run different parts + // of the query against different replicas, it would be possible for + // them to be at wildly different states, and we might unwittingly + // mix data from different block heights even if no reverts happen + // while the query is running. `self.store` can not be used after this + // point, and everything needs to go through the `store` we are + // setting up here + + let store = self.store.query_store(target.clone(), false).await?; + let state = store.deployment_state().await?; + let network = Some(store.network_name().to_string()); + let schema = store.api_schema()?; + + // Test only, see c435c25decbc4ad7bbbadf8e0ced0ff2 + #[cfg(debug_assertions)] + let state = INITIAL_DEPLOYMENT_STATE_FOR_TESTS + .lock() + .unwrap() + .clone() + .unwrap_or(state); + + let max_depth = max_depth.unwrap_or(ENV_VARS.graphql.max_depth); + let query = crate::execution::Query::new( + &self.logger, + schema, + network, + query, + max_complexity, + max_depth, + metrics.cheap_clone(), + )?; + self.load_manager + .decide( + &store.wait_stats().map_err(QueryExecutionError::from)?, + query.shape_hash, + query.query_text.as_ref(), + ) + .to_result()?; + let by_block_constraint = query.block_constraint()?; + let mut max_block = 0; + let mut result: QueryResults = QueryResults::empty(); + + // Note: This will always iterate at least once. + for (bc, (selection_set, error_policy)) in by_block_constraint { + let query_start = Instant::now(); + let resolver = StoreResolver::at_block( + &self.logger, + store.cheap_clone(), + &state, + self.subscription_manager.cheap_clone(), + bc, + error_policy, + query.schema.id().clone(), + metrics.cheap_clone(), + ) + .await?; + max_block = max_block.max(resolver.block_number()); + let query_res = execute_query( + query.clone(), + Some(selection_set), + resolver.block_ptr.as_ref().map(Into::into).clone(), + QueryExecutionOptions { + resolver, + deadline: ENV_VARS.graphql.query_timeout.map(|t| Instant::now() + t), + max_first: max_first.unwrap_or(ENV_VARS.graphql.max_first), + max_skip: max_skip.unwrap_or(ENV_VARS.graphql.max_skip), + load_manager: self.load_manager.clone(), + }, + ) + .await; + query_res.trace.finish(query_start.elapsed()); + result.append(query_res); + } + + query.log_execution(max_block); + self.deployment_changed(store.as_ref(), state, max_block as u64) + .await + .map_err(QueryResults::from) + .map(|()| result) + } +} + +#[async_trait] +impl GraphQlRunnerTrait for GraphQlRunner +where + S: QueryStoreManager, + SM: SubscriptionManager, +{ + async fn run_query(self: Arc, query: Query, target: QueryTarget) -> QueryResults { + self.run_query_with_complexity( + query, + target, + ENV_VARS.graphql.max_complexity, + Some(ENV_VARS.graphql.max_depth), + Some(ENV_VARS.graphql.max_first), + Some(ENV_VARS.graphql.max_skip), + ) + .await + } + + async fn run_query_with_complexity( + self: Arc, + query: Query, + target: QueryTarget, + max_complexity: Option, + max_depth: Option, + max_first: Option, + max_skip: Option, + ) -> QueryResults { + self.execute( + query, + target, + max_complexity, + max_depth, + max_first, + max_skip, + self.graphql_metrics.clone(), + ) + .await + .unwrap_or_else(|e| e) + } + + async fn run_subscription( + self: Arc, + subscription: Subscription, + target: QueryTarget, + ) -> Result { + let store = self.store.query_store(target.clone(), true).await?; + let schema = store.api_schema()?; + let network = store.network_name().to_string(); + + let query = crate::execution::Query::new( + &self.logger, + schema, + Some(network), + subscription.query, + ENV_VARS.graphql.max_complexity, + ENV_VARS.graphql.max_depth, + self.graphql_metrics.cheap_clone(), + )?; + + if let Err(err) = self + .load_manager + .decide( + &store.wait_stats().map_err(QueryExecutionError::from)?, + query.shape_hash, + query.query_text.as_ref(), + ) + .to_result() + { + return Err(SubscriptionError::GraphQLError(vec![err])); + } + + execute_prepared_subscription( + query, + SubscriptionExecutionOptions { + logger: self.logger.clone(), + store, + subscription_manager: self.subscription_manager.cheap_clone(), + timeout: ENV_VARS.graphql.query_timeout, + max_complexity: ENV_VARS.graphql.max_complexity, + max_depth: ENV_VARS.graphql.max_depth, + max_first: ENV_VARS.graphql.max_first, + max_skip: ENV_VARS.graphql.max_skip, + graphql_metrics: self.graphql_metrics.clone(), + }, + ) + } + + fn load_manager(&self) -> Arc { + self.load_manager.clone() + } + + fn metrics(&self) -> Arc { + self.graphql_metrics.clone() + } +} diff --git a/graphql/src/schema/api.rs b/graphql/src/schema/api.rs new file mode 100644 index 0000000..03d061c --- /dev/null +++ b/graphql/src/schema/api.rs @@ -0,0 +1,1403 @@ +use std::str::FromStr; + +use graphql_parser::Pos; +use inflector::Inflector; +use lazy_static::lazy_static; + +use crate::schema::ast; + +use graph::data::{ + graphql::ext::{DirectiveExt, DocumentExt, ValueExt}, + schema::{META_FIELD_NAME, META_FIELD_TYPE, SCHEMA_TYPE_NAME}, +}; +use graph::prelude::s::{Value, *}; +use graph::prelude::*; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum APISchemaError { + #[error("type {0} already exists in the input schema")] + TypeExists(String), + #[error("Type {0} not found")] + TypeNotFound(String), + #[error("Fulltext search is not yet deterministic")] + FulltextSearchNonDeterministic, +} + +// The followoing types are defined in meta.graphql +const BLOCK_HEIGHT: &str = "Block_height"; +const CHANGE_BLOCK_FILTER_NAME: &str = "BlockChangedFilter"; +const ERROR_POLICY_TYPE: &str = "_SubgraphErrorPolicy_"; + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub enum ErrorPolicy { + Allow, + Deny, +} + +impl std::str::FromStr for ErrorPolicy { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "allow" => Ok(ErrorPolicy::Allow), + "deny" => Ok(ErrorPolicy::Deny), + _ => Err(anyhow::anyhow!("failed to parse `{}` as ErrorPolicy", s)), + } + } +} + +impl TryFrom<&q::Value> for ErrorPolicy { + type Error = anyhow::Error; + + /// `value` should be the output of input value coercion. + fn try_from(value: &q::Value) -> Result { + match value { + q::Value::Enum(s) => ErrorPolicy::from_str(s), + _ => Err(anyhow::anyhow!("invalid `ErrorPolicy`")), + } + } +} + +impl TryFrom<&r::Value> for ErrorPolicy { + type Error = anyhow::Error; + + /// `value` should be the output of input value coercion. + fn try_from(value: &r::Value) -> Result { + match value { + r::Value::Enum(s) => ErrorPolicy::from_str(s), + _ => Err(anyhow::anyhow!("invalid `ErrorPolicy`")), + } + } +} + +/// Derives a full-fledged GraphQL API schema from an input schema. +/// +/// The input schema should only have type/enum/interface/union definitions +/// and must not include a root Query type. This Query type is derived, with +/// all its fields and their input arguments, based on the existing types. +pub fn api_schema(input_schema: &Document) -> Result { + // Refactor: Take `input_schema` by value. + let object_types = input_schema.get_object_type_definitions(); + let interface_types = input_schema.get_interface_type_definitions(); + + // Refactor: Don't clone the schema. + let mut schema = input_schema.clone(); + add_meta_field_type(&mut schema); + add_types_for_object_types(&mut schema, &object_types)?; + add_types_for_interface_types(&mut schema, &interface_types)?; + add_field_arguments(&mut schema, input_schema)?; + add_query_type(&mut schema, &object_types, &interface_types)?; + add_subscription_type(&mut schema, &object_types, &interface_types)?; + + // Remove the `_Schema_` type from the generated schema. + schema.definitions.retain(|d| match d { + Definition::TypeDefinition(def @ TypeDefinition::Object(_)) => match def { + TypeDefinition::Object(t) if t.name.eq(SCHEMA_TYPE_NAME) => false, + _ => true, + }, + _ => true, + }); + + Ok(schema) +} + +/// Adds a global `_Meta_` type to the schema. The `_meta` field +/// accepts values of this type +fn add_meta_field_type(schema: &mut Document) { + lazy_static! { + static ref META_FIELD_SCHEMA: Document = { + let schema = include_str!("meta.graphql"); + parse_schema(schema).expect("the schema `meta.graphql` is invalid") + }; + } + + schema + .definitions + .extend(META_FIELD_SCHEMA.definitions.iter().cloned()); +} + +fn add_types_for_object_types( + schema: &mut Document, + object_types: &[&ObjectType], +) -> Result<(), APISchemaError> { + for object_type in object_types { + if !object_type.name.eq(SCHEMA_TYPE_NAME) { + add_order_by_type(schema, &object_type.name, &object_type.fields)?; + add_filter_type(schema, &object_type.name, &object_type.fields)?; + } + } + Ok(()) +} + +/// Adds `*_orderBy` and `*_filter` enum types for the given interfaces to the schema. +fn add_types_for_interface_types( + schema: &mut Document, + interface_types: &[&InterfaceType], +) -> Result<(), APISchemaError> { + for interface_type in interface_types { + add_order_by_type(schema, &interface_type.name, &interface_type.fields)?; + add_filter_type(schema, &interface_type.name, &interface_type.fields)?; + } + Ok(()) +} + +/// Adds a `_orderBy` enum type for the given fields to the schema. +fn add_order_by_type( + schema: &mut Document, + type_name: &str, + fields: &[Field], +) -> Result<(), APISchemaError> { + let type_name = format!("{}_orderBy", type_name); + + match schema.get_named_type(&type_name) { + None => { + let typedef = TypeDefinition::Enum(EnumType { + position: Pos::default(), + description: None, + name: type_name, + directives: vec![], + values: fields + .iter() + .map(|field| &field.name) + .map(|name| EnumValue { + position: Pos::default(), + description: None, + name: name.to_owned(), + directives: vec![], + }) + .collect(), + }); + let def = Definition::TypeDefinition(typedef); + schema.definitions.push(def); + } + Some(_) => return Err(APISchemaError::TypeExists(type_name)), + } + Ok(()) +} + +/// Adds a `_filter` enum type for the given fields to the schema. +fn add_filter_type( + schema: &mut Document, + type_name: &str, + fields: &[Field], +) -> Result<(), APISchemaError> { + let filter_type_name = format!("{}_filter", type_name); + match schema.get_named_type(&filter_type_name) { + None => { + let mut generated_filter_fields = field_input_values(schema, fields)?; + generated_filter_fields.push(block_changed_filter_argument()); + + let typedef = TypeDefinition::InputObject(InputObjectType { + position: Pos::default(), + description: None, + name: filter_type_name, + directives: vec![], + fields: generated_filter_fields, + }); + let def = Definition::TypeDefinition(typedef); + schema.definitions.push(def); + } + Some(_) => return Err(APISchemaError::TypeExists(filter_type_name)), + } + + Ok(()) +} + +/// Generates `*_filter` input values for the given set of fields. +fn field_input_values( + schema: &Document, + fields: &[Field], +) -> Result, APISchemaError> { + let mut input_values = vec![]; + for field in fields { + input_values.extend(field_filter_input_values(schema, field, &field.field_type)?); + } + Ok(input_values) +} + +/// Generates `*_filter` input values for the given field. +fn field_filter_input_values( + schema: &Document, + field: &Field, + field_type: &Type, +) -> Result, APISchemaError> { + match field_type { + Type::NamedType(ref name) => { + let named_type = schema + .get_named_type(name) + .ok_or_else(|| APISchemaError::TypeNotFound(name.clone()))?; + Ok(match named_type { + TypeDefinition::Object(_) | TypeDefinition::Interface(_) => { + let mut input_values = match ast::get_derived_from_directive(field) { + // Only add `where` filter fields for object and interface fields + // if they are not @derivedFrom + Some(_) => vec![], + // We allow filtering with `where: { other: "some-id" }` and + // `where: { others: ["some-id", "other-id"] }`. In both cases, + // we allow ID strings as the values to be passed to these + // filters. + None => field_scalar_filter_input_values( + schema, + field, + &ScalarType::new(String::from("String")), + ), + }; + extend_with_child_filter_input_value(field, name, &mut input_values); + input_values + } + TypeDefinition::Scalar(ref t) => field_scalar_filter_input_values(schema, field, t), + TypeDefinition::Enum(ref t) => field_enum_filter_input_values(schema, field, t), + _ => vec![], + }) + } + Type::ListType(ref t) => { + Ok(field_list_filter_input_values(schema, field, t).unwrap_or(vec![])) + } + Type::NonNullType(ref t) => field_filter_input_values(schema, field, t), + } +} + +/// Generates `*_filter` input values for the given scalar field. +fn field_scalar_filter_input_values( + _schema: &Document, + field: &Field, + field_type: &ScalarType, +) -> Vec { + match field_type.name.as_ref() { + "BigInt" => vec!["", "not", "gt", "lt", "gte", "lte", "in", "not_in"], + "Boolean" => vec!["", "not", "in", "not_in"], + "Bytes" => vec!["", "not", "in", "not_in", "contains", "not_contains"], + "BigDecimal" => vec!["", "not", "gt", "lt", "gte", "lte", "in", "not_in"], + "ID" => vec!["", "not", "gt", "lt", "gte", "lte", "in", "not_in"], + "Int" => vec!["", "not", "gt", "lt", "gte", "lte", "in", "not_in"], + "String" => vec![ + "", + "not", + "gt", + "lt", + "gte", + "lte", + "in", + "not_in", + "contains", + "contains_nocase", + "not_contains", + "not_contains_nocase", + "starts_with", + "starts_with_nocase", + "not_starts_with", + "not_starts_with_nocase", + "ends_with", + "ends_with_nocase", + "not_ends_with", + "not_ends_with_nocase", + ], + _ => vec!["", "not"], + } + .into_iter() + .map(|filter_type| { + let field_type = Type::NamedType(field_type.name.to_owned()); + let value_type = match filter_type { + "in" | "not_in" => Type::ListType(Box::new(Type::NonNullType(Box::new(field_type)))), + _ => field_type, + }; + input_value(&field.name, filter_type, value_type) + }) + .collect() +} + +/// Appends a child filter to input values +fn extend_with_child_filter_input_value( + field: &Field, + field_type_name: &String, + input_values: &mut Vec, +) { + input_values.push(input_value( + &format!("{}_", field.name), + "", + Type::NamedType(format!("{}_filter", field_type_name)), + )); +} + +/// Generates `*_filter` input values for the given enum field. +fn field_enum_filter_input_values( + _schema: &Document, + field: &Field, + field_type: &EnumType, +) -> Vec { + vec!["", "not", "in", "not_in"] + .into_iter() + .map(|filter_type| { + let field_type = Type::NamedType(field_type.name.to_owned()); + let value_type = match filter_type { + "in" | "not_in" => { + Type::ListType(Box::new(Type::NonNullType(Box::new(field_type)))) + } + _ => field_type, + }; + input_value(&field.name, filter_type, value_type) + }) + .collect() +} + +/// Generates `*_filter` input values for the given list field. +fn field_list_filter_input_values( + schema: &Document, + field: &Field, + field_type: &Type, +) -> Option> { + // Only add a filter field if the type of the field exists in the schema + ast::get_type_definition_from_type(schema, field_type).and_then(|typedef| { + // Decide what type of values can be passed to the filter. In the case + // one-to-many or many-to-many object or interface fields that are not + // derived, we allow ID strings to be passed on. + // Adds child filter only to object types. + let (input_field_type, parent_type_name) = match typedef { + TypeDefinition::Object(ObjectType { name, .. }) + | TypeDefinition::Interface(InterfaceType { name, .. }) => { + if ast::get_derived_from_directive(field).is_some() { + (None, Some(name.clone())) + } else { + (Some(Type::NamedType("String".into())), Some(name.clone())) + } + } + TypeDefinition::Scalar(ref t) => (Some(Type::NamedType(t.name.to_owned())), None), + TypeDefinition::Enum(ref t) => (Some(Type::NamedType(t.name.to_owned())), None), + TypeDefinition::InputObject(_) | TypeDefinition::Union(_) => (None, None), + }; + + let mut input_values: Vec = match input_field_type { + None => { + vec![] + } + Some(input_field_type) => vec![ + "", + "not", + "contains", + "contains_nocase", + "not_contains", + "not_contains_nocase", + ] + .into_iter() + .map(|filter_type| { + input_value( + &field.name, + filter_type, + Type::ListType(Box::new(Type::NonNullType(Box::new( + input_field_type.clone(), + )))), + ) + }) + .collect(), + }; + + if let Some(parent) = parent_type_name { + extend_with_child_filter_input_value(field, &parent, &mut input_values); + } + + Some(input_values) + }) +} + +/// Generates a `*_filter` input value for the given field name, suffix and value type. +fn input_value(name: &str, suffix: &'static str, value_type: Type) -> InputValue { + InputValue { + position: Pos::default(), + description: None, + name: if suffix.is_empty() { + name.to_owned() + } else { + format!("{}_{}", name, suffix) + }, + value_type, + default_value: None, + directives: vec![], + } +} + +/// Adds a root `Query` object type to the schema. +fn add_query_type( + schema: &mut Document, + object_types: &[&ObjectType], + interface_types: &[&InterfaceType], +) -> Result<(), APISchemaError> { + let type_name = String::from("Query"); + + if schema.get_named_type(&type_name).is_some() { + return Err(APISchemaError::TypeExists(type_name)); + } + + let mut fields = object_types + .iter() + .map(|t| t.name.as_str()) + .filter(|name| !name.eq(&SCHEMA_TYPE_NAME)) + .chain(interface_types.iter().map(|t| t.name.as_str())) + .flat_map(query_fields_for_type) + .collect::>(); + let mut fulltext_fields = schema + .get_fulltext_directives() + .map_err(|_| APISchemaError::FulltextSearchNonDeterministic)? + .iter() + .filter_map(|fulltext| query_field_for_fulltext(fulltext)) + .collect(); + fields.append(&mut fulltext_fields); + fields.push(meta_field()); + + let typedef = TypeDefinition::Object(ObjectType { + position: Pos::default(), + description: None, + name: type_name, + implements_interfaces: vec![], + directives: vec![], + fields, + }); + let def = Definition::TypeDefinition(typedef); + schema.definitions.push(def); + Ok(()) +} + +fn query_field_for_fulltext(fulltext: &Directive) -> Option { + let name = fulltext.argument("name").unwrap().as_str().unwrap().into(); + + let includes = fulltext.argument("include").unwrap().as_list().unwrap(); + // Only one include is allowed per fulltext directive + let include = includes.iter().next().unwrap(); + let included_entity = include.as_object().unwrap(); + let entity_name = included_entity.get("entity").unwrap().as_str().unwrap(); + + let mut arguments = vec![ + // text: String + InputValue { + position: Pos::default(), + description: None, + name: String::from("text"), + value_type: Type::NonNullType(Box::new(Type::NamedType(String::from("String")))), + default_value: None, + directives: vec![], + }, + // first: Int + InputValue { + position: Pos::default(), + description: None, + name: String::from("first"), + value_type: Type::NamedType(String::from("Int")), + default_value: Some(Value::Int(100.into())), + directives: vec![], + }, + // skip: Int + InputValue { + position: Pos::default(), + description: None, + name: String::from("skip"), + value_type: Type::NamedType(String::from("Int")), + default_value: Some(Value::Int(0.into())), + directives: vec![], + }, + // block: BlockHeight + block_argument(), + ]; + + arguments.push(subgraph_error_argument()); + + Some(Field { + position: Pos::default(), + description: None, + name, + arguments, + field_type: Type::NonNullType(Box::new(Type::ListType(Box::new(Type::NonNullType( + Box::new(Type::NamedType(entity_name.into())), + ))))), // included entity type name + directives: vec![fulltext.clone()], + }) +} + +/// Adds a root `Subscription` object type to the schema. +fn add_subscription_type( + schema: &mut Document, + object_types: &[&ObjectType], + interface_types: &[&InterfaceType], +) -> Result<(), APISchemaError> { + let type_name = String::from("Subscription"); + + if schema.get_named_type(&type_name).is_some() { + return Err(APISchemaError::TypeExists(type_name)); + } + + let mut fields: Vec = object_types + .iter() + .map(|t| &t.name) + .filter(|name| !name.eq(&SCHEMA_TYPE_NAME)) + .chain(interface_types.iter().map(|t| &t.name)) + .flat_map(|name| query_fields_for_type(name)) + .collect(); + fields.push(meta_field()); + + let typedef = TypeDefinition::Object(ObjectType { + position: Pos::default(), + description: None, + name: type_name, + implements_interfaces: vec![], + directives: vec![], + fields, + }); + let def = Definition::TypeDefinition(typedef); + schema.definitions.push(def); + Ok(()) +} + +fn block_argument() -> InputValue { + InputValue { + position: Pos::default(), + description: Some( + "The block at which the query should be executed. \ + Can either be a `{ hash: Bytes }` value containing a block hash, \ + a `{ number: Int }` containing the block number, \ + or a `{ number_gte: Int }` containing the minimum block number. \ + In the case of `number_gte`, the query will be executed on the latest block only if \ + the subgraph has progressed to or past the minimum block number. \ + Defaults to the latest block when omitted." + .to_owned(), + ), + name: "block".to_string(), + value_type: Type::NamedType(BLOCK_HEIGHT.to_owned()), + default_value: None, + directives: vec![], + } +} + +fn block_changed_filter_argument() -> InputValue { + InputValue { + position: Pos::default(), + description: Some("Filter for the block changed event.".to_owned()), + name: "_change_block".to_string(), + value_type: Type::NamedType(CHANGE_BLOCK_FILTER_NAME.to_owned()), + default_value: None, + directives: vec![], + } +} + +fn subgraph_error_argument() -> InputValue { + InputValue { + position: Pos::default(), + description: Some( + "Set to `allow` to receive data even if the subgraph has skipped over errors while syncing." + .to_owned(), + ), + name: "subgraphError".to_string(), + value_type: Type::NonNullType(Box::new(Type::NamedType(ERROR_POLICY_TYPE.to_string()))), + default_value: Some(Value::Enum("deny".to_string())), + directives: vec![], + } +} + +/// Generates `Query` fields for the given type name (e.g. `users` and `user`). +fn query_fields_for_type(type_name: &str) -> Vec { + let mut collection_arguments = collection_arguments_for_named_type(type_name); + collection_arguments.push(block_argument()); + + let mut by_id_arguments = vec![ + InputValue { + position: Pos::default(), + description: None, + name: "id".to_string(), + value_type: Type::NonNullType(Box::new(Type::NamedType("ID".to_string()))), + default_value: None, + directives: vec![], + }, + block_argument(), + ]; + + collection_arguments.push(subgraph_error_argument()); + by_id_arguments.push(subgraph_error_argument()); + + vec![ + Field { + position: Pos::default(), + description: None, + name: type_name.to_camel_case(), // Name formatting must be updated in sync with `graph::data::schema::validate_fulltext_directive_name()` + arguments: by_id_arguments, + field_type: Type::NamedType(type_name.to_owned()), + directives: vec![], + }, + Field { + position: Pos::default(), + description: None, + name: type_name.to_plural().to_camel_case(), // Name formatting must be updated in sync with `graph::data::schema::validate_fulltext_directive_name()` + arguments: collection_arguments, + field_type: Type::NonNullType(Box::new(Type::ListType(Box::new(Type::NonNullType( + Box::new(Type::NamedType(type_name.to_owned())), + ))))), + directives: vec![], + }, + ] +} + +fn meta_field() -> Field { + lazy_static! { + static ref META_FIELD: Field = Field { + position: Pos::default(), + description: Some("Access to subgraph metadata".to_string()), + name: META_FIELD_NAME.to_string(), + arguments: vec![ + // block: BlockHeight + InputValue { + position: Pos::default(), + description: None, + name: String::from("block"), + value_type: Type::NamedType(BLOCK_HEIGHT.to_string()), + default_value: None, + directives: vec![], + }, + ], + field_type: Type::NamedType(META_FIELD_TYPE.to_string()), + directives: vec![], + }; + } + META_FIELD.clone() +} + +/// Generates arguments for collection queries of a named type (e.g. User). +fn collection_arguments_for_named_type(type_name: &str) -> Vec { + // `first` and `skip` should be non-nullable, but the Apollo graphql client + // exhibts non-conforming behaviour by erroing if no value is provided for a + // non-nullable field, regardless of the presence of a default. + let mut skip = input_value(&"skip".to_string(), "", Type::NamedType("Int".to_string())); + skip.default_value = Some(Value::Int(0.into())); + + let mut first = input_value(&"first".to_string(), "", Type::NamedType("Int".to_string())); + first.default_value = Some(Value::Int(100.into())); + + let args = vec![ + skip, + first, + input_value( + &"orderBy".to_string(), + "", + Type::NamedType(format!("{}_orderBy", type_name)), + ), + input_value( + &"orderDirection".to_string(), + "", + Type::NamedType("OrderDirection".to_string()), + ), + input_value( + &"where".to_string(), + "", + Type::NamedType(format!("{}_filter", type_name)), + ), + ]; + + args +} + +fn add_field_arguments( + schema: &mut Document, + input_schema: &Document, +) -> Result<(), APISchemaError> { + // Refactor: Remove the `input_schema` argument and do a mutable iteration + // over the definitions in `schema`. Also the duplication between this and + // the loop for interfaces below. + for input_object_type in input_schema.get_object_type_definitions() { + for input_field in &input_object_type.fields { + if let Some(input_reference_type) = + ast::get_referenced_entity_type(input_schema, input_field) + { + if ast::is_list_or_non_null_list_field(input_field) { + // Get corresponding object type and field in the output schema + let object_type = ast::get_object_type_mut(schema, &input_object_type.name) + .expect("object type from input schema is missing in API schema"); + let mut field = object_type + .fields + .iter_mut() + .find(|field| field.name == input_field.name) + .expect("field from input schema is missing in API schema"); + + match input_reference_type { + TypeDefinition::Object(ot) => { + field.arguments = collection_arguments_for_named_type(&ot.name); + } + TypeDefinition::Interface(it) => { + field.arguments = collection_arguments_for_named_type(&it.name); + } + _ => unreachable!( + "referenced entity types can only be object or interface types" + ), + } + } + } + } + } + + for input_interface_type in input_schema.get_interface_type_definitions() { + for input_field in &input_interface_type.fields { + if let Some(input_reference_type) = + ast::get_referenced_entity_type(input_schema, input_field) + { + if ast::is_list_or_non_null_list_field(input_field) { + // Get corresponding interface type and field in the output schema + let interface_type = + ast::get_interface_type_mut(schema, &input_interface_type.name) + .expect("interface type from input schema is missing in API schema"); + let mut field = interface_type + .fields + .iter_mut() + .find(|field| field.name == input_field.name) + .expect("field from input schema is missing in API schema"); + + match input_reference_type { + TypeDefinition::Object(ot) => { + field.arguments = collection_arguments_for_named_type(&ot.name); + } + TypeDefinition::Interface(it) => { + field.arguments = collection_arguments_for_named_type(&it.name); + } + _ => unreachable!( + "referenced entity types can only be object or interface types" + ), + } + } + } + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use graph::data::graphql::DocumentExt; + use graphql_parser::schema::*; + + use super::api_schema; + use crate::schema::ast; + + #[test] + fn api_schema_contains_built_in_scalar_types() { + let input_schema = + parse_schema("type User { id: ID! }").expect("Failed to parse input schema"); + let schema = api_schema(&input_schema).expect("Failed to derive API schema"); + + schema + .get_named_type("Boolean") + .expect("Boolean type is missing in API schema"); + schema + .get_named_type("ID") + .expect("ID type is missing in API schema"); + schema + .get_named_type("Int") + .expect("Int type is missing in API schema"); + schema + .get_named_type("BigDecimal") + .expect("BigDecimal type is missing in API schema"); + schema + .get_named_type("String") + .expect("String type is missing in API schema"); + } + + #[test] + fn api_schema_contains_order_direction_enum() { + let input_schema = parse_schema("type User { id: ID!, name: String! }") + .expect("Failed to parse input schema"); + let schema = api_schema(&input_schema).expect("Failed to derived API schema"); + + let order_direction = schema + .get_named_type("OrderDirection") + .expect("OrderDirection type is missing in derived API schema"); + let enum_type = match order_direction { + TypeDefinition::Enum(t) => Some(t), + _ => None, + } + .expect("OrderDirection type is not an enum"); + + let values: Vec<&str> = enum_type + .values + .iter() + .map(|value| value.name.as_str()) + .collect(); + assert_eq!(values, ["asc", "desc"]); + } + + #[test] + fn api_schema_contains_query_type() { + let input_schema = + parse_schema("type User { id: ID! }").expect("Failed to parse input schema"); + let schema = api_schema(&input_schema).expect("Failed to derive API schema"); + schema + .get_named_type("Query") + .expect("Root Query type is missing in API schema"); + } + + #[test] + fn api_schema_contains_field_order_by_enum() { + let input_schema = parse_schema("type User { id: ID!, name: String! }") + .expect("Failed to parse input schema"); + let schema = api_schema(&input_schema).expect("Failed to derived API schema"); + + let user_order_by = schema + .get_named_type("User_orderBy") + .expect("User_orderBy type is missing in derived API schema"); + + let enum_type = match user_order_by { + TypeDefinition::Enum(t) => Some(t), + _ => None, + } + .expect("User_orderBy type is not an enum"); + + let values: Vec<&str> = enum_type + .values + .iter() + .map(|value| value.name.as_str()) + .collect(); + assert_eq!(values, ["id", "name"]); + } + + #[test] + fn api_schema_contains_object_type_filter_enum() { + let input_schema = parse_schema( + r#" + enum FurType { + NONE + FLUFFY + BRISTLY + } + + type Pet { + id: ID! + name: String! + mostHatedBy: [User!]! + mostLovedBy: [User!]! + } + + type User { + id: ID! + name: String! + favoritePetNames: [String!] + pets: [Pet!]! + favoriteFurType: FurType! + favoritePet: Pet! + leastFavoritePet: Pet @derivedFrom(field: "mostHatedBy") + mostFavoritePets: [Pet!] @derivedFrom(field: "mostLovedBy") + } + "#, + ) + .expect("Failed to parse input schema"); + let schema = api_schema(&input_schema).expect("Failed to derived API schema"); + + let user_filter = schema + .get_named_type("User_filter") + .expect("User_filter type is missing in derived API schema"); + + let user_filter_type = match user_filter { + TypeDefinition::InputObject(t) => Some(t), + _ => None, + } + .expect("User_filter type is not an input object"); + + assert_eq!( + user_filter_type + .fields + .iter() + .map(|field| field.name.to_owned()) + .collect::>(), + [ + "id", + "id_not", + "id_gt", + "id_lt", + "id_gte", + "id_lte", + "id_in", + "id_not_in", + "name", + "name_not", + "name_gt", + "name_lt", + "name_gte", + "name_lte", + "name_in", + "name_not_in", + "name_contains", + "name_contains_nocase", + "name_not_contains", + "name_not_contains_nocase", + "name_starts_with", + "name_starts_with_nocase", + "name_not_starts_with", + "name_not_starts_with_nocase", + "name_ends_with", + "name_ends_with_nocase", + "name_not_ends_with", + "name_not_ends_with_nocase", + "favoritePetNames", + "favoritePetNames_not", + "favoritePetNames_contains", + "favoritePetNames_contains_nocase", + "favoritePetNames_not_contains", + "favoritePetNames_not_contains_nocase", + "pets", + "pets_not", + "pets_contains", + "pets_contains_nocase", + "pets_not_contains", + "pets_not_contains_nocase", + "pets_", + "favoriteFurType", + "favoriteFurType_not", + "favoriteFurType_in", + "favoriteFurType_not_in", + "favoritePet", + "favoritePet_not", + "favoritePet_gt", + "favoritePet_lt", + "favoritePet_gte", + "favoritePet_lte", + "favoritePet_in", + "favoritePet_not_in", + "favoritePet_contains", + "favoritePet_contains_nocase", + "favoritePet_not_contains", + "favoritePet_not_contains_nocase", + "favoritePet_starts_with", + "favoritePet_starts_with_nocase", + "favoritePet_not_starts_with", + "favoritePet_not_starts_with_nocase", + "favoritePet_ends_with", + "favoritePet_ends_with_nocase", + "favoritePet_not_ends_with", + "favoritePet_not_ends_with_nocase", + "favoritePet_", + "leastFavoritePet_", + "mostFavoritePets_", + "_change_block" + ] + .iter() + .map(ToString::to_string) + .collect::>() + ); + + let pets_field = user_filter_type + .fields + .iter() + .find(|field| field.name == "pets_") + .expect("pets_ field is missing"); + + assert_eq!( + pets_field.value_type.to_string(), + String::from("Pet_filter") + ); + + let pet_filter = schema + .get_named_type("Pet_filter") + .expect("Pet_filter type is missing in derived API schema"); + + let pet_filter_type = match pet_filter { + TypeDefinition::InputObject(t) => Some(t), + _ => None, + } + .expect("Pet_filter type is not an input object"); + + assert_eq!( + pet_filter_type + .fields + .iter() + .map(|field| field.name.to_owned()) + .collect::>(), + [ + "id", + "id_not", + "id_gt", + "id_lt", + "id_gte", + "id_lte", + "id_in", + "id_not_in", + "name", + "name_not", + "name_gt", + "name_lt", + "name_gte", + "name_lte", + "name_in", + "name_not_in", + "name_contains", + "name_contains_nocase", + "name_not_contains", + "name_not_contains_nocase", + "name_starts_with", + "name_starts_with_nocase", + "name_not_starts_with", + "name_not_starts_with_nocase", + "name_ends_with", + "name_ends_with_nocase", + "name_not_ends_with", + "name_not_ends_with_nocase", + "mostHatedBy", + "mostHatedBy_not", + "mostHatedBy_contains", + "mostHatedBy_contains_nocase", + "mostHatedBy_not_contains", + "mostHatedBy_not_contains_nocase", + "mostHatedBy_", + "mostLovedBy", + "mostLovedBy_not", + "mostLovedBy_contains", + "mostLovedBy_contains_nocase", + "mostLovedBy_not_contains", + "mostLovedBy_not_contains_nocase", + "mostLovedBy_", + "_change_block" + ] + .iter() + .map(ToString::to_string) + .collect::>() + ); + + let change_block_filter = user_filter_type + .fields + .iter() + .find(move |p| match p.name.as_str() { + "_change_block" => true, + _ => false, + }) + .expect("_change_block field is missing in User_filter"); + + match &change_block_filter.value_type { + Type::NamedType(name) => assert_eq!(name.as_str(), "BlockChangedFilter"), + _ => panic!("_change_block field is not a named type"), + } + + schema + .get_named_type("BlockChangedFilter") + .expect("BlockChangedFilter type is missing in derived API schema"); + } + + #[test] + fn api_schema_contains_object_type_with_field_interface() { + let input_schema = parse_schema( + r#" + interface Pet { + id: ID! + name: String! + } + + type Dog implements Pet { + id: ID! + name: String! + } + + type Cat implements Pet { + id: ID! + name: String! + owner: User! + } + + type User { + id: ID! + name: String! + pets: [Pet!]! @derivedFrom(field: "owner") + favoritePet: Pet! + } + "#, + ) + .expect("Failed to parse input schema"); + let schema = api_schema(&input_schema).expect("Failed to derived API schema"); + + let user_filter = schema + .get_named_type("User_filter") + .expect("User_filter type is missing in derived API schema"); + + let user_filter_type = match user_filter { + TypeDefinition::InputObject(t) => Some(t), + _ => None, + } + .expect("User_filter type is not an input object"); + + assert_eq!( + user_filter_type + .fields + .iter() + .map(|field| field.name.to_owned()) + .collect::>(), + [ + "id", + "id_not", + "id_gt", + "id_lt", + "id_gte", + "id_lte", + "id_in", + "id_not_in", + "name", + "name_not", + "name_gt", + "name_lt", + "name_gte", + "name_lte", + "name_in", + "name_not_in", + "name_contains", + "name_contains_nocase", + "name_not_contains", + "name_not_contains_nocase", + "name_starts_with", + "name_starts_with_nocase", + "name_not_starts_with", + "name_not_starts_with_nocase", + "name_ends_with", + "name_ends_with_nocase", + "name_not_ends_with", + "name_not_ends_with_nocase", + "pets_", + "favoritePet", + "favoritePet_not", + "favoritePet_gt", + "favoritePet_lt", + "favoritePet_gte", + "favoritePet_lte", + "favoritePet_in", + "favoritePet_not_in", + "favoritePet_contains", + "favoritePet_contains_nocase", + "favoritePet_not_contains", + "favoritePet_not_contains_nocase", + "favoritePet_starts_with", + "favoritePet_starts_with_nocase", + "favoritePet_not_starts_with", + "favoritePet_not_starts_with_nocase", + "favoritePet_ends_with", + "favoritePet_ends_with_nocase", + "favoritePet_not_ends_with", + "favoritePet_not_ends_with_nocase", + "favoritePet_", + "_change_block" + ] + .iter() + .map(ToString::to_string) + .collect::>() + ); + + let change_block_filter = user_filter_type + .fields + .iter() + .find(move |p| match p.name.as_str() { + "_change_block" => true, + _ => false, + }) + .expect("_change_block field is missing in User_filter"); + + match &change_block_filter.value_type { + Type::NamedType(name) => assert_eq!(name.as_str(), "BlockChangedFilter"), + _ => panic!("_change_block field is not a named type"), + } + + schema + .get_named_type("BlockChangedFilter") + .expect("BlockChangedFilter type is missing in derived API schema"); + } + + #[test] + fn api_schema_contains_object_fields_on_query_type() { + let input_schema = parse_schema( + "type User { id: ID!, name: String! } type UserProfile { id: ID!, title: String! }", + ) + .expect("Failed to parse input schema"); + let schema = api_schema(&input_schema).expect("Failed to derive API schema"); + + let query_type = schema + .get_named_type("Query") + .expect("Query type is missing in derived API schema"); + + let user_singular_field = match query_type { + TypeDefinition::Object(t) => ast::get_field(t, &"user".to_string()), + _ => None, + } + .expect("\"user\" field is missing on Query type"); + + assert_eq!( + user_singular_field.field_type, + Type::NamedType("User".to_string()) + ); + + assert_eq!( + user_singular_field + .arguments + .iter() + .map(|input_value| input_value.name.to_owned()) + .collect::>(), + vec![ + "id".to_string(), + "block".to_string(), + "subgraphError".to_string() + ], + ); + + let user_plural_field = match query_type { + TypeDefinition::Object(t) => ast::get_field(t, &"users".to_string()), + _ => None, + } + .expect("\"users\" field is missing on Query type"); + + assert_eq!( + user_plural_field.field_type, + Type::NonNullType(Box::new(Type::ListType(Box::new(Type::NonNullType( + Box::new(Type::NamedType("User".to_string())) + ))))) + ); + + assert_eq!( + user_plural_field + .arguments + .iter() + .map(|input_value| input_value.name.to_owned()) + .collect::>(), + [ + "skip", + "first", + "orderBy", + "orderDirection", + "where", + "block", + "subgraphError", + ] + .iter() + .map(ToString::to_string) + .collect::>() + ); + + let user_profile_singular_field = match query_type { + TypeDefinition::Object(t) => ast::get_field(t, &"userProfile".to_string()), + _ => None, + } + .expect("\"userProfile\" field is missing on Query type"); + + assert_eq!( + user_profile_singular_field.field_type, + Type::NamedType("UserProfile".to_string()) + ); + + let user_profile_plural_field = match query_type { + TypeDefinition::Object(t) => ast::get_field(t, &"userProfiles".to_string()), + _ => None, + } + .expect("\"userProfiles\" field is missing on Query type"); + + assert_eq!( + user_profile_plural_field.field_type, + Type::NonNullType(Box::new(Type::ListType(Box::new(Type::NonNullType( + Box::new(Type::NamedType("UserProfile".to_string())) + ))))) + ); + } + + #[test] + fn api_schema_contains_interface_fields_on_query_type() { + let input_schema = parse_schema( + " + interface Node { id: ID!, name: String! } + type User implements Node { id: ID!, name: String!, email: String } + ", + ) + .expect("Failed to parse input schema"); + let schema = api_schema(&input_schema).expect("Failed to derived API schema"); + + let query_type = schema + .get_named_type("Query") + .expect("Query type is missing in derived API schema"); + + let singular_field = match query_type { + TypeDefinition::Object(ref t) => ast::get_field(t, &"node".to_string()), + _ => None, + } + .expect("\"node\" field is missing on Query type"); + + assert_eq!( + singular_field.field_type, + Type::NamedType("Node".to_string()) + ); + + assert_eq!( + singular_field + .arguments + .iter() + .map(|input_value| input_value.name.to_owned()) + .collect::>(), + vec![ + "id".to_string(), + "block".to_string(), + "subgraphError".to_string() + ], + ); + + let plural_field = match query_type { + TypeDefinition::Object(ref t) => ast::get_field(t, &"nodes".to_string()), + _ => None, + } + .expect("\"nodes\" field is missing on Query type"); + + assert_eq!( + plural_field.field_type, + Type::NonNullType(Box::new(Type::ListType(Box::new(Type::NonNullType( + Box::new(Type::NamedType("Node".to_string())) + ))))) + ); + + assert_eq!( + plural_field + .arguments + .iter() + .map(|input_value| input_value.name.to_owned()) + .collect::>(), + [ + "skip", + "first", + "orderBy", + "orderDirection", + "where", + "block", + "subgraphError" + ] + .iter() + .map(ToString::to_string) + .collect::>() + ); + } + + #[test] + fn api_schema_contains_fulltext_query_field_on_query_type() { + const SCHEMA: &str = r#" +type _Schema_ @fulltext( + name: "metadata" + language: en + algorithm: rank + include: [ + { + entity: "Gravatar", + fields: [ + { name: "displayName"}, + { name: "imageUrl"}, + ] + } + ] +) +type Gravatar @entity { + id: ID! + owner: Bytes! + displayName: String! + imageUrl: String! +} +"#; + let input_schema = parse_schema(SCHEMA).expect("Failed to parse input schema"); + let schema = api_schema(&input_schema).expect("Failed to derive API schema"); + + let query_type = schema + .get_named_type("Query") + .expect("Query type is missing in derived API schema"); + + let _metadata_field = match query_type { + TypeDefinition::Object(t) => ast::get_field(t, &String::from("metadata")), + _ => None, + } + .expect("\"metadata\" field is missing on Query type"); + } +} diff --git a/graphql/src/schema/ast.rs b/graphql/src/schema/ast.rs new file mode 100644 index 0000000..6460f24 --- /dev/null +++ b/graphql/src/schema/ast.rs @@ -0,0 +1,511 @@ +use graph::cheap_clone::CheapClone; +use graphql_parser::Pos; +use lazy_static::lazy_static; +use std::ops::Deref; +use std::str::FromStr; +use std::sync::Arc; + +use graph::data::graphql::ext::DirectiveFinder; +use graph::data::graphql::{DocumentExt, ObjectOrInterface}; +use graph::prelude::anyhow::anyhow; +use graph::prelude::{s, Error, ValueType}; + +use crate::query::ast as qast; + +pub(crate) enum FilterOp { + Not, + GreaterThan, + LessThan, + GreaterOrEqual, + LessOrEqual, + In, + NotIn, + Contains, + ContainsNoCase, + NotContains, + NotContainsNoCase, + StartsWith, + StartsWithNoCase, + NotStartsWith, + NotStartsWithNoCase, + EndsWith, + EndsWithNoCase, + NotEndsWith, + NotEndsWithNoCase, + Equal, + Child, +} + +/// Split a "name_eq" style name into an attribute ("name") and a filter op (`Equal`). +pub(crate) fn parse_field_as_filter(key: &str) -> (String, FilterOp) { + let (suffix, op) = match key { + k if k.ends_with("_not") => ("_not", FilterOp::Not), + k if k.ends_with("_gt") => ("_gt", FilterOp::GreaterThan), + k if k.ends_with("_lt") => ("_lt", FilterOp::LessThan), + k if k.ends_with("_gte") => ("_gte", FilterOp::GreaterOrEqual), + k if k.ends_with("_lte") => ("_lte", FilterOp::LessOrEqual), + k if k.ends_with("_not_in") => ("_not_in", FilterOp::NotIn), + k if k.ends_with("_in") => ("_in", FilterOp::In), + k if k.ends_with("_not_contains") => ("_not_contains", FilterOp::NotContains), + k if k.ends_with("_not_contains_nocase") => { + ("_not_contains_nocase", FilterOp::NotContainsNoCase) + } + k if k.ends_with("_contains") => ("_contains", FilterOp::Contains), + k if k.ends_with("_contains_nocase") => ("_contains_nocase", FilterOp::ContainsNoCase), + k if k.ends_with("_not_starts_with") => ("_not_starts_with", FilterOp::NotStartsWith), + k if k.ends_with("_not_starts_with_nocase") => { + ("_not_starts_with_nocase", FilterOp::NotStartsWithNoCase) + } + k if k.ends_with("_not_ends_with") => ("_not_ends_with", FilterOp::NotEndsWith), + k if k.ends_with("_not_ends_with_nocase") => { + ("_not_ends_with_nocase", FilterOp::NotEndsWithNoCase) + } + k if k.ends_with("_starts_with") => ("_starts_with", FilterOp::StartsWith), + k if k.ends_with("_starts_with_nocase") => { + ("_starts_with_nocase", FilterOp::StartsWithNoCase) + } + k if k.ends_with("_ends_with") => ("_ends_with", FilterOp::EndsWith), + k if k.ends_with("_ends_with_nocase") => ("_ends_with_nocase", FilterOp::EndsWithNoCase), + k if k.ends_with("_") => ("_", FilterOp::Child), + _ => ("", FilterOp::Equal), + }; + + // Strip the operator suffix to get the attribute. + (key.trim_end_matches(suffix).to_owned(), op) +} + +/// An `ObjectType` with `Hash` and `Eq` derived from the name. +#[derive(Clone, Debug)] +pub struct ObjectType(Arc); + +impl Ord for ObjectType { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.0.name.cmp(&other.0.name) + } +} + +impl PartialOrd for ObjectType { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.0.name.cmp(&other.0.name)) + } +} + +impl std::hash::Hash for ObjectType { + fn hash(&self, state: &mut H) { + self.0.name.hash(state) + } +} + +impl PartialEq for ObjectType { + fn eq(&self, other: &Self) -> bool { + self.0.name.eq(&other.0.name) + } +} + +impl Eq for ObjectType {} + +impl From> for ObjectType { + fn from(object: Arc) -> Self { + ObjectType(object) + } +} + +impl<'a> From<&'a ObjectType> for ObjectOrInterface<'a> { + fn from(cond: &'a ObjectType) -> Self { + ObjectOrInterface::Object(cond.0.as_ref()) + } +} + +impl Deref for ObjectType { + type Target = s::ObjectType; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl CheapClone for ObjectType {} + +impl ObjectType { + pub fn name(&self) -> &str { + &self.0.name + } +} + +/// Returns all type definitions in the schema. +pub fn get_type_definitions(schema: &s::Document) -> Vec<&s::TypeDefinition> { + schema + .definitions + .iter() + .filter_map(|d| match d { + s::Definition::TypeDefinition(typedef) => Some(typedef), + _ => None, + }) + .collect() +} + +/// Returns the object type with the given name. +pub fn get_object_type_mut<'a>( + schema: &'a mut s::Document, + name: &str, +) -> Option<&'a mut s::ObjectType> { + use graphql_parser::schema::TypeDefinition::*; + + get_named_type_definition_mut(schema, name).and_then(|type_def| match type_def { + Object(object_type) => Some(object_type), + _ => None, + }) +} + +/// Returns the interface type with the given name. +pub fn get_interface_type_mut<'a>( + schema: &'a mut s::Document, + name: &str, +) -> Option<&'a mut s::InterfaceType> { + use graphql_parser::schema::TypeDefinition::*; + + get_named_type_definition_mut(schema, name).and_then(|type_def| match type_def { + Interface(interface_type) => Some(interface_type), + _ => None, + }) +} + +/// Returns the type of a field of an object type. +pub fn get_field<'a>( + object_type: impl Into>, + name: &str, +) -> Option<&'a s::Field> { + lazy_static! { + pub static ref TYPENAME_FIELD: s::Field = s::Field { + position: Pos::default(), + description: None, + name: "__typename".to_owned(), + field_type: s::Type::NonNullType(Box::new(s::Type::NamedType("String".to_owned()))), + arguments: vec![], + directives: vec![], + }; + } + + if name == TYPENAME_FIELD.name { + Some(&TYPENAME_FIELD) + } else { + object_type + .into() + .fields() + .iter() + .find(|field| field.name == name) + } +} + +/// Returns the value type for a GraphQL field type. +pub fn get_field_value_type(field_type: &s::Type) -> Result { + match field_type { + s::Type::NamedType(ref name) => ValueType::from_str(&name), + s::Type::NonNullType(inner) => get_field_value_type(&inner), + s::Type::ListType(_) => Err(anyhow!("Only scalar values are supported in this context")), + } +} + +/// Returns the value type for a GraphQL field type. +pub fn get_field_name(field_type: &s::Type) -> String { + match field_type { + s::Type::NamedType(name) => name.to_string(), + s::Type::NonNullType(inner) => get_field_name(inner), + s::Type::ListType(inner) => get_field_name(inner), + } +} + +/// Returns a mutable version of the type with the given name. +fn get_named_type_definition_mut<'a>( + schema: &'a mut s::Document, + name: &str, +) -> Option<&'a mut s::TypeDefinition> { + schema + .definitions + .iter_mut() + .filter_map(|def| match def { + s::Definition::TypeDefinition(typedef) => Some(typedef), + _ => None, + }) + .find(|typedef| get_type_name(typedef) == name) +} + +/// Returns the name of a type. +pub fn get_type_name(t: &s::TypeDefinition) -> &str { + match t { + s::TypeDefinition::Enum(t) => &t.name, + s::TypeDefinition::InputObject(t) => &t.name, + s::TypeDefinition::Interface(t) => &t.name, + s::TypeDefinition::Object(t) => &t.name, + s::TypeDefinition::Scalar(t) => &t.name, + s::TypeDefinition::Union(t) => &t.name, + } +} + +/// Returns the argument definitions for a field of an object type. +pub fn get_argument_definitions<'a>( + object_type: impl Into>, + name: &str, +) -> Option<&'a Vec> { + lazy_static! { + pub static ref NAME_ARGUMENT: Vec = vec![s::InputValue { + position: Pos::default(), + description: None, + name: "name".to_owned(), + value_type: s::Type::NonNullType(Box::new(s::Type::NamedType("String".to_owned()))), + default_value: None, + directives: vec![], + }]; + } + + // Introspection: `__type(name: String!): __Type` + if name == "__type" { + Some(&NAME_ARGUMENT) + } else { + get_field(object_type, name).map(|field| &field.arguments) + } +} + +/// Returns the type definition for a type. +pub fn get_type_definition_from_type<'a>( + schema: &'a s::Document, + t: &s::Type, +) -> Option<&'a s::TypeDefinition> { + match t { + s::Type::NamedType(name) => schema.get_named_type(name), + s::Type::ListType(inner) => get_type_definition_from_type(schema, inner), + s::Type::NonNullType(inner) => get_type_definition_from_type(schema, inner), + } +} + +/// Looks up a directive in a object type, if it is provided. +pub fn get_object_type_directive( + object_type: &s::ObjectType, + name: String, +) -> Option<&s::Directive> { + object_type + .directives + .iter() + .find(|directive| directive.name == name) +} + +// Returns true if the given type is a non-null type. +pub fn is_non_null_type(t: &s::Type) -> bool { + match t { + s::Type::NonNullType(_) => true, + _ => false, + } +} + +/// Returns true if the given type is an input type. +/// +/// Uses the algorithm outlined on +/// https://facebook.github.io/graphql/draft/#IsInputType(). +pub fn is_input_type(schema: &s::Document, t: &s::Type) -> bool { + match t { + s::Type::NamedType(name) => { + let named_type = schema.get_named_type(name); + named_type.map_or(false, |type_def| match type_def { + s::TypeDefinition::Scalar(_) + | s::TypeDefinition::Enum(_) + | s::TypeDefinition::InputObject(_) => true, + _ => false, + }) + } + s::Type::ListType(inner) => is_input_type(schema, inner), + s::Type::NonNullType(inner) => is_input_type(schema, inner), + } +} + +pub fn is_entity_type(schema: &s::Document, t: &s::Type) -> bool { + match t { + s::Type::NamedType(name) => schema + .get_named_type(name) + .map_or(false, is_entity_type_definition), + s::Type::ListType(inner_type) => is_entity_type(schema, inner_type), + s::Type::NonNullType(inner_type) => is_entity_type(schema, inner_type), + } +} + +pub fn is_entity_type_definition(type_def: &s::TypeDefinition) -> bool { + match type_def { + // Entity types are obvious + s::TypeDefinition::Object(object_type) => { + get_object_type_directive(object_type, String::from("entity")).is_some() + } + + // For now, we'll assume that only entities can implement interfaces; + // thus, any interface type definition is automatically an entity type + s::TypeDefinition::Interface(_) => true, + + // Everything else (unions, scalars, enums) are not considered entity + // types for now + _ => false, + } +} + +pub fn is_list_or_non_null_list_field(field: &s::Field) -> bool { + match &field.field_type { + s::Type::ListType(_) => true, + s::Type::NonNullType(inner_type) => match inner_type.deref() { + s::Type::ListType(_) => true, + _ => false, + }, + _ => false, + } +} + +fn unpack_type<'a>(schema: &'a s::Document, t: &s::Type) -> Option<&'a s::TypeDefinition> { + match t { + s::Type::NamedType(name) => schema.get_named_type(name), + s::Type::ListType(inner_type) => unpack_type(schema, inner_type), + s::Type::NonNullType(inner_type) => unpack_type(schema, inner_type), + } +} + +pub fn get_referenced_entity_type<'a>( + schema: &'a s::Document, + field: &s::Field, +) -> Option<&'a s::TypeDefinition> { + unpack_type(schema, &field.field_type).filter(|ty| is_entity_type_definition(ty)) +} + +/// If the field has a `@derivedFrom(field: "foo")` directive, obtain the +/// name of the field (e.g. `"foo"`) +pub fn get_derived_from_directive(field_definition: &s::Field) -> Option<&s::Directive> { + field_definition.find_directive("derivedFrom") +} + +pub fn get_derived_from_field<'a>( + object_type: impl Into>, + field_definition: &'a s::Field, +) -> Option<&'a s::Field> { + get_derived_from_directive(field_definition) + .and_then(|directive| qast::get_argument_value(&directive.arguments, "field")) + .and_then(|value| match value { + s::Value::String(s) => Some(s), + _ => None, + }) + .and_then(|derived_from_field_name| get_field(object_type, derived_from_field_name)) +} + +pub fn is_list(field_type: &s::Type) -> bool { + match field_type { + s::Type::NamedType(_) => false, + s::Type::NonNullType(inner) => is_list(inner), + s::Type::ListType(_) => true, + } +} + +#[test] +fn entity_validation() { + use graph::components::store::EntityKey; + use graph::data::store; + use graph::prelude::{DeploymentHash, Entity}; + + fn make_thing(name: &str) -> Entity { + let mut thing = Entity::new(); + thing.set("id", name); + thing.set("name", name); + thing.set("stuff", "less"); + thing.set("favorite_color", "red"); + thing.set("things", store::Value::List(vec![])); + thing + } + + fn check(thing: Entity, errmsg: &str) { + const DOCUMENT: &str = " + enum Color { red, yellow, blue } + interface Stuff { id: ID!, name: String! } + type Cruft @entity { + id: ID!, + thing: Thing! + } + type Thing @entity { + id: ID!, + name: String!, + favorite_color: Color, + stuff: Stuff, + things: [Thing!]! + # Make sure we do not validate derived fields; it's ok + # to store a thing with a null Cruft + cruft: Cruft! @derivedFrom(field: \"thing\") + }"; + let subgraph = DeploymentHash::new("doesntmatter").unwrap(); + let schema = + graph::prelude::Schema::parse(DOCUMENT, subgraph).expect("Failed to parse test schema"); + let id = thing.id().unwrap_or("none".to_owned()); + let key = EntityKey::data("Thing".to_owned(), id.clone()); + + let err = thing.validate(&schema, &key); + if errmsg == "" { + assert!( + err.is_ok(), + "checking entity {}: expected ok but got {}", + id, + err.unwrap_err() + ); + } else { + if let Err(e) = err { + assert_eq!(errmsg, e.to_string(), "checking entity {}", id); + } else { + panic!( + "Expected error `{}` but got ok when checking entity {}", + errmsg, id + ); + } + } + } + + let mut thing = make_thing("t1"); + thing.set("things", store::Value::from(vec!["thing1", "thing2"])); + check(thing, ""); + + let thing = make_thing("t2"); + check(thing, ""); + + let mut thing = make_thing("t3"); + thing.remove("name"); + check( + thing, + "Entity Thing[t3]: missing value for non-nullable field `name`", + ); + + let mut thing = make_thing("t4"); + thing.remove("things"); + check( + thing, + "Entity Thing[t4]: missing value for non-nullable field `things`", + ); + + let mut thing = make_thing("t5"); + thing.set("name", store::Value::Int(32)); + check( + thing, + "Entity Thing[t5]: the value `32` for field `name` must \ + have type String! but has type Int", + ); + + let mut thing = make_thing("t6"); + thing.set( + "things", + store::Value::List(vec!["thing1".into(), 17.into()]), + ); + check( + thing, + "Entity Thing[t6]: field `things` is of type [Thing!]!, \ + but the value `[thing1, 17]` contains a Int at index 1", + ); + + let mut thing = make_thing("t7"); + thing.remove("favorite_color"); + thing.remove("stuff"); + check(thing, ""); + + let mut thing = make_thing("t8"); + thing.set("cruft", "wat"); + check( + thing, + "Entity Thing[t8]: field `cruft` is derived and can not be set", + ); +} diff --git a/graphql/src/schema/meta.graphql b/graphql/src/schema/meta.graphql new file mode 100644 index 0000000..b2b5ffd --- /dev/null +++ b/graphql/src/schema/meta.graphql @@ -0,0 +1,89 @@ +# GraphQL core functionality +scalar Boolean +scalar ID +scalar Int +scalar Float +scalar String + +directive @skip(if: Boolean!) on FIELD | FRAGMENT_SPREAD | INLINE_FRAGMENT +directive @include(if: Boolean!) on FIELD | FRAGMENT_SPREAD | INLINE_FRAGMENT + +# The Graph extensions + +"Marks the GraphQL type as indexable entity. Each type that should be an entity is required to be annotated with this directive." +directive @entity on OBJECT + +"Defined a Subgraph ID for an object type" +directive @subgraphId(id: String!) on OBJECT + +"creates a virtual field on the entity that may be queried but cannot be set manually through the mappings API." +directive @derivedFrom(field: String!) on FIELD_DEFINITION + +scalar BigDecimal +scalar Bytes +scalar BigInt + +# The type names are purposely awkward to minimize the risk of them +# colliding with user-supplied types +"The type for the top-level _meta field" +type _Meta_ { + """ + Information about a specific subgraph block. The hash of the block + will be null if the _meta field has a block constraint that asks for + a block number. It will be filled if the _meta field has no block constraint + and therefore asks for the latest block + """ + block: _Block_! + "The deployment ID" + deployment: String! + "If `true`, the subgraph encountered indexing errors at some past block" + hasIndexingErrors: Boolean! +} + +input BlockChangedFilter { + number_gte: Int! +} + +input Block_height { + hash: Bytes + number: Int + number_gte: Int +} + +type _Block_ { + "The hash of the block" + hash: Bytes + "The block number" + number: Int! + "Integer representation of the timestamp stored in blocks for the chain" + timestamp: Int +} + +enum _SubgraphErrorPolicy_ { + "Data will be returned even if the subgraph has indexing errors" + allow, + + "If the subgraph has indexing errors, data will be omitted. The default." + deny +} + +"The block at which the query should be executed." +input Block_height { + "Value containing a block hash" + hash: Bytes + "Value containing a block number" + number: Int + """ + Value containing the minimum block number. + In the case of `number_gte`, the query will be executed on the latest block only if + the subgraph has progressed to or past the minimum block number. + Defaults to the latest block when omitted. + """ + number_gte: Int +} + +"Defines the order direction, either ascending or descending" +enum OrderDirection { + asc + desc +} \ No newline at end of file diff --git a/graphql/src/schema/mod.rs b/graphql/src/schema/mod.rs new file mode 100644 index 0000000..6df5190 --- /dev/null +++ b/graphql/src/schema/mod.rs @@ -0,0 +1,7 @@ +/// Generate full-fledged API schemas from existing GraphQL schemas. +pub mod api; + +/// Utilities for working with GraphQL schema ASTs. +pub mod ast; + +pub use self::api::{api_schema, APISchemaError}; diff --git a/graphql/src/store/mod.rs b/graphql/src/store/mod.rs new file mode 100644 index 0000000..85ceb42 --- /dev/null +++ b/graphql/src/store/mod.rs @@ -0,0 +1,6 @@ +mod prefetch; +mod query; +mod resolver; + +pub use self::query::parse_subgraph_id; +pub use self::resolver::StoreResolver; diff --git a/graphql/src/store/prefetch.rs b/graphql/src/store/prefetch.rs new file mode 100644 index 0000000..e480181 --- /dev/null +++ b/graphql/src/store/prefetch.rs @@ -0,0 +1,777 @@ +//! Run a GraphQL query and fetch all the entitied needed to build the +//! final result + +use anyhow::{anyhow, Error}; +use graph::constraint_violation; +use graph::data::query::Trace; +use graph::data::value::{Object, Word}; +use graph::prelude::{r, CacheWeight}; +use graph::slog::warn; +use graph::util::cache_weight; +use lazy_static::lazy_static; +use std::collections::BTreeMap; +use std::rc::Rc; +use std::time::Instant; + +use graph::{components::store::EntityType, data::graphql::*}; +use graph::{ + data::graphql::ext::DirectiveFinder, + prelude::{ + s, ApiSchema, AttributeNames, BlockNumber, ChildMultiplicity, EntityCollection, + EntityFilter, EntityLink, EntityOrder, EntityWindow, Logger, ParentLink, + QueryExecutionError, QueryStore, StoreError, Value as StoreValue, WindowAttribute, + ENV_VARS, + }, +}; + +use crate::execution::{ast as a, ExecutionContext, Resolver}; +use crate::metrics::GraphQLMetrics; +use crate::schema::ast as sast; +use crate::store::query::build_query; +use crate::store::StoreResolver; + +lazy_static! { + static ref ARG_FIRST: String = String::from("first"); + static ref ARG_SKIP: String = String::from("skip"); + static ref ARG_ID: String = String::from("id"); +} + +/// Intermediate data structure to hold the results of prefetching entities +/// and their nested associations. For each association of `entity`, `children` +/// has an entry mapping the response key to the list of nodes. +#[derive(Debug, Clone)] +struct Node { + /// Estimate the size of the children using their `CacheWeight`. This + /// field will have the cache weight of the `entity` plus the weight of + /// the keys and values of the `children` map, but not of the map itself + children_weight: usize, + + entity: BTreeMap, + /// We are using an `Rc` here for two reasons: it allows us to defer + /// copying objects until the end, when converting to `q::Value` forces + /// us to copy any child that is referenced by multiple parents. It also + /// makes it possible to avoid unnecessary copying of a child that is + /// referenced by only one parent - without the `Rc` we would have to + /// copy since we do not know that only one parent uses it. + /// + /// Multiple parents can reference a single child in the following + /// situation: assume a GraphQL query `balances { token { issuer {id}}}` + /// where `balances` stores the `id` of the `token`, and `token` stores + /// the `id` of its `issuer`. Execution of the query when all `balances` + /// reference the same `token` will happen through several invocations + /// of `fetch`. For the purposes of this comment, we can think of + /// `fetch` as taking a list of `(parent_id, child_id)` pairs and + /// returning entities that are identified by this pair, i.e., there + /// will be one entity for each unique `(parent_id, child_id)` + /// combination, rather than one for each unique `child_id`. In reality, + /// of course, we will usually not know the `child_id` yet until we + /// actually run the query. + /// + /// Query execution works as follows: + /// 1. Fetch all `balances`, returning `#b` `Balance` entities. The + /// `Balance.token` field will be the same for all these entities. + /// 2. Fetch `#b` `Token` entities, identified through `(Balance.id, + /// Balance.token)` resulting in one `Token` entity + /// 3. Fetch 1 `Issuer` entity, identified through `(Token.id, + /// Token.issuer)` + /// 4. Glue all these results together into a DAG through invocations of + /// `Join::perform` + /// + /// We now have `#b` `Node` instances representing the same `Token`, but + /// each the child of a different `Node` for the `#b` balances. Each of + /// those `#b` `Token` nodes points to the same `Issuer` node. It's + /// important to note that the issuer node could itself be the root of a + /// large tree and could therefore take up a lot of memory. When we + /// convert this DAG into `q::Value`, we need to make `#b` copies of the + /// `Issuer` node. Using an `Rc` in `Node` allows us to defer these + /// copies to the point where we need to convert to `q::Value`, and it + /// would be desirable to base the data structure that GraphQL execution + /// uses on a DAG rather than a tree, but that's a good amount of work + children: BTreeMap>>, +} + +impl From> for Node { + fn from(entity: BTreeMap) -> Self { + Node { + children_weight: entity.weight(), + entity, + children: BTreeMap::default(), + } + } +} + +impl CacheWeight for Node { + fn indirect_weight(&self) -> usize { + self.children_weight + cache_weight::btree::node_size(&self.children) + } +} + +/// Convert a list of nodes into a `q::Value::List` where each node has also +/// been converted to a `q::Value` +fn node_list_as_value(nodes: Vec>) -> r::Value { + r::Value::List( + nodes + .into_iter() + .map(|node| Rc::try_unwrap(node).unwrap_or_else(|rc| rc.as_ref().clone())) + .map(Into::into) + .collect(), + ) +} + +/// We pass the root node of the result around as a vec of nodes, not as +/// a single node so that we can use the same functions on interior node +/// lists which are the result of querying the database. The root list +/// consists of exactly one entry, and that entry has an empty +/// (not even a `__typename`) entity. +/// +/// That distinguishes it from both the result of a query that matches +/// nothing (an empty `Vec`), and a result that finds just one entity +/// (the entity is not completely empty) +fn is_root_node<'a>(mut nodes: impl Iterator) -> bool { + if let Some(node) = nodes.next() { + node.entity.is_empty() + } else { + false + } +} + +fn make_root_node() -> Vec { + let entity = BTreeMap::new(); + vec![Node { + children_weight: entity.weight(), + entity, + children: BTreeMap::default(), + }] +} + +/// Recursively convert a `Node` into the corresponding `q::Value`, which is +/// always a `q::Value::Object`. The entity's associations are mapped to +/// entries `r:{response_key}` as that name is guaranteed to not conflict +/// with any field of the entity. +impl From for r::Value { + fn from(node: Node) -> Self { + let mut map = node.entity; + for (key, nodes) in node.children.into_iter() { + map.insert( + format!("prefetch:{}", key).into(), + node_list_as_value(nodes), + ); + } + r::Value::object(map) + } +} + +trait ValueExt { + fn as_str(&self) -> Option<&str>; +} + +impl ValueExt for r::Value { + fn as_str(&self) -> Option<&str> { + match self { + r::Value::String(s) => Some(s), + _ => None, + } + } +} + +impl Node { + fn id(&self) -> Result { + match self.get("id") { + None => Err(anyhow!("Entity is missing an `id` attribute")), + Some(r::Value::String(s)) => Ok(s.to_owned()), + _ => Err(anyhow!("Entity has non-string `id` attribute")), + } + } + + fn get(&self, key: &str) -> Option<&r::Value> { + self.entity.get(&key.into()) + } + + fn typename(&self) -> &str { + self.get("__typename") + .expect("all entities have a __typename") + .as_str() + .expect("__typename must be a string") + } + + fn set_children(&mut self, response_key: String, nodes: Vec>) { + fn nodes_weight(nodes: &Vec>) -> usize { + let vec_weight = nodes.capacity() * std::mem::size_of::>(); + let children_weight = nodes.iter().map(|node| node.weight()).sum::(); + vec_weight + children_weight + } + + let key_weight = response_key.weight(); + + self.children_weight += nodes_weight(&nodes) + key_weight; + let old = self.children.insert(response_key.into(), nodes); + if let Some(old) = old { + self.children_weight -= nodes_weight(&old) + key_weight; + } + } +} + +/// Describe a field that we join on. The distinction between scalar and +/// list is important for generating the right filter, and handling results +/// correctly +#[derive(Debug)] +enum JoinField<'a> { + List(&'a str), + Scalar(&'a str), +} + +impl<'a> JoinField<'a> { + fn new(field: &'a s::Field) -> Self { + let name = field.name.as_str(); + if sast::is_list_or_non_null_list_field(field) { + JoinField::List(name) + } else { + JoinField::Scalar(name) + } + } + + fn window_attribute(&self) -> WindowAttribute { + match self { + JoinField::Scalar(name) => WindowAttribute::Scalar(name.to_string()), + JoinField::List(name) => WindowAttribute::List(name.to_string()), + } + } +} + +#[derive(Debug)] +enum JoinRelation<'a> { + // Name of field in which child stores parent ids + Direct(JoinField<'a>), + // Name of the field in the parent type containing child ids + Derived(JoinField<'a>), +} + +#[derive(Debug)] +struct JoinCond<'a> { + /// The (concrete) object type of the parent, interfaces will have + /// one `JoinCond` for each implementing type + parent_type: EntityType, + /// The (concrete) object type of the child, interfaces will have + /// one `JoinCond` for each implementing type + child_type: EntityType, + relation: JoinRelation<'a>, +} + +impl<'a> JoinCond<'a> { + fn new( + parent_type: &'a s::ObjectType, + child_type: &'a s::ObjectType, + field_name: &str, + ) -> Self { + let field = parent_type + .field(field_name) + .expect("field_name is a valid field of parent_type"); + let relation = + if let Some(derived_from_field) = sast::get_derived_from_field(child_type, field) { + JoinRelation::Direct(JoinField::new(derived_from_field)) + } else { + JoinRelation::Derived(JoinField::new(field)) + }; + JoinCond { + parent_type: parent_type.into(), + child_type: child_type.into(), + relation, + } + } + + fn entity_link( + &self, + parents_by_id: Vec<(String, &Node)>, + multiplicity: ChildMultiplicity, + ) -> (Vec, EntityLink) { + match &self.relation { + JoinRelation::Direct(field) => { + // we only need the parent ids + let ids = parents_by_id.into_iter().map(|(id, _)| id).collect(); + ( + ids, + EntityLink::Direct(field.window_attribute(), multiplicity), + ) + } + JoinRelation::Derived(field) => { + let (ids, parent_link) = match field { + JoinField::Scalar(child_field) => { + // child_field contains a String id of the child; extract + // those and the parent ids + let (ids, child_ids): (Vec<_>, Vec<_>) = parents_by_id + .into_iter() + .filter_map(|(id, node)| { + node.get(*child_field) + .and_then(|value| value.as_str()) + .map(|child_id| (id, child_id.to_owned())) + }) + .unzip(); + + (ids, ParentLink::Scalar(child_ids)) + } + JoinField::List(child_field) => { + // child_field stores a list of child ids; extract them, + // turn them into a list of strings and combine with the + // parent ids + let (ids, child_ids): (Vec<_>, Vec<_>) = parents_by_id + .into_iter() + .filter_map(|(id, node)| { + node.get(*child_field) + .and_then(|value| match value { + r::Value::List(values) => { + let values: Vec<_> = values + .iter() + .filter_map(|value| { + value.as_str().map(|value| value.to_owned()) + }) + .collect(); + if values.is_empty() { + None + } else { + Some(values) + } + } + _ => None, + }) + .map(|child_ids| (id, child_ids)) + }) + .unzip(); + (ids, ParentLink::List(child_ids)) + } + }; + ( + ids, + EntityLink::Parent(self.parent_type.clone(), parent_link), + ) + } + } + } +} + +/// Encapsulate how we should join a list of parent entities with a list of +/// child entities. +#[derive(Debug)] +struct Join<'a> { + /// The object type of the child entities + child_type: ObjectOrInterface<'a>, + conds: Vec>, +} + +impl<'a> Join<'a> { + /// Construct a `Join` based on the parent field pointing to the child + fn new( + schema: &'a ApiSchema, + parent_type: &'a s::ObjectType, + child_type: ObjectOrInterface<'a>, + field_name: &str, + ) -> Self { + let child_types = child_type + .object_types(schema.schema()) + .expect("the name of the child type is valid"); + + let conds = child_types + .iter() + .map(|child_type| JoinCond::new(parent_type, child_type, field_name)) + .collect(); + + Join { child_type, conds } + } + + /// Perform the join. The child nodes are distributed into the parent nodes + /// according to the `parent_id` returned by the database in each child as + /// attribute `g$parent_id`, and are stored in the `response_key` entry + /// in each parent's `children` map. + /// + /// The `children` must contain the nodes in the correct order for each + /// parent; we simply pick out matching children for each parent but + /// otherwise maintain the order in `children` + fn perform(parents: &mut [&mut Node], children: Vec, response_key: &str) { + let children: Vec<_> = children.into_iter().map(Rc::new).collect(); + + if parents.len() == 1 { + let parent = parents.first_mut().expect("we just checked"); + parent.set_children(response_key.to_owned(), children); + return; + } + + // Build a map parent_id -> Vec that we will use to add + // children to their parent. This relies on the fact that interfaces + // make sure that id's are distinct across all implementations of the + // interface. + let mut grouped: BTreeMap<&str, Vec>> = BTreeMap::default(); + for child in children.iter() { + match child + .get("g$parent_id") + .expect("the query that produces 'child' ensures there is always a g$parent_id") + { + r::Value::String(key) => grouped.entry(key).or_default().push(child.clone()), + _ => unreachable!("the parent_id returned by the query is always a string"), + } + } + + // Add appropriate children using grouped map + for parent in parents { + // Set the `response_key` field in `parent`. Make sure that even if `parent` has no + // matching `children`, the field gets set (to an empty `Vec`). + // + // This `insert` will overwrite in the case where the response key occurs both at the + // interface level and in nested object type conditions. The values for the interface + // query are always joined first, and may then be overwritten by the merged selection + // set under the object type condition. See also: e0d6da3e-60cf-41a5-b83c-b60a7a766d4a + let values = parent.id().ok().and_then(|id| grouped.get(&*id).cloned()); + parent.set_children(response_key.to_owned(), values.unwrap_or(vec![])); + } + } + + fn windows( + &self, + parents: &[&mut Node], + multiplicity: ChildMultiplicity, + previous_collection: &EntityCollection, + ) -> Vec { + let mut windows = vec![]; + let column_names_map = previous_collection.entity_types_and_column_names(); + for cond in &self.conds { + let mut parents_by_id = parents + .iter() + .filter(|parent| parent.typename() == cond.parent_type.as_str()) + .filter_map(|parent| parent.id().ok().map(|id| (id, &**parent))) + .collect::>(); + + if !parents_by_id.is_empty() { + parents_by_id.sort_unstable_by(|(id1, _), (id2, _)| id1.cmp(id2)); + parents_by_id.dedup_by(|(id1, _), (id2, _)| id1 == id2); + + let (ids, link) = cond.entity_link(parents_by_id, multiplicity); + let child_type: EntityType = cond.child_type.to_owned(); + let column_names = match column_names_map.get(&child_type) { + Some(column_names) => column_names.clone(), + None => AttributeNames::All, + }; + windows.push(EntityWindow { + child_type, + ids, + link, + column_names, + }); + } + } + windows + } +} + +/// Run the query in `ctx` in such a manner that we only perform one query +/// per 'level' in the query. A query like `musicians { id bands { id } }` +/// will perform two queries: one for musicians, and one for bands, regardless +/// of how many musicians there are. +/// +/// The returned value contains a `q::Value::Object` that contains a tree of +/// all the entities (converted into objects) in the form in which they need +/// to be returned. Nested object fields appear under the key `r:response_key` +/// in these objects, and are always `q::Value::List` of objects. +/// +/// For the above example, the returned object would have one entry under +/// `r:musicians`, which is a list of all the musicians; each musician has an +/// entry `r:bands` that contains a list of the bands for that musician. Note +/// that even for single-object fields, we return a list so that we can spot +/// cases where the store contains data that violates the data model by having +/// multiple values for what should be a relationship to a single object in +/// @derivedFrom fields +pub fn run( + resolver: &StoreResolver, + ctx: &ExecutionContext, + selection_set: &a::SelectionSet, + graphql_metrics: &GraphQLMetrics, +) -> Result<(r::Value, Trace), Vec> { + execute_root_selection_set(resolver, ctx, selection_set).map(|(nodes, trace)| { + graphql_metrics.observe_query_result_size(nodes.weight()); + let obj = Object::from_iter( + nodes + .into_iter() + .map(|node| { + node.children.into_iter().map(|(key, nodes)| { + (format!("prefetch:{}", key), node_list_as_value(nodes)) + }) + }) + .flatten(), + ); + (r::Value::Object(obj), trace) + }) +} + +/// Executes the root selection set of a query. +fn execute_root_selection_set( + resolver: &StoreResolver, + ctx: &ExecutionContext, + selection_set: &a::SelectionSet, +) -> Result<(Vec, Trace), Vec> { + let trace = Trace::root(ctx.query.query_text.clone()); + // Execute the root selection set against the root query type + execute_selection_set(resolver, ctx, make_root_node(), trace, selection_set) +} + +fn check_result_size<'a>( + ctx: &'a ExecutionContext, + size: usize, +) -> Result<(), QueryExecutionError> { + if size > ENV_VARS.graphql.error_result_size { + return Err(QueryExecutionError::ResultTooBig( + size, + ENV_VARS.graphql.error_result_size, + )); + } + if size > ENV_VARS.graphql.warn_result_size { + warn!(ctx.logger, "Large query result"; "size" => size, "query_id" => &ctx.query.query_id); + } + Ok(()) +} + +fn execute_selection_set<'a>( + resolver: &StoreResolver, + ctx: &'a ExecutionContext, + mut parents: Vec, + mut parent_trace: Trace, + selection_set: &a::SelectionSet, +) -> Result<(Vec, Trace), Vec> { + let schema = &ctx.query.schema; + let mut errors: Vec = Vec::new(); + + // Process all field groups in order + for (object_type, fields) in selection_set.interior_fields() { + if let Some(deadline) = ctx.deadline { + if deadline < Instant::now() { + errors.push(QueryExecutionError::Timeout); + break; + } + } + + // Filter out parents that do not match the type condition. + let mut parents: Vec<&mut Node> = if is_root_node(parents.iter()) { + parents.iter_mut().collect() + } else { + parents + .iter_mut() + .filter(|p| object_type.name == p.typename()) + .collect() + }; + + if parents.is_empty() { + continue; + } + + for field in fields { + let field_type = object_type + .field(&field.name) + .expect("field names are valid"); + let child_type = schema + .object_or_interface(field_type.field_type.get_base_type()) + .expect("we only collect fields that are objects or interfaces"); + + let join = Join::new( + ctx.query.schema.as_ref(), + object_type, + child_type, + &field.name, + ); + + // "Select by Specific Attribute Names" is an experimental feature and can be disabled completely. + // If this environment variable is set, the program will use an empty collection that, + // effectively, causes the `AttributeNames::All` variant to be used as a fallback value for all + // queries. + let collected_columns = if !ENV_VARS.enable_select_by_specific_attributes { + SelectedAttributes(BTreeMap::new()) + } else { + SelectedAttributes::for_field(field)? + }; + + match execute_field( + resolver, + ctx, + &parents, + &join, + field, + field_type, + collected_columns, + ) { + Ok((children, trace)) => { + match execute_selection_set( + resolver, + ctx, + children, + trace, + &field.selection_set, + ) { + Ok((children, trace)) => { + Join::perform(&mut parents, children, field.response_key()); + let weight = + parents.iter().map(|parent| parent.weight()).sum::(); + check_result_size(ctx, weight)?; + parent_trace.push(field.response_key(), trace); + } + Err(mut e) => errors.append(&mut e), + } + } + Err(mut e) => { + errors.append(&mut e); + } + }; + } + } + + if errors.is_empty() { + Ok((parents, parent_trace)) + } else { + Err(errors) + } +} + +/// Executes a field. +fn execute_field( + resolver: &StoreResolver, + ctx: &ExecutionContext, + parents: &[&mut Node], + join: &Join<'_>, + field: &a::Field, + field_definition: &s::Field, + selected_attrs: SelectedAttributes, +) -> Result<(Vec, Trace), Vec> { + let multiplicity = if sast::is_list_or_non_null_list_field(field_definition) { + ChildMultiplicity::Many + } else { + ChildMultiplicity::Single + }; + + fetch( + ctx.logger.clone(), + resolver.store.as_ref(), + parents, + join, + ctx.query.schema.as_ref(), + field, + multiplicity, + ctx.query.schema.types_for_interface(), + resolver.block_number(), + ctx.max_first, + ctx.max_skip, + ctx.query.query_id.clone(), + selected_attrs, + ) + .map_err(|e| vec![e]) +} + +/// Query child entities for `parents` from the store. The `join` indicates +/// in which child field to look for the parent's id/join field. When +/// `is_single` is `true`, there is at most one child per parent. +fn fetch( + logger: Logger, + store: &(impl QueryStore + ?Sized), + parents: &[&mut Node], + join: &Join<'_>, + schema: &ApiSchema, + field: &a::Field, + multiplicity: ChildMultiplicity, + types_for_interface: &BTreeMap>, + block: BlockNumber, + max_first: u32, + max_skip: u32, + query_id: String, + selected_attrs: SelectedAttributes, +) -> Result<(Vec, Trace), QueryExecutionError> { + let mut query = build_query( + join.child_type, + block, + field, + types_for_interface, + max_first, + max_skip, + selected_attrs, + schema, + )?; + query.query_id = Some(query_id); + + if multiplicity == ChildMultiplicity::Single { + // Suppress 'order by' in lookups of scalar values since + // that causes unnecessary work in the database + query.order = EntityOrder::Unordered; + } + + query.logger = Some(logger); + if let Some(r::Value::String(id)) = field.argument_value(ARG_ID.as_str()) { + query.filter = Some( + EntityFilter::Equal(ARG_ID.to_owned(), StoreValue::from(id.to_owned())) + .and_maybe(query.filter), + ); + } + + if !is_root_node(parents.iter().map(|p| &**p)) { + // For anything but the root node, restrict the children we select + // by the parent list + let windows = join.windows(parents, multiplicity, &query.collection); + if windows.is_empty() { + return Ok((vec![], Trace::None)); + } + query.collection = EntityCollection::Window(windows); + } + store.find_query_values(query).map(|(values, trace)| { + ( + values.into_iter().map(|entity| entity.into()).collect(), + trace, + ) + }) +} + +#[derive(Debug, Default, Clone)] +pub(crate) struct SelectedAttributes(BTreeMap); + +impl SelectedAttributes { + /// Extract the attributes we should select from `selection_set`. In + /// particular, disregard derived fields since they are not stored + fn for_field(field: &a::Field) -> Result> { + let mut map = BTreeMap::new(); + for (object_type, fields) in field.selection_set.fields() { + let column_names = fields + .filter(|field| { + // Keep fields that are not derived and for which we + // can find the field type + sast::get_field(object_type, &field.name) + .map(|field_type| !field_type.is_derived()) + .unwrap_or(false) + }) + .filter_map(|field| { + if field.name.starts_with("__") { + None + } else { + Some(field.name.clone()) + } + }) + .collect(); + map.insert( + object_type.name().to_string(), + AttributeNames::Select(column_names), + ); + } + // We need to also select the `orderBy` field if there is one. + // Because of how the API Schema is set up, `orderBy` can only have + // an enum value + match field.argument_value("orderBy") { + None => { /* nothing to do */ } + Some(r::Value::Enum(e)) => { + for columns in map.values_mut() { + columns.add_str(e); + } + } + Some(v) => { + return Err(vec![constraint_violation!( + "'orderBy' attribute must be an enum but is {:?}", + v + ) + .into()]); + } + } + Ok(SelectedAttributes(map)) + } + + pub fn get(&mut self, obj_type: &s::ObjectType) -> AttributeNames { + self.0.remove(&obj_type.name).unwrap_or(AttributeNames::All) + } +} diff --git a/graphql/src/store/query.rs b/graphql/src/store/query.rs new file mode 100644 index 0000000..c31b00b --- /dev/null +++ b/graphql/src/store/query.rs @@ -0,0 +1,952 @@ +use std::collections::{BTreeMap, BTreeSet, HashSet, VecDeque}; +use std::mem::discriminant; + +use graph::data::graphql::ext::DirectiveFinder; +use graph::data::graphql::TypeExt as _; +use graph::data::value::Object; +use graph::data::value::Value as DataValue; +use graph::prelude::*; +use graph::{components::store::EntityType, data::graphql::ObjectOrInterface}; + +use crate::execution::ast as a; +use crate::schema::ast as sast; + +use super::prefetch::SelectedAttributes; + +#[derive(Debug)] +enum OrderDirection { + Ascending, + Descending, +} + +/// Builds a EntityQuery from GraphQL arguments. +/// +/// Panics if `entity` is not present in `schema`. +pub(crate) fn build_query<'a>( + entity: impl Into>, + block: BlockNumber, + field: &a::Field, + types_for_interface: &'a BTreeMap>, + max_first: u32, + max_skip: u32, + mut column_names: SelectedAttributes, + schema: &ApiSchema, +) -> Result { + let entity = entity.into(); + let entity_types = EntityCollection::All(match &entity { + ObjectOrInterface::Object(object) => { + let selected_columns = column_names.get(object); + vec![((*object).into(), selected_columns)] + } + ObjectOrInterface::Interface(interface) => types_for_interface + [&EntityType::from(*interface)] + .iter() + .map(|o| { + let selected_columns = column_names.get(o); + (o.into(), selected_columns) + }) + .collect(), + }); + let mut query = EntityQuery::new(parse_subgraph_id(entity)?, block, entity_types) + .range(build_range(field, max_first, max_skip)?); + if let Some(filter) = build_filter(entity, field, schema)? { + query = query.filter(filter); + } + let order = match ( + build_order_by(entity, field)?, + build_order_direction(field)?, + ) { + (Some((attr, value_type)), OrderDirection::Ascending) => { + EntityOrder::Ascending(attr, value_type) + } + (Some((attr, value_type)), OrderDirection::Descending) => { + EntityOrder::Descending(attr, value_type) + } + (None, _) => EntityOrder::Default, + }; + query = query.order(order); + Ok(query) +} + +/// Parses GraphQL arguments into a EntityRange, if present. +fn build_range( + field: &a::Field, + max_first: u32, + max_skip: u32, +) -> Result { + let first = match field.argument_value("first") { + Some(r::Value::Int(n)) => { + let n = *n; + if n > 0 && n <= (max_first as i64) { + n as u32 + } else { + return Err(QueryExecutionError::RangeArgumentsError( + "first", max_first, n, + )); + } + } + Some(r::Value::Null) | None => 100, + _ => unreachable!("first is an Int with a default value"), + }; + + let skip = match field.argument_value("skip") { + Some(r::Value::Int(n)) => { + let n = *n; + if n >= 0 && n <= (max_skip as i64) { + n as u32 + } else { + return Err(QueryExecutionError::RangeArgumentsError( + "skip", max_skip, n, + )); + } + } + Some(r::Value::Null) | None => 0, + _ => unreachable!("skip is an Int with a default value"), + }; + + Ok(EntityRange { + first: Some(first), + skip, + }) +} + +/// Parses GraphQL arguments into an EntityFilter, if present. +fn build_filter( + entity: ObjectOrInterface, + field: &a::Field, + schema: &ApiSchema, +) -> Result, QueryExecutionError> { + match field.argument_value("where") { + Some(r::Value::Object(object)) => match build_filter_from_object(entity, object, schema) { + Ok(filter) => Ok(Some(filter)), + Err(e) => Err(e), + }, + Some(r::Value::Null) => Ok(None), + None => match field.argument_value("text") { + Some(r::Value::Object(filter)) => build_fulltext_filter_from_object(filter), + None => Ok(None), + _ => Err(QueryExecutionError::InvalidFilterError), + }, + _ => Err(QueryExecutionError::InvalidFilterError), + } +} + +fn build_fulltext_filter_from_object( + object: &Object, +) -> Result, QueryExecutionError> { + object.iter().next().map_or( + Err(QueryExecutionError::FulltextQueryRequiresFilter), + |(key, value)| { + if let r::Value::String(s) = value { + Ok(Some(EntityFilter::Equal( + key.to_string(), + Value::String(s.clone()), + ))) + } else { + Err(QueryExecutionError::FulltextQueryRequiresFilter) + } + }, + ) +} + +fn parse_change_block_filter(value: &r::Value) -> Result { + match value { + r::Value::Object(object) => i32::try_from_value( + object + .get("number_gte") + .ok_or_else(|| QueryExecutionError::InvalidFilterError)?, + ) + .map_err(|_| QueryExecutionError::InvalidFilterError), + _ => Err(QueryExecutionError::InvalidFilterError), + } +} + +/// Parses a GraphQL input object into an EntityFilter, if present. +fn build_filter_from_object( + entity: ObjectOrInterface, + object: &Object, + schema: &ApiSchema, +) -> Result { + Ok(EntityFilter::And({ + object + .iter() + .map(|(key, value)| { + // Special handling for _change_block input filter since its not a + // standard entity filter that is based on entity structure/fields + if key == "_change_block" { + return match parse_change_block_filter(value) { + Ok(block_number) => Ok(EntityFilter::ChangeBlockGte(block_number)), + Err(e) => Err(e), + }; + } + + use self::sast::FilterOp::*; + let (field_name, op) = sast::parse_field_as_filter(key); + + let field = sast::get_field(entity, &field_name).ok_or_else(|| { + QueryExecutionError::EntityFieldError( + entity.name().to_owned(), + field_name.clone(), + ) + })?; + + let ty = &field.field_type; + + Ok(match op { + Child => match value { + DataValue::Object(obj) => { + build_child_filter_from_object(entity, field_name, obj, schema)? + } + _ => { + return Err(QueryExecutionError::AttributeTypeError( + value.to_string(), + ty.to_string(), + )) + } + }, + _ => { + let store_value = Value::from_query_value(value, ty)?; + + match op { + Not => EntityFilter::Not(field_name, store_value), + GreaterThan => EntityFilter::GreaterThan(field_name, store_value), + LessThan => EntityFilter::LessThan(field_name, store_value), + GreaterOrEqual => EntityFilter::GreaterOrEqual(field_name, store_value), + LessOrEqual => EntityFilter::LessOrEqual(field_name, store_value), + In => EntityFilter::In(field_name, list_values(store_value, "_in")?), + NotIn => EntityFilter::NotIn( + field_name, + list_values(store_value, "_not_in")?, + ), + Contains => EntityFilter::Contains(field_name, store_value), + ContainsNoCase => EntityFilter::ContainsNoCase(field_name, store_value), + NotContains => EntityFilter::NotContains(field_name, store_value), + NotContainsNoCase => { + EntityFilter::NotContainsNoCase(field_name, store_value) + } + StartsWith => EntityFilter::StartsWith(field_name, store_value), + StartsWithNoCase => { + EntityFilter::StartsWithNoCase(field_name, store_value) + } + NotStartsWith => EntityFilter::NotStartsWith(field_name, store_value), + NotStartsWithNoCase => { + EntityFilter::NotStartsWithNoCase(field_name, store_value) + } + EndsWith => EntityFilter::EndsWith(field_name, store_value), + EndsWithNoCase => EntityFilter::EndsWithNoCase(field_name, store_value), + NotEndsWith => EntityFilter::NotEndsWith(field_name, store_value), + NotEndsWithNoCase => { + EntityFilter::NotEndsWithNoCase(field_name, store_value) + } + Equal => EntityFilter::Equal(field_name, store_value), + _ => unreachable!(), + } + } + }) + }) + .collect::, QueryExecutionError>>()? + })) +} + +fn build_child_filter_from_object( + entity: ObjectOrInterface, + field_name: String, + object: &Object, + schema: &ApiSchema, +) -> Result { + let field = entity + .field(&field_name) + .ok_or(QueryExecutionError::InvalidFilterError)?; + let type_name = &field.field_type.get_base_type(); + let child_entity = schema + .object_or_interface(type_name) + .ok_or(QueryExecutionError::InvalidFilterError)?; + let filter = Box::new(build_filter_from_object(child_entity, object, schema)?); + let derived = field.is_derived(); + let attr = match derived { + true => sast::get_derived_from_field(child_entity, field) + .ok_or(QueryExecutionError::InvalidFilterError)? + .name + .to_string(), + false => field_name.clone(), + }; + + if child_entity.is_interface() { + Ok(EntityFilter::Or( + child_entity + .object_types(schema.schema()) + .ok_or(QueryExecutionError::AbstractTypeError( + "Interface is not implemented by any types".to_string(), + ))? + .iter() + .map(|object_type| { + EntityFilter::Child(Child { + attr: attr.clone(), + entity_type: EntityType::new(object_type.name.to_string()), + filter: filter.clone(), + derived, + }) + }) + .collect(), + )) + } else if entity.is_interface() { + Ok(EntityFilter::Or( + entity + .object_types(schema.schema()) + .ok_or(QueryExecutionError::AbstractTypeError( + "Interface is not implemented by any types".to_string(), + ))? + .iter() + .map(|object_type| { + let field = object_type + .fields + .iter() + .find(|f| f.name == field_name.clone()) + .ok_or(QueryExecutionError::InvalidFilterError)?; + let derived = field.is_derived(); + + let attr = match derived { + true => sast::get_derived_from_field(child_entity, field) + .ok_or(QueryExecutionError::InvalidFilterError)? + .name + .to_string(), + false => field_name.clone(), + }; + + Ok(EntityFilter::Child(Child { + attr: attr.clone(), + entity_type: EntityType::new(child_entity.name().to_string()), + filter: filter.clone(), + derived, + })) + }) + .collect::, QueryExecutionError>>()?, + )) + } else { + Ok(EntityFilter::Child(Child { + attr, + entity_type: EntityType::new(type_name.to_string()), + filter, + derived, + })) + } +} + +/// Parses a list of GraphQL values into a vector of entity field values. +fn list_values(value: Value, filter_type: &str) -> Result, QueryExecutionError> { + match value { + Value::List(ref values) if !values.is_empty() => { + // Check that all values in list are of the same type + let root_discriminant = discriminant(&values[0]); + values + .iter() + .map(|value| { + let current_discriminant = discriminant(value); + if root_discriminant == current_discriminant { + Ok(value.clone()) + } else { + Err(QueryExecutionError::ListTypesError( + filter_type.to_string(), + vec![values[0].to_string(), value.to_string()], + )) + } + }) + .collect::, _>>() + } + Value::List(ref values) if values.is_empty() => Ok(vec![]), + _ => Err(QueryExecutionError::ListFilterError( + filter_type.to_string(), + )), + } +} + +/// Parses GraphQL arguments into an field name to order by, if present. +fn build_order_by( + entity: ObjectOrInterface, + field: &a::Field, +) -> Result, QueryExecutionError> { + match field.argument_value("orderBy") { + Some(r::Value::Enum(name)) => { + let field = sast::get_field(entity, name).ok_or_else(|| { + QueryExecutionError::EntityFieldError(entity.name().to_owned(), name.clone()) + })?; + sast::get_field_value_type(&field.field_type) + .map(|value_type| Some((name.to_owned(), value_type))) + .map_err(|_| { + QueryExecutionError::OrderByNotSupportedError( + entity.name().to_owned(), + name.clone(), + ) + }) + } + _ => match field.argument_value("text") { + Some(r::Value::Object(filter)) => build_fulltext_order_by_from_object(filter), + None => Ok(None), + _ => Err(QueryExecutionError::InvalidFilterError), + }, + } +} + +fn build_fulltext_order_by_from_object( + object: &Object, +) -> Result, QueryExecutionError> { + object.iter().next().map_or( + Err(QueryExecutionError::FulltextQueryRequiresFilter), + |(key, value)| { + if let r::Value::String(_) = value { + Ok(Some((key.to_string(), ValueType::String))) + } else { + Err(QueryExecutionError::FulltextQueryRequiresFilter) + } + }, + ) +} + +/// Parses GraphQL arguments into a EntityOrder, if present. +fn build_order_direction(field: &a::Field) -> Result { + Ok(field + .argument_value("orderDirection") + .map(|value| match value { + r::Value::Enum(name) if name == "asc" => OrderDirection::Ascending, + r::Value::Enum(name) if name == "desc" => OrderDirection::Descending, + _ => OrderDirection::Ascending, + }) + .unwrap_or(OrderDirection::Ascending)) +} + +/// Parses the subgraph ID from the ObjectType directives. +pub fn parse_subgraph_id<'a>( + entity: impl Into>, +) -> Result { + let entity = entity.into(); + let entity_name = entity.name(); + entity + .directives() + .iter() + .find(|directive| directive.name == "subgraphId") + .and_then(|directive| directive.arguments.iter().find(|(name, _)| name == "id")) + .and_then(|(_, value)| match value { + s::Value::String(id) => Some(id), + _ => None, + }) + .ok_or(()) + .and_then(|id| DeploymentHash::new(id).map_err(|_| ())) + .map_err(|_| QueryExecutionError::SubgraphDeploymentIdError(entity_name.to_owned())) +} + +/// Recursively collects entities involved in a query field as `(subgraph ID, name)` tuples. +pub(crate) fn collect_entities_from_query_field( + schema: &ApiSchema, + object_type: sast::ObjectType, + field: &a::Field, +) -> Result, QueryExecutionError> { + // Output entities + let mut entities = HashSet::new(); + + // List of objects/fields to visit next + let mut queue = VecDeque::new(); + queue.push_back((object_type, field)); + + while let Some((object_type, field)) = queue.pop_front() { + // Check if the field exists on the object type + if let Some(field_type) = sast::get_field(&object_type, &field.name) { + // Check if the field type corresponds to a type definition (in a valid schema, + // this should always be the case) + if let Some(type_definition) = schema.get_type_definition_from_field(field_type) { + // If the field's type definition is an object type, extract that type + if let s::TypeDefinition::Object(object_type) = type_definition { + // Only collect whether the field's type has an @entity directive + if sast::get_object_type_directive(object_type, String::from("entity")) + .is_some() + { + // Obtain the subgraph ID from the object type + if let Ok(subgraph_id) = parse_subgraph_id(object_type) { + // Add the (subgraph_id, entity_name) tuple to the result set + entities.insert((subgraph_id, object_type.name.to_owned())); + } + } + + // If the query field has a non-empty selection set, this means we + // need to recursively process it + let object_type = schema.object_type(object_type).into(); + for sub_field in field.selection_set.fields_for(&object_type)? { + queue.push_back((object_type.cheap_clone(), sub_field)) + } + } + } + } + } + + Ok(entities + .into_iter() + .map(|(id, entity_type)| SubscriptionFilter::Entities(id, EntityType::new(entity_type))) + .collect()) +} + +#[cfg(test)] +mod tests { + use graph::{ + components::store::EntityType, + data::value::Object, + prelude::{ + r, ApiSchema, AttributeNames, DeploymentHash, EntityCollection, EntityFilter, + EntityRange, Schema, Value, ValueType, BLOCK_NUMBER_MAX, + }, + prelude::{ + s::{self, Directive, Field, InputValue, ObjectType, Type, Value as SchemaValue}, + EntityOrder, + }, + }; + use graphql_parser::Pos; + use std::{collections::BTreeMap, iter::FromIterator, sync::Arc}; + + use super::{a, build_query}; + + fn default_object() -> ObjectType { + let subgraph_id_argument = ( + String::from("id"), + s::Value::String("QmZ5dsusHwD1PEbx6L4dLCWkDsk1BLhrx9mPsGyPvTxPCM".to_string()), + ); + let subgraph_id_directive = Directive { + name: "subgraphId".to_string(), + position: Pos::default(), + arguments: vec![subgraph_id_argument], + }; + let name_input_value = InputValue { + position: Pos::default(), + description: Some("name input".to_string()), + name: "name".to_string(), + value_type: Type::NamedType("String".to_string()), + default_value: Some(SchemaValue::String("name".to_string())), + directives: vec![], + }; + let name_field = Field { + position: Pos::default(), + description: Some("name field".to_string()), + name: "name".to_string(), + arguments: vec![name_input_value.clone()], + field_type: Type::NamedType("String".to_string()), + directives: vec![], + }; + let email_field = Field { + position: Pos::default(), + description: Some("email field".to_string()), + name: "email".to_string(), + arguments: vec![name_input_value], + field_type: Type::NamedType("String".to_string()), + directives: vec![], + }; + + ObjectType { + position: Default::default(), + description: None, + name: String::new(), + implements_interfaces: vec![], + directives: vec![subgraph_id_directive], + fields: vec![name_field, email_field], + } + } + + fn object(name: &str) -> ObjectType { + ObjectType { + name: name.to_owned(), + ..default_object() + } + } + + fn field(name: &str, field_type: Type) -> Field { + Field { + position: Default::default(), + description: None, + name: name.to_owned(), + arguments: vec![], + field_type, + directives: vec![], + } + } + + fn default_field() -> a::Field { + let arguments = vec![ + ("first".to_string(), r::Value::Int(100.into())), + ("skip".to_string(), r::Value::Int(0.into())), + ]; + let obj_type = Arc::new(object("SomeType")).into(); + a::Field { + position: Default::default(), + alias: None, + name: "aField".to_string(), + arguments, + directives: vec![], + selection_set: a::SelectionSet::new(vec![obj_type]), + } + } + + fn default_field_with(arg_name: &str, arg_value: r::Value) -> a::Field { + let mut field = default_field(); + field.arguments.push((arg_name.to_string(), arg_value)); + field + } + + fn default_field_with_vec(args: Vec<(&str, r::Value)>) -> a::Field { + let mut field = default_field(); + for (name, value) in args { + field.arguments.push((name.to_string(), value)); + } + field + } + + fn build_schema(raw_schema: &str) -> ApiSchema { + let document = graphql_parser::parse_schema(raw_schema) + .expect("Failed to parse raw schema") + .into_static(); + + let schema = Schema::new(DeploymentHash::new("id").unwrap(), document).unwrap(); + ApiSchema::from_api_schema(schema).expect("Failed to build schema") + } + + fn build_default_schema() -> ApiSchema { + build_schema( + r#" + type Query { + aField(first: Int, skip: Int): [SomeType] + } + + type SomeType @entity { + id: ID! + name: String! + } + "#, + ) + } + + #[test] + fn build_query_uses_the_entity_name() { + let schema = build_default_schema(); + assert_eq!( + build_query( + &object("Entity1"), + BLOCK_NUMBER_MAX, + &default_field(), + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema + ) + .unwrap() + .collection, + EntityCollection::All(vec![(EntityType::from("Entity1"), AttributeNames::All)]) + ); + assert_eq!( + build_query( + &object("Entity2"), + BLOCK_NUMBER_MAX, + &default_field(), + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema, + ) + .unwrap() + .collection, + EntityCollection::All(vec![(EntityType::from("Entity2"), AttributeNames::All)]) + ); + } + + #[test] + fn build_query_yields_no_order_if_order_arguments_are_missing() { + let schema = build_default_schema(); + assert_eq!( + build_query( + &default_object(), + BLOCK_NUMBER_MAX, + &default_field(), + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema, + ) + .unwrap() + .order, + EntityOrder::Default, + ); + } + + #[test] + fn build_query_parses_order_by_from_enum_values_correctly() { + let schema = build_default_schema(); + let field = default_field_with("orderBy", r::Value::Enum("name".to_string())); + assert_eq!( + build_query( + &default_object(), + BLOCK_NUMBER_MAX, + &field, + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema, + ) + .unwrap() + .order, + EntityOrder::Ascending("name".to_string(), ValueType::String) + ); + + let field = default_field_with("orderBy", r::Value::Enum("email".to_string())); + assert_eq!( + build_query( + &default_object(), + BLOCK_NUMBER_MAX, + &field, + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema, + ) + .unwrap() + .order, + EntityOrder::Ascending("email".to_string(), ValueType::String) + ); + } + + #[test] + fn build_query_ignores_order_by_from_non_enum_values() { + let schema = build_default_schema(); + let field = default_field_with("orderBy", r::Value::String("name".to_string())); + assert_eq!( + build_query( + &default_object(), + BLOCK_NUMBER_MAX, + &field, + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema + ) + .unwrap() + .order, + EntityOrder::Default + ); + + let field = default_field_with("orderBy", r::Value::String("email".to_string())); + assert_eq!( + build_query( + &default_object(), + BLOCK_NUMBER_MAX, + &field, + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema, + ) + .unwrap() + .order, + EntityOrder::Default + ); + } + + #[test] + fn build_query_parses_order_direction_from_enum_values_correctly() { + let schema = build_default_schema(); + let field = default_field_with_vec(vec![ + ("orderBy", r::Value::Enum("name".to_string())), + ("orderDirection", r::Value::Enum("asc".to_string())), + ]); + assert_eq!( + build_query( + &default_object(), + BLOCK_NUMBER_MAX, + &field, + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema, + ) + .unwrap() + .order, + EntityOrder::Ascending("name".to_string(), ValueType::String) + ); + + let field = default_field_with_vec(vec![ + ("orderBy", r::Value::Enum("name".to_string())), + ("orderDirection", r::Value::Enum("desc".to_string())), + ]); + assert_eq!( + build_query( + &default_object(), + BLOCK_NUMBER_MAX, + &field, + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema, + ) + .unwrap() + .order, + EntityOrder::Descending("name".to_string(), ValueType::String) + ); + + let field = default_field_with_vec(vec![ + ("orderBy", r::Value::Enum("name".to_string())), + ( + "orderDirection", + r::Value::Enum("descending...".to_string()), + ), + ]); + assert_eq!( + build_query( + &default_object(), + BLOCK_NUMBER_MAX, + &field, + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema, + ) + .unwrap() + .order, + EntityOrder::Ascending("name".to_string(), ValueType::String) + ); + + // No orderBy -> EntityOrder::Default + let field = default_field_with( + "orderDirection", + r::Value::Enum("descending...".to_string()), + ); + assert_eq!( + build_query( + &default_object(), + BLOCK_NUMBER_MAX, + &field, + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema + ) + .unwrap() + .order, + EntityOrder::Default + ); + } + + #[test] + fn build_query_yields_default_range_if_none_is_present() { + let schema = build_default_schema(); + assert_eq!( + build_query( + &default_object(), + BLOCK_NUMBER_MAX, + &default_field(), + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema + ) + .unwrap() + .range, + EntityRange::first(100) + ); + } + + #[test] + fn build_query_yields_default_first_if_only_skip_is_present() { + let schema = build_default_schema(); + let mut field = default_field(); + field.arguments = vec![("skip".to_string(), r::Value::Int(50))]; + + assert_eq!( + build_query( + &default_object(), + BLOCK_NUMBER_MAX, + &field, + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema + ) + .unwrap() + .range, + EntityRange { + first: Some(100), + skip: 50, + }, + ); + } + + #[test] + fn build_query_yields_filters() { + let schema = build_default_schema(); + let query_field = default_field_with( + "where", + r::Value::Object(Object::from_iter(vec![( + "name_ends_with".to_string(), + r::Value::String("ello".to_string()), + )])), + ); + assert_eq!( + build_query( + &ObjectType { + fields: vec![field("name", Type::NamedType("string".to_owned()))], + ..default_object() + }, + BLOCK_NUMBER_MAX, + &query_field, + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema + ) + .unwrap() + .filter, + Some(EntityFilter::And(vec![EntityFilter::EndsWith( + "name".to_string(), + Value::String("ello".to_string()), + )])) + ) + } + + #[test] + fn build_query_yields_block_change_gte_filter() { + let schema = build_default_schema(); + let query_field = default_field_with( + "where", + r::Value::Object(Object::from_iter(vec![( + "_change_block".to_string(), + r::Value::Object(Object::from_iter(vec![( + "number_gte".to_string(), + r::Value::Int(10), + )])), + )])), + ); + assert_eq!( + build_query( + &ObjectType { + fields: vec![field("name", Type::NamedType("string".to_owned()))], + ..default_object() + }, + BLOCK_NUMBER_MAX, + &query_field, + &BTreeMap::new(), + std::u32::MAX, + std::u32::MAX, + Default::default(), + &schema + ) + .unwrap() + .filter, + Some(EntityFilter::And(vec![EntityFilter::ChangeBlockGte(10)])) + ) + } +} diff --git a/graphql/src/store/resolver.rs b/graphql/src/store/resolver.rs new file mode 100644 index 0000000..d95a626 --- /dev/null +++ b/graphql/src/store/resolver.rs @@ -0,0 +1,390 @@ +use std::collections::BTreeMap; +use std::result; +use std::sync::Arc; + +use graph::data::query::Trace; +use graph::data::value::Object; +use graph::data::{ + graphql::{object, ObjectOrInterface}, + schema::META_FIELD_TYPE, +}; +use graph::prelude::*; +use graph::{components::store::*, data::schema::BLOCK_FIELD_TYPE}; + +use crate::execution::ast as a; +use crate::metrics::GraphQLMetrics; +use crate::query::ext::BlockConstraint; +use crate::schema::ast as sast; +use crate::{prelude::*, schema::api::ErrorPolicy}; + +use crate::store::query::collect_entities_from_query_field; + +/// A resolver that fetches entities from a `Store`. +#[derive(Clone)] +pub struct StoreResolver { + #[allow(dead_code)] + logger: Logger, + pub(crate) store: Arc, + subscription_manager: Arc, + pub(crate) block_ptr: Option, + deployment: DeploymentHash, + has_non_fatal_errors: bool, + error_policy: ErrorPolicy, + graphql_metrics: Arc, +} + +#[derive(Clone, Debug)] +pub(crate) struct BlockPtrTs { + pub ptr: BlockPtr, + pub timestamp: Option, +} + +impl From for BlockPtrTs { + fn from(ptr: BlockPtr) -> Self { + Self { + ptr, + timestamp: None, + } + } +} + +impl From<&BlockPtrTs> for BlockPtr { + fn from(ptr: &BlockPtrTs) -> Self { + ptr.ptr.cheap_clone() + } +} + +impl CheapClone for StoreResolver {} + +impl StoreResolver { + /// Create a resolver that looks up entities at whatever block is the + /// latest when the query is run. That means that multiple calls to find + /// entities into this resolver might return entities from different + /// blocks + pub fn for_subscription( + logger: &Logger, + deployment: DeploymentHash, + store: Arc, + subscription_manager: Arc, + graphql_metrics: Arc, + ) -> Self { + StoreResolver { + logger: logger.new(o!("component" => "StoreResolver")), + store, + subscription_manager, + block_ptr: None, + deployment, + + // Checking for non-fatal errors does not work with subscriptions. + has_non_fatal_errors: false, + error_policy: ErrorPolicy::Deny, + graphql_metrics, + } + } + + /// Create a resolver that looks up entities at the block specified + /// by `bc`. Any calls to find objects will always return entities as + /// of that block. Note that if `bc` is `BlockConstraint::Latest` we use + /// whatever the latest block for the subgraph was when the resolver was + /// created + pub async fn at_block( + logger: &Logger, + store: Arc, + state: &DeploymentState, + subscription_manager: Arc, + bc: BlockConstraint, + error_policy: ErrorPolicy, + deployment: DeploymentHash, + graphql_metrics: Arc, + ) -> Result { + let store_clone = store.cheap_clone(); + let block_ptr = Self::locate_block(store_clone.as_ref(), bc, state).await?; + + let has_non_fatal_errors = store + .has_deterministic_errors(block_ptr.ptr.block_number()) + .await?; + + let resolver = StoreResolver { + logger: logger.new(o!("component" => "StoreResolver")), + store, + subscription_manager, + block_ptr: Some(block_ptr), + deployment, + has_non_fatal_errors, + error_policy, + graphql_metrics, + }; + Ok(resolver) + } + + pub fn block_number(&self) -> BlockNumber { + self.block_ptr + .as_ref() + .map(|ptr| ptr.ptr.number as BlockNumber) + .unwrap_or(BLOCK_NUMBER_MAX) + } + + /// locate_block returns the block pointer and it's timestamp when available. + async fn locate_block( + store: &dyn QueryStore, + bc: BlockConstraint, + state: &DeploymentState, + ) -> Result { + fn block_queryable( + state: &DeploymentState, + block: BlockNumber, + ) -> Result<(), QueryExecutionError> { + state + .block_queryable(block) + .map_err(|msg| QueryExecutionError::ValueParseError("block.number".to_owned(), msg)) + } + + async fn get_block_ts( + store: &dyn QueryStore, + ptr: &BlockPtr, + ) -> Result, QueryExecutionError> { + match store + .block_number_with_timestamp(&ptr.hash) + .await + .map_err(Into::::into)? + { + Some((_, Some(ts))) => Ok(Some(ts)), + _ => Ok(None), + } + } + + match bc { + BlockConstraint::Hash(hash) => { + let ptr = store + .block_number_with_timestamp(&hash) + .await + .map_err(Into::into) + .and_then(|result| { + result + .ok_or_else(|| { + QueryExecutionError::ValueParseError( + "block.hash".to_owned(), + "no block with that hash found".to_owned(), + ) + }) + .map(|(number, ts)| BlockPtrTs { + ptr: BlockPtr::new(hash, number), + timestamp: ts, + }) + })?; + + block_queryable(state, ptr.ptr.number)?; + Ok(ptr) + } + BlockConstraint::Number(number) => { + block_queryable(state, number)?; + // We don't have a way here to look the block hash up from + // the database, and even if we did, there is no guarantee + // that we have the block in our cache. We therefore + // always return an all zeroes hash when users specify + // a block number + // See 7a7b9708-adb7-4fc2-acec-88680cb07ec1 + Ok(BlockPtr::from((web3::types::H256::zero(), number as u64)).into()) + } + BlockConstraint::Min(min) => { + let ptr = state.latest_block.cheap_clone(); + if ptr.number < min { + return Err(QueryExecutionError::ValueParseError( + "block.number_gte".to_owned(), + format!( + "subgraph {} has only indexed up to block number {} \ + and data for block number {} is therefore not yet available", + state.id, ptr.number, min + ), + )); + } + let timestamp = get_block_ts(store, &state.latest_block).await?; + + Ok(BlockPtrTs { ptr, timestamp }) + } + BlockConstraint::Latest => { + let timestamp = get_block_ts(store, &state.latest_block).await?; + + Ok(BlockPtrTs { + ptr: state.latest_block.cheap_clone(), + timestamp, + }) + } + } + } + + fn handle_meta( + &self, + prefetched_object: Option, + object_type: &ObjectOrInterface<'_>, + ) -> Result<(Option, Option), QueryExecutionError> { + // Pretend that the whole `_meta` field was loaded by prefetch. Eager + // loading this is ok until we add more information to this field + // that would force us to query the database; when that happens, we + // need to switch to loading on demand + if object_type.is_meta() { + let hash = self + .block_ptr + .as_ref() + .and_then(|ptr| { + // locate_block indicates that we do not have a block hash + // by setting the hash to `zero` + // See 7a7b9708-adb7-4fc2-acec-88680cb07ec1 + let hash_h256 = ptr.ptr.hash_as_h256(); + if hash_h256 == web3::types::H256::zero() { + None + } else { + Some(r::Value::String(format!("0x{:x}", hash_h256))) + } + }) + .unwrap_or(r::Value::Null); + let number = self + .block_ptr + .as_ref() + .map(|ptr| r::Value::Int((ptr.ptr.number as i32).into())) + .unwrap_or(r::Value::Null); + + let timestamp = self.block_ptr.as_ref().map(|ptr| { + ptr.timestamp + .clone() + .map(|ts| r::Value::Int(ts as i64)) + .unwrap_or(r::Value::Null) + }); + + let mut map = BTreeMap::new(); + let block = object! { + hash: hash, + number: number, + timestamp: timestamp, + __typename: BLOCK_FIELD_TYPE + }; + map.insert("prefetch:block".into(), r::Value::List(vec![block])); + map.insert( + "deployment".into(), + r::Value::String(self.deployment.to_string()), + ); + map.insert( + "hasIndexingErrors".into(), + r::Value::Boolean(self.has_non_fatal_errors), + ); + map.insert( + "__typename".into(), + r::Value::String(META_FIELD_TYPE.to_string()), + ); + return Ok((None, Some(r::Value::object(map)))); + } + Ok((prefetched_object, None)) + } +} + +#[async_trait] +impl Resolver for StoreResolver { + const CACHEABLE: bool = true; + + async fn query_permit(&self) -> Result { + self.store.query_permit().await.map_err(Into::into) + } + + fn prefetch( + &self, + ctx: &ExecutionContext, + selection_set: &a::SelectionSet, + ) -> Result<(Option, Trace), Vec> { + super::prefetch::run(self, ctx, selection_set, &self.graphql_metrics) + .map(|(value, trace)| (Some(value), trace)) + } + + async fn resolve_objects( + &self, + prefetched_objects: Option, + field: &a::Field, + _field_definition: &s::Field, + object_type: ObjectOrInterface<'_>, + ) -> Result { + if let Some(child) = prefetched_objects { + Ok(child) + } else { + Err(QueryExecutionError::ResolveEntitiesError(format!( + "internal error resolving {}.{}: \ + expected prefetched result, but found nothing", + object_type.name(), + &field.name, + ))) + } + } + + async fn resolve_object( + &self, + prefetched_object: Option, + field: &a::Field, + field_definition: &s::Field, + object_type: ObjectOrInterface<'_>, + ) -> Result { + let (prefetched_object, meta) = self.handle_meta(prefetched_object, &object_type)?; + if let Some(meta) = meta { + return Ok(meta); + } + if let Some(r::Value::List(children)) = prefetched_object { + if children.len() > 1 { + let derived_from_field = + sast::get_derived_from_field(object_type, field_definition) + .expect("only derived fields can lead to multiple children here"); + + return Err(QueryExecutionError::AmbiguousDerivedFromResult( + field.position, + field.name.to_owned(), + object_type.name().to_owned(), + derived_from_field.name.to_owned(), + )); + } else { + Ok(children.into_iter().next().unwrap_or(r::Value::Null)) + } + } else { + return Err(QueryExecutionError::ResolveEntitiesError(format!( + "internal error resolving {}.{}: \ + expected prefetched result, but found nothing", + object_type.name(), + &field.name, + ))); + } + } + + fn resolve_field_stream( + &self, + schema: &ApiSchema, + object_type: &s::ObjectType, + field: &a::Field, + ) -> result::Result { + // Collect all entities involved in the query field + let object_type = schema.object_type(object_type).into(); + let entities = collect_entities_from_query_field(schema, object_type, field)?; + + // Subscribe to the store and return the entity change stream + Ok(self.subscription_manager.subscribe_no_payload(entities)) + } + + fn post_process(&self, result: &mut QueryResult) -> Result<(), anyhow::Error> { + // Post-processing is only necessary for queries with indexing errors, and no query errors. + if !self.has_non_fatal_errors || result.has_errors() { + return Ok(()); + } + + // Add the "indexing_error" to the response. + assert!(result.errors_mut().is_empty()); + *result.errors_mut() = vec![QueryError::IndexingError]; + + match self.error_policy { + // If indexing errors are denied, we omit results, except for the `_meta` response. + // Note that the meta field could have been queried under a different response key, + // or a different field queried under the response key `_meta`. + ErrorPolicy::Deny => { + let data = result.take_data(); + let meta = + data.and_then(|mut d| d.remove("_meta").map(|m| ("_meta".to_string(), m))); + result.set_data(meta.map(|m| Object::from_iter(Some(m)))); + } + ErrorPolicy::Allow => (), + } + Ok(()) + } +} diff --git a/graphql/src/subscription/mod.rs b/graphql/src/subscription/mod.rs new file mode 100644 index 0000000..c07bf27 --- /dev/null +++ b/graphql/src/subscription/mod.rs @@ -0,0 +1,239 @@ +use std::result::Result; +use std::time::{Duration, Instant}; + +use graph::components::store::UnitStream; +use graph::{components::store::SubscriptionManager, prelude::*}; + +use crate::metrics::GraphQLMetrics; +use crate::{ + execution::ast as a, + execution::*, + prelude::{BlockConstraint, StoreResolver}, + schema::api::ErrorPolicy, +}; + +/// Options available for subscription execution. +pub struct SubscriptionExecutionOptions { + /// The logger to use during subscription execution. + pub logger: Logger, + + /// The store to use. + pub store: Arc, + + pub subscription_manager: Arc, + + /// Individual timeout for each subscription query. + pub timeout: Option, + + /// Maximum complexity for a subscription query. + pub max_complexity: Option, + + /// Maximum depth for a subscription query. + pub max_depth: u8, + + /// Maximum value for the `first` argument. + pub max_first: u32, + + /// Maximum value for the `skip` argument. + pub max_skip: u32, + + pub graphql_metrics: Arc, +} + +pub fn execute_subscription( + subscription: Subscription, + schema: Arc, + options: SubscriptionExecutionOptions, +) -> Result { + let query = crate::execution::Query::new( + &options.logger, + schema, + None, + subscription.query, + options.max_complexity, + options.max_depth, + options.graphql_metrics.cheap_clone(), + )?; + execute_prepared_subscription(query, options) +} + +pub(crate) fn execute_prepared_subscription( + query: Arc, + options: SubscriptionExecutionOptions, +) -> Result { + if !query.is_subscription() { + return Err(SubscriptionError::from(QueryExecutionError::NotSupported( + "Only subscriptions are supported".to_string(), + ))); + } + + info!( + options.logger, + "Execute subscription"; + "query" => &query.query_text, + ); + + let source_stream = create_source_event_stream(query.clone(), &options)?; + let response_stream = map_source_to_response_stream(query, options, source_stream); + Ok(response_stream) +} + +fn create_source_event_stream( + query: Arc, + options: &SubscriptionExecutionOptions, +) -> Result { + let resolver = StoreResolver::for_subscription( + &options.logger, + query.schema.id().clone(), + options.store.clone(), + options.subscription_manager.cheap_clone(), + options.graphql_metrics.cheap_clone(), + ); + let ctx = ExecutionContext { + logger: options.logger.cheap_clone(), + resolver, + query, + deadline: None, + max_first: options.max_first, + max_skip: options.max_skip, + cache_status: Default::default(), + }; + + let subscription_type = ctx + .query + .schema + .subscription_type + .as_ref() + .ok_or(QueryExecutionError::NoRootSubscriptionObjectType)?; + + let field = if ctx.query.selection_set.is_empty() { + return Err(SubscriptionError::from(QueryExecutionError::EmptyQuery)); + } else { + match ctx.query.selection_set.single_field() { + Some(field) => field, + None => { + return Err(SubscriptionError::from( + QueryExecutionError::MultipleSubscriptionFields, + )); + } + } + }; + + resolve_field_stream(&ctx, subscription_type, field) +} + +fn resolve_field_stream( + ctx: &ExecutionContext, + object_type: &s::ObjectType, + field: &a::Field, +) -> Result { + ctx.resolver + .resolve_field_stream(&ctx.query.schema, object_type, field) + .map_err(SubscriptionError::from) +} + +fn map_source_to_response_stream( + query: Arc, + options: SubscriptionExecutionOptions, + source_stream: UnitStream, +) -> QueryResultStream { + // Create a stream with a single empty event. By chaining this in front + // of the real events, we trick the subscription into executing its query + // at least once. This satisfies the GraphQL over Websocket protocol + // requirement of "respond[ing] with at least one GQL_DATA message", see + // https://github.com/apollographql/subscriptions-transport-ws/blob/master/PROTOCOL.md#gql_data + let trigger_stream = futures03::stream::once(async {}); + + let SubscriptionExecutionOptions { + logger, + store, + subscription_manager, + timeout, + max_complexity: _, + max_depth: _, + max_first, + max_skip, + graphql_metrics, + } = options; + + trigger_stream + .chain(source_stream) + .then(move |()| { + execute_subscription_event( + logger.clone(), + store.clone(), + subscription_manager.cheap_clone(), + query.clone(), + timeout, + max_first, + max_skip, + graphql_metrics.cheap_clone(), + ) + .boxed() + }) + .boxed() +} + +async fn execute_subscription_event( + logger: Logger, + store: Arc, + subscription_manager: Arc, + query: Arc, + timeout: Option, + max_first: u32, + max_skip: u32, + metrics: Arc, +) -> Arc { + async fn make_resolver( + store: Arc, + logger: &Logger, + subscription_manager: Arc, + query: &Arc, + metrics: Arc, + ) -> Result { + let state = store.deployment_state().await?; + StoreResolver::at_block( + logger, + store, + &state, + subscription_manager, + BlockConstraint::Latest, + ErrorPolicy::Deny, + query.schema.id().clone(), + metrics, + ) + .await + } + + let resolver = match make_resolver(store, &logger, subscription_manager, &query, metrics).await + { + Ok(resolver) => resolver, + Err(e) => return Arc::new(e.into()), + }; + + let block_ptr = resolver.block_ptr.as_ref().map(Into::into); + + // Create a fresh execution context with deadline. + let ctx = Arc::new(ExecutionContext { + logger, + resolver, + query, + deadline: timeout.map(|t| Instant::now() + t), + max_first, + max_skip, + cache_status: Default::default(), + }); + + let subscription_type = match ctx.query.schema.subscription_type.as_ref() { + Some(t) => t.cheap_clone(), + None => return Arc::new(QueryExecutionError::NoRootSubscriptionObjectType.into()), + }; + + execute_root_selection_set( + ctx.cheap_clone(), + ctx.query.selection_set.cheap_clone(), + subscription_type.into(), + block_ptr, + ) + .await +} diff --git a/graphql/src/values/coercion.rs b/graphql/src/values/coercion.rs new file mode 100644 index 0000000..b7ed80b --- /dev/null +++ b/graphql/src/values/coercion.rs @@ -0,0 +1,424 @@ +use crate::schema; +use graph::prelude::s::{EnumType, InputValue, ScalarType, Type, TypeDefinition}; +use graph::prelude::{q, r, QueryExecutionError}; +use std::collections::BTreeMap; +use std::convert::TryFrom; + +/// A GraphQL value that can be coerced according to a type. +pub trait MaybeCoercible { + /// On error, `self` is returned as `Err(self)`. + fn coerce(self, using_type: &T) -> Result; +} + +impl MaybeCoercible for q::Value { + fn coerce(self, using_type: &EnumType) -> Result { + match self { + q::Value::Null => Ok(r::Value::Null), + q::Value::String(name) | q::Value::Enum(name) + if using_type.values.iter().any(|value| value.name == name) => + { + Ok(r::Value::Enum(name)) + } + _ => Err(self), + } + } +} + +impl MaybeCoercible for q::Value { + fn coerce(self, using_type: &ScalarType) -> Result { + match (using_type.name.as_str(), self) { + (_, q::Value::Null) => Ok(r::Value::Null), + ("Boolean", q::Value::Boolean(b)) => Ok(r::Value::Boolean(b)), + ("BigDecimal", q::Value::Float(f)) => Ok(r::Value::String(f.to_string())), + ("BigDecimal", q::Value::Int(i)) => Ok(r::Value::String( + i.as_i64().ok_or(q::Value::Int(i))?.to_string(), + )), + ("BigDecimal", q::Value::String(s)) => Ok(r::Value::String(s)), + ("Int", q::Value::Int(num)) => { + let n = num.as_i64().ok_or_else(|| q::Value::Int(num.clone()))?; + if i32::min_value() as i64 <= n && n <= i32::max_value() as i64 { + Ok(r::Value::Int((n as i32).into())) + } else { + Err(q::Value::Int(num)) + } + } + ("String", q::Value::String(s)) => Ok(r::Value::String(s)), + ("ID", q::Value::String(s)) => Ok(r::Value::String(s)), + ("ID", q::Value::Int(n)) => Ok(r::Value::String( + n.as_i64().ok_or(q::Value::Int(n))?.to_string(), + )), + ("Bytes", q::Value::String(s)) => Ok(r::Value::String(s)), + ("BigInt", q::Value::String(s)) => Ok(r::Value::String(s)), + ("BigInt", q::Value::Int(n)) => Ok(r::Value::String( + n.as_i64().ok_or(q::Value::Int(n))?.to_string(), + )), + (_, v) => Err(v), + } + } +} + +/// On error, the `value` is returned as `Err(value)`. +fn coerce_to_definition<'a>( + value: r::Value, + definition: &str, + resolver: &impl Fn(&str) -> Option<&'a TypeDefinition>, +) -> Result { + match resolver(definition).ok_or_else(|| value.clone())? { + // Accept enum values if they match a value in the enum type + TypeDefinition::Enum(t) => value.coerce_enum(t), + + // Try to coerce Scalar values + TypeDefinition::Scalar(t) => value.coerce_scalar(t), + + // Try to coerce InputObject values + TypeDefinition::InputObject(t) => match value { + r::Value::Object(object) => { + let object_for_error = r::Value::Object(object.clone()); + let mut coerced_object = BTreeMap::new(); + for (name, value) in object { + let def = t + .fields + .iter() + .find(|f| f.name == &*name) + .ok_or_else(|| object_for_error.clone())?; + coerced_object.insert( + name.clone(), + match coerce_input_value(Some(value), def, resolver) { + Err(_) | Ok(None) => return Err(object_for_error), + Ok(Some(v)) => v, + }, + ); + } + Ok(r::Value::object(coerced_object)) + } + _ => Err(value), + }, + + // Everything else remains unimplemented + _ => Err(value), + } +} + +/// Coerces an argument into a GraphQL value. +/// +/// `Ok(None)` happens when no value is found for a nullable type. +pub(crate) fn coerce_input_value<'a>( + mut value: Option, + def: &InputValue, + resolver: &impl Fn(&str) -> Option<&'a TypeDefinition>, +) -> Result, QueryExecutionError> { + // Use the default value if necessary and present. + value = match value { + Some(value) => Some(value), + None => def + .default_value + .clone() + .map(r::Value::try_from) + .transpose() + .map_err(|value| { + QueryExecutionError::Panic(format!( + "internal error: failed to convert default value {:?}", + value + )) + })?, + }; + + // Extract value, checking for null or missing. + let value = match value { + None => { + return if schema::ast::is_non_null_type(&def.value_type) { + Err(QueryExecutionError::MissingArgumentError( + def.position, + def.name.to_owned(), + )) + } else { + Ok(None) + }; + } + Some(value) => value, + }; + + Ok(Some( + coerce_value(value, &def.value_type, resolver).map_err(|val| { + QueryExecutionError::InvalidArgumentError(def.position, def.name.to_owned(), val.into()) + })?, + )) +} + +/// On error, the `value` is returned as `Err(value)`. +pub(crate) fn coerce_value<'a>( + value: r::Value, + ty: &Type, + resolver: &impl Fn(&str) -> Option<&'a TypeDefinition>, +) -> Result { + match (ty, value) { + // Null values cannot be coerced into non-null types. + (Type::NonNullType(_), r::Value::Null) => Err(r::Value::Null), + + // Non-null values may be coercible into non-null types + (Type::NonNullType(_), val) => { + // We cannot bind `t` in the pattern above because "binding by-move and by-ref in the + // same pattern is unstable". Refactor this and the others when Rust fixes this. + let t = match ty { + Type::NonNullType(ty) => ty, + _ => unreachable!(), + }; + coerce_value(val, t, resolver) + } + + // Nullable types can be null. + (_, r::Value::Null) => Ok(r::Value::Null), + + // Resolve named types, then try to coerce the value into the resolved type + (Type::NamedType(_), val) => { + let name = match ty { + Type::NamedType(name) => name, + _ => unreachable!(), + }; + coerce_to_definition(val, name, resolver) + } + + // List values are coercible if their values are coercible into the + // inner type. + (Type::ListType(_), r::Value::List(values)) => { + let t = match ty { + Type::ListType(ty) => ty, + _ => unreachable!(), + }; + let mut coerced_values = vec![]; + + // Coerce the list values individually + for value in values { + coerced_values.push(coerce_value(value, t, resolver)?); + } + + Ok(r::Value::List(coerced_values)) + } + + // Otherwise the list type is not coercible. + (Type::ListType(_), value) => Err(value), + } +} + +#[cfg(test)] +mod tests { + use graph::prelude::r::Value; + use graphql_parser::schema::{EnumType, EnumValue, ScalarType, TypeDefinition}; + use graphql_parser::Pos; + + use super::coerce_to_definition; + + #[test] + fn coercion_using_enum_type_definitions_is_correct() { + let enum_type = TypeDefinition::Enum(EnumType { + name: "Enum".to_string(), + description: None, + directives: vec![], + position: Pos::default(), + values: vec![EnumValue { + name: "ValidVariant".to_string(), + position: Pos::default(), + description: None, + directives: vec![], + }], + }); + let resolver = |_: &str| Some(&enum_type); + + // We can coerce from Value::Enum -> TypeDefinition::Enum if the variant is valid + assert_eq!( + coerce_to_definition(Value::Enum("ValidVariant".to_string()), "", &resolver,), + Ok(Value::Enum("ValidVariant".to_string())) + ); + + // We cannot coerce from Value::Enum -> TypeDefinition::Enum if the variant is invalid + assert!( + coerce_to_definition(Value::Enum("InvalidVariant".to_string()), "", &resolver,) + .is_err() + ); + + // We also support going from Value::String -> TypeDefinition::Scalar(Enum) + assert_eq!( + coerce_to_definition(Value::String("ValidVariant".to_string()), "", &resolver,), + Ok(Value::Enum("ValidVariant".to_string())), + ); + + // But we don't support invalid variants + assert!( + coerce_to_definition(Value::String("InvalidVariant".to_string()), "", &resolver,) + .is_err() + ); + } + + #[test] + fn coercion_using_boolean_type_definitions_is_correct() { + let bool_type = TypeDefinition::Scalar(ScalarType { + name: "Boolean".to_string(), + description: None, + directives: vec![], + position: Pos::default(), + }); + let resolver = |_: &str| Some(&bool_type); + + // We can coerce from Value::Boolean -> TypeDefinition::Scalar(Boolean) + assert_eq!( + coerce_to_definition(Value::Boolean(true), "", &resolver), + Ok(Value::Boolean(true)) + ); + assert_eq!( + coerce_to_definition(Value::Boolean(false), "", &resolver), + Ok(Value::Boolean(false)) + ); + + // We don't support going from Value::String -> TypeDefinition::Scalar(Boolean) + assert!(coerce_to_definition(Value::String("true".to_string()), "", &resolver,).is_err()); + assert!(coerce_to_definition(Value::String("false".to_string()), "", &resolver,).is_err()); + + // We don't support going from Value::Float -> TypeDefinition::Scalar(Boolean) + assert!(coerce_to_definition(Value::Float(1.0), "", &resolver).is_err()); + assert!(coerce_to_definition(Value::Float(0.0), "", &resolver).is_err()); + } + + #[test] + fn coercion_using_big_decimal_type_definitions_is_correct() { + let big_decimal_type = TypeDefinition::Scalar(ScalarType::new("BigDecimal".to_string())); + let resolver = |_: &str| Some(&big_decimal_type); + + // We can coerce from Value::Float -> TypeDefinition::Scalar(BigDecimal) + assert_eq!( + coerce_to_definition(Value::Float(23.7), "", &resolver), + Ok(Value::String("23.7".to_string())) + ); + assert_eq!( + coerce_to_definition(Value::Float(-5.879), "", &resolver), + Ok(Value::String("-5.879".to_string())) + ); + + // We can coerce from Value::String -> TypeDefinition::Scalar(BigDecimal) + assert_eq!( + coerce_to_definition(Value::String("23.7".to_string()), "", &resolver,), + Ok(Value::String("23.7".to_string())) + ); + assert_eq!( + coerce_to_definition(Value::String("-5.879".to_string()), "", &resolver,), + Ok(Value::String("-5.879".to_string())), + ); + + // We can coerce from Value::Int -> TypeDefinition::Scalar(BigDecimal) + assert_eq!( + coerce_to_definition(Value::Int(23.into()), "", &resolver), + Ok(Value::String("23".to_string())) + ); + assert_eq!( + coerce_to_definition(Value::Int((-5 as i32).into()), "", &resolver,), + Ok(Value::String("-5".to_string())), + ); + + // We don't support going from Value::Boolean -> TypeDefinition::Scalar(Boolean) + assert!(coerce_to_definition(Value::Boolean(true), "", &resolver).is_err()); + assert!(coerce_to_definition(Value::Boolean(false), "", &resolver).is_err()); + } + + #[test] + fn coercion_using_string_type_definitions_is_correct() { + let string_type = TypeDefinition::Scalar(ScalarType::new("String".to_string())); + let resolver = |_: &str| Some(&string_type); + + // We can coerce from Value::String -> TypeDefinition::Scalar(String) + assert_eq!( + coerce_to_definition(Value::String("foo".to_string()), "", &resolver,), + Ok(Value::String("foo".to_string())) + ); + assert_eq!( + coerce_to_definition(Value::String("bar".to_string()), "", &resolver,), + Ok(Value::String("bar".to_string())) + ); + + // We don't support going from Value::Boolean -> TypeDefinition::Scalar(String) + assert!(coerce_to_definition(Value::Boolean(true), "", &resolver).is_err()); + assert!(coerce_to_definition(Value::Boolean(false), "", &resolver).is_err()); + + // We don't support going from Value::Float -> TypeDefinition::Scalar(String) + assert!(coerce_to_definition(Value::Float(23.7), "", &resolver).is_err()); + assert!(coerce_to_definition(Value::Float(-5.879), "", &resolver).is_err()); + } + + #[test] + fn coercion_using_id_type_definitions_is_correct() { + let string_type = TypeDefinition::Scalar(ScalarType::new("ID".to_owned())); + let resolver = |_: &str| Some(&string_type); + + // We can coerce from Value::String -> TypeDefinition::Scalar(ID) + assert_eq!( + coerce_to_definition(Value::String("foo".to_string()), "", &resolver,), + Ok(Value::String("foo".to_string())) + ); + assert_eq!( + coerce_to_definition(Value::String("bar".to_string()), "", &resolver,), + Ok(Value::String("bar".to_string())) + ); + + // And also from Value::Int + assert_eq!( + coerce_to_definition(Value::Int(1234.into()), "", &resolver), + Ok(Value::String("1234".to_string())) + ); + + // We don't support going from Value::Boolean -> TypeDefinition::Scalar(ID) + assert!(coerce_to_definition(Value::Boolean(true), "", &resolver).is_err()); + + assert!(coerce_to_definition(Value::Boolean(false), "", &resolver).is_err()); + + // We don't support going from Value::Float -> TypeDefinition::Scalar(ID) + assert!(coerce_to_definition(Value::Float(23.7), "", &resolver).is_err()); + assert!(coerce_to_definition(Value::Float(-5.879), "", &resolver).is_err()); + } + + #[test] + fn coerce_big_int_scalar() { + let big_int_type = TypeDefinition::Scalar(ScalarType::new("BigInt".to_string())); + let resolver = |_: &str| Some(&big_int_type); + + // We can coerce from Value::String -> TypeDefinition::Scalar(BigInt) + assert_eq!( + coerce_to_definition(Value::String("1234".to_string()), "", &resolver,), + Ok(Value::String("1234".to_string())) + ); + + // And also from Value::Int + assert_eq!( + coerce_to_definition(Value::Int(1234.into()), "", &resolver), + Ok(Value::String("1234".to_string())) + ); + assert_eq!( + coerce_to_definition(Value::Int((-1234 as i32).into()), "", &resolver,), + Ok(Value::String("-1234".to_string())) + ); + } + + #[test] + fn coerce_bytes_scalar() { + let bytes_type = TypeDefinition::Scalar(ScalarType::new("Bytes".to_string())); + let resolver = |_: &str| Some(&bytes_type); + + // We can coerce from Value::String -> TypeDefinition::Scalar(Bytes) + assert_eq!( + coerce_to_definition(Value::String("0x21f".to_string()), "", &resolver,), + Ok(Value::String("0x21f".to_string())) + ); + } + + #[test] + fn coerce_int_scalar() { + let int_type = TypeDefinition::Scalar(ScalarType::new("Int".to_string())); + let resolver = |_: &str| Some(&int_type); + + assert_eq!( + coerce_to_definition(Value::Int(13289123.into()), "", &resolver,), + Ok(Value::Int(13289123.into())) + ); + assert_eq!( + coerce_to_definition(Value::Int((-13289123 as i32).into()), "", &resolver,), + Ok(Value::Int((-13289123 as i32).into())) + ); + } +} diff --git a/graphql/src/values/mod.rs b/graphql/src/values/mod.rs new file mode 100644 index 0000000..db78a5b --- /dev/null +++ b/graphql/src/values/mod.rs @@ -0,0 +1,4 @@ +/// Utilities for coercing GraphQL values based on GraphQL types. +pub mod coercion; + +pub use self::coercion::MaybeCoercible; diff --git a/graphql/tests/introspection.rs b/graphql/tests/introspection.rs new file mode 100644 index 0000000..1056684 --- /dev/null +++ b/graphql/tests/introspection.rs @@ -0,0 +1,1286 @@ +#[macro_use] +extern crate pretty_assertions; + +use std::sync::Arc; + +use graph::data::graphql::{object, object_value, ObjectOrInterface}; +use graph::data::query::Trace; +use graph::prelude::{ + async_trait, o, r, s, slog, tokio, ApiSchema, DeploymentHash, Logger, Query, + QueryExecutionError, QueryResult, Schema, +}; +use graph_graphql::prelude::{ + a, api_schema, execute_query, ExecutionContext, Query as PreparedQuery, QueryExecutionOptions, + Resolver, +}; +use test_store::graphql_metrics; +use test_store::LOAD_MANAGER; + +/// Mock resolver used in tests that don't need a resolver. +#[derive(Clone)] +pub struct MockResolver; + +#[async_trait] +impl Resolver for MockResolver { + const CACHEABLE: bool = false; + + fn prefetch( + &self, + _: &ExecutionContext, + _: &a::SelectionSet, + ) -> Result<(Option, Trace), Vec> { + Ok((None, Trace::None)) + } + + async fn resolve_objects( + &self, + _: Option, + _field: &a::Field, + _field_definition: &s::Field, + _object_type: ObjectOrInterface<'_>, + ) -> Result { + Ok(r::Value::Null) + } + + async fn resolve_object( + &self, + __: Option, + _field: &a::Field, + _field_definition: &s::Field, + _object_type: ObjectOrInterface<'_>, + ) -> Result { + Ok(r::Value::Null) + } + + async fn query_permit(&self) -> Result { + Ok(Arc::new(tokio::sync::Semaphore::new(1)) + .acquire_owned() + .await + .unwrap()) + } +} + +/// Creates a basic GraphQL schema that exercies scalars, directives, +/// enums, interfaces, input objects, object types and field arguments. +fn mock_schema() -> Schema { + Schema::parse( + " + scalar ID + scalar Int + scalar String + scalar Boolean + + directive @language( + language: String = \"English\" + ) on FIELD_DEFINITION + + enum Role { + USER + ADMIN + } + + interface Node { + id: ID! + } + + type User implements Node @entity { + id: ID! + name: String! @language(language: \"English\") + role: Role! + } + + enum User_orderBy { + id + name + } + + input User_filter { + name_eq: String = \"default name\", + name_not: String, + } + + type Query @entity { + allUsers(orderBy: User_orderBy, filter: User_filter): [User!] + anyUserWithAge(age: Int = 99): User + User: User + } + ", + DeploymentHash::new("mockschema").unwrap(), + ) + .unwrap() +} + +/// Builds the expected result for GraphiQL's introspection query that we are +/// using for testing. +fn expected_mock_schema_introspection() -> r::Value { + let string_type = object! { + kind: r::Value::Enum("SCALAR".to_string()), + name: "String", + description: r::Value::Null, + fields: r::Value::Null, + inputFields: r::Value::Null, + interfaces: r::Value::Null, + enumValues: r::Value::Null, + possibleTypes: r::Value::Null, + }; + + let id_type = object! { + kind: r::Value::Enum("SCALAR".to_string()), + name: "ID", + description: r::Value::Null, + fields: r::Value::Null, + inputFields: r::Value::Null, + interfaces: r::Value::Null, + enumValues: r::Value::Null, + possibleTypes: r::Value::Null, + }; + + let int_type = object! { + kind: r::Value::Enum("SCALAR".to_string()), + name: "Int", + description: r::Value::Null, + fields: r::Value::Null, + inputFields: r::Value::Null, + interfaces: r::Value::Null, + enumValues: r::Value::Null, + possibleTypes: r::Value::Null, + }; + + let boolean_type = object! { + kind: r::Value::Enum("SCALAR".to_string()), + name: "Boolean", + description: r::Value::Null, + fields: r::Value::Null, + inputFields: r::Value::Null, + interfaces: r::Value::Null, + enumValues: r::Value::Null, + possibleTypes: r::Value::Null, + }; + + let role_type = object! { + kind: r::Value::Enum("ENUM".to_string()), + name: "Role", + description: r::Value::Null, + fields: r::Value::Null, + inputFields: r::Value::Null, + interfaces: r::Value::Null, + enumValues: + r::Value::List(vec![ + object! { + name: "USER", + description: r::Value::Null, + isDeprecated: r::Value::Boolean(false), + deprecationReason: r::Value::Null, + }, + object! { + name: "ADMIN", + description: r::Value::Null, + isDeprecated: false, + deprecationReason: r::Value::Null, + }, + ]), + possibleTypes: r::Value::Null, + }; + + let node_type = object_value(vec![ + ("kind", r::Value::Enum("INTERFACE".to_string())), + ("name", r::Value::String("Node".to_string())), + ("description", r::Value::Null), + ( + "fields", + r::Value::List(vec![object_value(vec![ + ("name", r::Value::String("id".to_string())), + ("description", r::Value::Null), + ("args", r::Value::List(vec![])), + ( + "type", + object_value(vec![ + ("kind", r::Value::Enum("NON_NULL".to_string())), + ("name", r::Value::Null), + ( + "ofType", + object_value(vec![ + ("kind", r::Value::Enum("SCALAR".to_string())), + ("name", r::Value::String("ID".to_string())), + ("ofType", r::Value::Null), + ]), + ), + ]), + ), + ("isDeprecated", r::Value::Boolean(false)), + ("deprecationReason", r::Value::Null), + ])]), + ), + ("inputFields", r::Value::Null), + ("interfaces", r::Value::Null), + ("enumValues", r::Value::Null), + ( + "possibleTypes", + r::Value::List(vec![object_value(vec![ + ("kind", r::Value::Enum("OBJECT".to_string())), + ("name", r::Value::String("User".to_string())), + ("ofType", r::Value::Null), + ])]), + ), + ]); + + let user_orderby_type = object_value(vec![ + ("kind", r::Value::Enum("ENUM".to_string())), + ("name", r::Value::String("User_orderBy".to_string())), + ("description", r::Value::Null), + ("fields", r::Value::Null), + ("inputFields", r::Value::Null), + ("interfaces", r::Value::Null), + ( + "enumValues", + r::Value::List(vec![ + object_value(vec![ + ("name", r::Value::String("id".to_string())), + ("description", r::Value::Null), + ("isDeprecated", r::Value::Boolean(false)), + ("deprecationReason", r::Value::Null), + ]), + object_value(vec![ + ("name", r::Value::String("name".to_string())), + ("description", r::Value::Null), + ("isDeprecated", r::Value::Boolean(false)), + ("deprecationReason", r::Value::Null), + ]), + ]), + ), + ("possibleTypes", r::Value::Null), + ]); + + let user_filter_type = object_value(vec![ + ("kind", r::Value::Enum("INPUT_OBJECT".to_string())), + ("name", r::Value::String("User_filter".to_string())), + ("description", r::Value::Null), + ("fields", r::Value::Null), + ( + "inputFields", + r::Value::List(vec![ + object_value(vec![ + ("name", r::Value::String("name_eq".to_string())), + ("description", r::Value::Null), + ( + "type", + object_value(vec![ + ("kind", r::Value::Enum("SCALAR".to_string())), + ("name", r::Value::String("String".to_string())), + ("ofType", r::Value::Null), + ]), + ), + ( + "defaultValue", + r::Value::String("\"default name\"".to_string()), + ), + ]), + object_value(vec![ + ("name", r::Value::String("name_not".to_string())), + ("description", r::Value::Null), + ( + "type", + object_value(vec![ + ("kind", r::Value::Enum("SCALAR".to_string())), + ("name", r::Value::String("String".to_string())), + ("ofType", r::Value::Null), + ]), + ), + ("defaultValue", r::Value::Null), + ]), + ]), + ), + ("interfaces", r::Value::Null), + ("enumValues", r::Value::Null), + ("possibleTypes", r::Value::Null), + ]); + + let user_type = object_value(vec![ + ("kind", r::Value::Enum("OBJECT".to_string())), + ("name", r::Value::String("User".to_string())), + ("description", r::Value::Null), + ( + "fields", + r::Value::List(vec![ + object_value(vec![ + ("name", r::Value::String("id".to_string())), + ("description", r::Value::Null), + ("args", r::Value::List(vec![])), + ( + "type", + object_value(vec![ + ("kind", r::Value::Enum("NON_NULL".to_string())), + ("name", r::Value::Null), + ( + "ofType", + object_value(vec![ + ("kind", r::Value::Enum("SCALAR".to_string())), + ("name", r::Value::String("ID".to_string())), + ("ofType", r::Value::Null), + ]), + ), + ]), + ), + ("isDeprecated", r::Value::Boolean(false)), + ("deprecationReason", r::Value::Null), + ]), + object_value(vec![ + ("name", r::Value::String("name".to_string())), + ("description", r::Value::Null), + ("args", r::Value::List(vec![])), + ( + "type", + object_value(vec![ + ("kind", r::Value::Enum("NON_NULL".to_string())), + ("name", r::Value::Null), + ( + "ofType", + object_value(vec![ + ("kind", r::Value::Enum("SCALAR".to_string())), + ("name", r::Value::String("String".to_string())), + ("ofType", r::Value::Null), + ]), + ), + ]), + ), + ("isDeprecated", r::Value::Boolean(false)), + ("deprecationReason", r::Value::Null), + ]), + object_value(vec![ + ("name", r::Value::String("role".to_string())), + ("description", r::Value::Null), + ("args", r::Value::List(vec![])), + ( + "type", + object_value(vec![ + ("kind", r::Value::Enum("NON_NULL".to_string())), + ("name", r::Value::Null), + ( + "ofType", + object_value(vec![ + ("kind", r::Value::Enum("ENUM".to_string())), + ("name", r::Value::String("Role".to_string())), + ("ofType", r::Value::Null), + ]), + ), + ]), + ), + ("isDeprecated", r::Value::Boolean(false)), + ("deprecationReason", r::Value::Null), + ]), + ]), + ), + ("inputFields", r::Value::Null), + ( + "interfaces", + r::Value::List(vec![object_value(vec![ + ("kind", r::Value::Enum("INTERFACE".to_string())), + ("name", r::Value::String("Node".to_string())), + ("ofType", r::Value::Null), + ])]), + ), + ("enumValues", r::Value::Null), + ("possibleTypes", r::Value::Null), + ]); + + let query_type = object_value(vec![ + ("kind", r::Value::Enum("OBJECT".to_string())), + ("name", r::Value::String("Query".to_string())), + ("description", r::Value::Null), + ( + "fields", + r::Value::List(vec![ + object_value(vec![ + ("name", r::Value::String("allUsers".to_string())), + ("description", r::Value::Null), + ( + "args", + r::Value::List(vec![ + object_value(vec![ + ("name", r::Value::String("orderBy".to_string())), + ("description", r::Value::Null), + ( + "type", + object_value(vec![ + ("kind", r::Value::Enum("ENUM".to_string())), + ("name", r::Value::String("User_orderBy".to_string())), + ("ofType", r::Value::Null), + ]), + ), + ("defaultValue", r::Value::Null), + ]), + object_value(vec![ + ("name", r::Value::String("filter".to_string())), + ("description", r::Value::Null), + ( + "type", + object_value(vec![ + ("kind", r::Value::Enum("INPUT_OBJECT".to_string())), + ("name", r::Value::String("User_filter".to_string())), + ("ofType", r::Value::Null), + ]), + ), + ("defaultValue", r::Value::Null), + ]), + ]), + ), + ( + "type", + object_value(vec![ + ("kind", r::Value::Enum("LIST".to_string())), + ("name", r::Value::Null), + ( + "ofType", + object_value(vec![ + ("kind", r::Value::Enum("NON_NULL".to_string())), + ("name", r::Value::Null), + ( + "ofType", + object_value(vec![ + ("kind", r::Value::Enum("OBJECT".to_string())), + ("name", r::Value::String("User".to_string())), + ("ofType", r::Value::Null), + ]), + ), + ]), + ), + ]), + ), + ("isDeprecated", r::Value::Boolean(false)), + ("deprecationReason", r::Value::Null), + ]), + object_value(vec![ + ("name", r::Value::String("anyUserWithAge".to_string())), + ("description", r::Value::Null), + ( + "args", + r::Value::List(vec![object_value(vec![ + ("name", r::Value::String("age".to_string())), + ("description", r::Value::Null), + ( + "type", + object_value(vec![ + ("kind", r::Value::Enum("SCALAR".to_string())), + ("name", r::Value::String("Int".to_string())), + ("ofType", r::Value::Null), + ]), + ), + ("defaultValue", r::Value::String("99".to_string())), + ])]), + ), + ( + "type", + object_value(vec![ + ("kind", r::Value::Enum("OBJECT".to_string())), + ("name", r::Value::String("User".to_string())), + ("ofType", r::Value::Null), + ]), + ), + ("isDeprecated", r::Value::Boolean(false)), + ("deprecationReason", r::Value::Null), + ]), + object_value(vec![ + ("name", r::Value::String("User".to_string())), + ("description", r::Value::Null), + ("args", r::Value::List(vec![])), + ( + "type", + object_value(vec![ + ("kind", r::Value::Enum("OBJECT".to_string())), + ("name", r::Value::String("User".to_string())), + ("ofType", r::Value::Null), + ]), + ), + ("isDeprecated", r::Value::Boolean(false)), + ("deprecationReason", r::Value::Null), + ]), + ]), + ), + ("inputFields", r::Value::Null), + ("interfaces", r::Value::List(vec![])), + ("enumValues", r::Value::Null), + ("possibleTypes", r::Value::Null), + ]); + + let expected_types = r::Value::List(vec![ + boolean_type, + id_type, + int_type, + node_type, + query_type, + role_type, + string_type, + user_type, + user_filter_type, + user_orderby_type, + ]); + + let expected_directives = r::Value::List(vec![object_value(vec![ + ("name", r::Value::String("language".to_string())), + ("description", r::Value::Null), + ( + "locations", + r::Value::List(vec![r::Value::Enum(String::from("FIELD_DEFINITION"))]), + ), + ( + "args", + r::Value::List(vec![object_value(vec![ + ("name", r::Value::String("language".to_string())), + ("description", r::Value::Null), + ( + "type", + object_value(vec![ + ("kind", r::Value::Enum("SCALAR".to_string())), + ("name", r::Value::String("String".to_string())), + ("ofType", r::Value::Null), + ]), + ), + ("defaultValue", r::Value::String("\"English\"".to_string())), + ])]), + ), + ])]); + + let schema_type = object_value(vec![ + ( + "queryType", + object_value(vec![("name", r::Value::String("Query".to_string()))]), + ), + ("mutationType", r::Value::Null), + ("subscriptionType", r::Value::Null), + ("types", expected_types), + ("directives", expected_directives), + ]); + + object_value(vec![("__schema", schema_type)]) +} + +/// Execute an introspection query. +async fn introspection_query(schema: Schema, query: &str) -> QueryResult { + // Create the query + let query = Query::new( + graphql_parser::parse_query(query).unwrap().into_static(), + None, + ); + + // Execute it + let logger = Logger::root(slog::Discard, o!()); + let options = QueryExecutionOptions { + resolver: MockResolver, + deadline: None, + max_first: std::u32::MAX, + max_skip: std::u32::MAX, + load_manager: LOAD_MANAGER.clone(), + }; + + let schema = Arc::new(ApiSchema::from_api_schema(schema).unwrap()); + let result = + match PreparedQuery::new(&logger, schema, None, query, None, 100, graphql_metrics()) { + Ok(query) => { + Ok(Arc::try_unwrap(execute_query(query, None, None, options).await).unwrap()) + } + Err(e) => Err(e), + }; + QueryResult::from(result) +} + +#[tokio::test] +async fn satisfies_graphiql_introspection_query_without_fragments() { + let result = introspection_query( + mock_schema(), + " + query IntrospectionQuery { + __schema { + queryType { name } + mutationType { name } + subscriptionType { name} + types { + kind + name + description + fields(includeDeprecated: true) { + name + description + args { + name + description + type { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + } + } + } + } + } + } + } + } + defaultValue + } + type { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + } + } + } + } + } + } + } + } + isDeprecated + deprecationReason + } + inputFields { + name + description + type { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + } + } + } + } + } + } + } + } + defaultValue + } + interfaces { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + } + } + } + } + } + } + } + } + enumValues(includeDeprecated: true) { + name + description + isDeprecated + deprecationReason + } + possibleTypes { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + } + } + } + } + } + } + } + } + } + directives { + name + description + locations + args { + name + description + type { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + } + } + } + } + } + } + } + } + defaultValue + } + } + } + } + ", + ) + .await; + + let data = result + .to_result() + .expect("Introspection query returned no result") + .unwrap(); + assert_eq!(data, expected_mock_schema_introspection()); +} + +#[tokio::test] +async fn satisfies_graphiql_introspection_query_with_fragments() { + let result = introspection_query( + mock_schema(), + " + query IntrospectionQuery { + __schema { + queryType { name } + mutationType { name } + subscriptionType { name } + types { + ...FullType + } + directives { + name + description + locations + args { + ...InputValue + } + } + } + } + + fragment FullType on __Type { + kind + name + description + fields(includeDeprecated: true) { + name + description + args { + ...InputValue + } + type { + ...TypeRef + } + isDeprecated + deprecationReason + } + inputFields { + ...InputValue + } + interfaces { + ...TypeRef + } + enumValues(includeDeprecated: true) { + name + description + isDeprecated + deprecationReason + } + possibleTypes { + ...TypeRef + } + } + + fragment InputValue on __InputValue { + name + description + type { ...TypeRef } + defaultValue + } + + fragment TypeRef on __Type { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + } + } + } + } + } + } + } + } + ", + ) + .await; + + let data = result + .to_result() + .expect("Introspection query returned no result") + .unwrap(); + assert_eq!(data, expected_mock_schema_introspection()); +} + +const COMPLEX_SCHEMA: &str = " +enum RegEntryStatus { + regEntry_status_challengePeriod + regEntry_status_commitPeriod + regEntry_status_revealPeriod + regEntry_status_blacklisted + regEntry_status_whitelisted +} + +interface RegEntry { + regEntry_address: ID + regEntry_version: Int + regEntry_status: RegEntryStatus + regEntry_creator: User + regEntry_deposit: Int + regEntry_createdOn: String + regEntry_challengePeriodEnd: String + challenge_challenger: User + challenge_createdOn: String + challenge_comment: String + challenge_votingToken: String + challenge_rewardPool: Int + challenge_commitPeriodEnd: String + challenge_revealPeriodEnd: String + challenge_votesFor: Int + challenge_votesAgainst: Int + challenge_votesTotal: Int + challenge_claimedRewardOn: String + challenge_vote(vote_voter: ID!): Vote +} + +enum VoteOption { + voteOption_noVote + voteOption_voteFor + voteOption_voteAgainst +} + +type Vote @entity { + vote_secretHash: String + vote_option: VoteOption + vote_amount: Int + vote_revealedOn: String + vote_claimedRewardOn: String + vote_reward: Int +} + +type Meme implements RegEntry @entity { + regEntry_address: ID + regEntry_version: Int + regEntry_status: RegEntryStatus + regEntry_creator: User + regEntry_deposit: Int + regEntry_createdOn: String + regEntry_challengePeriodEnd: String + challenge_challenger: User + challenge_createdOn: String + challenge_comment: String + challenge_votingToken: String + challenge_rewardPool: Int + challenge_commitPeriodEnd: String + challenge_revealPeriodEnd: String + challenge_votesFor: Int + challenge_votesAgainst: Int + challenge_votesTotal: Int + challenge_claimedRewardOn: String + challenge_vote(vote_voter: ID!): Vote + # Balance of voting token of a voter. This is client-side only, server doesn't return this + challenge_availableVoteAmount(voter: ID!): Int + meme_title: String + meme_number: Int + meme_metaHash: String + meme_imageHash: String + meme_totalSupply: Int + meme_totalMinted: Int + meme_tokenIdStart: Int + meme_totalTradeVolume: Int + meme_totalTradeVolumeRank: Int + meme_ownedMemeTokens(owner: String): [MemeToken] + meme_tags: [Tag] +} + +type Tag @entity { + tag_id: ID + tag_name: String +} + +type MemeToken @entity { + memeToken_tokenId: ID + memeToken_number: Int + memeToken_owner: User + memeToken_meme: Meme +} + +enum MemeAuctionStatus { + memeAuction_status_active + memeAuction_status_canceled + memeAuction_status_done +} + +type MemeAuction @entity { + memeAuction_address: ID + memeAuction_seller: User + memeAuction_buyer: User + memeAuction_startPrice: Int + memeAuction_endPrice: Int + memeAuction_duration: Int + memeAuction_startedOn: String + memeAuction_boughtOn: String + memeAuction_status: MemeAuctionStatus + memeAuction_memeToken: MemeToken +} + +type ParamChange implements RegEntry @entity { + regEntry_address: ID + regEntry_version: Int + regEntry_status: RegEntryStatus + regEntry_creator: User + regEntry_deposit: Int + regEntry_createdOn: String + regEntry_challengePeriodEnd: String + challenge_challenger: User + challenge_createdOn: String + challenge_comment: String + challenge_votingToken: String + challenge_rewardPool: Int + challenge_commitPeriodEnd: String + challenge_revealPeriodEnd: String + challenge_votesFor: Int + challenge_votesAgainst: Int + challenge_votesTotal: Int + challenge_claimedRewardOn: String + challenge_vote(vote_voter: ID!): Vote + # Balance of voting token of a voter. This is client-side only, server doesn't return this + challenge_availableVoteAmount(voter: ID!): Int + paramChange_db: String + paramChange_key: String + paramChange_value: Int + paramChange_originalValue: Int + paramChange_appliedOn: String +} + +type User @entity { + # Ethereum address of an user + user_address: ID + # Total number of memes submitted by user + user_totalCreatedMemes: Int + # Total number of memes submitted by user, which successfully got into TCR + user_totalCreatedMemesWhitelisted: Int + # Largest sale creator has done with his newly minted meme + user_creatorLargestSale: MemeAuction + # Position of a creator in leaderboard according to user_totalCreatedMemesWhitelisted + user_creatorRank: Int + # Amount of meme tokenIds owned by user + user_totalCollectedTokenIds: Int + # Amount of unique memes owned by user + user_totalCollectedMemes: Int + # Largest auction user sold, in terms of price + user_largestSale: MemeAuction + # Largest auction user bought into, in terms of price + user_largestBuy: MemeAuction + # Amount of challenges user created + user_totalCreatedChallenges: Int + # Amount of challenges user created and ended up in his favor + user_totalCreatedChallengesSuccess: Int + # Total amount of DANK token user received from challenger rewards + user_challengerTotalEarned: Int + # Total amount of DANK token user received from challenger rewards + user_challengerRank: Int + # Amount of different votes user participated in + user_totalParticipatedVotes: Int + # Amount of different votes user voted for winning option + user_totalParticipatedVotesSuccess: Int + # Amount of DANK token user received for voting for winning option + user_voterTotalEarned: Int + # Position of voter in leaderboard according to user_voterTotalEarned + user_voterRank: Int + # Sum of user_challengerTotalEarned and user_voterTotalEarned + user_curatorTotalEarned: Int + # Position of curator in leaderboard according to user_curatorTotalEarned + user_curatorRank: Int +} + +type Parameter @entity { + param_db: ID + param_key: ID + param_value: Int +} +"; + +#[tokio::test] +async fn successfully_runs_introspection_query_against_complex_schema() { + let mut schema = Schema::parse( + COMPLEX_SCHEMA, + DeploymentHash::new("complexschema").unwrap(), + ) + .unwrap(); + schema.document = api_schema(&schema.document).unwrap(); + + let result = introspection_query( + schema.clone(), + " + query IntrospectionQuery { + __schema { + queryType { name } + mutationType { name } + subscriptionType { name } + types { + ...FullType + } + directives { + name + description + locations + args { + ...InputValue + } + } + } + } + + fragment FullType on __Type { + kind + name + description + fields(includeDeprecated: true) { + name + description + args { + ...InputValue + } + type { + ...TypeRef + } + isDeprecated + deprecationReason + } + inputFields { + ...InputValue + } + interfaces { + ...TypeRef + } + enumValues(includeDeprecated: true) { + name + description + isDeprecated + deprecationReason + } + possibleTypes { + ...TypeRef + } + } + + fragment InputValue on __InputValue { + name + description + type { ...TypeRef } + defaultValue + } + + fragment TypeRef on __Type { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + } + } + } + } + } + } + } + } + ", + ) + .await; + + assert!(!result.has_errors(), "{:#?}", result); +} + +#[tokio::test] +async fn introspection_possible_types() { + let mut schema = Schema::parse( + COMPLEX_SCHEMA, + DeploymentHash::new("complexschema").unwrap(), + ) + .unwrap(); + schema.document = api_schema(&schema.document).unwrap(); + + // Test "possibleTypes" introspection in interfaces + let response = introspection_query( + schema, + "query { + __type(name: \"RegEntry\") { + name + possibleTypes { + name + } + } + }", + ) + .await + .to_result() + .unwrap() + .unwrap(); + + assert_eq!( + response, + object_value(vec![( + "__type", + object_value(vec![ + ("name", r::Value::String("RegEntry".to_string())), + ( + "possibleTypes", + r::Value::List(vec![ + object_value(vec![("name", r::Value::String("Meme".to_owned()))]), + object_value(vec![("name", r::Value::String("ParamChange".to_owned()))]) + ]) + ) + ]) + )]) + ) +} diff --git a/graphql/tests/query.rs b/graphql/tests/query.rs new file mode 100644 index 0000000..0c0b01d --- /dev/null +++ b/graphql/tests/query.rs @@ -0,0 +1,2125 @@ +#[macro_use] +extern crate pretty_assertions; + +use graph::components::store::{EntityKey, EntityType}; +use graph::data::subgraph::schema::DeploymentCreate; +use graph::entity; +use graph::prelude::SubscriptionResult; +use graphql_parser::Pos; +use std::iter::FromIterator; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use std::{ + collections::{BTreeSet, HashMap}, + marker::PhantomData, +}; + +use graph::{ + components::store::DeploymentLocator, + data::graphql::{object, object_value}, + data::subgraph::schema::SubgraphError, + data::{ + query::{QueryResults, QueryTarget}, + subgraph::SubgraphFeature, + }, + prelude::{ + futures03::stream::StreamExt, lazy_static, o, q, r, serde_json, slog, BlockPtr, + DeploymentHash, Entity, EntityOperation, FutureExtension, GraphQlRunner as _, Logger, + NodeId, Query, QueryError, QueryExecutionError, QueryResult, QueryStoreManager, + QueryVariables, Schema, SubgraphManifest, SubgraphName, SubgraphStore, + SubgraphVersionSwitchingMode, Subscription, SubscriptionError, + }, + semver::Version, +}; +use graph_graphql::{prelude::*, subscription::execute_subscription}; +use test_store::{ + deployment_state, execute_subgraph_query_with_deadline, graphql_metrics, revert_block, + run_test_sequentially, transact_errors, Store, BLOCK_ONE, GENESIS_PTR, LOAD_MANAGER, LOGGER, + METRICS_REGISTRY, STORE, SUBSCRIPTION_MANAGER, +}; + +const NETWORK_NAME: &str = "fake_network"; +const SONGS_STRING: [&str; 5] = ["s0", "s1", "s2", "s3", "s4"]; +const SONGS_BYTES: [&str; 5] = ["0xf0", "0xf1", "0xf2", "0xf3", "0xf4"]; +const MEDIA_STRING: [&str; 7] = ["md0", "md1", "md2", "md3", "md4", "md5", "md6"]; +const MEDIA_BYTES: [&str; 7] = ["0xf0", "0xf1", "0xf2", "0xf3", "0xf4", "0xf5", "0xf6"]; + +#[derive(Clone, Copy, Debug)] +enum IdType { + String, + #[allow(dead_code)] + Bytes, +} + +impl IdType { + fn songs(&self) -> &[&str] { + match self { + IdType::String => SONGS_STRING.as_slice(), + IdType::Bytes => SONGS_BYTES.as_slice(), + } + } + + fn medias(&self) -> &[&str] { + match self { + IdType::String => MEDIA_STRING.as_slice(), + IdType::Bytes => MEDIA_BYTES.as_slice(), + } + } + + fn as_str(&self) -> &str { + match self { + IdType::String => "String", + IdType::Bytes => "Bytes", + } + } + + fn deployment_id(&self) -> &str { + match self { + IdType::String => "graphqlTestsQuery", + IdType::Bytes => "graphqlTestsQueryBytes", + } + } +} + +/// Setup a basic deployment. The test using the deployment must not modify +/// the deployment at all +async fn setup_readonly(store: &Store) -> DeploymentLocator { + setup(store, "graphqlTestsQuery", BTreeSet::new(), IdType::String).await +} + +/// Set up a deployment `id` with the test schema and populate it with test +/// data. If the `id` is the same as `id_type.deployment_id()`, the test +/// must not modify the deployment in any way as these are reused for other +/// tests that expect pristine data +async fn setup( + store: &Store, + id: &str, + features: BTreeSet, + id_type: IdType, +) -> DeploymentLocator { + use test_store::block_store::{self, BLOCK_ONE, BLOCK_TWO, GENESIS_BLOCK}; + + /// Make sure we get rid of all subgraphs once for the entire test run + fn global_init() { + lazy_static! { + static ref STORE_CLEAN: AtomicBool = AtomicBool::new(false); + } + if !STORE_CLEAN.load(Ordering::SeqCst) { + let chain = vec![&*GENESIS_BLOCK, &*BLOCK_ONE, &*BLOCK_TWO]; + block_store::set_chain(chain, NETWORK_NAME); + test_store::remove_subgraphs(); + STORE_CLEAN.store(true, Ordering::SeqCst); + } + } + + async fn initialize( + store: &Store, + id: DeploymentHash, + features: BTreeSet, + id_type: IdType, + ) -> DeploymentLocator { + let schema = test_schema(id.clone(), id_type); + let manifest = SubgraphManifest:: { + id: id.clone(), + spec_version: Version::new(1, 0, 0), + features, + description: None, + repository: None, + schema: schema.clone(), + data_sources: vec![], + graft: None, + templates: vec![], + chain: PhantomData, + }; + + insert_test_entities(store.subgraph_store().as_ref(), manifest, id_type).await + } + + global_init(); + let id = DeploymentHash::new(id).unwrap(); + let loc = store.subgraph_store().locators(&id).unwrap().pop(); + + match loc { + Some(loc) if id_type.deployment_id() == loc.hash.as_str() => loc, + Some(loc) => { + test_store::remove_subgraph(&loc.hash); + initialize(store, id, features, id_type).await + } + None => initialize(store, id, features, id_type).await, + } +} + +fn test_schema(id: DeploymentHash, id_type: IdType) -> Schema { + const SCHEMA: &str = " + type Musician @entity { + id: ID! + name: String! + mainBand: Band + bands: [Band!]! + writtenSongs: [Song!]! @derivedFrom(field: \"writtenBy\") + } + + type Band @entity { + id: ID! + name: String! + members: [Musician!]! @derivedFrom(field: \"bands\") + reviews: [BandReview!]! @derivedFrom(field: \"band\") + originalSongs: [Song!]! + } + + type Song @entity { + id: @ID@! + title: String! + writtenBy: Musician! + publisher: Publisher! + band: Band @derivedFrom(field: \"originalSongs\") + reviews: [SongReview!]! @derivedFrom(field: \"song\") + media: [Media!]! + release: Release! @derivedFrom(field: \"songs\") + } + + type SongStat @entity { + id: @ID@! + song: Song @derivedFrom(field: \"id\") + played: Int! + } + + type Publisher { + id: Bytes! + } + + interface Review { + id: ID! + body: String! + author: User! + } + + type SongReview implements Review @entity { + id: ID! + body: String! + song: Song + author: User! + } + + type BandReview implements Review @entity { + id: ID! + body: String! + band: Band + author: User! + } + + type User @entity { + id: ID! + name: String! + bandReviews: [BandReview!]! @derivedFrom(field: \"author\") + songReviews: [SongReview!]! @derivedFrom(field: \"author\") + reviews: [Review!]! @derivedFrom(field: \"author\") + latestSongReview: SongReview! + latestBandReview: BandReview! + latestReview: Review! + } + + interface Media { + id: ID! + title: String! + song: Song! + } + + type Photo implements Media @entity { + id: ID! + title: String! + song: Song! @derivedFrom(field: \"media\") + } + + type Video implements Media @entity { + id: ID! + title: String! + song: Song! @derivedFrom(field: \"media\") + } + + interface Release { + id: ID! + title: String! + songs: [Song!]! + } + + type Single implements Release @entity { + id: ID! + title: String! + # It could be a single song + # but let's say a Single represents one song + bonus tracks + songs: [Song!]! + } + + type Album implements Release @entity { + id: ID! + title: String! + songs: [Song!]! + } + "; + + Schema::parse(&SCHEMA.replace("@ID@", id_type.as_str()), id).expect("Test schema invalid") +} + +async fn insert_test_entities( + store: &impl SubgraphStore, + manifest: SubgraphManifest, + id_type: IdType, +) -> DeploymentLocator { + let deployment = DeploymentCreate::new(&manifest, None); + let name = SubgraphName::new(manifest.id.as_str()).unwrap(); + let node_id = NodeId::new("test").unwrap(); + let deployment = store + .create_subgraph_deployment( + name, + &manifest.schema, + deployment, + node_id, + NETWORK_NAME.to_string(), + SubgraphVersionSwitchingMode::Instant, + ) + .unwrap(); + + let s = id_type.songs(); + let md = id_type.medias(); + let entities0 = vec![ + entity! { __typename: "Musician", id: "m1", name: "John", mainBand: "b1", bands: vec!["b1", "b2"] }, + entity! { __typename: "Musician", id: "m2", name: "Lisa", mainBand: "b1", bands: vec!["b1"] }, + entity! { __typename: "Publisher", id: "0xb1" }, + entity! { __typename: "Band", id: "b1", name: "The Musicians", originalSongs: vec![s[1], s[2]] }, + entity! { __typename: "Band", id: "b2", name: "The Amateurs", originalSongs: vec![s[1], s[3], s[4]] }, + entity! { __typename: "Song", id: s[1], title: "Cheesy Tune", publisher: "0xb1", writtenBy: "m1", media: vec![md[1], md[2]] }, + entity! { __typename: "Song", id: s[2], title: "Rock Tune", publisher: "0xb1", writtenBy: "m2", media: vec![md[3], md[4]] }, + entity! { __typename: "Song", id: s[3], title: "Pop Tune", publisher: "0xb1", writtenBy: "m1", media: vec![md[5]] }, + entity! { __typename: "Song", id: s[4], title: "Folk Tune", publisher: "0xb1", writtenBy: "m3", media: vec![md[6]] }, + entity! { __typename: "SongStat", id: s[1], played: 10 }, + entity! { __typename: "SongStat", id: s[2], played: 15 }, + entity! { __typename: "BandReview", id: "r1", body: "Bad musicians", band: "b1", author: "u1" }, + entity! { __typename: "BandReview", id: "r2", body: "Good amateurs", band: "b2", author: "u2" }, + entity! { __typename: "SongReview", id: "r3", body: "Bad", song: s[2], author: "u1" }, + entity! { __typename: "SongReview", id: "r4", body: "Good", song: s[3], author: "u2" }, + entity! { __typename: "User", id: "u1", name: "Baden", latestSongReview: "r3", latestBandReview: "r1", latestReview: "r1" }, + entity! { __typename: "User", id: "u2", name: "Goodwill", latestSongReview: "r4", latestBandReview: "r2", latestReview: "r2" }, + entity! { __typename: "Photo", id: md[1], title: "Cheesy Tune Single Cover" }, + entity! { __typename: "Video", id: md[2], title: "Cheesy Tune Music Video" }, + entity! { __typename: "Photo", id: md[3], title: "Rock Tune Single Cover" }, + entity! { __typename: "Video", id: md[4], title: "Rock Tune Music Video" }, + entity! { __typename: "Photo", id: md[5], title: "Pop Tune Single Cover" }, + entity! { __typename: "Video", id: md[6], title: "Folk Tune Music Video" }, + entity! { __typename: "Album", id: "rl1", title: "Pop and Folk", songs: vec![s[3], s[4]] }, + entity! { __typename: "Single", id: "rl2", title: "Rock", songs: vec![s[2]] }, + entity! { __typename: "Single", id: "rl3", title: "Cheesy", songs: vec![s[1]] }, + ]; + + let entities1 = vec![ + entity! { __typename: "Musician", id: "m3", name: "Tom", mainBand: "b2", bands: vec!["b1", "b2"] }, + entity! { __typename: "Musician", id: "m4", name: "Valerie", bands: Vec::::new() }, + ]; + + async fn insert_at(entities: Vec, deployment: &DeploymentLocator, block_ptr: BlockPtr) { + let insert_ops = entities.into_iter().map(|data| EntityOperation::Set { + key: EntityKey { + entity_type: EntityType::new( + data.get("__typename").unwrap().clone().as_string().unwrap(), + ), + entity_id: data.get("id").unwrap().clone().as_string().unwrap().into(), + }, + data, + }); + + test_store::transact_and_wait( + &STORE.subgraph_store(), + &deployment, + block_ptr, + insert_ops.collect::>(), + ) + .await + .unwrap(); + } + + insert_at(entities0, &deployment, GENESIS_PTR.clone()).await; + insert_at(entities1, &deployment, BLOCK_ONE.clone()).await; + deployment +} + +async fn execute_query(loc: &DeploymentLocator, query: &str) -> QueryResult { + let query = graphql_parser::parse_query(query) + .expect("invalid test query") + .into_static(); + execute_query_document_with_variables(&loc.hash, query, None).await +} + +async fn execute_query_document_with_variables( + id: &DeploymentHash, + query: q::Document, + variables: Option, +) -> QueryResult { + let runner = Arc::new(GraphQlRunner::new( + &*LOGGER, + STORE.clone(), + SUBSCRIPTION_MANAGER.clone(), + LOAD_MANAGER.clone(), + METRICS_REGISTRY.clone(), + )); + let target = QueryTarget::Deployment(id.clone(), Default::default()); + let query = Query::new(query, variables); + + runner + .run_query_with_complexity(query, target, None, None, None, None) + .await + .first() + .unwrap() + .duplicate() +} + +async fn first_result(f: QueryResults) -> QueryResult { + f.first().unwrap().duplicate() +} + +/// Extract the data from a `QueryResult`, and panic if it has errors +macro_rules! extract_data { + ($result: expr) => { + match $result.to_result() { + Err(errors) => panic!("Unexpected errors return for query: {:#?}", errors), + Ok(data) => data, + } + }; +} + +struct QueryArgs { + query: String, + variables: Option, + max_complexity: Option, +} + +impl From<&str> for QueryArgs { + fn from(query: &str) -> Self { + QueryArgs { + query: query.to_owned(), + variables: None, + max_complexity: None, + } + } +} + +impl From<(&str, r::Value)> for QueryArgs { + fn from((query, vars): (&str, r::Value)) -> Self { + let vars = match vars { + r::Value::Object(map) => map, + _ => panic!("vars must be an object"), + }; + let vars = QueryVariables::new(HashMap::from_iter( + vars.into_iter().map(|(k, v)| (k.to_string(), v)), + )); + QueryArgs { + query: query.to_owned(), + variables: Some(vars), + max_complexity: None, + } + } +} + +/// Run a GraphQL query against the `test_schema` and call the `test` +/// function with the result. The query is actually run twice: once against +/// the test schema where the `id` of `Song` and `SongStats` has type +/// `String`, and once where it has type `Bytes`. The second argument to +/// `test` indicates which type is being used for the id. +/// +/// The query can contain placeholders `@S1@` .. `@S4@` which will be +/// replaced with the id's of songs 1 through 4 before running the query. +fn run_query(args: impl Into, test: F) +where + F: Fn(QueryResult, IdType) -> () + Send + 'static, +{ + let QueryArgs { + query, + variables, + max_complexity, + } = args.into(); + run_test_sequentially(move |store| async move { + for id_type in [IdType::String, IdType::Bytes] { + let name = id_type.deployment_id(); + + let deployment = setup(store.as_ref(), name, BTreeSet::new(), id_type).await; + + let mut query = query.clone(); + for (i, id) in id_type.songs().iter().enumerate() { + let pat = format!("@S{i}@"); + let repl = format!("\"{id}\""); + query = query.replace(&pat, &repl); + } + + let result = { + let id = &deployment.hash; + let query = graphql_parser::parse_query(&query) + .expect("Invalid test query") + .into_static(); + let variables = variables.clone(); + let runner = Arc::new(GraphQlRunner::new( + &*LOGGER, + STORE.clone(), + SUBSCRIPTION_MANAGER.clone(), + LOAD_MANAGER.clone(), + METRICS_REGISTRY.clone(), + )); + let target = QueryTarget::Deployment(id.clone(), Default::default()); + let query = Query::new(query, variables); + + runner + .run_query_with_complexity(query, target, max_complexity, None, None, None) + .await + .first() + .unwrap() + .duplicate() + }; + test(result, id_type); + } + }) +} + +/// Helper to run a subscription +async fn run_subscription( + store: &Arc, + query: &str, + max_complexity: Option, +) -> Result { + let deployment = setup_readonly(store.as_ref()).await; + let logger = Logger::root(slog::Discard, o!()); + let query_store = store + .query_store( + QueryTarget::Deployment(deployment.hash.clone(), Default::default()), + true, + ) + .await + .unwrap(); + + let query = Query::new( + graphql_parser::parse_query(query).unwrap().into_static(), + None, + ); + let options = SubscriptionExecutionOptions { + logger: logger.clone(), + store: query_store.clone(), + subscription_manager: SUBSCRIPTION_MANAGER.clone(), + timeout: None, + max_complexity, + max_depth: 100, + max_first: std::u32::MAX, + max_skip: std::u32::MAX, + graphql_metrics: graphql_metrics(), + }; + let schema = STORE + .subgraph_store() + .api_schema(&deployment.hash, &Default::default()) + .unwrap(); + + execute_subscription(Subscription { query }, schema.clone(), options) +} + +#[test] +fn can_query_one_to_one_relationship() { + const QUERY: &str = " + query { + musicians(first: 100, orderBy: id) { + name + mainBand { + name + } + } + songStats(first: 100, orderBy: id) { + id + song { + id + title + } + played + } + } + "; + + run_query(QUERY, |result, id_type| { + let s = id_type.songs(); + let exp = object! { + musicians: vec![ + object! { name: "John", mainBand: object! { name: "The Musicians" } }, + object! { name: "Lisa", mainBand: object! { name: "The Musicians" } }, + object! { name: "Tom", mainBand: object! { name: "The Amateurs"} }, + object! { name: "Valerie", mainBand: r::Value::Null } + ], + songStats: vec![ + object! { + id: s[1], + song: object! { id: s[1], title: "Cheesy Tune" }, + played: 10, + }, + object! { + id: s[2], + song: object! { id: s[2], title: "Rock Tune" }, + played: 15 + } + ] + }; + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_one_to_many_relationships_in_both_directions() { + const QUERY: &str = " + query { + musicians(first: 100, orderBy: id) { + name + writtenSongs(first: 100, orderBy: id) { + title + writtenBy { name } + } + } + }"; + + run_query(QUERY, |result, _| { + fn song(title: &str, author: &str) -> r::Value { + object! { + title: title, + writtenBy: object! { name: author } + } + } + + let exp = object! { + musicians: vec![ + object! { + name: "John", + writtenSongs: vec![ + song("Cheesy Tune", "John"), + song("Pop Tune", "John"), + ] + }, + object! { + name: "Lisa", writtenSongs: vec![ song("Rock Tune", "Lisa") ] + }, + object! { + name: "Tom", writtenSongs: vec![ song("Folk Tune", "Tom") ] + }, + object! { + name: "Valerie", writtenSongs: Vec::::new() + }, + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_many_to_many_relationship() { + const QUERY: &str = " + query { + musicians(first: 100, orderBy: id) { + name + bands(first: 100, orderBy: id) { + name + members(first: 100, orderBy: id) { + name + } + } + } + }"; + + run_query(QUERY, |result, _| { + fn members(names: Vec<&str>) -> Vec { + names + .into_iter() + .map(|name| object! { name: name }) + .collect() + } + + let the_musicians = object! { + name: "The Musicians", + members: members(vec!["John", "Lisa", "Tom"]) + }; + + let the_amateurs = object! { + name: "The Amateurs", + members: members(vec![ "John", "Tom" ]) + }; + + let exp = object! { + musicians: vec![ + object! { name: "John", bands: vec![ the_musicians.clone(), the_amateurs.clone() ]}, + object! { name: "Lisa", bands: vec![ the_musicians.clone() ] }, + object! { name: "Tom", bands: vec![ the_musicians.clone(), the_amateurs.clone() ] }, + object! { name: "Valerie", bands: Vec::::new() } + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_with_child_filter_on_list_type_field() { + const QUERY: &str = " + query { + musicians(first: 100, orderBy: id, where: { bands_: { name: \"The Amateurs\" } }) { + name + bands(first: 100, orderBy: id) { + name + } + } + }"; + + run_query(QUERY, |result, _| { + let the_musicians = object! { + name: "The Musicians", + }; + + let the_amateurs = object! { + name: "The Amateurs", + }; + + let exp = object! { + musicians: vec![ + object! { name: "John", bands: vec![ the_musicians.clone(), the_amateurs.clone() ]}, + object! { name: "Tom", bands: vec![ the_musicians.clone(), the_amateurs.clone() ] }, + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_with_child_filter_on_derived_list_type_field() { + const QUERY: &str = " + query { + musicians(first: 100, orderBy: id, where: { writtenSongs_: { title_contains: \"Rock\" } }) { + name + } + }"; + + run_query(QUERY, |result, _| { + let exp = object! { + musicians: vec![ + object! { name: "Lisa" }, + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_with_child_filter_on_named_type_field() { + const QUERY: &str = " + query { + musicians(first: 100, orderBy: id, where: { mainBand_: { name_contains: \"The Amateurs\" } }) { + name + mainBand { + id + } + } + }"; + + run_query(QUERY, |result, _| { + let exp = object! { + musicians: vec![ + object! { name: "Tom", mainBand: object! { id: "b2"} } + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_with_child_filter_on_derived_named_type_field() { + const QUERY: &str = " + query { + songs(first: 100, orderBy: id, where: { band_: { name_contains: \"The Musicians\" } }) { + title + } + }"; + + run_query(QUERY, |result, _| { + let exp = object! { + songs: vec![ + object! { title: "Cheesy Tune" }, + object! { title: "Rock Tune" }, + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_an_interface_with_child_filter_on_named_type_field() { + const QUERY: &str = " + query { + reviews(first: 100, orderBy: id, where: { author_: { name_starts_with: \"Good\" } }) { + body + author { + name + } + } + }"; + + run_query(QUERY, |result, _| { + let exp = object! { + reviews: vec![ + object! { body: "Good amateurs", author: object! { name: "Goodwill" } }, + object! { body: "Good", author: object! { name: "Goodwill" } }, + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_with_child_filter_on_derived_interface_list_field() { + const QUERY: &str = " + query { + users(first: 100, orderBy: id, where: { reviews_: { body_starts_with: \"Good\" } }) { + name + reviews { + body + } + } + }"; + + run_query(QUERY, |result, _| { + let exp = object! { + users: vec![ + object! { name: "Goodwill", reviews: vec![ object! { body: "Good amateurs" }, object! { body: "Good" } ] }, + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_entity_by_child_entity_field() { + const QUERY: &str = " + query { + users(first: 100, orderBy: id, where: { latestSongReview_: { body_starts_with: \"Good\" } }) { + name + latestSongReview { + body + } + } + }"; + + run_query(QUERY, |result, _| { + let exp = object! { + users: vec![ + object! { name: "Goodwill", latestSongReview: object! { body: "Good" } }, + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_entity_by_child_interface_field() { + const QUERY: &str = " + query { + users(first: 100, orderBy: id, where: { latestReview_: { body_starts_with: \"Good\" } }) { + name + latestReview { + body + } + } + }"; + + run_query(QUERY, |result, _| { + let exp = object! { + users: vec![ + object! { name: "Goodwill", latestReview: object! { body: "Good amateurs" } }, + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_interface_by_child_entity_field() { + const QUERY: &str = " + query { + reviews(first: 100, orderBy: id, where: { author_: { name_starts_with: \"Good\" } }) { + body + author { + name + } + } + }"; + + run_query(QUERY, |result, _| { + let exp = object! { + reviews: vec![ + object! { body: "Good amateurs", author: object! { name: "Goodwill" } }, + object! { body: "Good", author: object! { name: "Goodwill" } }, + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_entity_by_child_interface_derived_field() { + const QUERY: &str = " + query { + songs(first: 100, orderBy: id, where: { release_: { title_starts_with: \"Pop\" } }) { + title + release { + title + } + } + }"; + + run_query(QUERY, |result, _| { + let exp = object! { + songs: vec![ + object! { title: "Pop Tune", release: object! { title: "Pop and Folk" } }, + object! { title: "Folk Tune", release: object! { title: "Pop and Folk" } }, + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_interface_by_child_entity_derived_field() { + const QUERY: &str = " + query { + medias(first: 100, orderBy: id, where: { song_: { title_starts_with: \"Folk\" } }) { + title + song { + title + } + } + }"; + + run_query(QUERY, |result, _| { + let exp = object! { + medias: vec![ + object! { title: "Folk Tune Music Video", song: object! { title: "Folk Tune" } }, + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_entity_by_child_interface_list_field() { + const QUERY: &str = " + query { + songs(first: 100, orderBy: id, where: { media_: { title_starts_with: \"Cheesy Tune\" } }) { + title + media { + title + } + } + }"; + + run_query(QUERY, |result, _| { + let exp = object! { + songs: vec![ + object! { title: "Cheesy Tune", media: vec![ + object! { title: "Cheesy Tune Single Cover" }, + object! { title: "Cheesy Tune Music Video" } + ] }, + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn can_query_entity_by_child_interface_list_derived_field() { + const QUERY: &str = " + query { + songs(first: 100, orderBy: id, where: { reviews_: { body_starts_with: \"Good\" } }) { + title + reviews { + body + } + } + }"; + + run_query(QUERY, |result, _| { + let exp = object! { + songs: vec![ + object! { title: "Pop Tune", reviews: vec![ + object! { body: "Good" }, + ] }, + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn root_fragments_are_expanded() { + const QUERY: &str = r#" + fragment Musicians on Query { + musicians(first: 100, where: { name: "Tom" }) { + name + } + } + query MyQuery { + ...Musicians + }"#; + + run_query(QUERY, |result, _| { + let exp = object! { musicians: vec![ object! { name: "Tom" }]}; + assert_eq!(extract_data!(result), Some(exp)); + }) +} + +#[test] +fn query_variables_are_used() { + const QUERY: &str = " + query musicians($where: Musician_filter!) { + musicians(first: 100, where: $where) { + name + } + }"; + + run_query( + (QUERY, object![ where: object! { name: "Tom"} ]), + |result, _| { + let exp = object! { + musicians: vec![ object! { name: "Tom" }] + }; + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }, + ); +} + +#[test] +fn mixed_parent_child_id() { + // Check that any combination of parent and child id type (String or + // Bytes) works in queries + + // `Publisher` has `id` of type `Bytes`, which used to lead to + // `NonNullError` when `Song` used `String` + const QUERY: &str = " + query amxx { + songs(first: 2) { + publisher { id } + } + }"; + + run_query(QUERY, |result, _| { + let exp = object! { + songs: vec![ + object! { publisher: object! { id: "0xb1" } }, + object! { publisher: object! { id: "0xb1" } } + ] + }; + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }); + + const QUERY2: &str = " + query bytes_string { + songs(first: 2) { + writtenBy { id } + } + } + "; + run_query(QUERY2, |result, _| { + let exp = object! { + songs: vec![ + object! { writtenBy: object! { id: "m1" } }, + object! { writtenBy: object! { id: "m2" } } + ] + }; + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn skip_directive_works_with_query_variables() { + const QUERY: &str = " + query musicians($skip: Boolean!) { + musicians(first: 100, orderBy: id) { + id @skip(if: $skip) + name + } + } +"; + + run_query((QUERY, object! { skip: true }), |result, _| { + // Assert that only names are returned + let musicians: Vec<_> = ["John", "Lisa", "Tom", "Valerie"] + .into_iter() + .map(|name| object! { name: name }) + .collect(); + let exp = object! { musicians: musicians }; + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }); + + run_query((QUERY, object! { skip: false }), |result, _| { + // Assert that IDs and names are returned + let exp = object! { + musicians: vec![ + object! { id: "m1", name: "John" }, + object! { id: "m2", name: "Lisa"}, + object! { id: "m3", name: "Tom" }, + object! { id: "m4", name: "Valerie" } + ] + }; + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }); +} + +#[test] +fn include_directive_works_with_query_variables() { + const QUERY: &str = " + query musicians($include: Boolean!) { + musicians(first: 100, orderBy: id) { + id @include(if: $include) + name + } + } +"; + + run_query((QUERY, object! { include: true }), |result, _| { + // Assert that IDs and names are returned + let exp = object! { + musicians: vec![ + object! { id: "m1", name: "John" }, + object! { id: "m2", name: "Lisa"}, + object! { id: "m3", name: "Tom" }, + object! { id: "m4", name: "Valerie" } + ] + }; + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }); + + run_query((QUERY, object! { include: false }), |result, _| { + // Assert that only names are returned + let musicians: Vec<_> = ["John", "Lisa", "Tom", "Valerie"] + .into_iter() + .map(|name| object! { name: name }) + .collect(); + let exp = object! { musicians: musicians }; + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }); +} + +#[test] +fn query_complexity() { + const QUERY1: &str = "query { + musicians(orderBy: id) { + name + bands(first: 100, orderBy: id) { + name + members(first: 100, orderBy: id) { + name + } + } + } + }"; + let args = QueryArgs { + query: QUERY1.to_owned(), + variables: None, + max_complexity: Some(1_010_100), + }; + run_query(args, |result, _| { + // This query is exactly at the maximum complexity. + assert!(!result.has_errors()); + }); + + const QUERY2: &str = "query { + musicians(orderBy: id) { + name + bands(first: 100, orderBy: id) { + name + members(first: 100, orderBy: id) { + name + } + } + } + __schema { + types { + name + } + } + }"; + let args = QueryArgs { + query: QUERY2.to_owned(), + variables: None, + max_complexity: Some(1_010_100), + }; + run_query(args, |result, _| { + // The extra introspection causes the complexity to go over. + match result.to_result().unwrap_err()[0] { + QueryError::ExecutionError(QueryExecutionError::TooComplex(1_010_200, _)) => (), + _ => panic!("did not catch complexity"), + }; + }) +} + +#[test] +fn query_complexity_subscriptions() { + run_test_sequentially(|store| async move { + const QUERY1: &str = "subscription { + musicians(orderBy: id) { + name + bands(first: 100, orderBy: id) { + name + members(first: 100, orderBy: id) { + name + } + } + } + }"; + let max_complexity = Some(1_010_100); + + // This query is exactly at the maximum complexity. + // FIXME: Not collecting the stream because that will hang the test. + let _ignore_stream = run_subscription(&store, QUERY1, max_complexity) + .await + .unwrap(); + + const QUERY2: &str = "subscription { + musicians(orderBy: id) { + name + t1: bands(first: 100, orderBy: id) { + name + members(first: 100, orderBy: id) { + name + } + } + t2: bands(first: 200, orderBy: id) { + name + members(first: 100, orderBy: id) { + name + } + } + } + }"; + + let result = run_subscription(&store, QUERY2, max_complexity).await; + + match result { + Err(SubscriptionError::GraphQLError(e)) => match &e[0] { + QueryExecutionError::TooComplex(3_030_100, _) => (), // Expected + e => panic!("did not catch complexity: {:?}", e), + }, + _ => panic!("did not catch complexity"), + } + }) +} + +#[test] +fn instant_timeout() { + run_test_sequentially(|store| async move { + let deployment = setup_readonly(store.as_ref()).await; + let query = Query::new( + graphql_parser::parse_query("query { musicians(first: 100) { name } }") + .unwrap() + .into_static(), + None, + ); + + match first_result( + execute_subgraph_query_with_deadline( + query, + QueryTarget::Deployment(deployment.hash.into(), Default::default()), + Some(Instant::now()), + ) + .await, + ) + .await + .to_result() + .unwrap_err()[0] + { + QueryError::ExecutionError(QueryExecutionError::Timeout) => (), // Expected + _ => panic!("did not time out"), + }; + }) +} + +#[test] +fn variable_defaults() { + const QUERY: &str = " + query musicians($orderDir: OrderDirection = desc) { + bands(first: 2, orderBy: id, orderDirection: $orderDir) { + id + } + } +"; + + run_query((QUERY, object! {}), |result, _| { + let exp = object! { + bands: vec![ + object! { id: "b2" }, + object! { id: "b1" } + ] + }; + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }); + + run_query( + (QUERY, object! { orderDir: r::Value::Null }), + |result, _| { + let exp = object! { + bands: vec![ + object! { id: "b1" }, + object! { id: "b2" } + ] + }; + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }, + ) +} + +#[test] +fn skip_is_nullable() { + const QUERY: &str = " + query musicians { + musicians(orderBy: id, skip: null) { + name + } + } +"; + + run_query(QUERY, |result, _| { + let musicians: Vec<_> = ["John", "Lisa", "Tom", "Valerie"] + .into_iter() + .map(|name| object! { name: name }) + .collect(); + let exp = object! { musicians: musicians }; + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn first_is_nullable() { + const QUERY: &str = " + query musicians { + musicians(first: null, orderBy: id) { + name + } + } +"; + + run_query(QUERY, |result, _| { + let musicians: Vec<_> = ["John", "Lisa", "Tom", "Valerie"] + .into_iter() + .map(|name| object! { name: name }) + .collect(); + let exp = object! { musicians: musicians }; + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn nested_variable() { + const QUERY: &str = " + query musicians($name: String) { + musicians(first: 100, where: { name: $name }) { + name + } + } +"; + + run_query((QUERY, object! { name: "Lisa" }), |result, _| { + let exp = object! { + musicians: vec! { object! { name: "Lisa" }} + }; + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn ambiguous_derived_from_result() { + const QUERY: &str = "{ songs(first: 100, orderBy: id) { id band { id } } }"; + + run_query(QUERY, |result, _| { + match &result.to_result().unwrap_err()[0] { + QueryError::ExecutionError(QueryExecutionError::AmbiguousDerivedFromResult( + pos, + derived_from_field, + target_type, + target_field, + )) => { + assert_eq!( + pos, + &Pos { + line: 1, + column: 39 + } + ); + assert_eq!(derived_from_field.as_str(), "band"); + assert_eq!(target_type.as_str(), "Band"); + assert_eq!(target_field.as_str(), "originalSongs"); + } + e => panic!("expected AmbiguousDerivedFromResult error, got {}", e), + } + }) +} + +#[test] +fn can_filter_by_relationship_fields() { + const QUERY: &str = " + query { + musicians(orderBy: id, where: { mainBand: \"b2\" }) { + id name + mainBand { id } + } + bands(orderBy: id, where: { originalSongs: [@S1@, @S3@, @S4@] }) { + id name + originalSongs { id } + } + } + "; + + run_query(QUERY, |result, id_type| { + let s = id_type.songs(); + + let exp = object! { + musicians: vec![ + object! { id: "m3", name: "Tom", mainBand: object! { id: "b2"} } + ], + bands: vec![ + object! { + id: "b2", + name: "The Amateurs", + originalSongs: vec! [ + object! { id: s[1] }, + object! { id: s[3] }, + object! { id: s[4] } + ] + } + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +#[test] +fn cannot_filter_by_derved_relationship_fields() { + const QUERY: &str = " + query { + musicians(orderBy: id, where: { writtenSongs: [@S1@] }) { + id name + mainBand { id } + } + } + "; + + run_query(QUERY, |result, _id_type| { + match &result.to_result().unwrap_err()[0] { + // With validations + QueryError::ExecutionError(QueryExecutionError::ValidationError(_, error_message)) => { + assert_eq!( + error_message, + "Field \"writtenSongs\" is not defined by type \"Musician_filter\"." + ); + } + // Without validations + QueryError::ExecutionError(QueryExecutionError::InvalidArgumentError( + _pos, + error_message, + _value, + )) => { + assert_eq!(error_message, "where"); + } + e => panic!("expected a runtime/validation error, got {:?}", e), + }; + }) +} + +#[test] +fn subscription_gets_result_even_without_events() { + run_test_sequentially(|store| async move { + const QUERY: &str = "subscription { + musicians(orderBy: id, first: 2) { + name + } + }"; + + // Execute the subscription and expect at least one result to be + // available in the result stream + let stream = run_subscription(&store, QUERY, None).await.unwrap(); + let results: Vec<_> = stream + .take(1) + .collect() + .timeout(Duration::from_secs(3)) + .await + .unwrap(); + + assert_eq!(results.len(), 1); + let result = Arc::try_unwrap(results.into_iter().next().unwrap()).unwrap(); + let data = extract_data!(result).unwrap(); + let exp = object! { + musicians: vec![ + object! { name: "John" }, + object! { name: "Lisa" } + ] + }; + assert_eq!(data, exp); + }) +} + +#[test] +fn can_use_nested_filter() { + const QUERY: &str = " + query { + musicians(orderBy: id) { + name + bands(where: { originalSongs: [@S1@, @S3@, @S4@] }) { id } + } + } + "; + + run_query(QUERY, |result, _| { + let exp = object! { + musicians: vec![ + object! { + name: "John", + bands: vec![ object! { id: "b2" }] + }, + object! { + name: "Lisa", + bands: Vec::::new(), + }, + object! { + name: "Tom", + bands: vec![ object! { id: "b2" }] + }, + object! { + name: "Valerie", + bands: Vec::::new(), + } + ] + }; + + let data = extract_data!(result).unwrap(); + assert_eq!(data, exp); + }) +} + +// see: graphql-bug-compat +#[test] +fn ignores_invalid_field_arguments() { + // This query has to return all the musicians since `id` is not a + // valid argument for the `musicians` field and must therefore be + // ignored + const QUERY: &str = "query { musicians(id: \"m1\") { id } } "; + + run_query(QUERY, |result, _| { + match &result.to_result() { + // Without validations + Ok(Some(r::Value::Object(obj))) => match obj.get("musicians").unwrap() { + r::Value::List(lst) => { + assert_eq!(4, lst.len()); + } + _ => panic!("expected a list of values"), + }, + // With validations + Err(e) => { + match e.get(0).unwrap() { + QueryError::ExecutionError(QueryExecutionError::ValidationError( + _pos, + message, + )) => { + assert_eq!( + message, + "Unknown argument \"id\" on field \"Query.musicians\"." + ); + } + r => panic!("unexpexted query error: {:?}", r), + }; + } + r => { + panic!("unexpexted result: {:?}", r); + } + } + }) +} + +// see: graphql-bug-compat +#[test] +fn leaf_selection_mismatch() { + const QUERY1: &str = "query { musician(id: \"m1\") { id name { wat }} } "; + + run_query(QUERY1, |result, _| { + let exp = object! { musician: object! { id: "m1", name: "John" } }; + + match &result.to_result() { + // Without validations + Ok(Some(data)) => { + assert_eq!(exp, *data); + } + // With validations + Err(e) => { + match e.get(0).unwrap() { + QueryError::ExecutionError(QueryExecutionError::ValidationError( + _pos, + message, + )) => { + assert_eq!(message, "Field \"name\" must not have a selection since type \"String!\" has no subfields."); + } + r => panic!("unexpexted query error: {:?}", r), + }; + match e.get(1).unwrap() { + QueryError::ExecutionError(QueryExecutionError::ValidationError( + _pos, + message, + )) => { + assert_eq!(message, "Cannot query field \"wat\" on type \"String\"."); + } + r => panic!("unexpexted query error: {:?}", r), + } + } + r => { + panic!("unexpexted result: {:?}", r); + } + } + }); + + const QUERY2: &str = "query { musician(id: \"m1\") { id name mainBand } } "; + run_query(QUERY2, |result, _| { + let exp = object! { musician: object! { id: "m1", name: "John" } }; + + match &result.to_result() { + // Without validations + Ok(Some(data)) => { + assert_eq!(exp, *data); + } + // With validations + Err(e) => { + match e.get(0).unwrap() { + QueryError::ExecutionError(QueryExecutionError::ValidationError( + _pos, + message, + )) => { + assert_eq!(message, "Field \"mainBand\" of type \"Band\" must have a selection of subfields. Did you mean \"mainBand { ... }\"?"); + } + r => panic!("unexpexted query error: {:?}", r), + }; + } + r => { + panic!("unexpexted result: {:?}", r); + } + } + }) +} + +// see: graphql-bug-compat +#[test] +fn missing_variable() { + // '$first' is not defined, use its default from the schema + const QUERY1: &str = "query { musicians(first: $first) { id } }"; + run_query(QUERY1, |result, _| { + let exp = object! { + musicians: vec![ + object! { id: "m1" }, + object! { id: "m2" }, + object! { id: "m3" }, + object! { id: "m4" }, + ] + }; + + match &result.to_result() { + // We silently set `$first` to 100 and `$skip` to 0, and therefore + Ok(Some(data)) => { + assert_eq!(exp, *data); + } + // With GraphQL validations active, this query fails + Err(e) => match e.get(0).unwrap() { + QueryError::ExecutionError(QueryExecutionError::ValidationError(_pos, message)) => { + assert_eq!(message, "Variable \"$first\" is not defined."); + } + r => panic!("unexpexted query error: {:?}", r), + }, + r => { + panic!("unexpexted result: {:?}", r); + } + } + }); + + // '$where' is not defined but nullable, ignore the argument + const QUERY2: &str = "query { musicians(where: $where) { id } }"; + run_query(QUERY2, |result, _| { + let exp = object! { + musicians: vec![ + object! { id: "m1" }, + object! { id: "m2" }, + object! { id: "m3" }, + object! { id: "m4" }, + ] + }; + + match &result.to_result() { + // '$where' is not defined but nullable, ignore the argument + Ok(Some(data)) => { + assert_eq!(exp, *data); + } + // With GraphQL validations active, this query fails + Err(e) => match e.get(0).unwrap() { + QueryError::ExecutionError(QueryExecutionError::ValidationError(_pos, message)) => { + assert_eq!(message, "Variable \"$where\" is not defined."); + } + r => panic!("unexpexted query error: {:?}", r), + }, + r => { + panic!("unexpexted result: {:?}", r); + } + } + }) +} + +// see: graphql-bug-compat +// Test that queries with nonmergeable fields do not cause a panic. Can be +// deleted once queries are validated +#[test] +fn invalid_field_merge() { + const QUERY: &str = "query { musicians { t: id t: mainBand { id } } }"; + + run_query(QUERY, |result, _| { + assert!(result.has_errors()); + }) +} + +/// What we expect the query to return: either a list of musician ids when +/// the query should succeed (`Ok`) or a string that should appear in the +/// error message when the query should return an `Err`. The error string +/// can contain `@DEPLOYMENT@` which will be replaced with the deployment id +type Expected = Result, &'static str>; + +fn check_musicians_at(query0: &str, block_var: r::Value, expected: Expected, qid: &'static str) { + run_query((query0, block_var), move |result, id_type| { + match &expected { + Ok(ids) => { + let ids: Vec<_> = ids.into_iter().map(|id| object! { id: *id }).collect(); + let expected = Some(object_value(vec![("musicians", r::Value::List(ids))])); + let data = match result.to_result() { + Err(errors) => panic!("unexpected error: {:?} ({})\n", errors, qid), + Ok(data) => data, + }; + assert_eq!(data, expected, "failed query: ({})", qid); + } + Err(msg) => { + let errors = match result.to_result() { + Err(errors) => errors, + Ok(_) => panic!( + "expected error `{}` but got successful result ({})", + msg, qid + ), + }; + let actual = errors + .first() + .expect("we expect one error message") + .to_string(); + let msg = msg.replace("@DEPLOYMENT@", id_type.deployment_id()); + assert!( + actual.contains(&msg), + "expected error message `{}` but got {:?} ({})", + msg, + errors, + qid + ); + } + }; + }); +} + +#[test] +fn query_at_block() { + use test_store::block_store::{FakeBlock, BLOCK_ONE, BLOCK_THREE, BLOCK_TWO, GENESIS_BLOCK}; + + fn musicians_at(block: &str, expected: Expected, qid: &'static str) { + let query = format!("query {{ musicians(block: {{ {} }}) {{ id }} }}", block); + check_musicians_at(&query, object! {}, expected, qid); + } + + fn hash(block: &FakeBlock) -> String { + format!("hash : \"0x{}\"", block.hash) + } + + const BLOCK_NOT_INDEXED: &str = "subgraph @DEPLOYMENT@ has only indexed \ + up to block number 1 and data for block number 7000 is therefore not yet available"; + const BLOCK_NOT_INDEXED2: &str = "subgraph @DEPLOYMENT@ has only indexed \ + up to block number 1 and data for block number 2 is therefore not yet available"; + const BLOCK_HASH_NOT_FOUND: &str = "no block with that hash found"; + + musicians_at("number: 7000", Err(BLOCK_NOT_INDEXED), "n7000"); + musicians_at("number: 0", Ok(vec!["m1", "m2"]), "n0"); + musicians_at("number: 1", Ok(vec!["m1", "m2", "m3", "m4"]), "n1"); + + musicians_at(&hash(&*GENESIS_BLOCK), Ok(vec!["m1", "m2"]), "h0"); + musicians_at(&hash(&*BLOCK_ONE), Ok(vec!["m1", "m2", "m3", "m4"]), "h1"); + musicians_at(&hash(&*BLOCK_TWO), Err(BLOCK_NOT_INDEXED2), "h2"); + musicians_at(&hash(&*BLOCK_THREE), Err(BLOCK_HASH_NOT_FOUND), "h3"); +} + +#[test] +fn query_at_block_with_vars() { + use test_store::block_store::{FakeBlock, BLOCK_ONE, BLOCK_THREE, BLOCK_TWO, GENESIS_BLOCK}; + + fn musicians_at_nr(block: i32, expected: Expected, qid: &'static str) { + let query = "query by_nr($block: Int!) { musicians(block: { number: $block }) { id } }"; + let var = object! { block: block }; + + check_musicians_at(query, var, expected.clone(), qid); + + let query = "query by_nr($block: Block_height!) { musicians(block: $block) { id } }"; + let var = object! { block: object! { number: block } }; + + check_musicians_at(query, var, expected, qid); + } + + fn musicians_at_nr_gte(block: i32, expected: Expected, qid: &'static str) { + let query = "query by_nr($block: Int!) { musicians(block: { number_gte: $block }) { id } }"; + let var = object! { block: block }; + + check_musicians_at(query, var, expected, qid); + } + + fn musicians_at_hash(block: &FakeBlock, expected: Expected, qid: &'static str) { + let query = "query by_hash($block: Bytes!) { musicians(block: { hash: $block }) { id } }"; + let var = object! { block: block.hash.to_string() }; + + check_musicians_at(query, var, expected, qid); + } + + const BLOCK_NOT_INDEXED: &str = "subgraph @DEPLOYMENT@ has only indexed \ + up to block number 1 and data for block number 7000 is therefore not yet available"; + const BLOCK_NOT_INDEXED2: &str = "subgraph @DEPLOYMENT@ has only indexed \ + up to block number 1 and data for block number 2 is therefore not yet available"; + const BLOCK_HASH_NOT_FOUND: &str = "no block with that hash found"; + + musicians_at_nr(7000, Err(BLOCK_NOT_INDEXED), "n7000"); + musicians_at_nr(0, Ok(vec!["m1", "m2"]), "n0"); + musicians_at_nr(1, Ok(vec!["m1", "m2", "m3", "m4"]), "n1"); + + musicians_at_nr_gte(7000, Err(BLOCK_NOT_INDEXED), "ngte7000"); + musicians_at_nr_gte(0, Ok(vec!["m1", "m2", "m3", "m4"]), "ngte0"); + musicians_at_nr_gte(1, Ok(vec!["m1", "m2", "m3", "m4"]), "ngte1"); + + musicians_at_hash(&GENESIS_BLOCK, Ok(vec!["m1", "m2"]), "h0"); + musicians_at_hash(&BLOCK_ONE, Ok(vec!["m1", "m2", "m3", "m4"]), "h1"); + musicians_at_hash(&BLOCK_TWO, Err(BLOCK_NOT_INDEXED2), "h2"); + musicians_at_hash(&BLOCK_THREE, Err(BLOCK_HASH_NOT_FOUND), "h3"); +} + +#[test] +fn query_detects_reorg() { + async fn query_at(deployment: &DeploymentLocator, block: i32) -> QueryResult { + let query = + format!("query {{ musician(id: \"m1\", block: {{ number: {block} }}) {{ id }} }}"); + execute_query(&deployment, &query).await + } + + run_test_sequentially(|store| async move { + let deployment = setup( + store.as_ref(), + "graphqlQueryDetectsReorg", + BTreeSet::new(), + IdType::String, + ) + .await; + // Initial state with latest block at block 1 + let state = deployment_state(STORE.as_ref(), &deployment.hash).await; + + // Inject a fake initial state; c435c25decbc4ad7bbbadf8e0ced0ff2 + *graph_graphql::test_support::INITIAL_DEPLOYMENT_STATE_FOR_TESTS + .lock() + .unwrap() = Some(state); + + // When there is no revert, queries work fine + let result = query_at(&deployment, 1).await; + + assert_eq!( + extract_data!(result), + Some(object!(musician: object!(id: "m1"))) + ); + + // Revert one block + revert_block(&*STORE, &deployment, &*GENESIS_PTR).await; + + // A query is still fine since we query at block 0; we were at block + // 1 when we got `state`, and reorged once by one block, which can + // not affect block 0, and it's therefore ok to query at block 0 + // even with a concurrent reorg + let result = query_at(&deployment, 0).await; + assert_eq!( + extract_data!(result), + Some(object!(musician: object!(id: "m1"))) + ); + + // We move the subgraph head forward. The state we have is also for + // block 1, but with a smaller reorg count and we therefore report + // an error + test_store::transact_and_wait( + &STORE.subgraph_store(), + &deployment, + BLOCK_ONE.clone(), + vec![], + ) + .await + .unwrap(); + + let result = query_at(&deployment, 1).await; + match result.to_result().unwrap_err()[0] { + QueryError::ExecutionError(QueryExecutionError::DeploymentReverted) => { /* expected */ + } + _ => panic!("unexpected error from block reorg"), + } + + // Reset the fake initial state; c435c25decbc4ad7bbbadf8e0ced0ff2 + *graph_graphql::test_support::INITIAL_DEPLOYMENT_STATE_FOR_TESTS + .lock() + .unwrap() = None; + }) +} + +#[test] +fn can_query_meta() { + // metadata for the latest block (block 1) + const QUERY1: &str = + "query { _meta { deployment block { hash number __typename } __typename } }"; + run_query(QUERY1, |result, id_type| { + let exp = object! { + _meta: object! { + deployment: id_type.deployment_id(), + block: object! { + hash: "0x8511fa04b64657581e3f00e14543c1d522d5d7e771b54aa3060b662ade47da13", + number: 1, + __typename: "_Block_" + }, + __typename: "_Meta_" + }, + }; + assert_eq!(extract_data!(result), Some(exp)); + }); + + // metadata for block 0 by number + const QUERY2: &str = + "query { _meta(block: { number: 0 }) { deployment block { hash number } } }"; + run_query(QUERY2, |result, id_type| { + let exp = object! { + _meta: object! { + deployment: id_type.deployment_id(), + block: object! { + hash: r::Value::Null, + number: 0 + }, + }, + }; + assert_eq!(extract_data!(result), Some(exp)); + }); + + // metadata for block 0 by hash + const QUERY3: &str = "query { _meta(block: { hash: \"bd34884280958002c51d3f7b5f853e6febeba33de0f40d15b0363006533c924f\" }) { \ + deployment block { hash number } } }"; + run_query(QUERY3, |result, id_type| { + let exp = object! { + _meta: object! { + deployment: id_type.deployment_id(), + block: object! { + hash: "0xbd34884280958002c51d3f7b5f853e6febeba33de0f40d15b0363006533c924f", + number: 0 + }, + }, + }; + assert_eq!(extract_data!(result), Some(exp)); + }); + + // metadata for block 2, which is beyond what the subgraph has indexed + const QUERY4: &str = + "query { _meta(block: { number: 2 }) { deployment block { hash number } } }"; + run_query(QUERY4, |result, _| { + assert!(result.has_errors()); + }); +} + +#[test] +fn non_fatal_errors() { + use serde_json::json; + use test_store::block_store::BLOCK_TWO; + + run_test_sequentially(|store| async move { + let deployment = setup( + store.as_ref(), + "testNonFatalErrors", + BTreeSet::from_iter(Some(SubgraphFeature::NonFatalErrors)), + IdType::String, + ) + .await; + + let err = SubgraphError { + subgraph_id: deployment.hash.clone(), + message: "cow template handler could not moo event transaction".to_string(), + block_ptr: Some(BLOCK_TWO.block_ptr()), + handler: Some("handleMoo".to_string()), + deterministic: true, + }; + + transact_errors(&*STORE, &deployment, BLOCK_TWO.block_ptr(), vec![err]) + .await + .unwrap(); + + // `subgraphError` is implicitly `deny`, data is omitted. + let query = "query { musician(id: \"m1\") { id } }"; + let result = execute_query(&deployment, query).await; + let expected = json!({ + "errors": [ + { + "message": "indexing_error" + } + ] + }); + assert_eq!(expected, serde_json::to_value(&result).unwrap()); + + // Same result for explicit `deny`. + let query = "query { musician(id: \"m1\", subgraphError: deny) { id } }"; + let result = execute_query(&deployment, query).await; + assert_eq!(expected, serde_json::to_value(&result).unwrap()); + + // But `_meta` is still returned. + let query = "query { musician(id: \"m1\") { id } _meta { hasIndexingErrors } }"; + let result = execute_query(&deployment, query).await; + let expected = json!({ + "data": { + "_meta": { + "hasIndexingErrors": true + } + }, + "errors": [ + { + "message": "indexing_error" + } + ] + }); + assert_eq!(expected, serde_json::to_value(&result).unwrap()); + + // With `allow`, the error remains but the data is included. + let query = "query { musician(id: \"m1\", subgraphError: allow) { id } }"; + let result = execute_query(&deployment, query).await; + let expected = json!({ + "data": { + "musician": { + "id": "m1" + } + }, + "errors": [ + { + "message": "indexing_error" + } + ] + }); + assert_eq!(expected, serde_json::to_value(&result).unwrap()); + + // Test error reverts. + revert_block(&*STORE, &deployment, &*BLOCK_ONE).await; + let query = "query { musician(id: \"m1\") { id } _meta { hasIndexingErrors } }"; + let result = execute_query(&deployment, query).await; + let expected = json!({ + "data": { + "musician": { + "id": "m1" + }, + "_meta": { + "hasIndexingErrors": false + } + } + }); + assert_eq!(expected, serde_json::to_value(&result).unwrap()); + }) +} + +#[test] +fn can_query_root_typename() { + const QUERY: &str = "query { __typename }"; + run_query(QUERY, |result, _| { + let exp = object! { + __typename: "Query" + }; + assert_eq!(extract_data!(result), Some(exp)); + }) +} + +#[test] +fn deterministic_error() { + use serde_json::json; + use test_store::block_store::BLOCK_TWO; + + run_test_sequentially(|store| async move { + let deployment = setup( + store.as_ref(), + "testDeterministicError", + BTreeSet::new(), + IdType::String, + ) + .await; + + let err = SubgraphError { + subgraph_id: deployment.hash.clone(), + message: "cow template handler could not moo event transaction".to_string(), + block_ptr: Some(BLOCK_TWO.block_ptr()), + handler: Some("handleMoo".to_string()), + deterministic: true, + }; + + transact_errors(&*STORE, &deployment, BLOCK_TWO.block_ptr(), vec![err]) + .await + .unwrap(); + + // `subgraphError` is implicitly `deny`, data is omitted. + let query = "query { musician(id: \"m1\") { id } }"; + let result = execute_query(&deployment, query).await; + let expected = json!({ + "errors": [ + { + "message": "indexing_error" + } + ] + }); + assert_eq!(expected, serde_json::to_value(&result).unwrap()); + }) +} diff --git a/mock/Cargo.toml b/mock/Cargo.toml new file mode 100644 index 0000000..7a3a79a --- /dev/null +++ b/mock/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "graph-mock" +version = "0.27.0" +edition = "2021" + +[dependencies] +graph = { path = "../graph" } diff --git a/mock/src/lib.rs b/mock/src/lib.rs new file mode 100644 index 0000000..8d4df4b --- /dev/null +++ b/mock/src/lib.rs @@ -0,0 +1,3 @@ +mod metrics_registry; + +pub use self::metrics_registry::MockMetricsRegistry; diff --git a/mock/src/metrics_registry.rs b/mock/src/metrics_registry.rs new file mode 100644 index 0000000..0b45052 --- /dev/null +++ b/mock/src/metrics_registry.rs @@ -0,0 +1,87 @@ +use graph::components::metrics::{Collector, Counter, Gauge, Opts, PrometheusError}; +use graph::prelude::MetricsRegistry as MetricsRegistryTrait; +use graph::prometheus::{CounterVec, GaugeVec, HistogramOpts, HistogramVec}; + +use std::collections::HashMap; + +#[derive(Clone)] +pub struct MockMetricsRegistry {} + +impl MockMetricsRegistry { + pub fn new() -> Self { + Self {} + } +} + +impl MetricsRegistryTrait for MockMetricsRegistry { + fn register(&self, _name: &str, _c: Box) { + // Ignore, we do not register metrics + } + + fn global_counter( + &self, + name: &str, + help: &str, + const_labels: HashMap, + ) -> Result { + let opts = Opts::new(name, help).const_labels(const_labels); + Counter::with_opts(opts) + } + + fn global_gauge( + &self, + name: &str, + help: &str, + const_labels: HashMap, + ) -> Result { + let opts = Opts::new(name, help).const_labels(const_labels); + Gauge::with_opts(opts) + } + + fn unregister(&self, _: Box) {} + + fn global_counter_vec( + &self, + name: &str, + help: &str, + variable_labels: &[&str], + ) -> Result { + let opts = Opts::new(name, help); + let counters = CounterVec::new(opts, variable_labels)?; + Ok(counters) + } + + fn global_deployment_counter_vec( + &self, + name: &str, + help: &str, + subgraph: &str, + variable_labels: &[&str], + ) -> Result { + let opts = Opts::new(name, help).const_label("deployment", subgraph); + let counters = CounterVec::new(opts, variable_labels)?; + Ok(counters) + } + + fn global_gauge_vec( + &self, + name: &str, + help: &str, + variable_labels: &[&str], + ) -> Result { + let opts = Opts::new(name, help); + let gauges = GaugeVec::new(opts, variable_labels)?; + Ok(gauges) + } + + fn global_histogram_vec( + &self, + name: &str, + help: &str, + variable_labels: &[&str], + ) -> Result { + let opts = HistogramOpts::new(name, help); + let histograms = HistogramVec::new(opts, variable_labels)?; + Ok(histograms) + } +} diff --git a/node/Cargo.toml b/node/Cargo.toml new file mode 100644 index 0000000..ddee868 --- /dev/null +++ b/node/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "graph-node" +version = "0.27.0" +edition = "2021" +default-run = "graph-node" + +[[bin]] +name = "graph-node" +path = "src/main.rs" + +[[bin]] +name = "graphman" +path = "src/bin/manager.rs" + +[dependencies] +clap = { version = "3.2.22", features = ["derive", "env"] } +env_logger = "0.9.0" +git-testament = "0.2" +graphql-parser = "0.4.0" +futures = { version = "0.3.1", features = ["compat"] } +lazy_static = "1.2.0" +url = "2.2.1" +crossbeam-channel = "0.5.5" +graph = { path = "../graph" } +graph-core = { path = "../core" } +graph-chain-arweave = { path = "../chain/arweave" } +graph-chain-ethereum = { path = "../chain/ethereum" } +graph-chain-near = { path = "../chain/near" } +graph-chain-cosmos = { path = "../chain/cosmos" } +graph-chain-substreams= { path = "../chain/substreams" } +graph-graphql = { path = "../graphql" } +graph-runtime-wasm = { path = "../runtime/wasm" } +graph-server-http = { path = "../server/http" } +graph-server-index-node = { path = "../server/index-node" } +graph-server-json-rpc = { path = "../server/json-rpc"} +graph-server-websocket = { path = "../server/websocket" } +graph-server-metrics = { path = "../server/metrics" } +graph-store-postgres = { path = "../store/postgres" } +regex = "1.5.4" +serde = { version = "1.0.126", features = ["derive", "rc"] } +serde_regex = "1.1.0" +toml = "0.5.7" +shellexpand = "2.1.0" +diesel = "1.4.8" +http = "0.2.5" # must be compatible with the version rust-web3 uses +prometheus = { version ="0.13.2", features = ["push"] } +json-structural-diff = {version = "0.1", features = ["colorize"] } diff --git a/node/resources/tests/full_config.toml b/node/resources/tests/full_config.toml new file mode 100644 index 0000000..97d3be6 --- /dev/null +++ b/node/resources/tests/full_config.toml @@ -0,0 +1,70 @@ +[general] +query = "query_node_.*" + +[store] +[store.primary] +connection = "postgresql://postgres:1.1.1.1@test/primary" +pool_size = [ + { node = "index_node_1_.*", size = 2 }, + { node = "index_node_2_.*", size = 10 }, + { node = "index_node_3_.*", size = 10 }, + { node = "index_node_4_.*", size = 2 }, + { node = "query_node_.*", size = 10 } +] + +[store.shard_a] +connection = "postgresql://postgres:1.1.1.1@test/shard-a" +pool_size = [ + { node = "index_node_1_.*", size = 2 }, + { node = "index_node_2_.*", size = 10 }, + { node = "index_node_3_.*", size = 10 }, + { node = "index_node_4_.*", size = 2 }, + { node = "query_node_.*", size = 10 } +] + +[deployment] +# Studio subgraphs +[[deployment.rule]] +match = { name = "^prefix/" } +shard = "shard_a" +indexers = [ "index_prefix_0", + "index_prefix_1" ] + +[[deployment.rule]] +match = { name = "^custom/.*" } +indexers = [ "index_custom_0" ] + +[[deployment.rule]] +shards = [ "primary", "shard_a" ] +indexers = [ "index_node_1_a", + "index_node_2_a", + "index_node_3_a" ] + +[chains] +ingestor = "index_0" + +[chains.mainnet] +shard = "primary" +provider = [ + { label = "mainnet-0", url = "http://rpc.mainnet.io", features = ["archive", "traces"] }, + { label = "firehose", details = { type = "firehose", url = "http://localhost:9000", features = [] }}, + { label = "substreams", details = { type = "substreams", url = "http://localhost:9000", features = [] }}, +] + +[chains.ropsten] +shard = "primary" +provider = [ + { label = "ropsten-0", url = "http://rpc.ropsten.io", transport = "rpc", features = ["archive", "traces"] } +] + +[chains.goerli] +shard = "primary" +provider = [ + { label = "goerli-0", url = "http://rpc.goerli.io", transport = "ipc", features = ["archive"] } +] + +[chains.kovan] +shard = "primary" +provider = [ + { label = "kovan-0", url = "http://rpc.kovan.io", transport = "ws", features = [] } +] diff --git a/node/src/bin/manager.rs b/node/src/bin/manager.rs new file mode 100644 index 0000000..f68a2db --- /dev/null +++ b/node/src/bin/manager.rs @@ -0,0 +1,1073 @@ +use clap::{Parser, Subcommand}; +use config::PoolSize; +use git_testament::{git_testament, render_testament}; +use graph::{data::graphql::effort::LoadManager, prelude::chrono, prometheus::Registry}; +use graph::{ + log::logger, + prelude::{ + anyhow::{self, Context as AnyhowContextTrait}, + info, o, slog, tokio, Logger, NodeId, ENV_VARS, + }, + url::Url, +}; +use graph_chain_ethereum::{EthereumAdapter, EthereumNetworks}; +use graph_core::MetricsRegistry; +use graph_graphql::prelude::GraphQlRunner; +use graph_node::config::{self, Config as Cfg}; +use graph_node::manager::commands; +use graph_node::{ + chain::create_ethereum_networks, + manager::{deployment::DeploymentSearch, PanicSubscriptionManager}, + store_builder::StoreBuilder, + MetricsContext, +}; +use graph_store_postgres::ChainStore; +use graph_store_postgres::{ + connection_pool::ConnectionPool, BlockStore, NotificationSender, Shard, Store, SubgraphStore, + SubscriptionManager, PRIMARY_SHARD, +}; +use lazy_static::lazy_static; +use std::{collections::HashMap, env, num::ParseIntError, sync::Arc, time::Duration}; +const VERSION_LABEL_KEY: &str = "version"; + +git_testament!(TESTAMENT); + +lazy_static! { + static ref RENDERED_TESTAMENT: String = render_testament!(TESTAMENT); +} + +#[derive(Parser, Clone, Debug)] +#[clap( + name = "graphman", + about = "Management tool for a graph-node infrastructure", + author = "Graph Protocol, Inc.", + version = RENDERED_TESTAMENT.as_str() +)] +pub struct Opt { + #[clap( + long, + short, + env = "GRAPH_NODE_CONFIG", + help = "the name of the configuration file\n" + )] + pub config: String, + #[clap( + long, + default_value = "default", + value_name = "NODE_ID", + env = "GRAPH_NODE_ID", + help = "a unique identifier for this node.\nShould have the same value between consecutive node restarts\n" + )] + pub node_id: String, + #[clap( + long, + value_name = "{HOST:PORT|URL}", + default_value = "https://api.thegraph.com/ipfs/", + env = "IPFS", + help = "HTTP addresses of IPFS nodes" + )] + pub ipfs: Vec, + #[clap( + long, + default_value = "3", + help = "the size for connection pools. Set to 0\n to use pool size from configuration file\n corresponding to NODE_ID" + )] + pub pool_size: u32, + #[clap(long, value_name = "URL", help = "Base URL for forking subgraphs")] + pub fork_base: Option, + #[clap(long, help = "version label, used for prometheus metrics")] + pub version_label: Option, + #[clap(subcommand)] + pub cmd: Command, +} + +#[derive(Clone, Debug, Subcommand)] +pub enum Command { + /// Calculate the transaction speed + TxnSpeed { + #[clap(long, short, default_value = "60")] + delay: u64, + }, + /// Print details about a deployment + /// + /// The deployment can be specified as either a subgraph name, an IPFS + /// hash `Qm..`, or the database namespace `sgdNNN`. Since the same IPFS + /// hash can be deployed in multiple shards, it is possible to specify + /// the shard by adding `:shard` to the IPFS hash. + Info { + /// The deployment (see above) + deployment: DeploymentSearch, + /// List only current version + #[clap(long, short)] + current: bool, + /// List only pending versions + #[clap(long, short)] + pending: bool, + /// Include status information + #[clap(long, short)] + status: bool, + /// List only used (current and pending) versions + #[clap(long, short)] + used: bool, + }, + /// Manage unused deployments + /// + /// Record which deployments are unused with `record`, then remove them + /// with `remove` + #[clap(subcommand)] + Unused(UnusedCommand), + /// Remove a named subgraph + Remove { + /// The name of the subgraph to remove + name: String, + }, + /// Create a subgraph name + Create { + /// The name of the subgraph to create + name: String, + }, + /// Assign or reassign a deployment + Reassign { + /// The deployment (see `help info`) + deployment: DeploymentSearch, + /// The name of the node that should index the deployment + node: String, + }, + /// Unassign a deployment + Unassign { + /// The deployment (see `help info`) + deployment: DeploymentSearch, + }, + /// Rewind a subgraph to a specific block + Rewind { + /// Force rewinding even if the block hash is not found in the local + /// database + #[clap(long, short)] + force: bool, + /// Sleep for this many seconds after pausing subgraphs + #[clap( + long, + short, + default_value = "10", + parse(try_from_str = parse_duration_in_secs) + )] + sleep: Duration, + /// The block hash of the target block + block_hash: String, + /// The block number of the target block + block_number: i32, + /// The deployments to rewind (see `help info`) + deployments: Vec, + }, + /// Deploy and run an arbitrary subgraph up to a certain block + /// + /// The run can surpass it by a few blocks, it's not exact (use for dev + /// and testing purposes) -- WARNING: WILL RUN MIGRATIONS ON THE DB, DO + /// NOT USE IN PRODUCTION + /// + /// Also worth noting that the deployed subgraph will be removed at the + /// end. + Run { + /// Network name (must fit one of the chain) + network_name: String, + + /// Subgraph in the form `` or `:` + subgraph: String, + + /// Highest block number to process before stopping (inclusive) + stop_block: i32, + + /// Prometheus push gateway endpoint. + prometheus_host: Option, + }, + /// Check and interrogate the configuration + /// + /// Print information about a configuration file without + /// actually connecting to databases or network clients + #[clap(subcommand)] + Config(ConfigCommand), + /// Listen for store events and print them + #[clap(subcommand)] + Listen(ListenCommand), + /// Manage deployment copies and grafts + #[clap(subcommand)] + Copy(CopyCommand), + /// Run a GraphQL query + Query { + /// Save the JSON query result in this file + #[clap(long, short)] + output: Option, + /// Save the query trace in this file + #[clap(long, short)] + trace: Option, + + /// The subgraph to query + /// + /// Either a deployment id `Qm..` or a subgraph name + target: String, + /// The GraphQL query + query: String, + /// The variables in the form `key=value` + vars: Vec, + }, + /// Get information about chains and manipulate them + #[clap(subcommand)] + Chain(ChainCommand), + /// Manipulate internal subgraph statistics + #[clap(subcommand)] + Stats(StatsCommand), + + /// Manage database indexes + #[clap(subcommand)] + Index(IndexCommand), + + /// Prune deployments + Prune { + /// The deployment to prune (see `help info`) + deployment: DeploymentSearch, + /// Prune tables with a ratio of entities to entity versions lower than this + #[clap(long, short, default_value = "0.20")] + prune_ratio: f64, + /// How much history to keep in blocks + #[clap(long, short, default_value = "10000")] + history: usize, + }, +} + +impl Command { + /// Return `true` if the command should not override connection pool + /// sizes, in general only when we will not actually connect to any + /// databases + fn use_configured_pool_size(&self) -> bool { + matches!(self, Command::Config(_)) + } +} + +#[derive(Clone, Debug, Subcommand)] +pub enum UnusedCommand { + /// List unused deployments + List { + /// Only list unused deployments that still exist + #[clap(short, long)] + existing: bool, + }, + /// Update and record currently unused deployments + Record, + /// Remove deployments that were marked as unused with `record`. + /// + /// Deployments are removed in descending order of number of entities, + /// i.e., smaller deployments are removed before larger ones + Remove { + /// How many unused deployments to remove (default: all) + #[clap(short, long)] + count: Option, + /// Remove a specific deployment + #[clap(short, long, conflicts_with = "count")] + deployment: Option, + /// Remove unused deployments that were recorded at least this many minutes ago + #[clap(short, long)] + older: Option, + }, +} + +#[derive(Clone, Debug, Subcommand)] +pub enum ConfigCommand { + /// Check and validate the configuration file + Check { + /// Print the configuration as JSON + #[clap(long)] + print: bool, + }, + /// Print how a specific subgraph would be placed + Place { + /// The name of the subgraph + name: String, + /// The network the subgraph indexes + network: String, + }, + /// Information about the size of database pools + Pools { + /// The names of the nodes that are going to run + nodes: Vec, + /// Print connections by shard rather than by node + #[clap(short, long)] + shard: bool, + }, + /// Show eligible providers + /// + /// Prints the providers that can be used for a deployment on a given + /// network with the given features. Set the name of the node for which + /// to simulate placement with the toplevel `--node-id` option + Provider { + #[clap(short, long, default_value = "")] + features: String, + network: String, + }, +} + +#[derive(Clone, Debug, Subcommand)] +pub enum ListenCommand { + /// Listen only to assignment events + Assignments, + /// Listen to events for entities in a specific deployment + Entities { + /// The deployment (see `help info`). + deployment: DeploymentSearch, + /// The entity types for which to print change notifications + entity_types: Vec, + }, +} +#[derive(Clone, Debug, Subcommand)] +pub enum CopyCommand { + /// Create a copy of an existing subgraph + /// + /// The copy will be treated as its own deployment. The deployment with + /// IPFS hash `src` will be copied to a new deployment in the database + /// shard `shard` and will be assigned to `node` for indexing. The new + /// subgraph will start as a copy of all blocks of `src` that are + /// `offset` behind the current subgraph head of `src`. The offset + /// should be chosen such that only final blocks are copied + Create { + /// How far behind `src` subgraph head to copy + #[clap(long, short, default_value = "200")] + offset: u32, + /// The source deployment (see `help info`) + src: DeploymentSearch, + /// The name of the database shard into which to copy + shard: String, + /// The name of the node that should index the copy + node: String, + }, + /// Activate the copy of a deployment. + /// + /// This will route queries to that specific copy (with some delay); the + /// previously active copy will become inactive. Only copies that have + /// progressed at least as far as the original should be activated. + Activate { + /// The IPFS hash of the deployment to activate + deployment: String, + /// The name of the database shard that holds the copy + shard: String, + }, + /// List all currently running copy and graft operations + List, + /// Print the progress of a copy operation + Status { + /// The destination deployment of the copy operation (see `help info`) + dst: DeploymentSearch, + }, +} + +#[derive(Clone, Debug, Subcommand)] +pub enum ChainCommand { + /// List all chains that are in the database + List, + /// Show information about a chain + Info { + #[clap( + long, + short, + default_value = "50", + env = "ETHEREUM_REORG_THRESHOLD", + help = "the reorg threshold to check\n" + )] + reorg_threshold: i32, + #[clap(long, help = "display block hashes\n")] + hashes: bool, + name: String, + }, + /// Remove a chain and all its data + /// + /// There must be no deployments using that chain. If there are, the + /// subgraphs and/or deployments using the chain must first be removed + Remove { name: String }, + + /// Compares cached blocks with fresh ones and clears the block cache when they differ. + CheckBlocks { + #[clap(subcommand)] // Note that we mark a field as a subcommand + method: CheckBlockMethod, + + /// Chain name (must be an existing chain, see 'chain list') + #[clap(empty_values = false)] + chain_name: String, + }, + /// Truncates the whole block cache for the given chain. + Truncate { + /// Chain name (must be an existing chain, see 'chain list') + #[clap(empty_values = false)] + chain_name: String, + /// Skips confirmation prompt + #[clap(long, short)] + force: bool, + }, +} + +#[derive(Clone, Debug, Subcommand)] +pub enum StatsCommand { + /// Toggle whether a table is account-like + /// + /// Setting a table to 'account-like' enables a query optimization that + /// is very effective for tables with a high ratio of entity versions + /// to distinct entities. It can take up to 5 minutes for this to take + /// effect. + AccountLike { + #[clap(long, short, help = "do not set but clear the account-like flag\n")] + clear: bool, + /// The deployment (see `help info`). + deployment: DeploymentSearch, + /// The name of the database table + table: String, + }, + /// Show statistics for the tables of a deployment + /// + /// Show how many distinct entities and how many versions the tables of + /// each subgraph have. The data is based on the statistics that + /// Postgres keeps, and only refreshed when a table is analyzed. + Show { + /// The deployment (see `help info`). + deployment: DeploymentSearch, + }, + /// Perform a SQL ANALYZE in a Entity table + Analyze { + /// The deployment (see `help info`). + deployment: DeploymentSearch, + /// The name of the Entity to ANALYZE, in camel case + entity: String, + }, +} + +#[derive(Clone, Debug, Subcommand)] +pub enum IndexCommand { + /// Creates a new database index. + /// + /// The new index will be created concurrenly for the provided entity and its fields. whose + /// names must be declared the in camel case, following GraphQL conventions. + /// + /// The index will have its validity checked after the operation and will be dropped if it is + /// invalid. + /// + /// This command may be time-consuming. + Create { + /// The deployment (see `help info`). + #[clap(empty_values = false)] + deployment: DeploymentSearch, + /// The Entity name. + /// + /// Can be expressed either in upper camel case (as its GraphQL definition) or in snake case + /// (as its SQL table name). + #[clap(empty_values = false)] + entity: String, + /// The Field names. + /// + /// Each field can be expressed either in camel case (as its GraphQL definition) or in snake + /// case (as its SQL colmun name). + #[clap(min_values = 1, required = true)] + fields: Vec, + /// The index method. Defaults to `btree`. + #[clap( + short, long, default_value = "btree", + possible_values = &["btree", "hash", "gist", "spgist", "gin", "brin"] + )] + method: String, + }, + /// Lists existing indexes for a given Entity + List { + /// The deployment (see `help info`). + #[clap(empty_values = false)] + deployment: DeploymentSearch, + /// The Entity name. + /// + /// Can be expressed either in upper camel case (as its GraphQL definition) or in snake case + /// (as its SQL table name). + #[clap(empty_values = false)] + entity: String, + }, + + /// Drops an index for a given deployment, concurrently + Drop { + /// The deployment (see `help info`). + #[clap(empty_values = false)] + deployment: DeploymentSearch, + /// The name of the index to be dropped + #[clap(empty_values = false)] + index_name: String, + }, +} + +#[derive(Clone, Debug, Subcommand)] +pub enum CheckBlockMethod { + /// The number of the target block + ByHash { hash: String }, + + /// The hash of the target block + ByNumber { number: i32 }, + + /// A block number range, inclusive on both ends. + ByRange { + #[clap(long, short)] + from: Option, + #[clap(long, short)] + to: Option, + }, +} + +impl From for config::Opt { + fn from(opt: Opt) -> Self { + let mut config_opt = config::Opt::default(); + config_opt.config = Some(opt.config); + config_opt.store_connection_pool_size = 5; + config_opt.node_id = opt.node_id; + config_opt + } +} + +/// Utilities to interact mostly with the store and build the parts of the +/// store we need for specific commands +struct Context { + logger: Logger, + node_id: NodeId, + config: Cfg, + ipfs_url: Vec, + fork_base: Option, + registry: Arc, + pub prometheus_registry: Arc, +} + +impl Context { + fn new( + logger: Logger, + node_id: NodeId, + config: Cfg, + ipfs_url: Vec, + fork_base: Option, + version_label: Option, + ) -> Self { + let prometheus_registry = Arc::new( + Registry::new_custom( + None, + version_label.map(|label| { + let mut m = HashMap::::new(); + m.insert(VERSION_LABEL_KEY.into(), label); + m + }), + ) + .expect("unable to build prometheus registry"), + ); + let registry = Arc::new(MetricsRegistry::new( + logger.clone(), + prometheus_registry.clone(), + )); + + Self { + logger, + node_id, + config, + ipfs_url, + fork_base, + registry, + prometheus_registry, + } + } + + fn metrics_registry(&self) -> Arc { + self.registry.clone() + } + + fn config(&self) -> Cfg { + self.config.clone() + } + + fn node_id(&self) -> NodeId { + self.node_id.clone() + } + + fn notification_sender(&self) -> Arc { + Arc::new(NotificationSender::new(self.registry.clone())) + } + + fn primary_pool(self) -> ConnectionPool { + let primary = self.config.primary_store(); + let pool = StoreBuilder::main_pool( + &self.logger, + &self.node_id, + PRIMARY_SHARD.as_str(), + primary, + self.metrics_registry(), + Arc::new(vec![]), + ); + pool.skip_setup(); + pool + } + + fn subgraph_store(self) -> Arc { + self.store_and_pools().0.subgraph_store() + } + + fn subscription_manager(&self) -> Arc { + let primary = self.config.primary_store(); + + Arc::new(SubscriptionManager::new( + self.logger.clone(), + primary.connection.to_owned(), + self.registry.clone(), + )) + } + + fn primary_and_subscription_manager(self) -> (ConnectionPool, Arc) { + let mgr = self.subscription_manager(); + let primary_pool = self.primary_pool(); + + (primary_pool, mgr) + } + + fn store(self) -> Arc { + let (store, _) = self.store_and_pools(); + store + } + + fn pools(self) -> HashMap { + let (_, pools) = self.store_and_pools(); + pools + } + + async fn store_builder(&self) -> StoreBuilder { + StoreBuilder::new( + &self.logger, + &self.node_id, + &self.config, + self.fork_base.clone(), + self.registry.clone(), + ) + .await + } + + fn store_and_pools(self) -> (Arc, HashMap) { + let (subgraph_store, pools) = StoreBuilder::make_subgraph_store_and_pools( + &self.logger, + &self.node_id, + &self.config, + self.fork_base, + self.registry, + ); + + for pool in pools.values() { + pool.skip_setup(); + } + + let store = StoreBuilder::make_store( + &self.logger, + pools.clone(), + subgraph_store, + HashMap::default(), + vec![], + ); + + (store, pools) + } + + fn store_and_primary(self) -> (Arc, ConnectionPool) { + let (store, pools) = self.store_and_pools(); + let primary = pools.get(&*PRIMARY_SHARD).expect("there is a primary pool"); + (store, primary.clone()) + } + + fn block_store_and_primary_pool(self) -> (Arc, ConnectionPool) { + let (store, pools) = self.store_and_pools(); + + let primary = pools.get(&*PRIMARY_SHARD).unwrap(); + (store.block_store(), primary.clone()) + } + + fn graphql_runner(self) -> Arc> { + let logger = self.logger.clone(); + let registry = self.registry.clone(); + + let store = self.store(); + + let subscription_manager = Arc::new(PanicSubscriptionManager); + let load_manager = Arc::new(LoadManager::new(&logger, vec![], registry.clone())); + + Arc::new(GraphQlRunner::new( + &logger, + store, + subscription_manager, + load_manager, + registry, + )) + } + + async fn ethereum_networks(&self) -> anyhow::Result { + let logger = self.logger.clone(); + let registry = self.metrics_registry(); + create_ethereum_networks(logger, registry, &self.config).await + } + + fn chain_store(self, chain_name: &str) -> anyhow::Result> { + use graph::components::store::BlockStore; + self.store() + .block_store() + .chain_store(&chain_name) + .ok_or_else(|| anyhow::anyhow!("Could not find a network named '{}'", chain_name)) + } + + async fn chain_store_and_adapter( + self, + chain_name: &str, + ) -> anyhow::Result<(Arc, Arc)> { + let ethereum_networks = self.ethereum_networks().await?; + let chain_store = self.chain_store(chain_name)?; + let ethereum_adapter = ethereum_networks + .networks + .get(chain_name) + .map(|adapters| adapters.cheapest()) + .flatten() + .ok_or(anyhow::anyhow!( + "Failed to obtain an Ethereum adapter for chain '{}'", + chain_name + ))?; + Ok((chain_store, ethereum_adapter)) + } +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let opt = Opt::parse(); + + let version_label = opt.version_label.clone(); + // Set up logger + let logger = match ENV_VARS.log_levels { + Some(_) => logger(false), + None => Logger::root(slog::Discard, o!()), + }; + + // Log version information + info!( + logger, + "Graph Node version: {}", + render_testament!(TESTAMENT) + ); + + let mut config = Cfg::load(&logger, &opt.clone().into()).context("Configuration error")?; + + if opt.pool_size > 0 && !opt.cmd.use_configured_pool_size() { + // Override pool size from configuration + for shard in config.stores.values_mut() { + shard.pool_size = PoolSize::Fixed(opt.pool_size); + for replica in shard.replicas.values_mut() { + replica.pool_size = PoolSize::Fixed(opt.pool_size); + } + } + } + + let node = match NodeId::new(&opt.node_id) { + Err(()) => { + eprintln!("invalid node id: {}", opt.node_id); + std::process::exit(1); + } + Ok(node) => node, + }; + + let fork_base = match &opt.fork_base { + Some(url) => { + // Make sure the endpoint ends with a terminating slash. + let url = if !url.ends_with("/") { + let mut url = url.clone(); + url.push('/'); + Url::parse(&url) + } else { + Url::parse(url) + }; + + match url { + Err(e) => { + eprintln!("invalid fork base URL: {}", e); + std::process::exit(1); + } + Ok(url) => Some(url), + } + } + None => None, + }; + + let ctx = Context::new( + logger.clone(), + node, + config, + opt.ipfs, + fork_base, + version_label.clone(), + ); + + use Command::*; + match opt.cmd { + TxnSpeed { delay } => commands::txn_speed::run(ctx.primary_pool(), delay), + Info { + deployment, + current, + pending, + status, + used, + } => { + let (primary, store) = if status { + let (store, primary) = ctx.store_and_primary(); + (primary.clone(), Some(store)) + } else { + (ctx.primary_pool(), None) + }; + commands::info::run(primary, store, deployment, current, pending, used) + } + Unused(cmd) => { + let store = ctx.subgraph_store(); + use UnusedCommand::*; + + match cmd { + List { existing } => commands::unused_deployments::list(store, existing), + Record => commands::unused_deployments::record(store), + Remove { + count, + deployment, + older, + } => { + let count = count.unwrap_or(1_000_000); + let older = older.map(|older| chrono::Duration::minutes(older as i64)); + commands::unused_deployments::remove(store, count, deployment, older) + } + } + } + Config(cmd) => { + use ConfigCommand::*; + + match cmd { + Place { name, network } => { + commands::config::place(&ctx.config.deployment, &name, &network) + } + Check { print } => commands::config::check(&ctx.config, print), + Pools { nodes, shard } => commands::config::pools(&ctx.config, nodes, shard), + Provider { features, network } => { + let logger = ctx.logger.clone(); + let registry = ctx.registry.clone(); + commands::config::provider(logger, &ctx.config, registry, features, network) + .await + } + } + } + Remove { name } => commands::remove::run(ctx.subgraph_store(), name), + Create { name } => commands::create::run(ctx.subgraph_store(), name), + Unassign { deployment } => { + let sender = ctx.notification_sender(); + commands::assign::unassign(ctx.primary_pool(), &sender, &deployment).await + } + Reassign { deployment, node } => { + let sender = ctx.notification_sender(); + commands::assign::reassign(ctx.primary_pool(), &sender, &deployment, node) + } + Rewind { + force, + sleep, + block_hash, + block_number, + deployments, + } => { + let (store, primary) = ctx.store_and_primary(); + commands::rewind::run( + primary, + store, + deployments, + block_hash, + block_number, + force, + sleep, + ) + .await + } + Run { + network_name, + subgraph, + stop_block, + prometheus_host, + } => { + let logger = ctx.logger.clone(); + let config = ctx.config(); + let registry = ctx.metrics_registry().clone(); + let node_id = ctx.node_id().clone(); + let store_builder = ctx.store_builder().await; + let job_name = version_label.clone(); + let ipfs_url = ctx.ipfs_url.clone(); + let metrics_ctx = MetricsContext { + prometheus: ctx.prometheus_registry.clone(), + registry: registry.clone(), + prometheus_host, + job_name, + }; + + commands::run::run( + logger, + store_builder, + network_name, + ipfs_url, + config, + metrics_ctx, + node_id, + subgraph, + stop_block, + ) + .await + } + Listen(cmd) => { + use ListenCommand::*; + match cmd { + Assignments => commands::listen::assignments(ctx.subscription_manager()).await, + Entities { + deployment, + entity_types, + } => { + let (primary, mgr) = ctx.primary_and_subscription_manager(); + commands::listen::entities(primary, mgr, &deployment, entity_types).await + } + } + } + Copy(cmd) => { + use CopyCommand::*; + match cmd { + Create { + src, + shard, + node, + offset, + } => { + let shards: Vec<_> = ctx.config.stores.keys().cloned().collect(); + let (store, primary) = ctx.store_and_primary(); + commands::copy::create(store, primary, src, shard, shards, node, offset).await + } + Activate { deployment, shard } => { + commands::copy::activate(ctx.subgraph_store(), deployment, shard) + } + List => commands::copy::list(ctx.pools()), + Status { dst } => commands::copy::status(ctx.pools(), &dst), + } + } + Query { + output, + trace, + target, + query, + vars, + } => commands::query::run(ctx.graphql_runner(), target, query, vars, output, trace).await, + Chain(cmd) => { + use ChainCommand::*; + match cmd { + List => { + let (block_store, primary) = ctx.block_store_and_primary_pool(); + commands::chain::list(primary, block_store).await + } + Info { + name, + reorg_threshold, + hashes, + } => { + let (block_store, primary) = ctx.block_store_and_primary_pool(); + commands::chain::info(primary, block_store, name, reorg_threshold, hashes).await + } + Remove { name } => { + let (block_store, primary) = ctx.block_store_and_primary_pool(); + commands::chain::remove(primary, block_store, name) + } + CheckBlocks { method, chain_name } => { + use commands::check_blocks::{by_hash, by_number, by_range}; + use CheckBlockMethod::*; + let logger = ctx.logger.clone(); + let (chain_store, ethereum_adapter) = + ctx.chain_store_and_adapter(&chain_name).await?; + match method { + ByHash { hash } => { + by_hash(&hash, chain_store, ðereum_adapter, &logger).await + } + ByNumber { number } => { + by_number(number, chain_store, ðereum_adapter, &logger).await + } + ByRange { from, to } => { + by_range(chain_store, ðereum_adapter, from, to, &logger).await + } + } + } + Truncate { chain_name, force } => { + use commands::check_blocks::truncate; + let chain_store = ctx.chain_store(&chain_name)?; + truncate(chain_store, force) + } + } + } + Stats(cmd) => { + use StatsCommand::*; + match cmd { + AccountLike { + clear, + deployment, + table, + } => { + let (store, primary_pool) = ctx.store_and_primary(); + let subgraph_store = store.subgraph_store(); + commands::stats::account_like( + subgraph_store, + primary_pool, + clear, + &deployment, + table, + ) + .await + } + Show { deployment } => commands::stats::show(ctx.pools(), &deployment), + Analyze { deployment, entity } => { + let (store, primary_pool) = ctx.store_and_primary(); + let subgraph_store = store.subgraph_store(); + commands::stats::analyze(subgraph_store, primary_pool, deployment, &entity) + } + } + } + Index(cmd) => { + use IndexCommand::*; + let (store, primary_pool) = ctx.store_and_primary(); + let subgraph_store = store.subgraph_store(); + match cmd { + Create { + deployment, + entity, + fields, + method, + } => { + commands::index::create( + subgraph_store, + primary_pool, + deployment, + &entity, + fields, + method, + ) + .await + } + List { deployment, entity } => { + commands::index::list(subgraph_store, primary_pool, deployment, &entity).await + } + Drop { + deployment, + index_name, + } => { + commands::index::drop(subgraph_store, primary_pool, deployment, &index_name) + .await + } + } + } + Prune { + deployment, + history, + prune_ratio, + } => { + let (store, primary_pool) = ctx.store_and_primary(); + commands::prune::run(store, primary_pool, deployment, history, prune_ratio).await + } + } +} + +fn parse_duration_in_secs(s: &str) -> Result { + Ok(Duration::from_secs(s.parse()?)) +} diff --git a/node/src/chain.rs b/node/src/chain.rs new file mode 100644 index 0000000..a7713d2 --- /dev/null +++ b/node/src/chain.rs @@ -0,0 +1,522 @@ +use crate::config::{Config, ProviderDetails}; +use ethereum::{EthereumNetworks, ProviderEthRpcMetrics}; +use futures::future::join_all; +use futures::TryFutureExt; +use graph::anyhow::Error; +use graph::blockchain::{Block as BlockchainBlock, BlockchainKind, ChainIdentifier}; +use graph::cheap_clone::CheapClone; +use graph::firehose::{FirehoseEndpoint, FirehoseNetworks}; +use graph::ipfs_client::IpfsClient; +use graph::prelude::{anyhow, tokio}; +use graph::prelude::{prost, MetricsRegistry as MetricsRegistryTrait}; +use graph::slog::{debug, error, info, o, Logger}; +use graph::url::Url; +use graph::util::security::SafeDisplay; +use graph_chain_ethereum::{self as ethereum, EthereumAdapterTrait, Transport}; +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; +use std::time::Duration; + +// The status of a provider that we learned from connecting to it +#[derive(PartialEq)] +enum ProviderNetworkStatus { + Broken { + chain_id: String, + provider: String, + }, + Version { + chain_id: String, + ident: ChainIdentifier, + }, +} + +/// How long we will hold up node startup to get the net version and genesis +/// hash from the client. If we can't get it within that time, we'll try and +/// continue regardless. +const NET_VERSION_WAIT_TIME: Duration = Duration::from_secs(30); + +pub fn create_ipfs_clients(logger: &Logger, ipfs_addresses: &Vec) -> Vec { + // Parse the IPFS URL from the `--ipfs` command line argument + let ipfs_addresses: Vec<_> = ipfs_addresses + .iter() + .map(|uri| { + if uri.starts_with("http://") || uri.starts_with("https://") { + String::from(uri) + } else { + format!("http://{}", uri) + } + }) + .collect(); + + ipfs_addresses + .into_iter() + .map(|ipfs_address| { + info!( + logger, + "Trying IPFS node at: {}", + SafeDisplay(&ipfs_address) + ); + + let ipfs_client = match IpfsClient::new(&ipfs_address) { + Ok(ipfs_client) => ipfs_client, + Err(e) => { + error!( + logger, + "Failed to create IPFS client for `{}`: {}", + SafeDisplay(&ipfs_address), + e + ); + panic!("Could not connect to IPFS"); + } + }; + + // Test the IPFS client by getting the version from the IPFS daemon + let ipfs_test = ipfs_client.cheap_clone(); + let ipfs_ok_logger = logger.clone(); + let ipfs_err_logger = logger.clone(); + let ipfs_address_for_ok = ipfs_address.clone(); + let ipfs_address_for_err = ipfs_address.clone(); + graph::spawn(async move { + ipfs_test + .test() + .map_err(move |e| { + error!( + ipfs_err_logger, + "Is there an IPFS node running at \"{}\"?", + SafeDisplay(ipfs_address_for_err), + ); + panic!("Failed to connect to IPFS: {}", e); + }) + .map_ok(move |_| { + info!( + ipfs_ok_logger, + "Successfully connected to IPFS node at: {}", + SafeDisplay(ipfs_address_for_ok) + ); + }) + .await + }); + + ipfs_client + }) + .collect() +} + +/// Parses an Ethereum connection string and returns the network name and Ethereum adapter. +pub async fn create_ethereum_networks( + logger: Logger, + registry: Arc, + config: &Config, +) -> Result { + let eth_rpc_metrics = Arc::new(ProviderEthRpcMetrics::new(registry)); + let mut parsed_networks = EthereumNetworks::new(); + for (name, chain) in &config.chains.chains { + if chain.protocol != BlockchainKind::Ethereum { + continue; + } + + for provider in &chain.providers { + if let ProviderDetails::Web3(web3) = &provider.details { + let capabilities = web3.node_capabilities(); + + let logger = logger.new(o!("provider" => provider.label.clone())); + info!( + logger, + "Creating transport"; + "url" => &web3.url, + "capabilities" => capabilities + ); + + use crate::config::Transport::*; + + let transport = match web3.transport { + Rpc => Transport::new_rpc(Url::parse(&web3.url)?, web3.headers.clone()), + Ipc => Transport::new_ipc(&web3.url).await, + Ws => Transport::new_ws(&web3.url).await, + }; + + let supports_eip_1898 = !web3.features.contains("no_eip1898"); + + parsed_networks.insert( + name.to_string(), + capabilities, + Arc::new( + graph_chain_ethereum::EthereumAdapter::new( + logger, + provider.label.clone(), + &web3.url, + transport, + eth_rpc_metrics.clone(), + supports_eip_1898, + ) + .await, + ), + web3.limit_for(&config.node), + ); + } + } + } + parsed_networks.sort(); + Ok(parsed_networks) +} + +pub fn create_substreams_networks( + logger: Logger, + config: &Config, +) -> BTreeMap { + debug!( + logger, + "Creating firehose networks [{} chains, ingestor {}]", + config.chains.chains.len(), + config.chains.ingestor, + ); + + let mut networks_by_kind = BTreeMap::new(); + + for (name, chain) in &config.chains.chains { + for provider in &chain.providers { + if let ProviderDetails::Substreams(ref firehose) = provider.details { + info!( + logger, + "Configuring firehose endpoint"; + "provider" => &provider.label, + ); + + let endpoint = FirehoseEndpoint::new( + &provider.label, + &firehose.url, + firehose.token.clone(), + firehose.filters_enabled(), + firehose.compression_enabled(), + firehose.conn_pool_size, + ); + + let parsed_networks = networks_by_kind + .entry(chain.protocol) + .or_insert_with(|| FirehoseNetworks::new()); + parsed_networks.insert(name.to_string(), Arc::new(endpoint)); + } + } + } + + networks_by_kind +} + +pub fn create_firehose_networks( + logger: Logger, + config: &Config, +) -> BTreeMap { + debug!( + logger, + "Creating firehose networks [{} chains, ingestor {}]", + config.chains.chains.len(), + config.chains.ingestor, + ); + + let mut networks_by_kind = BTreeMap::new(); + + for (name, chain) in &config.chains.chains { + for provider in &chain.providers { + if let ProviderDetails::Firehose(ref firehose) = provider.details { + info!( + logger, + "Configuring firehose endpoint"; + "provider" => &provider.label, + ); + + let endpoint = FirehoseEndpoint::new( + &provider.label, + &firehose.url, + firehose.token.clone(), + firehose.filters_enabled(), + firehose.compression_enabled(), + firehose.conn_pool_size, + ); + + let parsed_networks = networks_by_kind + .entry(chain.protocol) + .or_insert_with(|| FirehoseNetworks::new()); + parsed_networks.insert(name.to_string(), Arc::new(endpoint)); + } + } + } + + networks_by_kind +} + +/// Try to connect to all the providers in `eth_networks` and get their net +/// version and genesis block. Return the same `eth_networks` and the +/// retrieved net identifiers grouped by network name. Remove all providers +/// for which trying to connect resulted in an error from the returned +/// `EthereumNetworks`, since it's likely pointless to try and connect to +/// them. If the connection attempt to a provider times out after +/// `NET_VERSION_WAIT_TIME`, keep the provider, but don't report a +/// version for it. +pub async fn connect_ethereum_networks( + logger: &Logger, + mut eth_networks: EthereumNetworks, +) -> (EthereumNetworks, Vec<(String, Vec)>) { + // This has one entry for each provider, and therefore multiple entries + // for each network + let statuses = join_all( + eth_networks + .flatten() + .into_iter() + .map(|(network_name, capabilities, eth_adapter)| { + (network_name, capabilities, eth_adapter, logger.clone()) + }) + .map(|(network, capabilities, eth_adapter, logger)| async move { + let logger = logger.new(o!("provider" => eth_adapter.provider().to_string())); + info!( + logger, "Connecting to Ethereum to get network identifier"; + "capabilities" => &capabilities + ); + match tokio::time::timeout(NET_VERSION_WAIT_TIME, eth_adapter.net_identifiers()) + .await + .map_err(Error::from) + { + // An `Err` means a timeout, an `Ok(Err)` means some other error (maybe a typo + // on the URL) + Ok(Err(e)) | Err(e) => { + error!(logger, "Connection to provider failed. Not using this provider"; + "error" => e.to_string()); + ProviderNetworkStatus::Broken { + chain_id: network, + provider: eth_adapter.provider().to_string(), + } + } + Ok(Ok(ident)) => { + info!( + logger, + "Connected to Ethereum"; + "network_version" => &ident.net_version, + "capabilities" => &capabilities + ); + ProviderNetworkStatus::Version { + chain_id: network, + ident, + } + } + } + }), + ) + .await; + + // Group identifiers by network name + let idents: HashMap> = + statuses + .into_iter() + .fold(HashMap::new(), |mut networks, status| { + match status { + ProviderNetworkStatus::Broken { + chain_id: network, + provider, + } => eth_networks.remove(&network, &provider), + ProviderNetworkStatus::Version { + chain_id: network, + ident, + } => networks.entry(network.to_string()).or_default().push(ident), + } + networks + }); + let idents: Vec<_> = idents.into_iter().collect(); + (eth_networks, idents) +} + +/// Try to connect to all the providers in `firehose_networks` and get their net +/// version and genesis block. Return the same `eth_networks` and the +/// retrieved net identifiers grouped by network name. Remove all providers +/// for which trying to connect resulted in an error from the returned +/// `EthereumNetworks`, since it's likely pointless to try and connect to +/// them. If the connection attempt to a provider times out after +/// `NET_VERSION_WAIT_TIME`, keep the provider, but don't report a +/// version for it. +pub async fn connect_firehose_networks( + logger: &Logger, + mut firehose_networks: FirehoseNetworks, +) -> (FirehoseNetworks, Vec<(String, Vec)>) +where + M: prost::Message + BlockchainBlock + Default + 'static, +{ + // This has one entry for each provider, and therefore multiple entries + // for each network + let statuses = join_all( + firehose_networks + .flatten() + .into_iter() + .map(|(chain_id, endpoint)| (chain_id, endpoint, logger.clone())) + .map(|(chain_id, endpoint, logger)| async move { + let logger = logger.new(o!("provider" => endpoint.provider.to_string())); + info!( + logger, "Connecting to Firehose to get chain identifier"; + "provider" => &endpoint.provider, + ); + match tokio::time::timeout( + NET_VERSION_WAIT_TIME, + endpoint.genesis_block_ptr::(&logger), + ) + .await + .map_err(Error::from) + { + // An `Err` means a timeout, an `Ok(Err)` means some other error (maybe a typo + // on the URL) + Ok(Err(e)) | Err(e) => { + error!(logger, "Connection to provider failed. Not using this provider"; + "error" => format!("{:#}", e)); + ProviderNetworkStatus::Broken { + chain_id, + provider: endpoint.provider.to_string(), + } + } + Ok(Ok(ptr)) => { + info!( + logger, + "Connected to Firehose"; + "provider" => &endpoint.provider, + "genesis_block" => format_args!("{}", &ptr), + ); + + let ident = ChainIdentifier { + net_version: "0".to_string(), + genesis_block_hash: ptr.hash, + }; + + ProviderNetworkStatus::Version { chain_id, ident } + } + } + }), + ) + .await; + + // Group identifiers by chain id + let idents: HashMap> = + statuses + .into_iter() + .fold(HashMap::new(), |mut networks, status| { + match status { + ProviderNetworkStatus::Broken { chain_id, provider } => { + firehose_networks.remove(&chain_id, &provider) + } + ProviderNetworkStatus::Version { chain_id, ident } => networks + .entry(chain_id.to_string()) + .or_default() + .push(ident), + } + networks + }); + + // Clean-up chains with 0 provider + firehose_networks.networks.retain(|chain_id, endpoints| { + if endpoints.len() == 0 { + error!( + logger, + "No non-broken providers available for chain {}; ignoring this chain", chain_id + ); + } + + endpoints.len() > 0 + }); + + let idents: Vec<_> = idents.into_iter().collect(); + (firehose_networks, idents) +} + +#[cfg(test)] +mod test { + use crate::chain::create_ethereum_networks; + use crate::config::{Config, Opt}; + use graph::log::logger; + use graph::prelude::tokio; + use graph::prometheus::Registry; + use graph_chain_ethereum::NodeCapabilities; + use graph_core::MetricsRegistry; + use std::sync::Arc; + + #[tokio::test] + async fn correctly_parse_ethereum_networks() { + let logger = logger(true); + + let network_args = vec![ + "mainnet:traces:http://localhost:8545/".to_string(), + "goerli:archive:http://localhost:8546/".to_string(), + ]; + + let opt = Opt { + postgres_url: Some("not needed".to_string()), + config: None, + store_connection_pool_size: 5, + postgres_secondary_hosts: vec![], + postgres_host_weights: vec![], + disable_block_ingestor: true, + node_id: "default".to_string(), + ethereum_rpc: network_args, + ethereum_ws: vec![], + ethereum_ipc: vec![], + unsafe_config: false, + }; + + let config = Config::load(&logger, &opt).expect("can create config"); + let prometheus_registry = Arc::new(Registry::new()); + let metrics_registry = Arc::new(MetricsRegistry::new( + logger.clone(), + prometheus_registry.clone(), + )); + + let ethereum_networks = create_ethereum_networks(logger, metrics_registry, &config) + .await + .expect("Correctly parse Ethereum network args"); + let mut network_names = ethereum_networks.networks.keys().collect::>(); + network_names.sort(); + + let traces = NodeCapabilities { + archive: false, + traces: true, + }; + let archive = NodeCapabilities { + archive: true, + traces: false, + }; + let has_mainnet_with_traces = ethereum_networks + .adapter_with_capabilities("mainnet".to_string(), &traces) + .is_ok(); + let has_goerli_with_archive = ethereum_networks + .adapter_with_capabilities("goerli".to_string(), &archive) + .is_ok(); + let has_mainnet_with_archive = ethereum_networks + .adapter_with_capabilities("mainnet".to_string(), &archive) + .is_ok(); + let has_goerli_with_traces = ethereum_networks + .adapter_with_capabilities("goerli".to_string(), &traces) + .is_ok(); + + assert_eq!(has_mainnet_with_traces, true); + assert_eq!(has_goerli_with_archive, true); + assert_eq!(has_mainnet_with_archive, false); + assert_eq!(has_goerli_with_traces, false); + + let goerli_capability = ethereum_networks + .networks + .get("goerli") + .unwrap() + .adapters + .iter() + .next() + .unwrap() + .capabilities; + let mainnet_capability = ethereum_networks + .networks + .get("mainnet") + .unwrap() + .adapters + .iter() + .next() + .unwrap() + .capabilities; + assert_eq!( + network_names, + vec![&"goerli".to_string(), &"mainnet".to_string()] + ); + assert_eq!(goerli_capability, archive); + assert_eq!(mainnet_capability, traces); + } +} diff --git a/node/src/config.rs b/node/src/config.rs new file mode 100644 index 0000000..067b2dc --- /dev/null +++ b/node/src/config.rs @@ -0,0 +1,1444 @@ +use graph::{ + anyhow::Error, + blockchain::BlockchainKind, + prelude::{ + anyhow::{anyhow, bail, Context, Result}, + info, + serde::{ + de::{self, value, SeqAccess, Visitor}, + Deserialize, Deserializer, Serialize, + }, + serde_json, Logger, NodeId, StoreError, + }, +}; +use graph_chain_ethereum::{self as ethereum, NodeCapabilities}; +use graph_store_postgres::{DeploymentPlacer, Shard as ShardName, PRIMARY_SHARD}; + +use http::{HeaderMap, Uri}; +use regex::Regex; +use std::fs::read_to_string; +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt, +}; +use url::Url; + +const ANY_NAME: &str = ".*"; +/// A regular expression that matches nothing +const NO_NAME: &str = ".^"; + +pub struct Opt { + pub postgres_url: Option, + pub config: Option, + // This is only used when we cosntruct a config purely from command + // line options. When using a configuration file, pool sizes must be + // set in the configuration file alone + pub store_connection_pool_size: u32, + pub postgres_secondary_hosts: Vec, + pub postgres_host_weights: Vec, + pub disable_block_ingestor: bool, + pub node_id: String, + pub ethereum_rpc: Vec, + pub ethereum_ws: Vec, + pub ethereum_ipc: Vec, + pub unsafe_config: bool, +} + +impl Default for Opt { + fn default() -> Self { + Opt { + postgres_url: None, + config: None, + store_connection_pool_size: 10, + postgres_secondary_hosts: vec![], + postgres_host_weights: vec![], + disable_block_ingestor: true, + node_id: "default".to_string(), + ethereum_rpc: vec![], + ethereum_ws: vec![], + ethereum_ipc: vec![], + unsafe_config: false, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Config { + #[serde(skip, default = "default_node_id")] + pub node: NodeId, + pub general: Option, + #[serde(rename = "store")] + pub stores: BTreeMap, + pub chains: ChainSection, + pub deployment: Deployment, +} + +fn validate_name(s: &str) -> Result<()> { + if s.is_empty() { + return Err(anyhow!("names must not be empty")); + } + if s.len() > 30 { + return Err(anyhow!( + "names can be at most 30 characters, but `{}` has {} characters", + s, + s.len() + )); + } + + if !s + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') + { + return Err(anyhow!( + "name `{}` is invalid: names can only contain lowercase alphanumeric characters or '-'", + s + )); + } + Ok(()) +} + +impl Config { + /// Check that the config is valid. + fn validate(&mut self) -> Result<()> { + if !self.stores.contains_key(PRIMARY_SHARD.as_str()) { + return Err(anyhow!("missing a primary store")); + } + if self.stores.len() > 1 && ethereum::ENV_VARS.cleanup_blocks { + // See 8b6ad0c64e244023ac20ced7897fe666 + return Err(anyhow!( + "GRAPH_ETHEREUM_CLEANUP_BLOCKS can not be used with a sharded store" + )); + } + for (key, shard) in self.stores.iter_mut() { + shard.validate(&key)?; + } + self.deployment.validate()?; + + // Check that deployment rules only reference existing stores and chains + for (i, rule) in self.deployment.rules.iter().enumerate() { + for shard in &rule.shards { + if !self.stores.contains_key(shard) { + return Err(anyhow!("unknown shard {} in deployment rule {}", shard, i)); + } + } + if let Some(networks) = &rule.pred.network { + for network in networks.to_vec() { + if !self.chains.chains.contains_key(&network) { + return Err(anyhow!( + "unknown network {} in deployment rule {}", + network, + i + )); + } + } + } + } + + // Check that chains only reference existing stores + for (name, chain) in &self.chains.chains { + if !self.stores.contains_key(&chain.shard) { + return Err(anyhow!("unknown shard {} in chain {}", chain.shard, name)); + } + } + + self.chains.validate()?; + + Ok(()) + } + + /// Load a configuration file if `opt.config` is set. If not, generate + /// a config from the command line arguments in `opt` + pub fn load(logger: &Logger, opt: &Opt) -> Result { + if let Some(config) = &opt.config { + Self::from_file(logger, config, &opt.node_id) + } else { + info!( + logger, + "Generating configuration from command line arguments" + ); + Self::from_opt(opt) + } + } + + pub fn from_file(logger: &Logger, path: &str, node: &str) -> Result { + info!(logger, "Reading configuration file `{}`", path); + Self::from_str(&read_to_string(path)?, node) + } + + pub fn from_str(config: &str, node: &str) -> Result { + let mut config: Config = toml::from_str(&config)?; + config.node = + NodeId::new(node.clone()).map_err(|()| anyhow!("invalid node id {}", node))?; + config.validate()?; + Ok(config) + } + + fn from_opt(opt: &Opt) -> Result { + let deployment = Deployment::from_opt(opt); + let mut stores = BTreeMap::new(); + let chains = ChainSection::from_opt(opt)?; + let node = NodeId::new(opt.node_id.to_string()) + .map_err(|()| anyhow!("invalid node id {}", opt.node_id))?; + stores.insert(PRIMARY_SHARD.to_string(), Shard::from_opt(true, opt)?); + Ok(Config { + node, + general: None, + stores, + chains, + deployment, + }) + } + + /// Generate a JSON representation of the config. + pub fn to_json(&self) -> Result { + // It would be nice to produce a TOML representation, but that runs + // into this error: https://github.com/alexcrichton/toml-rs/issues/142 + // and fixing it as described in the issue didn't fix it. Since serializing + // this data isn't crucial and only needed for debugging, we'll + // just stick with JSON + Ok(serde_json::to_string_pretty(&self)?) + } + + pub fn primary_store(&self) -> &Shard { + self.stores + .get(PRIMARY_SHARD.as_str()) + .expect("a validated config has a primary store") + } + + pub fn query_only(&self, node: &NodeId) -> bool { + self.general + .as_ref() + .map(|g| match g.query.find(node.as_str()) { + None => false, + Some(m) => m.as_str() == node.as_str(), + }) + .unwrap_or(false) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct GeneralSection { + #[serde(with = "serde_regex", default = "no_name")] + query: Regex, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Shard { + pub connection: String, + #[serde(default = "one")] + pub weight: usize, + #[serde(default)] + pub pool_size: PoolSize, + #[serde(default = "PoolSize::five")] + pub fdw_pool_size: PoolSize, + #[serde(default)] + pub replicas: BTreeMap, +} + +impl Shard { + fn validate(&mut self, name: &str) -> Result<()> { + ShardName::new(name.to_string()).map_err(|e| anyhow!(e))?; + + self.connection = shellexpand::env(&self.connection)?.into_owned(); + + if matches!(self.pool_size, PoolSize::None) { + return Err(anyhow!("missing pool size definition for shard `{}`", name)); + } + + self.pool_size + .validate(name == PRIMARY_SHARD.as_str(), &self.connection)?; + for (name, replica) in self.replicas.iter_mut() { + validate_name(name).context("illegal replica name")?; + replica.validate(name == PRIMARY_SHARD.as_str(), &self.pool_size)?; + } + + let no_weight = + self.weight == 0 && self.replicas.values().all(|replica| replica.weight == 0); + if no_weight { + return Err(anyhow!( + "all weights for shard `{}` are 0; \ + remove explicit weights or set at least one of them to a value bigger than 0", + name + )); + } + Ok(()) + } + + fn from_opt(is_primary: bool, opt: &Opt) -> Result { + let postgres_url = opt + .postgres_url + .as_ref() + .expect("validation checked that postgres_url is set"); + let pool_size = PoolSize::Fixed(opt.store_connection_pool_size); + pool_size.validate(is_primary, &postgres_url)?; + let mut replicas = BTreeMap::new(); + for (i, host) in opt.postgres_secondary_hosts.iter().enumerate() { + let replica = Replica { + connection: replace_host(&postgres_url, &host), + weight: opt.postgres_host_weights.get(i + 1).cloned().unwrap_or(1), + pool_size: pool_size.clone(), + }; + replicas.insert(format!("replica{}", i + 1), replica); + } + Ok(Self { + connection: postgres_url.clone(), + weight: opt.postgres_host_weights.get(0).cloned().unwrap_or(1), + pool_size, + fdw_pool_size: PoolSize::five(), + replicas, + }) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(untagged)] +pub enum PoolSize { + None, + Fixed(u32), + Rule(Vec), +} + +impl Default for PoolSize { + fn default() -> Self { + Self::None + } +} + +impl PoolSize { + fn five() -> Self { + Self::Fixed(5) + } + + fn validate(&self, is_primary: bool, connection: &str) -> Result<()> { + use PoolSize::*; + + let pool_size = match self { + None => bail!("missing pool size for {}", connection), + Fixed(s) => *s, + Rule(rules) => rules.iter().map(|rule| rule.size).min().unwrap_or(0u32), + }; + + match pool_size { + 0 if is_primary => Err(anyhow!( + "the pool size for the primary shard must be at least 2" + )), + 0 => Ok(()), + 1 => Err(anyhow!( + "connection pool size must be at least 2, but is {} for {}", + pool_size, + connection + )), + _ => Ok(()), + } + } + + pub fn size_for(&self, node: &NodeId, name: &str) -> Result { + use PoolSize::*; + match self { + None => unreachable!("validation ensures we have a pool size"), + Fixed(s) => Ok(*s), + Rule(rules) => rules + .iter() + .find(|rule| rule.matches(node.as_str())) + .map(|rule| rule.size) + .ok_or_else(|| { + anyhow!( + "no rule matches node id `{}` for the pool of shard {}", + node.as_str(), + name + ) + }), + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct PoolSizeRule { + #[serde(with = "serde_regex", default = "any_name")] + node: Regex, + size: u32, +} + +impl PoolSizeRule { + fn matches(&self, name: &str) -> bool { + match self.node.find(name) { + None => false, + Some(m) => m.as_str() == name, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Replica { + pub connection: String, + #[serde(default = "one")] + pub weight: usize, + #[serde(default)] + pub pool_size: PoolSize, +} + +impl Replica { + fn validate(&mut self, is_primary: bool, pool_size: &PoolSize) -> Result<()> { + self.connection = shellexpand::env(&self.connection)?.into_owned(); + if matches!(self.pool_size, PoolSize::None) { + self.pool_size = pool_size.clone(); + } + + self.pool_size.validate(is_primary, &self.connection)?; + Ok(()) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct ChainSection { + pub ingestor: String, + #[serde(flatten)] + pub chains: BTreeMap, +} + +impl ChainSection { + fn validate(&mut self) -> Result<()> { + NodeId::new(&self.ingestor) + .map_err(|()| anyhow!("invalid node id for ingestor {}", &self.ingestor))?; + for (_, chain) in self.chains.iter_mut() { + chain.validate()? + } + Ok(()) + } + + fn from_opt(opt: &Opt) -> Result { + // If we are not the block ingestor, set the node name + // to something that is definitely not our node_id + let ingestor = if opt.disable_block_ingestor { + format!("{} is not ingesting", opt.node_id) + } else { + opt.node_id.clone() + }; + let mut chains = BTreeMap::new(); + Self::parse_networks(&mut chains, Transport::Rpc, &opt.ethereum_rpc)?; + Self::parse_networks(&mut chains, Transport::Ws, &opt.ethereum_ws)?; + Self::parse_networks(&mut chains, Transport::Ipc, &opt.ethereum_ipc)?; + Ok(Self { ingestor, chains }) + } + + fn parse_networks( + chains: &mut BTreeMap, + transport: Transport, + args: &Vec, + ) -> Result<()> { + for (nr, arg) in args.iter().enumerate() { + if arg.starts_with("wss://") + || arg.starts_with("http://") + || arg.starts_with("https://") + { + return Err(anyhow!( + "Is your Ethereum node string missing a network name? \ + Try 'mainnet:' + the Ethereum node URL." + )); + } else { + // Parse string (format is "NETWORK_NAME:NETWORK_CAPABILITIES:URL" OR + // "NETWORK_NAME::URL" which will default to NETWORK_CAPABILITIES="archive,traces") + let colon = arg.find(':').ok_or_else(|| { + return anyhow!( + "A network name must be provided alongside the \ + Ethereum node location. Try e.g. 'mainnet:URL'." + ); + })?; + + let (name, rest_with_delim) = arg.split_at(colon); + let rest = &rest_with_delim[1..]; + if name.is_empty() { + return Err(anyhow!("Ethereum network name cannot be an empty string")); + } + if rest.is_empty() { + return Err(anyhow!("Ethereum node URL cannot be an empty string")); + } + + let colon = rest.find(':').ok_or_else(|| { + return anyhow!( + "A network name must be provided alongside the \ + Ethereum node location. Try e.g. 'mainnet:URL'." + ); + })?; + + let (features, url_str) = rest.split_at(colon); + let (url, features) = if vec!["http", "https", "ws", "wss"].contains(&features) { + (rest, DEFAULT_PROVIDER_FEATURES.to_vec()) + } else { + (&url_str[1..], features.split(',').collect()) + }; + let features = features.into_iter().map(|s| s.to_string()).collect(); + let provider = Provider { + label: format!("{}-{}-{}", name, transport, nr), + details: ProviderDetails::Web3(Web3Provider { + transport, + url: url.to_string(), + features, + headers: Default::default(), + rules: Vec::new(), + }), + }; + let entry = chains.entry(name.to_string()).or_insert_with(|| Chain { + shard: PRIMARY_SHARD.to_string(), + protocol: BlockchainKind::Ethereum, + providers: vec![], + }); + entry.providers.push(provider); + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +pub struct Chain { + pub shard: String, + #[serde(default = "default_blockchain_kind")] + pub protocol: BlockchainKind, + #[serde(rename = "provider")] + pub providers: Vec, +} + +fn default_blockchain_kind() -> BlockchainKind { + BlockchainKind::Ethereum +} + +impl Chain { + fn validate(&mut self) -> Result<()> { + // `Config` validates that `self.shard` references a configured shard + + for provider in self.providers.iter_mut() { + provider.validate()? + } + Ok(()) + } +} + +fn deserialize_http_headers<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let kvs: BTreeMap = Deserialize::deserialize(deserializer)?; + Ok(btree_map_to_http_headers(kvs)) +} + +fn btree_map_to_http_headers(kvs: BTreeMap) -> HeaderMap { + let mut headers = HeaderMap::new(); + for (k, v) in kvs.into_iter() { + headers.insert( + k.parse::() + .expect(&format!("invalid HTTP header name: {}", k)), + v.parse::() + .expect(&format!("invalid HTTP header value: {}: {}", k, v)), + ); + } + headers +} + +#[derive(Clone, Debug, Serialize, PartialEq)] +pub struct Provider { + pub label: String, + pub details: ProviderDetails, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum ProviderDetails { + Firehose(FirehoseProvider), + Web3(Web3Provider), + Substreams(FirehoseProvider), +} + +const FIREHOSE_FILTER_FEATURE: &str = "filters"; +const FIREHOSE_COMPRESSION_FEATURE: &str = "compression"; +const FIREHOSE_PROVIDER_FEATURES: [&str; 2] = + [FIREHOSE_FILTER_FEATURE, FIREHOSE_COMPRESSION_FEATURE]; + +fn ten() -> u16 { + 10 +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub struct FirehoseProvider { + pub url: String, + pub token: Option, + #[serde(default = "ten")] + pub conn_pool_size: u16, + #[serde(default)] + pub features: BTreeSet, +} + +impl FirehoseProvider { + pub fn filters_enabled(&self) -> bool { + self.features.contains(FIREHOSE_FILTER_FEATURE) + } + pub fn compression_enabled(&self) -> bool { + self.features.contains(FIREHOSE_COMPRESSION_FEATURE) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct Web3Rule { + #[serde(with = "serde_regex")] + name: Regex, + limit: usize, +} + +impl PartialEq for Web3Rule { + fn eq(&self, other: &Self) -> bool { + self.name.to_string() == other.name.to_string() && self.limit == other.limit + } +} + +impl Web3Rule { + fn limit_for(&self, node: &NodeId) -> Option { + match self.name.find(node.as_str()) { + Some(m) if m.as_str() == node.as_str() => Some(self.limit), + _ => None, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub struct Web3Provider { + #[serde(default)] + pub transport: Transport, + pub url: String, + pub features: BTreeSet, + + // TODO: This should be serialized. + #[serde( + skip_serializing, + default, + deserialize_with = "deserialize_http_headers" + )] + pub headers: HeaderMap, + + #[serde(default, rename = "match")] + rules: Vec, +} + +impl Web3Provider { + pub fn node_capabilities(&self) -> NodeCapabilities { + NodeCapabilities { + archive: self.features.contains("archive"), + traces: self.features.contains("traces"), + } + } + + pub fn limit_for(&self, node: &NodeId) -> usize { + self.rules + .iter() + .find_map(|l| l.limit_for(node)) + .unwrap_or(usize::MAX) + } +} + +const PROVIDER_FEATURES: [&str; 3] = ["traces", "archive", "no_eip1898"]; +const DEFAULT_PROVIDER_FEATURES: [&str; 2] = ["traces", "archive"]; + +impl Provider { + fn validate(&mut self) -> Result<()> { + validate_name(&self.label).context("illegal provider name")?; + + match self.details { + ProviderDetails::Firehose(ref mut firehose) + | ProviderDetails::Substreams(ref mut firehose) => { + firehose.url = shellexpand::env(&firehose.url)?.into_owned(); + + // A Firehose url must be a valid Uri since gRPC library we use (Tonic) + // works with Uri. + let label = &self.label; + firehose.url.parse::().map_err(|e| { + anyhow!( + "the url `{}` for firehose provider {} is not a legal URI: {}", + firehose.url, + label, + e + ) + })?; + + if let Some(token) = &firehose.token { + firehose.token = Some(shellexpand::env(token)?.into_owned()); + } + + if firehose + .features + .iter() + .any(|feature| !FIREHOSE_PROVIDER_FEATURES.contains(&feature.as_str())) + { + return Err(anyhow!( + "supported firehose endpoint filters are: {:?}", + FIREHOSE_PROVIDER_FEATURES + )); + } + } + + ProviderDetails::Web3(ref mut web3) => { + for feature in &web3.features { + if !PROVIDER_FEATURES.contains(&feature.as_str()) { + return Err(anyhow!( + "illegal feature `{}` for provider {}. Features must be one of {}", + feature, + self.label, + PROVIDER_FEATURES.join(", ") + )); + } + } + + web3.url = shellexpand::env(&web3.url)?.into_owned(); + + let label = &self.label; + Url::parse(&web3.url).map_err(|e| { + anyhow!( + "the url `{}` for provider {} is not a legal URL: {}", + web3.url, + label, + e + ) + })?; + } + } + + Ok(()) + } +} + +impl<'de> Deserialize<'de> for Provider { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct ProviderVisitor; + + impl<'de> serde::de::Visitor<'de> for ProviderVisitor { + type Value = Provider; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("struct Provider") + } + + fn visit_map(self, mut map: V) -> Result + where + V: serde::de::MapAccess<'de>, + { + let mut label = None; + let mut details = None; + + let mut url = None; + let mut transport = None; + let mut features = None; + let mut headers = None; + let mut nodes = Vec::new(); + + while let Some(key) = map.next_key()? { + match key { + ProviderField::Label => { + if label.is_some() { + return Err(serde::de::Error::duplicate_field("label")); + } + label = Some(map.next_value()?); + } + ProviderField::Details => { + if details.is_some() { + return Err(serde::de::Error::duplicate_field("details")); + } + details = Some(map.next_value()?); + } + ProviderField::Url => { + if url.is_some() { + return Err(serde::de::Error::duplicate_field("url")); + } + url = Some(map.next_value()?); + } + ProviderField::Transport => { + if transport.is_some() { + return Err(serde::de::Error::duplicate_field("transport")); + } + transport = Some(map.next_value()?); + } + ProviderField::Features => { + if features.is_some() { + return Err(serde::de::Error::duplicate_field("features")); + } + features = Some(map.next_value()?); + } + ProviderField::Headers => { + if headers.is_some() { + return Err(serde::de::Error::duplicate_field("headers")); + } + + let raw_headers: BTreeMap = map.next_value()?; + headers = Some(btree_map_to_http_headers(raw_headers)); + } + ProviderField::Match => { + nodes = map.next_value()?; + } + } + } + + let label = label.ok_or_else(|| serde::de::Error::missing_field("label"))?; + let details = match details { + Some(v) => { + if url.is_some() + || transport.is_some() + || features.is_some() + || headers.is_some() + { + return Err(serde::de::Error::custom("when `details` field is provided, deprecated `url`, `transport`, `features` and `headers` cannot be specified")); + } + + v + } + None => ProviderDetails::Web3(Web3Provider { + url: url.ok_or_else(|| serde::de::Error::missing_field("url"))?, + transport: transport.unwrap_or(Transport::Rpc), + features: features + .ok_or_else(|| serde::de::Error::missing_field("features"))?, + headers: headers.unwrap_or_else(|| HeaderMap::new()), + rules: nodes, + }), + }; + + Ok(Provider { label, details }) + } + } + + const FIELDS: &'static [&'static str] = &[ + "label", + "details", + "transport", + "url", + "features", + "headers", + ]; + deserializer.deserialize_struct("Provider", FIELDS, ProviderVisitor) + } +} + +#[derive(Deserialize)] +#[serde(field_identifier, rename_all = "lowercase")] +enum ProviderField { + Label, + Details, + Match, + + // Deprecated fields + Url, + Transport, + Features, + Headers, +} + +#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq)] +pub enum Transport { + #[serde(rename = "rpc")] + Rpc, + #[serde(rename = "ws")] + Ws, + #[serde(rename = "ipc")] + Ipc, +} + +impl Default for Transport { + fn default() -> Self { + Self::Rpc + } +} + +impl std::fmt::Display for Transport { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use Transport::*; + + match self { + Rpc => write!(f, "rpc"), + Ws => write!(f, "ws"), + Ipc => write!(f, "ipc"), + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Deployment { + #[serde(rename = "rule")] + rules: Vec, +} + +impl Deployment { + fn validate(&self) -> Result<()> { + if self.rules.is_empty() { + return Err(anyhow!( + "there must be at least one deployment rule".to_string() + )); + } + let mut default_rule = false; + for rule in &self.rules { + rule.validate()?; + if default_rule { + return Err(anyhow!("rules after a default rule are useless")); + } + default_rule = rule.is_default(); + } + if !default_rule { + return Err(anyhow!( + "the rules do not contain a default rule that matches everything" + )); + } + Ok(()) + } + + fn from_opt(_: &Opt) -> Self { + Self { rules: vec![] } + } +} + +impl DeploymentPlacer for Deployment { + fn place( + &self, + name: &str, + network: &str, + ) -> Result, Vec)>, String> { + // Errors here are really programming errors. We should have validated + // everything already so that the various conversions can't fail. We + // still return errors so that they bubble up to the deployment request + // rather than crashing the node and burying the crash in the logs + let placement = match self.rules.iter().find(|rule| rule.matches(name, network)) { + Some(rule) => { + let shards = rule.shard_names().map_err(|e| e.to_string())?; + let indexers: Vec<_> = rule + .indexers + .iter() + .map(|idx| { + NodeId::new(idx.clone()) + .map_err(|()| format!("{} is not a valid node name", idx)) + }) + .collect::, _>>()?; + Some((shards, indexers)) + } + None => None, + }; + Ok(placement) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct Rule { + #[serde(rename = "match", default)] + pred: Predicate, + // For backwards compatibility, we also accept 'shard' for the shards + #[serde( + alias = "shard", + default = "primary_store", + deserialize_with = "string_or_vec" + )] + shards: Vec, + indexers: Vec, +} + +impl Rule { + fn is_default(&self) -> bool { + self.pred.matches_anything() + } + + fn matches(&self, name: &str, network: &str) -> bool { + self.pred.matches(name, network) + } + + fn shard_names(&self) -> Result, StoreError> { + self.shards + .iter() + .cloned() + .map(ShardName::new) + .collect::>() + } + + fn validate(&self) -> Result<()> { + if self.indexers.is_empty() { + return Err(anyhow!("useless rule without indexers")); + } + for indexer in &self.indexers { + NodeId::new(indexer).map_err(|()| anyhow!("invalid node id {}", &indexer))?; + } + self.shard_names().map_err(Error::from)?; + Ok(()) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct Predicate { + #[serde(with = "serde_regex", default = "any_name")] + name: Regex, + network: Option, +} + +impl Predicate { + fn matches_anything(&self) -> bool { + self.name.as_str() == ANY_NAME && self.network.is_none() + } + + pub fn matches(&self, name: &str, network: &str) -> bool { + if let Some(n) = &self.network { + if !n.matches(network) { + return false; + } + } + + match self.name.find(name) { + None => false, + Some(m) => m.as_str() == name, + } + } +} + +impl Default for Predicate { + fn default() -> Self { + Predicate { + name: any_name(), + network: None, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(untagged)] +enum NetworkPredicate { + Single(String), + Many(Vec), +} + +impl NetworkPredicate { + fn matches(&self, network: &str) -> bool { + use NetworkPredicate::*; + match self { + Single(n) => n == network, + Many(ns) => ns.iter().any(|n| n == network), + } + } + + fn to_vec(&self) -> Vec { + use NetworkPredicate::*; + match self { + Single(n) => vec![n.clone()], + Many(ns) => ns.clone(), + } + } +} + +/// Replace the host portion of `url` and return a new URL with `host` +/// as the host portion +/// +/// Panics if `url` is not a valid URL (which won't happen in our case since +/// we would have paniced before getting here as `url` is the connection for +/// the primary Postgres instance) +fn replace_host(url: &str, host: &str) -> String { + let mut url = match Url::parse(url) { + Ok(url) => url, + Err(_) => panic!("Invalid Postgres URL {}", url), + }; + if let Err(e) = url.set_host(Some(host)) { + panic!("Invalid Postgres url {}: {}", url, e.to_string()); + } + String::from(url) +} + +// Various default functions for deserialization +fn any_name() -> Regex { + Regex::new(ANY_NAME).unwrap() +} + +fn no_name() -> Regex { + Regex::new(NO_NAME).unwrap() +} + +fn primary_store() -> Vec { + vec![PRIMARY_SHARD.to_string()] +} + +fn one() -> usize { + 1 +} + +fn default_node_id() -> NodeId { + NodeId::new("default").unwrap() +} + +// From https://github.com/serde-rs/serde/issues/889#issuecomment-295988865 +fn string_or_vec<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + struct StringOrVec; + + impl<'de> Visitor<'de> for StringOrVec { + type Value = Vec; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("string or list of strings") + } + + fn visit_str(self, s: &str) -> Result + where + E: de::Error, + { + Ok(vec![s.to_owned()]) + } + + fn visit_seq(self, seq: S) -> Result + where + S: SeqAccess<'de>, + { + Deserialize::deserialize(value::SeqAccessDeserializer::new(seq)) + } + } + + deserializer.deserialize_any(StringOrVec) +} + +#[cfg(test)] +mod tests { + + use super::{ + Chain, Config, FirehoseProvider, Provider, ProviderDetails, Transport, Web3Provider, + }; + use graph::blockchain::BlockchainKind; + use graph::prelude::NodeId; + use http::{HeaderMap, HeaderValue}; + use std::collections::BTreeSet; + use std::fs::read_to_string; + use std::path::{Path, PathBuf}; + + #[test] + fn it_works_on_standard_config() { + let content = read_resource_as_string("full_config.toml"); + let actual: Config = toml::from_str(&content).unwrap(); + + // We do basic checks because writing the full equality method is really too long + + assert_eq!( + "query_node_.*".to_string(), + actual.general.unwrap().query.to_string() + ); + assert_eq!(4, actual.chains.chains.len()); + assert_eq!(2, actual.stores.len()); + assert_eq!(3, actual.deployment.rules.len()); + } + + #[test] + fn it_works_on_chain_without_protocol() { + let actual = toml::from_str( + r#" + shard = "primary" + provider = [] + "#, + ) + .unwrap(); + + assert_eq!( + Chain { + shard: "primary".to_string(), + protocol: BlockchainKind::Ethereum, + providers: vec![], + }, + actual + ); + } + + #[test] + fn it_works_on_chain_with_protocol() { + let actual = toml::from_str( + r#" + shard = "primary" + protocol = "near" + provider = [] + "#, + ) + .unwrap(); + + assert_eq!( + Chain { + shard: "primary".to_string(), + protocol: BlockchainKind::Near, + providers: vec![], + }, + actual + ); + } + + #[test] + fn it_works_on_deprecated_provider_from_toml() { + let actual = toml::from_str( + r#" + transport = "rpc" + label = "peering" + url = "http://localhost:8545" + features = [] + "#, + ) + .unwrap(); + + assert_eq!( + Provider { + label: "peering".to_owned(), + details: ProviderDetails::Web3(Web3Provider { + transport: Transport::Rpc, + url: "http://localhost:8545".to_owned(), + features: BTreeSet::new(), + headers: HeaderMap::new(), + rules: Vec::new(), + }), + }, + actual + ); + } + + #[test] + fn it_works_on_deprecated_provider_without_transport_from_toml() { + let actual = toml::from_str( + r#" + label = "peering" + url = "http://localhost:8545" + features = [] + "#, + ) + .unwrap(); + + assert_eq!( + Provider { + label: "peering".to_owned(), + details: ProviderDetails::Web3(Web3Provider { + transport: Transport::Rpc, + url: "http://localhost:8545".to_owned(), + features: BTreeSet::new(), + headers: HeaderMap::new(), + rules: Vec::new(), + }), + }, + actual + ); + } + + #[test] + fn it_errors_on_deprecated_provider_missing_url_from_toml() { + let actual = toml::from_str::( + r#" + transport = "rpc" + label = "peering" + features = [] + "#, + ); + + assert_eq!(true, actual.is_err()); + assert_eq!( + actual.unwrap_err().to_string(), + "missing field `url` at line 1 column 1" + ); + } + + #[test] + fn it_errors_on_deprecated_provider_missing_features_from_toml() { + let actual = toml::from_str::( + r#" + transport = "rpc" + url = "http://localhost:8545" + label = "peering" + "#, + ); + + assert_eq!(true, actual.is_err()); + assert_eq!( + actual.unwrap_err().to_string(), + "missing field `features` at line 1 column 1" + ); + } + + #[test] + fn it_works_on_new_web3_provider_from_toml() { + let actual = toml::from_str( + r#" + label = "peering" + details = { type = "web3", transport = "ipc", url = "http://localhost:8545", features = ["archive"], headers = { x-test = "value" } } + "#, + ) + .unwrap(); + + let mut features = BTreeSet::new(); + features.insert("archive".to_string()); + + let mut headers = HeaderMap::new(); + headers.insert("x-test", HeaderValue::from_static("value")); + + assert_eq!( + Provider { + label: "peering".to_owned(), + details: ProviderDetails::Web3(Web3Provider { + transport: Transport::Ipc, + url: "http://localhost:8545".to_owned(), + features, + headers, + rules: Vec::new(), + }), + }, + actual + ); + } + + #[test] + fn it_works_on_new_web3_provider_without_transport_from_toml() { + let actual = toml::from_str( + r#" + label = "peering" + details = { type = "web3", url = "http://localhost:8545", features = [] } + "#, + ) + .unwrap(); + + assert_eq!( + Provider { + label: "peering".to_owned(), + details: ProviderDetails::Web3(Web3Provider { + transport: Transport::Rpc, + url: "http://localhost:8545".to_owned(), + features: BTreeSet::new(), + headers: HeaderMap::new(), + rules: Vec::new(), + }), + }, + actual + ); + } + + #[test] + fn it_errors_on_new_provider_with_deprecated_fields_from_toml() { + let actual = toml::from_str::( + r#" + label = "peering" + url = "http://localhost:8545" + details = { type = "web3", url = "http://localhost:8545", features = [] } + "#, + ); + + assert_eq!(true, actual.is_err()); + assert_eq!(actual.unwrap_err().to_string(), "when `details` field is provided, deprecated `url`, `transport`, `features` and `headers` cannot be specified at line 1 column 1"); + } + + #[test] + fn it_works_on_new_firehose_provider_from_toml() { + let actual = toml::from_str( + r#" + label = "firehose" + details = { type = "firehose", url = "http://localhost:9000", features = [] } + "#, + ) + .unwrap(); + + assert_eq!( + Provider { + label: "firehose".to_owned(), + details: ProviderDetails::Firehose(FirehoseProvider { + url: "http://localhost:9000".to_owned(), + token: None, + features: BTreeSet::new(), + conn_pool_size: 10, + }), + }, + actual + ); + } + + #[test] + fn it_works_on_substreams_provider_from_toml() { + let actual = toml::from_str( + r#" + label = "bananas" + details = { type = "substreams", url = "http://localhost:9000", features = [] } + "#, + ) + .unwrap(); + + assert_eq!( + Provider { + label: "bananas".to_owned(), + details: ProviderDetails::Substreams(FirehoseProvider { + url: "http://localhost:9000".to_owned(), + token: None, + features: BTreeSet::new(), + conn_pool_size: 10, + }), + }, + actual + ); + } + #[test] + fn it_works_on_new_firehose_provider_from_toml_no_features() { + let actual = toml::from_str( + r#" + label = "firehose" + details = { type = "firehose", url = "http://localhost:9000" } + "#, + ) + .unwrap(); + + assert_eq!( + Provider { + label: "firehose".to_owned(), + details: ProviderDetails::Firehose(FirehoseProvider { + url: "http://localhost:9000".to_owned(), + token: None, + features: BTreeSet::new(), + conn_pool_size: 10, + }), + }, + actual + ); + } + + #[test] + fn it_works_on_new_firehose_provider_from_toml_unsupported_features() { + let actual = toml::from_str::( + r#" + label = "firehose" + details = { type = "firehose", url = "http://localhost:9000", features = ["bananas"]} + "#, + ).unwrap().validate(); + assert_eq!(true, actual.is_err(), "{:?}", actual); + + if let Err(error) = actual { + assert_eq!( + true, + error + .to_string() + .starts_with("supported firehose endpoint filters are:") + ) + } + } + + #[test] + fn it_parses_web3_provider_rules() { + fn limit_for(node: &str) -> usize { + let prov = toml::from_str::( + r#" + label = "something" + url = "http://example.com" + features = [] + match = [ { name = "some_node_.*", limit = 10 }, + { name = "other_node_.*", limit = 0 } ] + "#, + ) + .unwrap(); + + prov.limit_for(&NodeId::new(node.to_string()).unwrap()) + } + + assert_eq!(10, limit_for("some_node_0")); + assert_eq!(0, limit_for("other_node_0")); + assert_eq!(usize::MAX, limit_for("default")); + } + + fn read_resource_as_string>(path: P) -> String { + let mut d = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + d.push("resources/tests"); + d.push(path); + + read_to_string(&d).expect(&format!("resource {:?} not found", &d)) + } +} diff --git a/node/src/lib.rs b/node/src/lib.rs new file mode 100644 index 0000000..2d4f8ca --- /dev/null +++ b/node/src/lib.rs @@ -0,0 +1,21 @@ +use std::sync::Arc; + +use graph::prometheus::Registry; +use graph_core::MetricsRegistry; + +#[macro_use] +extern crate diesel; + +pub mod chain; +pub mod config; +pub mod opt; +pub mod store_builder; + +pub mod manager; + +pub struct MetricsContext { + pub prometheus: Arc, + pub registry: Arc, + pub prometheus_host: Option, + pub job_name: Option, +} diff --git a/node/src/main.rs b/node/src/main.rs new file mode 100644 index 0000000..326d5e8 --- /dev/null +++ b/node/src/main.rs @@ -0,0 +1,952 @@ +use clap::Parser as _; +use ethereum::chain::{EthereumAdapterSelector, EthereumStreamBuilder}; +use ethereum::{ + BlockIngestor as EthereumBlockIngestor, EthereumAdapterTrait, EthereumNetworks, RuntimeAdapter, +}; +use git_testament::{git_testament, render_testament}; +use graph::blockchain::firehose_block_ingestor::FirehoseBlockIngestor; +use graph::blockchain::{Block as BlockchainBlock, Blockchain, BlockchainKind, BlockchainMap}; +use graph::components::store::BlockStore; +use graph::data::graphql::effort::LoadManager; +use graph::env::EnvVars; +use graph::firehose::{FirehoseEndpoints, FirehoseNetworks}; +use graph::log::logger; +use graph::prelude::{IndexNodeServer as _, *}; +use graph::prometheus::Registry; +use graph::url::Url; +use graph_chain_arweave::{self as arweave, Block as ArweaveBlock}; +use graph_chain_cosmos::{self as cosmos, Block as CosmosFirehoseBlock}; +use graph_chain_ethereum as ethereum; +use graph_chain_near::{self as near, HeaderOnlyBlock as NearFirehoseHeaderOnlyBlock}; +use graph_chain_substreams as substreams; +use graph_core::polling_monitor::ipfs_service::IpfsService; +use graph_core::{ + LinkResolver, MetricsRegistry, SubgraphAssignmentProvider as IpfsSubgraphAssignmentProvider, + SubgraphInstanceManager, SubgraphRegistrar as IpfsSubgraphRegistrar, +}; +use graph_graphql::prelude::GraphQlRunner; +use graph_node::chain::{ + connect_ethereum_networks, connect_firehose_networks, create_ethereum_networks, + create_firehose_networks, create_ipfs_clients, create_substreams_networks, +}; +use graph_node::config::Config; +use graph_node::opt; +use graph_node::store_builder::StoreBuilder; +use graph_server_http::GraphQLServer as GraphQLQueryServer; +use graph_server_index_node::IndexNodeServer; +use graph_server_json_rpc::JsonRpcServer; +use graph_server_metrics::PrometheusMetricsServer; +use graph_server_websocket::SubscriptionServer as GraphQLSubscriptionServer; +use graph_store_postgres::{register_jobs as register_store_jobs, ChainHeadUpdateListener, Store}; +use near::NearStreamBuilder; +use std::collections::BTreeMap; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::sync::atomic; +use std::time::Duration; +use std::{collections::HashMap, env}; +use tokio::sync::mpsc; + +git_testament!(TESTAMENT); + +fn read_expensive_queries( + logger: &Logger, + expensive_queries_filename: String, +) -> Result>, std::io::Error> { + // A file with a list of expensive queries, one query per line + // Attempts to run these queries will return a + // QueryExecutionError::TooExpensive to clients + let path = Path::new(&expensive_queries_filename); + let mut queries = Vec::new(); + if path.exists() { + info!( + logger, + "Reading expensive queries file: {}", expensive_queries_filename + ); + let file = std::fs::File::open(path)?; + let reader = BufReader::new(file); + for line in reader.lines() { + let line = line?; + let query = graphql_parser::parse_query(&line) + .map_err(|e| { + let msg = format!( + "invalid GraphQL query in {}: {}\n{}", + expensive_queries_filename, + e.to_string(), + line + ); + std::io::Error::new(std::io::ErrorKind::InvalidData, msg) + })? + .into_static(); + queries.push(Arc::new(query)); + } + } else { + warn!( + logger, + "Expensive queries file not set to a valid file: {}", expensive_queries_filename + ); + } + Ok(queries) +} + +#[tokio::main] +async fn main() { + env_logger::init(); + + let opt = opt::Opt::parse(); + + // Set up logger + let logger = logger(opt.debug); + + // Log version information + info!( + logger, + "Graph Node version: {}", + render_testament!(TESTAMENT) + ); + + if opt.unsafe_config { + warn!(logger, "allowing unsafe configurations"); + graph::env::UNSAFE_CONFIG.store(true, atomic::Ordering::SeqCst); + } + + if !graph_server_index_node::PoiProtection::from_env(&ENV_VARS).is_active() { + warn!( + logger, + "GRAPH_POI_ACCESS_TOKEN not set; might leak POIs to the public via GraphQL" + ); + } + + let config = match Config::load(&logger, &opt.clone().into()) { + Err(e) => { + eprintln!("configuration error: {}", e); + std::process::exit(1); + } + Ok(config) => config, + }; + if opt.check_config { + match config.to_json() { + Ok(txt) => println!("{}", txt), + Err(e) => eprintln!("error serializing config: {}", e), + } + eprintln!("Successfully validated configuration"); + std::process::exit(0); + } + + let node_id = NodeId::new(opt.node_id.clone()) + .expect("Node ID must be between 1 and 63 characters in length"); + let query_only = config.query_only(&node_id); + + // Obtain subgraph related command-line arguments + let subgraph = opt.subgraph.clone(); + + // Obtain ports to use for the GraphQL server(s) + let http_port = opt.http_port; + let ws_port = opt.ws_port; + + // Obtain JSON-RPC server port + let json_rpc_port = opt.admin_port; + + // Obtain index node server port + let index_node_port = opt.index_node_port; + + // Obtain metrics server port + let metrics_port = opt.metrics_port; + + // Obtain the fork base URL + let fork_base = match &opt.fork_base { + Some(url) => { + // Make sure the endpoint ends with a terminating slash. + let url = if !url.ends_with("/") { + let mut url = url.clone(); + url.push('/'); + Url::parse(&url) + } else { + Url::parse(url) + }; + + Some(url.expect("Failed to parse the fork base URL")) + } + None => { + warn!( + logger, + "No fork base URL specified, subgraph forking is disabled" + ); + None + } + }; + + info!(logger, "Starting up"); + + // Optionally, identify the Elasticsearch logging configuration + let elastic_config = opt + .elasticsearch_url + .clone() + .map(|endpoint| ElasticLoggingConfig { + endpoint: endpoint.clone(), + username: opt.elasticsearch_user.clone(), + password: opt.elasticsearch_password.clone(), + client: reqwest::Client::new(), + }); + + // Create a component and subgraph logger factory + let logger_factory = LoggerFactory::new(logger.clone(), elastic_config); + + // Try to create IPFS clients for each URL specified in `--ipfs` + let ipfs_clients: Vec<_> = create_ipfs_clients(&logger, &opt.ipfs); + let ipfs_client = ipfs_clients.first().cloned().expect("Missing IPFS client"); + let ipfs_service = IpfsService::new( + ipfs_client, + ENV_VARS.mappings.max_ipfs_file_bytes as u64, + ENV_VARS.mappings.ipfs_timeout, + ENV_VARS.mappings.max_ipfs_concurrent_requests, + ); + + // Convert the clients into a link resolver. Since we want to get past + // possible temporary DNS failures, make the resolver retry + let link_resolver = Arc::new(LinkResolver::new( + ipfs_clients, + Arc::new(EnvVars::default()), + )); + + // Set up Prometheus registry + let prometheus_registry = Arc::new(Registry::new()); + let metrics_registry = Arc::new(MetricsRegistry::new( + logger.clone(), + prometheus_registry.clone(), + )); + let mut metrics_server = + PrometheusMetricsServer::new(&logger_factory, prometheus_registry.clone()); + + // Ethereum clients; query nodes ignore all ethereum clients and never + // connect to them directly + let eth_networks = if query_only { + EthereumNetworks::new() + } else { + create_ethereum_networks(logger.clone(), metrics_registry.clone(), &config) + .await + .expect("Failed to parse Ethereum networks") + }; + + let mut firehose_networks_by_kind = if query_only { + BTreeMap::new() + } else { + create_firehose_networks(logger.clone(), &config) + }; + + let substreams_networks_by_kind = if query_only { + BTreeMap::new() + } else { + create_substreams_networks(logger.clone(), &config) + }; + + let graphql_metrics_registry = metrics_registry.clone(); + + let contention_logger = logger.clone(); + + // TODO: make option loadable from configuration TOML and environment: + let expensive_queries = + read_expensive_queries(&logger, opt.expensive_queries_filename).unwrap(); + + let store_builder = StoreBuilder::new( + &logger, + &node_id, + &config, + fork_base, + metrics_registry.cheap_clone(), + ) + .await; + + let launch_services = |logger: Logger| async move { + let subscription_manager = store_builder.subscription_manager(); + let chain_head_update_listener = store_builder.chain_head_update_listener(); + let primary_pool = store_builder.primary_pool(); + + // To support the ethereum block ingestor, ethereum networks are referenced both by the + // `blockchain_map` and `ethereum_chains`. Future chains should be referred to only in + // `blockchain_map`. + let mut blockchain_map = BlockchainMap::new(); + + let (arweave_networks, arweave_idents) = connect_firehose_networks::( + &logger, + firehose_networks_by_kind + .remove(&BlockchainKind::Arweave) + .unwrap_or_else(|| FirehoseNetworks::new()), + ) + .await; + + let (eth_networks, ethereum_idents) = + connect_ethereum_networks(&logger, eth_networks).await; + + let (near_networks, near_idents) = + connect_firehose_networks::( + &logger, + firehose_networks_by_kind + .remove(&BlockchainKind::Near) + .unwrap_or_else(|| FirehoseNetworks::new()), + ) + .await; + + let (cosmos_networks, cosmos_idents) = connect_firehose_networks::( + &logger, + firehose_networks_by_kind + .remove(&BlockchainKind::Cosmos) + .unwrap_or_else(|| FirehoseNetworks::new()), + ) + .await; + + let network_identifiers = ethereum_idents + .into_iter() + .chain(arweave_idents) + .chain(near_idents) + .chain(cosmos_idents) + .collect(); + + let network_store = store_builder.network_store(network_identifiers); + + let arweave_chains = arweave_networks_as_chains( + &mut blockchain_map, + &logger, + &arweave_networks, + network_store.as_ref(), + &logger_factory, + metrics_registry.clone(), + ); + + let ethereum_chains = ethereum_networks_as_chains( + &mut blockchain_map, + &logger, + node_id.clone(), + metrics_registry.clone(), + firehose_networks_by_kind.get(&BlockchainKind::Ethereum), + substreams_networks_by_kind.get(&BlockchainKind::Ethereum), + ð_networks, + network_store.as_ref(), + chain_head_update_listener, + &logger_factory, + metrics_registry.clone(), + ); + + let near_chains = near_networks_as_chains( + &mut blockchain_map, + &logger, + &near_networks, + network_store.as_ref(), + &logger_factory, + metrics_registry.clone(), + ); + + let cosmos_chains = cosmos_networks_as_chains( + &mut blockchain_map, + &logger, + &cosmos_networks, + network_store.as_ref(), + &logger_factory, + metrics_registry.clone(), + ); + + let blockchain_map = Arc::new(blockchain_map); + + let load_manager = Arc::new(LoadManager::new( + &logger, + expensive_queries, + metrics_registry.clone(), + )); + let graphql_runner = Arc::new(GraphQlRunner::new( + &logger, + network_store.clone(), + subscription_manager.clone(), + load_manager, + graphql_metrics_registry, + )); + let mut graphql_server = + GraphQLQueryServer::new(&logger_factory, graphql_runner.clone(), node_id.clone()); + let subscription_server = + GraphQLSubscriptionServer::new(&logger, graphql_runner.clone(), network_store.clone()); + + let mut index_node_server = IndexNodeServer::new( + &logger_factory, + blockchain_map.clone(), + graphql_runner.clone(), + network_store.clone(), + link_resolver.clone(), + ); + + if !opt.disable_block_ingestor { + if ethereum_chains.len() > 0 { + let block_polling_interval = Duration::from_millis(opt.ethereum_polling_interval); + + start_block_ingestor( + &logger, + &logger_factory, + block_polling_interval, + ethereum_chains, + ); + } + + start_firehose_block_ingestor::<_, ArweaveBlock>( + &logger, + &network_store, + arweave_chains, + ); + + start_firehose_block_ingestor::<_, NearFirehoseHeaderOnlyBlock>( + &logger, + &network_store, + near_chains, + ); + start_firehose_block_ingestor::<_, CosmosFirehoseBlock>( + &logger, + &network_store, + cosmos_chains, + ); + + // Start a task runner + let mut job_runner = graph::util::jobs::Runner::new(&logger); + register_store_jobs( + &mut job_runner, + network_store.clone(), + primary_pool, + metrics_registry.clone(), + ); + graph::spawn_blocking(job_runner.start()); + } + let static_filters = ENV_VARS.experimental_static_filters; + + let subgraph_instance_manager = SubgraphInstanceManager::new( + &logger_factory, + network_store.subgraph_store(), + blockchain_map.cheap_clone(), + metrics_registry.clone(), + link_resolver.clone(), + ipfs_service, + static_filters, + ); + + // Create IPFS-based subgraph provider + let subgraph_provider = IpfsSubgraphAssignmentProvider::new( + &logger_factory, + link_resolver.clone(), + subgraph_instance_manager, + ); + + // Check version switching mode environment variable + let version_switching_mode = ENV_VARS.subgraph_version_switching_mode; + + // Create named subgraph provider for resolving subgraph name->ID mappings + let subgraph_registrar = Arc::new(IpfsSubgraphRegistrar::new( + &logger_factory, + link_resolver, + Arc::new(subgraph_provider), + network_store.subgraph_store(), + subscription_manager, + blockchain_map, + node_id.clone(), + version_switching_mode, + )); + graph::spawn( + subgraph_registrar + .start() + .map_err(|e| panic!("failed to initialize subgraph provider {}", e)) + .compat(), + ); + + // Start admin JSON-RPC server. + let json_rpc_server = JsonRpcServer::serve( + json_rpc_port, + http_port, + ws_port, + subgraph_registrar.clone(), + node_id.clone(), + logger.clone(), + ) + .await + .expect("failed to start JSON-RPC admin server"); + + // Let the server run forever. + std::mem::forget(json_rpc_server); + + // Add the CLI subgraph with a REST request to the admin server. + if let Some(subgraph) = subgraph { + let (name, hash) = if subgraph.contains(':') { + let mut split = subgraph.split(':'); + (split.next().unwrap(), split.next().unwrap().to_owned()) + } else { + ("cli", subgraph) + }; + + let name = SubgraphName::new(name) + .expect("Subgraph name must contain only a-z, A-Z, 0-9, '-' and '_'"); + let subgraph_id = + DeploymentHash::new(hash).expect("Subgraph hash must be a valid IPFS hash"); + let debug_fork = opt + .debug_fork + .map(DeploymentHash::new) + .map(|h| h.expect("Debug fork hash must be a valid IPFS hash")); + let start_block = opt + .start_block + .map(|block| { + let mut split = block.split(":"); + ( + // BlockHash + split.next().unwrap().to_owned(), + // BlockNumber + split.next().unwrap().parse::().unwrap(), + ) + }) + .map(|(hash, number)| BlockPtr::try_from((hash.as_str(), number))) + .map(Result::unwrap); + + graph::spawn( + async move { + subgraph_registrar.create_subgraph(name.clone()).await?; + subgraph_registrar + .create_subgraph_version( + name, + subgraph_id, + node_id, + debug_fork, + start_block, + None, + ) + .await + } + .map_err(|e| panic!("Failed to deploy subgraph from `--subgraph` flag: {}", e)), + ); + } + + // Serve GraphQL queries over HTTP + graph::spawn( + graphql_server + .serve(http_port, ws_port) + .expect("Failed to start GraphQL query server") + .compat(), + ); + + // Serve GraphQL subscriptions over WebSockets + graph::spawn(subscription_server.serve(ws_port)); + + // Run the index node server + graph::spawn( + index_node_server + .serve(index_node_port) + .expect("Failed to start index node server") + .compat(), + ); + + graph::spawn( + metrics_server + .serve(metrics_port) + .expect("Failed to start metrics server") + .compat(), + ); + }; + + graph::spawn(launch_services(logger.clone())); + + // Periodically check for contention in the tokio threadpool. First spawn a + // task that simply responds to "ping" requests. Then spawn a separate + // thread to periodically ping it and check responsiveness. + let (ping_send, mut ping_receive) = mpsc::channel::>(1); + graph::spawn(async move { + while let Some(pong_send) = ping_receive.recv().await { + let _ = pong_send.clone().send(()); + } + panic!("ping sender dropped"); + }); + std::thread::spawn(move || loop { + std::thread::sleep(Duration::from_secs(1)); + let (pong_send, pong_receive) = crossbeam_channel::bounded(1); + if futures::executor::block_on(ping_send.clone().send(pong_send)).is_err() { + debug!(contention_logger, "Shutting down contention checker thread"); + break; + } + let mut timeout = Duration::from_millis(10); + while pong_receive.recv_timeout(timeout) + == Err(crossbeam_channel::RecvTimeoutError::Timeout) + { + debug!(contention_logger, "Possible contention in tokio threadpool"; + "timeout_ms" => timeout.as_millis(), + "code" => LogCode::TokioContention); + if timeout < Duration::from_secs(10) { + timeout *= 10; + } else if ENV_VARS.kill_if_unresponsive { + // The node is unresponsive, kill it in hopes it will be restarted. + crit!(contention_logger, "Node is unresponsive, killing process"); + std::process::abort() + } + } + }); + + futures::future::pending::<()>().await; +} + +/// Return the hashmap of Arweave chains and also add them to `blockchain_map`. +fn arweave_networks_as_chains( + blockchain_map: &mut BlockchainMap, + logger: &Logger, + firehose_networks: &FirehoseNetworks, + store: &Store, + logger_factory: &LoggerFactory, + metrics_registry: Arc, +) -> HashMap> { + let chains: Vec<_> = firehose_networks + .networks + .iter() + .filter_map(|(chain_id, endpoints)| { + store + .block_store() + .chain_store(chain_id) + .map(|chain_store| (chain_id, chain_store, endpoints)) + .or_else(|| { + error!( + logger, + "No store configured for Arweave chain {}; ignoring this chain", chain_id + ); + None + }) + }) + .map(|(chain_id, chain_store, endpoints)| { + ( + chain_id.clone(), + FirehoseChain { + chain: Arc::new(arweave::Chain::new( + logger_factory.clone(), + chain_id.clone(), + chain_store, + endpoints.clone(), + metrics_registry.clone(), + )), + firehose_endpoints: endpoints.clone(), + }, + ) + }) + .collect(); + + for (chain_id, firehose_chain) in chains.iter() { + blockchain_map.insert::(chain_id.clone(), firehose_chain.chain.clone()) + } + + HashMap::from_iter(chains) +} + +/// Return the hashmap of ethereum chains and also add them to `blockchain_map`. +fn ethereum_networks_as_chains( + blockchain_map: &mut BlockchainMap, + logger: &Logger, + node_id: NodeId, + registry: Arc, + firehose_networks: Option<&FirehoseNetworks>, + substreams_networks: Option<&FirehoseNetworks>, + eth_networks: &EthereumNetworks, + store: &Store, + chain_head_update_listener: Arc, + logger_factory: &LoggerFactory, + metrics_registry: Arc, +) -> HashMap> { + let chains: Vec<_> = eth_networks + .networks + .iter() + .filter_map(|(network_name, eth_adapters)| { + store + .block_store() + .chain_store(network_name) + .map(|chain_store| { + let is_ingestible = chain_store.is_ingestible(); + (network_name, eth_adapters, chain_store, is_ingestible) + }) + .or_else(|| { + error!( + logger, + "No store configured for Ethereum chain {}; ignoring this chain", + network_name + ); + None + }) + }) + .map(|(network_name, eth_adapters, chain_store, is_ingestible)| { + let firehose_endpoints = firehose_networks.and_then(|v| v.networks.get(network_name)); + + let adapter_selector = EthereumAdapterSelector::new( + logger_factory.clone(), + Arc::new(eth_adapters.clone()), + Arc::new( + firehose_endpoints + .map(|fe| fe.clone()) + .unwrap_or(FirehoseEndpoints::new()), + ), + registry.clone(), + chain_store.clone(), + ); + + let runtime_adapter = Arc::new(RuntimeAdapter { + eth_adapters: Arc::new(eth_adapters.clone()), + call_cache: chain_store.cheap_clone(), + }); + + let chain = ethereum::Chain::new( + logger_factory.clone(), + network_name.clone(), + node_id.clone(), + registry.clone(), + chain_store.cheap_clone(), + chain_store, + firehose_endpoints.map_or_else(|| FirehoseEndpoints::new(), |v| v.clone()), + eth_adapters.clone(), + chain_head_update_listener.clone(), + Arc::new(EthereumStreamBuilder {}), + Arc::new(adapter_selector), + runtime_adapter, + ethereum::ENV_VARS.reorg_threshold, + is_ingestible, + ); + (network_name.clone(), Arc::new(chain)) + }) + .collect(); + + for (network_name, chain) in chains.iter().cloned() { + blockchain_map.insert::(network_name, chain) + } + + if let Some(substreams_networks) = substreams_networks { + for (network_name, firehose_endpoints) in substreams_networks.networks.iter() { + let chain_store = blockchain_map + .get::(network_name.clone()) + .expect("any substreams endpoint needs an rpc or firehose chain defined") + .chain_store(); + + blockchain_map.insert::( + network_name.clone(), + Arc::new(substreams::Chain::new( + logger_factory.clone(), + firehose_endpoints.clone(), + metrics_registry.clone(), + chain_store, + Arc::new(substreams::BlockStreamBuilder::new()), + )), + ); + } + } + + HashMap::from_iter(chains) +} + +fn cosmos_networks_as_chains( + blockchain_map: &mut BlockchainMap, + logger: &Logger, + firehose_networks: &FirehoseNetworks, + store: &Store, + logger_factory: &LoggerFactory, + metrics_registry: Arc, +) -> HashMap> { + let chains: Vec<_> = firehose_networks + .networks + .iter() + .filter_map(|(network_name, firehose_endpoints)| { + store + .block_store() + .chain_store(network_name) + .map(|chain_store| (network_name, chain_store, firehose_endpoints)) + .or_else(|| { + error!( + logger, + "No store configured for Cosmos chain {}; ignoring this chain", + network_name + ); + None + }) + }) + .map(|(network_name, chain_store, firehose_endpoints)| { + ( + network_name.clone(), + FirehoseChain { + chain: Arc::new(cosmos::Chain::new( + logger_factory.clone(), + network_name.clone(), + chain_store, + firehose_endpoints.clone(), + metrics_registry.clone(), + )), + firehose_endpoints: firehose_endpoints.clone(), + }, + ) + }) + .collect(); + + for (network_name, firehose_chain) in chains.iter() { + blockchain_map.insert::(network_name.clone(), firehose_chain.chain.clone()) + } + + HashMap::from_iter(chains) +} + +/// Return the hashmap of NEAR chains and also add them to `blockchain_map`. +fn near_networks_as_chains( + blockchain_map: &mut BlockchainMap, + logger: &Logger, + firehose_networks: &FirehoseNetworks, + store: &Store, + logger_factory: &LoggerFactory, + metrics_registry: Arc, +) -> HashMap> { + let chains: Vec<_> = firehose_networks + .networks + .iter() + .filter_map(|(chain_id, endpoints)| { + store + .block_store() + .chain_store(chain_id) + .map(|chain_store| (chain_id, chain_store, endpoints)) + .or_else(|| { + error!( + logger, + "No store configured for NEAR chain {}; ignoring this chain", chain_id + ); + None + }) + }) + .map(|(chain_id, chain_store, endpoints)| { + ( + chain_id.clone(), + FirehoseChain { + chain: Arc::new(near::Chain::new( + logger_factory.clone(), + chain_id.clone(), + chain_store, + endpoints.clone(), + metrics_registry.clone(), + Arc::new(NearStreamBuilder {}), + )), + firehose_endpoints: endpoints.clone(), + }, + ) + }) + .collect(); + + for (chain_id, firehose_chain) in chains.iter() { + blockchain_map + .insert::(chain_id.clone(), firehose_chain.chain.clone()) + } + + HashMap::from_iter(chains) +} + +fn start_block_ingestor( + logger: &Logger, + logger_factory: &LoggerFactory, + block_polling_interval: Duration, + chains: HashMap>, +) { + info!( + logger, + "Starting block ingestors with {} chains [{}]", + chains.len(), + chains + .keys() + .map(|v| v.clone()) + .collect::>() + .join(", ") + ); + + // Create Ethereum block ingestors and spawn a thread to run each + chains + .iter() + .filter(|(network_name, chain)| { + if !chain.is_ingestible { + error!(logger, "Not starting block ingestor (chain is defective)"; "network_name" => &network_name); + } + chain.is_ingestible + }) + .for_each(|(network_name, chain)| { + info!( + logger, + "Starting block ingestor for network"; + "network_name" => &network_name + ); + + let eth_adapter = chain.cheapest_adapter(); + let logger = logger_factory + .component_logger( + "BlockIngestor", + Some(ComponentLoggerConfig { + elastic: Some(ElasticComponentLoggerConfig { + index: String::from("block-ingestor-logs"), + }), + }), + ) + .new(o!("provider" => eth_adapter.provider().to_string())); + + // The block ingestor must be configured to keep at least REORG_THRESHOLD ancestors, + // because the json-rpc BlockStream expects blocks after the reorg threshold to be + // present in the DB. + let block_ingestor = EthereumBlockIngestor::new( + logger, + ethereum::ENV_VARS.reorg_threshold, + eth_adapter, + chain.chain_store(), + block_polling_interval, + ) + .expect("failed to create Ethereum block ingestor"); + + // Run the Ethereum block ingestor in the background + graph::spawn(block_ingestor.into_polling_stream()); + }); +} + +#[derive(Clone)] +struct FirehoseChain { + chain: Arc, + firehose_endpoints: FirehoseEndpoints, +} + +fn start_firehose_block_ingestor( + logger: &Logger, + store: &Store, + chains: HashMap>, +) where + C: Blockchain, + M: prost::Message + BlockchainBlock + Default + 'static, +{ + info!( + logger, + "Starting firehose block ingestors with {} chains [{}]", + chains.len(), + chains + .keys() + .map(|v| v.clone()) + .collect::>() + .join(", ") + ); + + // Create Firehose block ingestors and spawn a thread to run each + chains + .iter() + .for_each(|(network_name, chain)| { + info!( + logger, + "Starting firehose block ingestor for network"; + "network_name" => &network_name + ); + + let endpoint = chain + .firehose_endpoints + .random() + .expect("One Firehose endpoint should exist at that execution point"); + + match store.block_store().chain_store(network_name.as_ref()) { + Some(s) => { + let block_ingestor = FirehoseBlockIngestor::::new( + s, + endpoint.clone(), + logger.new(o!("component" => "FirehoseBlockIngestor", "provider" => endpoint.provider.clone())), + ); + + // Run the Firehose block ingestor in the background + graph::spawn(block_ingestor.run()); + }, + None => { + error!(logger, "Not starting firehose block ingestor (no chain store available)"; "network_name" => &network_name); + } + } + }); +} diff --git a/node/src/manager/catalog.rs b/node/src/manager/catalog.rs new file mode 100644 index 0000000..af17a38 --- /dev/null +++ b/node/src/manager/catalog.rs @@ -0,0 +1,71 @@ +pub mod pg_catalog { + diesel::table! { + pg_catalog.pg_stat_database (datid) { + datid -> Oid, + datname -> Nullable, + numbackends -> Nullable, + xact_commit -> Nullable, + xact_rollback -> Nullable, + blks_read -> Nullable, + blks_hit -> Nullable, + tup_returned -> Nullable, + tup_fetched -> Nullable, + tup_inserted -> Nullable, + tup_updated -> Nullable, + tup_deleted -> Nullable, + conflicts -> Nullable, + temp_files -> Nullable, + temp_bytes -> Nullable, + deadlocks -> Nullable, + blk_read_time -> Nullable, + blk_write_time -> Nullable, + stats_reset -> Nullable, + } + } + + diesel::table! { + pg_catalog.pg_stat_user_indexes (relid) { + relid -> Oid, + indexrelid -> Nullable, + schemaname -> Nullable, + relname -> Nullable, + indexrelname -> Nullable, + idx_scan -> Nullable, + idx_tup_read -> Nullable, + idx_tup_fetch -> Nullable, + } + } + + diesel::table! { + pg_catalog.pg_stat_user_tables (relid) { + relid -> Oid, + schemaname -> Nullable, + relname -> Nullable, + seq_scan -> Nullable, + seq_tup_read -> Nullable, + idx_scan -> Nullable, + idx_tup_fetch -> Nullable, + n_tup_ins -> Nullable, + n_tup_upd -> Nullable, + n_tup_del -> Nullable, + n_tup_hot_upd -> Nullable, + n_live_tup -> Nullable, + n_dead_tup -> Nullable, + n_mod_since_analyze -> Nullable, + last_vacuum -> Nullable, + last_autovacuum -> Nullable, + last_analyze -> Nullable, + last_autoanalyze -> Nullable, + vacuum_count -> Nullable, + autovacuum_count -> Nullable, + analyze_count -> Nullable, + autoanalyze_count -> Nullable, + } + } + + diesel::allow_tables_to_appear_in_same_query!( + pg_stat_database, + pg_stat_user_indexes, + pg_stat_user_tables, + ); +} diff --git a/node/src/manager/commands/assign.rs b/node/src/manager/commands/assign.rs new file mode 100644 index 0000000..aa045a1 --- /dev/null +++ b/node/src/manager/commands/assign.rs @@ -0,0 +1,62 @@ +use graph::prelude::{anyhow::anyhow, Error, NodeId, StoreEvent}; +use graph_store_postgres::{ + command_support::catalog, connection_pool::ConnectionPool, NotificationSender, +}; + +use crate::manager::deployment::DeploymentSearch; + +pub async fn unassign( + primary: ConnectionPool, + sender: &NotificationSender, + search: &DeploymentSearch, +) -> Result<(), Error> { + let locator = search.locate_unique(&primary)?; + + let conn = primary.get()?; + let conn = catalog::Connection::new(conn); + + let site = conn + .locate_site(locator.clone())? + .ok_or_else(|| anyhow!("failed to locate site for {locator}"))?; + + println!("unassigning {locator}"); + let changes = conn.unassign_subgraph(&site)?; + conn.send_store_event(sender, &StoreEvent::new(changes))?; + + Ok(()) +} + +pub fn reassign( + primary: ConnectionPool, + sender: &NotificationSender, + search: &DeploymentSearch, + node: String, +) -> Result<(), Error> { + let node = NodeId::new(node.clone()).map_err(|()| anyhow!("illegal node id `{}`", node))?; + let locator = search.locate_unique(&primary)?; + + let conn = primary.get()?; + let conn = catalog::Connection::new(conn); + + let site = conn + .locate_site(locator.clone())? + .ok_or_else(|| anyhow!("failed to locate site for {locator}"))?; + let changes = match conn.assigned_node(&site)? { + Some(cur) => { + if cur == node { + println!("deployment {locator} is already assigned to {cur}"); + vec![] + } else { + println!("reassigning {locator} to {node} (was {cur})"); + conn.reassign_subgraph(&site, &node)? + } + } + None => { + println!("assigning {locator} to {node}"); + conn.assign_subgraph(&site, &node)? + } + }; + conn.send_store_event(sender, &StoreEvent::new(changes))?; + + Ok(()) +} diff --git a/node/src/manager/commands/chain.rs b/node/src/manager/commands/chain.rs new file mode 100644 index 0000000..764ae06 --- /dev/null +++ b/node/src/manager/commands/chain.rs @@ -0,0 +1,131 @@ +use std::sync::Arc; + +use graph::blockchain::BlockPtr; +use graph::cheap_clone::CheapClone; +use graph::prelude::BlockNumber; +use graph::prelude::ChainStore as _; +use graph::prelude::EthereumBlock; +use graph::prelude::LightEthereumBlockExt as _; +use graph::prelude::{anyhow, anyhow::bail}; +use graph::{ + components::store::BlockStore as _, prelude::anyhow::Error, prelude::serde_json as json, +}; +use graph_store_postgres::BlockStore; +use graph_store_postgres::{ + command_support::catalog::block_store, connection_pool::ConnectionPool, +}; + +pub async fn list(primary: ConnectionPool, store: Arc) -> Result<(), Error> { + let mut chains = { + let conn = primary.get()?; + block_store::load_chains(&conn)? + }; + chains.sort_by_key(|chain| chain.name.clone()); + + if !chains.is_empty() { + println!( + "{:^20} | {:^10} | {:^10} | {:^7} | {:^10}", + "name", "shard", "namespace", "version", "head block" + ); + println!( + "{:-^20}-+-{:-^10}-+-{:-^10}-+-{:-^7}-+-{:-^10}", + "", "", "", "", "" + ); + } + for chain in chains { + let head_block = match store.chain_store(&chain.name) { + None => "no chain".to_string(), + Some(chain_store) => chain_store + .chain_head_ptr() + .await? + .map(|ptr| ptr.number.to_string()) + .unwrap_or("none".to_string()), + }; + println!( + "{:<20} | {:<10} | {:<10} | {:>7} | {:>10}", + chain.name, chain.shard, chain.storage, chain.net_version, head_block + ); + } + Ok(()) +} + +pub async fn info( + primary: ConnectionPool, + store: Arc, + name: String, + offset: BlockNumber, + hashes: bool, +) -> Result<(), Error> { + fn row(label: &str, value: impl std::fmt::Display) { + println!("{:<16} | {}", label, value.to_string()); + } + + fn print_ptr(label: &str, ptr: Option, hashes: bool) { + match ptr { + None => { + row(label, "ø"); + } + Some(ptr) => { + row(label, ptr.number); + if hashes { + row("", ptr.hash); + } + } + } + } + + let conn = primary.get()?; + + let chain = + block_store::find_chain(&conn, &name)?.ok_or_else(|| anyhow!("unknown chain: {}", name))?; + + let chain_store = store + .chain_store(&chain.name) + .ok_or_else(|| anyhow!("unknown chain: {}", name))?; + let head_block = chain_store.cheap_clone().chain_head_ptr().await?; + let ancestor = match &head_block { + None => None, + Some(head_block) => chain_store + .ancestor_block(head_block.clone(), offset) + .await? + .map(json::from_value::) + .transpose()? + .map(|b| b.block.block_ptr()), + }; + + row("name", chain.name); + row("shard", chain.shard); + row("namespace", chain.storage); + row("net_version", chain.net_version); + if hashes { + row("genesis", chain.genesis_block); + } + print_ptr("head block", head_block, hashes); + row("reorg threshold", offset); + print_ptr("reorg ancestor", ancestor, hashes); + + Ok(()) +} + +pub fn remove(primary: ConnectionPool, store: Arc, name: String) -> Result<(), Error> { + let sites = { + let conn = graph_store_postgres::command_support::catalog::Connection::new(primary.get()?); + conn.find_sites_for_network(&name)? + }; + + if !sites.is_empty() { + println!( + "there are {} deployments using chain {}:", + sites.len(), + name + ); + for site in sites { + println!("{:<8} | {} ", site.namespace, site.deployment); + } + bail!("remove all deployments using chain {} first", name); + } + + store.drop_chain(&name)?; + + Ok(()) +} diff --git a/node/src/manager/commands/check_blocks.rs b/node/src/manager/commands/check_blocks.rs new file mode 100644 index 0000000..e5d2c0c --- /dev/null +++ b/node/src/manager/commands/check_blocks.rs @@ -0,0 +1,269 @@ +use graph::{ + anyhow::{bail, ensure}, + components::store::ChainStore as ChainStoreTrait, + prelude::{ + anyhow::{self, anyhow, Context}, + web3::types::H256, + }, + slog::Logger, +}; +use graph_chain_ethereum::{EthereumAdapter, EthereumAdapterTrait}; +use graph_store_postgres::ChainStore; +use std::sync::Arc; + +pub async fn by_hash( + hash: &str, + chain_store: Arc, + ethereum_adapter: &EthereumAdapter, + logger: &Logger, +) -> anyhow::Result<()> { + let block_hash = helpers::parse_block_hash(hash)?; + run(&block_hash, &chain_store, ethereum_adapter, logger).await +} + +pub async fn by_number( + number: i32, + chain_store: Arc, + ethereum_adapter: &EthereumAdapter, + logger: &Logger, +) -> anyhow::Result<()> { + let block_hash = steps::resolve_block_hash_from_block_number(number, &chain_store)?; + run(&block_hash, &chain_store, ethereum_adapter, logger).await +} + +pub async fn by_range( + chain_store: Arc, + ethereum_adapter: &EthereumAdapter, + range_from: Option, + range_to: Option, + logger: &Logger, +) -> anyhow::Result<()> { + // Resolve a range of block numbers into a collection of blocks hashes + let range = ranges::Range::new(range_from, range_to)?; + let max = match range.upper_bound { + // When we have an open upper bound, we use the chain head's block number + None => steps::find_chain_head(&chain_store)?, + Some(x) => x, + }; + // FIXME: This performs poorly. + // TODO: This could be turned into async code + for block_number in range.lower_bound..=max { + println!("Fixing block [{block_number}/{max}]"); + let block_hash = steps::resolve_block_hash_from_block_number(block_number, &chain_store)?; + run(&block_hash, &chain_store, ethereum_adapter, logger).await? + } + Ok(()) +} + +pub fn truncate(chain_store: Arc, skip_confirmation: bool) -> anyhow::Result<()> { + if !skip_confirmation && !helpers::prompt_for_confirmation()? { + println!("Aborting."); + return Ok(()); + } + + chain_store + .truncate_block_cache() + .with_context(|| format!("Failed to truncate block cache for {}", chain_store.chain)) +} + +async fn run( + block_hash: &H256, + chain_store: &ChainStore, + ethereum_adapter: &EthereumAdapter, + logger: &Logger, +) -> anyhow::Result<()> { + let cached_block = steps::fetch_single_cached_block(*block_hash, &chain_store)?; + let provider_block = + steps::fetch_single_provider_block(&block_hash, ethereum_adapter, logger).await?; + let diff = steps::diff_block_pair(&cached_block, &provider_block); + steps::report_difference(diff.as_deref(), &block_hash); + if diff.is_some() { + steps::delete_block(&block_hash, &chain_store)?; + } + Ok(()) +} + +mod steps { + use super::*; + use futures::compat::Future01CompatExt; + use graph::prelude::serde_json::{self, Value}; + use json_structural_diff::{colorize as diff_to_string, JsonDiff}; + + /// Queries the [`ChainStore`] about the block hash for the given block number. + /// + /// Errors on a non-unary result. + pub(super) fn resolve_block_hash_from_block_number( + number: i32, + chain_store: &ChainStore, + ) -> anyhow::Result { + let block_hashes = chain_store.block_hashes_by_block_number(number)?; + let hash = helpers::get_single_item("block hash", block_hashes) + .with_context(|| format!("Failed to locate block number {} in store", number))?; + Ok(H256(hash.as_slice().try_into()?)) + } + + /// Queries the [`ChainStore`] for a cached block given a block hash. + /// + /// Errors on a non-unary result. + pub(super) fn fetch_single_cached_block( + block_hash: H256, + chain_store: &ChainStore, + ) -> anyhow::Result { + let blocks = chain_store.blocks(&[block_hash.into()])?; + if blocks.is_empty() { + bail!("Could not find a block with hash={block_hash:?} in cache") + } + helpers::get_single_item("block", blocks) + .with_context(|| format!("Failed to locate block {} in store.", block_hash)) + } + + /// Fetches a block from a JRPC endpoint. + /// + /// Errors on a non-unary result. + pub(super) async fn fetch_single_provider_block( + block_hash: &H256, + ethereum_adapter: &EthereumAdapter, + logger: &Logger, + ) -> anyhow::Result { + let provider_block = ethereum_adapter + .block_by_hash(&logger, *block_hash) + .compat() + .await + .with_context(|| format!("failed to fetch block {block_hash}"))? + .ok_or_else(|| anyhow!("JRPC provider found no block {block_hash}"))?; + ensure!( + provider_block.hash == Some(*block_hash), + "Provider responded with a different block hash" + ); + serde_json::to_value(provider_block) + .context("failed to parse provider block as a JSON value") + } + + /// Compares two [`serde_json::Value`] values. + /// + /// If they are different, returns a user-friendly string ready to be displayed. + pub(super) fn diff_block_pair(a: &Value, b: &Value) -> Option { + if a == b { + None + } else { + match JsonDiff::diff(a, &b, false).diff { + // The diff could potentially be a `Value::Null`, which is equivalent to not being + // different at all. + None | Some(Value::Null) => None, + Some(diff) => { + // Convert the JSON diff to a pretty-formatted text that will be displayed to + // the user + Some(diff_to_string(&diff, false)) + } + } + } + } + + /// Prints the difference between two [`serde_json::Value`] values to the user. + pub(super) fn report_difference(difference: Option<&str>, hash: &H256) { + if let Some(diff) = difference { + eprintln!("block {hash} diverges from cache:"); + eprintln!("{diff}"); + } else { + println!("Cached block is equal to the same block from provider.") + } + } + + /// Attempts to delete a block from the block cache. + pub(super) fn delete_block(hash: &H256, chain_store: &ChainStore) -> anyhow::Result<()> { + println!("Deleting block {hash} from cache."); + chain_store.delete_blocks(&[&hash])?; + println!("Done."); + Ok(()) + } + + /// Queries the [`ChainStore`] about the chain head. + pub(super) fn find_chain_head(chain_store: &ChainStore) -> anyhow::Result { + let chain_head: Option = chain_store.chain_head_block(&chain_store.chain)?; + chain_head.ok_or_else(|| anyhow!("Could not find the chain head for {}", chain_store.chain)) + } +} + +mod helpers { + use super::*; + use graph::prelude::hex; + use std::io::{self, Write}; + + /// Tries to parse a [`H256`] from a hex string. + pub(super) fn parse_block_hash(hash: &str) -> anyhow::Result { + let hash = hash.trim_start_matches("0x"); + let hash = hex::decode(hash)?; + Ok(H256::from_slice(&hash)) + } + + /// Asks users if they are certain about truncating the whole block cache. + pub(super) fn prompt_for_confirmation() -> anyhow::Result { + print!("This will delete all cached blocks.\nProceed? [y/N] "); + io::stdout().flush()?; + + let mut answer = String::new(); + io::stdin().read_line(&mut answer)?; + answer.make_ascii_lowercase(); + + match answer.trim() { + "y" | "yes" => Ok(true), + _ => Ok(false), + } + } + + /// Convenience function for extracting values from unary sets. + pub(super) fn get_single_item(name: &'static str, collection: I) -> anyhow::Result + where + I: IntoIterator, + { + let mut iterator = collection.into_iter(); + match (iterator.next(), iterator.next()) { + (Some(a), None) => Ok(a), + (None, None) => bail!("Expected a single {name} but found none."), + _ => bail!("Expected a single {name} but found multiple occurrences."), + } + } +} + +/// Custom range type +mod ranges { + use graph::prelude::anyhow::{self, bail}; + + pub(super) struct Range { + pub(super) lower_bound: i32, + pub(super) upper_bound: Option, + } + + impl Range { + pub fn new(lower_bound: Option, upper_bound: Option) -> anyhow::Result { + let (lower_bound, upper_bound) = match (lower_bound, upper_bound) { + // Invalid cases: + (None, None) => { + bail!( + "This would wipe the whole cache. \ + Use `graphman chain truncate` instead" + ) + } + (Some(0), _) => bail!("Genesis block can't be removed"), + (Some(x), _) | (_, Some(x)) if x < 0 => { + bail!("Negative block number used as range bound: {}", x) + } + (Some(lower), Some(upper)) if upper < lower => bail!( + "Upper bound ({}) can't be smaller than lower bound ({})", + upper, + lower + ), + + // Valid cases: + // Open lower bounds are set to the lowest possible block number + (None, upper @ Some(_)) => (1, upper), + (Some(lower), upper) => (lower, upper), + }; + + Ok(Self { + lower_bound, + upper_bound, + }) + } + } +} diff --git a/node/src/manager/commands/config.rs b/node/src/manager/commands/config.rs new file mode 100644 index 0000000..f33552b --- /dev/null +++ b/node/src/manager/commands/config.rs @@ -0,0 +1,142 @@ +use std::{collections::BTreeMap, sync::Arc}; + +use graph::{ + anyhow::bail, + components::metrics::MetricsRegistry, + itertools::Itertools, + prelude::{ + anyhow::{anyhow, Error}, + NodeId, + }, + slog::Logger, +}; +use graph_chain_ethereum::{EthereumAdapterTrait, NodeCapabilities}; +use graph_store_postgres::DeploymentPlacer; + +use crate::config::Config; + +pub fn place(placer: &dyn DeploymentPlacer, name: &str, network: &str) -> Result<(), Error> { + match placer.place(name, network).map_err(|s| anyhow!(s))? { + None => { + println!( + "no matching placement rule; default placement from JSON RPC call would be used" + ); + } + Some((shards, nodes)) => { + let nodes: Vec<_> = nodes.into_iter().map(|n| n.to_string()).collect(); + let shards: Vec<_> = shards.into_iter().map(|s| s.to_string()).collect(); + println!("subgraph: {}", name); + println!("network: {}", network); + println!("shard: {}", shards.join(", ")); + println!("nodes: {}", nodes.join(", ")); + } + } + Ok(()) +} + +pub fn check(config: &Config, print: bool) -> Result<(), Error> { + match config.to_json() { + Ok(txt) => { + if print { + println!("{}", txt); + } else { + println!("Successfully validated configuration"); + } + Ok(()) + } + Err(e) => Err(anyhow!("error serializing config: {}", e)), + } +} + +pub fn pools(config: &Config, nodes: Vec, shard: bool) -> Result<(), Error> { + // Quietly replace `-` with `_` in node names to make passing in pod names + // from k8s less annoying + let nodes: Vec<_> = nodes + .into_iter() + .map(|name| { + NodeId::new(name.replace("-", "_")) + .map_err(|()| anyhow!("illegal node name `{}`", name)) + }) + .collect::>()?; + // node -> shard_name -> size + let mut sizes = BTreeMap::new(); + for node in &nodes { + let mut shard_sizes = BTreeMap::new(); + for (name, shard) in &config.stores { + let size = shard.pool_size.size_for(node, name)?; + shard_sizes.insert(name.to_string(), size); + for (replica_name, replica) in &shard.replicas { + let qname = format!("{}.{}", name, replica_name); + let size = replica.pool_size.size_for(node, &qname)?; + shard_sizes.insert(qname, size); + } + } + sizes.insert(node.to_string(), shard_sizes); + } + + if shard { + let mut by_shard: BTreeMap<&str, u32> = BTreeMap::new(); + for shard_sizes in sizes.values() { + for (shard_name, size) in shard_sizes { + *by_shard.entry(shard_name).or_default() += size; + } + } + for (shard_name, size) in by_shard { + println!("{}: {}", shard_name, size); + } + } else { + for node in &nodes { + let empty = BTreeMap::new(); + println!("{}:", node); + let node_sizes = sizes.get(node.as_str()).unwrap_or(&empty); + for (shard, size) in node_sizes { + println!(" {}: {}", shard, size); + } + } + } + Ok(()) +} + +pub async fn provider( + logger: Logger, + config: &Config, + registry: Arc, + features: String, + network: String, +) -> Result<(), Error> { + // Like NodeCapabilities::from_str but with error checking for typos etc. + fn caps_from_features(features: String) -> Result { + let mut caps = NodeCapabilities { + archive: false, + traces: false, + }; + for feature in features.split(',') { + match feature { + "archive" => caps.archive = true, + "traces" => caps.traces = true, + _ => bail!("unknown feature {}", feature), + } + } + Ok(caps) + } + + let caps = caps_from_features(features)?; + let networks = + crate::manager::commands::run::create_ethereum_networks(logger, registry, config, &network) + .await?; + let adapters = networks + .networks + .get(&network) + .ok_or_else(|| anyhow!("unknown network {}", network))?; + let adapters = adapters.all_cheapest_with(&caps); + println!( + "deploy on network {} with features [{}] on node {}\neligible providers: {}", + network, + caps, + config.node.as_str(), + adapters + .map(|adapter| adapter.provider().to_string()) + .join(", ") + ); + Ok(()) +} diff --git a/node/src/manager/commands/copy.rs b/node/src/manager/commands/copy.rs new file mode 100644 index 0000000..c832c57 --- /dev/null +++ b/node/src/manager/commands/copy.rs @@ -0,0 +1,346 @@ +use diesel::{ExpressionMethods, JoinOnDsl, OptionalExtension, QueryDsl, RunQueryDsl}; +use std::{collections::HashMap, sync::Arc, time::SystemTime}; + +use graph::{ + components::store::BlockStore as _, + data::query::QueryTarget, + prelude::{ + anyhow::{anyhow, bail, Error}, + chrono::{DateTime, Duration, SecondsFormat, Utc}, + BlockPtr, ChainStore, DeploymentHash, NodeId, QueryStoreManager, + }, +}; +use graph_store_postgres::{ + command_support::catalog::{self, copy_state, copy_table_state}, + PRIMARY_SHARD, +}; +use graph_store_postgres::{connection_pool::ConnectionPool, Shard, Store, SubgraphStore}; + +use crate::manager::deployment::DeploymentSearch; +use crate::manager::display::List; + +type UtcDateTime = DateTime; + +#[derive(Queryable, QueryableByName, Debug)] +#[table_name = "copy_state"] +struct CopyState { + src: i32, + dst: i32, + #[allow(dead_code)] + target_block_hash: Vec, + target_block_number: i32, + started_at: UtcDateTime, + finished_at: Option, + cancelled_at: Option, +} + +#[derive(Queryable, QueryableByName, Debug)] +#[table_name = "copy_table_state"] +struct CopyTableState { + #[allow(dead_code)] + id: i32, + entity_type: String, + #[allow(dead_code)] + dst: i32, + next_vid: i64, + target_vid: i64, + batch_size: i64, + #[allow(dead_code)] + started_at: UtcDateTime, + finished_at: Option, + duration_ms: i64, +} + +impl CopyState { + fn find( + pools: &HashMap, + shard: &Shard, + dst: i32, + ) -> Result)>, Error> { + use copy_state as cs; + use copy_table_state as cts; + + let dpool = pools + .get(shard) + .ok_or_else(|| anyhow!("can not find pool for shard {}", shard))?; + + let dconn = dpool.get()?; + + let tables = cts::table + .filter(cts::dst.eq(dst)) + .order_by(cts::entity_type) + .load::(&dconn)?; + + Ok(cs::table + .filter(cs::dst.eq(dst)) + .get_result::(&dconn) + .optional()? + .map(|state| (state, tables))) + } +} + +pub async fn create( + store: Arc, + primary: ConnectionPool, + src: DeploymentSearch, + shard: String, + shards: Vec, + node: String, + block_offset: u32, +) -> Result<(), Error> { + let block_offset = block_offset as i32; + let subgraph_store = store.subgraph_store(); + let src = src.locate_unique(&primary)?; + let query_store = store + .query_store( + QueryTarget::Deployment(src.hash.clone(), Default::default()), + true, + ) + .await?; + let network = query_store.network_name(); + + let src_ptr = query_store.block_ptr().await?.ok_or_else(|| anyhow!("subgraph {} has not indexed any blocks yet and can not be used as the source of a copy", src))?; + let src_number = if src_ptr.number <= block_offset { + bail!("subgraph {} has only indexed up to block {}, but we need at least block {} before we can copy from it", src, src_ptr.number, block_offset); + } else { + src_ptr.number - block_offset + }; + + let chain_store = store + .block_store() + .chain_store(&network) + .ok_or_else(|| anyhow!("could not find chain store for network {}", network))?; + let mut hashes = chain_store.block_hashes_by_block_number(src_number)?; + let hash = match hashes.len() { + 0 => bail!( + "could not find a block with number {} in our cache", + src_number + ), + 1 => hashes.pop().unwrap(), + n => bail!( + "the cache contains {} hashes for block number {}", + n, + src_number + ), + }; + let base_ptr = BlockPtr::new(hash, src_number); + + if !shards.contains(&shard) { + bail!( + "unknown shard {shard}, only shards {} are configured", + shards.join(", ") + ) + } + let shard = Shard::new(shard)?; + let node = NodeId::new(node.clone()).map_err(|()| anyhow!("invalid node id `{}`", node))?; + + let dst = subgraph_store.copy_deployment(&src, shard, node, base_ptr)?; + + println!("created deployment {} as copy of {}", dst, src); + Ok(()) +} + +pub fn activate(store: Arc, deployment: String, shard: String) -> Result<(), Error> { + let shard = Shard::new(shard)?; + let deployment = + DeploymentHash::new(deployment).map_err(|s| anyhow!("illegal deployment hash `{}`", s))?; + let deployment = store + .locate_in_shard(&deployment, shard.clone())? + .ok_or_else(|| { + anyhow!( + "could not find a copy for {} in shard {}", + deployment, + shard + ) + })?; + store.activate(&deployment)?; + println!("activated copy {}", deployment); + Ok(()) +} + +pub fn list(pools: HashMap) -> Result<(), Error> { + use catalog::active_copies as ac; + use catalog::deployment_schemas as ds; + + let primary = pools.get(&*PRIMARY_SHARD).expect("there is a primary pool"); + let conn = primary.get()?; + + let copies = ac::table + .inner_join(ds::table.on(ds::id.eq(ac::dst))) + .select(( + ac::src, + ac::dst, + ac::cancelled_at, + ac::queued_at, + ds::subgraph, + ds::shard, + )) + .load::<(i32, i32, Option, UtcDateTime, String, Shard)>(&conn)?; + if copies.is_empty() { + println!("no active copies"); + } else { + fn status(name: &str, at: UtcDateTime) { + println!( + "{:20} | {}", + name, + at.to_rfc3339_opts(SecondsFormat::Secs, false) + ); + } + + for (src, dst, cancelled_at, queued_at, deployment_hash, shard) in copies { + println!("{:-<78}", ""); + + println!("{:20} | {}", "deployment", deployment_hash); + println!("{:20} | sgd{} -> sgd{} ({})", "action", src, dst, shard); + match CopyState::find(&pools, &shard, dst)? { + Some((state, tables)) => match cancelled_at { + Some(cancel_requested) => match state.cancelled_at { + Some(cancelled_at) => status("cancelled", cancelled_at), + None => status("cancel requested", cancel_requested), + }, + None => match state.finished_at { + Some(finished_at) => status("finished", finished_at), + None => { + let target: i64 = tables.iter().map(|table| table.target_vid).sum(); + let next: i64 = tables.iter().map(|table| table.next_vid).sum(); + let done = next as f64 / target as f64 * 100.0; + status("started", state.started_at); + println!("{:20} | {:.2}% done, {}/{}", "progress", done, next, target) + } + }, + }, + None => status("queued", queued_at), + }; + } + } + Ok(()) +} + +pub fn status(pools: HashMap, dst: &DeploymentSearch) -> Result<(), Error> { + use catalog::active_copies as ac; + use catalog::deployment_schemas as ds; + + fn done(ts: &Option) -> String { + ts.map(|_| "✓").unwrap_or(".").to_string() + } + + fn duration(start: &UtcDateTime, end: &Option) -> String { + let start = *start; + let end = *end; + + let end = end.unwrap_or(UtcDateTime::from(SystemTime::now())); + let duration = end - start; + + human_duration(duration) + } + + fn human_duration(duration: Duration) -> String { + if duration.num_seconds() < 5 { + format!("{}ms", duration.num_milliseconds()) + } else if duration.num_minutes() < 5 { + format!("{}s", duration.num_seconds()) + } else { + format!("{}m", duration.num_minutes()) + } + } + + let primary = pools + .get(&*PRIMARY_SHARD) + .ok_or_else(|| anyhow!("can not find deployment with id {}", dst))?; + let pconn = primary.get()?; + let dst = dst.locate_unique(&primary)?.id.0; + + let (shard, deployment) = ds::table + .filter(ds::id.eq(dst as i32)) + .select((ds::shard, ds::subgraph)) + .get_result::<(Shard, String)>(&pconn)?; + + let (active, cancelled_at) = ac::table + .filter(ac::dst.eq(dst)) + .select((ac::src, ac::cancelled_at)) + .get_result::<(i32, Option)>(&pconn) + .optional()? + .map(|(_, cancelled_at)| (true, cancelled_at)) + .unwrap_or((false, None)); + + let (state, tables) = match CopyState::find(&pools, &shard, dst)? { + Some((state, tables)) => (state, tables), + None => { + if active { + println!("copying is queued but has not started"); + return Ok(()); + } else { + bail!("no copy operation for {} exists", dst); + } + } + }; + + let progress = match &state.finished_at { + Some(_) => done(&state.finished_at), + None => { + let target: i64 = tables.iter().map(|table| table.target_vid).sum(); + let next: i64 = tables.iter().map(|table| table.next_vid).sum(); + let pct = next as f64 / target as f64 * 100.0; + format!("{:.2}% done, {}/{}", pct, next, target) + } + }; + + let mut lst = vec![ + "deployment", + "src", + "dst", + "target block", + "duration", + "status", + ]; + let mut vals = vec![ + deployment, + state.src.to_string(), + state.dst.to_string(), + state.target_block_number.to_string(), + duration(&state.started_at, &state.finished_at), + progress, + ]; + match (cancelled_at, state.cancelled_at) { + (Some(c), None) => { + lst.push("cancel"); + vals.push(format!("requested at {}", c)); + } + (_, Some(c)) => { + lst.push("cancel"); + vals.push(format!("cancelled at {}", c)); + } + (None, None) => {} + } + let mut lst = List::new(lst); + lst.append(vals); + lst.render(); + println!(""); + + println!( + "{:^30} | {:^8} | {:^8} | {:^8} | {:^8}", + "entity type", "next", "target", "batch", "duration" + ); + println!("{:-<74}", "-"); + for table in tables { + let status = if table.next_vid > 0 && table.next_vid < table.target_vid { + ">".to_string() + } else if table.target_vid < 0 { + // empty source table + "✓".to_string() + } else { + done(&table.finished_at) + }; + println!( + "{} {:<28} | {:>8} | {:>8} | {:>8} | {:>8}", + status, + table.entity_type, + table.next_vid, + table.target_vid, + table.batch_size, + human_duration(Duration::milliseconds(table.duration_ms)), + ); + } + + Ok(()) +} diff --git a/node/src/manager/commands/create.rs b/node/src/manager/commands/create.rs new file mode 100644 index 0000000..02e1184 --- /dev/null +++ b/node/src/manager/commands/create.rs @@ -0,0 +1,14 @@ +use std::sync::Arc; + +use graph::prelude::{anyhow, Error, SubgraphName, SubgraphStore as _}; +use graph_store_postgres::SubgraphStore; + +pub fn run(store: Arc, name: String) -> Result<(), Error> { + let name = SubgraphName::new(name.clone()) + .map_err(|()| anyhow!("illegal subgraph name `{}`", name))?; + + println!("creating subgraph {}", name); + store.create_subgraph(name)?; + + Ok(()) +} diff --git a/node/src/manager/commands/index.rs b/node/src/manager/commands/index.rs new file mode 100644 index 0000000..2ef9582 --- /dev/null +++ b/node/src/manager/commands/index.rs @@ -0,0 +1,70 @@ +use crate::manager::deployment::DeploymentSearch; +use graph::prelude::{anyhow, StoreError}; +use graph_store_postgres::{connection_pool::ConnectionPool, SubgraphStore}; +use std::{collections::HashSet, sync::Arc}; + +fn validate_fields>(fields: &[T]) -> Result<(), anyhow::Error> { + // Must be non-empty. Double checking, since [`StructOpt`] already checks this. + if fields.is_empty() { + anyhow::bail!("at least one field must be informed") + } + // All values must be unique + let unique: HashSet<_> = fields.iter().map(AsRef::as_ref).collect(); + if fields.len() != unique.len() { + anyhow::bail!("entity fields must be unique") + } + Ok(()) +} +pub async fn create( + store: Arc, + pool: ConnectionPool, + search: DeploymentSearch, + entity_name: &str, + field_names: Vec, + index_method: String, +) -> Result<(), anyhow::Error> { + validate_fields(&field_names)?; + let deployment_locator = search.locate_unique(&pool)?; + println!("Index creation started. Please wait."); + match store + .create_manual_index(&deployment_locator, entity_name, field_names, index_method) + .await + { + Ok(()) => Ok(()), + Err(StoreError::Canceled) => { + eprintln!("Index creation attempt faield. Please retry."); + ::std::process::exit(1); + } + Err(other) => Err(anyhow::anyhow!(other)), + } +} + +pub async fn list( + store: Arc, + pool: ConnectionPool, + search: DeploymentSearch, + entity_name: &str, +) -> Result<(), anyhow::Error> { + let deployment_locator = search.locate_unique(&pool)?; + let indexes: Vec = store + .indexes_for_entity(&deployment_locator, entity_name) + .await?; + for index in &indexes { + println!("{index}") + } + Ok(()) +} + +pub async fn drop( + store: Arc, + pool: ConnectionPool, + search: DeploymentSearch, + index_name: &str, +) -> Result<(), anyhow::Error> { + let deployment_locator = search.locate_unique(&pool)?; + store + .drop_index_for_deployment(&deployment_locator, &index_name) + .await?; + println!("Dropped index {index_name}"); + Ok(()) +} diff --git a/node/src/manager/commands/info.rs b/node/src/manager/commands/info.rs new file mode 100644 index 0000000..19994b6 --- /dev/null +++ b/node/src/manager/commands/info.rs @@ -0,0 +1,54 @@ +use std::sync::Arc; + +use graph::{components::store::StatusStore, data::subgraph::status, prelude::anyhow}; +use graph_store_postgres::{connection_pool::ConnectionPool, Store}; + +use crate::manager::deployment::{Deployment, DeploymentSearch}; + +fn find( + pool: ConnectionPool, + search: DeploymentSearch, + current: bool, + pending: bool, + used: bool, +) -> Result, anyhow::Error> { + let current = current || used; + let pending = pending || used; + + let deployments = search.lookup(&pool)?; + // Filter by status; if neither `current` or `pending` are set, list + // all deployments + let deployments: Vec<_> = deployments + .into_iter() + .filter(|deployment| match (current, pending) { + (true, false) => deployment.status == "current", + (false, true) => deployment.status == "pending", + (true, true) => deployment.status == "current" || deployment.status == "pending", + (false, false) => true, + }) + .collect(); + Ok(deployments) +} + +pub fn run( + pool: ConnectionPool, + store: Option>, + search: DeploymentSearch, + current: bool, + pending: bool, + used: bool, +) -> Result<(), anyhow::Error> { + let deployments = find(pool, search, current, pending, used)?; + let ids: Vec<_> = deployments.iter().map(|d| d.locator().id).collect(); + let statuses = match store { + Some(store) => store.status(status::Filter::DeploymentIds(ids))?, + None => vec![], + }; + + if deployments.is_empty() { + println!("No matches"); + } else { + Deployment::print_table(deployments, statuses); + } + Ok(()) +} diff --git a/node/src/manager/commands/listen.rs b/node/src/manager/commands/listen.rs new file mode 100644 index 0000000..3193a4a --- /dev/null +++ b/node/src/manager/commands/listen.rs @@ -0,0 +1,71 @@ +use std::iter::FromIterator; +use std::sync::Arc; +use std::{collections::BTreeSet, io::Write}; + +use futures::compat::Future01CompatExt; +//use futures::future; +use graph::{ + components::store::{EntityType, SubscriptionManager as _}, + prelude::{serde_json, Error, Stream, SubscriptionFilter}, +}; +use graph_store_postgres::connection_pool::ConnectionPool; +use graph_store_postgres::SubscriptionManager; + +use crate::manager::deployment::DeploymentSearch; + +async fn listen( + mgr: Arc, + filter: BTreeSet, +) -> Result<(), Error> { + let events = mgr.subscribe(filter); + println!("press ctrl-c to stop"); + let res = events + .inspect(move |event| { + serde_json::to_writer_pretty(std::io::stdout(), event) + .expect("event can be serialized to JSON"); + writeln!(std::io::stdout(), "").unwrap(); + std::io::stdout().flush().unwrap(); + }) + .collect() + .compat() + .await; + + match res { + Ok(_) => { + println!("stream finished") + } + Err(()) => { + eprintln!("stream failed") + } + } + Ok(()) +} + +pub async fn assignments(mgr: Arc) -> Result<(), Error> { + println!("waiting for assignment events"); + listen( + mgr, + FromIterator::from_iter([SubscriptionFilter::Assignment]), + ) + .await?; + + Ok(()) +} + +pub async fn entities( + primary_pool: ConnectionPool, + mgr: Arc, + search: &DeploymentSearch, + entity_types: Vec, +) -> Result<(), Error> { + let locator = search.locate_unique(&primary_pool)?; + let filter = entity_types + .into_iter() + .map(|et| SubscriptionFilter::Entities(locator.hash.clone(), EntityType::new(et))) + .collect(); + + println!("waiting for store events from {}", locator); + listen(mgr, filter).await?; + + Ok(()) +} diff --git a/node/src/manager/commands/mod.rs b/node/src/manager/commands/mod.rs new file mode 100644 index 0000000..eab1f1a --- /dev/null +++ b/node/src/manager/commands/mod.rs @@ -0,0 +1,17 @@ +pub mod assign; +pub mod chain; +pub mod check_blocks; +pub mod config; +pub mod copy; +pub mod create; +pub mod index; +pub mod info; +pub mod listen; +pub mod prune; +pub mod query; +pub mod remove; +pub mod rewind; +pub mod run; +pub mod stats; +pub mod txn_speed; +pub mod unused_deployments; diff --git a/node/src/manager/commands/prune.rs b/node/src/manager/commands/prune.rs new file mode 100644 index 0000000..e635df7 --- /dev/null +++ b/node/src/manager/commands/prune.rs @@ -0,0 +1,183 @@ +use std::{ + collections::HashSet, + io::Write, + sync::Arc, + time::{Duration, Instant}, +}; + +use graph::{ + components::store::{PruneReporter, StatusStore}, + data::subgraph::status, + prelude::{anyhow, BlockNumber}, +}; +use graph_chain_ethereum::ENV_VARS as ETH_ENV; +use graph_store_postgres::{connection_pool::ConnectionPool, Store}; + +use crate::manager::{ + commands::stats::{abbreviate_table_name, show_stats}, + deployment::DeploymentSearch, +}; + +struct Progress { + start: Instant, + analyze_start: Instant, + switch_start: Instant, + final_start: Instant, + final_table_start: Instant, + nonfinal_start: Instant, +} + +impl Progress { + fn new() -> Self { + Self { + start: Instant::now(), + analyze_start: Instant::now(), + switch_start: Instant::now(), + final_start: Instant::now(), + final_table_start: Instant::now(), + nonfinal_start: Instant::now(), + } + } +} + +fn print_copy_header() { + println!("{:^30} | {:^10} | {:^11}", "table", "versions", "time"); + println!("{:-^30}-+-{:-^10}-+-{:-^11}", "", "", ""); + std::io::stdout().flush().ok(); +} + +fn print_copy_row(table: &str, total_rows: usize, elapsed: Duration) { + print!( + "\r{:<30} | {:>10} | {:>9}s", + abbreviate_table_name(table, 30), + total_rows, + elapsed.as_secs() + ); + std::io::stdout().flush().ok(); +} + +impl PruneReporter for Progress { + fn start_analyze(&mut self) { + print!("Analyze tables"); + self.analyze_start = Instant::now(); + } + + fn start_analyze_table(&mut self, table: &str) { + print!("\rAnalyze {table:48} "); + std::io::stdout().flush().ok(); + } + + fn finish_analyze(&mut self, stats: &[graph::components::store::VersionStats]) { + println!( + "\rAnalyzed {} tables in {}s", + stats.len(), + self.analyze_start.elapsed().as_secs() + ); + show_stats(stats, HashSet::new()).ok(); + println!(""); + } + + fn copy_final_start(&mut self, earliest_block: BlockNumber, final_block: BlockNumber) { + println!("Copy final entities (versions live between {earliest_block} and {final_block})"); + print_copy_header(); + + self.final_start = Instant::now(); + self.final_table_start = self.final_start; + } + + fn copy_final_batch(&mut self, table: &str, _rows: usize, total_rows: usize, finished: bool) { + print_copy_row(table, total_rows, self.final_table_start.elapsed()); + if finished { + println!(""); + self.final_table_start = Instant::now(); + } + std::io::stdout().flush().ok(); + } + + fn copy_final_finish(&mut self) { + println!( + "Finished copying final entity versions in {}s\n", + self.final_start.elapsed().as_secs() + ); + } + + fn start_switch(&mut self) { + println!("Blocking writes and switching tables"); + print_copy_header(); + self.switch_start = Instant::now(); + } + + fn finish_switch(&mut self) { + println!( + "Enabling writes. Switching took {}s\n", + self.switch_start.elapsed().as_secs() + ); + } + + fn copy_nonfinal_start(&mut self, table: &str) { + print_copy_row(table, 0, Duration::from_secs(0)); + self.nonfinal_start = Instant::now(); + } + + fn copy_nonfinal_finish(&mut self, table: &str, rows: usize) { + print_copy_row(table, rows, self.nonfinal_start.elapsed()); + println!(""); + std::io::stdout().flush().ok(); + } + + fn finish_prune(&mut self) { + println!("Finished pruning in {}s", self.start.elapsed().as_secs()); + } +} + +pub async fn run( + store: Arc, + primary_pool: ConnectionPool, + search: DeploymentSearch, + history: usize, + prune_ratio: f64, +) -> Result<(), anyhow::Error> { + let history = history as BlockNumber; + let deployment = search.locate_unique(&primary_pool)?; + let mut info = store + .status(status::Filter::DeploymentIds(vec![deployment.id]))? + .pop() + .ok_or_else(|| anyhow!("deployment {deployment} not found"))?; + if info.chains.len() > 1 { + return Err(anyhow!( + "deployment {deployment} indexes {} chains, not sure how to deal with more than one chain", + info.chains.len() + )); + } + let status = info + .chains + .pop() + .ok_or_else(|| anyhow!("deployment {} does not index any chain", deployment))?; + let latest = status.latest_block.map(|ptr| ptr.number()).unwrap_or(0); + if latest <= history { + return Err(anyhow!("deployment {deployment} has only indexed up to block {latest} and we can't preserve {history} blocks of history")); + } + + println!("prune {deployment}"); + println!(" latest: {latest}"); + println!(" final: {}", latest - ETH_ENV.reorg_threshold); + println!(" earliest: {}\n", latest - history); + + let reporter = Box::new(Progress::new()); + store + .subgraph_store() + .prune( + reporter, + &deployment, + latest - history, + // Using the setting for eth chains is a bit lazy; the value + // should really depend on the chain, but we don't have a + // convenient way to figure out how each chain deals with + // finality + ETH_ENV.reorg_threshold, + prune_ratio, + ) + .await?; + + Ok(()) +} diff --git a/node/src/manager/commands/query.rs b/node/src/manager/commands/query.rs new file mode 100644 index 0000000..a57ca14 --- /dev/null +++ b/node/src/manager/commands/query.rs @@ -0,0 +1,143 @@ +use std::fs::File; +use std::io::Write; +use std::iter::FromIterator; +use std::time::Duration; +use std::{collections::HashMap, sync::Arc}; + +use graph::data::query::Trace; +use graph::prelude::r; +use graph::{ + data::query::QueryTarget, + prelude::{ + anyhow::{self, anyhow}, + serde_json, DeploymentHash, GraphQlRunner as _, Query, QueryVariables, SubgraphName, + }, +}; +use graph_graphql::prelude::GraphQlRunner; +use graph_store_postgres::Store; + +use crate::manager::PanicSubscriptionManager; + +pub async fn run( + runner: Arc>, + target: String, + query: String, + vars: Vec, + output: Option, + trace: Option, +) -> Result<(), anyhow::Error> { + let target = if target.starts_with("Qm") { + let id = + DeploymentHash::new(target).map_err(|id| anyhow!("illegal deployment id `{}`", id))?; + QueryTarget::Deployment(id, Default::default()) + } else { + let name = SubgraphName::new(target.clone()) + .map_err(|()| anyhow!("illegal subgraph name `{}`", target))?; + QueryTarget::Name(name, Default::default()) + }; + + let document = graphql_parser::parse_query(&query)?.into_static(); + let vars: Vec<(String, r::Value)> = vars + .into_iter() + .map(|v| { + let mut pair = v.splitn(2, '=').map(|s| s.to_string()); + let key = pair.next(); + let value = pair + .next() + .map(|s| r::Value::String(s)) + .unwrap_or(r::Value::Null); + match key { + Some(key) => Ok((key, value)), + None => Err(anyhow!( + "malformed variable `{}`, it must be of the form `key=value`", + v + )), + } + }) + .collect::>()?; + let query = Query::new( + document, + Some(QueryVariables::new(HashMap::from_iter(vars))), + ); + + let res = runner.run_query(query, target).await; + if let Some(output) = output { + let mut f = File::create(output)?; + let json = serde_json::to_string(&res)?; + writeln!(f, "{}", json)?; + } + + // The format of this file is pretty awful, but good enough to fish out + // interesting SQL queries + if let Some(trace) = trace { + let mut f = File::create(trace)?; + let json = serde_json::to_string(&res.traces())?; + writeln!(f, "{}", json)?; + } + + for trace in res.traces() { + print_brief_trace("root", trace, 0)?; + } + Ok(()) +} + +fn print_brief_trace(name: &str, trace: &Trace, indent: usize) -> Result<(), anyhow::Error> { + use Trace::*; + + fn query_time(trace: &Trace) -> Duration { + match trace { + None => Duration::from_millis(0), + Root { children, .. } => children.iter().map(|(_, trace)| query_time(trace)).sum(), + Query { + elapsed, children, .. + } => *elapsed + children.iter().map(|(_, trace)| query_time(trace)).sum(), + } + } + + match trace { + None => { /* do nothing */ } + Root { + elapsed, children, .. + } => { + let elapsed = *elapsed.lock().unwrap(); + let qt = query_time(trace); + let pt = elapsed - qt; + + println!( + "{space:indent$}{name:rest$} {elapsed:7}ms", + space = " ", + indent = indent, + rest = 48 - indent, + name = name, + elapsed = elapsed.as_millis(), + ); + for (name, trace) in children { + print_brief_trace(name, trace, indent + 2)?; + } + println!("\nquery: {:7}ms", qt.as_millis()); + println!("other: {:7}ms", pt.as_millis()); + println!("total: {:7}ms", elapsed.as_millis()) + } + Query { + elapsed, + entity_count, + children, + .. + } => { + println!( + "{space:indent$}{name:rest$} {elapsed:7}ms [{count:7} entities]", + space = " ", + indent = indent, + rest = 50 - indent, + name = name, + elapsed = elapsed.as_millis(), + count = entity_count + ); + for (name, trace) in children { + print_brief_trace(name, trace, indent + 2)?; + } + } + } + + Ok(()) +} diff --git a/node/src/manager/commands/remove.rs b/node/src/manager/commands/remove.rs new file mode 100644 index 0000000..36d0b61 --- /dev/null +++ b/node/src/manager/commands/remove.rs @@ -0,0 +1,14 @@ +use std::sync::Arc; + +use graph::prelude::{anyhow, Error, SubgraphName, SubgraphStore as _}; +use graph_store_postgres::SubgraphStore; + +pub fn run(store: Arc, name: String) -> Result<(), Error> { + let name = SubgraphName::new(name.clone()) + .map_err(|()| anyhow!("illegal subgraph name `{}`", name))?; + + println!("Removing subgraph {}", name); + store.remove_subgraph(name)?; + + Ok(()) +} diff --git a/node/src/manager/commands/rewind.rs b/node/src/manager/commands/rewind.rs new file mode 100644 index 0000000..393d778 --- /dev/null +++ b/node/src/manager/commands/rewind.rs @@ -0,0 +1,134 @@ +use std::sync::Arc; +use std::thread; +use std::time::Duration; +use std::{collections::HashSet, convert::TryFrom}; + +use graph::anyhow::bail; +use graph::components::store::{BlockStore as _, ChainStore as _}; +use graph::prelude::{anyhow, BlockNumber, BlockPtr, NodeId, SubgraphStore}; +use graph_store_postgres::BlockStore; +use graph_store_postgres::{connection_pool::ConnectionPool, Store}; + +use crate::manager::deployment::{Deployment, DeploymentSearch}; + +async fn block_ptr( + store: Arc, + searches: &[DeploymentSearch], + deployments: &[Deployment], + hash: &str, + number: BlockNumber, + force: bool, +) -> Result { + let block_ptr_to = BlockPtr::try_from((hash, number as i64)) + .map_err(|e| anyhow!("error converting to block pointer: {}", e))?; + + let chains = deployments.iter().map(|d| &d.chain).collect::>(); + if chains.len() > 1 { + let names = searches + .into_iter() + .map(|s| s.to_string()) + .collect::>() + .join(", "); + bail!("the deployments matching `{names}` are on different chains"); + } + let chain = chains.iter().next().unwrap(); + let chain_store = match store.chain_store(chain) { + None => bail!("can not find chain store for {}", chain), + Some(store) => store, + }; + if let Some((_, number, _)) = chain_store.block_number(&block_ptr_to.hash).await? { + if number != block_ptr_to.number { + bail!( + "the given hash is for block number {} but the command specified block number {}", + number, + block_ptr_to.number + ); + } + } else { + if !force { + bail!( + "the chain {} does not have a block with hash {} \ + (run with --force to avoid this error)", + chain, + block_ptr_to.hash + ); + } + } + Ok(block_ptr_to) +} + +pub async fn run( + primary: ConnectionPool, + store: Arc, + searches: Vec, + block_hash: String, + block_number: BlockNumber, + force: bool, + sleep: Duration, +) -> Result<(), anyhow::Error> { + const PAUSED: &str = "paused_"; + + let subgraph_store = store.subgraph_store(); + let block_store = store.block_store(); + + let deployments = searches + .iter() + .map(|search| search.lookup(&primary)) + .collect::, _>>()? + .into_iter() + .flatten() + .collect::>(); + if deployments.is_empty() { + println!("nothing to do"); + return Ok(()); + } + + let block_ptr_to = block_ptr( + block_store, + &searches, + &deployments, + &block_hash, + block_number, + force, + ) + .await?; + + println!("Pausing deployments"); + let mut paused = false; + for deployment in &deployments { + if let Some(node) = &deployment.node_id { + if !node.starts_with(PAUSED) { + let loc = deployment.locator(); + let node = + NodeId::new(format!("{}{}", PAUSED, node)).expect("paused_ node id is valid"); + subgraph_store.reassign_subgraph(&loc, &node)?; + println!(" ... paused {}", loc); + paused = true; + } + } + } + + if paused { + // There's no good way to tell that a subgraph has in fact stopped + // indexing. We sleep and hope for the best. + println!("\nWaiting 10s to make sure pausing was processed"); + thread::sleep(sleep); + } + + println!("\nRewinding deployments"); + for deployment in &deployments { + let loc = deployment.locator(); + subgraph_store.rewind(loc.hash.clone(), block_ptr_to.clone())?; + println!(" ... rewound {}", loc); + } + + println!("Resuming deployments"); + for deployment in &deployments { + if let Some(node) = &deployment.node_id { + let loc = deployment.locator(); + let node = NodeId::new(node.clone()).expect("node id is valid"); + subgraph_store.reassign_subgraph(&loc, &node)?; + } + } + Ok(()) +} diff --git a/node/src/manager/commands/run.rs b/node/src/manager/commands/run.rs new file mode 100644 index 0000000..df0db21 --- /dev/null +++ b/node/src/manager/commands/run.rs @@ -0,0 +1,489 @@ +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use crate::chain::create_firehose_networks; +use crate::config::{Config, ProviderDetails}; +use crate::manager::PanicSubscriptionManager; +use crate::store_builder::StoreBuilder; +use crate::MetricsContext; +use ethereum::chain::{EthereumAdapterSelector, EthereumStreamBuilder}; +use ethereum::{EthereumNetworks, ProviderEthRpcMetrics, RuntimeAdapter as EthereumRuntimeAdapter}; +use futures::future::join_all; +use futures::TryFutureExt; +use graph::anyhow::{bail, format_err, Error}; +use graph::blockchain::{BlockchainKind, BlockchainMap, ChainIdentifier}; +use graph::cheap_clone::CheapClone; +use graph::components::store::{BlockStore as _, DeploymentLocator}; +use graph::env::EnvVars; +use graph::firehose::FirehoseEndpoints; +use graph::ipfs_client::IpfsClient; +use graph::prelude::{ + anyhow, tokio, BlockNumber, DeploymentHash, LoggerFactory, + MetricsRegistry as MetricsRegistryTrait, NodeId, SubgraphAssignmentProvider, SubgraphName, + SubgraphRegistrar, SubgraphStore, SubgraphVersionSwitchingMode, ENV_VARS, +}; +use graph::slog::{debug, error, info, o, Logger}; +use graph::util::security::SafeDisplay; +use graph_chain_ethereum::{self as ethereum, EthereumAdapterTrait, Transport}; +use graph_core::polling_monitor::ipfs_service::IpfsService; +use graph_core::{ + LinkResolver, SubgraphAssignmentProvider as IpfsSubgraphAssignmentProvider, + SubgraphInstanceManager, SubgraphRegistrar as IpfsSubgraphRegistrar, +}; +use url::Url; + +fn locate(store: &dyn SubgraphStore, hash: &str) -> Result { + let mut locators = store.locators(&hash)?; + match locators.len() { + 0 => bail!("could not find subgraph {hash} we just created"), + 1 => Ok(locators.pop().unwrap()), + n => bail!("there are {n} subgraphs with hash {hash}"), + } +} + +pub async fn run( + logger: Logger, + store_builder: StoreBuilder, + network_name: String, + ipfs_url: Vec, + config: Config, + metrics_ctx: MetricsContext, + node_id: NodeId, + subgraph: String, + stop_block: BlockNumber, +) -> Result<(), anyhow::Error> { + println!( + "Run command: starting subgraph => {}, stop_block = {}", + subgraph, stop_block + ); + + let metrics_registry = metrics_ctx.registry.clone(); + let logger_factory = LoggerFactory::new(logger.clone(), None); + + // FIXME: Hard-coded IPFS config, take it from config file instead? + let ipfs_clients: Vec<_> = create_ipfs_clients(&logger, &ipfs_url); + let ipfs_client = ipfs_clients.first().cloned().expect("Missing IPFS client"); + let ipfs_service = IpfsService::new( + ipfs_client, + ENV_VARS.mappings.max_ipfs_file_bytes as u64, + ENV_VARS.mappings.ipfs_timeout, + ENV_VARS.mappings.max_ipfs_concurrent_requests, + ); + + // Convert the clients into a link resolver. Since we want to get past + // possible temporary DNS failures, make the resolver retry + let link_resolver = Arc::new(LinkResolver::new( + ipfs_clients, + Arc::new(EnvVars::default()), + )); + + let eth_networks = create_ethereum_networks( + logger.clone(), + metrics_registry.clone(), + &config, + &network_name, + ) + .await + .expect("Failed to parse Ethereum networks"); + let firehose_networks_by_kind = create_firehose_networks(logger.clone(), &config); + let firehose_networks = firehose_networks_by_kind.get(&BlockchainKind::Ethereum); + let firehose_endpoints = firehose_networks + .and_then(|v| v.networks.get(&network_name)) + .map_or_else(|| FirehoseEndpoints::new(), |v| v.clone()); + + let eth_adapters = match eth_networks.networks.get(&network_name) { + Some(adapters) => adapters.clone(), + None => { + return Err(format_err!( + "No ethereum adapters found, but required in this state of graphman run command" + )) + } + }; + + let eth_adapters2 = eth_adapters.clone(); + + let (_, ethereum_idents) = connect_ethereum_networks(&logger, eth_networks).await; + // let (near_networks, near_idents) = connect_firehose_networks::( + // &logger, + // firehose_networks_by_kind + // .remove(&BlockchainKind::Near) + // .unwrap_or_else(|| FirehoseNetworks::new()), + // ) + // .await; + + let chain_head_update_listener = store_builder.chain_head_update_listener(); + let network_identifiers = ethereum_idents.into_iter().collect(); + let network_store = store_builder.network_store(network_identifiers); + + let subgraph_store = network_store.subgraph_store(); + let chain_store = network_store + .block_store() + .chain_store(network_name.as_ref()) + .expect(format!("No chain store for {}", &network_name).as_ref()); + + let chain = ethereum::Chain::new( + logger_factory.clone(), + network_name.clone(), + node_id.clone(), + metrics_registry.clone(), + chain_store.cheap_clone(), + chain_store.cheap_clone(), + firehose_endpoints.clone(), + eth_adapters.clone(), + chain_head_update_listener, + Arc::new(EthereumStreamBuilder {}), + Arc::new(EthereumAdapterSelector::new( + logger_factory.clone(), + Arc::new(eth_adapters), + Arc::new(firehose_endpoints.clone()), + metrics_registry.clone(), + chain_store.cheap_clone(), + )), + Arc::new(EthereumRuntimeAdapter { + call_cache: chain_store.cheap_clone(), + eth_adapters: Arc::new(eth_adapters2), + }), + ethereum::ENV_VARS.reorg_threshold, + // We assume the tested chain is always ingestible for now + true, + ); + + let mut blockchain_map = BlockchainMap::new(); + blockchain_map.insert(network_name.clone(), Arc::new(chain)); + + let static_filters = ENV_VARS.experimental_static_filters; + + let blockchain_map = Arc::new(blockchain_map); + let subgraph_instance_manager = SubgraphInstanceManager::new( + &logger_factory, + subgraph_store.clone(), + blockchain_map.clone(), + metrics_registry.clone(), + link_resolver.cheap_clone(), + ipfs_service, + static_filters, + ); + + // Create IPFS-based subgraph provider + let subgraph_provider = Arc::new(IpfsSubgraphAssignmentProvider::new( + &logger_factory, + link_resolver.cheap_clone(), + subgraph_instance_manager, + )); + + let panicking_subscription_manager = Arc::new(PanicSubscriptionManager {}); + + let subgraph_registrar = Arc::new(IpfsSubgraphRegistrar::new( + &logger_factory, + link_resolver.cheap_clone(), + subgraph_provider.clone(), + subgraph_store.clone(), + panicking_subscription_manager, + blockchain_map, + node_id.clone(), + SubgraphVersionSwitchingMode::Instant, + )); + + let (name, hash) = if subgraph.contains(':') { + let mut split = subgraph.split(':'); + (split.next().unwrap(), split.next().unwrap().to_owned()) + } else { + ("cli", subgraph) + }; + + let subgraph_name = SubgraphName::new(name) + .expect("Subgraph name must contain only a-z, A-Z, 0-9, '-' and '_'"); + let subgraph_hash = + DeploymentHash::new(hash.clone()).expect("Subgraph hash must be a valid IPFS hash"); + + info!(&logger, "Creating subgraph {}", name); + let create_result = + SubgraphRegistrar::create_subgraph(subgraph_registrar.as_ref(), subgraph_name.clone()) + .await?; + + info!( + &logger, + "Looking up subgraph deployment {} (Deployment hash => {}, id => {})", + name, + subgraph_hash, + create_result.id, + ); + + SubgraphRegistrar::create_subgraph_version( + subgraph_registrar.as_ref(), + subgraph_name.clone(), + subgraph_hash.clone(), + node_id.clone(), + None, + None, + None, + ) + .await?; + + let locator = locate(subgraph_store.as_ref(), &hash)?; + + SubgraphAssignmentProvider::start(subgraph_provider.as_ref(), locator, Some(stop_block)) + .await?; + + loop { + tokio::time::sleep(Duration::from_millis(1000)).await; + + let block_ptr = subgraph_store + .least_block_ptr(&subgraph_hash) + .await + .unwrap() + .unwrap(); + + debug!(&logger, "subgraph block: {:?}", block_ptr); + + if block_ptr.number >= stop_block { + info!( + &logger, + "subgraph now at block {}, reached stop block {}", block_ptr.number, stop_block + ); + break; + } + } + + info!(&logger, "Removing subgraph {}", name); + subgraph_store.clone().remove_subgraph(subgraph_name)?; + + if let Some(host) = metrics_ctx.prometheus_host { + let mfs = metrics_ctx.prometheus.gather(); + let job_name = match metrics_ctx.job_name { + Some(name) => name, + None => "graphman run".into(), + }; + + tokio::task::spawn_blocking(move || { + prometheus::push_metrics(&job_name, HashMap::new(), &host, mfs, None) + }) + .await??; + } + + Ok(()) +} + +// Stuff copied directly moslty from `main.rs` +// +// FIXME: Share that with `main.rs` stuff + +// The status of a provider that we learned from connecting to it +#[derive(PartialEq)] +enum ProviderNetworkStatus { + Broken { + network: String, + provider: String, + }, + Version { + network: String, + ident: ChainIdentifier, + }, +} + +/// How long we will hold up node startup to get the net version and genesis +/// hash from the client. If we can't get it within that time, we'll try and +/// continue regardless. +const NET_VERSION_WAIT_TIME: Duration = Duration::from_secs(30); + +fn create_ipfs_clients(logger: &Logger, ipfs_addresses: &Vec) -> Vec { + // Parse the IPFS URL from the `--ipfs` command line argument + let ipfs_addresses: Vec<_> = ipfs_addresses + .iter() + .map(|uri| { + if uri.starts_with("http://") || uri.starts_with("https://") { + String::from(uri) + } else { + format!("http://{}", uri) + } + }) + .collect(); + + ipfs_addresses + .into_iter() + .map(|ipfs_address| { + info!( + logger, + "Trying IPFS node at: {}", + SafeDisplay(&ipfs_address) + ); + + let ipfs_client = match IpfsClient::new(&ipfs_address) { + Ok(ipfs_client) => ipfs_client, + Err(e) => { + error!( + logger, + "Failed to create IPFS client for `{}`: {}", + SafeDisplay(&ipfs_address), + e + ); + panic!("Could not connect to IPFS"); + } + }; + + // Test the IPFS client by getting the version from the IPFS daemon + let ipfs_test = ipfs_client.cheap_clone(); + let ipfs_ok_logger = logger.clone(); + let ipfs_err_logger = logger.clone(); + let ipfs_address_for_ok = ipfs_address.clone(); + let ipfs_address_for_err = ipfs_address.clone(); + graph::spawn(async move { + ipfs_test + .test() + .map_err(move |e| { + error!( + ipfs_err_logger, + "Is there an IPFS node running at \"{}\"?", + SafeDisplay(ipfs_address_for_err), + ); + panic!("Failed to connect to IPFS: {}", e); + }) + .map_ok(move |_| { + info!( + ipfs_ok_logger, + "Successfully connected to IPFS node at: {}", + SafeDisplay(ipfs_address_for_ok) + ); + }) + .await + }); + + ipfs_client + }) + .collect() +} + +/// Parses an Ethereum connection string and returns the network name and Ethereum adapter. +pub async fn create_ethereum_networks( + logger: Logger, + registry: Arc, + config: &Config, + network_name: &str, +) -> Result { + let eth_rpc_metrics = Arc::new(ProviderEthRpcMetrics::new(registry)); + let mut parsed_networks = EthereumNetworks::new(); + let chain = config + .chains + .chains + .get(network_name) + .ok_or_else(|| anyhow!("unknown network {}", network_name))?; + if chain.protocol == BlockchainKind::Ethereum { + for provider in &chain.providers { + if let ProviderDetails::Web3(web3) = &provider.details { + let capabilities = web3.node_capabilities(); + + let logger = logger.new(o!("provider" => provider.label.clone())); + info!( + logger, + "Creating transport"; + "url" => &web3.url, + "capabilities" => capabilities + ); + + use crate::config::Transport::*; + + let transport = match web3.transport { + Rpc => Transport::new_rpc(Url::parse(&web3.url)?, web3.headers.clone()), + Ipc => Transport::new_ipc(&web3.url).await, + Ws => Transport::new_ws(&web3.url).await, + }; + + let supports_eip_1898 = !web3.features.contains("no_eip1898"); + + parsed_networks.insert( + network_name.to_string(), + capabilities, + Arc::new( + graph_chain_ethereum::EthereumAdapter::new( + logger, + provider.label.clone(), + &web3.url, + transport, + eth_rpc_metrics.clone(), + supports_eip_1898, + ) + .await, + ), + web3.limit_for(&config.node), + ); + } + } + } + parsed_networks.sort(); + Ok(parsed_networks) +} + +/// Try to connect to all the providers in `eth_networks` and get their net +/// version and genesis block. Return the same `eth_networks` and the +/// retrieved net identifiers grouped by network name. Remove all providers +/// for which trying to connect resulted in an error from the returned +/// `EthereumNetworks`, since it's likely pointless to try and connect to +/// them. If the connection attempt to a provider times out after +/// `NET_VERSION_WAIT_TIME`, keep the provider, but don't report a +/// version for it. +async fn connect_ethereum_networks( + logger: &Logger, + mut eth_networks: EthereumNetworks, +) -> (EthereumNetworks, Vec<(String, Vec)>) { + // This has one entry for each provider, and therefore multiple entries + // for each network + let statuses = join_all( + eth_networks + .flatten() + .into_iter() + .map(|(network_name, capabilities, eth_adapter)| { + (network_name, capabilities, eth_adapter, logger.clone()) + }) + .map(|(network, capabilities, eth_adapter, logger)| async move { + let logger = logger.new(o!("provider" => eth_adapter.provider().to_string())); + info!( + logger, "Connecting to Ethereum to get network identifier"; + "capabilities" => &capabilities + ); + match tokio::time::timeout(NET_VERSION_WAIT_TIME, eth_adapter.net_identifiers()) + .await + .map_err(Error::from) + { + // An `Err` means a timeout, an `Ok(Err)` means some other error (maybe a typo + // on the URL) + Ok(Err(e)) | Err(e) => { + error!(logger, "Connection to provider failed. Not using this provider"; + "error" => e.to_string()); + ProviderNetworkStatus::Broken { + network, + provider: eth_adapter.provider().to_string(), + } + } + Ok(Ok(ident)) => { + info!( + logger, + "Connected to Ethereum"; + "network_version" => &ident.net_version, + "capabilities" => &capabilities + ); + ProviderNetworkStatus::Version { network, ident } + } + } + }), + ) + .await; + + // Group identifiers by network name + let idents: HashMap> = + statuses + .into_iter() + .fold(HashMap::new(), |mut networks, status| { + match status { + ProviderNetworkStatus::Broken { network, provider } => { + eth_networks.remove(&network, &provider) + } + ProviderNetworkStatus::Version { network, ident } => { + networks.entry(network.to_string()).or_default().push(ident) + } + } + networks + }); + let idents: Vec<_> = idents.into_iter().collect(); + (eth_networks, idents) +} diff --git a/node/src/manager/commands/stats.rs b/node/src/manager/commands/stats.rs new file mode 100644 index 0000000..ca07cb1 --- /dev/null +++ b/node/src/manager/commands/stats.rs @@ -0,0 +1,126 @@ +use std::collections::HashMap; +use std::collections::HashSet; +use std::sync::Arc; + +use crate::manager::deployment::DeploymentSearch; +use diesel::r2d2::ConnectionManager; +use diesel::r2d2::PooledConnection; +use diesel::PgConnection; +use graph::components::store::VersionStats; +use graph::prelude::anyhow; +use graph_store_postgres::command_support::catalog as store_catalog; +use graph_store_postgres::command_support::catalog::Site; +use graph_store_postgres::connection_pool::ConnectionPool; +use graph_store_postgres::Shard; +use graph_store_postgres::SubgraphStore; +use graph_store_postgres::PRIMARY_SHARD; + +fn site_and_conn( + pools: HashMap, + search: &DeploymentSearch, +) -> Result<(Site, PooledConnection>), anyhow::Error> { + let primary_pool = pools.get(&*PRIMARY_SHARD).unwrap(); + let locator = search.locate_unique(primary_pool)?; + + let conn = primary_pool.get()?; + let conn = store_catalog::Connection::new(conn); + + let site = conn + .locate_site(locator)? + .ok_or_else(|| anyhow!("deployment `{}` does not exist", search))?; + + let conn = pools.get(&site.shard).unwrap().get()?; + + Ok((site, conn)) +} + +pub async fn account_like( + store: Arc, + primary_pool: ConnectionPool, + clear: bool, + search: &DeploymentSearch, + table: String, +) -> Result<(), anyhow::Error> { + let locator = search.locate_unique(&primary_pool)?; + + store.set_account_like(&locator, &table, !clear).await?; + let clear_text = if clear { "cleared" } else { "set" }; + println!("{}: account-like flag {}", table, clear_text); + + Ok(()) +} + +pub fn abbreviate_table_name(table: &str, size: usize) -> String { + if table.len() > size { + let fragment = size / 2 - 2; + let last = table.len() - fragment; + let mut table = table.to_string(); + table.replace_range(fragment..last, ".."); + let table = table.trim().to_string(); + table + } else { + table.to_string() + } +} + +pub fn show_stats( + stats: &[VersionStats], + account_like: HashSet, +) -> Result<(), anyhow::Error> { + fn header() { + println!( + "{:^30} | {:^10} | {:^10} | {:^7}", + "table", "entities", "versions", "ratio" + ); + println!("{:-^30}-+-{:-^10}-+-{:-^10}-+-{:-^7}", "", "", "", ""); + } + + fn footer() { + println!(" (a): account-like flag set"); + } + + fn print_stats(s: &VersionStats, account_like: bool) { + println!( + "{:<26} {:3} | {:>10} | {:>10} | {:>5.1}%", + abbreviate_table_name(&s.tablename, 26), + if account_like { "(a)" } else { " " }, + s.entities, + s.versions, + s.ratio * 100.0 + ); + } + + header(); + for s in stats { + print_stats(s, account_like.contains(&s.tablename)); + } + if !account_like.is_empty() { + footer(); + } + + Ok(()) +} + +pub fn show( + pools: HashMap, + search: &DeploymentSearch, +) -> Result<(), anyhow::Error> { + let (site, conn) = site_and_conn(pools, search)?; + + let stats = store_catalog::stats(&conn, &site.namespace)?; + + let account_like = store_catalog::account_like(&conn, &site)?; + + show_stats(stats.as_slice(), account_like) +} + +pub fn analyze( + store: Arc, + pool: ConnectionPool, + search: DeploymentSearch, + entity_name: &str, +) -> Result<(), anyhow::Error> { + let locator = search.locate_unique(&pool)?; + println!("Analyzing table sgd{}.{entity_name}", locator.id); + store.analyze(&locator, entity_name).map_err(|e| anyhow!(e)) +} diff --git a/node/src/manager/commands/txn_speed.rs b/node/src/manager/commands/txn_speed.rs new file mode 100644 index 0000000..795483b --- /dev/null +++ b/node/src/manager/commands/txn_speed.rs @@ -0,0 +1,58 @@ +use diesel::PgConnection; +use std::{collections::HashMap, thread::sleep, time::Duration}; + +use graph::prelude::anyhow; +use graph_store_postgres::connection_pool::ConnectionPool; + +use crate::manager::catalog; + +pub fn run(pool: ConnectionPool, delay: u64) -> Result<(), anyhow::Error> { + fn query(conn: &PgConnection) -> Result, anyhow::Error> { + use catalog::pg_catalog::pg_stat_database as d; + use diesel::dsl::*; + use diesel::sql_types::BigInt; + use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; + + let rows = d::table + .filter(d::datname.eq(any(vec!["explorer", "graph"]))) + .select(( + d::datname, + sql::("(xact_commit + xact_rollback)::bigint"), + sql::("txid_current()::bigint"), + )) + //.select((d::datname)) + .load::<(Option, i64, i64)>(conn)?; + Ok(rows + .into_iter() + .map(|(datname, all_txn, write_txn)| { + (datname.unwrap_or("none".to_string()), all_txn, write_txn) + }) + .collect()) + } + + let mut speeds = HashMap::new(); + let conn = pool.get()?; + for (datname, all_txn, write_txn) in query(&conn)? { + speeds.insert(datname, (all_txn, write_txn)); + } + println!( + "Looking for number of transactions performed in {}s ...", + delay + ); + sleep(Duration::from_secs(delay)); + println!("Number of transactions/minute"); + println!("{:10} {:>7} {}", "database", "all", "write"); + for (datname, all_txn, write_txn) in query(&conn)? { + let (all_speed, write_speed) = speeds + .get(&datname) + .map(|(all_txn_old, write_txn_old)| { + (all_txn - *all_txn_old, write_txn - *write_txn_old) + }) + .unwrap_or((0, 0)); + let all_speed = all_speed as f64 * 60.0 / delay as f64; + let write_speed = write_speed as f64 * 60.0 / delay as f64; + println!("{:10} {:>7} {}", datname, all_speed, write_speed); + } + + Ok(()) +} diff --git a/node/src/manager/commands/unused_deployments.rs b/node/src/manager/commands/unused_deployments.rs new file mode 100644 index 0000000..632b205 --- /dev/null +++ b/node/src/manager/commands/unused_deployments.rs @@ -0,0 +1,139 @@ +use std::{sync::Arc, time::Instant}; + +use graph::prelude::{anyhow::Error, chrono}; +use graph_store_postgres::{unused, SubgraphStore, UnusedDeployment}; + +use crate::manager::display::List; + +fn make_list() -> List { + List::new(vec!["id", "shard", "namespace", "subgraphs", "entities"]) +} + +fn add_row(list: &mut List, deployment: UnusedDeployment) { + let UnusedDeployment { + id, + shard, + namespace, + subgraphs, + entity_count, + .. + } = deployment; + let subgraphs = subgraphs.unwrap_or(vec![]).join(", "); + + list.append(vec![ + id.to_string(), + shard, + namespace, + subgraphs, + entity_count.to_string(), + ]) +} + +pub fn list(store: Arc, existing: bool) -> Result<(), Error> { + let mut list = make_list(); + + let filter = if existing { + unused::Filter::New + } else { + unused::Filter::All + }; + + for deployment in store.list_unused_deployments(filter)? { + add_row(&mut list, deployment); + } + + if list.is_empty() { + println!("no unused deployments"); + } else { + list.render(); + } + + Ok(()) +} + +pub fn record(store: Arc) -> Result<(), Error> { + let mut list = make_list(); + + println!("Recording unused deployments. This might take a while."); + let recorded = store.record_unused_deployments()?; + + for unused in store.list_unused_deployments(unused::Filter::New)? { + if recorded + .iter() + .find(|r| r.deployment == unused.deployment) + .is_some() + { + add_row(&mut list, unused); + } + } + + list.render(); + println!("Recorded {} unused deployments", recorded.len()); + + Ok(()) +} + +pub fn remove( + store: Arc, + count: usize, + deployment: Option, + older: Option, +) -> Result<(), Error> { + let filter = match older { + Some(duration) => unused::Filter::UnusedLongerThan(duration), + None => unused::Filter::New, + }; + let unused = store.list_unused_deployments(filter)?; + let unused = match &deployment { + None => unused, + Some(deployment) => unused + .into_iter() + .filter(|u| u.deployment.as_str() == deployment) + .collect::>(), + }; + + if unused.is_empty() { + match &deployment { + Some(s) => println!("No unused subgraph matches `{}`", s), + None => println!("Nothing to remove."), + } + return Ok(()); + } + + for (i, deployment) in unused.iter().take(count).enumerate() { + println!("{:=<36} {:4} {:=<36}", "", i + 1, ""); + println!( + "removing {} from {}", + deployment.namespace, deployment.shard + ); + println!(" {:>14}: {}", "deployment id", deployment.deployment); + println!(" {:>14}: {}", "entities", deployment.entity_count); + if let Some(subgraphs) = &deployment.subgraphs { + let mut first = true; + for name in subgraphs { + if first { + println!(" {:>14}: {}", "subgraphs", name); + } else { + println!(" {:>14} {}", "", name); + } + first = false; + } + } + + let start = Instant::now(); + match store.remove_deployment(deployment.id) { + Ok(()) => { + println!( + "done removing {} from {} in {:.1}s\n", + deployment.namespace, + deployment.shard, + start.elapsed().as_millis() as f64 / 1000.0 + ); + } + Err(e) => { + println!("removal failed: {}", e) + } + } + } + Ok(()) +} diff --git a/node/src/manager/deployment.rs b/node/src/manager/deployment.rs new file mode 100644 index 0000000..db17db1 --- /dev/null +++ b/node/src/manager/deployment.rs @@ -0,0 +1,219 @@ +use std::collections::HashSet; +use std::fmt; +use std::str::FromStr; + +use diesel::{dsl::sql, prelude::*}; +use diesel::{sql_types::Text, PgConnection}; +use regex::Regex; + +use graph::components::store::DeploymentId; +use graph::{ + components::store::DeploymentLocator, + data::subgraph::status, + prelude::{ + anyhow::{self}, + lazy_static, DeploymentHash, + }, +}; +use graph_store_postgres::command_support::catalog as store_catalog; +use graph_store_postgres::connection_pool::ConnectionPool; + +use crate::manager::display::List; + +lazy_static! { + // `Qm...` optionally follow by `:$shard` + static ref HASH_RE: Regex = Regex::new("\\A(?PQm[^:]+)(:(?P[a-z0-9_]+))?\\z").unwrap(); + // `sgdNNN` + static ref DEPLOYMENT_RE: Regex = Regex::new("\\A(?Psgd[0-9]+)\\z").unwrap(); +} + +/// A search for one or multiple deployments to make it possible to search +/// by subgraph name, IPFS hash, or namespace. Since there can be multiple +/// deployments for the same IPFS hash, the search term for a hash can +/// optionally specify a shard. +#[derive(Clone, Debug)] +pub enum DeploymentSearch { + Name { name: String }, + Hash { hash: String, shard: Option }, + Deployment { namespace: String }, +} + +impl fmt::Display for DeploymentSearch { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DeploymentSearch::Name { name } => write!(f, "{}", name), + DeploymentSearch::Hash { + hash, + shard: Some(shard), + } => write!(f, "{}:{}", hash, shard), + DeploymentSearch::Hash { hash, shard: None } => write!(f, "{}", hash), + DeploymentSearch::Deployment { namespace } => write!(f, "{}", namespace), + } + } +} + +impl FromStr for DeploymentSearch { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + if let Some(caps) = HASH_RE.captures(s) { + let hash = caps.name("hash").unwrap().as_str().to_string(); + let shard = caps.name("shard").map(|shard| shard.as_str().to_string()); + Ok(DeploymentSearch::Hash { hash, shard }) + } else if let Some(caps) = DEPLOYMENT_RE.captures(s) { + let namespace = caps.name("nsp").unwrap().as_str().to_string(); + Ok(DeploymentSearch::Deployment { namespace }) + } else { + Ok(DeploymentSearch::Name { + name: s.to_string(), + }) + } + } +} + +impl DeploymentSearch { + pub fn lookup(&self, primary: &ConnectionPool) -> Result, anyhow::Error> { + let conn = primary.get()?; + self.lookup_with_conn(&conn) + } + + pub fn lookup_with_conn(&self, conn: &PgConnection) -> Result, anyhow::Error> { + use store_catalog::deployment_schemas as ds; + use store_catalog::subgraph as s; + use store_catalog::subgraph_deployment_assignment as a; + use store_catalog::subgraph_version as v; + + let query = ds::table + .inner_join(v::table.on(v::deployment.eq(ds::subgraph))) + .inner_join(s::table.on(v::subgraph.eq(s::id))) + .left_outer_join(a::table.on(a::id.eq(ds::id))) + .select(( + s::name, + sql::( + "(case + when subgraphs.subgraph.pending_version = subgraphs.subgraph_version.id then 'pending' + when subgraphs.subgraph.current_version = subgraphs.subgraph_version.id then 'current' + else 'unused' end) status", + ), + v::deployment, + ds::name, + ds::id, + a::node_id.nullable(), + ds::shard, + ds::network, + ds::active, + )); + + let deployments: Vec = match self { + DeploymentSearch::Name { name } => { + let pattern = format!("%{}%", name); + query.filter(s::name.ilike(&pattern)).load(conn)? + } + DeploymentSearch::Hash { hash, shard } => { + let query = query.filter(ds::subgraph.eq(&hash)); + match shard { + Some(shard) => query.filter(ds::shard.eq(shard)).load(conn)?, + None => query.load(conn)?, + } + } + DeploymentSearch::Deployment { namespace } => { + query.filter(ds::name.eq(&namespace)).load(conn)? + } + }; + Ok(deployments) + } + + /// Finds a single deployment locator for the given deployment identifier. + pub fn locate_unique(&self, pool: &ConnectionPool) -> anyhow::Result { + let mut locators: Vec = HashSet::::from_iter( + self.lookup(pool)? + .into_iter() + .map(|deployment| deployment.locator()), + ) + .into_iter() + .collect(); + let deployment_locator = match locators.len() { + 0 => anyhow::bail!("Found no deployment for `{}`", self), + 1 => locators.pop().unwrap(), + n => anyhow::bail!("Found {} deployments for `{}`", n, self), + }; + Ok(deployment_locator) + } +} + +#[derive(Queryable, PartialEq, Eq, Hash, Debug)] +pub struct Deployment { + pub name: String, + pub status: String, + pub deployment: String, + pub namespace: String, + pub id: i32, + pub node_id: Option, + pub shard: String, + pub chain: String, + pub active: bool, +} + +impl Deployment { + pub fn locator(&self) -> DeploymentLocator { + DeploymentLocator::new( + DeploymentId(self.id), + DeploymentHash::new(self.deployment.clone()).unwrap(), + ) + } + + pub fn print_table(deployments: Vec, statuses: Vec) { + let mut rows = vec![ + "name", + "status", + "id", + "namespace", + "shard", + "active", + "chain", + "node_id", + ]; + if !statuses.is_empty() { + rows.extend(vec!["synced", "health", "latest block", "chain head block"]); + } + + let mut list = List::new(rows); + + for deployment in deployments { + let status = statuses + .iter() + .find(|status| &status.id.0 == &deployment.id); + + let mut rows = vec![ + deployment.name, + deployment.status, + deployment.deployment, + deployment.namespace, + deployment.shard, + deployment.active.to_string(), + deployment.chain, + deployment.node_id.unwrap_or("---".to_string()), + ]; + if let Some(status) = status { + let chain = &status.chains[0]; + rows.extend(vec![ + status.synced.to_string(), + status.health.as_str().to_string(), + chain + .latest_block + .as_ref() + .map(|b| b.number().to_string()) + .unwrap_or("-".to_string()), + chain + .chain_head_block + .as_ref() + .map(|b| b.number().to_string()) + .unwrap_or("-".to_string()), + ]) + } + list.append(rows); + } + + list.render(); + } +} diff --git a/node/src/manager/display.rs b/node/src/manager/display.rs new file mode 100644 index 0000000..694eaf6 --- /dev/null +++ b/node/src/manager/display.rs @@ -0,0 +1,54 @@ +pub struct List { + pub headers: Vec, + pub rows: Vec>, +} + +impl List { + pub fn new(headers: Vec<&str>) -> Self { + let headers = headers.into_iter().map(|s| s.to_string()).collect(); + Self { + headers, + rows: Vec::new(), + } + } + + pub fn append(&mut self, row: Vec) { + if row.len() != self.headers.len() { + panic!( + "there are {} headers but the row has {} entries: {:?}", + self.headers.len(), + row.len(), + row + ); + } + self.rows.push(row); + } + + pub fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + pub fn render(&self) { + const LINE_WIDTH: usize = 78; + + let header_width = self.headers.iter().map(|h| h.len()).max().unwrap_or(0); + let header_width = if header_width < 5 { 5 } else { header_width }; + let mut first = true; + for row in &self.rows { + if !first { + println!( + "{:-) -> StoreEventStreamBox { + panic!("we were never meant to call `subscribe`"); + } + + fn subscribe_no_payload(&self, _: BTreeSet) -> UnitStream { + panic!("we were never meant to call `subscribe_no_payload`"); + } +} diff --git a/node/src/opt.rs b/node/src/opt.rs new file mode 100644 index 0000000..c40c1c5 --- /dev/null +++ b/node/src/opt.rs @@ -0,0 +1,259 @@ +use clap::Parser; +use git_testament::{git_testament, render_testament}; +use lazy_static::lazy_static; + +use crate::config; + +git_testament!(TESTAMENT); +lazy_static! { + static ref RENDERED_TESTAMENT: String = render_testament!(TESTAMENT); +} + +#[derive(Clone, Debug, Parser)] +#[clap( + name = "graph-node", + about = "Scalable queries for a decentralized future", + author = "Graph Protocol, Inc.", + version = RENDERED_TESTAMENT.as_str() +)] +pub struct Opt { + #[clap( + long, + env = "GRAPH_NODE_CONFIG", + conflicts_with_all = &["postgres-url", "postgres-secondary-hosts", "postgres-host-weights"], + required_unless = "postgres-url", + help = "the name of the configuration file", + )] + pub config: Option, + #[clap(long, help = "validate the configuration and exit")] + pub check_config: bool, + #[clap( + long, + value_name = "[NAME:]IPFS_HASH", + env = "SUBGRAPH", + help = "name and IPFS hash of the subgraph manifest" + )] + pub subgraph: Option, + + #[clap( + long, + value_name = "BLOCK_HASH:BLOCK_NUMBER", + help = "block hash and number that the subgraph passed will start indexing at" + )] + pub start_block: Option, + + #[clap( + long, + value_name = "URL", + env = "POSTGRES_URL", + conflicts_with = "config", + required_unless = "config", + help = "Location of the Postgres database used for storing entities" + )] + pub postgres_url: Option, + #[clap( + long, + value_name = "URL,", + use_delimiter = true, + env = "GRAPH_POSTGRES_SECONDARY_HOSTS", + conflicts_with = "config", + help = "Comma-separated list of host names/IP's for read-only Postgres replicas, \ + which will share the load with the primary server" + )] + // FIXME: Make sure delimiter is ',' + pub postgres_secondary_hosts: Vec, + #[clap( + long, + value_name = "WEIGHT,", + use_delimiter = true, + env = "GRAPH_POSTGRES_HOST_WEIGHTS", + conflicts_with = "config", + help = "Comma-separated list of relative weights for selecting the main database \ + and secondary databases. The list is in the order MAIN,REPLICA1,REPLICA2,...\ + A host will receive approximately WEIGHT/SUM(WEIGHTS) fraction of total queries. \ + Defaults to weight 1 for each host" + )] + pub postgres_host_weights: Vec, + #[clap( + long, + min_values=0, + required_unless_one = &["ethereum-ws", "ethereum-ipc", "config"], + conflicts_with_all = &["ethereum-ws", "ethereum-ipc", "config"], + value_name="NETWORK_NAME:[CAPABILITIES]:URL", + env="ETHEREUM_RPC", + help= "Ethereum network name (e.g. 'mainnet'), optional comma-seperated capabilities (eg 'full,archive'), and an Ethereum RPC URL, separated by a ':'", + )] + pub ethereum_rpc: Vec, + #[clap(long, min_values=0, + required_unless_one = &["ethereum-rpc", "ethereum-ipc", "config"], + conflicts_with_all = &["ethereum-rpc", "ethereum-ipc", "config"], + value_name="NETWORK_NAME:[CAPABILITIES]:URL", + env="ETHEREUM_WS", + help= "Ethereum network name (e.g. 'mainnet'), optional comma-seperated capabilities (eg 'full,archive`, and an Ethereum WebSocket URL, separated by a ':'", + )] + pub ethereum_ws: Vec, + #[clap(long, min_values=0, + required_unless_one = &["ethereum-rpc", "ethereum-ws", "config"], + conflicts_with_all = &["ethereum-rpc", "ethereum-ws", "config"], + value_name="NETWORK_NAME:[CAPABILITIES]:FILE", + env="ETHEREUM_IPC", + help= "Ethereum network name (e.g. 'mainnet'), optional comma-seperated capabilities (eg 'full,archive'), and an Ethereum IPC pipe, separated by a ':'", + )] + pub ethereum_ipc: Vec, + #[clap( + long, + value_name = "HOST:PORT", + env = "IPFS", + help = "HTTP addresses of IPFS nodes" + )] + pub ipfs: Vec, + #[clap( + long, + default_value = "8000", + value_name = "PORT", + help = "Port for the GraphQL HTTP server", + env = "GRAPH_GRAPHQL_HTTP_PORT" + )] + pub http_port: u16, + #[clap( + long, + default_value = "8030", + value_name = "PORT", + help = "Port for the index node server" + )] + pub index_node_port: u16, + #[clap( + long, + default_value = "8001", + value_name = "PORT", + help = "Port for the GraphQL WebSocket server", + env = "GRAPH_GRAPHQL_WS_PORT" + )] + pub ws_port: u16, + #[clap( + long, + default_value = "8020", + value_name = "PORT", + help = "Port for the JSON-RPC admin server" + )] + pub admin_port: u16, + #[clap( + long, + default_value = "8040", + value_name = "PORT", + help = "Port for the Prometheus metrics server" + )] + pub metrics_port: u16, + #[clap( + long, + default_value = "default", + value_name = "NODE_ID", + env = "GRAPH_NODE_ID", + help = "a unique identifier for this node. Should have the same value between consecutive node restarts" + )] + pub node_id: String, + #[clap( + long, + value_name = "FILE", + env = "GRAPH_NODE_EXPENSIVE_QUERIES_FILE", + default_value = "/etc/graph-node/expensive-queries.txt", + help = "a file with a list of expensive queries, one query per line. Attempts to run these queries will return a QueryExecutionError::TooExpensive to clients" + )] + pub expensive_queries_filename: String, + #[clap(long, help = "Enable debug logging")] + pub debug: bool, + + #[clap( + long, + value_name = "URL", + env = "ELASTICSEARCH_URL", + help = "Elasticsearch service to write subgraph logs to" + )] + pub elasticsearch_url: Option, + #[clap( + long, + value_name = "USER", + env = "ELASTICSEARCH_USER", + help = "User to use for Elasticsearch logging" + )] + pub elasticsearch_user: Option, + #[clap( + long, + value_name = "PASSWORD", + env = "ELASTICSEARCH_PASSWORD", + hide_env_values = true, + help = "Password to use for Elasticsearch logging" + )] + pub elasticsearch_password: Option, + #[clap( + long, + value_name = "MILLISECONDS", + default_value = "1000", + env = "ETHEREUM_POLLING_INTERVAL", + help = "How often to poll the Ethereum node for new blocks" + )] + pub ethereum_polling_interval: u64, + #[clap( + long, + value_name = "DISABLE_BLOCK_INGESTOR", + env = "DISABLE_BLOCK_INGESTOR", + help = "Ensures that the block ingestor component does not execute" + )] + pub disable_block_ingestor: bool, + #[clap( + long, + value_name = "STORE_CONNECTION_POOL_SIZE", + default_value = "10", + env = "STORE_CONNECTION_POOL_SIZE", + help = "Limits the number of connections in the store's connection pool" + )] + pub store_connection_pool_size: u32, + #[clap( + long, + help = "Allows setting configurations that may result in incorrect Proofs of Indexing." + )] + pub unsafe_config: bool, + + #[clap( + long, + value_name = "IPFS_HASH", + help = "IPFS hash of the subgraph manifest that you want to fork" + )] + pub debug_fork: Option, + + #[clap(long, value_name = "URL", help = "Base URL for forking subgraphs")] + pub fork_base: Option, +} + +impl From for config::Opt { + fn from(opt: Opt) -> Self { + let Opt { + postgres_url, + config, + store_connection_pool_size, + postgres_host_weights, + postgres_secondary_hosts, + disable_block_ingestor, + node_id, + ethereum_rpc, + ethereum_ws, + ethereum_ipc, + unsafe_config, + .. + } = opt; + + config::Opt { + postgres_url, + config, + store_connection_pool_size, + postgres_host_weights, + postgres_secondary_hosts, + disable_block_ingestor, + node_id, + ethereum_rpc, + ethereum_ws, + ethereum_ipc, + unsafe_config, + } + } +} diff --git a/node/src/store_builder.rs b/node/src/store_builder.rs new file mode 100644 index 0000000..205f182 --- /dev/null +++ b/node/src/store_builder.rs @@ -0,0 +1,292 @@ +use std::iter::FromIterator; +use std::{collections::HashMap, sync::Arc}; + +use futures::future::join_all; +use graph::blockchain::ChainIdentifier; +use graph::prelude::{o, MetricsRegistry, NodeId}; +use graph::url::Url; +use graph::{ + prelude::{info, CheapClone, Logger}, + util::security::SafeDisplay, +}; +use graph_store_postgres::connection_pool::{ConnectionPool, ForeignServer, PoolName}; +use graph_store_postgres::{ + BlockStore as DieselBlockStore, ChainHeadUpdateListener as PostgresChainHeadUpdateListener, + NotificationSender, Shard as ShardName, Store as DieselStore, SubgraphStore, + SubscriptionManager, PRIMARY_SHARD, +}; + +use crate::config::{Config, Shard}; + +pub struct StoreBuilder { + logger: Logger, + subgraph_store: Arc, + pools: HashMap, + subscription_manager: Arc, + chain_head_update_listener: Arc, + /// Map network names to the shards where they are/should be stored + chains: HashMap, +} + +impl StoreBuilder { + /// Set up all stores, and run migrations. This does a complete store + /// setup whereas other methods here only get connections for an already + /// initialized store + pub async fn new( + logger: &Logger, + node: &NodeId, + config: &Config, + fork_base: Option, + registry: Arc, + ) -> Self { + let primary_shard = config.primary_store().clone(); + + let subscription_manager = Arc::new(SubscriptionManager::new( + logger.cheap_clone(), + primary_shard.connection.to_owned(), + registry.clone(), + )); + + let (store, pools) = Self::make_subgraph_store_and_pools( + logger, + node, + config, + fork_base, + registry.cheap_clone(), + ); + + // Try to perform setup (migrations etc.) for all the pools. If this + // attempt doesn't work for all of them because the database is + // unavailable, they will try again later in the normal course of + // using the pool + join_all(pools.iter().map(|(_, pool)| pool.setup())).await; + + let chains = HashMap::from_iter(config.chains.chains.iter().map(|(name, chain)| { + let shard = ShardName::new(chain.shard.to_string()) + .expect("config validation catches invalid names"); + (name.to_string(), shard) + })); + + let chain_head_update_listener = Arc::new(PostgresChainHeadUpdateListener::new( + &logger, + registry.cheap_clone(), + primary_shard.connection.to_owned(), + )); + + Self { + logger: logger.cheap_clone(), + subgraph_store: store, + pools, + subscription_manager, + chain_head_update_listener, + chains, + } + } + + /// Make a `ShardedStore` across all configured shards, and also return + /// the main connection pools for each shard, but not any pools for + /// replicas + pub fn make_subgraph_store_and_pools( + logger: &Logger, + node: &NodeId, + config: &Config, + fork_base: Option, + registry: Arc, + ) -> (Arc, HashMap) { + let notification_sender = Arc::new(NotificationSender::new(registry.cheap_clone())); + + let servers = config + .stores + .iter() + .map(|(name, shard)| ForeignServer::new_from_raw(name.to_string(), &shard.connection)) + .collect::, _>>() + .expect("connection url's contain enough detail"); + let servers = Arc::new(servers); + + let shards: Vec<_> = config + .stores + .iter() + .map(|(name, shard)| { + let logger = logger.new(o!("shard" => name.to_string())); + let conn_pool = Self::main_pool( + &logger, + node, + name, + shard, + registry.cheap_clone(), + servers.clone(), + ); + + let (read_only_conn_pools, weights) = Self::replica_pools( + &logger, + node, + name, + shard, + registry.cheap_clone(), + servers.clone(), + ); + + let name = + ShardName::new(name.to_string()).expect("shard names have been validated"); + (name, conn_pool, read_only_conn_pools, weights) + }) + .collect(); + + let pools: HashMap<_, _> = HashMap::from_iter( + shards + .iter() + .map(|(name, pool, _, _)| (name.clone(), pool.clone())), + ); + + let store = Arc::new(SubgraphStore::new( + logger, + shards, + Arc::new(config.deployment.clone()), + notification_sender, + fork_base, + registry, + )); + + (store, pools) + } + + pub fn make_store( + logger: &Logger, + pools: HashMap, + subgraph_store: Arc, + chains: HashMap, + networks: Vec<(String, Vec)>, + ) -> Arc { + let networks = networks + .into_iter() + .map(|(name, idents)| { + let shard = chains.get(&name).unwrap_or(&*PRIMARY_SHARD).clone(); + (name, idents, shard) + }) + .collect(); + + let logger = logger.new(o!("component" => "BlockStore")); + + let block_store = Arc::new( + DieselBlockStore::new( + logger, + networks, + pools.clone(), + subgraph_store.notification_sender(), + ) + .expect("Creating the BlockStore works"), + ); + block_store + .update_db_version() + .expect("Updating `db_version` works"); + + Arc::new(DieselStore::new(subgraph_store, block_store)) + } + + /// Create a connection pool for the main database of the primary shard + /// without connecting to all the other configured databases + pub fn main_pool( + logger: &Logger, + node: &NodeId, + name: &str, + shard: &Shard, + registry: Arc, + servers: Arc>, + ) -> ConnectionPool { + let logger = logger.new(o!("pool" => "main")); + let pool_size = shard.pool_size.size_for(node, name).expect(&format!( + "cannot determine the pool size for store {}", + name + )); + let fdw_pool_size = shard.fdw_pool_size.size_for(node, name).expect(&format!( + "cannot determine the fdw pool size for store {}", + name + )); + info!( + logger, + "Connecting to Postgres"; + "url" => SafeDisplay(shard.connection.as_str()), + "conn_pool_size" => pool_size, + "weight" => shard.weight + ); + ConnectionPool::create( + name, + PoolName::Main, + shard.connection.to_owned(), + pool_size, + Some(fdw_pool_size), + &logger, + registry.cheap_clone(), + servers, + ) + } + + /// Create connection pools for each of the replicas + fn replica_pools( + logger: &Logger, + node: &NodeId, + name: &str, + shard: &Shard, + registry: Arc, + servers: Arc>, + ) -> (Vec, Vec) { + let mut weights: Vec<_> = vec![shard.weight]; + ( + shard + .replicas + .values() + .enumerate() + .map(|(i, replica)| { + let pool = format!("replica{}", i + 1); + let logger = logger.new(o!("pool" => pool.clone())); + info!( + &logger, + "Connecting to Postgres (read replica {})", i+1; + "url" => SafeDisplay(replica.connection.as_str()), + "weight" => replica.weight + ); + weights.push(replica.weight); + let pool_size = replica.pool_size.size_for(node, name).expect(&format!( + "we can determine the pool size for replica {}", + name + )); + ConnectionPool::create( + name, + PoolName::Replica(pool), + replica.connection.clone(), + pool_size, + None, + &logger, + registry.cheap_clone(), + servers.clone(), + ) + }) + .collect(), + weights, + ) + } + + /// Return a store that combines both a `Store` for subgraph data + /// and a `BlockStore` for all chain related data + pub fn network_store(self, networks: Vec<(String, Vec)>) -> Arc { + Self::make_store( + &self.logger, + self.pools, + self.subgraph_store, + self.chains, + networks, + ) + } + + pub fn subscription_manager(&self) -> Arc { + self.subscription_manager.cheap_clone() + } + + pub fn chain_head_update_listener(&self) -> Arc { + self.chain_head_update_listener.clone() + } + + pub fn primary_pool(&self) -> ConnectionPool { + self.pools.get(&*PRIMARY_SHARD).unwrap().clone() + } +} diff --git a/resources/construction.svg b/resources/construction.svg new file mode 100644 index 0000000..e4d4ce9 --- /dev/null +++ b/resources/construction.svg @@ -0,0 +1,168 @@ + + + + under consctruction 2 + Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/runtime/derive/Cargo.toml b/runtime/derive/Cargo.toml new file mode 100644 index 0000000..13013d7 --- /dev/null +++ b/runtime/derive/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "graph-runtime-derive" +version = "0.27.0" +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "1.0.98", features = ["full"] } +quote = "1.0" +proc-macro2 = "1.0.43" +heck = "0.4" diff --git a/runtime/derive/src/generate_array_type.rs b/runtime/derive/src/generate_array_type.rs new file mode 100644 index 0000000..91a7d1d --- /dev/null +++ b/runtime/derive/src/generate_array_type.rs @@ -0,0 +1,80 @@ +use proc_macro::TokenStream; +use proc_macro2::{Ident, Span}; +use quote::quote; +use syn::{self, parse_macro_input, AttributeArgs, ItemStruct, Meta, NestedMeta, Path}; + +pub fn generate_array_type(metadata: TokenStream, input: TokenStream) -> TokenStream { + let item_struct = parse_macro_input!(input as ItemStruct); + let name = item_struct.ident.clone(); + + let asc_name = Ident::new(&format!("Asc{}", name.to_string()), Span::call_site()); + let asc_name_array = Ident::new(&format!("Asc{}Array", name.to_string()), Span::call_site()); + + let args = parse_macro_input!(metadata as AttributeArgs); + + let args = args + .iter() + .filter_map(|a| { + if let NestedMeta::Meta(Meta::Path(Path { segments, .. })) = a { + if let Some(p) = segments.last() { + return Some(p.ident.to_string().to_owned()); + } + } + None + }) + .collect::>(); + + assert!( + args.len() > 0, + "arguments not found! generate_array_type()" + ); + + let no_asc_name = if name.to_string().to_uppercase().starts_with("ASC") { + name.to_string()[3..].to_owned() + } else { + name.to_string() + }; + + let index_asc_type_id_array = format!("{}{}Array", args[0], no_asc_name) + .parse::() + .unwrap(); + + quote! { + #item_struct + + #[automatically_derived] + pub struct #asc_name_array(pub graph_runtime_wasm::asc_abi::class::Array>); + + impl graph::runtime::ToAscObj<#asc_name_array> for Vec<#name> { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &graph::runtime::gas::GasCounter, + ) -> Result<#asc_name_array, graph::runtime::DeterministicHostError> { + let content: Result, _> = self.iter().map(|x| graph::runtime::asc_new(heap, x, gas)).collect(); + + Ok(#asc_name_array(graph_runtime_wasm::asc_abi::class::Array::new(&content?, heap, gas)?)) + } + } + + impl graph::runtime::AscType for #asc_name_array { + fn to_asc_bytes(&self) -> Result, graph::runtime::DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &graph::semver::Version, + ) -> Result { + Ok(Self(graph_runtime_wasm::asc_abi::class::Array::from_asc_bytes(asc_obj, api_version)?)) + } + } + + #[automatically_derived] + impl graph::runtime::AscIndexId for #asc_name_array { + const INDEX_ASC_TYPE_ID: graph::runtime::IndexForAscTypeId = graph::runtime::IndexForAscTypeId::#index_asc_type_id_array ; + } + + } + .into() +} diff --git a/runtime/derive/src/generate_asc_type.rs b/runtime/derive/src/generate_asc_type.rs new file mode 100644 index 0000000..0d133a3 --- /dev/null +++ b/runtime/derive/src/generate_asc_type.rs @@ -0,0 +1,172 @@ +use proc_macro::TokenStream; +use proc_macro2::{Ident, Span}; +use quote::quote; +use syn::{self, parse_macro_input, Field, ItemStruct}; + +pub fn generate_asc_type(metadata: TokenStream, input: TokenStream) -> TokenStream { + let item_struct = parse_macro_input!(input as ItemStruct); + let args = parse_macro_input!(metadata as super::Args); + + let name = item_struct.ident.clone(); + let asc_name = Ident::new(&format!("Asc{}", name.to_string()), Span::call_site()); + + let enum_names = args + .vars + .iter() + .filter(|f| f.ident.to_string() != super::REQUIRED_IDENT_NAME) + .map(|f| f.ident.to_string()) + .collect::>(); + + //struct's fields -> need to skip enum fields + let mut fields = item_struct + .fields + .iter() + .filter(|f| !enum_names.contains(&f.ident.as_ref().unwrap().to_string())) + .collect::>(); + + //extend fields list with enum's variants + args.vars + .iter() + .filter(|f| f.ident.to_string() != super::REQUIRED_IDENT_NAME) + .flat_map(|f| f.fields.named.iter()) + .for_each(|f| fields.push(f)); + + let m_fields: Vec = fields + .iter() + .map(|f| { + let fld_name = f.ident.clone().unwrap(); + let typ = field_type_map(field_type(f)); + let fld_type = typ.parse::().unwrap(); + + quote! { + pub #fld_name : #fld_type , + } + }) + .collect(); + + let expanded = quote! { + + #item_struct + + #[automatically_derived] + + #[repr(C)] + #[derive(graph_runtime_derive::AscType)] + #[derive(Debug, Default)] + pub struct #asc_name { + #(#m_fields)* + } + }; + + expanded.into() +} + +fn is_scalar(nm: &str) -> bool { + match nm { + "i8" | "u8" => true, + "i16" | "u16" => true, + "i32" | "u32" => true, + "i64" | "u64" => true, + "usize" | "isize" => true, + "bool" => true, + _ => false, + } +} + +fn field_type_map(tp: String) -> String { + if is_scalar(&tp) { + tp + } else { + match tp.as_ref() { + "String" => "graph_runtime_wasm::asc_abi::class::AscString".into(), + _ => tp.to_owned(), + } + } +} + +fn field_type(fld: &syn::Field) -> String { + if let syn::Type::Path(tp) = &fld.ty { + if let Some(ps) = tp.path.segments.last() { + let name = ps.ident.to_string(); + //TODO - this must be optimized + match name.as_ref() { + "Vec" => match &ps.arguments { + syn::PathArguments::AngleBracketed(v) => { + if let syn::GenericArgument::Type(syn::Type::Path(p)) = &v.args[0] { + let nm = path_to_string(&p.path); + + match nm.as_ref(){ + "u8" => "graph::runtime::AscPtr".to_owned(), + "Vec" => "graph::runtime::AscPtr".to_owned(), + "String" => "graph::runtime::AscPtr>>".to_owned(), + _ => format!("graph::runtime::AscPtr", path_to_string(&p.path)) + } + } else { + name + } + } + + syn::PathArguments::None => name, + syn::PathArguments::Parenthesized(_v) => { + panic!("syn::PathArguments::Parenthesized is not implemented") + } + }, + "Option" => match &ps.arguments { + syn::PathArguments::AngleBracketed(v) => { + if let syn::GenericArgument::Type(syn::Type::Path(p)) = &v.args[0] { + let tp_nm = path_to_string(&p.path); + if is_scalar(&tp_nm) { + format!("Option<{}>", tp_nm) + } else { + format!("graph::runtime::AscPtr", tp_nm) + } + } else { + name + } + } + + syn::PathArguments::None => name, + syn::PathArguments::Parenthesized(_v) => { + panic!("syn::PathArguments::Parenthesized is not implemented") + } + }, + "String" => { + //format!("graph::runtime::AscPtr", name) + "graph::runtime::AscPtr" + .to_owned() + } + + _ => { + if is_scalar(&name) { + name + } else { + format!("graph::runtime::AscPtr", name) + } + } + } + } else { + "N/A".into() + } + } else { + "N/A".into() + } +} + +//recursive +fn path_to_string(path: &syn::Path) -> String { + if let Some(ps) = path.segments.last() { + let nm = ps.ident.to_string(); + + if let syn::PathArguments::AngleBracketed(v) = &ps.arguments { + if let syn::GenericArgument::Type(syn::Type::Path(p)) = &v.args[0] { + format!("{}<{}>", nm, path_to_string(&p.path)) + } else { + nm + } + } else { + nm + } + } else { + panic!("path_to_string - can't get last segment!") + } +} diff --git a/runtime/derive/src/generate_from_rust_type.rs b/runtime/derive/src/generate_from_rust_type.rs new file mode 100644 index 0000000..a21de91 --- /dev/null +++ b/runtime/derive/src/generate_from_rust_type.rs @@ -0,0 +1,233 @@ +use proc_macro::TokenStream; +use proc_macro2::{Ident, Span}; +use quote::quote; +use syn::{self, parse_macro_input, Field, ItemStruct}; + +pub fn generate_from_rust_type(metadata: TokenStream, input: TokenStream) -> TokenStream { + let item_struct = parse_macro_input!(input as ItemStruct); + let args = parse_macro_input!(metadata as super::Args); + + let enum_names = args + .vars + .iter() + .filter(|f| f.ident.to_string() != super::REQUIRED_IDENT_NAME) + .map(|f| f.ident.to_string()) + .collect::>(); + + let required_flds = args + .vars + .iter() + .filter(|f| f.ident.to_string() == super::REQUIRED_IDENT_NAME) + .flat_map(|f| f.fields.named.iter()) + .map(|f| f.ident.as_ref().unwrap().to_string()) + .collect::>(); + + //struct's standard fields + let fields = item_struct + .fields + .iter() + .filter(|f| { + let nm = f.ident.as_ref().unwrap().to_string(); + !enum_names.contains(&nm) && !nm.starts_with("_") + }) + .collect::>(); + + //struct's enum fields + let enum_fields = item_struct + .fields + .iter() + .filter(|f| enum_names.contains(&f.ident.as_ref().unwrap().to_string())) + .collect::>(); + + //module name + let mod_name = Ident::new( + &format!("__{}__", item_struct.ident.to_string().to_lowercase()), + item_struct.ident.span(), + ); + + let name = item_struct.ident.clone(); + let asc_name = Ident::new(&format!("Asc{}", name.to_string()), Span::call_site()); + + //generate enum fields validator + let enum_validation = enum_fields.iter().map(|f|{ + let fld_name = f.ident.as_ref().unwrap(); //empty, maybe call it "sum"? + let type_nm = format!("\"{}\"", name.to_string()).parse::().unwrap(); + let fld_nm = format!("\"{}\"", fld_name.to_string()).to_string().parse::().unwrap(); + + quote! { + let #fld_name = self.#fld_name.as_ref() + .ok_or_else(|| graph::runtime::DeterministicHostError::from(anyhow::anyhow!("{} missing {}", #type_nm, #fld_nm)))?; + } + }); + + let mut methods:Vec = + fields.iter().map(|f| { + let fld_name = f.ident.as_ref().unwrap(); + let self_ref = + if is_byte_array(f){ + quote! { graph_runtime_wasm::asc_abi::class::Bytes(&self.#fld_name) } + }else{ + quote!{ self.#fld_name } + }; + + let is_required = is_required(f, &required_flds); + + let setter = + if is_nullable(&f) { + if is_required{ + let type_nm = format!("\"{}\"", name.to_string()).parse::().unwrap(); + let fld_nm = format!("\"{}\"", fld_name.to_string()).parse::().unwrap(); + + quote! { + #fld_name: graph::runtime::asc_new_or_missing(heap, &#self_ref, gas, #type_nm, #fld_nm)?, + } + }else{ + quote! { + #fld_name: graph::runtime::asc_new_or_null(heap, &#self_ref, gas)?, + } + } + } else { + if is_scalar(&field_type(f)){ + quote!{ + #fld_name: #self_ref, + } + }else{ + quote! { + #fld_name: graph::runtime::asc_new(heap, &#self_ref, gas)?, + } + } + }; + setter + }) + .collect(); + + for var in args.vars { + let var_nm = var.ident.to_string(); + if var_nm == super::REQUIRED_IDENT_NAME { + continue; + } + + let mut c = var_nm.chars(); + let var_type_name = c.next().unwrap().to_uppercase().collect::() + c.as_str(); + + var.fields.named.iter().map(|f|{ + + let fld_nm = f.ident.as_ref().unwrap(); + let var_nm = var.ident.clone(); + + use heck::{ToUpperCamelCase, ToSnakeCase}; + + let varian_type_name = fld_nm.to_string().to_upper_camel_case(); + let mod_name = item_struct.ident.to_string().to_snake_case(); + let varian_type_name = format!("{}::{}::{}",mod_name, var_type_name, varian_type_name).parse::().unwrap(); + + let setter = + if is_byte_array(f){ + quote! { + #fld_nm: if let #varian_type_name(v) = #var_nm {graph::runtime::asc_new(heap, &graph_runtime_wasm::asc_abi::class::Bytes(v), gas)? } else {graph::runtime::AscPtr::null()}, + } + }else{ + quote! { + #fld_nm: if let #varian_type_name(v) = #var_nm {graph::runtime::asc_new(heap, v, gas)? } else {graph::runtime::AscPtr::null()}, + } + }; + + setter + }) + .for_each(|ts| methods.push(ts)); + } + + let expanded = quote! { + #item_struct + + #[automatically_derived] + mod #mod_name{ + use super::*; + + use crate::protobuf::*; + + impl graph::runtime::ToAscObj<#asc_name> for #name { + + #[allow(unused_variables)] + fn to_asc_obj( + &self, + heap: &mut H, + gas: &graph::runtime::gas::GasCounter, + ) -> Result<#asc_name, graph::runtime::DeterministicHostError> { + + #(#enum_validation)* + + Ok( + #asc_name { + #(#methods)* + ..Default::default() + } + ) + } + } + } // -------- end of mod + + + }; + + expanded.into() +} + +fn is_scalar(fld: &str) -> bool { + match fld { + "i8" | "u8" => true, + "i16" | "u16" => true, + "i32" | "u32" => true, + "i64" | "u64" => true, + "usize" | "isize" => true, + "bool" => true, + _ => false, + } +} + +fn field_type(fld: &syn::Field) -> String { + if let syn::Type::Path(tp) = &fld.ty { + if let Some(ps) = tp.path.segments.last() { + return ps.ident.to_string(); + } else { + "N/A".into() + } + } else { + "N/A".into() + } +} + +fn is_required(fld: &syn::Field, req_list: &[String]) -> bool { + let fld_name = fld.ident.as_ref().unwrap().to_string(); + req_list.iter().find(|r| *r == &fld_name).is_some() +} + +fn is_nullable(fld: &syn::Field) -> bool { + if let syn::Type::Path(tp) = &fld.ty { + if let Some(last) = tp.path.segments.last() { + return last.ident == "Option"; + } + } + false +} + +fn is_byte_array(fld: &syn::Field) -> bool { + if let syn::Type::Path(tp) = &fld.ty { + if let Some(last) = tp.path.segments.last() { + if last.ident == "Vec" { + if let syn::PathArguments::AngleBracketed(ref v) = last.arguments { + if let Some(last) = v.args.last() { + if let syn::GenericArgument::Type(t) = last { + if let syn::Type::Path(p) = t { + if let Some(a) = p.path.segments.last() { + return a.ident == "u8"; + } + } + } + } + } + } + } + } + false +} diff --git a/runtime/derive/src/generate_network_type_id.rs b/runtime/derive/src/generate_network_type_id.rs new file mode 100644 index 0000000..1b66ca8 --- /dev/null +++ b/runtime/derive/src/generate_network_type_id.rs @@ -0,0 +1,56 @@ +use proc_macro::TokenStream; +use proc_macro2::{Ident, Span}; +use quote::quote; +use syn::{self, parse_macro_input, AttributeArgs, ItemStruct, Meta, NestedMeta, Path}; + +pub fn generate_network_type_id(metadata: TokenStream, input: TokenStream) -> TokenStream { + let item_struct = parse_macro_input!(input as ItemStruct); + let name = item_struct.ident.clone(); + + let asc_name = if name.to_string().to_uppercase().starts_with("ASC") { + name.clone() + } else { + Ident::new(&format!("Asc{}", name.to_string()), Span::call_site()) + }; + + let no_asc_name = if name.to_string().to_uppercase().starts_with("ASC") { + name.to_string()[3..].to_owned() + } else { + name.to_string() + }; + + let args = parse_macro_input!(metadata as AttributeArgs); + + let args = args + .iter() + .filter_map(|a| { + if let NestedMeta::Meta(Meta::Path(Path { segments, .. })) = a { + if let Some(p) = segments.last() { + return Some(p.ident.to_string().to_owned()); + } + } + None + }) + .collect::>(); + + assert!( + args.len() > 0, + "arguments not found! generate_network_type_id()" + ); + + //type_id variant name + let index_asc_type_id = format!("{}{}", args[0], no_asc_name) + .parse::() + .unwrap(); + + let expanded = quote! { + #item_struct + + #[automatically_derived] + impl graph::runtime::AscIndexId for #asc_name { + const INDEX_ASC_TYPE_ID: graph::runtime::IndexForAscTypeId = graph::runtime::IndexForAscTypeId::#index_asc_type_id ; + } + }; + + expanded.into() +} diff --git a/runtime/derive/src/lib.rs b/runtime/derive/src/lib.rs new file mode 100644 index 0000000..f95c11f --- /dev/null +++ b/runtime/derive/src/lib.rs @@ -0,0 +1,446 @@ +#![recursion_limit = "128"] + +extern crate proc_macro; + +use proc_macro::TokenStream; +use quote::quote; +use syn::{ + parse::{Parse, ParseStream}, + Fields, FieldsNamed, Ident, Item, ItemEnum, ItemStruct, Token, +}; + +const REQUIRED_IDENT_NAME: &str = "__required__"; + +struct Args { + vars: Vec, +} + +struct ArgsField { + ident: Ident, + fields: FieldsNamed, +} + +impl Parse for Args { + fn parse(input: ParseStream) -> syn::Result { + let mut idents = Vec::::new(); + + while input.peek(syn::Ident) { + let ident = input.call(Ident::parse)?; + idents.push(ArgsField { + ident, + fields: input.call(FieldsNamed::parse)?, + }); + let _: Option = input.parse()?; + } + + Ok(Args { vars: idents }) + } +} + +#[derive(Debug)] +struct TypeParam(syn::Ident); + +impl syn::parse::Parse for TypeParam { + fn parse(input: syn::parse::ParseStream) -> syn::Result { + let content; + syn::parenthesized!(content in input); + let typ = content.parse()?; + Ok(TypeParam(typ)) + } +} + +#[derive(Debug)] +struct TypeParamList(Vec); + +impl syn::parse::Parse for TypeParamList { + fn parse(input: syn::parse::ParseStream) -> syn::Result { + let content; + syn::parenthesized!(content in input); + + let mut params: Vec = Vec::new(); + + while !content.is_empty() { + let typ = content.parse()?; + params.push(typ); + + if !content.is_empty() { + let _comma: syn::Token![,] = content.parse()?; + } + } + + Ok(TypeParamList(params)) + } +} + +//generates graph::runtime::ToAscObj implementation for the type +//takes optional optional list of required fields '__required__{name:TypeName}' and enumerations field decraration with types, i.e. sum{single: ModeInfoSingle,multi: ModeInfoMulti} +//intended use is in build.rs with tonic_build's type_attribute(<...>, <...>) to generate type implementation of graph::runtime::ToAscObj +//Annotation example: +//#[graph_runtime_derive::generate_from_rust_type(...)] +// pub struct MyMessageType { +// .. +// } +//the above annotation will produce following implementation +// impl graph::runtime::ToAscObj for MyMessageType { +// ... +// } +mod generate_from_rust_type; +#[proc_macro_attribute] +pub fn generate_from_rust_type(args: TokenStream, input: TokenStream) -> TokenStream { + generate_from_rust_type::generate_from_rust_type(args, input) +} + +//generates graph::runtime::AscIndexId implementation for the type +//takes required network name attribute to form variant name graph::runtime::IndexForAscTypeId::+ +//Annotation example: +//#[graph_runtime_derive::generate_network_type_id(Cosmos)] +// pub struct MyMessageType { +// .. +// } +//the above annotation will produce following implementation +// impl graph::runtime::AscIndexId for AscMyMessageType { +// const INDEX_ASC_TYPE_ID: graph::runtime::IndexForAscTypeId = graph::runtime::IndexForAscTypeId::CosmosMyMessageType ; +// } + +mod generate_network_type_id; +#[proc_macro_attribute] +pub fn generate_network_type_id(args: TokenStream, input: TokenStream) -> TokenStream { + generate_network_type_id::generate_network_type_id(args, input) +} + +//generates AscType for Type. Takes optional list of non-optional field+type +//Annotation example: +//#[graph_runtime_derive::generate_asc_type(non-optional-field-name: non-optional-field-type,...)] +// pub struct MyMessageType { +// .. +// } +//the above annotation will produce following implementation +// #[repr(C)] +// #[derive(graph_runtime_derive::AscType)] +// #[derive(Debug, Default)] +// pub struct AscMyMessageType { +// ... +// } +// +//Note: this macro makes heavy reliance on types to be available via crate::protobuf (network chain crate root/src/protobuf/lib.rs) +//please see usage exmple in chain::cosmos crate... lib.rs imports generates protobuf bindings, as well as any other needed types +mod generate_asc_type; +#[proc_macro_attribute] +pub fn generate_asc_type(args: TokenStream, input: TokenStream) -> TokenStream { + generate_asc_type::generate_asc_type(args, input) +} + +//generates array type for a type. +//Annotation example: +// #[graph_runtime_derive::generate_array_type(>)] +// pub struct MyMessageType { +// .. +// } +//the above annoation will generate code for MyMessageType type +//Example: +// pub struct AscMyMessageTypeArray(pub graph_runtime_wasm::asc_abi::class::Array>) +//where "AscMyMessageTypeArray" is an array type for "AscMyMessageType" (AscMyMessageType is generated by asc_type derive macro above) +//Macro, also, will generate code for the following 3 trait implementations +//1. graph::runtime::ToAscObj trait +//Example: +// impl graph::runtime::ToAscObj for Vec { +// ... +// } +//2. graph::runtime::AscType +//Example: +// impl graph::runtime::AscType for AscMyMessageTypeArray { +// ... +// } +//3. graph::runtime::AscIndexId (adding expected >Array (CosmosMyMessageTypeArray) variant to graph::runtime::IndexForAscTypeId is manual step) +//impl graph::runtime::AscIndexId for MyMessageTypeArray { +// const INDEX_ASC_TYPE_ID: graph::runtime::IndexForAscTypeId = graph::runtime::IndexForAscTypeId::CosmosMyMessageTypeArray ; +//} +mod generate_array_type; +#[proc_macro_attribute] +pub fn generate_array_type(args: TokenStream, input: TokenStream) -> TokenStream { + generate_array_type::generate_array_type(args, input) +} + +#[proc_macro_derive(AscType)] +pub fn asc_type_derive(input: TokenStream) -> TokenStream { + let item: Item = syn::parse(input).unwrap(); + match item { + Item::Struct(item_struct) => asc_type_derive_struct(item_struct), + Item::Enum(item_enum) => asc_type_derive_enum(item_enum), + _ => panic!("AscType can only be derived for structs and enums"), + } +} + +// Example input: +// #[repr(C)] +// #[derive(AscType)] +// struct AscTypedMapEntry { +// pub key: AscPtr, +// pub value: AscPtr, +// } +// +// Example output: +// impl graph::runtime::AscType for AscTypedMapEntry { +// fn to_asc_bytes(&self) -> Vec { +// let mut bytes = Vec::new(); +// bytes.extend_from_slice(&self.key.to_asc_bytes()); +// bytes.extend_from_slice(&self.value.to_asc_bytes()); +// assert_eq!(&bytes.len(), &size_of::()); +// bytes +// } + +// #[allow(unused_variables)] +// fn from_asc_bytes(asc_obj: &[u8], api_version: graph::semver::Version) -> Self { +// assert_eq!(&asc_obj.len(), &size_of::()); +// let mut offset = 0; +// let field_size = std::mem::size_of::>(); +// let key = graph::runtime::AscType::from_asc_bytes(&asc_obj[offset..(offset + field_size)], +// api_version.clone()); +// offset += field_size; +// let field_size = std::mem::size_of::>(); +// let value = graph::runtime::AscType::from_asc_bytes(&asc_obj[offset..(offset + field_size)], api_version); +// offset += field_size; +// Self { key, value } +// } +// } +// +// padding logic inspired by: +// https://doc.rust-lang.org/reference/type-layout.html#reprc-structs +// +// start with offset 0 +// for each field: +// * if offset is not multiple of field alignment add padding bytes with size needed to fill +// the gap to the next alignment multiple: alignment - (offset % alignment) +// * increase offset by size of padding, if any +// * increase offset by size of field bytes +// * keep track of maximum field alignment to determine alignment of struct +// if end offset is not multiple of struct alignment add padding field with required size. +fn asc_type_derive_struct(item_struct: ItemStruct) -> TokenStream { + let struct_name = &item_struct.ident; + let (impl_generics, ty_generics, where_clause) = item_struct.generics.split_for_impl(); + let field_names: Vec<_> = match &item_struct.fields { + Fields::Named(fields) => fields + .named + .iter() + .map(|field| field.ident.as_ref().unwrap()) + .collect(), + _ => panic!("AscType can only be derived for structs with named fields"), + }; + let field_names2 = field_names.clone(); + let field_names3 = field_names.clone(); + let field_types = match &item_struct.fields { + Fields::Named(fields) => fields.named.iter().map(|field| &field.ty), + _ => panic!("AscType can only be derived for structs with named fields"), + }; + let field_types2 = field_types.clone(); + + // if no fields, return immediately + let (no_fields_return_bytes, no_fields_return_self) = if field_names.is_empty() { + ( + quote! { + #![allow(unreachable_code)] + return Ok(Vec::new()); + }, + quote! { + #![allow(unreachable_code)] + return Ok(Self {}); + }, + ) + } else { + (quote! {}, quote! {}) + }; + + TokenStream::from(quote! { + impl #impl_generics graph::runtime::AscType for #struct_name #ty_generics #where_clause { + fn to_asc_bytes(&self) -> Result, graph::runtime::DeterministicHostError> { + #no_fields_return_bytes + + let in_memory_byte_count = std::mem::size_of::(); + let mut bytes = Vec::with_capacity(in_memory_byte_count); + + let mut offset = 0; + // max field alignment will also be struct alignment which we need to pad the end + let mut max_align = 0; + + #( + let field_align = std::mem::align_of::<#field_types>(); + let misalignment = offset % field_align; + + if misalignment > 0 { + let padding_size = field_align - misalignment; + + bytes.extend_from_slice(&vec![0; padding_size]); + + offset += padding_size; + } + + let field_bytes = self.#field_names.to_asc_bytes()?; + + bytes.extend_from_slice(&field_bytes); + + offset += field_bytes.len(); + + if max_align < field_align { + max_align = field_align; + } + )* + + // pad end of struct data if needed + let struct_misalignment = offset % max_align; + + if struct_misalignment > 0 { + let padding_size = max_align - struct_misalignment; + + bytes.extend_from_slice(&vec![0; padding_size]); + } + + // **Important** AssemblyScript and `repr(C)` in Rust does not follow exactly + // the same rules always. One caveats is that some struct are packed in AssemblyScript + // but padded for alignment in `repr(C)` like a struct `{ one: AscPtr, two: AscPtr, three: AscPtr, four: u64 }`, + // it appears this struct is always padded in `repr(C)` by Rust whatever order is tried. + // However, it's possible to packed completely this struct in AssemblyScript and avoid + // any padding. + // + // To overcome those cases where re-ordering never work, you will need to add an explicit + // _padding field to account for missing padding and pass this check. + assert_eq!(bytes.len(), in_memory_byte_count, "Alignment mismatch for {}, re-order fields or explicitely add a _padding field", stringify!(#struct_name)); + Ok(bytes) + } + + #[allow(unused_variables)] + fn from_asc_bytes(asc_obj: &[u8], api_version: &graph::semver::Version) -> Result { + #no_fields_return_self + + // Sanity check + match api_version { + api_version if *api_version <= graph::semver::Version::new(0, 0, 4) => { + // This was using an double equal sign before instead of less than. + // This happened because of the new apiVersion support. + // Since some structures need different implementations for each + // version, their memory size got bigger because we're using an enum + // that contains both versions (each in a variant), and that increased + // the memory size, so that's why we use less than. + if asc_obj.len() < std::mem::size_of::() { + return Err(graph::runtime::DeterministicHostError::from(graph::prelude::anyhow::anyhow!("Size does not match"))); + } + } + _ => { + let content_size = std::mem::size_of::(); + let aligned_size = graph::runtime::padding_to_16(content_size); + + if graph::runtime::HEADER_SIZE + asc_obj.len() == aligned_size + content_size { + return Err(graph::runtime::DeterministicHostError::from(graph::prelude::anyhow::anyhow!("Size does not match"))); + } + }, + }; + + let mut offset = 0; + + #( + // skip padding + let field_align = std::mem::align_of::<#field_types2>(); + let misalignment = offset % field_align; + if misalignment > 0 { + let padding_size = field_align - misalignment; + + offset += padding_size; + } + + let field_size = std::mem::size_of::<#field_types2>(); + let field_data = asc_obj.get(offset..(offset + field_size)).ok_or_else(|| { + graph::runtime::DeterministicHostError::from(graph::prelude::anyhow::anyhow!("Attempted to read past end of array")) + })?; + let #field_names2 = graph::runtime::AscType::from_asc_bytes(&field_data, api_version)?; + offset += field_size; + )* + + Ok(Self { + #(#field_names3,)* + }) + } + } + }) +} + +// Example input: +// #[repr(u32)] +// #[derive(AscType)] +// enum JsonValueKind { +// Null, +// Bool, +// Number, +// String, +// Array, +// Object, +// } +// +// Example output: +// impl graph::runtime::AscType for JsonValueKind { +// fn to_asc_bytes(&self) -> Result, graph::runtime::DeterministicHostError> { +// let discriminant: u32 = match *self { +// JsonValueKind::Null => 0u32, +// JsonValueKind::Bool => 1u32, +// JsonValueKind::Number => 2u32, +// JsonValueKind::String => 3u32, +// JsonValueKind::Array => 4u32, +// JsonValueKind::Object => 5u32, +// }; +// Ok(discriminant.to_asc_bytes()) +// } +// +// fn from_asc_bytes(asc_obj: &[u8], _api_version: graph::semver::Version) -> Result { +// let mut u32_bytes: [u8; size_of::()] = [0; size_of::()]; +// if std::mem::size_of_val(&u32_bytes) != std::mem::size_of_val(&asc_obj) { +// return Err(graph::runtime::DeterministicHostError::from(graph::prelude::anyhow::anyhow!("Invalid asc bytes size"))); +// } +// u32_bytes.copy_from_slice(&asc_obj); +// let discr = u32::from_le_bytes(u32_bytes); +// match discr { +// 0u32 => JsonValueKind::Null, +// 1u32 => JsonValueKind::Bool, +// 2u32 => JsonValueKind::Number, +// 3u32 => JsonValueKind::String, +// 4u32 => JsonValueKind::Array, +// 5u32 => JsonValueKind::Object, +// _ => Err(graph::runtime::DeterministicHostError::from(graph::prelude::anyhow::anyhow!("value {} is out of range for {}", discr, "JsonValueKind"))), +// } +// } +// } +fn asc_type_derive_enum(item_enum: ItemEnum) -> TokenStream { + let enum_name = &item_enum.ident; + let enum_name_iter = std::iter::repeat(enum_name); + let enum_name_iter2 = enum_name_iter.clone(); + let (impl_generics, ty_generics, where_clause) = item_enum.generics.split_for_impl(); + let variant_paths: Vec<_> = item_enum + .variants + .iter() + .map(|v| { + assert!(v.discriminant.is_none()); + &v.ident + }) + .collect(); + let variant_paths2 = variant_paths.clone(); + let variant_discriminant = 0..(variant_paths.len() as u32); + let variant_discriminant2 = variant_discriminant.clone(); + + TokenStream::from(quote! { + impl #impl_generics graph::runtime::AscType for #enum_name #ty_generics #where_clause { + fn to_asc_bytes(&self) -> Result, graph::runtime::DeterministicHostError> { + let discriminant: u32 = match self { + #(#enum_name_iter::#variant_paths => #variant_discriminant,)* + }; + discriminant.to_asc_bytes() + } + + fn from_asc_bytes(asc_obj: &[u8], _api_version: &graph::semver::Version) -> Result { + let u32_bytes = ::std::convert::TryFrom::try_from(asc_obj) + .map_err(|_| graph::runtime::DeterministicHostError::from(graph::prelude::anyhow::anyhow!("Invalid asc bytes size")))?; + let discr = u32::from_le_bytes(u32_bytes); + match discr { + #(#variant_discriminant2 => Ok(#enum_name_iter2::#variant_paths2),)* + _ => Err(graph::runtime::DeterministicHostError::from(graph::prelude::anyhow::anyhow!("value {} is out of range for {}", discr, stringify!(#enum_name)))) + } + } + } + }) +} diff --git a/runtime/test/Cargo.toml b/runtime/test/Cargo.toml new file mode 100644 index 0000000..cd85bbc --- /dev/null +++ b/runtime/test/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "graph-runtime-test" +version = "0.27.0" +edition = "2021" + +[dependencies] +semver = "1.0" +wasmtime = "0.27.0" +graph = { path = "../../graph" } +graph-chain-ethereum = { path = "../../chain/ethereum" } +graph-runtime-wasm = { path = "../wasm" } +graph-core = { path = "../../core" } +graph-runtime-derive = { path = "../derive" } +rand = "0.8.5" + + +[dev-dependencies] +test-store = { path = "../../store/test-store" } +graph-mock = { path = "../../mock" } diff --git a/runtime/test/README.md b/runtime/test/README.md new file mode 100644 index 0000000..c555617 --- /dev/null +++ b/runtime/test/README.md @@ -0,0 +1,86 @@ +# Runtime tests + +These are the unit tests that check if the WASM runtime code is working. For now we only run code compiled from the [`AssemblyScript`](https://www.assemblyscript.org/) language, which is done by [`asc`](https://github.com/AssemblyScript/assemblyscript) (the AssemblyScript Compiler) in our [`CLI`](https://github.com/graphprotocol/graph-cli). + +We support two versions of their compiler/language for now: + +- [`v0.6`](https://github.com/AssemblyScript/assemblyscript/releases/tag/v0.6) +- +[`v0.19.10`](https://github.com/AssemblyScript/assemblyscript/releases/tag/v0.19.10) + +Because the internal ABIs changed between these versions, the runtime was added, etc, we had to duplicate the source files used for the tests (`.ts` and `.wasm`). + +If you look into the [`wasm_test`](https://github.com/graphprotocol/graph-node/tree/master/runtime/test/wasm_test) folder you'll find two other folders: + +- [`api_version_0_0_4`](https://github.com/graphprotocol/graph-node/tree/master/runtime/test/wasm_test/api_version_0_0_4) +- [`api_version_0_0_5`](https://github.com/graphprotocol/graph-node/tree/master/runtime/test/wasm_test/api_version_0_0_5) + +This is because the first one (`0.0.4` `apiVersion`) is related to the `v0.6` of `AssemblyScript` and the second (`0.0.5` `apiVersion`) to +`v0.19.10`. + +## How to change the `.ts`/`.wasm` files + +### Api Version 0.0.4 + +First make sure your `asc` version is `v0.6`, to check use `asc --version`. + +To install the correct one use: + +``` +npm install -g AssemblyScript/assemblyscript#v0.6 +``` + +And to compile/change the desired test use the command below (just rename the files to the correct ones): + +``` +asc wasm_test/api_version_0_0_4/abi_classes.ts -b wasm_test/api_version_0_0_4/abi_classes.wasm +``` + +### Api Version 0.0.5 + +First make sure your `asc` version is +`v0.19.10`, to check use `asc --version`. + +To install the correct one use: + +``` +# for the precise one +npm install -g assemblyscript@0.19.10 + +# for the latest one, it should work as well +npm install -g assemblyscript +``` + +And to compile/change the desired test use the command below (just rename the files to the correct ones): + +``` +asc --explicitStart --exportRuntime --runtime stub wasm_test/api_version_0_0_5/abi_classes.ts -b wasm_test/api_version_0_0_5/abi_classes.wasm +``` + +## Caveats + +### Api Version 0.0.4 + +You'll always have to put this at the beginning of your `.ts` files: + +```typescript +import "allocator/arena"; + +export { memory }; +``` + +So the runtime can use the allocator properly. + +### Api Version 0.0.5 + +Since in this version we started: + +- Using the `--explicitStart` flag, that requires `__alloc(0)` to always be called before any global be defined +- To add the necessary variants for `TypeId` and using on `id_of_type` function. References from [`here`](https://github.com/graphprotocol/graph-node/blob/8bef4c005f5b1357fe29ca091c9188e1395cc227/graph/src/runtime/mod.rs#L140) + +Instead of having to add this manually to all of the files, you can just import and re-export this [`common`](https://github.com/graphprotocol/graph-node/tree/master/runtime/test/wasm_test/api_version_0_0_5/common/global.ts) file like this: + +```typescript +export * from './common/global' +``` + +And import the types you need from [`here`](https://github.com/graphprotocol/graph-node/tree/master/runtime/test/wasm_test/api_version_0_0_5/common/types.ts). If the type you need is missing, just add them there. + +This way the runtime can both properly generate the headers with proper class identifiers and do memory allocations. diff --git a/runtime/test/src/common.rs b/runtime/test/src/common.rs new file mode 100644 index 0000000..8b0d77b --- /dev/null +++ b/runtime/test/src/common.rs @@ -0,0 +1,149 @@ +use ethabi::Contract; +use graph::components::store::DeploymentLocator; +use graph::data::subgraph::*; +use graph::data_source; +use graph::env::EnvVars; +use graph::ipfs_client::IpfsClient; +use graph::log; +use graph::prelude::*; +use graph_chain_ethereum::{ + Chain, DataSource, DataSourceTemplate, Mapping, MappingABI, TemplateSource, +}; +use graph_runtime_wasm::{HostExports, MappingContext}; +use semver::Version; +use std::env; +use std::str::FromStr; +use web3::types::Address; + +lazy_static! { + pub static ref LOGGER: Logger = match env::var_os("GRAPH_LOG") { + Some(_) => log::logger(false), + None => Logger::root(slog::Discard, o!()), + }; +} + +fn mock_host_exports( + subgraph_id: DeploymentHash, + data_source: DataSource, + store: Arc, + api_version: Version, +) -> HostExports { + let templates = vec![data_source::DataSourceTemplate::Onchain( + DataSourceTemplate { + kind: String::from("ethereum/contract"), + name: String::from("example template"), + manifest_idx: 0, + network: Some(String::from("mainnet")), + source: TemplateSource { + abi: String::from("foo"), + }, + mapping: Mapping { + kind: String::from("ethereum/events"), + api_version, + language: String::from("wasm/assemblyscript"), + entities: vec![], + abis: vec![], + event_handlers: vec![], + call_handlers: vec![], + block_handlers: vec![], + link: Link { + link: "link".to_owned(), + }, + runtime: Arc::new(vec![]), + }, + }, + )]; + + let network = data_source.network.clone().unwrap(); + let ens_lookup = store.ens_lookup(); + HostExports::new( + subgraph_id, + &data_source::DataSource::Onchain(data_source), + network, + Arc::new(templates), + Arc::new(graph_core::LinkResolver::new( + vec![IpfsClient::localhost()], + Arc::new(EnvVars::default()), + )), + ens_lookup, + ) +} + +fn mock_abi() -> MappingABI { + MappingABI { + name: "mock_abi".to_string(), + contract: Contract::load( + r#"[ + { + "inputs": [ + { + "name": "a", + "type": "address" + } + ], + "type": "constructor" + } + ]"# + .as_bytes(), + ) + .unwrap(), + } +} + +pub fn mock_context( + deployment: DeploymentLocator, + data_source: DataSource, + store: Arc, + api_version: Version, +) -> MappingContext { + MappingContext { + logger: Logger::root(slog::Discard, o!()), + block_ptr: BlockPtr { + hash: Default::default(), + number: 0, + }, + host_exports: Arc::new(mock_host_exports( + deployment.hash.clone(), + data_source, + store.clone(), + api_version, + )), + state: BlockState::new( + futures03::executor::block_on(store.writable(LOGGER.clone(), deployment.id)).unwrap(), + Default::default(), + ), + proof_of_indexing: None, + host_fns: Arc::new(Vec::new()), + debug_fork: None, + } +} + +pub fn mock_data_source(path: &str, api_version: Version) -> DataSource { + let runtime = std::fs::read(path).unwrap(); + + DataSource { + kind: String::from("ethereum/contract"), + name: String::from("example data source"), + manifest_idx: 0, + network: Some(String::from("mainnet")), + address: Some(Address::from_str("0123123123012312312301231231230123123123").unwrap()), + start_block: 0, + mapping: Mapping { + kind: String::from("ethereum/events"), + api_version, + language: String::from("wasm/assemblyscript"), + entities: vec![], + abis: vec![], + event_handlers: vec![], + call_handlers: vec![], + block_handlers: vec![], + link: Link { + link: "link".to_owned(), + }, + runtime: Arc::new(runtime.clone()), + }, + context: Default::default(), + creation_block: None, + contract_abi: Arc::new(mock_abi()), + } +} diff --git a/runtime/test/src/lib.rs b/runtime/test/src/lib.rs new file mode 100644 index 0000000..9bdc7b7 --- /dev/null +++ b/runtime/test/src/lib.rs @@ -0,0 +1,13 @@ +#![cfg(test)] +pub mod common; +mod test; + +#[cfg(test)] +pub mod test_padding; + +// this used in crate::test_padding module +// graph_runtime_derive::generate_from_rust_type looks for types in crate::protobuf, +// hence this mod presence in crate that uses ASC related macros is required +pub mod protobuf { + pub use super::test_padding::data::*; +} diff --git a/runtime/test/src/test.rs b/runtime/test/src/test.rs new file mode 100644 index 0000000..cad61b5 --- /dev/null +++ b/runtime/test/src/test.rs @@ -0,0 +1,1163 @@ +use graph::data::store::scalar; +use graph::data::subgraph::*; +use graph::prelude::web3::types::U256; +use graph::prelude::*; +use graph::runtime::{AscIndexId, AscType}; +use graph::runtime::{AscPtr, ToAscObj}; +use graph::{components::store::*, ipfs_client::IpfsClient}; +use graph_chain_ethereum::{Chain, DataSource}; +use graph_mock::MockMetricsRegistry; +use graph_runtime_wasm::asc_abi::class::{Array, AscBigInt, AscEntity, AscString, Uint8Array}; +use graph_runtime_wasm::{ExperimentalFeatures, ValidModule, WasmInstance}; +use hex; +use semver::Version; +use std::collections::{BTreeMap, HashMap}; +use std::str::FromStr; +use test_store::{LOGGER, STORE}; +use web3::types::H160; + +use crate::common::{mock_context, mock_data_source}; + +mod abi; + +pub const API_VERSION_0_0_4: Version = Version::new(0, 0, 4); +pub const API_VERSION_0_0_5: Version = Version::new(0, 0, 5); + +pub fn wasm_file_path(wasm_file: &str, api_version: Version) -> String { + format!( + "wasm_test/api_version_{}_{}_{}/{}", + api_version.major, api_version.minor, api_version.patch, wasm_file + ) +} + +fn subgraph_id_with_api_version(subgraph_id: &str, api_version: Version) -> String { + format!( + "{}_{}_{}_{}", + subgraph_id, api_version.major, api_version.minor, api_version.patch + ) +} + +async fn test_valid_module_and_store( + subgraph_id: &str, + data_source: DataSource, + api_version: Version, +) -> ( + WasmInstance, + Arc, + DeploymentLocator, +) { + test_valid_module_and_store_with_timeout(subgraph_id, data_source, api_version, None).await +} + +async fn test_valid_module_and_store_with_timeout( + subgraph_id: &str, + data_source: DataSource, + api_version: Version, + timeout: Option, +) -> ( + WasmInstance, + Arc, + DeploymentLocator, +) { + let logger = Logger::root(slog::Discard, o!()); + let subgraph_id_with_api_version = + subgraph_id_with_api_version(subgraph_id, api_version.clone()); + + let store = STORE.clone(); + let metrics_registry = Arc::new(MockMetricsRegistry::new()); + let deployment_id = DeploymentHash::new(&subgraph_id_with_api_version).unwrap(); + let deployment = test_store::create_test_subgraph( + &deployment_id, + "type User @entity { + id: ID!, + name: String, + } + + type Thing @entity { + id: ID!, + value: String, + extra: String + }", + ) + .await; + let stopwatch_metrics = StopwatchMetrics::new( + logger.clone(), + deployment_id.clone(), + "test", + metrics_registry.clone(), + ); + let host_metrics = Arc::new(HostMetrics::new( + metrics_registry, + deployment_id.as_str(), + stopwatch_metrics, + )); + + let experimental_features = ExperimentalFeatures { + allow_non_deterministic_ipfs: true, + }; + + let module = WasmInstance::from_valid_module_with_ctx( + Arc::new(ValidModule::new(&logger, data_source.mapping.runtime.as_ref()).unwrap()), + mock_context( + deployment.clone(), + data_source, + store.subgraph_store(), + api_version, + ), + host_metrics, + timeout, + experimental_features, + ) + .unwrap(); + + (module, store.subgraph_store(), deployment) +} + +pub async fn test_module( + subgraph_id: &str, + data_source: DataSource, + api_version: Version, +) -> WasmInstance { + test_valid_module_and_store(subgraph_id, data_source, api_version) + .await + .0 +} + +// A test module using the latest API version +pub async fn test_module_latest(subgraph_id: &str, wasm_file: &str) -> WasmInstance { + let version = ENV_VARS.mappings.max_api_version.clone(); + let ds = mock_data_source( + &wasm_file_path(wasm_file, API_VERSION_0_0_5.clone()), + version.clone(), + ); + test_valid_module_and_store(subgraph_id, ds, version) + .await + .0 +} + +pub trait WasmInstanceExt { + fn invoke_export0_void(&self, f: &str) -> Result<(), wasmtime::Trap>; + fn invoke_export1_val_void( + &self, + f: &str, + v: V, + ) -> Result<(), wasmtime::Trap>; + fn invoke_export0(&self, f: &str) -> AscPtr; + fn invoke_export1(&mut self, f: &str, arg: &T) -> AscPtr + where + C: AscType + AscIndexId, + T: ToAscObj + ?Sized; + fn invoke_export2(&mut self, f: &str, arg0: &T1, arg1: &T2) -> AscPtr + where + C1: AscType + AscIndexId, + C2: AscType + AscIndexId, + T1: ToAscObj + ?Sized, + T2: ToAscObj + ?Sized; + fn invoke_export2_void( + &mut self, + f: &str, + arg0: &T1, + arg1: &T2, + ) -> Result<(), wasmtime::Trap> + where + C1: AscType + AscIndexId, + C2: AscType + AscIndexId, + T1: ToAscObj + ?Sized, + T2: ToAscObj + ?Sized; + fn invoke_export0_val(&mut self, func: &str) -> V; + fn invoke_export1_val(&mut self, func: &str, v: &T) -> V + where + C: AscType + AscIndexId, + T: ToAscObj + ?Sized; + fn takes_ptr_returns_ptr(&self, f: &str, arg: AscPtr) -> AscPtr; + fn takes_val_returns_ptr

(&mut self, fn_name: &str, val: impl wasmtime::WasmTy) -> AscPtr

; +} + +impl WasmInstanceExt for WasmInstance { + fn invoke_export0_void(&self, f: &str) -> Result<(), wasmtime::Trap> { + let func = self.get_func(f).typed().unwrap().clone(); + func.call(()) + } + + fn invoke_export0(&self, f: &str) -> AscPtr { + let func = self.get_func(f).typed().unwrap().clone(); + let ptr: u32 = func.call(()).unwrap(); + ptr.into() + } + + fn takes_ptr_returns_ptr(&self, f: &str, arg: AscPtr) -> AscPtr { + let func = self.get_func(f).typed().unwrap().clone(); + let ptr: u32 = func.call(arg.wasm_ptr()).unwrap(); + ptr.into() + } + + fn invoke_export1(&mut self, f: &str, arg: &T) -> AscPtr + where + C: AscType + AscIndexId, + T: ToAscObj + ?Sized, + { + let func = self.get_func(f).typed().unwrap().clone(); + let ptr = self.asc_new(arg).unwrap(); + let ptr: u32 = func.call(ptr.wasm_ptr()).unwrap(); + ptr.into() + } + + fn invoke_export1_val_void( + &self, + f: &str, + v: V, + ) -> Result<(), wasmtime::Trap> { + let func = self.get_func(f).typed().unwrap().clone(); + func.call(v)?; + Ok(()) + } + + fn invoke_export2(&mut self, f: &str, arg0: &T1, arg1: &T2) -> AscPtr + where + C1: AscType + AscIndexId, + C2: AscType + AscIndexId, + T1: ToAscObj + ?Sized, + T2: ToAscObj + ?Sized, + { + let func = self.get_func(f).typed().unwrap().clone(); + let arg0 = self.asc_new(arg0).unwrap(); + let arg1 = self.asc_new(arg1).unwrap(); + let ptr: u32 = func.call((arg0.wasm_ptr(), arg1.wasm_ptr())).unwrap(); + ptr.into() + } + + fn invoke_export2_void( + &mut self, + f: &str, + arg0: &T1, + arg1: &T2, + ) -> Result<(), wasmtime::Trap> + where + C1: AscType + AscIndexId, + C2: AscType + AscIndexId, + T1: ToAscObj + ?Sized, + T2: ToAscObj + ?Sized, + { + let func = self.get_func(f).typed().unwrap().clone(); + let arg0 = self.asc_new(arg0).unwrap(); + let arg1 = self.asc_new(arg1).unwrap(); + func.call((arg0.wasm_ptr(), arg1.wasm_ptr())) + } + + fn invoke_export0_val(&mut self, func: &str) -> V { + let func = self.get_func(func).typed().unwrap().clone(); + func.call(()).unwrap() + } + + fn invoke_export1_val(&mut self, func: &str, v: &T) -> V + where + C: AscType + AscIndexId, + T: ToAscObj + ?Sized, + { + let func = self.get_func(func).typed().unwrap().clone(); + let ptr = self.asc_new(v).unwrap(); + func.call(ptr.wasm_ptr()).unwrap() + } + + fn takes_val_returns_ptr

(&mut self, fn_name: &str, val: impl wasmtime::WasmTy) -> AscPtr

{ + let func = self.get_func(fn_name).typed().unwrap().clone(); + let ptr: u32 = func.call(val).unwrap(); + ptr.into() + } +} + +async fn test_json_conversions(api_version: Version, gas_used: u64) { + let mut module = test_module( + "jsonConversions", + mock_data_source( + &wasm_file_path("string_to_number.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + // test u64 conversion + let number = 9223372036850770800; + let converted: i64 = module.invoke_export1_val("testToU64", &number.to_string()); + assert_eq!(number, u64::from_le_bytes(converted.to_le_bytes())); + + // test i64 conversion + let number = -9223372036850770800; + let converted: i64 = module.invoke_export1_val("testToI64", &number.to_string()); + assert_eq!(number, converted); + + // test f64 conversion + let number = -9223372036850770.92345034; + let converted: f64 = module.invoke_export1_val("testToF64", &number.to_string()); + assert_eq!(number, converted); + + // test BigInt conversion + let number = "-922337203685077092345034"; + let big_int_obj: AscPtr = module.invoke_export1("testToBigInt", number); + let bytes: Vec = module.asc_get(big_int_obj).unwrap(); + + assert_eq!( + scalar::BigInt::from_str(number).unwrap(), + scalar::BigInt::from_signed_bytes_le(&bytes) + ); + + assert_eq!(module.gas_used(), gas_used); +} + +#[tokio::test] +async fn json_conversions_v0_0_4() { + test_json_conversions(API_VERSION_0_0_4, 52976429).await; +} + +#[tokio::test] +async fn json_conversions_v0_0_5() { + test_json_conversions(API_VERSION_0_0_5, 2289897).await; +} + +async fn test_json_parsing(api_version: Version, gas_used: u64) { + let mut module = test_module( + "jsonParsing", + mock_data_source( + &wasm_file_path("json_parsing.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + // Parse invalid JSON and handle the error gracefully + let s = "foo"; // Invalid because there are no quotes around `foo` + let bytes: &[u8] = s.as_ref(); + let return_value: AscPtr = module.invoke_export1("handleJsonError", bytes); + let output: String = module.asc_get(return_value).unwrap(); + assert_eq!(output, "ERROR: true"); + + // Parse valid JSON and get it back + let s = "\"foo\""; // Valid because there are quotes around `foo` + let bytes: &[u8] = s.as_ref(); + let return_value: AscPtr = module.invoke_export1("handleJsonError", bytes); + + let output: String = module.asc_get(return_value).unwrap(); + assert_eq!(output, "OK: foo, ERROR: false"); + assert_eq!(module.gas_used(), gas_used); +} + +#[tokio::test] +async fn json_parsing_v0_0_4() { + test_json_parsing(API_VERSION_0_0_4, 2722284).await; +} + +#[tokio::test] +async fn json_parsing_v0_0_5() { + test_json_parsing(API_VERSION_0_0_5, 3862933).await; +} + +async fn test_ipfs_cat(api_version: Version) { + // Ipfs host functions use `block_on` which must be called from a sync context, + // so we replicate what we do `spawn_module`. + let runtime = tokio::runtime::Handle::current(); + std::thread::spawn(move || { + let _runtime_guard = runtime.enter(); + + let ipfs = IpfsClient::localhost(); + let hash = graph::block_on(ipfs.add("42".into())).unwrap().hash; + + let mut module = graph::block_on(test_module( + "ipfsCat", + mock_data_source( + &wasm_file_path("ipfs_cat.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + )); + let converted: AscPtr = module.invoke_export1("ipfsCatString", &hash); + let data: String = module.asc_get(converted).unwrap(); + assert_eq!(data, "42"); + }) + .join() + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ipfs_cat_v0_0_4() { + test_ipfs_cat(API_VERSION_0_0_4).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn ipfs_cat_v0_0_5() { + test_ipfs_cat(API_VERSION_0_0_5).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_ipfs_block() { + // Ipfs host functions use `block_on` which must be called from a sync context, + // so we replicate what we do `spawn_module`. + let runtime = tokio::runtime::Handle::current(); + std::thread::spawn(move || { + let _runtime_guard = runtime.enter(); + + let ipfs = IpfsClient::localhost(); + let hash = graph::block_on(ipfs.add("42".into())).unwrap().hash; + let mut module = graph::block_on(test_module( + "ipfsBlock", + mock_data_source( + &wasm_file_path("ipfs_block.wasm", API_VERSION_0_0_5), + API_VERSION_0_0_5, + ), + API_VERSION_0_0_5, + )); + let converted: AscPtr = module.invoke_export1("ipfsBlockHex", &hash); + let data: String = module.asc_get(converted).unwrap(); + assert_eq!(data, "0x0a080802120234321802"); + }) + .join() + .unwrap(); +} + +// The user_data value we use with calls to ipfs_map +const USER_DATA: &str = "user_data"; + +fn make_thing(id: &str, value: &str) -> (String, EntityModification) { + let mut data = Entity::new(); + data.set("id", id); + data.set("value", value); + data.set("extra", USER_DATA); + let key = EntityKey { + entity_type: EntityType::new("Thing".to_string()), + entity_id: id.into(), + }; + ( + format!("{{ \"id\": \"{}\", \"value\": \"{}\"}}", id, value), + EntityModification::Insert { key, data }, + ) +} + +const BAD_IPFS_HASH: &str = "bad-ipfs-hash"; + +async fn run_ipfs_map( + ipfs: IpfsClient, + subgraph_id: &'static str, + json_string: String, + api_version: Version, +) -> Result, anyhow::Error> { + let hash = if json_string == BAD_IPFS_HASH { + "Qm".to_string() + } else { + ipfs.add(json_string.into()).await.unwrap().hash + }; + + // Ipfs host functions use `block_on` which must be called from a sync context, + // so we replicate what we do `spawn_module`. + let runtime = tokio::runtime::Handle::current(); + std::thread::spawn(move || { + let _runtime_guard = runtime.enter(); + + let (mut module, _, _) = graph::block_on(test_valid_module_and_store( + &subgraph_id, + mock_data_source( + &wasm_file_path("ipfs_map.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + )); + + let value = module.asc_new(&hash).unwrap(); + let user_data = module.asc_new(USER_DATA).unwrap(); + + // Invoke the callback + let func = module.get_func("ipfsMap").typed().unwrap().clone(); + let _: () = func.call((value.wasm_ptr(), user_data.wasm_ptr()))?; + let mut mods = module + .take_ctx() + .ctx + .state + .entity_cache + .as_modifications()? + .modifications; + + // Bring the modifications into a predictable order (by entity_id) + mods.sort_by(|a, b| { + a.entity_ref() + .entity_id + .partial_cmp(&b.entity_ref().entity_id) + .unwrap() + }); + Ok(mods) + }) + .join() + .unwrap() +} + +async fn test_ipfs_map(api_version: Version, json_error_msg: &str) { + let ipfs = IpfsClient::localhost(); + let subgraph_id = "ipfsMap"; + + // Try it with two valid objects + let (str1, thing1) = make_thing("one", "eins"); + let (str2, thing2) = make_thing("two", "zwei"); + let ops = run_ipfs_map( + ipfs.clone(), + subgraph_id, + format!("{}\n{}", str1, str2), + api_version.clone(), + ) + .await + .expect("call failed"); + let expected = vec![thing1, thing2]; + assert_eq!(expected, ops); + + // Valid JSON, but not what the callback expected; it will + // fail on an assertion + let err = run_ipfs_map( + ipfs.clone(), + subgraph_id, + format!("{}\n[1,2]", str1), + api_version.clone(), + ) + .await + .unwrap_err(); + assert!( + format!("{:#}", err).contains("JSON value is not an object."), + "{:#}", + err + ); + + // Malformed JSON + let errmsg = run_ipfs_map( + ipfs.clone(), + subgraph_id, + format!("{}\n[", str1), + api_version.clone(), + ) + .await + .unwrap_err() + .to_string(); + assert!(errmsg.contains("EOF while parsing a list")); + + // Empty input + let ops = run_ipfs_map( + ipfs.clone(), + subgraph_id, + "".to_string(), + api_version.clone(), + ) + .await + .expect("call failed for emoty string"); + assert_eq!(0, ops.len()); + + // Missing entry in the JSON object + let errmsg = format!( + "{:#}", + run_ipfs_map( + ipfs.clone(), + subgraph_id, + "{\"value\": \"drei\"}".to_string(), + api_version.clone(), + ) + .await + .unwrap_err() + ); + assert!(errmsg.contains(json_error_msg)); + + // Bad IPFS hash. + let errmsg = run_ipfs_map( + ipfs.clone(), + subgraph_id, + BAD_IPFS_HASH.to_string(), + api_version.clone(), + ) + .await + .unwrap_err() + .to_string(); + assert!(errmsg.contains("500 Internal Server Error")); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ipfs_map_v0_0_4() { + test_ipfs_map(API_VERSION_0_0_4, "JSON value is not a string.").await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn ipfs_map_v0_0_5() { + test_ipfs_map(API_VERSION_0_0_5, "'id' should not be null").await; +} + +async fn test_ipfs_fail(api_version: Version) { + let runtime = tokio::runtime::Handle::current(); + + // Ipfs host functions use `block_on` which must be called from a sync context, + // so we replicate what we do `spawn_module`. + std::thread::spawn(move || { + let _runtime_guard = runtime.enter(); + + let mut module = graph::block_on(test_module( + "ipfsFail", + mock_data_source( + &wasm_file_path("ipfs_cat.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + )); + + assert!(module + .invoke_export1::<_, _, AscString>("ipfsCat", "invalid hash") + .is_null()); + }) + .join() + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ipfs_fail_v0_0_4() { + test_ipfs_fail(API_VERSION_0_0_4).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn ipfs_fail_v0_0_5() { + test_ipfs_fail(API_VERSION_0_0_5).await; +} + +async fn test_crypto_keccak256(api_version: Version) { + let mut module = test_module( + "cryptoKeccak256", + mock_data_source( + &wasm_file_path("crypto.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + let input: &[u8] = "eth".as_ref(); + + let hash: AscPtr = module.invoke_export1("hash", input); + let hash: Vec = module.asc_get(hash).unwrap(); + assert_eq!( + hex::encode(hash), + "4f5b812789fc606be1b3b16908db13fc7a9adf7ca72641f84d75b47069d3d7f0" + ); +} + +#[tokio::test] +async fn crypto_keccak256_v0_0_4() { + test_crypto_keccak256(API_VERSION_0_0_4).await; +} + +#[tokio::test] +async fn crypto_keccak256_v0_0_5() { + test_crypto_keccak256(API_VERSION_0_0_5).await; +} + +async fn test_big_int_to_hex(api_version: Version, gas_used: u64) { + let mut module = test_module( + "BigIntToHex", + mock_data_source( + &wasm_file_path("big_int_to_hex.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + // Convert zero to hex + let zero = BigInt::from_unsigned_u256(&U256::zero()); + let zero_hex_ptr: AscPtr = module.invoke_export1("big_int_to_hex", &zero); + let zero_hex_str: String = module.asc_get(zero_hex_ptr).unwrap(); + assert_eq!(zero_hex_str, "0x0"); + + // Convert 1 to hex + let one = BigInt::from_unsigned_u256(&U256::one()); + let one_hex_ptr: AscPtr = module.invoke_export1("big_int_to_hex", &one); + let one_hex_str: String = module.asc_get(one_hex_ptr).unwrap(); + assert_eq!(one_hex_str, "0x1"); + + // Convert U256::max_value() to hex + let u256_max = BigInt::from_unsigned_u256(&U256::max_value()); + let u256_max_hex_ptr: AscPtr = module.invoke_export1("big_int_to_hex", &u256_max); + let u256_max_hex_str: String = module.asc_get(u256_max_hex_ptr).unwrap(); + assert_eq!( + u256_max_hex_str, + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + ); + + assert_eq!(module.gas_used(), gas_used); +} + +#[tokio::test] +async fn big_int_to_hex_v0_0_4() { + test_big_int_to_hex(API_VERSION_0_0_4, 53113760).await; +} + +#[tokio::test] +async fn big_int_to_hex_v0_0_5() { + test_big_int_to_hex(API_VERSION_0_0_5, 2858580).await; +} + +async fn test_big_int_arithmetic(api_version: Version, gas_used: u64) { + let mut module = test_module( + "BigIntArithmetic", + mock_data_source( + &wasm_file_path("big_int_arithmetic.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + // 0 + 1 = 1 + let zero = BigInt::from(0); + let one = BigInt::from(1); + let result_ptr: AscPtr = module.invoke_export2("plus", &zero, &one); + let result: BigInt = module.asc_get(result_ptr).unwrap(); + assert_eq!(result, BigInt::from(1)); + + // 127 + 1 = 128 + let zero = BigInt::from(127); + let one = BigInt::from(1); + let result_ptr: AscPtr = module.invoke_export2("plus", &zero, &one); + let result: BigInt = module.asc_get(result_ptr).unwrap(); + assert_eq!(result, BigInt::from(128)); + + // 5 - 10 = -5 + let five = BigInt::from(5); + let ten = BigInt::from(10); + let result_ptr: AscPtr = module.invoke_export2("minus", &five, &ten); + let result: BigInt = module.asc_get(result_ptr).unwrap(); + assert_eq!(result, BigInt::from(-5)); + + // -20 * 5 = -100 + let minus_twenty = BigInt::from(-20); + let five = BigInt::from(5); + let result_ptr: AscPtr = module.invoke_export2("times", &minus_twenty, &five); + let result: BigInt = module.asc_get(result_ptr).unwrap(); + assert_eq!(result, BigInt::from(-100)); + + // 5 / 2 = 2 + let five = BigInt::from(5); + let two = BigInt::from(2); + let result_ptr: AscPtr = module.invoke_export2("dividedBy", &five, &two); + let result: BigInt = module.asc_get(result_ptr).unwrap(); + assert_eq!(result, BigInt::from(2)); + + // 5 % 2 = 1 + let five = BigInt::from(5); + let two = BigInt::from(2); + let result_ptr: AscPtr = module.invoke_export2("mod", &five, &two); + let result: BigInt = module.asc_get(result_ptr).unwrap(); + assert_eq!(result, BigInt::from(1)); + + assert_eq!(module.gas_used(), gas_used); +} + +#[tokio::test] +async fn big_int_arithmetic_v0_0_4() { + test_big_int_arithmetic(API_VERSION_0_0_4, 54962411).await; +} + +#[tokio::test] +async fn big_int_arithmetic_v0_0_5() { + test_big_int_arithmetic(API_VERSION_0_0_5, 7318364).await; +} + +async fn test_abort(api_version: Version, error_msg: &str) { + let module = test_module( + "abort", + mock_data_source( + &wasm_file_path("abort.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + let res: Result<(), _> = module.get_func("abort").typed().unwrap().call(()); + assert!(res.unwrap_err().to_string().contains(error_msg)); +} + +#[tokio::test] +async fn abort_v0_0_4() { + test_abort( + API_VERSION_0_0_4, + "line 6, column 2, with message: not true", + ) + .await; +} + +#[tokio::test] +async fn abort_v0_0_5() { + test_abort( + API_VERSION_0_0_5, + "line 4, column 3, with message: not true", + ) + .await; +} + +async fn test_bytes_to_base58(api_version: Version, gas_used: u64) { + let mut module = test_module( + "bytesToBase58", + mock_data_source( + &wasm_file_path("bytes_to_base58.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + let bytes = hex::decode("12207D5A99F603F231D53A4F39D1521F98D2E8BB279CF29BEBFD0687DC98458E7F89") + .unwrap(); + let result_ptr: AscPtr = module.invoke_export1("bytes_to_base58", bytes.as_slice()); + let base58: String = module.asc_get(result_ptr).unwrap(); + + assert_eq!(base58, "QmWmyoMoctfbAaiEs2G46gpeUmhqFRDW6KWo64y5r581Vz"); + assert_eq!(module.gas_used(), gas_used); +} + +#[tokio::test] +async fn bytes_to_base58_v0_0_4() { + test_bytes_to_base58(API_VERSION_0_0_4, 52301689).await; +} + +#[tokio::test] +async fn bytes_to_base58_v0_0_5() { + test_bytes_to_base58(API_VERSION_0_0_5, 1310019).await; +} + +async fn test_data_source_create(api_version: Version, gas_used: u64) { + // Test with a valid template + let template = String::from("example template"); + let params = vec![String::from("0xc0a47dFe034B400B47bDaD5FecDa2621de6c4d95")]; + let result = run_data_source_create( + template.clone(), + params.clone(), + api_version.clone(), + gas_used, + ) + .await + .expect("unexpected error returned from dataSourceCreate"); + assert_eq!(result[0].params, params.clone()); + assert_eq!(result[0].template.name(), template); + + // Test with a template that doesn't exist + let template = String::from("nonexistent template"); + let params = vec![String::from("0xc000000000000000000000000000000000000000")]; + match run_data_source_create(template.clone(), params.clone(), api_version, gas_used).await { + Ok(_) => panic!("expected an error because the template does not exist"), + Err(e) => assert!(e.to_string().contains( + "Failed to create data source from name `nonexistent template`: \ + No template with this name in parent data source `example data source`. \ + Available names: example template." + )), + }; +} + +async fn run_data_source_create( + name: String, + params: Vec, + api_version: Version, + gas_used: u64, +) -> Result>, wasmtime::Trap> { + let mut module = test_module( + "DataSourceCreate", + mock_data_source( + &wasm_file_path("data_source_create.wasm", api_version.clone()), + api_version.clone(), + ), + api_version.clone(), + ) + .await; + + module.instance_ctx_mut().ctx.state.enter_handler(); + module.invoke_export2_void("dataSourceCreate", &name, ¶ms)?; + module.instance_ctx_mut().ctx.state.exit_handler(); + + assert_eq!(module.gas_used(), gas_used); + + Ok(module.take_ctx().ctx.state.drain_created_data_sources()) +} + +#[tokio::test] +async fn data_source_create_v0_0_4() { + test_data_source_create(API_VERSION_0_0_4, 152102833).await; +} + +#[tokio::test] +async fn data_source_create_v0_0_5() { + test_data_source_create(API_VERSION_0_0_5, 101450079).await; +} + +async fn test_ens_name_by_hash(api_version: Version) { + let mut module = test_module( + "EnsNameByHash", + mock_data_source( + &wasm_file_path("ens_name_by_hash.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + let hash = "0x7f0c1b04d1a4926f9c635a030eeb611d4c26e5e73291b32a1c7a4ac56935b5b3"; + let name = "dealdrafts"; + test_store::insert_ens_name(hash, name); + let converted: AscPtr = module.invoke_export1("nameByHash", hash); + let data: String = module.asc_get(converted).unwrap(); + assert_eq!(data, name); + + assert!(module + .invoke_export1::<_, _, AscString>("nameByHash", "impossible keccak hash") + .is_null()); +} + +#[tokio::test] +async fn ens_name_by_hash_v0_0_4() { + test_ens_name_by_hash(API_VERSION_0_0_4).await; +} + +#[tokio::test] +async fn ens_name_by_hash_v0_0_5() { + test_ens_name_by_hash(API_VERSION_0_0_5).await; +} + +async fn test_entity_store(api_version: Version) { + let (mut module, store, deployment) = test_valid_module_and_store( + "entityStore", + mock_data_source( + &wasm_file_path("store.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + let mut alex = Entity::new(); + alex.set("id", "alex"); + alex.set("name", "Alex"); + let mut steve = Entity::new(); + steve.set("id", "steve"); + steve.set("name", "Steve"); + let user_type = EntityType::from("User"); + test_store::insert_entities( + &deployment, + vec![(user_type.clone(), alex), (user_type, steve)], + ) + .await + .unwrap(); + + let get_user = move |module: &mut WasmInstance, id: &str| -> Option { + let entity_ptr: AscPtr = module.invoke_export1("getUser", id); + if entity_ptr.is_null() { + None + } else { + Some(Entity::from( + module + .asc_get::, _>(entity_ptr) + .unwrap(), + )) + } + }; + + let load_and_set_user_name = |module: &mut WasmInstance, id: &str, name: &str| { + module + .invoke_export2_void("loadAndSetUserName", id, name) + .unwrap(); + }; + + // store.get of a nonexistent user + assert_eq!(None, get_user(&mut module, "herobrine")); + // store.get of an existing user + let steve = get_user(&mut module, "steve").unwrap(); + assert_eq!(Some(&Value::from("Steve")), steve.get("name")); + + // Load, set, save cycle for an existing entity + load_and_set_user_name(&mut module, "steve", "Steve-O"); + + // We need to empty the cache for the next test + let writable = store.writable(LOGGER.clone(), deployment.id).await.unwrap(); + let cache = std::mem::replace( + &mut module.instance_ctx_mut().ctx.state.entity_cache, + EntityCache::new(Arc::new(writable.clone())), + ); + let mut mods = cache.as_modifications().unwrap().modifications; + assert_eq!(1, mods.len()); + match mods.pop().unwrap() { + EntityModification::Overwrite { data, .. } => { + assert_eq!(Some(&Value::from("steve")), data.get("id")); + assert_eq!(Some(&Value::from("Steve-O")), data.get("name")); + } + _ => assert!(false, "expected Overwrite modification"), + } + + // Load, set, save cycle for a new entity with fulltext API + load_and_set_user_name(&mut module, "herobrine", "Brine-O"); + let mut fulltext_entities = BTreeMap::new(); + let mut fulltext_fields = BTreeMap::new(); + fulltext_fields.insert("name".to_string(), vec!["search".to_string()]); + fulltext_entities.insert("User".to_string(), fulltext_fields); + let mut mods = module + .take_ctx() + .ctx + .state + .entity_cache + .as_modifications() + .unwrap() + .modifications; + assert_eq!(1, mods.len()); + match mods.pop().unwrap() { + EntityModification::Insert { data, .. } => { + assert_eq!(Some(&Value::from("herobrine")), data.get("id")); + assert_eq!(Some(&Value::from("Brine-O")), data.get("name")); + } + _ => assert!(false, "expected Insert modification"), + }; +} + +#[tokio::test] +async fn entity_store_v0_0_4() { + test_entity_store(API_VERSION_0_0_4).await; +} + +#[tokio::test] +async fn entity_store_v0_0_5() { + test_entity_store(API_VERSION_0_0_5).await; +} + +fn test_detect_contract_calls(api_version: Version) { + let data_source_without_calls = mock_data_source( + &wasm_file_path("abi_store_value.wasm", api_version.clone()), + api_version.clone(), + ); + assert_eq!( + data_source_without_calls + .mapping + .requires_archive() + .unwrap(), + false + ); + + let data_source_with_calls = mock_data_source( + &wasm_file_path("contract_calls.wasm", api_version.clone()), + api_version, + ); + assert_eq!( + data_source_with_calls.mapping.requires_archive().unwrap(), + true + ); +} + +#[tokio::test] +async fn detect_contract_calls_v0_0_4() { + test_detect_contract_calls(API_VERSION_0_0_4); +} + +#[tokio::test] +async fn detect_contract_calls_v0_0_5() { + test_detect_contract_calls(API_VERSION_0_0_5); +} + +async fn test_allocate_global(api_version: Version) { + let module = test_module( + "AllocateGlobal", + mock_data_source( + &wasm_file_path("allocate_global.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + // Assert globals can be allocated and don't break the heap + module.invoke_export0_void("assert_global_works").unwrap(); +} + +#[tokio::test] +async fn allocate_global_v0_0_5() { + // Only in apiVersion v0.0.5 because there's no issue in older versions. + // The problem with the new one is related to the AS stub runtime `offset` + // variable not being initialized (lazy) before we use it so this test checks + // that it works (at the moment using __alloc call to force offset to be eagerly + // evaluated). + test_allocate_global(API_VERSION_0_0_5).await; +} + +async fn test_null_ptr_read(api_version: Version) { + let module = test_module( + "NullPtrRead", + mock_data_source( + &wasm_file_path("null_ptr_read.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + module.invoke_export0_void("nullPtrRead").unwrap(); +} + +#[tokio::test] +#[should_panic(expected = "Tried to read AssemblyScript value that is 'null'")] +async fn null_ptr_read_0_0_5() { + test_null_ptr_read(API_VERSION_0_0_5).await; +} + +async fn test_safe_null_ptr_read(api_version: Version) { + let module = test_module( + "SafeNullPtrRead", + mock_data_source( + &wasm_file_path("null_ptr_read.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + module.invoke_export0_void("safeNullPtrRead").unwrap(); +} + +#[tokio::test] +#[should_panic(expected = "Failed to sum BigInts because left hand side is 'null'")] +async fn safe_null_ptr_read_0_0_5() { + test_safe_null_ptr_read(API_VERSION_0_0_5).await; +} + +#[ignore] // Ignored because of long run time in debug build. +#[tokio::test] +async fn test_array_blowup() { + let module = test_module_latest("ArrayBlowup", "array_blowup.wasm").await; + + assert!(module + .invoke_export0_void("arrayBlowup") + .unwrap_err() + .to_string() + .contains("Gas limit exceeded. Used: 11286295575421")); +} + +#[tokio::test] +async fn test_boolean() { + let mut module = test_module_latest("boolean", "boolean.wasm").await; + + let true_: i32 = module.invoke_export0_val("testReturnTrue"); + assert_eq!(true_, 1); + + let false_: i32 = module.invoke_export0_val("testReturnFalse"); + assert_eq!(false_, 0); + + // non-zero values are true + for x in (-10i32..10).filter(|&x| x != 0) { + assert!(module.invoke_export1_val_void("testReceiveTrue", x).is_ok(),); + } + + // zero is not true + assert!(module + .invoke_export1_val_void("testReceiveTrue", 0i32) + .is_err()); + + // zero is false + assert!(module + .invoke_export1_val_void("testReceiveFalse", 0i32) + .is_ok()); + + // non-zero values are not false + for x in (-10i32..10).filter(|&x| x != 0) { + assert!(module + .invoke_export1_val_void("testReceiveFalse", x) + .is_err()); + } +} diff --git a/runtime/test/src/test/abi.rs b/runtime/test/src/test/abi.rs new file mode 100644 index 0000000..dc62f44 --- /dev/null +++ b/runtime/test/src/test/abi.rs @@ -0,0 +1,529 @@ +use graph::prelude::{ethabi::Token, web3::types::U256}; +use graph_runtime_wasm::{ + asc_abi::class::{ + ArrayBuffer, AscAddress, AscEnum, AscEnumArray, EthereumValueKind, StoreValueKind, + TypedArray, + }, + TRAP_TIMEOUT, +}; + +use super::*; + +async fn test_unbounded_loop(api_version: Version) { + // Set handler timeout to 3 seconds. + let module = test_valid_module_and_store_with_timeout( + "unboundedLoop", + mock_data_source( + &wasm_file_path("non_terminating.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + Some(Duration::from_secs(3)), + ) + .await + .0; + let res: Result<(), _> = module.get_func("loop").typed().unwrap().call(()); + assert!(res.unwrap_err().to_string().contains(TRAP_TIMEOUT)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn unbounded_loop_v0_0_4() { + test_unbounded_loop(API_VERSION_0_0_4).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn unbounded_loop_v0_0_5() { + test_unbounded_loop(API_VERSION_0_0_5).await; +} + +async fn test_unbounded_recursion(api_version: Version) { + let module = test_module( + "unboundedRecursion", + mock_data_source( + &wasm_file_path("non_terminating.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + let res: Result<(), _> = module.get_func("rabbit_hole").typed().unwrap().call(()); + let err_msg = res.unwrap_err().to_string(); + assert!(err_msg.contains("call stack exhausted"), "{:#?}", err_msg); +} + +#[tokio::test] +async fn unbounded_recursion_v0_0_4() { + test_unbounded_recursion(API_VERSION_0_0_4).await; +} + +#[tokio::test] +async fn unbounded_recursion_v0_0_5() { + test_unbounded_recursion(API_VERSION_0_0_5).await; +} + +async fn test_abi_array(api_version: Version, gas_used: u64) { + let mut module = test_module( + "abiArray", + mock_data_source( + &wasm_file_path("abi_classes.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + let vec = vec![ + "1".to_owned(), + "2".to_owned(), + "3".to_owned(), + "4".to_owned(), + ]; + let new_vec_obj: AscPtr>> = module.invoke_export1("test_array", &vec); + let new_vec: Vec = module.asc_get(new_vec_obj).unwrap(); + + assert_eq!(module.gas_used(), gas_used); + assert_eq!( + new_vec, + vec![ + "1".to_owned(), + "2".to_owned(), + "3".to_owned(), + "4".to_owned(), + "5".to_owned(), + ] + ); +} + +#[tokio::test] +async fn abi_array_v0_0_4() { + test_abi_array(API_VERSION_0_0_4, 695935).await; +} + +#[tokio::test] +async fn abi_array_v0_0_5() { + test_abi_array(API_VERSION_0_0_5, 1636130).await; +} + +async fn test_abi_subarray(api_version: Version) { + let mut module = test_module( + "abiSubarray", + mock_data_source( + &wasm_file_path("abi_classes.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + let vec: Vec = vec![1, 2, 3, 4]; + let new_vec_obj: AscPtr> = + module.invoke_export1("byte_array_third_quarter", vec.as_slice()); + let new_vec: Vec = module.asc_get(new_vec_obj).unwrap(); + + assert_eq!(new_vec, vec![3]); +} + +#[tokio::test] +async fn abi_subarray_v0_0_4() { + test_abi_subarray(API_VERSION_0_0_4).await; +} + +#[tokio::test] +async fn abi_subarray_v0_0_5() { + test_abi_subarray(API_VERSION_0_0_5).await; +} + +async fn test_abi_bytes_and_fixed_bytes(api_version: Version) { + let mut module = test_module( + "abiBytesAndFixedBytes", + mock_data_source( + &wasm_file_path("abi_classes.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + let bytes1: Vec = vec![42, 45, 7, 245, 45]; + let bytes2: Vec = vec![3, 12, 0, 1, 255]; + let new_vec_obj: AscPtr = module.invoke_export2("concat", &*bytes1, &*bytes2); + + // This should be bytes1 and bytes2 concatenated. + let new_vec: Vec = module.asc_get(new_vec_obj).unwrap(); + + let mut concated = bytes1.clone(); + concated.extend(bytes2.clone()); + assert_eq!(new_vec, concated); +} + +#[tokio::test] +async fn abi_bytes_and_fixed_bytes_v0_0_4() { + test_abi_bytes_and_fixed_bytes(API_VERSION_0_0_4).await; +} + +#[tokio::test] +async fn abi_bytes_and_fixed_bytes_v0_0_5() { + test_abi_bytes_and_fixed_bytes(API_VERSION_0_0_5).await; +} + +async fn test_abi_ethabi_token_identity(api_version: Version) { + let mut module = test_module( + "abiEthabiTokenIdentity", + mock_data_source( + &wasm_file_path("abi_token.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + // Token::Address + let address = H160([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]); + let token_address = Token::Address(address); + + let new_address_obj: AscPtr = + module.invoke_export1("token_to_address", &token_address); + + let new_token_ptr = module.takes_ptr_returns_ptr("token_from_address", new_address_obj); + let new_token = module.asc_get(new_token_ptr).unwrap(); + + assert_eq!(token_address, new_token); + + // Token::Bytes + let token_bytes = Token::Bytes(vec![42, 45, 7, 245, 45]); + let new_bytes_obj: AscPtr = module.invoke_export1("token_to_bytes", &token_bytes); + let new_token_ptr = module.takes_ptr_returns_ptr("token_from_bytes", new_bytes_obj); + let new_token = module.asc_get(new_token_ptr).unwrap(); + + assert_eq!(token_bytes, new_token); + + // Token::Int + let int_token = Token::Int(U256([256, 453452345, 0, 42])); + let new_int_obj: AscPtr = module.invoke_export1("token_to_int", &int_token); + + let new_token_ptr = module.takes_ptr_returns_ptr("token_from_int", new_int_obj); + let new_token = module.asc_get(new_token_ptr).unwrap(); + + assert_eq!(int_token, new_token); + + // Token::Uint + let uint_token = Token::Uint(U256([256, 453452345, 0, 42])); + + let new_uint_obj: AscPtr = module.invoke_export1("token_to_uint", &uint_token); + let new_token_ptr = module.takes_ptr_returns_ptr("token_from_uint", new_uint_obj); + let new_token = module.asc_get(new_token_ptr).unwrap(); + + assert_eq!(uint_token, new_token); + assert_ne!(uint_token, int_token); + + // Token::Bool + let token_bool = Token::Bool(true); + + let token_bool_ptr = module.asc_new(&token_bool).unwrap(); + let func = module.get_func("token_to_bool").typed().unwrap().clone(); + let boolean: i32 = func.call(token_bool_ptr.wasm_ptr()).unwrap(); + + let new_token_ptr = module.takes_val_returns_ptr("token_from_bool", boolean); + let new_token = module.asc_get(new_token_ptr).unwrap(); + + assert_eq!(token_bool, new_token); + + // Token::String + let token_string = Token::String("漢字Go🇧🇷".into()); + let new_string_obj: AscPtr = module.invoke_export1("token_to_string", &token_string); + let new_token_ptr = module.takes_ptr_returns_ptr("token_from_string", new_string_obj); + let new_token = module.asc_get(new_token_ptr).unwrap(); + + assert_eq!(token_string, new_token); + + // Token::Array + let token_array = Token::Array(vec![token_address, token_bytes, token_bool]); + let token_array_nested = Token::Array(vec![token_string, token_array]); + let new_array_obj: AscEnumArray = + module.invoke_export1("token_to_array", &token_array_nested); + + let new_token_ptr = module.takes_ptr_returns_ptr("token_from_array", new_array_obj); + let new_token: Token = module.asc_get(new_token_ptr).unwrap(); + + assert_eq!(new_token, token_array_nested); +} + +/// Test a roundtrip Token -> Payload -> Token identity conversion through asc, +/// and assert the final token is the same as the starting one. +#[tokio::test] +async fn abi_ethabi_token_identity_v0_0_4() { + test_abi_ethabi_token_identity(API_VERSION_0_0_4).await; +} + +/// Test a roundtrip Token -> Payload -> Token identity conversion through asc, +/// and assert the final token is the same as the starting one. +#[tokio::test] +async fn abi_ethabi_token_identity_v0_0_5() { + test_abi_ethabi_token_identity(API_VERSION_0_0_5).await; +} + +async fn test_abi_store_value(api_version: Version) { + let mut module = test_module( + "abiStoreValue", + mock_data_source( + &wasm_file_path("abi_store_value.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + // Value::Null + let func = module.get_func("value_null").typed().unwrap().clone(); + let ptr: u32 = func.call(()).unwrap(); + let null_value_ptr: AscPtr> = ptr.into(); + let null_value: Value = module.asc_get(null_value_ptr).unwrap(); + assert_eq!(null_value, Value::Null); + + // Value::String + let string = "some string"; + let new_value_ptr = module.invoke_export1("value_from_string", string); + let new_value: Value = module.asc_get(new_value_ptr).unwrap(); + assert_eq!(new_value, Value::from(string)); + + // Value::Int + let int = i32::min_value(); + let new_value_ptr = module.takes_val_returns_ptr("value_from_int", int); + let new_value: Value = module.asc_get(new_value_ptr).unwrap(); + assert_eq!(new_value, Value::Int(int)); + + // Value::BigDecimal + let big_decimal = BigDecimal::from_str("3.14159001").unwrap(); + let new_value_ptr = module.invoke_export1("value_from_big_decimal", &big_decimal); + let new_value: Value = module.asc_get(new_value_ptr).unwrap(); + assert_eq!(new_value, Value::BigDecimal(big_decimal)); + + let big_decimal = BigDecimal::new(10.into(), 5); + let new_value_ptr = module.invoke_export1("value_from_big_decimal", &big_decimal); + let new_value: Value = module.asc_get(new_value_ptr).unwrap(); + assert_eq!(new_value, Value::BigDecimal(1_000_000.into())); + + // Value::Bool + let boolean = true; + let new_value_ptr = + module.takes_val_returns_ptr("value_from_bool", if boolean { 1 } else { 0 }); + let new_value: Value = module.asc_get(new_value_ptr).unwrap(); + assert_eq!(new_value, Value::Bool(boolean)); + + // Value::List + let func = module + .get_func("array_from_values") + .typed() + .unwrap() + .clone(); + let new_value_ptr: u32 = func + .call((module.asc_new(string).unwrap().wasm_ptr(), int)) + .unwrap(); + let new_value_ptr = AscPtr::from(new_value_ptr); + let new_value: Value = module.asc_get(new_value_ptr).unwrap(); + assert_eq!( + new_value, + Value::List(vec![Value::from(string), Value::Int(int)]) + ); + + let array: &[Value] = &[ + Value::String("foo".to_owned()), + Value::String("bar".to_owned()), + ]; + let new_value_ptr = module.invoke_export1("value_from_array", array); + let new_value: Value = module.asc_get(new_value_ptr).unwrap(); + assert_eq!( + new_value, + Value::List(vec![ + Value::String("foo".to_owned()), + Value::String("bar".to_owned()), + ]) + ); + + // Value::Bytes + let bytes: &[u8] = &[0, 2, 5]; + let new_value_ptr = module.invoke_export1("value_from_bytes", bytes); + let new_value: Value = module.asc_get(new_value_ptr).unwrap(); + assert_eq!(new_value, Value::Bytes(bytes.into())); + + // Value::BigInt + let bytes: &[u8] = &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]; + let new_value_ptr = module.invoke_export1("value_from_bigint", bytes); + let new_value: Value = module.asc_get(new_value_ptr).unwrap(); + assert_eq!( + new_value, + Value::BigInt(::graph::data::store::scalar::BigInt::from_unsigned_bytes_le(bytes)) + ); +} + +#[tokio::test] +async fn abi_store_value_v0_0_4() { + test_abi_store_value(API_VERSION_0_0_4).await; +} + +#[tokio::test] +async fn abi_store_value_v0_0_5() { + test_abi_store_value(API_VERSION_0_0_5).await; +} + +async fn test_abi_h160(api_version: Version) { + let mut module = test_module( + "abiH160", + mock_data_source( + &wasm_file_path("abi_classes.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + let address = H160::zero(); + + // As an `Uint8Array` + let new_address_obj: AscPtr = module.invoke_export1("test_address", &address); + + // This should have 1 added to the first and last byte. + let new_address: H160 = module.asc_get(new_address_obj).unwrap(); + + assert_eq!( + new_address, + H160([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]) + ) +} + +#[tokio::test] +async fn abi_h160_v0_0_4() { + test_abi_h160(API_VERSION_0_0_4).await; +} + +#[tokio::test] +async fn abi_h160_v0_0_5() { + test_abi_h160(API_VERSION_0_0_5).await; +} + +async fn test_string(api_version: Version) { + let mut module = test_module( + "string", + mock_data_source( + &wasm_file_path("abi_classes.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + let string = " 漢字Double_Me🇧🇷 "; + let trimmed_string_obj: AscPtr = module.invoke_export1("repeat_twice", string); + let doubled_string: String = module.asc_get(trimmed_string_obj).unwrap(); + assert_eq!(doubled_string, string.repeat(2)); +} + +#[tokio::test] +async fn string_v0_0_4() { + test_string(API_VERSION_0_0_4).await; +} + +#[tokio::test] +async fn string_v0_0_5() { + test_string(API_VERSION_0_0_5).await; +} + +async fn test_abi_big_int(api_version: Version) { + let mut module = test_module( + "abiBigInt", + mock_data_source( + &wasm_file_path("abi_classes.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + // Test passing in 0 and increment it by 1 + let old_uint = U256::zero(); + let new_uint_obj: AscPtr = + module.invoke_export1("test_uint", &BigInt::from_unsigned_u256(&old_uint)); + let new_uint: BigInt = module.asc_get(new_uint_obj).unwrap(); + assert_eq!(new_uint, BigInt::from(1 as i32)); + let new_uint = new_uint.to_unsigned_u256(); + assert_eq!(new_uint, U256([1, 0, 0, 0])); + + // Test passing in -50 and increment it by 1 + let old_uint = BigInt::from(-50); + let new_uint_obj: AscPtr = module.invoke_export1("test_uint", &old_uint); + let new_uint: BigInt = module.asc_get(new_uint_obj).unwrap(); + assert_eq!(new_uint, BigInt::from(-49 as i32)); + let new_uint_from_u256 = BigInt::from_signed_u256(&new_uint.to_signed_u256()); + assert_eq!(new_uint, new_uint_from_u256); +} + +#[tokio::test] +async fn abi_big_int_v0_0_4() { + test_abi_big_int(API_VERSION_0_0_4).await; +} + +#[tokio::test] +async fn abi_big_int_v0_0_5() { + test_abi_big_int(API_VERSION_0_0_5).await; +} + +async fn test_big_int_to_string(api_version: Version) { + let mut module = test_module( + "bigIntToString", + mock_data_source( + &wasm_file_path("big_int_to_string.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + let big_int_str = "30145144166666665000000000000000000"; + let big_int = BigInt::from_str(big_int_str).unwrap(); + let string_obj: AscPtr = module.invoke_export1("big_int_to_string", &big_int); + let string: String = module.asc_get(string_obj).unwrap(); + assert_eq!(string, big_int_str); +} + +#[tokio::test] +async fn big_int_to_string_v0_0_4() { + test_big_int_to_string(API_VERSION_0_0_4).await; +} + +#[tokio::test] +async fn big_int_to_string_v0_0_5() { + test_big_int_to_string(API_VERSION_0_0_5).await; +} + +async fn test_invalid_discriminant(api_version: Version) { + let module = test_module( + "invalidDiscriminant", + mock_data_source( + &wasm_file_path("abi_store_value.wasm", api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + let func = module + .get_func("invalid_discriminant") + .typed() + .unwrap() + .clone(); + let ptr: u32 = func.call(()).unwrap(); + let _value: Value = module.asc_get(ptr.into()).unwrap(); +} + +// This should panic rather than exhibiting UB. It's hard to test for UB, but +// when reproducing a SIGILL was observed which would be caught by this. +#[tokio::test] +#[should_panic] +async fn invalid_discriminant_v0_0_4() { + test_invalid_discriminant(API_VERSION_0_0_4).await; +} + +// This should panic rather than exhibiting UB. It's hard to test for UB, but +// when reproducing a SIGILL was observed which would be caught by this. +#[tokio::test] +#[should_panic] +async fn invalid_discriminant_v0_0_5() { + test_invalid_discriminant(API_VERSION_0_0_5).await; +} diff --git a/runtime/test/src/test_padding.rs b/runtime/test/src/test_padding.rs new file mode 100644 index 0000000..85da8f3 --- /dev/null +++ b/runtime/test/src/test_padding.rs @@ -0,0 +1,429 @@ +use crate::protobuf; +use graph::prelude::tokio; + +use self::data::BadFixed; + +const WASM_FILE_NAME: &str = "test_padding.wasm"; + +//for tests, to run in parallel, sub graph name has be unique +fn rnd_sub_graph_name(size: usize) -> String { + use rand::{distributions::Alphanumeric, Rng}; + rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(size) + .map(char::from) + .collect() +} + +pub mod data { + #[graph_runtime_derive::generate_asc_type()] + #[graph_runtime_derive::generate_network_type_id(UnitTestNetwork)] + #[graph_runtime_derive::generate_from_rust_type()] + #[graph_runtime_derive::generate_array_type(UnitTestNetwork)] + #[derive(Debug, PartialEq)] + pub struct UnitTestTypeBool { + pub str_pref: String, + pub under_test: bool, + pub str_suff: String, + pub large: i64, + pub tail: bool, + } + + #[graph_runtime_derive::generate_asc_type()] + #[graph_runtime_derive::generate_network_type_id(UnitTestNetwork)] + #[graph_runtime_derive::generate_from_rust_type()] + #[graph_runtime_derive::generate_array_type(UnitTestNetwork)] + #[derive(Debug, PartialEq)] + pub struct UnitTestTypeI8 { + pub str_pref: String, + pub under_test: i8, + pub str_suff: String, + pub large: i64, + pub tail: bool, + } + #[graph_runtime_derive::generate_asc_type()] + #[graph_runtime_derive::generate_network_type_id(UnitTestNetwork)] + #[graph_runtime_derive::generate_from_rust_type()] + #[graph_runtime_derive::generate_array_type(UnitTestNetwork)] + #[derive(Debug, PartialEq)] + pub struct UnitTestTypeU16 { + pub str_pref: String, + pub under_test: u16, + pub str_suff: String, + pub large: i64, + pub tail: bool, + } + #[graph_runtime_derive::generate_asc_type()] + #[graph_runtime_derive::generate_network_type_id(UnitTestNetwork)] + #[graph_runtime_derive::generate_from_rust_type()] + #[graph_runtime_derive::generate_array_type(UnitTestNetwork)] + #[derive(Debug, PartialEq)] + pub struct UnitTestTypeU32 { + pub str_pref: String, + pub under_test: u32, + pub str_suff: String, + pub large: i64, + pub tail: bool, + } + + pub struct Bad { + pub nonce: u64, + pub str_suff: String, + pub tail: u64, + } + + #[repr(C)] + pub struct AscBad { + pub nonce: u64, + pub str_suff: graph::runtime::AscPtr, + pub tail: u64, + } + + impl AscType for AscBad { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + let in_memory_byte_count = std::mem::size_of::(); + let mut bytes = Vec::with_capacity(in_memory_byte_count); + + bytes.extend_from_slice(&self.nonce.to_asc_bytes()?); + bytes.extend_from_slice(&self.str_suff.to_asc_bytes()?); + bytes.extend_from_slice(&self.tail.to_asc_bytes()?); + + //ensure misaligned + assert!( + bytes.len() != in_memory_byte_count, + "struct is intentionally misaligned", + ); + Ok(bytes) + } + + fn from_asc_bytes( + _asc_obj: &[u8], + _api_version: &graph::semver::Version, + ) -> Result { + unimplemented!(); + } + } + + impl AscIndexId for AscBad { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = + IndexForAscTypeId::UnitTestNetworkUnitTestTypeBool; + } + + pub use graph::runtime::{ + asc_new, gas::GasCounter, AscHeap, AscIndexId, AscPtr, AscType, AscValue, + DeterministicHostError, IndexForAscTypeId, ToAscObj, + }; + + impl ToAscObj for Bad { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscBad { + nonce: self.nonce, + str_suff: asc_new(heap, &self.str_suff, gas)?, + tail: self.tail, + }) + } + } + + pub struct BadFixed { + pub nonce: u64, + pub str_suff: String, + pub tail: u64, + } + #[repr(C)] + pub struct AscBadFixed { + pub nonce: u64, + pub str_suff: graph::runtime::AscPtr, + pub _padding: u32, + pub tail: u64, + } + + impl AscType for AscBadFixed { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + let in_memory_byte_count = std::mem::size_of::(); + let mut bytes = Vec::with_capacity(in_memory_byte_count); + + bytes.extend_from_slice(&self.nonce.to_asc_bytes()?); + bytes.extend_from_slice(&self.str_suff.to_asc_bytes()?); + bytes.extend_from_slice(&self._padding.to_asc_bytes()?); + bytes.extend_from_slice(&self.tail.to_asc_bytes()?); + + assert_eq!( + bytes.len(), + in_memory_byte_count, + "Alignment mismatch for AscBadFixed, re-order fields or explicitely add a _padding field", + ); + Ok(bytes) + } + + fn from_asc_bytes( + _asc_obj: &[u8], + _api_version: &graph::semver::Version, + ) -> Result { + unimplemented!(); + } + } + + //we will have to keep this chain specific (Inner/Outer) + impl AscIndexId for AscBadFixed { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = + IndexForAscTypeId::UnitTestNetworkUnitTestTypeBool; + } + + impl ToAscObj for BadFixed { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscBadFixed { + nonce: self.nonce, + str_suff: asc_new(heap, &self.str_suff, gas)?, + _padding: 0, + tail: self.tail, + }) + } + } +} + +#[tokio::test] +async fn test_v5_manual_padding_manualy_fixed_ok() { + manual_padding_manualy_fixed_ok(super::test::API_VERSION_0_0_5).await +} + +#[tokio::test] +async fn test_v4_manual_padding_manualy_fixed_ok() { + manual_padding_manualy_fixed_ok(super::test::API_VERSION_0_0_4).await +} + +#[tokio::test] +async fn test_v5_manual_padding_should_fail() { + manual_padding_should_fail(super::test::API_VERSION_0_0_5).await +} + +#[tokio::test] +async fn test_v4_manual_padding_should_fail() { + manual_padding_should_fail(super::test::API_VERSION_0_0_4).await +} + +#[tokio::test] +async fn test_v5_bool_padding_ok() { + bool_padding_ok(super::test::API_VERSION_0_0_5).await +} + +#[tokio::test] +async fn test_v4_bool_padding_ok() { + bool_padding_ok(super::test::API_VERSION_0_0_4).await +} + +#[tokio::test] +async fn test_v5_i8_padding_ok() { + i8_padding_ok(super::test::API_VERSION_0_0_5).await +} + +#[tokio::test] +async fn test_v4_i8_padding_ok() { + i8_padding_ok(super::test::API_VERSION_0_0_4).await +} + +#[tokio::test] +async fn test_v5_u16_padding_ok() { + u16_padding_ok(super::test::API_VERSION_0_0_5).await +} + +#[tokio::test] +async fn test_v4_u16_padding_ok() { + u16_padding_ok(super::test::API_VERSION_0_0_4).await +} + +#[tokio::test] +async fn test_v5_u32_padding_ok() { + u32_padding_ok(super::test::API_VERSION_0_0_5).await +} + +#[tokio::test] +async fn test_v4_u32_padding_ok() { + u32_padding_ok(super::test::API_VERSION_0_0_4).await +} + +async fn manual_padding_should_fail(api_version: semver::Version) { + let mut module = super::test::test_module( + &rnd_sub_graph_name(12), + super::common::mock_data_source( + &super::test::wasm_file_path(WASM_FILE_NAME, api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + let parm = protobuf::Bad { + nonce: i64::MAX as u64, + str_suff: "suff".into(), + tail: i64::MAX as u64, + }; + + let new_obj = module.asc_new(&parm).unwrap(); + + let func = module + .get_func("test_padding_manual") + .typed() + .unwrap() + .clone(); + + let res: Result<(), _> = func.call(new_obj.wasm_ptr()); + + assert!( + res.is_err(), + "suposed to fail due to WASM memory padding error" + ); +} + +async fn manual_padding_manualy_fixed_ok(api_version: semver::Version) { + let parm = BadFixed { + nonce: i64::MAX as u64, + str_suff: "suff".into(), + tail: i64::MAX as u64, + }; + + let mut module = super::test::test_module( + &rnd_sub_graph_name(12), + super::common::mock_data_source( + &super::test::wasm_file_path(WASM_FILE_NAME, api_version.clone()), + api_version.clone(), + ), + api_version.clone(), + ) + .await; + + let new_obj = module.asc_new(&parm).unwrap(); + + let func = module + .get_func("test_padding_manual") + .typed() + .unwrap() + .clone(); + + let res: Result<(), _> = func.call(new_obj.wasm_ptr()); + + assert!(res.is_ok(), "{:?}", res.err()); +} + +async fn bool_padding_ok(api_version: semver::Version) { + let mut module = super::test::test_module( + &rnd_sub_graph_name(12), + super::common::mock_data_source( + &super::test::wasm_file_path(WASM_FILE_NAME, api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + let parm = protobuf::UnitTestTypeBool { + str_pref: "pref".into(), + under_test: true, + str_suff: "suff".into(), + large: i64::MAX, + tail: true, + }; + + let new_obj = module.asc_new(&parm).unwrap(); + + let func = module + .get_func("test_padding_bool") + .typed() + .unwrap() + .clone(); + + let res: Result<(), _> = func.call(new_obj.wasm_ptr()); + + assert!(res.is_ok(), "{:?}", res.err()); +} + +async fn i8_padding_ok(api_version: semver::Version) { + let mut module = super::test::test_module( + &rnd_sub_graph_name(12), + super::common::mock_data_source( + &super::test::wasm_file_path(WASM_FILE_NAME, api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + let parm = protobuf::UnitTestTypeI8 { + str_pref: "pref".into(), + under_test: i8::MAX, + str_suff: "suff".into(), + large: i64::MAX, + tail: true, + }; + + let new_obj = module.asc_new(&parm).unwrap(); + + let func = module.get_func("test_padding_i8").typed().unwrap().clone(); + + let res: Result<(), _> = func.call(new_obj.wasm_ptr()); + + assert!(res.is_ok(), "{:?}", res.err()); +} + +async fn u16_padding_ok(api_version: semver::Version) { + let mut module = super::test::test_module( + &rnd_sub_graph_name(12), + super::common::mock_data_source( + &super::test::wasm_file_path(WASM_FILE_NAME, api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + let parm = protobuf::UnitTestTypeU16 { + str_pref: "pref".into(), + under_test: i16::MAX as u16, + str_suff: "suff".into(), + large: i64::MAX, + tail: true, + }; + + let new_obj = module.asc_new(&parm).unwrap(); + + let func = module.get_func("test_padding_i16").typed().unwrap().clone(); + + let res: Result<(), _> = func.call(new_obj.wasm_ptr()); + + assert!(res.is_ok(), "{:?}", res.err()); +} + +async fn u32_padding_ok(api_version: semver::Version) { + let mut module = super::test::test_module( + &rnd_sub_graph_name(12), + super::common::mock_data_source( + &super::test::wasm_file_path(WASM_FILE_NAME, api_version.clone()), + api_version.clone(), + ), + api_version, + ) + .await; + + let parm = protobuf::UnitTestTypeU32 { + str_pref: "pref".into(), + under_test: i32::MAX as u32, + str_suff: "suff".into(), + large: i64::MAX, + tail: true, + }; + + let new_obj = module.asc_new(&parm).unwrap(); + + let func = module.get_func("test_padding_i32").typed().unwrap().clone(); + + let res: Result<(), _> = func.call(new_obj.wasm_ptr()); + + assert!(res.is_ok(), "{:?}", res.err()); +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/abi_classes.ts b/runtime/test/wasm_test/api_version_0_0_4/abi_classes.ts new file mode 100644 index 0000000..bac8093 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/abi_classes.ts @@ -0,0 +1,72 @@ +import "allocator/arena"; + +export { memory }; + +// Sequence of 20 `u8`s. +type Address = Uint8Array; + +const array_buffer_header_size = 8; + +// Clone the address to a new buffer, add 1 to the first and last bytes of the +// address and return the new address. +export function test_address(address: Address): Address { + let new_address = address.subarray(); + + // Add 1 to the first and last byte. + new_address[0] += 1; + new_address[address.length - 1] += 1; + + return new_address +} + +// Sequence of 32 `u8`s. +type Uint = Uint8Array; + +// Clone the Uint to a new buffer, add 1 to the first and last `u8`s and return +// the new Uint. +export function test_uint(address: Uint): Uint { + let new_address = address.subarray(); + + // Add 1 to the first byte. + new_address[0] += 1; + + return new_address +} + +// Return the string repeated twice. +export function repeat_twice(original: string): string { + return original.repeat(2) +} + +// Sequences of `u8`s. +type FixedBytes = Uint8Array; +type Bytes = Uint8Array; + +// Concatenate two byte sequences into a new one. +export function concat(bytes1: Bytes, bytes2: FixedBytes): Bytes { + let concated = new ArrayBuffer(bytes1.byteLength + bytes2.byteLength); + let concated_offset = changetype(concated) + array_buffer_header_size; + let bytes1_start = load(changetype(bytes1)) + array_buffer_header_size; + let bytes2_start = load(changetype(bytes2)) + array_buffer_header_size; + + // Move bytes1. + memory.copy(concated_offset, bytes1_start, bytes1.byteLength); + concated_offset += bytes1.byteLength + + // Move bytes2. + memory.copy(concated_offset, bytes2_start, bytes2.byteLength); + + let new_typed_array = new Uint8Array(concated.byteLength); + store(changetype(new_typed_array), changetype(concated)); + + return new_typed_array; +} + +export function test_array(strings: Array): Array { + strings.push("5") + return strings +} + +export function byte_array_third_quarter(bytes: Uint8Array): Uint8Array { + return bytes.subarray(bytes.length * 2/4, bytes.length * 3/4) +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/abi_classes.wasm b/runtime/test/wasm_test/api_version_0_0_4/abi_classes.wasm new file mode 100644 index 0000000..274d1e3 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/abi_classes.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/abi_store_value.ts b/runtime/test/wasm_test/api_version_0_0_4/abi_store_value.ts new file mode 100644 index 0000000..69e67ea --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/abi_store_value.ts @@ -0,0 +1,104 @@ +import "allocator/arena"; + +export { memory }; + +enum ValueKind { + STRING = 0, + INT = 1, + BIG_DECIMAL = 2, + BOOL = 3, + ARRAY = 4, + NULL = 5, + BYTES = 6, + BIG_INT = 7, +} + +// Big enough to fit any pointer or native `this.data`. +type Payload = u64 + +type Bytes = Uint8Array; +type BigInt = Uint8Array; + +export class BigDecimal { + exp: BigInt + digits: BigInt +} + +export class Value { + kind: ValueKind + data: Payload +} + +export function value_from_string(str: string): Value { + let token = new Value(); + token.kind = ValueKind.STRING; + token.data = str as u64; + return token +} + +export function value_from_int(int: i32): Value { + let value = new Value(); + value.kind = ValueKind.INT; + value.data = int as u64 + return value +} + +export function value_from_big_decimal(float: BigInt): Value { + let value = new Value(); + value.kind = ValueKind.BIG_DECIMAL; + value.data = float as u64; + return value +} + +export function value_from_bool(bool: boolean): Value { + let value = new Value(); + value.kind = ValueKind.BOOL; + value.data = bool ? 1 : 0; + return value +} + +export function array_from_values(str: string, i: i32): Value { + let array = new Array(); + array.push(value_from_string(str)); + array.push(value_from_int(i)); + + let value = new Value(); + value.kind = ValueKind.ARRAY; + value.data = array as u64; + return value +} + +export function value_null(): Value { + let value = new Value(); + value.kind = ValueKind.NULL; + return value +} + +export function value_from_bytes(bytes: Bytes): Value { + let value = new Value(); + value.kind = ValueKind.BYTES; + value.data = bytes as u64; + return value +} + +export function value_from_bigint(bigint: BigInt): Value { + let value = new Value(); + value.kind = ValueKind.BIG_INT; + value.data = bigint as u64; + return value +} + +export function value_from_array(array: Array): Value { + let value = new Value() + value.kind = ValueKind.ARRAY + value.data = array as u64 + return value +} + +// Test that this does not cause undefined behaviour in Rust. +export function invalid_discriminant(): Value { + let token = new Value(); + token.kind = 70; + token.data = "blebers" as u64; + return token +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/abi_store_value.wasm b/runtime/test/wasm_test/api_version_0_0_4/abi_store_value.wasm new file mode 100644 index 0000000..635271b Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/abi_store_value.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/abi_token.ts b/runtime/test/wasm_test/api_version_0_0_4/abi_token.ts new file mode 100644 index 0000000..7c59dc7 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/abi_token.ts @@ -0,0 +1,122 @@ +import "allocator/arena"; + +export { memory }; + +// Sequence of 20 `u8`s. +type Address = Uint8Array; + +// Sequences of `u8`s. +type Bytes = Uint8Array; + +// Sequence of 4 `u64`s. +type Int = Uint64Array; +type Uint = Uint64Array; + +enum TokenKind { + ADDRESS = 0, + FIXED_BYTES = 1, + BYTES = 2, + INT = 3, + UINT = 4, + BOOL = 5, + STRING = 6, + FIXED_ARRAY = 7, + ARRAY = 8 +} + +// Big enough to fit any pointer or native this.data. +type Payload = u64 + +export class Token { + kind: TokenKind + data: Payload +} + +export function token_to_address(token: Token): Address { + assert(token.kind == TokenKind.ADDRESS, "Token is not an address."); + return changetype

(token.data as u32); +} + +export function token_to_bytes(token: Token): Bytes { + assert(token.kind == TokenKind.FIXED_BYTES + || token.kind == TokenKind.BYTES, "Token is not bytes.") + return changetype(token.data as u32) +} + +export function token_to_int(token: Token): Int { + assert(token.kind == TokenKind.INT + || token.kind == TokenKind.UINT, "Token is not an int or uint.") + return changetype(token.data as u32) +} + +export function token_to_uint(token: Token): Uint { + assert(token.kind == TokenKind.INT + || token.kind == TokenKind.UINT, "Token is not an int or uint.") + return changetype(token.data as u32) +} + +export function token_to_bool(token: Token): boolean { + assert(token.kind == TokenKind.BOOL, "Token is not a boolean.") + return token.data != 0 +} + +export function token_to_string(token: Token): string { + assert(token.kind == TokenKind.STRING, "Token is not a string.") + return changetype(token.data as u32) +} + +export function token_to_array(token: Token): Array { + assert(token.kind == TokenKind.FIXED_ARRAY || + token.kind == TokenKind.ARRAY, "Token is not an array.") + return changetype>(token.data as u32) +} + + +export function token_from_address(address: Address): Token { + let token: Token; + token.kind = TokenKind.ADDRESS; + token.data = address as u64; + return token +} + +export function token_from_bytes(bytes: Bytes): Token { + let token: Token; + token.kind = TokenKind.BYTES; + token.data = bytes as u64; + return token +} + +export function token_from_int(int: Int): Token { + let token: Token; + token.kind = TokenKind.INT; + token.data = int as u64; + return token +} + +export function token_from_uint(uint: Uint): Token { + let token: Token; + token.kind = TokenKind.UINT; + token.data = uint as u64; + return token +} + +export function token_from_bool(bool: boolean): Token { + let token: Token; + token.kind = TokenKind.BOOL; + token.data = bool as u64; + return token +} + +export function token_from_string(str: string): Token { + let token: Token; + token.kind = TokenKind.STRING; + token.data = str as u64; + return token +} + +export function token_from_array(array: Token): Token { + let token: Token; + token.kind = TokenKind.ARRAY; + token.data = array as u64; + return token +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/abi_token.wasm b/runtime/test/wasm_test/api_version_0_0_4/abi_token.wasm new file mode 100644 index 0000000..edf14dc Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/abi_token.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/abort.ts b/runtime/test/wasm_test/api_version_0_0_4/abort.ts new file mode 100644 index 0000000..5ff4b1a --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/abort.ts @@ -0,0 +1,7 @@ +import "allocator/arena"; + +export { memory }; + +export function abort(): void { + assert(false, "not true") +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/abort.wasm b/runtime/test/wasm_test/api_version_0_0_4/abort.wasm new file mode 100644 index 0000000..2fbfbab Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/abort.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/big_int_arithmetic.ts b/runtime/test/wasm_test/api_version_0_0_4/big_int_arithmetic.ts new file mode 100644 index 0000000..1fe4615 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/big_int_arithmetic.ts @@ -0,0 +1,33 @@ +import "allocator/arena"; + +export { memory }; + +type BigInt = Uint8Array; + +declare namespace bigInt { + function plus(x: BigInt, y: BigInt): BigInt + function minus(x: BigInt, y: BigInt): BigInt + function times(x: BigInt, y: BigInt): BigInt + function dividedBy(x: BigInt, y: BigInt): BigInt + function mod(x: BigInt, y: BigInt): BigInt +} + +export function plus(x: BigInt, y: BigInt): BigInt { + return bigInt.plus(x, y) +} + +export function minus(x: BigInt, y: BigInt): BigInt { + return bigInt.minus(x, y) +} + +export function times(x: BigInt, y: BigInt): BigInt { + return bigInt.times(x, y) +} + +export function dividedBy(x: BigInt, y: BigInt): BigInt { + return bigInt.dividedBy(x, y) +} + +export function mod(x: BigInt, y: BigInt): BigInt { + return bigInt.mod(x, y) +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/big_int_arithmetic.wasm b/runtime/test/wasm_test/api_version_0_0_4/big_int_arithmetic.wasm new file mode 100644 index 0000000..df2f9e3 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/big_int_arithmetic.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/big_int_to_hex.ts b/runtime/test/wasm_test/api_version_0_0_4/big_int_to_hex.ts new file mode 100644 index 0000000..8e9b77d --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/big_int_to_hex.ts @@ -0,0 +1,11 @@ +import "allocator/arena"; + +export { memory }; + +declare namespace typeConversion { + function bigIntToHex(n: Uint8Array): String +} + +export function big_int_to_hex(n: Uint8Array): String { + return typeConversion.bigIntToHex(n) +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/big_int_to_hex.wasm b/runtime/test/wasm_test/api_version_0_0_4/big_int_to_hex.wasm new file mode 100644 index 0000000..9e30ff2 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/big_int_to_hex.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/big_int_to_string.ts b/runtime/test/wasm_test/api_version_0_0_4/big_int_to_string.ts new file mode 100644 index 0000000..d2c4651 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/big_int_to_string.ts @@ -0,0 +1,11 @@ +import "allocator/arena"; + +export { memory }; + +declare namespace typeConversion { + function bigIntToString(n: Uint8Array): String +} + +export function big_int_to_string(n: Uint8Array): String { + return typeConversion.bigIntToString(n) +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/big_int_to_string.wasm b/runtime/test/wasm_test/api_version_0_0_4/big_int_to_string.wasm new file mode 100644 index 0000000..40063ca Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/big_int_to_string.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/bytes_to_base58.ts b/runtime/test/wasm_test/api_version_0_0_4/bytes_to_base58.ts new file mode 100644 index 0000000..b867cee --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/bytes_to_base58.ts @@ -0,0 +1,11 @@ +import "allocator/arena"; + +export { memory }; + +declare namespace typeConversion { + function bytesToBase58(n: Uint8Array): string +} + +export function bytes_to_base58(n: Uint8Array): string { + return typeConversion.bytesToBase58(n) +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/bytes_to_base58.wasm b/runtime/test/wasm_test/api_version_0_0_4/bytes_to_base58.wasm new file mode 100644 index 0000000..ea85915 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/bytes_to_base58.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/contract_calls.ts b/runtime/test/wasm_test/api_version_0_0_4/contract_calls.ts new file mode 100644 index 0000000..003bd38 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/contract_calls.ts @@ -0,0 +1,9 @@ +type Address = Uint8Array; + +export declare namespace ethereum { + function call(call: Address): Array
| null +} + +export function callContract(address: Address): void { + ethereum.call(address) +} \ No newline at end of file diff --git a/runtime/test/wasm_test/api_version_0_0_4/contract_calls.wasm b/runtime/test/wasm_test/api_version_0_0_4/contract_calls.wasm new file mode 100644 index 0000000..6206608 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/contract_calls.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/crypto.ts b/runtime/test/wasm_test/api_version_0_0_4/crypto.ts new file mode 100644 index 0000000..8210831 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/crypto.ts @@ -0,0 +1,11 @@ +import "allocator/arena"; + +export { memory }; + +declare namespace crypto { + function keccak256(input: Uint8Array): Uint8Array +} + +export function hash(input: Uint8Array): Uint8Array { + return crypto.keccak256(input) +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/crypto.wasm b/runtime/test/wasm_test/api_version_0_0_4/crypto.wasm new file mode 100644 index 0000000..c683c05 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/crypto.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/data_source_create.ts b/runtime/test/wasm_test/api_version_0_0_4/data_source_create.ts new file mode 100644 index 0000000..ca12b37 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/data_source_create.ts @@ -0,0 +1,11 @@ +import "allocator/arena"; + +export { memory }; + +declare namespace dataSource { + function create(name: string, params: Array): void +} + +export function dataSourceCreate(name: string, params: Array): void { + dataSource.create(name, params) +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/data_source_create.wasm b/runtime/test/wasm_test/api_version_0_0_4/data_source_create.wasm new file mode 100644 index 0000000..02b302a Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/data_source_create.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/ens_name_by_hash.ts b/runtime/test/wasm_test/api_version_0_0_4/ens_name_by_hash.ts new file mode 100644 index 0000000..0e2748f --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/ens_name_by_hash.ts @@ -0,0 +1,11 @@ +import "allocator/arena"; + +export { memory }; + +declare namespace ens { + function nameByHash(hash: string): string|null +} + +export function nameByHash(hash: string): string|null { + return ens.nameByHash(hash) +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/ens_name_by_hash.wasm b/runtime/test/wasm_test/api_version_0_0_4/ens_name_by_hash.wasm new file mode 100644 index 0000000..c7e21be Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/ens_name_by_hash.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/ipfs_cat.ts b/runtime/test/wasm_test/api_version_0_0_4/ipfs_cat.ts new file mode 100644 index 0000000..76a06d7 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/ipfs_cat.ts @@ -0,0 +1,19 @@ +import "allocator/arena"; + +export { memory }; + +declare namespace typeConversion { + function bytesToString(bytes: Uint8Array): string +} + +declare namespace ipfs { + function cat(hash: String): Uint8Array +} + +export function ipfsCatString(hash: string): string { + return typeConversion.bytesToString(ipfs.cat(hash)) +} + +export function ipfsCat(hash: string): Uint8Array { + return ipfs.cat(hash) +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/ipfs_cat.wasm b/runtime/test/wasm_test/api_version_0_0_4/ipfs_cat.wasm new file mode 100644 index 0000000..dc52142 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/ipfs_cat.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/ipfs_map.ts b/runtime/test/wasm_test/api_version_0_0_4/ipfs_map.ts new file mode 100644 index 0000000..acab2eb --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/ipfs_map.ts @@ -0,0 +1,138 @@ +import "allocator/arena"; + +export { memory }; + +/* + * Declarations copied from graph-ts/input.ts and edited for brevity + */ + +declare namespace store { + function set(entity: string, id: string, data: Entity): void +} + +class TypedMapEntry { + key: K + value: V + + constructor(key: K, value: V) { + this.key = key + this.value = value + } +} + +class TypedMap { + entries: Array> + + constructor() { + this.entries = new Array>(0) + } + + set(key: K, value: V): void { + let entry = this.getEntry(key) + if (entry !== null) { + entry.value = value + } else { + let entry = new TypedMapEntry(key, value) + this.entries.push(entry) + } + } + + getEntry(key: K): TypedMapEntry | null { + for (let i: i32 = 0; i < this.entries.length; i++) { + if (this.entries[i].key == key) { + return this.entries[i] + } + } + return null + } + + get(key: K): V | null { + for (let i: i32 = 0; i < this.entries.length; i++) { + if (this.entries[i].key == key) { + return this.entries[i].value + } + } + return null + } +} + +enum ValueKind { + STRING = 0, + INT = 1, + FLOAT = 2, + BOOL = 3, + ARRAY = 4, + NULL = 5, + BYTES = 6, + BIGINT = 7, +} + +type ValuePayload = u64 + +class Value { + kind: ValueKind + data: ValuePayload + + toString(): string { + assert(this.kind == ValueKind.STRING, 'Value is not a string.') + return changetype(this.data as u32) + } + + static fromString(s: string): Value { + let value = new Value() + value.kind = ValueKind.STRING + value.data = s as u64 + return value + } +} + +class Entity extends TypedMap { } + +enum JSONValueKind { + NULL = 0, + BOOL = 1, + NUMBER = 2, + STRING = 3, + ARRAY = 4, + OBJECT = 5, +} + +type JSONValuePayload = u64 +class JSONValue { + kind: JSONValueKind + data: JSONValuePayload + + toString(): string { + assert(this.kind == JSONValueKind.STRING, 'JSON value is not a string.') + return changetype(this.data as u32) + } + + toObject(): TypedMap { + assert(this.kind == JSONValueKind.OBJECT, 'JSON value is not an object.') + return changetype>(this.data as u32) + } +} + +/* + * Actual setup for the test + */ +declare namespace ipfs { + function map(hash: String, callback: String, userData: Value, flags: String[]): void +} + +export function echoToStore(data: JSONValue, userData: Value): void { + // expect a map of the form { "id": "anId", "value": "aValue" } + let map = data.toObject(); + let id = map.get("id").toString(); + let value = map.get("value").toString(); + + let entity = new Entity(); + entity.set("id", Value.fromString(id)); + entity.set("value", Value.fromString(value)); + entity.set("extra", userData); + store.set("Thing", id, entity); +} + +export function ipfsMap(hash: string, userData: string): void { + ipfs.map(hash, "echoToStore", Value.fromString(userData), ["json"]) +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/ipfs_map.wasm b/runtime/test/wasm_test/api_version_0_0_4/ipfs_map.wasm new file mode 100644 index 0000000..5776f0f Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/ipfs_map.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/json_parsing.ts b/runtime/test/wasm_test/api_version_0_0_4/json_parsing.ts new file mode 100644 index 0000000..0e55733 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/json_parsing.ts @@ -0,0 +1,78 @@ +import "allocator/arena"; +export { memory }; + +export class Wrapped { + inner: T; + + constructor(inner: T) { + this.inner = inner; + } +} + +export class Result { + _value: Wrapped | null; + _error: Wrapped | null; + + get isOk(): boolean { + return this._value !== null; + } + + get isError(): boolean { + return this._error !== null; + } + + get value(): V { + assert(this._value != null, "Trying to get a value from an error result"); + return (this._value as Wrapped).inner; + } + + get error(): E { + assert( + this._error != null, + "Trying to get an error from a successful result" + ); + return (this._error as Wrapped).inner; + } +} + +/** Type hint for JSON values. */ +export enum JSONValueKind { + NULL = 0, + BOOL = 1, + NUMBER = 2, + STRING = 3, + ARRAY = 4, + OBJECT = 5 +} + +/** + * Pointer type for JSONValue data. + * + * Big enough to fit any pointer or native `this.data`. + */ +export type JSONValuePayload = u64; + +export class JSONValue { + kind: JSONValueKind; + data: JSONValuePayload; + + toString(): string { + assert(this.kind == JSONValueKind.STRING, "JSON value is not a string."); + return changetype(this.data as u32); + } +} + +export class Bytes extends Uint8Array {} + +declare namespace json { + function try_fromBytes(data: Bytes): Result; +} + +export function handleJsonError(data: Bytes): string { + let result = json.try_fromBytes(data); + if (result.isOk) { + return "OK: " + result.value.toString() + ", ERROR: " + (result.isError ? "true" : "false"); + } else { + return "ERROR: " + (result.error ? "true" : "false"); + } +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/json_parsing.wasm b/runtime/test/wasm_test/api_version_0_0_4/json_parsing.wasm new file mode 100644 index 0000000..ad6a932 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/json_parsing.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/non_terminating.ts b/runtime/test/wasm_test/api_version_0_0_4/non_terminating.ts new file mode 100644 index 0000000..e289416 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/non_terminating.ts @@ -0,0 +1,11 @@ +import "allocator/arena"; +export { memory }; + +// Test that non-terminating handlers are killed by timeout. +export function loop(): void { + while (true) {} +} + +export function rabbit_hole(): void { + rabbit_hole() +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/non_terminating.wasm b/runtime/test/wasm_test/api_version_0_0_4/non_terminating.wasm new file mode 100644 index 0000000..6e3eb54 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/non_terminating.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/store.ts b/runtime/test/wasm_test/api_version_0_0_4/store.ts new file mode 100644 index 0000000..15c3b8e --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/store.ts @@ -0,0 +1,526 @@ +import "allocator/arena"; + +export { memory }; + +/** Definitions copied from graph-ts/index.ts */ +declare namespace store { + function get(entity: string, id: string): Entity | null + function set(entity: string, id: string, data: Entity): void + function remove(entity: string, id: string): void +} + +/** Host Ethereum interface */ +declare namespace ethereum { + function call(call: SmartContractCall): Array +} + +/** Host interface for BigInt arithmetic */ +declare namespace bigInt { + function plus(x: BigInt, y: BigInt): BigInt + function minus(x: BigInt, y: BigInt): BigInt + function times(x: BigInt, y: BigInt): BigInt + function dividedBy(x: BigInt, y: BigInt): BigInt + function dividedByDecimal(x: BigInt, y: BigDecimal): BigDecimal + function mod(x: BigInt, y: BigInt): BigInt +} + +/** Host interface for BigDecimal */ +declare namespace bigDecimal { + function plus(x: BigDecimal, y: BigDecimal): BigDecimal + function minus(x: BigDecimal, y: BigDecimal): BigDecimal + function times(x: BigDecimal, y: BigDecimal): BigDecimal + function dividedBy(x: BigDecimal, y: BigDecimal): BigDecimal + function equals(x: BigDecimal, y: BigDecimal): boolean + function toString(bigDecimal: BigDecimal): string + function fromString(s: string): BigDecimal +} + +/** + * TypedMap entry. + */ +class TypedMapEntry { + key: K + value: V + + constructor(key: K, value: V) { + this.key = key + this.value = value + } +} + +/** Typed map */ +class TypedMap { + entries: Array> + + constructor() { + this.entries = new Array>(0) + } + + set(key: K, value: V): void { + let entry = this.getEntry(key) + if (entry !== null) { + entry.value = value + } else { + let entry = new TypedMapEntry(key, value) + this.entries.push(entry) + } + } + + getEntry(key: K): TypedMapEntry | null { + for (let i: i32 = 0; i < this.entries.length; i++) { + if (this.entries[i].key == key) { + return this.entries[i] + } + } + return null + } + + get(key: K): V | null { + for (let i: i32 = 0; i < this.entries.length; i++) { + if (this.entries[i].key == key) { + return this.entries[i].value + } + } + return null + } + + isSet(key: K): bool { + for (let i: i32 = 0; i < this.entries.length; i++) { + if (this.entries[i].key == key) { + return true + } + } + return false + } +} + +/** + * Byte array + */ +class ByteArray extends Uint8Array { + toHex(): string { + return typeConversion.bytesToHex(this) + } + + toHexString(): string { + return typeConversion.bytesToHex(this) + } + + toString(): string { + return typeConversion.bytesToString(this) + } + + toBase58(): string { + return typeConversion.bytesToBase58(this) + } +} + +/** A dynamically-sized byte array. */ +class Bytes extends ByteArray { } + +/** An Ethereum address (20 bytes). */ +class Address extends Bytes { + static fromString(s: string): Address { + return typeConversion.stringToH160(s) as Address + } +} + +/** An arbitrary size integer represented as an array of bytes. */ +class BigInt extends Uint8Array { + toHex(): string { + return typeConversion.bigIntToHex(this) + } + + toHexString(): string { + return typeConversion.bigIntToHex(this) + } + + toString(): string { + return typeConversion.bigIntToString(this) + } + + static fromI32(x: i32): BigInt { + return typeConversion.i32ToBigInt(x) as BigInt + } + + toI32(): i32 { + return typeConversion.bigIntToI32(this) + } + + @operator('+') + plus(other: BigInt): BigInt { + return bigInt.plus(this, other) + } + + @operator('-') + minus(other: BigInt): BigInt { + return bigInt.minus(this, other) + } + + @operator('*') + times(other: BigInt): BigInt { + return bigInt.times(this, other) + } + + @operator('/') + div(other: BigInt): BigInt { + return bigInt.dividedBy(this, other) + } + + divDecimal(other: BigDecimal): BigDecimal { + return bigInt.dividedByDecimal(this, other) + } + + @operator('%') + mod(other: BigInt): BigInt { + return bigInt.mod(this, other) + } + + @operator('==') + equals(other: BigInt): boolean { + if (this.length !== other.length) { + return false; + } + for (let i = 0; i < this.length; i++) { + if (this[i] !== other[i]) { + return false; + } + } + return true; + } + + toBigDecimal(): BigDecimal { + return new BigDecimal(this) + } +} + +class BigDecimal { + digits: BigInt + exp: BigInt + + constructor(bigInt: BigInt) { + this.digits = bigInt + this.exp = BigInt.fromI32(0) + } + + static fromString(s: string): BigDecimal { + return bigDecimal.fromString(s) + } + + toString(): string { + return bigDecimal.toString(this) + } + + truncate(decimals: i32): BigDecimal { + let digitsRightOfZero = this.digits.toString().length + this.exp.toI32() + let newDigitLength = decimals + digitsRightOfZero + let truncateLength = this.digits.toString().length - newDigitLength + if (truncateLength < 0) { + return this + } else { + for (let i = 0; i < truncateLength; i++) { + this.digits = this.digits.div(BigInt.fromI32(10)) + } + this.exp = BigInt.fromI32(decimals* -1) + return this + } + } + + @operator('+') + plus(other: BigDecimal): BigDecimal { + return bigDecimal.plus(this, other) + } + + @operator('-') + minus(other: BigDecimal): BigDecimal { + return bigDecimal.minus(this, other) + } + + @operator('*') + times(other: BigDecimal): BigDecimal { + return bigDecimal.times(this, other) + } + + @operator('/') + div(other: BigDecimal): BigDecimal { + return bigDecimal.dividedBy(this, other) + } + + @operator('==') + equals(other: BigDecimal): boolean { + return bigDecimal.equals(this, other) + } +} + +/** + * Enum for supported value types. + */ +enum ValueKind { + STRING = 0, + INT = 1, + BIGDECIMAL = 2, + BOOL = 3, + ARRAY = 4, + NULL = 5, + BYTES = 6, + BIGINT = 7, +} + +/** + * Pointer type for Value data. + * + * Big enough to fit any pointer or native `this.data`. + */ +type ValuePayload = u64 + +/** + * A dynamically typed value. + */ +class Value { + kind: ValueKind + data: ValuePayload + + toAddress(): Address { + assert(this.kind == ValueKind.BYTES, 'Value is not an address.') + return changetype
(this.data as u32) + } + + toBoolean(): boolean { + if (this.kind == ValueKind.NULL) { + return false; + } + assert(this.kind == ValueKind.BOOL, 'Value is not a boolean.') + return this.data != 0 + } + + toBytes(): Bytes { + assert(this.kind == ValueKind.BYTES, 'Value is not a byte array.') + return changetype(this.data as u32) + } + + toI32(): i32 { + if (this.kind == ValueKind.NULL) { + return 0; + } + assert(this.kind == ValueKind.INT, 'Value is not an i32.') + return this.data as i32 + } + + toString(): string { + assert(this.kind == ValueKind.STRING, 'Value is not a string.') + return changetype(this.data as u32) + } + + toBigInt(): BigInt { + assert(this.kind == ValueKind.BIGINT, 'Value is not a BigInt.') + return changetype(this.data as u32) + } + + toBigDecimal(): BigDecimal { + assert(this.kind == ValueKind.BIGDECIMAL, 'Value is not a BigDecimal.') + return changetype(this.data as u32) + } + + toArray(): Array { + assert(this.kind == ValueKind.ARRAY, 'Value is not an array.') + return changetype>(this.data as u32) + } + + toBooleanArray(): Array { + let values = this.toArray() + let output = new Array(values.length) + for (let i: i32; i < values.length; i++) { + output[i] = values[i].toBoolean() + } + return output + } + + toBytesArray(): Array { + let values = this.toArray() + let output = new Array(values.length) + for (let i: i32 = 0; i < values.length; i++) { + output[i] = values[i].toBytes() + } + return output + } + + toStringArray(): Array { + let values = this.toArray() + let output = new Array(values.length) + for (let i: i32 = 0; i < values.length; i++) { + output[i] = values[i].toString() + } + return output + } + + toI32Array(): Array { + let values = this.toArray() + let output = new Array(values.length) + for (let i: i32 = 0; i < values.length; i++) { + output[i] = values[i].toI32() + } + return output + } + + toBigIntArray(): Array { + let values = this.toArray() + let output = new Array(values.length) + for (let i: i32 = 0; i < values.length; i++) { + output[i] = values[i].toBigInt() + } + return output + } + + toBigDecimalArray(): Array { + let values = this.toArray() + let output = new Array(values.length) + for (let i: i32 = 0; i < values.length; i++) { + output[i] = values[i].toBigDecimal() + } + return output + } + + static fromBooleanArray(input: Array): Value { + let output = new Array(input.length) + for (let i: i32 = 0; i < input.length; i++) { + output[i] = Value.fromBoolean(input[i]) + } + return Value.fromArray(output) + } + + static fromBytesArray(input: Array): Value { + let output = new Array(input.length) + for (let i: i32 = 0; i < input.length; i++) { + output[i] = Value.fromBytes(input[i]) + } + return Value.fromArray(output) + } + + static fromI32Array(input: Array): Value { + let output = new Array(input.length) + for (let i: i32 = 0; i < input.length; i++) { + output[i] = Value.fromI32(input[i]) + } + return Value.fromArray(output) + } + + static fromBigIntArray(input: Array): Value { + let output = new Array(input.length) + for (let i: i32 = 0; i < input.length; i++) { + output[i] = Value.fromBigInt(input[i]) + } + return Value.fromArray(output) + } + + static fromBigDecimalArray(input: Array): Value { + let output = new Array(input.length) + for (let i: i32 = 0; i < input.length; i++) { + output[i] = Value.fromBigDecimal(input[i]) + } + return Value.fromArray(output) + } + + static fromStringArray(input: Array): Value { + let output = new Array(input.length) + for (let i: i32 = 0; i < input.length; i++) { + output[i] = Value.fromString(input[i]) + } + return Value.fromArray(output) + } + + static fromArray(input: Array): Value { + let value = new Value() + value.kind = ValueKind.ARRAY + value.data = input as u64 + return value + } + + static fromBigInt(n: BigInt): Value { + let value = new Value() + value.kind = ValueKind.BIGINT + value.data = n as u64 + return value + } + + static fromBoolean(b: boolean): Value { + let value = new Value() + value.kind = ValueKind.BOOL + value.data = b ? 1 : 0 + return value + } + + static fromBytes(bytes: Bytes): Value { + let value = new Value() + value.kind = ValueKind.BYTES + value.data = bytes as u64 + return value + } + + static fromNull(): Value { + let value = new Value() + value.kind = ValueKind.NULL + return value + } + + static fromI32(n: i32): Value { + let value = new Value() + value.kind = ValueKind.INT + value.data = n as u64 + return value + } + + static fromString(s: string): Value { + let value = new Value() + value.kind = ValueKind.STRING + value.data = s as u64 + return value + } + + static fromBigDecimal(n: BigDecimal): Value { + let value = new Value() + value.kind = ValueKind.BIGDECIMAL + value.data = n as u64 + return value + } +} + +/** + * Common representation for entity data, storing entity attributes + * as `string` keys and the attribute values as dynamically-typed + * `Value` objects. + */ +export class Entity extends TypedMap { + unset(key: string): void { + this.set(key, Value.fromNull()) + } + + /** Assigns properties from sources to this Entity in right-to-left order */ + merge(sources: Array): Entity { + var target = this + for (let i = 0; i < sources.length; i++) { + let entries = sources[i].entries + for (let j = 0; j < entries.length; j++) { + target.set(entries[j].key, entries[j].value) + } + } + return target + } +} + +/** + * Test functions + */ +export function getUser(id: string): Entity | null { + return store.get("User", id); +} + +export function loadAndSetUserName(id: string, name: string) : void { + let user = store.get("User", id); + if (user == null) { + user = new Entity(); + user.set("id", Value.fromString(id)); + } + user.set("name", Value.fromString(name)); + // save it + store.set("User", id, (user as Entity)); +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/store.wasm b/runtime/test/wasm_test/api_version_0_0_4/store.wasm new file mode 100644 index 0000000..ec7027c Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/store.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/string_to_number.ts b/runtime/test/wasm_test/api_version_0_0_4/string_to_number.ts new file mode 100644 index 0000000..1622c27 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/string_to_number.ts @@ -0,0 +1,29 @@ +import "allocator/arena"; + +export { memory }; + +type BigInt = Uint8Array; + +/** Host JSON interface */ +declare namespace json { + function toI64(decimal: string): i64 + function toU64(decimal: string): u64 + function toF64(decimal: string): f64 + function toBigInt(decimal: string): BigInt +} + +export function testToI64(decimal: string): i64 { + return json.toI64(decimal); +} + +export function testToU64(decimal: string): u64 { + return json.toU64(decimal); +} + +export function testToF64(decimal: string): f64 { + return json.toF64(decimal) +} + +export function testToBigInt(decimal: string): BigInt { + return json.toBigInt(decimal) +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/string_to_number.wasm b/runtime/test/wasm_test/api_version_0_0_4/string_to_number.wasm new file mode 100644 index 0000000..981539f Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/string_to_number.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_4/test_padding.ts b/runtime/test/wasm_test/api_version_0_0_4/test_padding.ts new file mode 100644 index 0000000..57af24d --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_4/test_padding.ts @@ -0,0 +1,129 @@ +import "allocator/arena"; + +export { memory }; + + + +export class UnitTestTypeBool{ + str_pref: string; + under_test: boolean; + str_suff: string; + large: i64 ; + tail: boolean ; + + + + constructor(str_pref: string, under_test: boolean, str_suff:string, large: i64, tail:boolean) { + this.str_pref = str_pref; + this.under_test = under_test; + this.str_suff = str_suff; + this.large = large; + this.tail = tail; + } +} + +export function test_padding_bool(p: UnitTestTypeBool): void { + assert(p.str_pref == "pref", "parm.str_pref: Assertion failed!"); + assert(p.under_test == true, "parm.under_test: Assertion failed!"); + assert(p.str_suff == "suff", "parm.str_suff: Assertion failed!"); + assert(p.large == 9223372036854775807, "parm.large: Assertion failed!"); + assert(p.tail == true, "parm.tail: Assertion failed!"); +} + +export class UnitTestTypeI8{ + str_pref: string; + under_test: i8; + str_suff: string; + large: i64 ; + tail: boolean ; + + + + constructor(str_pref: string, under_test: i8, str_suff:string, large: i64, tail:boolean) { + this.str_pref = str_pref; + this.under_test = under_test; + this.str_suff = str_suff; + this.large = large; + this.tail = tail; + } +} + +export function test_padding_i8(p: UnitTestTypeI8): void { + assert(p.str_pref == "pref", "parm.str_pref: Assertion failed!"); + assert(p.under_test == 127, "parm.under_test: Assertion failed!"); + assert(p.str_suff == "suff", "parm.str_suff: Assertion failed!"); + assert(p.large == 9223372036854775807, "parm.large: Assertion failed!"); + assert(p.tail == true, "parm.tail: Assertion failed!"); +} + + +export class UnitTestTypeU16{ + str_pref: string; + under_test: i16; + str_suff: string; + large: i64 ; + tail: boolean ; + + + + constructor(str_pref: string, under_test: i16, str_suff:string, large: i64, tail:boolean) { + this.str_pref = str_pref; + this.under_test = under_test; + this.str_suff = str_suff; + this.large = large; + this.tail = tail; + } +} + +export function test_padding_i16(p: UnitTestTypeU16): void { + assert(p.str_pref == "pref", "parm.str_pref: Assertion failed!"); + assert(p.under_test == 32767, "parm.under_test: Assertion failed!"); + assert(p.str_suff == "suff", "parm.str_suff: Assertion failed!"); + assert(p.large == 9223372036854775807, "parm.large: Assertion failed!"); + assert(p.tail == true, "parm.tail: Assertion failed!"); +} + +export class UnitTestTypeU32{ + str_pref: string; + under_test: i32; + str_suff: string; + large: i64 ; + tail: boolean ; + + + + constructor(str_pref: string, under_test: i32, str_suff:string, large: i64, tail:boolean) { + this.str_pref = str_pref; + this.under_test = under_test; + this.str_suff = str_suff; + this.large = large; + this.tail = tail; + } +} + +export function test_padding_i32(p: UnitTestTypeU32): void { + assert(p.str_pref == "pref", "parm.str_pref: Assertion failed!"); + assert(p.under_test == 2147483647, "parm.under_test: Assertion failed!"); + assert(p.str_suff == "suff", "parm.str_suff: Assertion failed!"); + assert(p.large == 9223372036854775807, "parm.large: Assertion failed!"); + assert(p.tail == true, "parm.tail: Assertion failed!"); +} + + +export class ManualPadding{ + nonce: i64 ; + str_suff: string; + tail: i64 ; + + constructor(nonce: i64, str_suff:string, tail:i64) { + this.nonce = nonce; + this.str_suff = str_suff; + this.tail = tail + } +} + +export function test_padding_manual(p: ManualPadding): void { + assert(p.nonce == 9223372036854775807, "parm.nonce: Assertion failed!"); + assert(p.str_suff == "suff", "parm.str_suff: Assertion failed!"); + assert(p.tail == 9223372036854775807, "parm.tail: Assertion failed!"); +} diff --git a/runtime/test/wasm_test/api_version_0_0_4/test_padding.wasm b/runtime/test/wasm_test/api_version_0_0_4/test_padding.wasm new file mode 100644 index 0000000..af72b0f Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_4/test_padding.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/abi_classes.ts b/runtime/test/wasm_test/api_version_0_0_5/abi_classes.ts new file mode 100644 index 0000000..037e0ef --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/abi_classes.ts @@ -0,0 +1,62 @@ +export * from './common/global' +import { Address, Uint8, FixedBytes, Bytes, Payload, Value } from './common/types' + +// Clone the address to a new buffer, add 1 to the first and last bytes of the +// address and return the new address. +export function test_address(address: Address): Address { + let new_address = address.subarray(); + + // Add 1 to the first and last byte. + new_address[0] += 1; + new_address[address.length - 1] += 1; + + return changetype
(new_address) +} + +// Clone the Uint8 to a new buffer, add 1 to the first and last `u8`s and return +// the new Uint8 +export function test_uint(address: Uint8): Uint8 { + let new_address = address.subarray(); + + // Add 1 to the first byte. + new_address[0] += 1; + + return new_address +} + +// Return the string repeated twice. +export function repeat_twice(original: string): string { + return original.repeat(2) +} + +// Concatenate two byte sequences into a new one. +export function concat(bytes1: Bytes, bytes2: FixedBytes): Bytes { + let concated_buff = new ArrayBuffer(bytes1.byteLength + bytes2.byteLength); + let concated_buff_ptr = changetype(concated_buff); + + let bytes1_ptr = changetype(bytes1); + let bytes1_buff_ptr = load(bytes1_ptr); + + let bytes2_ptr = changetype(bytes2); + let bytes2_buff_ptr = load(bytes2_ptr); + + // Move bytes1. + memory.copy(concated_buff_ptr, bytes1_buff_ptr, bytes1.byteLength); + concated_buff_ptr += bytes1.byteLength + + // Move bytes2. + memory.copy(concated_buff_ptr, bytes2_buff_ptr, bytes2.byteLength); + + let new_typed_array = Uint8Array.wrap(concated_buff); + + return changetype(new_typed_array); +} + +export function test_array(strings: Array): Array { + strings.push("5") + return strings +} + +export function byte_array_third_quarter(bytes: Uint8Array): Uint8Array { + return bytes.subarray(bytes.length * 2/4, bytes.length * 3/4) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/abi_classes.wasm b/runtime/test/wasm_test/api_version_0_0_5/abi_classes.wasm new file mode 100644 index 0000000..8a29ed1 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/abi_classes.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/abi_store_value.ts b/runtime/test/wasm_test/api_version_0_0_5/abi_store_value.ts new file mode 100644 index 0000000..4a2a58b --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/abi_store_value.ts @@ -0,0 +1,76 @@ +export * from './common/global' +import { BigInt, BigDecimal, Bytes, Value, ValueKind } from './common/types' + +export function value_from_string(str: string): Value { + let token = new Value(); + token.kind = ValueKind.STRING; + token.data = changetype(str); + return token +} + +export function value_from_int(int: i32): Value { + let value = new Value(); + value.kind = ValueKind.INT; + value.data = int as u64 + return value +} + +export function value_from_big_decimal(float: BigInt): Value { + let value = new Value(); + value.kind = ValueKind.BIG_DECIMAL; + value.data = changetype(float); + return value +} + +export function value_from_bool(bool: boolean): Value { + let value = new Value(); + value.kind = ValueKind.BOOL; + value.data = bool ? 1 : 0; + return value +} + +export function array_from_values(str: string, i: i32): Value { + let array = new Array(); + array.push(value_from_string(str)); + array.push(value_from_int(i)); + + let value = new Value(); + value.kind = ValueKind.ARRAY; + value.data = changetype(array); + return value +} + +export function value_null(): Value { + let value = new Value(); + value.kind = ValueKind.NULL; + return value +} + +export function value_from_bytes(bytes: Bytes): Value { + let value = new Value(); + value.kind = ValueKind.BYTES; + value.data = changetype(bytes); + return value +} + +export function value_from_bigint(bigint: BigInt): Value { + let value = new Value(); + value.kind = ValueKind.BIG_INT; + value.data = changetype(bigint); + return value +} + +export function value_from_array(array: Array): Value { + let value = new Value() + value.kind = ValueKind.ARRAY + value.data = changetype(array) + return value +} + +// Test that this does not cause undefined behaviour in Rust. +export function invalid_discriminant(): Value { + let token = new Value(); + token.kind = 70; + token.data = changetype("blebers"); + return token +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/abi_store_value.wasm b/runtime/test/wasm_test/api_version_0_0_5/abi_store_value.wasm new file mode 100644 index 0000000..5ac9f91 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/abi_store_value.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/abi_token.ts b/runtime/test/wasm_test/api_version_0_0_5/abi_token.ts new file mode 100644 index 0000000..5a170b5 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/abi_token.ts @@ -0,0 +1,91 @@ +export * from './common/global' +import { Address, Bytes, Token, TokenKind, Int64, Uint64 } from './common/types' + +export function token_to_address(token: Token): Address { + assert(token.kind == TokenKind.ADDRESS, "Token is not an address."); + return changetype
(token.data as u32); +} + +export function token_to_bytes(token: Token): Bytes { + assert(token.kind == TokenKind.FIXED_BYTES + || token.kind == TokenKind.BYTES, "Token is not bytes.") + return changetype(token.data as u32) +} + +export function token_to_int(token: Token): Int64 { + assert(token.kind == TokenKind.INT + || token.kind == TokenKind.UINT, "Token is not an int or uint.") + return changetype(token.data as u32) +} + +export function token_to_uint(token: Token): Uint64 { + assert(token.kind == TokenKind.INT + || token.kind == TokenKind.UINT, "Token is not an int or uint.") + return changetype(token.data as u32) +} + +export function token_to_bool(token: Token): boolean { + assert(token.kind == TokenKind.BOOL, "Token is not a boolean.") + return token.data != 0 +} + +export function token_to_string(token: Token): string { + assert(token.kind == TokenKind.STRING, "Token is not a string.") + return changetype(token.data as u32) +} + +export function token_to_array(token: Token): Array { + assert(token.kind == TokenKind.FIXED_ARRAY || + token.kind == TokenKind.ARRAY, "Token is not an array.") + return changetype>(token.data as u32) +} + + +export function token_from_address(address: Address): Token { + let token = new Token(); + token.kind = TokenKind.ADDRESS; + token.data = changetype(address); + return token +} + +export function token_from_bytes(bytes: Bytes): Token { + let token = new Token(); + token.kind = TokenKind.BYTES; + token.data = changetype(bytes); + return token +} + +export function token_from_int(int: Int64): Token { + let token = new Token(); + token.kind = TokenKind.INT; + token.data = changetype(int); + return token +} + +export function token_from_uint(uint: Uint64): Token { + let token = new Token(); + token.kind = TokenKind.UINT; + token.data = changetype(uint); + return token +} + +export function token_from_bool(bool: boolean): Token { + let token = new Token(); + token.kind = TokenKind.BOOL; + token.data = bool as u32; + return token +} + +export function token_from_string(str: string): Token { + let token = new Token(); + token.kind = TokenKind.STRING; + token.data = changetype(str); + return token +} + +export function token_from_array(array: Token): Token { + let token = new Token(); + token.kind = TokenKind.ARRAY; + token.data = changetype(array); + return token +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/abi_token.wasm b/runtime/test/wasm_test/api_version_0_0_5/abi_token.wasm new file mode 100644 index 0000000..0cd6cd2 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/abi_token.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/abort.ts b/runtime/test/wasm_test/api_version_0_0_5/abort.ts new file mode 100644 index 0000000..88f7dee --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/abort.ts @@ -0,0 +1,5 @@ +export * from './common/global' + +export function abort(): void { + assert(false, "not true") +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/abort.wasm b/runtime/test/wasm_test/api_version_0_0_5/abort.wasm new file mode 100644 index 0000000..b8e08d7 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/abort.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/allocate_global.ts b/runtime/test/wasm_test/api_version_0_0_5/allocate_global.ts new file mode 100644 index 0000000..c1f0a90 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/allocate_global.ts @@ -0,0 +1,13 @@ +export * from './common/global' +import { BigInt } from './common/types' + +let globalOne = bigInt.fromString("1") + +declare namespace bigInt { + function fromString(s: string): BigInt +} + +export function assert_global_works(): void { + let localOne = bigInt.fromString("1") + assert(globalOne != localOne) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/allocate_global.wasm b/runtime/test/wasm_test/api_version_0_0_5/allocate_global.wasm new file mode 100644 index 0000000..779d930 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/allocate_global.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/array_blowup.ts b/runtime/test/wasm_test/api_version_0_0_5/array_blowup.ts new file mode 100644 index 0000000..a7891af --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/array_blowup.ts @@ -0,0 +1,20 @@ +export * from './common/global' +import { Bytes, Entity, Value } from './common/types' + +/** Definitions copied from graph-ts/index.ts */ +declare namespace store { + function set(entity: string, id: string, data: Entity): void +} + +export function arrayBlowup(): void { + // 1 GB array. + let s = changetype(new Bytes(1_000_000_000).fill(1)); + + // Repeated 100 times. + let a = new Array(100).fill(s); + + let entity = new Entity(); + entity.set("field", Value.fromBytesArray(a)); + store.set("NonExisting", "foo", entity) +} + diff --git a/runtime/test/wasm_test/api_version_0_0_5/array_blowup.wasm b/runtime/test/wasm_test/api_version_0_0_5/array_blowup.wasm new file mode 100644 index 0000000..80c5aba Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/array_blowup.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/big_int_arithmetic.ts b/runtime/test/wasm_test/api_version_0_0_5/big_int_arithmetic.ts new file mode 100644 index 0000000..6cea38e --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/big_int_arithmetic.ts @@ -0,0 +1,30 @@ +export * from './common/global' +import { BigInt } from './common/types' + +declare namespace bigInt { + function plus(x: BigInt, y: BigInt): BigInt + function minus(x: BigInt, y: BigInt): BigInt + function times(x: BigInt, y: BigInt): BigInt + function dividedBy(x: BigInt, y: BigInt): BigInt + function mod(x: BigInt, y: BigInt): BigInt +} + +export function plus(x: BigInt, y: BigInt): BigInt { + return bigInt.plus(x, y) +} + +export function minus(x: BigInt, y: BigInt): BigInt { + return bigInt.minus(x, y) +} + +export function times(x: BigInt, y: BigInt): BigInt { + return bigInt.times(x, y) +} + +export function dividedBy(x: BigInt, y: BigInt): BigInt { + return bigInt.dividedBy(x, y) +} + +export function mod(x: BigInt, y: BigInt): BigInt { + return bigInt.mod(x, y) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/big_int_arithmetic.wasm b/runtime/test/wasm_test/api_version_0_0_5/big_int_arithmetic.wasm new file mode 100644 index 0000000..a69c405 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/big_int_arithmetic.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/big_int_to_hex.ts b/runtime/test/wasm_test/api_version_0_0_5/big_int_to_hex.ts new file mode 100644 index 0000000..4da1265 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/big_int_to_hex.ts @@ -0,0 +1,9 @@ +export * from './common/global' + +declare namespace typeConversion { + function bigIntToHex(n: Uint8Array): string +} + +export function big_int_to_hex(n: Uint8Array): string { + return typeConversion.bigIntToHex(n) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/big_int_to_hex.wasm b/runtime/test/wasm_test/api_version_0_0_5/big_int_to_hex.wasm new file mode 100644 index 0000000..fae821a Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/big_int_to_hex.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/big_int_to_string.ts b/runtime/test/wasm_test/api_version_0_0_5/big_int_to_string.ts new file mode 100644 index 0000000..8c9c5eb --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/big_int_to_string.ts @@ -0,0 +1,9 @@ +export * from './common/global' + +declare namespace typeConversion { + function bigIntToString(n: Uint8Array): string +} + +export function big_int_to_string(n: Uint8Array): string { + return typeConversion.bigIntToString(n) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/big_int_to_string.wasm b/runtime/test/wasm_test/api_version_0_0_5/big_int_to_string.wasm new file mode 100644 index 0000000..137414e Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/big_int_to_string.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/boolean.ts b/runtime/test/wasm_test/api_version_0_0_5/boolean.ts new file mode 100644 index 0000000..7bf85ca --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/boolean.ts @@ -0,0 +1,17 @@ +export * from "./common/global" + +export function testReceiveTrue(a: bool): void { + assert(a) +} + +export function testReceiveFalse(a: bool): void { + assert(!a) +} + +export function testReturnTrue(): bool { + return true +} + +export function testReturnFalse(): bool { + return false +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/boolean.wasm b/runtime/test/wasm_test/api_version_0_0_5/boolean.wasm new file mode 100644 index 0000000..ba80672 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/boolean.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/bytes_to_base58.ts b/runtime/test/wasm_test/api_version_0_0_5/bytes_to_base58.ts new file mode 100644 index 0000000..38c956d --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/bytes_to_base58.ts @@ -0,0 +1,9 @@ +export * from './common/global' + +declare namespace typeConversion { + function bytesToBase58(n: Uint8Array): string +} + +export function bytes_to_base58(n: Uint8Array): string { + return typeConversion.bytesToBase58(n) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/bytes_to_base58.wasm b/runtime/test/wasm_test/api_version_0_0_5/bytes_to_base58.wasm new file mode 100644 index 0000000..851bb60 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/bytes_to_base58.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/common/global.ts b/runtime/test/wasm_test/api_version_0_0_5/common/global.ts new file mode 100644 index 0000000..7979b14 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/common/global.ts @@ -0,0 +1,173 @@ +__alloc(0); + +import { BigDecimal, TypedMapEntry, Entity, TypedMap, Result, Wrapped, JSONValue, Value, Token } from './types' + +export enum TypeId { + String = 0, + ArrayBuffer = 1, + Int8Array = 2, + Int16Array = 3, + Int32Array = 4, + Int64Array = 5, + Uint8Array = 6, + Uint16Array = 7, + Uint32Array = 8, + Uint64Array = 9, + Float32Array = 10, + Float64Array = 11, + BigDecimal = 12, + ArrayBool = 13, + ArrayUint8Array = 14, + ArrayEthereumValue = 15, + ArrayStoreValue = 16, + ArrayJsonValue = 17, + ArrayString = 18, + ArrayEventParam = 19, + ArrayTypedMapEntryStringJsonValue = 20, + ArrayTypedMapEntryStringStoreValue = 21, + SmartContractCall = 22, + EventParam = 23, + // EthereumTransaction = 24, + // EthereumBlock = 25, + // EthereumCall = 26, + WrappedTypedMapStringJsonValue = 27, + WrappedBool = 28, + WrappedJsonValue = 29, + EthereumValue = 30, + StoreValue = 31, + JsonValue = 32, + // EthereumEvent = 33, + TypedMapEntryStringStoreValue = 34, + TypedMapEntryStringJsonValue = 35, + TypedMapStringStoreValue = 36, + TypedMapStringJsonValue = 37, + TypedMapStringTypedMapStringJsonValue = 38, + ResultTypedMapStringJsonValueBool = 39, + ResultJsonValueBool = 40, + ArrayU8 = 41, + ArrayU16 = 42, + ArrayU32 = 43, + ArrayU64 = 44, + ArrayI8 = 45, + ArrayI16 = 46, + ArrayI32 = 47, + ArrayI64 = 48, + ArrayF32 = 49, + ArrayF64 = 50, + ArrayBigDecimal = 51, +} + +export function id_of_type(typeId: TypeId): usize { + switch (typeId) { + case TypeId.String: + return idof() + case TypeId.ArrayBuffer: + return idof() + case TypeId.Int8Array: + return idof() + case TypeId.Int16Array: + return idof() + case TypeId.Int32Array: + return idof() + case TypeId.Int64Array: + return idof() + case TypeId.Uint8Array: + return idof() + case TypeId.Uint16Array: + return idof() + case TypeId.Uint32Array: + return idof() + case TypeId.Uint64Array: + return idof() + case TypeId.Float32Array: + return idof() + case TypeId.Float64Array: + return idof() + case TypeId.BigDecimal: + return idof() + case TypeId.ArrayBool: + return idof>() + case TypeId.ArrayUint8Array: + return idof>() + case TypeId.ArrayEthereumValue: + return idof>() + case TypeId.ArrayStoreValue: + return idof>() + case TypeId.ArrayJsonValue: + return idof>() + case TypeId.ArrayString: + return idof>() + // case TypeId.ArrayEventParam: + // return idof>() + case TypeId.ArrayTypedMapEntryStringJsonValue: + return idof>>() + case TypeId.ArrayTypedMapEntryStringStoreValue: + return idof>() + case TypeId.WrappedTypedMapStringJsonValue: + return idof>>() + case TypeId.WrappedBool: + return idof>() + case TypeId.WrappedJsonValue: + return idof>() + // case TypeId.SmartContractCall: + // return idof() + // case TypeId.EventParam: + // return idof() + // case TypeId.EthereumTransaction: + // return idof() + // case TypeId.EthereumBlock: + // return idof() + // case TypeId.EthereumCall: + // return idof() + case TypeId.EthereumValue: + return idof() + case TypeId.StoreValue: + return idof() + case TypeId.JsonValue: + return idof() + // case TypeId.EthereumEvent: + // return idof() + case TypeId.TypedMapEntryStringStoreValue: + return idof() + case TypeId.TypedMapEntryStringJsonValue: + return idof>() + case TypeId.TypedMapStringStoreValue: + return idof>() + case TypeId.TypedMapStringJsonValue: + return idof>() + case TypeId.TypedMapStringTypedMapStringJsonValue: + return idof>>() + case TypeId.ResultTypedMapStringJsonValueBool: + return idof, boolean>>() + case TypeId.ResultJsonValueBool: + return idof>() + case TypeId.ArrayU8: + return idof>() + case TypeId.ArrayU16: + return idof>() + case TypeId.ArrayU32: + return idof>() + case TypeId.ArrayU64: + return idof>() + case TypeId.ArrayI8: + return idof>() + case TypeId.ArrayI16: + return idof>() + case TypeId.ArrayI32: + return idof>() + case TypeId.ArrayI64: + return idof>() + case TypeId.ArrayF32: + return idof>() + case TypeId.ArrayF64: + return idof>() + case TypeId.ArrayBigDecimal: + return idof>() + default: + return 0 + } +} + +export function allocate(size: usize): usize { + return __alloc(size) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/common/types.ts b/runtime/test/wasm_test/api_version_0_0_5/common/types.ts new file mode 100644 index 0000000..73a6189 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/common/types.ts @@ -0,0 +1,560 @@ +/** A dynamically-sized byte array. */ +export class Bytes extends ByteArray { } + +/** An Ethereum address (20 bytes). */ +export class Address extends Bytes { + static fromString(s: string): Address { + return typeConversion.stringToH160(s) as Address + } +} + +// Sequence of 20 `u8`s. +// export type Address = Uint8Array; + +// Sequence of 32 `u8`s. +export type Uint8 = Uint8Array; + +// Sequences of `u8`s. +export type FixedBytes = Uint8Array; +// export type Bytes = Uint8Array; + +/** + * Enum for supported value types. + */ +export enum ValueKind { + STRING = 0, + INT = 1, + BIG_DECIMAL = 2, + BOOL = 3, + ARRAY = 4, + NULL = 5, + BYTES = 6, + BIG_INT = 7, +} +// Big enough to fit any pointer or native `this.data`. +export type Payload = u64 +/** + * A dynamically typed value. + */ +export class Value { + kind: ValueKind + data: Payload + + toAddress(): Address { + assert(this.kind == ValueKind.BYTES, 'Value is not an address.') + return changetype
(this.data as u32) + } + + toBoolean(): boolean { + if (this.kind == ValueKind.NULL) { + return false; + } + assert(this.kind == ValueKind.BOOL, 'Value is not a boolean.') + return this.data != 0 + } + + toBytes(): Bytes { + assert(this.kind == ValueKind.BYTES, 'Value is not a byte array.') + return changetype(this.data as u32) + } + + toI32(): i32 { + if (this.kind == ValueKind.NULL) { + return 0; + } + assert(this.kind == ValueKind.INT, 'Value is not an i32.') + return this.data as i32 + } + + toString(): string { + assert(this.kind == ValueKind.STRING, 'Value is not a string.') + return changetype(this.data as u32) + } + + toBigInt(): BigInt { + assert(this.kind == ValueKind.BIGINT, 'Value is not a BigInt.') + return changetype(this.data as u32) + } + + toBigDecimal(): BigDecimal { + assert(this.kind == ValueKind.BIGDECIMAL, 'Value is not a BigDecimal.') + return changetype(this.data as u32) + } + + toArray(): Array { + assert(this.kind == ValueKind.ARRAY, 'Value is not an array.') + return changetype>(this.data as u32) + } + + toBooleanArray(): Array { + let values = this.toArray() + let output = new Array(values.length) + for (let i: i32; i < values.length; i++) { + output[i] = values[i].toBoolean() + } + return output + } + + toBytesArray(): Array { + let values = this.toArray() + let output = new Array(values.length) + for (let i: i32 = 0; i < values.length; i++) { + output[i] = values[i].toBytes() + } + return output + } + + toStringArray(): Array { + let values = this.toArray() + let output = new Array(values.length) + for (let i: i32 = 0; i < values.length; i++) { + output[i] = values[i].toString() + } + return output + } + + toI32Array(): Array { + let values = this.toArray() + let output = new Array(values.length) + for (let i: i32 = 0; i < values.length; i++) { + output[i] = values[i].toI32() + } + return output + } + + toBigIntArray(): Array { + let values = this.toArray() + let output = new Array(values.length) + for (let i: i32 = 0; i < values.length; i++) { + output[i] = values[i].toBigInt() + } + return output + } + + toBigDecimalArray(): Array { + let values = this.toArray() + let output = new Array(values.length) + for (let i: i32 = 0; i < values.length; i++) { + output[i] = values[i].toBigDecimal() + } + return output + } + + static fromBooleanArray(input: Array): Value { + let output = new Array(input.length) + for (let i: i32 = 0; i < input.length; i++) { + output[i] = Value.fromBoolean(input[i]) + } + return Value.fromArray(output) + } + + static fromBytesArray(input: Array): Value { + let output = new Array(input.length) + for (let i: i32 = 0; i < input.length; i++) { + output[i] = Value.fromBytes(input[i]) + } + return Value.fromArray(output) + } + + static fromI32Array(input: Array): Value { + let output = new Array(input.length) + for (let i: i32 = 0; i < input.length; i++) { + output[i] = Value.fromI32(input[i]) + } + return Value.fromArray(output) + } + + static fromBigIntArray(input: Array): Value { + let output = new Array(input.length) + for (let i: i32 = 0; i < input.length; i++) { + output[i] = Value.fromBigInt(input[i]) + } + return Value.fromArray(output) + } + + static fromBigDecimalArray(input: Array): Value { + let output = new Array(input.length) + for (let i: i32 = 0; i < input.length; i++) { + output[i] = Value.fromBigDecimal(input[i]) + } + return Value.fromArray(output) + } + + static fromStringArray(input: Array): Value { + let output = new Array(input.length) + for (let i: i32 = 0; i < input.length; i++) { + output[i] = Value.fromString(input[i]) + } + return Value.fromArray(output) + } + + static fromArray(input: Array): Value { + let value = new Value() + value.kind = ValueKind.ARRAY + value.data = changetype(input) as u64 + return value + } + + static fromBigInt(n: BigInt): Value { + let value = new Value() + value.kind = ValueKind.BIGINT + value.data = n as u64 + return value + } + + static fromBoolean(b: boolean): Value { + let value = new Value() + value.kind = ValueKind.BOOL + value.data = b ? 1 : 0 + return value + } + + static fromBytes(bytes: Bytes): Value { + let value = new Value() + value.kind = ValueKind.BYTES + value.data = changetype(bytes) as u64 + return value + } + + static fromNull(): Value { + let value = new Value() + value.kind = ValueKind.NULL + return value + } + + static fromI32(n: i32): Value { + let value = new Value() + value.kind = ValueKind.INT + value.data = n as u64 + return value + } + + static fromString(s: string): Value { + let value = new Value() + value.kind = ValueKind.STRING + value.data = changetype(s) + return value + } + + static fromBigDecimal(n: BigDecimal): Value { + let value = new Value() + value.kind = ValueKind.BIGDECIMAL + value.data = n as u64 + return value + } +} + +/** An arbitrary size integer represented as an array of bytes. */ +export class BigInt extends Uint8Array { + toHex(): string { + return typeConversion.bigIntToHex(this) + } + + toHexString(): string { + return typeConversion.bigIntToHex(this) + } + + toString(): string { + return typeConversion.bigIntToString(this) + } + + static fromI32(x: i32): BigInt { + return typeConversion.i32ToBigInt(x) as BigInt + } + + toI32(): i32 { + return typeConversion.bigIntToI32(this) + } + + @operator('+') + plus(other: BigInt): BigInt { + return bigInt.plus(this, other) + } + + @operator('-') + minus(other: BigInt): BigInt { + return bigInt.minus(this, other) + } + + @operator('*') + times(other: BigInt): BigInt { + return bigInt.times(this, other) + } + + @operator('/') + div(other: BigInt): BigInt { + return bigInt.dividedBy(this, other) + } + + divDecimal(other: BigDecimal): BigDecimal { + return bigInt.dividedByDecimal(this, other) + } + + @operator('%') + mod(other: BigInt): BigInt { + return bigInt.mod(this, other) + } + + @operator('==') + equals(other: BigInt): boolean { + if (this.length !== other.length) { + return false; + } + for (let i = 0; i < this.length; i++) { + if (this[i] !== other[i]) { + return false; + } + } + return true; + } + + toBigDecimal(): BigDecimal { + return new BigDecimal(this) + } +} + +export class BigDecimal { + exp!: BigInt + digits!: BigInt + + constructor(bigInt: BigInt) { + this.digits = bigInt + this.exp = BigInt.fromI32(0) + } + + static fromString(s: string): BigDecimal { + return bigDecimal.fromString(s) + } + + toString(): string { + return bigDecimal.toString(this) + } + + truncate(decimals: i32): BigDecimal { + let digitsRightOfZero = this.digits.toString().length + this.exp.toI32() + let newDigitLength = decimals + digitsRightOfZero + let truncateLength = this.digits.toString().length - newDigitLength + if (truncateLength < 0) { + return this + } else { + for (let i = 0; i < truncateLength; i++) { + this.digits = this.digits.div(BigInt.fromI32(10)) + } + this.exp = BigInt.fromI32(decimals * -1) + return this + } + } + + @operator('+') + plus(other: BigDecimal): BigDecimal { + return bigDecimal.plus(this, other) + } + + @operator('-') + minus(other: BigDecimal): BigDecimal { + return bigDecimal.minus(this, other) + } + + @operator('*') + times(other: BigDecimal): BigDecimal { + return bigDecimal.times(this, other) + } + + @operator('/') + div(other: BigDecimal): BigDecimal { + return bigDecimal.dividedBy(this, other) + } + + @operator('==') + equals(other: BigDecimal): boolean { + return bigDecimal.equals(this, other) + } +} + +export enum TokenKind { + ADDRESS = 0, + FIXED_BYTES = 1, + BYTES = 2, + INT = 3, + UINT = 4, + BOOL = 5, + STRING = 6, + FIXED_ARRAY = 7, + ARRAY = 8 +} +export class Token { + kind: TokenKind + data: Payload +} + +// Sequence of 4 `u64`s. +export type Int64 = Uint64Array; +export type Uint64 = Uint64Array; + +/** + * TypedMap entry. + */ +export class TypedMapEntry { + key: K + value: V + + constructor(key: K, value: V) { + this.key = key + this.value = value + } +} + +/** Typed map */ +export class TypedMap { + entries: Array> + + constructor() { + this.entries = new Array>(0) + } + + set(key: K, value: V): void { + let entry = this.getEntry(key) + if (entry !== null) { + entry.value = value + } else { + let entry = new TypedMapEntry(key, value) + this.entries.push(entry) + } + } + + getEntry(key: K): TypedMapEntry | null { + for (let i: i32 = 0; i < this.entries.length; i++) { + if (this.entries[i].key == key) { + return this.entries[i] + } + } + return null + } + + get(key: K): V | null { + for (let i: i32 = 0; i < this.entries.length; i++) { + if (this.entries[i].key == key) { + return this.entries[i].value + } + } + return null + } + + isSet(key: K): bool { + for (let i: i32 = 0; i < this.entries.length; i++) { + if (this.entries[i].key == key) { + return true + } + } + return false + } +} + +/** + * Common representation for entity data, storing entity attributes + * as `string` keys and the attribute values as dynamically-typed + * `Value` objects. + */ +export class Entity extends TypedMap { + unset(key: string): void { + this.set(key, Value.fromNull()) + } + + /** Assigns properties from sources to this Entity in right-to-left order */ + merge(sources: Array): Entity { + var target = this + for (let i = 0; i < sources.length; i++) { + let entries = sources[i].entries + for (let j = 0; j < entries.length; j++) { + target.set(entries[j].key, entries[j].value) + } + } + return target + } +} + +/** Type hint for JSON values. */ +export enum JSONValueKind { + NULL = 0, + BOOL = 1, + NUMBER = 2, + STRING = 3, + ARRAY = 4, + OBJECT = 5, +} + +/** + * Pointer type for JSONValue data. + * + * Big enough to fit any pointer or native `this.data`. + */ +export type JSONValuePayload = u64 +export class JSONValue { + kind: JSONValueKind + data: JSONValuePayload + + toString(): string { + assert(this.kind == JSONValueKind.STRING, 'JSON value is not a string.') + return changetype(this.data as u32) + } + + toObject(): TypedMap { + assert(this.kind == JSONValueKind.OBJECT, 'JSON value is not an object.') + return changetype>(this.data as u32) + } +} + +export class Wrapped { + inner: T; + + constructor(inner: T) { + this.inner = inner; + } +} + +export class Result { + _value: Wrapped | null; + _error: Wrapped | null; + + get isOk(): boolean { + return this._value !== null; + } + + get isError(): boolean { + return this._error !== null; + } + + get value(): V { + assert(this._value != null, "Trying to get a value from an error result"); + return (this._value as Wrapped).inner; + } + + get error(): E { + assert( + this._error != null, + "Trying to get an error from a successful result" + ); + return (this._error as Wrapped).inner; + } +} + +/** + * Byte array + */ +class ByteArray extends Uint8Array { + toHex(): string { + return typeConversion.bytesToHex(this) + } + + toHexString(): string { + return typeConversion.bytesToHex(this) + } + + toString(): string { + return typeConversion.bytesToString(this) + } + + toBase58(): string { + return typeConversion.bytesToBase58(this) + } +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/contract_calls.ts b/runtime/test/wasm_test/api_version_0_0_5/contract_calls.ts new file mode 100644 index 0000000..223ea0c --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/contract_calls.ts @@ -0,0 +1,10 @@ +export * from './common/global' +import { Address } from './common/types' + +export declare namespace ethereum { + function call(call: Address): Array
| null +} + +export function callContract(address: Address): void { + ethereum.call(address) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/contract_calls.wasm b/runtime/test/wasm_test/api_version_0_0_5/contract_calls.wasm new file mode 100644 index 0000000..b2ab791 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/contract_calls.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/crypto.ts b/runtime/test/wasm_test/api_version_0_0_5/crypto.ts new file mode 100644 index 0000000..89a1781 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/crypto.ts @@ -0,0 +1,9 @@ +export * from './common/global' + +declare namespace crypto { + function keccak256(input: Uint8Array): Uint8Array +} + +export function hash(input: Uint8Array): Uint8Array { + return crypto.keccak256(input) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/crypto.wasm b/runtime/test/wasm_test/api_version_0_0_5/crypto.wasm new file mode 100644 index 0000000..393c57a Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/crypto.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/data_source_create.ts b/runtime/test/wasm_test/api_version_0_0_5/data_source_create.ts new file mode 100644 index 0000000..c071dd8 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/data_source_create.ts @@ -0,0 +1,9 @@ +export * from './common/global' + +declare namespace dataSource { + function create(name: string, params: Array): void +} + +export function dataSourceCreate(name: string, params: Array): void { + dataSource.create(name, params) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/data_source_create.wasm b/runtime/test/wasm_test/api_version_0_0_5/data_source_create.wasm new file mode 100644 index 0000000..6b3f7e6 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/data_source_create.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/ens_name_by_hash.ts b/runtime/test/wasm_test/api_version_0_0_5/ens_name_by_hash.ts new file mode 100644 index 0000000..8f11518 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/ens_name_by_hash.ts @@ -0,0 +1,9 @@ +export * from './common/global' + +declare namespace ens { + function nameByHash(hash: string): string|null +} + +export function nameByHash(hash: string): string|null { + return ens.nameByHash(hash) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/ens_name_by_hash.wasm b/runtime/test/wasm_test/api_version_0_0_5/ens_name_by_hash.wasm new file mode 100644 index 0000000..6fe9960 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/ens_name_by_hash.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/ipfs_block.ts b/runtime/test/wasm_test/api_version_0_0_5/ipfs_block.ts new file mode 100644 index 0000000..7fc1797 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/ipfs_block.ts @@ -0,0 +1,13 @@ +export * from './common/global' + +declare namespace typeConversion { + function bytesToHex(bytes: Uint8Array): string +} + +declare namespace ipfs { + function getBlock(hash: String): Uint8Array +} + +export function ipfsBlockHex(hash: string): string { + return typeConversion.bytesToHex(ipfs.getBlock(hash)) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/ipfs_block.wasm b/runtime/test/wasm_test/api_version_0_0_5/ipfs_block.wasm new file mode 100644 index 0000000..985b3bb Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/ipfs_block.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/ipfs_cat.ts b/runtime/test/wasm_test/api_version_0_0_5/ipfs_cat.ts new file mode 100644 index 0000000..a66540b --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/ipfs_cat.ts @@ -0,0 +1,17 @@ +export * from './common/global' + +declare namespace typeConversion { + function bytesToString(bytes: Uint8Array): string +} + +declare namespace ipfs { + function cat(hash: String): Uint8Array +} + +export function ipfsCatString(hash: string): string { + return typeConversion.bytesToString(ipfs.cat(hash)) +} + +export function ipfsCat(hash: string): Uint8Array { + return ipfs.cat(hash) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/ipfs_cat.wasm b/runtime/test/wasm_test/api_version_0_0_5/ipfs_cat.wasm new file mode 100644 index 0000000..b803eb6 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/ipfs_cat.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/ipfs_map.ts b/runtime/test/wasm_test/api_version_0_0_5/ipfs_map.ts new file mode 100644 index 0000000..b52d31b --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/ipfs_map.ts @@ -0,0 +1,41 @@ +export * from './common/global' +import { Value, ValueKind, TypedMapEntry, TypedMap, Entity, JSONValueKind, JSONValue } from './common/types' + +/* + * Declarations copied from graph-ts/input.ts and edited for brevity + */ + +declare namespace store { + function set(entity: string, id: string, data: Entity): void +} + +/* + * Actual setup for the test + */ +declare namespace ipfs { + function map(hash: String, callback: String, userData: Value, flags: String[]): void +} + +export function echoToStore(data: JSONValue, userData: Value): void { + // expect a map of the form { "id": "anId", "value": "aValue" } + let map = data.toObject(); + + let id = map.get("id"); + let value = map.get("value"); + + assert(id !== null, "'id' should not be null"); + assert(value !== null, "'value' should not be null"); + + let stringId = id!.toString(); + let stringValue = value!.toString(); + + let entity = new Entity(); + entity.set("id", Value.fromString(stringId)); + entity.set("value", Value.fromString(stringValue)); + entity.set("extra", userData); + store.set("Thing", stringId, entity); +} + +export function ipfsMap(hash: string, userData: string): void { + ipfs.map(hash, "echoToStore", Value.fromString(userData), ["json"]) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/ipfs_map.wasm b/runtime/test/wasm_test/api_version_0_0_5/ipfs_map.wasm new file mode 100644 index 0000000..e4b5cdc Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/ipfs_map.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/json_parsing.ts b/runtime/test/wasm_test/api_version_0_0_5/json_parsing.ts new file mode 100644 index 0000000..d1d27ff --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/json_parsing.ts @@ -0,0 +1,49 @@ +import { JSONValue, JSONValueKind, Bytes, Wrapped, Result } from './common/types' + +enum IndexForAscTypeId { + STRING = 0, + ARRAY_BUFFER = 1, + UINT8_ARRAY = 6, + WRAPPED_BOOL = 28, + WRAPPED_JSON_VALUE = 29, + JSON_VALUE = 32, + RESULT_JSON_VALUE_BOOL = 40, +} + +export function id_of_type(type_id_index: IndexForAscTypeId): usize { + switch (type_id_index) { + case IndexForAscTypeId.STRING: + return idof(); + case IndexForAscTypeId.ARRAY_BUFFER: + return idof(); + case IndexForAscTypeId.UINT8_ARRAY: + return idof(); + case IndexForAscTypeId.WRAPPED_BOOL: + return idof>(); + case IndexForAscTypeId.WRAPPED_JSON_VALUE: + return idof>(); + case IndexForAscTypeId.JSON_VALUE: + return idof(); + case IndexForAscTypeId.RESULT_JSON_VALUE_BOOL: + return idof>(); + default: + return 0; + } +} + +export function allocate(n: usize): usize { + return __alloc(n); +} + +declare namespace json { + function try_fromBytes(data: Bytes): Result; +} + +export function handleJsonError(data: Bytes): string { + let result = json.try_fromBytes(data); + if (result.isOk) { + return "OK: " + result.value.toString() + ", ERROR: " + (result.isError ? "true" : "false"); + } else { + return "ERROR: " + (result.error ? "true" : "false"); + } +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/json_parsing.wasm b/runtime/test/wasm_test/api_version_0_0_5/json_parsing.wasm new file mode 100644 index 0000000..99479a5 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/json_parsing.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/non_terminating.ts b/runtime/test/wasm_test/api_version_0_0_5/non_terminating.ts new file mode 100644 index 0000000..77866c3 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/non_terminating.ts @@ -0,0 +1,10 @@ +export * from './common/global' + +// Test that non-terminating handlers are killed by timeout. +export function loop(): void { + while (true) {} +} + +export function rabbit_hole(): void { + rabbit_hole() +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/non_terminating.wasm b/runtime/test/wasm_test/api_version_0_0_5/non_terminating.wasm new file mode 100644 index 0000000..bd9d564 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/non_terminating.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/null_ptr_read.ts b/runtime/test/wasm_test/api_version_0_0_5/null_ptr_read.ts new file mode 100644 index 0000000..f111e6a --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/null_ptr_read.ts @@ -0,0 +1,82 @@ +export * from './common/global'; + +declare namespace bigInt { + function plus(x: BigInt, y: BigInt): BigInt +} + +class BigInt extends Uint8Array { + static fromI32(x: i32): BigInt { + let self = new Uint8Array(4); + self[0] = x as u8; + self[1] = (x >> 8) as u8; + self[2] = (x >> 16) as u8; + self[3] = (x >> 24) as u8; + return changetype(self); + } + + @operator('+') + plus(other: BigInt): BigInt { + return bigInt.plus(this, other); + } +} + +class Wrapper { + public constructor( + public n: BigInt | null + ) {} +} + +export function nullPtrRead(): void { + let x = BigInt.fromI32(2); + let y: BigInt | null = null; + + let wrapper = new Wrapper(y); + + // Operator overloading works even on nullable types. + // To fix this, the type signature of the function should + // consider nullable types, like this: + // + // @operator('+') + // plus(other: BigInt | null): BigInt { + // // Do null checks + // } + // + // This test is proposidely doing this to make sure we give + // the correct error message to the user. + wrapper.n = wrapper.n + x; +} + +class SafeBigInt extends Uint8Array { + static fromI32(x: i32): SafeBigInt { + let self = new Uint8Array(4); + self[0] = x as u8; + self[1] = (x >> 8) as u8; + self[2] = (x >> 16) as u8; + self[3] = (x >> 24) as u8; + return changetype(self); + } + + @operator('+') + plus(other: SafeBigInt): SafeBigInt { + assert(this !== null, "Failed to sum BigInts because left hand side is 'null'"); + + return changetype(bigInt.plus(changetype(this), changetype(other))); + } +} + +class Wrapper2 { + public constructor( + public n: SafeBigInt | null + ) {} +} + +export function safeNullPtrRead(): void { + let x = SafeBigInt.fromI32(2); + let y: SafeBigInt | null = null; + + let wrapper2 = new Wrapper2(y); + + // Breaks as well, but by our assertion, before getting into + // the Rust code. + wrapper2.n = wrapper2.n + x; +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/null_ptr_read.wasm b/runtime/test/wasm_test/api_version_0_0_5/null_ptr_read.wasm new file mode 100644 index 0000000..a4e05ba Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/null_ptr_read.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/store.ts b/runtime/test/wasm_test/api_version_0_0_5/store.ts new file mode 100644 index 0000000..e755c27 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/store.ts @@ -0,0 +1,53 @@ +export * from './common/global' +import { TypedMap, Entity, BigDecimal, Value } from './common/types' + +/** Definitions copied from graph-ts/index.ts */ +declare namespace store { + function get(entity: string, id: string): Entity | null + function set(entity: string, id: string, data: Entity): void + function remove(entity: string, id: string): void +} + +/** Host Ethereum interface */ +declare namespace ethereum { + function call(call: SmartContractCall): Array +} + +/** Host interface for BigInt arithmetic */ +declare namespace bigInt { + function plus(x: BigInt, y: BigInt): BigInt + function minus(x: BigInt, y: BigInt): BigInt + function times(x: BigInt, y: BigInt): BigInt + function dividedBy(x: BigInt, y: BigInt): BigInt + function dividedByDecimal(x: BigInt, y: BigDecimal): BigDecimal + function mod(x: BigInt, y: BigInt): BigInt +} + +/** Host interface for BigDecimal */ +declare namespace bigDecimal { + function plus(x: BigDecimal, y: BigDecimal): BigDecimal + function minus(x: BigDecimal, y: BigDecimal): BigDecimal + function times(x: BigDecimal, y: BigDecimal): BigDecimal + function dividedBy(x: BigDecimal, y: BigDecimal): BigDecimal + function equals(x: BigDecimal, y: BigDecimal): boolean + function toString(bigDecimal: BigDecimal): string + function fromString(s: string): BigDecimal +} + +/** + * Test functions + */ +export function getUser(id: string): Entity | null { + return store.get("User", id); +} + +export function loadAndSetUserName(id: string, name: string) : void { + let user = store.get("User", id); + if (user == null) { + user = new Entity(); + user.set("id", Value.fromString(id)); + } + user.set("name", Value.fromString(name)); + // save it + store.set("User", id, (user as Entity)); +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/store.wasm b/runtime/test/wasm_test/api_version_0_0_5/store.wasm new file mode 100644 index 0000000..577d136 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/store.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/string_to_number.ts b/runtime/test/wasm_test/api_version_0_0_5/string_to_number.ts new file mode 100644 index 0000000..027b387 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/string_to_number.ts @@ -0,0 +1,26 @@ +export * from './common/global' +import { BigInt } from './common/types' + +/** Host JSON interface */ +declare namespace json { + function toI64(decimal: string): i64 + function toU64(decimal: string): u64 + function toF64(decimal: string): f64 + function toBigInt(decimal: string): BigInt +} + +export function testToI64(decimal: string): i64 { + return json.toI64(decimal); +} + +export function testToU64(decimal: string): u64 { + return json.toU64(decimal); +} + +export function testToF64(decimal: string): f64 { + return json.toF64(decimal) +} + +export function testToBigInt(decimal: string): BigInt { + return json.toBigInt(decimal) +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/string_to_number.wasm b/runtime/test/wasm_test/api_version_0_0_5/string_to_number.wasm new file mode 100644 index 0000000..bc46b67 Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/string_to_number.wasm differ diff --git a/runtime/test/wasm_test/api_version_0_0_5/test_padding.ts b/runtime/test/wasm_test/api_version_0_0_5/test_padding.ts new file mode 100644 index 0000000..7051de3 --- /dev/null +++ b/runtime/test/wasm_test/api_version_0_0_5/test_padding.ts @@ -0,0 +1,125 @@ +export * from './common/global' + +export class UnitTestTypeBool{ + str_pref: string; + under_test: boolean; + str_suff: string; + large: i64 ; + tail: boolean ; + + + + constructor(str_pref: string, under_test: boolean, str_suff:string, large: i64, tail:boolean) { + this.str_pref = str_pref; + this.under_test = under_test; + this.str_suff = str_suff; + this.large = large; + this.tail = tail; + } +} + +export function test_padding_bool(p: UnitTestTypeBool): void { + assert(p.str_pref == "pref", "parm.str_pref: Assertion failed!"); + assert(p.under_test == true, "parm.under_test: Assertion failed!"); + assert(p.str_suff == "suff", "parm.str_suff: Assertion failed!"); + assert(p.large == 9223372036854775807, "parm.large: Assertion failed!"); + assert(p.tail == true, "parm.tail: Assertion failed!"); +} + +export class UnitTestTypeI8{ + str_pref: string; + under_test: i8; + str_suff: string; + large: i64 ; + tail: boolean ; + + + + constructor(str_pref: string, under_test: i8, str_suff:string, large: i64, tail:boolean) { + this.str_pref = str_pref; + this.under_test = under_test; + this.str_suff = str_suff; + this.large = large; + this.tail = tail; + } +} + +export function test_padding_i8(p: UnitTestTypeI8): void { + assert(p.str_pref == "pref", "parm.str_pref: Assertion failed!"); + assert(p.under_test == 127, "parm.under_test: Assertion failed!"); + assert(p.str_suff == "suff", "parm.str_suff: Assertion failed!"); + assert(p.large == 9223372036854775807, "parm.large: Assertion failed!"); + assert(p.tail == true, "parm.tail: Assertion failed!"); +} + + +export class UnitTestTypeU16{ + str_pref: string; + under_test: i16; + str_suff: string; + large: i64 ; + tail: boolean ; + + + + constructor(str_pref: string, under_test: i16, str_suff:string, large: i64, tail:boolean) { + this.str_pref = str_pref; + this.under_test = under_test; + this.str_suff = str_suff; + this.large = large; + this.tail = tail; + } +} + +export function test_padding_i16(p: UnitTestTypeU16): void { + assert(p.str_pref == "pref", "parm.str_pref: Assertion failed!"); + assert(p.under_test == 32767, "parm.under_test: Assertion failed!"); + assert(p.str_suff == "suff", "parm.str_suff: Assertion failed!"); + assert(p.large == 9223372036854775807, "parm.large: Assertion failed!"); + assert(p.tail == true, "parm.tail: Assertion failed!"); +} + +export class UnitTestTypeU32{ + str_pref: string; + under_test: i32; + str_suff: string; + large: i64 ; + tail: boolean ; + + + + constructor(str_pref: string, under_test: i32, str_suff:string, large: i64, tail:boolean) { + this.str_pref = str_pref; + this.under_test = under_test; + this.str_suff = str_suff; + this.large = large; + this.tail = tail; + } +} + +export function test_padding_i32(p: UnitTestTypeU32): void { + assert(p.str_pref == "pref", "parm.str_pref: Assertion failed!"); + assert(p.under_test == 2147483647, "parm.under_test: Assertion failed!"); + assert(p.str_suff == "suff", "parm.str_suff: Assertion failed!"); + assert(p.large == 9223372036854775807, "parm.large: Assertion failed!"); + assert(p.tail == true, "parm.tail: Assertion failed!"); +} + + +export class ManualPadding{ + nonce: i64 ; + str_suff: string; + tail: i64 ; + + constructor(nonce: i64, str_suff:string, tail:i64) { + this.nonce = nonce; + this.str_suff = str_suff; + this.tail = tail + } +} + +export function test_padding_manual(p: ManualPadding): void { + assert(p.nonce == 9223372036854775807, "parm.nonce: Assertion failed!"); + assert(p.str_suff == "suff", "parm.str_suff: Assertion failed!"); + assert(p.tail == 9223372036854775807, "parm.tail: Assertion failed!"); +} diff --git a/runtime/test/wasm_test/api_version_0_0_5/test_padding.wasm b/runtime/test/wasm_test/api_version_0_0_5/test_padding.wasm new file mode 100644 index 0000000..f68205a Binary files /dev/null and b/runtime/test/wasm_test/api_version_0_0_5/test_padding.wasm differ diff --git a/runtime/wasm/Cargo.toml b/runtime/wasm/Cargo.toml new file mode 100644 index 0000000..32818eb --- /dev/null +++ b/runtime/wasm/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "graph-runtime-wasm" +version = "0.27.0" +edition = "2021" + +[dependencies] +async-trait = "0.1.50" +atomic_refcell = "0.1.8" +ethabi = "17.2" +futures = "0.1.21" +hex = "0.4.3" +graph = { path = "../../graph" } +bs58 = "0.4.0" +graph-runtime-derive = { path = "../derive" } +semver = "1.0.12" +lazy_static = "1.4" +uuid = { version = "1.1.2", features = ["v4"] } +strum = "0.21.0" +strum_macros = "0.21.1" +bytes = "1.0" +anyhow = "1.0" +wasmtime = "0.27.0" +defer = "0.1" +never = "0.1" + +wasm-instrument = { version = "0.2.0", features = ["std", "sign_ext"] } + +# AssemblyScript uses sign extensions +parity-wasm = { version = "0.45", features = ["std", "sign_ext"] } diff --git a/runtime/wasm/src/asc_abi/class.rs b/runtime/wasm/src/asc_abi/class.rs new file mode 100644 index 0000000..5298eee --- /dev/null +++ b/runtime/wasm/src/asc_abi/class.rs @@ -0,0 +1,723 @@ +use ethabi; +use semver::Version; + +use graph::{ + data::store, + runtime::{ + gas::GasCounter, AscHeap, AscIndexId, AscType, AscValue, IndexForAscTypeId, ToAscObj, + }, +}; +use graph::{prelude::serde_json, runtime::DeterministicHostError}; +use graph::{prelude::slog, runtime::AscPtr}; +use graph_runtime_derive::AscType; + +use crate::asc_abi::{v0_0_4, v0_0_5}; + +///! Rust types that have with a direct correspondence to an Asc class, +///! with their `AscType` implementations. + +/// Wrapper of ArrayBuffer for multiple AssemblyScript versions. +/// It just delegates its method calls to the correct mappings apiVersion. +pub enum ArrayBuffer { + ApiVersion0_0_4(v0_0_4::ArrayBuffer), + ApiVersion0_0_5(v0_0_5::ArrayBuffer), +} + +impl ArrayBuffer { + pub(crate) fn new( + values: &[T], + api_version: Version, + ) -> Result { + match api_version { + version if version <= Version::new(0, 0, 4) => { + Ok(Self::ApiVersion0_0_4(v0_0_4::ArrayBuffer::new(values)?)) + } + _ => Ok(Self::ApiVersion0_0_5(v0_0_5::ArrayBuffer::new(values)?)), + } + } +} + +impl AscType for ArrayBuffer { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + match self { + Self::ApiVersion0_0_4(a) => a.to_asc_bytes(), + Self::ApiVersion0_0_5(a) => a.to_asc_bytes(), + } + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + match api_version { + version if *version <= Version::new(0, 0, 4) => Ok(Self::ApiVersion0_0_4( + v0_0_4::ArrayBuffer::from_asc_bytes(asc_obj, api_version)?, + )), + _ => Ok(Self::ApiVersion0_0_5(v0_0_5::ArrayBuffer::from_asc_bytes( + asc_obj, + api_version, + )?)), + } + } + + fn asc_size( + ptr: AscPtr, + heap: &H, + gas: &GasCounter, + ) -> Result { + v0_0_4::ArrayBuffer::asc_size(AscPtr::new(ptr.wasm_ptr()), heap, gas) + } + + fn content_len(&self, asc_bytes: &[u8]) -> usize { + match self { + Self::ApiVersion0_0_5(a) => a.content_len(asc_bytes), + _ => unreachable!("Only called for apiVersion >=0.0.5"), + } + } +} + +impl AscIndexId for ArrayBuffer { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayBuffer; +} + +/// Wrapper of TypedArray for multiple AssemblyScript versions. +/// It just delegates its method calls to the correct mappings apiVersion. +pub enum TypedArray { + ApiVersion0_0_4(v0_0_4::TypedArray), + ApiVersion0_0_5(v0_0_5::TypedArray), +} + +impl TypedArray { + pub fn new( + content: &[T], + heap: &mut H, + gas: &GasCounter, + ) -> Result { + match heap.api_version() { + version if version <= Version::new(0, 0, 4) => Ok(Self::ApiVersion0_0_4( + v0_0_4::TypedArray::new(content, heap, gas)?, + )), + _ => Ok(Self::ApiVersion0_0_5(v0_0_5::TypedArray::new( + content, heap, gas, + )?)), + } + } + + pub fn to_vec( + &self, + heap: &H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + match self { + Self::ApiVersion0_0_4(t) => t.to_vec(heap, gas), + Self::ApiVersion0_0_5(t) => t.to_vec(heap, gas), + } + } +} + +impl AscType for TypedArray { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + match self { + Self::ApiVersion0_0_4(t) => t.to_asc_bytes(), + Self::ApiVersion0_0_5(t) => t.to_asc_bytes(), + } + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + match api_version { + version if *version <= Version::new(0, 0, 4) => Ok(Self::ApiVersion0_0_4( + v0_0_4::TypedArray::from_asc_bytes(asc_obj, api_version)?, + )), + _ => Ok(Self::ApiVersion0_0_5(v0_0_5::TypedArray::from_asc_bytes( + asc_obj, + api_version, + )?)), + } + } +} + +pub struct Bytes<'a>(pub &'a Vec); + +pub type Uint8Array = TypedArray; +impl ToAscObj for Bytes<'_> { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + self.0.to_asc_obj(heap, gas) + } +} + +impl AscIndexId for TypedArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::Int8Array; +} + +impl AscIndexId for TypedArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::Int16Array; +} + +impl AscIndexId for TypedArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::Int32Array; +} + +impl AscIndexId for TypedArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::Int64Array; +} + +impl AscIndexId for TypedArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::Uint8Array; +} + +impl AscIndexId for TypedArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::Uint16Array; +} + +impl AscIndexId for TypedArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::Uint32Array; +} + +impl AscIndexId for TypedArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::Uint64Array; +} + +impl AscIndexId for TypedArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::Float32Array; +} + +impl AscIndexId for TypedArray { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::Float64Array; +} + +/// Wrapper of String for multiple AssemblyScript versions. +/// It just delegates its method calls to the correct mappings apiVersion. +pub enum AscString { + ApiVersion0_0_4(v0_0_4::AscString), + ApiVersion0_0_5(v0_0_5::AscString), +} + +impl AscString { + pub fn new(content: &[u16], api_version: Version) -> Result { + match api_version { + version if version <= Version::new(0, 0, 4) => { + Ok(Self::ApiVersion0_0_4(v0_0_4::AscString::new(content)?)) + } + _ => Ok(Self::ApiVersion0_0_5(v0_0_5::AscString::new(content)?)), + } + } + + pub fn content(&self) -> &[u16] { + match self { + Self::ApiVersion0_0_4(s) => &s.content, + Self::ApiVersion0_0_5(s) => &s.content, + } + } +} + +impl AscIndexId for AscString { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::String; +} + +impl AscType for AscString { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + match self { + Self::ApiVersion0_0_4(s) => s.to_asc_bytes(), + Self::ApiVersion0_0_5(s) => s.to_asc_bytes(), + } + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + match api_version { + version if *version <= Version::new(0, 0, 4) => Ok(Self::ApiVersion0_0_4( + v0_0_4::AscString::from_asc_bytes(asc_obj, api_version)?, + )), + _ => Ok(Self::ApiVersion0_0_5(v0_0_5::AscString::from_asc_bytes( + asc_obj, + api_version, + )?)), + } + } + + fn asc_size( + ptr: AscPtr, + heap: &H, + gas: &GasCounter, + ) -> Result { + v0_0_4::AscString::asc_size(AscPtr::new(ptr.wasm_ptr()), heap, gas) + } + + fn content_len(&self, asc_bytes: &[u8]) -> usize { + match self { + Self::ApiVersion0_0_5(s) => s.content_len(asc_bytes), + _ => unreachable!("Only called for apiVersion >=0.0.5"), + } + } +} + +/// Wrapper of Array for multiple AssemblyScript versions. +/// It just delegates its method calls to the correct mappings apiVersion. +pub enum Array { + ApiVersion0_0_4(v0_0_4::Array), + ApiVersion0_0_5(v0_0_5::Array), +} + +impl Array { + pub fn new( + content: &[T], + heap: &mut H, + gas: &GasCounter, + ) -> Result { + match heap.api_version() { + version if version <= Version::new(0, 0, 4) => Ok(Self::ApiVersion0_0_4( + v0_0_4::Array::new(content, heap, gas)?, + )), + _ => Ok(Self::ApiVersion0_0_5(v0_0_5::Array::new( + content, heap, gas, + )?)), + } + } + + pub(crate) fn to_vec( + &self, + heap: &H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + match self { + Self::ApiVersion0_0_4(a) => a.to_vec(heap, gas), + Self::ApiVersion0_0_5(a) => a.to_vec(heap, gas), + } + } +} + +impl AscType for Array { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + match self { + Self::ApiVersion0_0_4(a) => a.to_asc_bytes(), + Self::ApiVersion0_0_5(a) => a.to_asc_bytes(), + } + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + match api_version { + version if *version <= Version::new(0, 0, 4) => Ok(Self::ApiVersion0_0_4( + v0_0_4::Array::from_asc_bytes(asc_obj, api_version)?, + )), + _ => Ok(Self::ApiVersion0_0_5(v0_0_5::Array::from_asc_bytes( + asc_obj, + api_version, + )?)), + } + } +} + +impl AscIndexId for Array { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayBool; +} + +impl AscIndexId for Array { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayUint8Array; +} + +impl AscIndexId for Array>> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayEthereumValue; +} + +impl AscIndexId for Array>> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayStoreValue; +} + +impl AscIndexId for Array>> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayJsonValue; +} + +impl AscIndexId for Array> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayString; +} + +impl AscIndexId for Array>>> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = + IndexForAscTypeId::ArrayTypedMapEntryStringJsonValue; +} + +impl AscIndexId for Array>>> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = + IndexForAscTypeId::ArrayTypedMapEntryStringStoreValue; +} + +impl AscIndexId for Array { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayU8; +} + +impl AscIndexId for Array { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayU16; +} + +impl AscIndexId for Array { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayU32; +} + +impl AscIndexId for Array { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayU64; +} + +impl AscIndexId for Array { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayI8; +} + +impl AscIndexId for Array { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayI16; +} + +impl AscIndexId for Array { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayI32; +} + +impl AscIndexId for Array { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayI64; +} + +impl AscIndexId for Array { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayF32; +} + +impl AscIndexId for Array { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayF64; +} + +impl AscIndexId for Array> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayBigDecimal; +} + +/// Represents any `AscValue` since they all fit in 64 bits. +#[repr(C)] +#[derive(Copy, Clone, Default)] +pub struct EnumPayload(pub u64); + +impl AscType for EnumPayload { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + self.0.to_asc_bytes() + } + + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + Ok(EnumPayload(u64::from_asc_bytes(asc_obj, api_version)?)) + } +} + +impl From for i32 { + fn from(payload: EnumPayload) -> i32 { + payload.0 as i32 + } +} + +impl From for f64 { + fn from(payload: EnumPayload) -> f64 { + f64::from_bits(payload.0) + } +} + +impl From for bool { + fn from(payload: EnumPayload) -> bool { + payload.0 != 0 + } +} + +impl From for EnumPayload { + fn from(x: i32) -> EnumPayload { + EnumPayload(x as u64) + } +} + +impl From for EnumPayload { + fn from(x: f64) -> EnumPayload { + EnumPayload(x.to_bits()) + } +} + +impl From for EnumPayload { + fn from(b: bool) -> EnumPayload { + EnumPayload(if b { 1 } else { 0 }) + } +} + +impl From for EnumPayload { + fn from(x: i64) -> EnumPayload { + EnumPayload(x as u64) + } +} + +impl From for AscPtr { + fn from(payload: EnumPayload) -> Self { + AscPtr::new(payload.0 as u32) + } +} + +impl From> for EnumPayload { + fn from(x: AscPtr) -> EnumPayload { + EnumPayload(x.wasm_ptr() as u64) + } +} + +/// In Asc, we represent a Rust enum as a discriminant `kind: D`, which is an +/// Asc enum so in Rust it's a `#[repr(u32)]` enum, plus an arbitrary `AscValue` +/// payload. +#[repr(C)] +#[derive(AscType)] +pub struct AscEnum { + pub kind: D, + pub _padding: u32, // Make padding explicit. + pub payload: EnumPayload, +} + +impl AscIndexId for AscEnum { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumValue; +} + +impl AscIndexId for AscEnum { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::StoreValue; +} + +impl AscIndexId for AscEnum { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::JsonValue; +} + +pub type AscEnumArray = AscPtr>>>; + +#[repr(u32)] +#[derive(AscType, Copy, Clone)] +pub enum EthereumValueKind { + Address, + FixedBytes, + Bytes, + Int, + Uint, + Bool, + String, + FixedArray, + Array, + Tuple, +} + +impl EthereumValueKind { + pub(crate) fn get_kind(token: ðabi::Token) -> Self { + match token { + ethabi::Token::Address(_) => EthereumValueKind::Address, + ethabi::Token::FixedBytes(_) => EthereumValueKind::FixedBytes, + ethabi::Token::Bytes(_) => EthereumValueKind::Bytes, + ethabi::Token::Int(_) => EthereumValueKind::Int, + ethabi::Token::Uint(_) => EthereumValueKind::Uint, + ethabi::Token::Bool(_) => EthereumValueKind::Bool, + ethabi::Token::String(_) => EthereumValueKind::String, + ethabi::Token::FixedArray(_) => EthereumValueKind::FixedArray, + ethabi::Token::Array(_) => EthereumValueKind::Array, + ethabi::Token::Tuple(_) => EthereumValueKind::Tuple, + } + } +} + +impl Default for EthereumValueKind { + fn default() -> Self { + EthereumValueKind::Address + } +} + +impl AscValue for EthereumValueKind {} + +#[repr(u32)] +#[derive(AscType, Copy, Clone)] +pub enum StoreValueKind { + String, + Int, + BigDecimal, + Bool, + Array, + Null, + Bytes, + BigInt, +} + +impl StoreValueKind { + pub(crate) fn get_kind(value: &store::Value) -> StoreValueKind { + use self::store::Value; + + match value { + Value::String(_) => StoreValueKind::String, + Value::Int(_) => StoreValueKind::Int, + Value::BigDecimal(_) => StoreValueKind::BigDecimal, + Value::Bool(_) => StoreValueKind::Bool, + Value::List(_) => StoreValueKind::Array, + Value::Null => StoreValueKind::Null, + Value::Bytes(_) => StoreValueKind::Bytes, + Value::BigInt(_) => StoreValueKind::BigInt, + } + } +} + +impl Default for StoreValueKind { + fn default() -> Self { + StoreValueKind::Null + } +} + +impl AscValue for StoreValueKind {} + +/// Big ints are represented using signed number representation. Note: This differs +/// from how U256 and U128 are represented (they use two's complement). So whenever +/// we convert between them, we need to make sure we handle signed and unsigned +/// cases correctly. +pub type AscBigInt = Uint8Array; + +pub type AscAddress = Uint8Array; +pub type AscH160 = Uint8Array; + +#[repr(C)] +#[derive(AscType)] +pub struct AscTypedMapEntry { + pub key: AscPtr, + pub value: AscPtr, +} + +impl AscIndexId for AscTypedMapEntry> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::TypedMapEntryStringStoreValue; +} + +impl AscIndexId for AscTypedMapEntry> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::TypedMapEntryStringJsonValue; +} + +pub(crate) type AscTypedMapEntryArray = Array>>; + +#[repr(C)] +#[derive(AscType)] +pub struct AscTypedMap { + pub entries: AscPtr>, +} + +impl AscIndexId for AscTypedMap> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::TypedMapStringStoreValue; +} + +impl AscIndexId for AscTypedMap> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::TypedMapStringJsonValue; +} + +impl AscIndexId for AscTypedMap>> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = + IndexForAscTypeId::TypedMapStringTypedMapStringJsonValue; +} + +pub type AscEntity = AscTypedMap>; +pub(crate) type AscJson = AscTypedMap>; + +#[repr(u32)] +#[derive(AscType, Copy, Clone)] +pub enum JsonValueKind { + Null, + Bool, + Number, + String, + Array, + Object, +} + +impl Default for JsonValueKind { + fn default() -> Self { + JsonValueKind::Null + } +} + +impl AscValue for JsonValueKind {} + +impl JsonValueKind { + pub(crate) fn get_kind(token: &serde_json::Value) -> Self { + use serde_json::Value; + + match token { + Value::Null => JsonValueKind::Null, + Value::Bool(_) => JsonValueKind::Bool, + Value::Number(_) => JsonValueKind::Number, + Value::String(_) => JsonValueKind::String, + Value::Array(_) => JsonValueKind::Array, + Value::Object(_) => JsonValueKind::Object, + } + } +} + +#[repr(C)] +#[derive(AscType)] +pub struct AscBigDecimal { + pub digits: AscPtr, + + // Decimal exponent. This is the opposite of `scale` in rust BigDecimal. + pub exp: AscPtr, +} + +impl AscIndexId for AscBigDecimal { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::BigDecimal; +} + +#[repr(u32)] +pub(crate) enum LogLevel { + Critical, + Error, + Warning, + Info, + Debug, +} + +impl From for slog::Level { + fn from(level: LogLevel) -> slog::Level { + match level { + LogLevel::Critical => slog::Level::Critical, + LogLevel::Error => slog::Level::Error, + LogLevel::Warning => slog::Level::Warning, + LogLevel::Info => slog::Level::Info, + LogLevel::Debug => slog::Level::Debug, + } + } +} + +#[repr(C)] +#[derive(AscType)] +pub struct AscResult { + pub value: AscPtr>, + pub error: AscPtr>, +} + +impl AscIndexId for AscResult, bool> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = + IndexForAscTypeId::ResultTypedMapStringJsonValueBool; +} + +impl AscIndexId for AscResult>, bool> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ResultJsonValueBool; +} + +#[repr(C)] +#[derive(AscType, Copy, Clone)] +pub struct AscWrapped { + pub inner: V, +} + +impl AscIndexId for AscWrapped> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::WrappedTypedMapStringJsonValue; +} + +impl AscIndexId for AscWrapped { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::WrappedBool; +} + +impl AscIndexId for AscWrapped>> { + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::WrappedJsonValue; +} diff --git a/runtime/wasm/src/asc_abi/mod.rs b/runtime/wasm/src/asc_abi/mod.rs new file mode 100644 index 0000000..4f69f52 --- /dev/null +++ b/runtime/wasm/src/asc_abi/mod.rs @@ -0,0 +1,4 @@ +// This unecessary nesting of the module should be resolved by further refactoring. +pub mod class; +pub mod v0_0_4; +pub mod v0_0_5; diff --git a/runtime/wasm/src/asc_abi/v0_0_4.rs b/runtime/wasm/src/asc_abi/v0_0_4.rs new file mode 100644 index 0000000..92f14ed --- /dev/null +++ b/runtime/wasm/src/asc_abi/v0_0_4.rs @@ -0,0 +1,330 @@ +use graph::runtime::gas::GasCounter; +use std::convert::TryInto as _; +use std::marker::PhantomData; +use std::mem::{size_of, size_of_val}; + +use anyhow::anyhow; +use semver::Version; + +use graph::runtime::{AscHeap, AscPtr, AscType, AscValue, DeterministicHostError}; +use graph_runtime_derive::AscType; + +use crate::asc_abi::class; + +/// Module related to AssemblyScript version v0.6. + +/// Asc std ArrayBuffer: "a generic, fixed-length raw binary data buffer". +/// See https://github.com/AssemblyScript/assemblyscript/wiki/Memory-Layout-&-Management/86447e88be5aa8ec633eaf5fe364651136d136ab#arrays +pub struct ArrayBuffer { + pub byte_length: u32, + // Asc allocators always align at 8 bytes, we already have 4 bytes from + // `byte_length_size` so with 4 more bytes we align the contents at 8 + // bytes. No Asc type has alignment greater than 8, so the + // elements in `content` will be aligned for any element type. + pub padding: [u8; 4], + // In Asc this slice is layed out inline with the ArrayBuffer. + pub content: Box<[u8]>, +} + +impl ArrayBuffer { + pub fn new(values: &[T]) -> Result { + let mut content = Vec::new(); + for value in values { + let asc_bytes = value.to_asc_bytes()?; + // An `AscValue` has size equal to alignment, no padding required. + content.extend(&asc_bytes); + } + + if content.len() > u32::max_value() as usize { + return Err(DeterministicHostError::from(anyhow::anyhow!( + "slice cannot fit in WASM memory" + ))); + } + Ok(ArrayBuffer { + byte_length: content.len() as u32, + padding: [0; 4], + content: content.into(), + }) + } + + /// Read `length` elements of type `T` starting at `byte_offset`. + /// + /// Panics if that tries to read beyond the length of `self.content`. + pub fn get( + &self, + byte_offset: u32, + length: u32, + api_version: Version, + ) -> Result, DeterministicHostError> { + let length = length as usize; + let byte_offset = byte_offset as usize; + + self.content[byte_offset..] + .chunks(size_of::()) + .take(length) + .map(|asc_obj| T::from_asc_bytes(asc_obj, &api_version)) + .collect() + + // TODO: This code is preferred as it validates the length of the array. + // But, some existing subgraphs were found to break when this was added. + // This needs to be root caused + /* + let range = byte_offset..byte_offset + length * size_of::(); + self.content + .get(range) + .ok_or_else(|| { + DeterministicHostError::from(anyhow::anyhow!("Attempted to read past end of array")) + })? + .chunks_exact(size_of::()) + .map(|bytes| T::from_asc_bytes(bytes)) + .collect() + */ + } +} + +impl AscType for ArrayBuffer { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + let mut asc_layout: Vec = Vec::new(); + + let byte_length: [u8; 4] = self.byte_length.to_le_bytes(); + asc_layout.extend(&byte_length); + asc_layout.extend(&self.padding); + asc_layout.extend(self.content.iter()); + + // Allocate extra capacity to next power of two, as required by asc. + let header_size = size_of_val(&byte_length) + size_of_val(&self.padding); + let total_size = self.byte_length as usize + header_size; + let total_capacity = total_size.next_power_of_two(); + let extra_capacity = total_capacity - total_size; + asc_layout.extend(std::iter::repeat(0).take(extra_capacity)); + assert_eq!(asc_layout.len(), total_capacity); + + Ok(asc_layout) + } + + /// The Rust representation of an Asc object as layed out in Asc memory. + fn from_asc_bytes( + asc_obj: &[u8], + api_version: &Version, + ) -> Result { + // Skip `byte_length` and the padding. + let content_offset = size_of::() + 4; + let byte_length = asc_obj.get(..size_of::()).ok_or_else(|| { + DeterministicHostError::from(anyhow!("Attempted to read past end of array")) + })?; + let content = asc_obj.get(content_offset..).ok_or_else(|| { + DeterministicHostError::from(anyhow!("Attempted to read past end of array")) + })?; + Ok(ArrayBuffer { + byte_length: u32::from_asc_bytes(&byte_length, api_version)?, + padding: [0; 4], + content: content.to_vec().into(), + }) + } + + fn asc_size( + ptr: AscPtr, + heap: &H, + gas: &GasCounter, + ) -> Result { + let byte_length = ptr.read_u32(heap, gas)?; + let byte_length_size = size_of::() as u32; + let padding_size = size_of::() as u32; + Ok(byte_length_size + padding_size + byte_length) + } +} + +/// A typed, indexable view of an `ArrayBuffer` of Asc primitives. In Asc it's +/// an abstract class with subclasses for each primitive, for example +/// `Uint8Array` is `TypedArray`. +/// See https://github.com/AssemblyScript/assemblyscript/wiki/Memory-Layout-&-Management/86447e88be5aa8ec633eaf5fe364651136d136ab#arrays +#[repr(C)] +#[derive(AscType)] +pub struct TypedArray { + pub buffer: AscPtr, + /// Byte position in `buffer` of the array start. + byte_offset: u32, + byte_length: u32, + ty: PhantomData, +} + +impl TypedArray { + pub(crate) fn new( + content: &[T], + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let buffer = class::ArrayBuffer::new(content, heap.api_version())?; + let buffer_byte_length = if let class::ArrayBuffer::ApiVersion0_0_4(ref a) = buffer { + a.byte_length + } else { + unreachable!("Only the correct ArrayBuffer will be constructed") + }; + let ptr = AscPtr::alloc_obj(buffer, heap, gas)?; + Ok(TypedArray { + byte_length: buffer_byte_length, + buffer: AscPtr::new(ptr.wasm_ptr()), + byte_offset: 0, + ty: PhantomData, + }) + } + + pub(crate) fn to_vec( + &self, + heap: &H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + self.buffer.read_ptr(heap, gas)?.get( + self.byte_offset, + self.byte_length / size_of::() as u32, + heap.api_version(), + ) + } +} + +/// Asc std string: "Strings are encoded as UTF-16LE in AssemblyScript, and are +/// prefixed with their length (in character codes) as a 32-bit integer". See +/// https://github.com/AssemblyScript/assemblyscript/wiki/Memory-Layout-&-Management/86447e88be5aa8ec633eaf5fe364651136d136ab#arrays +pub struct AscString { + // In number of UTF-16 code units (2 bytes each). + length: u32, + // The sequence of UTF-16LE code units that form the string. + pub content: Box<[u16]>, +} + +impl AscString { + pub fn new(content: &[u16]) -> Result { + if size_of_val(content) > u32::max_value() as usize { + return Err(DeterministicHostError::from(anyhow!( + "string cannot fit in WASM memory" + ))); + } + + Ok(AscString { + length: content.len() as u32, + content: content.into(), + }) + } +} + +impl AscType for AscString { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + let mut asc_layout: Vec = Vec::new(); + + let length: [u8; 4] = self.length.to_le_bytes(); + asc_layout.extend(&length); + + // Write the code points, in little-endian (LE) order. + for &code_unit in self.content.iter() { + let low_byte = code_unit as u8; + let high_byte = (code_unit >> 8) as u8; + asc_layout.push(low_byte); + asc_layout.push(high_byte); + } + + Ok(asc_layout) + } + + /// The Rust representation of an Asc object as layed out in Asc memory. + fn from_asc_bytes( + asc_obj: &[u8], + _api_version: &Version, + ) -> Result { + // Pointer for our current position within `asc_obj`, + // initially at the start of the content skipping `length`. + let mut offset = size_of::(); + + let length = asc_obj + .get(..offset) + .ok_or(DeterministicHostError::from(anyhow::anyhow!( + "String bytes not long enough to contain length" + )))?; + + // Does not panic - already validated slice length == size_of::. + let length = i32::from_le_bytes(length.try_into().unwrap()); + if length.checked_mul(2).and_then(|l| l.checked_add(4)) != asc_obj.len().try_into().ok() { + return Err(DeterministicHostError::from(anyhow::anyhow!( + "String length header does not equal byte length" + ))); + } + + // Prevents panic when accessing offset + 1 in the loop + if asc_obj.len() % 2 != 0 { + return Err(DeterministicHostError::from(anyhow::anyhow!( + "Invalid string length" + ))); + } + + // UTF-16 (used in assemblyscript) always uses one + // pair of bytes per code unit. + // https://mathiasbynens.be/notes/javascript-encoding + // UTF-16 (16-bit Unicode Transformation Format) is an + // extension of UCS-2 that allows representing code points + // outside the BMP. It produces a variable-length result + // of either one or two 16-bit code units per code point. + // This way, it can encode code points in the range from 0 + // to 0x10FFFF. + + // Read the content. + let mut content = Vec::new(); + while offset < asc_obj.len() { + let code_point_bytes = [asc_obj[offset], asc_obj[offset + 1]]; + let code_point = u16::from_le_bytes(code_point_bytes); + content.push(code_point); + offset += size_of::(); + } + AscString::new(&content) + } + + fn asc_size( + ptr: AscPtr, + heap: &H, + gas: &GasCounter, + ) -> Result { + let length = ptr.read_u32(heap, gas)?; + let length_size = size_of::() as u32; + let code_point_size = size_of::() as u32; + let data_size = code_point_size.checked_mul(length); + let total_size = data_size.and_then(|d| d.checked_add(length_size)); + total_size.ok_or_else(|| { + DeterministicHostError::from(anyhow::anyhow!("Overflowed when getting size of string")) + }) + } +} + +/// Growable array backed by an `ArrayBuffer`. +/// See https://github.com/AssemblyScript/assemblyscript/wiki/Memory-Layout-&-Management/86447e88be5aa8ec633eaf5fe364651136d136ab#arrays +#[repr(C)] +#[derive(AscType)] +pub struct Array { + buffer: AscPtr, + length: u32, + ty: PhantomData, +} + +impl Array { + pub fn new( + content: &[T], + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let arr_buffer = class::ArrayBuffer::new(content, heap.api_version())?; + let arr_buffer_ptr = AscPtr::alloc_obj(arr_buffer, heap, gas)?; + Ok(Array { + buffer: AscPtr::new(arr_buffer_ptr.wasm_ptr()), + // If this cast would overflow, the above line has already panicked. + length: content.len() as u32, + ty: PhantomData, + }) + } + + pub(crate) fn to_vec( + &self, + heap: &H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + self.buffer + .read_ptr(heap, gas)? + .get(0, self.length, heap.api_version()) + } +} diff --git a/runtime/wasm/src/asc_abi/v0_0_5.rs b/runtime/wasm/src/asc_abi/v0_0_5.rs new file mode 100644 index 0000000..31503af --- /dev/null +++ b/runtime/wasm/src/asc_abi/v0_0_5.rs @@ -0,0 +1,313 @@ +use std::marker::PhantomData; +use std::mem::{size_of, size_of_val}; + +use anyhow::anyhow; +use semver::Version; + +use graph::runtime::gas::GasCounter; +use graph::runtime::{AscHeap, AscPtr, AscType, AscValue, DeterministicHostError, HEADER_SIZE}; +use graph_runtime_derive::AscType; + +use crate::asc_abi::class; + +/// Module related to AssemblyScript version >=v0.19.2. +/// All `to_asc_bytes`/`from_asc_bytes` only consider the #data/content/payload +/// not the #header, that's handled on `AscPtr`. +/// Header in question: https://www.assemblyscript.org/memory.html#common-header-layout + +/// Similar as JS ArrayBuffer, "a generic, fixed-length raw binary data buffer". +/// See https://www.assemblyscript.org/memory.html#arraybuffer-layout +pub struct ArrayBuffer { + // Not included in memory layout + pub byte_length: u32, + // #data + pub content: Box<[u8]>, +} + +impl ArrayBuffer { + pub fn new(values: &[T]) -> Result { + let mut content = Vec::new(); + for value in values { + let asc_bytes = value.to_asc_bytes()?; + content.extend(&asc_bytes); + } + + if content.len() > u32::max_value() as usize { + return Err(DeterministicHostError::from(anyhow::anyhow!( + "slice cannot fit in WASM memory" + ))); + } + Ok(ArrayBuffer { + byte_length: content.len() as u32, + content: content.into(), + }) + } + + /// Read `length` elements of type `T` starting at `byte_offset`. + /// + /// Panics if that tries to read beyond the length of `self.content`. + pub fn get( + &self, + byte_offset: u32, + length: u32, + api_version: Version, + ) -> Result, DeterministicHostError> { + let length = length as usize; + let byte_offset = byte_offset as usize; + + self.content[byte_offset..] + .chunks(size_of::()) + .take(length) + .map(|asc_obj| T::from_asc_bytes(asc_obj, &api_version)) + .collect() + } +} + +impl AscType for ArrayBuffer { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + let mut asc_layout: Vec = Vec::new(); + + asc_layout.extend(self.content.iter()); + + // Allocate extra capacity to next power of two, as required by asc. + let total_size = self.byte_length as usize + HEADER_SIZE; + let total_capacity = total_size.next_power_of_two(); + let extra_capacity = total_capacity - total_size; + asc_layout.extend(std::iter::repeat(0).take(extra_capacity)); + + Ok(asc_layout) + } + + fn from_asc_bytes( + asc_obj: &[u8], + _api_version: &Version, + ) -> Result { + Ok(ArrayBuffer { + byte_length: asc_obj.len() as u32, + content: asc_obj.to_vec().into(), + }) + } + + fn content_len(&self, _asc_bytes: &[u8]) -> usize { + self.byte_length as usize // without extra_capacity + } +} + +/// A typed, indexable view of an `ArrayBuffer` of Asc primitives. In Asc it's +/// an abstract class with subclasses for each primitive, for example +/// `Uint8Array` is `TypedArray`. +/// Also known as `ArrayBufferView`. +/// See https://www.assemblyscript.org/memory.html#arraybufferview-layout +#[repr(C)] +#[derive(AscType)] +pub struct TypedArray { + // #data -> Backing buffer reference + pub buffer: AscPtr, + // #dataStart -> Start within the #data + data_start: u32, + // #dataLength -> Length of the data from #dataStart + byte_length: u32, + // Not included in memory layout, it's just for typings + ty: PhantomData, +} + +impl TypedArray { + pub(crate) fn new( + content: &[T], + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let buffer = class::ArrayBuffer::new(content, heap.api_version())?; + let byte_length = content.len() as u32; + let ptr = AscPtr::alloc_obj(buffer, heap, gas)?; + Ok(TypedArray { + buffer: AscPtr::new(ptr.wasm_ptr()), // new AscPtr necessary to convert type parameter + data_start: ptr.wasm_ptr(), + byte_length, + ty: PhantomData, + }) + } + + pub(crate) fn to_vec( + &self, + heap: &H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + // We're trying to read the pointer below, we should check it's + // not null before using it. + self.buffer.check_is_not_null()?; + + // This subtraction is needed because on the ArrayBufferView memory layout + // there are two pointers to the data. + // - The first (self.buffer) points to the related ArrayBuffer. + // - The second (self.data_start) points to where in this ArrayBuffer the data starts. + // So this is basically getting the offset. + // Related docs: https://www.assemblyscript.org/memory.html#arraybufferview-layout + let data_start_with_offset = self + .data_start + .checked_sub(self.buffer.wasm_ptr()) + .ok_or_else(|| { + DeterministicHostError::from(anyhow::anyhow!( + "Subtract overflow on pointer: {}", + self.data_start + )) + })?; + + self.buffer.read_ptr(heap, gas)?.get( + data_start_with_offset, + self.byte_length / size_of::() as u32, + heap.api_version(), + ) + } +} + +/// Asc std string: "Strings are encoded as UTF-16LE in AssemblyScript" +/// See https://www.assemblyscript.org/memory.html#string-layout +pub struct AscString { + // Not included in memory layout + // In number of UTF-16 code units (2 bytes each). + byte_length: u32, + // #data + // The sequence of UTF-16LE code units that form the string. + pub content: Box<[u16]>, +} + +impl AscString { + pub fn new(content: &[u16]) -> Result { + if size_of_val(content) > u32::max_value() as usize { + return Err(DeterministicHostError::from(anyhow!( + "string cannot fit in WASM memory" + ))); + } + + Ok(AscString { + byte_length: content.len() as u32, + content: content.into(), + }) + } +} + +impl AscType for AscString { + fn to_asc_bytes(&self) -> Result, DeterministicHostError> { + let mut content: Vec = Vec::new(); + + // Write the code points, in little-endian (LE) order. + for &code_unit in self.content.iter() { + let low_byte = code_unit as u8; + let high_byte = (code_unit >> 8) as u8; + content.push(low_byte); + content.push(high_byte); + } + + let header_size = 20; + let total_size = (self.byte_length as usize * 2) + header_size; + let total_capacity = total_size.next_power_of_two(); + let extra_capacity = total_capacity - total_size; + content.extend(std::iter::repeat(0).take(extra_capacity)); + + Ok(content) + } + + /// The Rust representation of an Asc object as layed out in Asc memory. + fn from_asc_bytes( + asc_obj: &[u8], + _api_version: &Version, + ) -> Result { + // UTF-16 (used in assemblyscript) always uses one + // pair of bytes per code unit. + // https://mathiasbynens.be/notes/javascript-encoding + // UTF-16 (16-bit Unicode Transformation Format) is an + // extension of UCS-2 that allows representing code points + // outside the BMP. It produces a variable-length result + // of either one or two 16-bit code units per code point. + // This way, it can encode code points in the range from 0 + // to 0x10FFFF. + + let mut content = Vec::new(); + for pair in asc_obj.chunks(2) { + let code_point_bytes = [ + pair[0], + *pair.get(1).ok_or_else(|| { + DeterministicHostError::from(anyhow!( + "Attempted to read past end of string content bytes chunk" + )) + })?, + ]; + let code_point = u16::from_le_bytes(code_point_bytes); + content.push(code_point); + } + AscString::new(&content) + } + + fn content_len(&self, _asc_bytes: &[u8]) -> usize { + self.byte_length as usize * 2 // without extra_capacity, and times 2 because the content is measured in u8s + } +} + +/// Growable array backed by an `ArrayBuffer`. +/// See https://www.assemblyscript.org/memory.html#array-layout +#[repr(C)] +#[derive(AscType)] +pub struct Array { + // #data -> Backing buffer reference + buffer: AscPtr, + // #dataStart -> Start of the data within #data + buffer_data_start: u32, + // #dataLength -> Length of the data from #dataStart + buffer_data_length: u32, + // #length -> Mutable length of the data the user is interested in + length: i32, + // Not included in memory layout, it's just for typings + ty: PhantomData, +} + +impl Array { + pub fn new( + content: &[T], + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let arr_buffer = class::ArrayBuffer::new(content, heap.api_version())?; + let buffer = AscPtr::alloc_obj(arr_buffer, heap, gas)?; + let buffer_data_length = buffer.read_len(heap, gas)?; + Ok(Array { + buffer: AscPtr::new(buffer.wasm_ptr()), + buffer_data_start: buffer.wasm_ptr(), + buffer_data_length, + length: content.len() as i32, + ty: PhantomData, + }) + } + + pub(crate) fn to_vec( + &self, + heap: &H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + // We're trying to read the pointer below, we should check it's + // not null before using it. + self.buffer.check_is_not_null()?; + + // This subtraction is needed because on the ArrayBufferView memory layout + // there are two pointers to the data. + // - The first (self.buffer) points to the related ArrayBuffer. + // - The second (self.buffer_data_start) points to where in this ArrayBuffer the data starts. + // So this is basically getting the offset. + // Related docs: https://www.assemblyscript.org/memory.html#arraybufferview-layout + let buffer_data_start_with_offset = self + .buffer_data_start + .checked_sub(self.buffer.wasm_ptr()) + .ok_or_else(|| { + DeterministicHostError::from(anyhow::anyhow!( + "Subtract overflow on pointer: {}", + self.buffer_data_start + )) + })?; + + self.buffer.read_ptr(heap, gas)?.get( + buffer_data_start_with_offset, + self.length as u32, + heap.api_version(), + ) + } +} diff --git a/runtime/wasm/src/error.rs b/runtime/wasm/src/error.rs new file mode 100644 index 0000000..e3852b3 --- /dev/null +++ b/runtime/wasm/src/error.rs @@ -0,0 +1,33 @@ +use wasmtime::Trap; + +use graph::runtime::DeterministicHostError; + +use crate::module::IntoTrap; + +pub enum DeterminismLevel { + /// This error is known to be deterministic. For example, divide by zero. + /// TODO: For these errors, a further designation should be created about the contents + /// of the actual message. + Deterministic, + + /// This error is known to be non-deterministic. For example, an intermittent http failure. + #[allow(dead_code)] + NonDeterministic, + + /// The runtime is processing a given block, but there is an indication that the blockchain client + /// might not consider that block to be on the main chain. So the block must be reprocessed. + PossibleReorg, + + /// An error has not yet been designated as deterministic or not. This should be phased out over time, + /// and is the default for errors like anyhow which are of an unknown origin. + Unimplemented, +} + +impl IntoTrap for DeterministicHostError { + fn determinism_level(&self) -> DeterminismLevel { + DeterminismLevel::Deterministic + } + fn into_trap(self) -> Trap { + Trap::from(self.inner()) + } +} diff --git a/runtime/wasm/src/gas_rules.rs b/runtime/wasm/src/gas_rules.rs new file mode 100644 index 0000000..10717f9 --- /dev/null +++ b/runtime/wasm/src/gas_rules.rs @@ -0,0 +1,168 @@ +use std::{convert::TryInto, num::NonZeroU32}; + +use graph::runtime::gas::CONST_MAX_GAS_PER_HANDLER; +use parity_wasm::elements::Instruction; +use wasm_instrument::gas_metering::{MemoryGrowCost, Rules}; + +pub const GAS_COST_STORE: u32 = 2263; +pub const GAS_COST_LOAD: u32 = 1573; + +pub struct GasRules; + +impl Rules for GasRules { + fn instruction_cost(&self, instruction: &Instruction) -> Option { + use Instruction::*; + let weight = match instruction { + // These are taken from this post: https://github.com/paritytech/substrate/pull/7361#issue-506217103 + // from the table under the "Schedule" dropdown. Each decimal is multiplied by 10. + // Note that those were calculated for wasi, not wasmtime, so they are likely very conservative. + I64Const(_) => 16, + I64Load(_, _) => GAS_COST_LOAD, + I64Store(_, _) => GAS_COST_STORE, + Select => 61, + Instruction::If(_) => 79, + Br(_) => 30, + BrIf(_) => 63, + BrTable(data) => 146 + data.table.len() as u32, + Call(_) => 951, + // TODO: To figure out the param cost we need to look up the function + CallIndirect(_, _) => 1995, + GetLocal(_) => 18, + SetLocal(_) => 21, + TeeLocal(_) => 21, + GetGlobal(_) => 66, + SetGlobal(_) => 107, + CurrentMemory(_) => 23, + GrowMemory(_) => 435000, + I64Clz => 23, + I64Ctz => 23, + I64Popcnt => 29, + I64Eqz => 24, + I64ExtendSI32 => 22, + I64ExtendUI32 => 22, + I32WrapI64 => 23, + I64Eq => 26, + I64Ne => 25, + I64LtS => 25, + I64LtU => 26, + I64GtS => 25, + I64GtU => 25, + I64LeS => 25, + I64LeU => 26, + I64GeS => 26, + I64GeU => 25, + I64Add => 25, + I64Sub => 26, + I64Mul => 25, + I64DivS => 82, + I64DivU => 72, + I64RemS => 81, + I64RemU => 73, + I64And => 25, + I64Or => 25, + I64Xor => 26, + I64Shl => 25, + I64ShrS => 26, + I64ShrU => 26, + I64Rotl => 25, + I64Rotr => 26, + + // These are similar enough to something above so just referencing a similar + // instruction + I32Load(_, _) + | F32Load(_, _) + | F64Load(_, _) + | I32Load8S(_, _) + | I32Load8U(_, _) + | I32Load16S(_, _) + | I32Load16U(_, _) + | I64Load8S(_, _) + | I64Load8U(_, _) + | I64Load16S(_, _) + | I64Load16U(_, _) + | I64Load32S(_, _) + | I64Load32U(_, _) => GAS_COST_LOAD, + + I32Store(_, _) + | F32Store(_, _) + | F64Store(_, _) + | I32Store8(_, _) + | I32Store16(_, _) + | I64Store8(_, _) + | I64Store16(_, _) + | I64Store32(_, _) => GAS_COST_STORE, + + I32Const(_) | F32Const(_) | F64Const(_) => 16, + I32Eqz => 26, + I32Eq => 26, + I32Ne => 25, + I32LtS => 25, + I32LtU => 26, + I32GtS => 25, + I32GtU => 25, + I32LeS => 25, + I32LeU => 26, + I32GeS => 26, + I32GeU => 25, + I32Add => 25, + I32Sub => 26, + I32Mul => 25, + I32DivS => 82, + I32DivU => 72, + I32RemS => 81, + I32RemU => 73, + I32And => 25, + I32Or => 25, + I32Xor => 26, + I32Shl => 25, + I32ShrS => 26, + I32ShrU => 26, + I32Rotl => 25, + I32Rotr => 26, + I32Clz => 23, + I32Popcnt => 29, + I32Ctz => 23, + + // Float weights not calculated by reference source material. Making up + // some conservative values. The point here is not to be perfect but just + // to have some reasonable upper bound. + F64ReinterpretI64 | F32ReinterpretI32 | F64PromoteF32 | F64ConvertUI64 + | F64ConvertSI64 | F64ConvertUI32 | F64ConvertSI32 | F32DemoteF64 | F32ConvertUI64 + | F32ConvertSI64 | F32ConvertUI32 | F32ConvertSI32 | I64TruncUF64 | I64TruncSF64 + | I64TruncUF32 | I64TruncSF32 | I32TruncUF64 | I32TruncSF64 | I32TruncUF32 + | I32TruncSF32 | F64Copysign | F64Max | F64Min | F64Mul | F64Sub | F64Add + | F64Trunc | F64Floor | F64Ceil | F64Neg | F64Abs | F64Nearest | F32Copysign + | F32Max | F32Min | F32Mul | F32Sub | F32Add | F32Nearest | F32Trunc | F32Floor + | F32Ceil | F32Neg | F32Abs | F32Eq | F32Ne | F32Lt | F32Gt | F32Le | F32Ge | F64Eq + | F64Ne | F64Lt | F64Gt | F64Le | F64Ge | I32ReinterpretF32 | I64ReinterpretF64 => 100, + F64Div | F64Sqrt | F32Div | F32Sqrt => 100, + + // More invented weights + Block(_) => 100, + Loop(_) => 100, + Else => 100, + End => 100, + Return => 100, + Drop => 100, + SignExt(_) => 100, + Nop => 1, + Unreachable => 1, + }; + Some(weight) + } + + fn memory_grow_cost(&self) -> MemoryGrowCost { + // Each page is 64KiB which is 65536 bytes. + const PAGE: u64 = 64 * 1024; + // 1 GB + const GIB: u64 = 1073741824; + // 12GiB to pages for the max memory allocation + // In practice this will never be hit unless we also + // free pages because this is 32bit WASM. + const MAX_PAGES: u64 = 12 * GIB / PAGE; + let gas_per_page = + NonZeroU32::new((CONST_MAX_GAS_PER_HANDLER / MAX_PAGES).try_into().unwrap()).unwrap(); + + MemoryGrowCost::Linear(gas_per_page) + } +} diff --git a/runtime/wasm/src/host.rs b/runtime/wasm/src/host.rs new file mode 100644 index 0000000..e6423a5 --- /dev/null +++ b/runtime/wasm/src/host.rs @@ -0,0 +1,263 @@ +use std::cmp::PartialEq; +use std::time::Instant; + +use async_trait::async_trait; +use futures::sync::mpsc::Sender; +use futures03::channel::oneshot::channel; + +use graph::blockchain::{Blockchain, HostFn, RuntimeAdapter}; +use graph::components::store::{EnsLookup, SubgraphFork}; +use graph::components::subgraph::{MappingError, SharedProofOfIndexing}; +use graph::data_source::{ + DataSource, DataSourceTemplate, MappingTrigger, TriggerData, TriggerWithHandler, +}; +use graph::prelude::{ + RuntimeHost as RuntimeHostTrait, RuntimeHostBuilder as RuntimeHostBuilderTrait, *, +}; + +use crate::mapping::{MappingContext, MappingRequest}; +use crate::module::ToAscPtr; +use crate::{host_exports::HostExports, module::ExperimentalFeatures}; +use graph::runtime::gas::Gas; + +pub struct RuntimeHostBuilder { + runtime_adapter: Arc>, + link_resolver: Arc, + ens_lookup: Arc, +} + +impl Clone for RuntimeHostBuilder { + fn clone(&self) -> Self { + RuntimeHostBuilder { + runtime_adapter: self.runtime_adapter.cheap_clone(), + link_resolver: self.link_resolver.cheap_clone(), + ens_lookup: self.ens_lookup.cheap_clone(), + } + } +} + +impl RuntimeHostBuilder { + pub fn new( + runtime_adapter: Arc>, + link_resolver: Arc, + ens_lookup: Arc, + ) -> Self { + RuntimeHostBuilder { + runtime_adapter, + link_resolver, + ens_lookup, + } + } +} + +impl RuntimeHostBuilderTrait for RuntimeHostBuilder +where + ::MappingTrigger: ToAscPtr, +{ + type Host = RuntimeHost; + type Req = MappingRequest; + + fn spawn_mapping( + raw_module: &[u8], + logger: Logger, + subgraph_id: DeploymentHash, + metrics: Arc, + ) -> Result, Error> { + let experimental_features = ExperimentalFeatures { + allow_non_deterministic_ipfs: ENV_VARS.mappings.allow_non_deterministic_ipfs, + }; + crate::mapping::spawn_module( + raw_module, + logger, + subgraph_id, + metrics, + tokio::runtime::Handle::current(), + ENV_VARS.mappings.timeout, + experimental_features, + ) + } + + fn build( + &self, + network_name: String, + subgraph_id: DeploymentHash, + data_source: DataSource, + templates: Arc>>, + mapping_request_sender: Sender>, + metrics: Arc, + ) -> Result { + RuntimeHost::new( + self.runtime_adapter.cheap_clone(), + self.link_resolver.clone(), + network_name, + subgraph_id, + data_source, + templates, + mapping_request_sender, + metrics, + self.ens_lookup.cheap_clone(), + ) + } +} + +pub struct RuntimeHost { + host_fns: Arc>, + data_source: DataSource, + mapping_request_sender: Sender>, + host_exports: Arc>, + metrics: Arc, +} + +impl RuntimeHost +where + C: Blockchain, +{ + fn new( + runtime_adapter: Arc>, + link_resolver: Arc, + network_name: String, + subgraph_id: DeploymentHash, + data_source: DataSource, + templates: Arc>>, + mapping_request_sender: Sender>, + metrics: Arc, + ens_lookup: Arc, + ) -> Result { + // Create new instance of externally hosted functions invoker. The `Arc` is simply to avoid + // implementing `Clone` for `HostExports`. + let host_exports = Arc::new(HostExports::new( + subgraph_id, + &data_source, + network_name, + templates, + link_resolver, + ens_lookup, + )); + + let host_fns = data_source + .as_onchain() + .map(|ds| runtime_adapter.host_fns(ds)) + .transpose()? + .unwrap_or_default(); + + Ok(RuntimeHost { + host_fns: Arc::new(host_fns), + data_source, + mapping_request_sender, + host_exports, + metrics, + }) + } + + /// Sends a MappingRequest to the thread which owns the host, + /// and awaits the result. + async fn send_mapping_request( + &self, + logger: &Logger, + state: BlockState, + trigger: TriggerWithHandler>, + block_ptr: BlockPtr, + proof_of_indexing: SharedProofOfIndexing, + debug_fork: &Option>, + ) -> Result, MappingError> { + let handler = trigger.handler_name().to_string(); + + let extras = trigger.logging_extras(); + trace!( + logger, "Start processing trigger"; + &extras, + "handler" => &handler, + "data_source" => &self.data_source.name(), + ); + + let (result_sender, result_receiver) = channel(); + let start_time = Instant::now(); + let metrics = self.metrics.clone(); + + self.mapping_request_sender + .clone() + .send(MappingRequest { + ctx: MappingContext { + logger: logger.cheap_clone(), + state, + host_exports: self.host_exports.cheap_clone(), + block_ptr, + proof_of_indexing, + host_fns: self.host_fns.cheap_clone(), + debug_fork: debug_fork.cheap_clone(), + }, + trigger, + result_sender, + }) + .compat() + .await + .context("Mapping terminated before passing in trigger")?; + + let result = result_receiver + .await + .context("Mapping terminated before handling trigger")?; + + let elapsed = start_time.elapsed(); + metrics.observe_handler_execution_time(elapsed.as_secs_f64(), &handler); + + // If there is an error, "gas_used" is incorrectly reported as 0. + let gas_used = result.as_ref().map(|(_, gas)| gas).unwrap_or(&Gas::ZERO); + info!( + logger, "Done processing trigger"; + &extras, + "total_ms" => elapsed.as_millis(), + "handler" => handler, + "data_source" => &self.data_source.name(), + "gas_used" => gas_used.to_string(), + ); + + // Discard the gas value + result.map(|(block_state, _)| block_state) + } +} + +#[async_trait] +impl RuntimeHostTrait for RuntimeHost { + fn data_source(&self) -> &DataSource { + &self.data_source + } + + fn match_and_decode( + &self, + trigger: &TriggerData, + block: &Arc, + logger: &Logger, + ) -> Result>>, Error> { + self.data_source.match_and_decode(trigger, block, logger) + } + + async fn process_mapping_trigger( + &self, + logger: &Logger, + block_ptr: BlockPtr, + trigger: TriggerWithHandler>, + state: BlockState, + proof_of_indexing: SharedProofOfIndexing, + debug_fork: &Option>, + ) -> Result, MappingError> { + self.send_mapping_request( + logger, + state, + trigger, + block_ptr, + proof_of_indexing, + debug_fork, + ) + .await + } + + fn creation_block_number(&self) -> Option { + self.data_source.creation_block() + } +} + +impl PartialEq for RuntimeHost { + fn eq(&self, other: &Self) -> bool { + self.data_source.is_duplicate_of(&other.data_source) + } +} diff --git a/runtime/wasm/src/host_exports.rs b/runtime/wasm/src/host_exports.rs new file mode 100644 index 0000000..28ef3b8 --- /dev/null +++ b/runtime/wasm/src/host_exports.rs @@ -0,0 +1,826 @@ +use std::collections::HashMap; +use std::ops::Deref; +use std::str::FromStr; +use std::time::{Duration, Instant}; + +use never::Never; +use semver::Version; +use wasmtime::Trap; +use web3::types::H160; + +use graph::blockchain::Blockchain; +use graph::components::store::EnsLookup; +use graph::components::store::{EntityKey, EntityType}; +use graph::components::subgraph::{CausalityRegion, ProofOfIndexingEvent, SharedProofOfIndexing}; +use graph::data::store; +use graph::data_source::{DataSource, DataSourceTemplate}; +use graph::ensure; +use graph::prelude::ethabi::param_type::Reader; +use graph::prelude::ethabi::{decode, encode, Token}; +use graph::prelude::serde_json; +use graph::prelude::{slog::b, slog::record_static, *}; +use graph::runtime::gas::{self, complexity, Gas, GasCounter}; +pub use graph::runtime::{DeterministicHostError, HostExportError}; + +use crate::module::{WasmInstance, WasmInstanceContext}; +use crate::{error::DeterminismLevel, module::IntoTrap}; + +fn write_poi_event( + proof_of_indexing: &SharedProofOfIndexing, + poi_event: &ProofOfIndexingEvent, + causality_region: &str, + logger: &Logger, +) { + if let Some(proof_of_indexing) = proof_of_indexing { + let mut proof_of_indexing = proof_of_indexing.deref().borrow_mut(); + proof_of_indexing.write(logger, causality_region, poi_event); + } +} + +impl IntoTrap for HostExportError { + fn determinism_level(&self) -> DeterminismLevel { + match self { + HostExportError::Deterministic(_) => DeterminismLevel::Deterministic, + HostExportError::Unknown(_) => DeterminismLevel::Unimplemented, + HostExportError::PossibleReorg(_) => DeterminismLevel::PossibleReorg, + } + } + fn into_trap(self) -> Trap { + match self { + HostExportError::Unknown(e) + | HostExportError::PossibleReorg(e) + | HostExportError::Deterministic(e) => Trap::from(e), + } + } +} + +pub struct HostExports { + pub(crate) subgraph_id: DeploymentHash, + pub api_version: Version, + data_source_name: String, + data_source_address: Vec, + data_source_network: String, + data_source_context: Arc>, + /// Some data sources have indeterminism or different notions of time. These + /// need to be each be stored separately to separate causality between them, + /// and merge the results later. Right now, this is just the ethereum + /// networks but will be expanded for ipfs and the availability chain. + causality_region: String, + templates: Arc>>, + pub(crate) link_resolver: Arc, + ens_lookup: Arc, +} + +impl HostExports { + pub fn new( + subgraph_id: DeploymentHash, + data_source: &DataSource, + data_source_network: String, + templates: Arc>>, + link_resolver: Arc, + ens_lookup: Arc, + ) -> Self { + Self { + subgraph_id, + api_version: data_source.api_version(), + data_source_name: data_source.name().to_owned(), + data_source_address: data_source.address().unwrap_or_default(), + data_source_context: data_source.context().cheap_clone(), + causality_region: CausalityRegion::from_network(&data_source_network), + data_source_network, + templates, + link_resolver, + ens_lookup, + } + } + + pub(crate) fn abort( + &self, + message: Option, + file_name: Option, + line_number: Option, + column_number: Option, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(Gas::new(gas::DEFAULT_BASE_COST))?; + + let message = message + .map(|message| format!("message: {}", message)) + .unwrap_or_else(|| "no message".into()); + let location = match (file_name, line_number, column_number) { + (None, None, None) => "an unknown location".into(), + (Some(file_name), None, None) => file_name, + (Some(file_name), Some(line_number), None) => { + format!("{}, line {}", file_name, line_number) + } + (Some(file_name), Some(line_number), Some(column_number)) => format!( + "{}, line {}, column {}", + file_name, line_number, column_number + ), + _ => unreachable!(), + }; + Err(DeterministicHostError::from(anyhow::anyhow!( + "Mapping aborted at {}, with {}", + location, + message + ))) + } + + pub(crate) fn store_set( + &self, + logger: &Logger, + state: &mut BlockState, + proof_of_indexing: &SharedProofOfIndexing, + entity_type: String, + entity_id: String, + data: HashMap, + stopwatch: &StopwatchMetrics, + gas: &GasCounter, + ) -> Result<(), anyhow::Error> { + let poi_section = stopwatch.start_section("host_export_store_set__proof_of_indexing"); + write_poi_event( + proof_of_indexing, + &ProofOfIndexingEvent::SetEntity { + entity_type: &entity_type, + id: &entity_id, + data: &data, + }, + &self.causality_region, + logger, + ); + poi_section.end(); + + let key = EntityKey { + entity_type: EntityType::new(entity_type), + entity_id: entity_id.into(), + }; + + gas.consume_host_fn(gas::STORE_SET.with_args(complexity::Linear, (&key, &data)))?; + + let entity = Entity::from(data); + state.entity_cache.set(key.clone(), entity)?; + + Ok(()) + } + + pub(crate) fn store_remove( + &self, + logger: &Logger, + state: &mut BlockState, + proof_of_indexing: &SharedProofOfIndexing, + entity_type: String, + entity_id: String, + gas: &GasCounter, + ) -> Result<(), HostExportError> { + write_poi_event( + proof_of_indexing, + &ProofOfIndexingEvent::RemoveEntity { + entity_type: &entity_type, + id: &entity_id, + }, + &self.causality_region, + logger, + ); + let key = EntityKey { + entity_type: EntityType::new(entity_type), + entity_id: entity_id.into(), + }; + + gas.consume_host_fn(gas::STORE_REMOVE.with_args(complexity::Size, &key))?; + + state.entity_cache.remove(key); + + Ok(()) + } + + pub(crate) fn store_get( + &self, + state: &mut BlockState, + entity_type: String, + entity_id: String, + gas: &GasCounter, + ) -> Result, anyhow::Error> { + let store_key = EntityKey { + entity_type: EntityType::new(entity_type), + entity_id: entity_id.into(), + }; + + let result = state.entity_cache.get(&store_key)?; + gas.consume_host_fn(gas::STORE_GET.with_args(complexity::Linear, (&store_key, &result)))?; + + Ok(result) + } + + /// Prints the module of `n` in hex. + /// Integers are encoded using the least amount of digits (no leading zero digits). + /// Their encoding may be of uneven length. The number zero encodes as "0x0". + /// + /// https://godoc.org/github.com/ethereum/go-ethereum/common/hexutil#hdr-Encoding_Rules + pub(crate) fn big_int_to_hex( + &self, + n: BigInt, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(complexity::Size, &n))?; + + if n == 0.into() { + return Ok("0x0".to_string()); + } + + let bytes = n.to_bytes_be().1; + Ok(format!( + "0x{}", + ::hex::encode(bytes).trim_start_matches('0') + )) + } + + pub(crate) fn ipfs_cat(&self, logger: &Logger, link: String) -> Result, anyhow::Error> { + // Does not consume gas because this is not a part of the deterministic feature set. + // Ideally this would first consume gas for fetching the file stats, and then again + // for the bytes of the file. + graph::block_on(self.link_resolver.cat(logger, &Link { link })) + } + + pub(crate) fn ipfs_get_block( + &self, + logger: &Logger, + link: String, + ) -> Result, anyhow::Error> { + // Does not consume gas because this is not a part of the deterministic feature set. + // Ideally this would first consume gas for fetching the file stats, and then again + // for the bytes of the file. + graph::block_on(self.link_resolver.get_block(logger, &Link { link })) + } + + // Read the IPFS file `link`, split it into JSON objects, and invoke the + // exported function `callback` on each JSON object. The successful return + // value contains the block state produced by each callback invocation. Each + // invocation of `callback` happens in its own instance of a WASM module, + // which is identical to `module` when it was first started. The signature + // of the callback must be `callback(JSONValue, Value)`, and the `userData` + // parameter is passed to the callback without any changes + pub(crate) fn ipfs_map( + link_resolver: &Arc, + module: &mut WasmInstanceContext, + link: String, + callback: &str, + user_data: store::Value, + flags: Vec, + ) -> Result>, anyhow::Error> { + // Does not consume gas because this is not a part of deterministic APIs. + // Ideally we would consume gas the same as ipfs_cat and then share + // gas across the spawned modules for callbacks. + + const JSON_FLAG: &str = "json"; + ensure!( + flags.contains(&JSON_FLAG.to_string()), + "Flags must contain 'json'" + ); + + let host_metrics = module.host_metrics.clone(); + let valid_module = module.valid_module.clone(); + let ctx = module.ctx.derive_with_empty_block_state(); + let callback = callback.to_owned(); + // Create a base error message to avoid borrowing headaches + let errmsg = format!( + "ipfs_map: callback '{}' failed when processing file '{}'", + &*callback, &link + ); + + let start = Instant::now(); + let mut last_log = start; + let logger = ctx.logger.new(o!("ipfs_map" => link.clone())); + + let result = { + let mut stream: JsonValueStream = + graph::block_on(link_resolver.json_stream(&logger, &Link { link }))?; + let mut v = Vec::new(); + while let Some(sv) = graph::block_on(stream.next()) { + let sv = sv?; + let module = WasmInstance::from_valid_module_with_ctx( + valid_module.clone(), + ctx.derive_with_empty_block_state(), + host_metrics.clone(), + module.timeout, + module.experimental_features, + )?; + let result = module.handle_json_callback(&callback, &sv.value, &user_data)?; + // Log progress every 15s + if last_log.elapsed() > Duration::from_secs(15) { + debug!( + logger, + "Processed {} lines in {}s so far", + sv.line, + start.elapsed().as_secs() + ); + last_log = Instant::now(); + } + v.push(result) + } + Ok(v) + }; + result.map_err(move |e: Error| anyhow::anyhow!("{}: {}", errmsg, e.to_string())) + } + + /// Expects a decimal string. + pub(crate) fn json_to_i64( + &self, + json: String, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(complexity::Size, &json))?; + i64::from_str(&json) + .with_context(|| format!("JSON `{}` cannot be parsed as i64", json)) + .map_err(DeterministicHostError::from) + } + + /// Expects a decimal string. + pub(crate) fn json_to_u64( + &self, + json: String, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(complexity::Size, &json))?; + + u64::from_str(&json) + .with_context(|| format!("JSON `{}` cannot be parsed as u64", json)) + .map_err(DeterministicHostError::from) + } + + /// Expects a decimal string. + pub(crate) fn json_to_f64( + &self, + json: String, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(complexity::Size, &json))?; + + f64::from_str(&json) + .with_context(|| format!("JSON `{}` cannot be parsed as f64", json)) + .map_err(DeterministicHostError::from) + } + + /// Expects a decimal string. + pub(crate) fn json_to_big_int( + &self, + json: String, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(complexity::Size, &json))?; + + let big_int = BigInt::from_str(&json) + .with_context(|| format!("JSON `{}` is not a decimal string", json)) + .map_err(DeterministicHostError::from)?; + Ok(big_int.to_signed_bytes_le()) + } + + pub(crate) fn crypto_keccak_256( + &self, + input: Vec, + gas: &GasCounter, + ) -> Result<[u8; 32], DeterministicHostError> { + let data = &input[..]; + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(complexity::Size, data))?; + Ok(tiny_keccak::keccak256(data)) + } + + pub(crate) fn big_int_plus( + &self, + x: BigInt, + y: BigInt, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::BIG_MATH_GAS_OP.with_args(complexity::Max, (&x, &y)))?; + Ok(x + y) + } + + pub(crate) fn big_int_minus( + &self, + x: BigInt, + y: BigInt, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::BIG_MATH_GAS_OP.with_args(complexity::Max, (&x, &y)))?; + Ok(x - y) + } + + pub(crate) fn big_int_times( + &self, + x: BigInt, + y: BigInt, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::BIG_MATH_GAS_OP.with_args(complexity::Mul, (&x, &y)))?; + Ok(x * y) + } + + pub(crate) fn big_int_divided_by( + &self, + x: BigInt, + y: BigInt, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::BIG_MATH_GAS_OP.with_args(complexity::Mul, (&x, &y)))?; + if y == 0.into() { + return Err(DeterministicHostError::from(anyhow!( + "attempted to divide BigInt `{}` by zero", + x + ))); + } + Ok(x / y) + } + + pub(crate) fn big_int_mod( + &self, + x: BigInt, + y: BigInt, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::BIG_MATH_GAS_OP.with_args(complexity::Mul, (&x, &y)))?; + if y == 0.into() { + return Err(DeterministicHostError::from(anyhow!( + "attempted to calculate the remainder of `{}` with a divisor of zero", + x + ))); + } + Ok(x % y) + } + + /// Limited to a small exponent to avoid creating huge BigInts. + pub(crate) fn big_int_pow( + &self, + x: BigInt, + exp: u8, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn( + gas::BIG_MATH_GAS_OP + .with_args(complexity::Exponential, (&x, (exp as f32).log2() as u8)), + )?; + Ok(x.pow(exp)) + } + + pub(crate) fn big_int_from_string( + &self, + s: String, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(complexity::Size, &s))?; + BigInt::from_str(&s) + .with_context(|| format!("string is not a BigInt: `{}`", s)) + .map_err(DeterministicHostError::from) + } + + pub(crate) fn big_int_bit_or( + &self, + x: BigInt, + y: BigInt, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::BIG_MATH_GAS_OP.with_args(complexity::Max, (&x, &y)))?; + Ok(x | y) + } + + pub(crate) fn big_int_bit_and( + &self, + x: BigInt, + y: BigInt, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::BIG_MATH_GAS_OP.with_args(complexity::Min, (&x, &y)))?; + Ok(x & y) + } + + pub(crate) fn big_int_left_shift( + &self, + x: BigInt, + bits: u8, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::BIG_MATH_GAS_OP.with_args(complexity::Linear, (&x, &bits)))?; + Ok(x << bits) + } + + pub(crate) fn big_int_right_shift( + &self, + x: BigInt, + bits: u8, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::BIG_MATH_GAS_OP.with_args(complexity::Linear, (&x, &bits)))?; + Ok(x >> bits) + } + + /// Useful for IPFS hashes stored as bytes + pub(crate) fn bytes_to_base58( + &self, + bytes: Vec, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(complexity::Size, &bytes))?; + Ok(::bs58::encode(&bytes).into_string()) + } + + pub(crate) fn big_decimal_plus( + &self, + x: BigDecimal, + y: BigDecimal, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::BIG_MATH_GAS_OP.with_args(complexity::Linear, (&x, &y)))?; + Ok(x + y) + } + + pub(crate) fn big_decimal_minus( + &self, + x: BigDecimal, + y: BigDecimal, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::BIG_MATH_GAS_OP.with_args(complexity::Linear, (&x, &y)))?; + Ok(x - y) + } + + pub(crate) fn big_decimal_times( + &self, + x: BigDecimal, + y: BigDecimal, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::BIG_MATH_GAS_OP.with_args(complexity::Mul, (&x, &y)))?; + Ok(x * y) + } + + /// Maximum precision of 100 decimal digits. + pub(crate) fn big_decimal_divided_by( + &self, + x: BigDecimal, + y: BigDecimal, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::BIG_MATH_GAS_OP.with_args(complexity::Mul, (&x, &y)))?; + if y == 0.into() { + return Err(DeterministicHostError::from(anyhow!( + "attempted to divide BigDecimal `{}` by zero", + x + ))); + } + Ok(x / y) + } + + pub(crate) fn big_decimal_equals( + &self, + x: BigDecimal, + y: BigDecimal, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::BIG_MATH_GAS_OP.with_args(complexity::Min, (&x, &y)))?; + Ok(x == y) + } + + pub(crate) fn big_decimal_to_string( + &self, + x: BigDecimal, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(complexity::Size, &x))?; + Ok(x.to_string()) + } + + pub(crate) fn big_decimal_from_string( + &self, + s: String, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(complexity::Size, &s))?; + BigDecimal::from_str(&s) + .with_context(|| format!("string is not a BigDecimal: '{}'", s)) + .map_err(DeterministicHostError::from) + } + + pub(crate) fn data_source_create( + &self, + logger: &Logger, + state: &mut BlockState, + name: String, + params: Vec, + context: Option, + creation_block: BlockNumber, + gas: &GasCounter, + ) -> Result<(), HostExportError> { + gas.consume_host_fn(gas::CREATE_DATA_SOURCE)?; + info!( + logger, + "Create data source"; + "name" => &name, + "params" => format!("{}", params.join(",")) + ); + + // Resolve the name into the right template + let template = self + .templates + .iter() + .find(|template| template.name() == name) + .with_context(|| { + format!( + "Failed to create data source from name `{}`: \ + No template with this name in parent data source `{}`. \ + Available names: {}.", + name, + self.data_source_name, + self.templates + .iter() + .map(|template| template.name()) + .collect::>() + .join(", ") + ) + }) + .map_err(DeterministicHostError::from)? + .clone(); + + // Remember that we need to create this data source + state.push_created_data_source(DataSourceTemplateInfo { + template, + params, + context, + creation_block, + }); + + Ok(()) + } + + pub(crate) fn ens_name_by_hash(&self, hash: &str) -> Result, anyhow::Error> { + Ok(self.ens_lookup.find_name(hash)?) + } + + pub(crate) fn log_log( + &self, + logger: &Logger, + level: slog::Level, + msg: String, + gas: &GasCounter, + ) -> Result<(), DeterministicHostError> { + gas.consume_host_fn(gas::LOG_OP.with_args(complexity::Size, &msg))?; + + let rs = record_static!(level, self.data_source_name.as_str()); + + logger.log(&slog::Record::new( + &rs, + &format_args!("{}", msg), + b!("data_source" => &self.data_source_name), + )); + + if level == slog::Level::Critical { + return Err(DeterministicHostError::from(anyhow!( + "Critical error logged in mapping" + ))); + } + Ok(()) + } + + pub(crate) fn data_source_address( + &self, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + gas.consume_host_fn(Gas::new(gas::DEFAULT_BASE_COST))?; + Ok(self.data_source_address.clone()) + } + + pub(crate) fn data_source_network( + &self, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(Gas::new(gas::DEFAULT_BASE_COST))?; + Ok(self.data_source_network.clone()) + } + + pub(crate) fn data_source_context( + &self, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(Gas::new(gas::DEFAULT_BASE_COST))?; + Ok(self + .data_source_context + .as_ref() + .clone() + .unwrap_or_default()) + } + + pub(crate) fn json_from_bytes( + &self, + bytes: &Vec, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(gas::complexity::Size, &bytes))?; + serde_json::from_reader(bytes.as_slice()) + .map_err(|e| DeterministicHostError::from(Error::from(e))) + } + + pub(crate) fn string_to_h160( + &self, + string: &str, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(complexity::Size, &string))?; + string_to_h160(string) + } + + pub(crate) fn bytes_to_string( + &self, + logger: &Logger, + bytes: Vec, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(complexity::Size, &bytes))?; + + Ok(bytes_to_string(logger, bytes)) + } + + pub(crate) fn ethereum_encode( + &self, + token: Token, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + let encoded = encode(&[token]); + + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(complexity::Size, &encoded))?; + + Ok(encoded) + } + + pub(crate) fn ethereum_decode( + &self, + types: String, + data: Vec, + gas: &GasCounter, + ) -> Result { + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(complexity::Size, &data))?; + + let param_types = + Reader::read(&types).map_err(|e| anyhow::anyhow!("Failed to read types: {}", e))?; + + decode(&[param_types], &data) + // The `.pop().unwrap()` here is ok because we're always only passing one + // `param_types` to `decode`, so the returned `Vec` has always size of one. + // We can't do `tokens[0]` because the value can't be moved out of the `Vec`. + .map(|mut tokens| tokens.pop().unwrap()) + .context("Failed to decode") + } +} + +fn string_to_h160(string: &str) -> Result { + // `H160::from_str` takes a hex string with no leading `0x`. + let s = string.trim_start_matches("0x"); + H160::from_str(s) + .with_context(|| format!("Failed to convert string to Address/H160: '{}'", s)) + .map_err(DeterministicHostError::from) +} + +fn bytes_to_string(logger: &Logger, bytes: Vec) -> String { + let s = String::from_utf8_lossy(&bytes); + + // If the string was re-allocated, that means it was not UTF8. + if matches!(s, std::borrow::Cow::Owned(_)) { + warn!( + logger, + "Bytes contain invalid UTF8. This may be caused by attempting \ + to convert a value such as an address that cannot be parsed to a unicode string. \ + You may want to use 'toHexString()' instead. String (truncated to 1024 chars): '{}'", + &s.chars().take(1024).collect::(), + ) + } + + // The string may have been encoded in a fixed length buffer and padded with null + // characters, so trim trailing nulls. + s.trim_end_matches('\u{0000}').to_string() +} + +#[test] +fn test_string_to_h160_with_0x() { + assert_eq!( + H160::from_str("A16081F360e3847006dB660bae1c6d1b2e17eC2A").unwrap(), + string_to_h160("0xA16081F360e3847006dB660bae1c6d1b2e17eC2A").unwrap() + ) +} + +#[test] +fn bytes_to_string_is_lossy() { + assert_eq!( + "Downcoin WETH-USDT", + bytes_to_string( + &graph::log::logger(true), + vec![68, 111, 119, 110, 99, 111, 105, 110, 32, 87, 69, 84, 72, 45, 85, 83, 68, 84], + ) + ); + + assert_eq!( + "Downcoin WETH-USDT�", + bytes_to_string( + &graph::log::logger(true), + vec![ + 68, 111, 119, 110, 99, 111, 105, 110, 32, 87, 69, 84, 72, 45, 85, 83, 68, 84, 160, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + ], + ) + ) +} diff --git a/runtime/wasm/src/lib.rs b/runtime/wasm/src/lib.rs new file mode 100644 index 0000000..2a365b9 --- /dev/null +++ b/runtime/wasm/src/lib.rs @@ -0,0 +1,26 @@ +pub mod asc_abi; + +mod host; +pub mod to_from; + +/// Public interface of the crate, receives triggers to be processed. + +/// Pre-processes modules and manages their threads. Serves as an interface from `host` to `module`. +pub mod mapping; + +/// WASM module instance. +pub mod module; + +/// Runtime-agnostic implementation of exports to WASM. +pub mod host_exports; + +pub mod error; +mod gas_rules; + +pub use host::RuntimeHostBuilder; +pub use host_exports::HostExports; +pub use mapping::{MappingContext, ValidModule}; +pub use module::{ExperimentalFeatures, WasmInstance}; + +#[cfg(debug_assertions)] +pub use module::TRAP_TIMEOUT; diff --git a/runtime/wasm/src/mapping.rs b/runtime/wasm/src/mapping.rs new file mode 100644 index 0000000..cd1198b --- /dev/null +++ b/runtime/wasm/src/mapping.rs @@ -0,0 +1,216 @@ +use crate::gas_rules::GasRules; +use crate::module::{ExperimentalFeatures, ToAscPtr, WasmInstance}; +use futures::sync::mpsc; +use futures03::channel::oneshot::Sender; +use graph::blockchain::{Blockchain, HostFn}; +use graph::components::store::SubgraphFork; +use graph::components::subgraph::{MappingError, SharedProofOfIndexing}; +use graph::data_source::{MappingTrigger, TriggerWithHandler}; +use graph::prelude::*; +use graph::runtime::gas::Gas; +use std::collections::BTreeMap; +use std::sync::Arc; +use std::thread; + +/// Spawn a wasm module in its own thread. +pub fn spawn_module( + raw_module: &[u8], + logger: Logger, + subgraph_id: DeploymentHash, + host_metrics: Arc, + runtime: tokio::runtime::Handle, + timeout: Option, + experimental_features: ExperimentalFeatures, +) -> Result>, anyhow::Error> +where + ::MappingTrigger: ToAscPtr, +{ + let valid_module = Arc::new(ValidModule::new(&logger, raw_module)?); + + // Create channel for event handling requests + let (mapping_request_sender, mapping_request_receiver) = mpsc::channel(100); + + // wasmtime instances are not `Send` therefore they cannot be scheduled by + // the regular tokio executor, so we create a dedicated thread. + // + // In case of failure, this thread may panic or simply terminate, + // dropping the `mapping_request_receiver` which ultimately causes the + // subgraph to fail the next time it tries to handle an event. + let conf = + thread::Builder::new().name(format!("mapping-{}-{}", &subgraph_id, uuid::Uuid::new_v4())); + conf.spawn(move || { + let _runtime_guard = runtime.enter(); + + // Pass incoming triggers to the WASM module and return entity changes; + // Stop when canceled because all RuntimeHosts and their senders were dropped. + match mapping_request_receiver + .map_err(|()| unreachable!()) + .for_each(move |request| { + let MappingRequest { + ctx, + trigger, + result_sender, + } = request; + + let result = instantiate_module_and_handle_trigger( + valid_module.cheap_clone(), + ctx, + trigger, + host_metrics.cheap_clone(), + timeout, + experimental_features, + ); + + result_sender + .send(result) + .map_err(|_| anyhow::anyhow!("WASM module result receiver dropped.")) + }) + .wait() + { + Ok(()) => debug!(logger, "Subgraph stopped, WASM runtime thread terminated"), + Err(e) => debug!(logger, "WASM runtime thread terminated abnormally"; + "error" => e.to_string()), + } + }) + .map(|_| ()) + .context("Spawning WASM runtime thread failed")?; + + Ok(mapping_request_sender) +} + +fn instantiate_module_and_handle_trigger( + valid_module: Arc, + ctx: MappingContext, + trigger: TriggerWithHandler>, + host_metrics: Arc, + timeout: Option, + experimental_features: ExperimentalFeatures, +) -> Result<(BlockState, Gas), MappingError> +where + ::MappingTrigger: ToAscPtr, +{ + let logger = ctx.logger.cheap_clone(); + + // Start the WASM module runtime. + let section = host_metrics.stopwatch.start_section("module_init"); + let module = WasmInstance::from_valid_module_with_ctx( + valid_module, + ctx, + host_metrics.cheap_clone(), + timeout, + experimental_features, + )?; + section.end(); + + let _section = host_metrics.stopwatch.start_section("run_handler"); + if ENV_VARS.log_trigger_data { + debug!(logger, "trigger data: {:?}", trigger); + } + module.handle_trigger(trigger) +} + +pub struct MappingRequest { + pub(crate) ctx: MappingContext, + pub(crate) trigger: TriggerWithHandler>, + pub(crate) result_sender: Sender, Gas), MappingError>>, +} + +pub struct MappingContext { + pub logger: Logger, + pub host_exports: Arc>, + pub block_ptr: BlockPtr, + pub state: BlockState, + pub proof_of_indexing: SharedProofOfIndexing, + pub host_fns: Arc>, + pub debug_fork: Option>, +} + +impl MappingContext { + pub fn derive_with_empty_block_state(&self) -> Self { + MappingContext { + logger: self.logger.cheap_clone(), + host_exports: self.host_exports.cheap_clone(), + block_ptr: self.block_ptr.cheap_clone(), + state: BlockState::new(self.state.entity_cache.store.clone(), Default::default()), + proof_of_indexing: self.proof_of_indexing.cheap_clone(), + host_fns: self.host_fns.cheap_clone(), + debug_fork: self.debug_fork.cheap_clone(), + } + } +} + +/// A pre-processed and valid WASM module, ready to be started as a WasmModule. +pub struct ValidModule { + pub module: wasmtime::Module, + + // A wasm import consists of a `module` and a `name`. AS will generate imports such that they + // have `module` set to the name of the file it is imported from and `name` set to the imported + // function name or `namespace.function` if inside a namespace. We'd rather not specify names of + // source files, so we consider that the import `name` uniquely identifies an import. Still we + // need to know the `module` to properly link it, so here we map import names to modules. + // + // AS now has an `@external("module", "name")` decorator which would make things cleaner, but + // the ship has sailed. + pub import_name_to_modules: BTreeMap>, +} + +impl ValidModule { + /// Pre-process and validate the module. + pub fn new(logger: &Logger, raw_module: &[u8]) -> Result { + // Add the gas calls here. Module name "gas" must match. See also + // e3f03e62-40e4-4f8c-b4a1-d0375cca0b76. We do this by round-tripping the module through + // parity - injecting gas then serializing again. + let parity_module = parity_wasm::elements::Module::from_bytes(raw_module)?; + let parity_module = match parity_module.parse_names() { + Ok(module) => module, + Err((errs, module)) => { + for (index, err) in errs { + warn!( + logger, + "unable to parse function name for index {}: {}", + index, + err.to_string() + ); + } + + module + } + }; + let parity_module = wasm_instrument::gas_metering::inject(parity_module, &GasRules, "gas") + .map_err(|_| anyhow!("Failed to inject gas counter"))?; + let raw_module = parity_module.into_bytes()?; + + // We currently use Cranelift as a compilation engine. Cranelift is an optimizing compiler, + // but that should not cause determinism issues since it adheres to the Wasm spec. Still we + // turn off optional optimizations to be conservative. + let mut config = wasmtime::Config::new(); + config.strategy(wasmtime::Strategy::Cranelift).unwrap(); + config.interruptable(true); // For timeouts. + config.cranelift_nan_canonicalization(true); // For NaN determinism. + config.cranelift_opt_level(wasmtime::OptLevel::None); + config + .max_wasm_stack(ENV_VARS.mappings.max_stack_size) + .unwrap(); // Safe because this only panics if size passed is 0. + + let engine = &wasmtime::Engine::new(&config)?; + let module = wasmtime::Module::from_binary(&engine, &raw_module)?; + + let mut import_name_to_modules: BTreeMap> = BTreeMap::new(); + + // Unwrap: Module linking is disabled. + for (name, module) in module + .imports() + .map(|import| (import.name().unwrap(), import.module())) + { + import_name_to_modules + .entry(name.to_string()) + .or_default() + .push(module.to_string()); + } + + Ok(ValidModule { + module, + import_name_to_modules, + }) + } +} diff --git a/runtime/wasm/src/module/into_wasm_ret.rs b/runtime/wasm/src/module/into_wasm_ret.rs new file mode 100644 index 0000000..444c9eb --- /dev/null +++ b/runtime/wasm/src/module/into_wasm_ret.rs @@ -0,0 +1,78 @@ +use never::Never; +use wasmtime::Trap; + +use graph::runtime::AscPtr; + +/// Helper trait for the `link!` macro. +pub trait IntoWasmRet { + type Ret: wasmtime::WasmRet; + + fn into_wasm_ret(self) -> Self::Ret; +} + +impl IntoWasmRet for () { + type Ret = Self; + fn into_wasm_ret(self) -> Self { + self + } +} + +impl IntoWasmRet for Never { + type Ret = (); + fn into_wasm_ret(self) -> Self::Ret { + unreachable!() + } +} + +impl IntoWasmRet for i32 { + type Ret = Self; + fn into_wasm_ret(self) -> Self { + self + } +} + +impl IntoWasmRet for i64 { + type Ret = Self; + fn into_wasm_ret(self) -> Self { + self + } +} + +impl IntoWasmRet for f64 { + type Ret = Self; + fn into_wasm_ret(self) -> Self { + self + } +} + +impl IntoWasmRet for u64 { + type Ret = u64; + fn into_wasm_ret(self) -> u64 { + self + } +} + +impl IntoWasmRet for bool { + type Ret = i32; + fn into_wasm_ret(self) -> i32 { + self.into() + } +} + +impl IntoWasmRet for AscPtr { + type Ret = u32; + fn into_wasm_ret(self) -> u32 { + self.wasm_ptr() + } +} + +impl IntoWasmRet for Result +where + T: IntoWasmRet, + T::Ret: wasmtime::WasmTy, +{ + type Ret = Result; + fn into_wasm_ret(self) -> Self::Ret { + self.map(|x| x.into_wasm_ret()) + } +} diff --git a/runtime/wasm/src/module/mod.rs b/runtime/wasm/src/module/mod.rs new file mode 100644 index 0000000..327b00e --- /dev/null +++ b/runtime/wasm/src/module/mod.rs @@ -0,0 +1,1763 @@ +use std::cell::RefCell; +use std::collections::HashMap; +use std::convert::TryFrom; +use std::mem::MaybeUninit; +use std::ops::{Deref, DerefMut}; +use std::rc::Rc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Instant; + +use anyhow::anyhow; +use anyhow::Error; +use never::Never; +use semver::Version; +use wasmtime::{Memory, Trap}; + +use graph::blockchain::{Blockchain, HostFnCtx}; +use graph::data::store; +use graph::data::subgraph::schema::SubgraphError; +use graph::data_source::{offchain, MappingTrigger, TriggerWithHandler}; +use graph::prelude::*; +use graph::runtime::{ + asc_get, asc_new, + gas::{self, Gas, GasCounter, SaturatingInto}, + AscHeap, AscIndexId, AscType, DeterministicHostError, FromAscObj, HostExportError, + IndexForAscTypeId, ToAscObj, +}; +use graph::util::mem::init_slice; +use graph::{components::subgraph::MappingError, runtime::AscPtr}; +pub use into_wasm_ret::IntoWasmRet; +pub use stopwatch::TimeoutStopwatch; + +use crate::asc_abi::class::*; +use crate::error::DeterminismLevel; +use crate::gas_rules::{GAS_COST_LOAD, GAS_COST_STORE}; +pub use crate::host_exports; +use crate::host_exports::HostExports; +use crate::mapping::MappingContext; +use crate::mapping::ValidModule; + +mod into_wasm_ret; +pub mod stopwatch; + +pub const TRAP_TIMEOUT: &str = "trap: interrupt"; + +pub trait IntoTrap { + fn determinism_level(&self) -> DeterminismLevel; + fn into_trap(self) -> Trap; +} + +/// A flexible interface for writing a type to AS memory, any pointer can be returned. +/// Use `AscPtr::erased` to convert `AscPtr` into `AscPtr<()>`. +pub trait ToAscPtr { + fn to_asc_ptr( + self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError>; +} + +impl ToAscPtr for offchain::TriggerData { + fn to_asc_ptr( + self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + asc_new(heap, self.data.as_ref() as &[u8], gas).map(|ptr| ptr.erase()) + } +} + +impl ToAscPtr for MappingTrigger +where + C::MappingTrigger: ToAscPtr, +{ + fn to_asc_ptr( + self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + match self { + MappingTrigger::Onchain(trigger) => trigger.to_asc_ptr(heap, gas), + MappingTrigger::Offchain(trigger) => trigger.to_asc_ptr(heap, gas), + } + } +} + +impl ToAscPtr for TriggerWithHandler { + fn to_asc_ptr( + self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + self.trigger.to_asc_ptr(heap, gas) + } +} + +/// Handle to a WASM instance, which is terminated if and only if this is dropped. +pub struct WasmInstance { + pub instance: wasmtime::Instance, + + // This is the only reference to `WasmInstanceContext` that's not within the instance itself, so + // we can always borrow the `RefCell` with no concern for race conditions. + // + // Also this is the only strong reference, so the instance will be dropped once this is dropped. + // The weak references are circulary held by instance itself through host exports. + pub instance_ctx: Rc>>>, + + // A reference to the gas counter used for reporting the gas used. + pub gas: GasCounter, +} + +impl Drop for WasmInstance { + fn drop(&mut self) { + // Assert that the instance will be dropped. + assert_eq!(Rc::strong_count(&self.instance_ctx), 1); + } +} + +impl WasmInstance { + pub fn asc_get(&self, asc_ptr: AscPtr

) -> Result + where + P: AscType + AscIndexId, + T: FromAscObj

, + { + asc_get(self.instance_ctx().deref(), asc_ptr, &self.gas) + } + + pub fn asc_new( + &mut self, + rust_obj: &T, + ) -> Result, DeterministicHostError> + where + P: AscType + AscIndexId, + T: ToAscObj

, + { + asc_new(self.instance_ctx_mut().deref_mut(), rust_obj, &self.gas) + } +} + +impl WasmInstance { + pub(crate) fn handle_json_callback( + mut self, + handler_name: &str, + value: &serde_json::Value, + user_data: &store::Value, + ) -> Result, anyhow::Error> { + let gas = GasCounter::default(); + let value = asc_new(self.instance_ctx_mut().deref_mut(), value, &gas)?; + let user_data = asc_new(self.instance_ctx_mut().deref_mut(), user_data, &gas)?; + + self.instance_ctx_mut().ctx.state.enter_handler(); + + // Invoke the callback + self.instance + .get_func(handler_name) + .with_context(|| format!("function {} not found", handler_name))? + .typed()? + .call((value.wasm_ptr(), user_data.wasm_ptr())) + .with_context(|| format!("Failed to handle callback '{}'", handler_name))?; + + self.instance_ctx_mut().ctx.state.exit_handler(); + + Ok(self.take_ctx().ctx.state) + } + + pub(crate) fn handle_trigger( + mut self, + trigger: TriggerWithHandler>, + ) -> Result<(BlockState, Gas), MappingError> + where + ::MappingTrigger: ToAscPtr, + { + let handler_name = trigger.handler_name().to_owned(); + let gas = self.gas.clone(); + let asc_trigger = trigger.to_asc_ptr(self.instance_ctx_mut().deref_mut(), &gas)?; + self.invoke_handler(&handler_name, asc_trigger) + } + + pub fn take_ctx(&mut self) -> WasmInstanceContext { + self.instance_ctx.borrow_mut().take().unwrap() + } + + pub(crate) fn instance_ctx(&self) -> std::cell::Ref<'_, WasmInstanceContext> { + std::cell::Ref::map(self.instance_ctx.borrow(), |i| i.as_ref().unwrap()) + } + + pub fn instance_ctx_mut(&self) -> std::cell::RefMut<'_, WasmInstanceContext> { + std::cell::RefMut::map(self.instance_ctx.borrow_mut(), |i| i.as_mut().unwrap()) + } + + #[cfg(debug_assertions)] + pub fn get_func(&self, func_name: &str) -> wasmtime::Func { + self.instance.get_func(func_name).unwrap() + } + + #[cfg(debug_assertions)] + pub fn gas_used(&self) -> u64 { + self.gas.get().value() + } + + fn invoke_handler( + &mut self, + handler: &str, + arg: AscPtr, + ) -> Result<(BlockState, Gas), MappingError> { + let func = self + .instance + .get_func(handler) + .with_context(|| format!("function {} not found", handler))?; + + // Caution: Make sure all exit paths from this function call `exit_handler`. + self.instance_ctx_mut().ctx.state.enter_handler(); + + // This `match` will return early if there was a non-deterministic trap. + let deterministic_error: Option = match func.typed()?.call(arg.wasm_ptr()) { + Ok(()) => None, + Err(trap) if self.instance_ctx().possible_reorg => { + self.instance_ctx_mut().ctx.state.exit_handler(); + return Err(MappingError::PossibleReorg(trap.into())); + } + Err(trap) if trap.to_string().contains(TRAP_TIMEOUT) => { + self.instance_ctx_mut().ctx.state.exit_handler(); + return Err(MappingError::Unknown(Error::from(trap).context(format!( + "Handler '{}' hit the timeout of '{}' seconds", + handler, + self.instance_ctx().timeout.unwrap().as_secs() + )))); + } + Err(trap) => { + use wasmtime::TrapCode::*; + let trap_code = trap.trap_code(); + let e = Error::from(trap); + match trap_code { + Some(MemoryOutOfBounds) + | Some(HeapMisaligned) + | Some(TableOutOfBounds) + | Some(IndirectCallToNull) + | Some(BadSignature) + | Some(IntegerOverflow) + | Some(IntegerDivisionByZero) + | Some(BadConversionToInteger) + | Some(UnreachableCodeReached) => Some(e), + _ if self.instance_ctx().deterministic_host_trap => Some(e), + _ => { + self.instance_ctx_mut().ctx.state.exit_handler(); + return Err(MappingError::Unknown(e)); + } + } + } + }; + + if let Some(deterministic_error) = deterministic_error { + let message = format!("{:#}", deterministic_error).replace("\n", "\t"); + + // Log the error and restore the updates snapshot, effectively reverting the handler. + error!(&self.instance_ctx().ctx.logger, + "Handler skipped due to execution failure"; + "handler" => handler, + "error" => &message, + ); + let subgraph_error = SubgraphError { + subgraph_id: self.instance_ctx().ctx.host_exports.subgraph_id.clone(), + message, + block_ptr: Some(self.instance_ctx().ctx.block_ptr.cheap_clone()), + handler: Some(handler.to_string()), + deterministic: true, + }; + self.instance_ctx_mut() + .ctx + .state + .exit_handler_and_discard_changes_due_to_error(subgraph_error); + } else { + self.instance_ctx_mut().ctx.state.exit_handler(); + } + + let gas = self.gas.get(); + Ok((self.take_ctx().ctx.state, gas)) + } +} + +#[derive(Copy, Clone)] +pub struct ExperimentalFeatures { + pub allow_non_deterministic_ipfs: bool, +} + +pub struct WasmInstanceContext { + // In the future there may be multiple memories, but currently there is only one memory per + // module. And at least AS calls it "memory". There is no uninitialized memory in Wasm, memory + // is zeroed when initialized or grown. + memory: Memory, + + // Function exported by the wasm module that will allocate the request number of bytes and + // return a pointer to the first byte of allocated space. + memory_allocate: wasmtime::TypedFunc, + + // Function wrapper for `idof` from AssemblyScript + id_of_type: Option>, + + pub ctx: MappingContext, + pub valid_module: Arc, + pub host_metrics: Arc, + pub(crate) timeout: Option, + + // Used by ipfs.map. + pub(crate) timeout_stopwatch: Arc>, + + // First free byte in the current arena. Set on the first call to `raw_new`. + arena_start_ptr: i32, + + // Number of free bytes starting from `arena_start_ptr`. + arena_free_size: i32, + + // A trap ocurred due to a possible reorg detection. + pub possible_reorg: bool, + + // A host export trap ocurred for a deterministic reason. + pub deterministic_host_trap: bool, + + pub(crate) experimental_features: ExperimentalFeatures, +} + +impl WasmInstance { + /// Instantiates the module and sets it to be interrupted after `timeout`. + pub fn from_valid_module_with_ctx( + valid_module: Arc, + ctx: MappingContext, + host_metrics: Arc, + timeout: Option, + experimental_features: ExperimentalFeatures, + ) -> Result, anyhow::Error> { + let mut linker = wasmtime::Linker::new(&wasmtime::Store::new(valid_module.module.engine())); + let host_fns = ctx.host_fns.cheap_clone(); + let api_version = ctx.host_exports.api_version.clone(); + + // Used by exports to access the instance context. There are two ways this can be set: + // - After instantiation, if no host export is called in the start function. + // - During the start function, if it calls a host export. + // Either way, after instantiation this will have been set. + let shared_ctx: Rc>>> = Rc::new(RefCell::new(None)); + + // We will move the ctx only once, to init `shared_ctx`. But we don't statically know where + // it will be moved so we need this ugly thing. + let ctx: Rc>>> = Rc::new(RefCell::new(Some(ctx))); + + // Start the timeout watchdog task. + let timeout_stopwatch = Arc::new(std::sync::Mutex::new(TimeoutStopwatch::start_new())); + if let Some(timeout) = timeout { + // This task is likely to outlive the instance, which is fine. + let interrupt_handle = linker.store().interrupt_handle().unwrap(); + let timeout_stopwatch = timeout_stopwatch.clone(); + graph::spawn_allow_panic(async move { + let minimum_wait = Duration::from_secs(1); + loop { + let time_left = + timeout.checked_sub(timeout_stopwatch.lock().unwrap().elapsed()); + match time_left { + None => break interrupt_handle.interrupt(), // Timed out. + + Some(time) if time < minimum_wait => break interrupt_handle.interrupt(), + Some(time) => tokio::time::sleep(time).await, + } + } + }); + } + + // Because `gas` and `deterministic_host_trap` need to be accessed from the gas + // host fn, they need to be separate from the rest of the context. + let gas = GasCounter::default(); + let deterministic_host_trap = Rc::new(AtomicBool::new(false)); + + macro_rules! link { + ($wasm_name:expr, $rust_name:ident, $($param:ident),*) => { + link!($wasm_name, $rust_name, "host_export_other", $($param),*) + }; + + ($wasm_name:expr, $rust_name:ident, $section:expr, $($param:ident),*) => { + let modules = valid_module + .import_name_to_modules + .get($wasm_name) + .into_iter() + .flatten(); + + // link an import with all the modules that require it. + for module in modules { + let func_shared_ctx = Rc::downgrade(&shared_ctx); + let valid_module = valid_module.cheap_clone(); + let host_metrics = host_metrics.cheap_clone(); + let timeout_stopwatch = timeout_stopwatch.cheap_clone(); + let ctx = ctx.cheap_clone(); + let gas = gas.cheap_clone(); + linker.func( + module, + $wasm_name, + move |caller: wasmtime::Caller, $($param: u32),*| { + let instance = func_shared_ctx.upgrade().unwrap(); + let mut instance = instance.borrow_mut(); + + // Happens when calling a host fn in Wasm start. + if instance.is_none() { + *instance = Some(WasmInstanceContext::from_caller( + caller, + ctx.borrow_mut().take().unwrap(), + valid_module.cheap_clone(), + host_metrics.cheap_clone(), + timeout, + timeout_stopwatch.cheap_clone(), + experimental_features.clone() + ).unwrap()) + } + + let instance = instance.as_mut().unwrap(); + let _section = instance.host_metrics.stopwatch.start_section($section); + + let result = instance.$rust_name( + &gas, + $($param.into()),* + ); + match result { + Ok(result) => Ok(result.into_wasm_ret()), + Err(e) => { + match IntoTrap::determinism_level(&e) { + DeterminismLevel::Deterministic => { + instance.deterministic_host_trap = true; + }, + DeterminismLevel::PossibleReorg => { + instance.possible_reorg = true; + }, + DeterminismLevel::Unimplemented | DeterminismLevel::NonDeterministic => {}, + } + + Err(IntoTrap::into_trap(e)) + } + } + } + )?; + } + }; + } + + // Link chain-specifc host fns. + for host_fn in host_fns.iter() { + let modules = valid_module + .import_name_to_modules + .get(host_fn.name) + .into_iter() + .flatten(); + + for module in modules { + let func_shared_ctx = Rc::downgrade(&shared_ctx); + let host_fn = host_fn.cheap_clone(); + let gas = gas.cheap_clone(); + linker.func(module, host_fn.name, move |call_ptr: u32| { + let start = Instant::now(); + let instance = func_shared_ctx.upgrade().unwrap(); + let mut instance = instance.borrow_mut(); + + let instance = match &mut *instance { + Some(instance) => instance, + + // Happens when calling a host fn in Wasm start. + None => { + return Err(anyhow!( + "{} is not allowed in global variables", + host_fn.name + ) + .into()); + } + }; + + let name_for_metrics = host_fn.name.replace('.', "_"); + let stopwatch = &instance.host_metrics.stopwatch; + let _section = + stopwatch.start_section(&format!("host_export_{}", name_for_metrics)); + + let ctx = HostFnCtx { + logger: instance.ctx.logger.cheap_clone(), + block_ptr: instance.ctx.block_ptr.cheap_clone(), + heap: instance, + gas: gas.cheap_clone(), + }; + let ret = (host_fn.func)(ctx, call_ptr).map_err(|e| match e { + HostExportError::Deterministic(e) => { + instance.deterministic_host_trap = true; + e + } + HostExportError::PossibleReorg(e) => { + instance.possible_reorg = true; + e + } + HostExportError::Unknown(e) => e, + })?; + instance.host_metrics.observe_host_fn_execution_time( + start.elapsed().as_secs_f64(), + &name_for_metrics, + ); + Ok(ret) + })?; + } + } + + link!("ethereum.encode", ethereum_encode, params_ptr); + link!("ethereum.decode", ethereum_decode, params_ptr, data_ptr); + + link!("abort", abort, message_ptr, file_name_ptr, line, column); + + link!("store.get", store_get, "host_export_store_get", entity, id); + link!( + "store.set", + store_set, + "host_export_store_set", + entity, + id, + data + ); + + // All IPFS-related functions exported by the host WASM runtime should be listed in the + // graph::data::subgraph::features::IPFS_ON_ETHEREUM_CONTRACTS_FUNCTION_NAMES array for + // automatic feature detection to work. + // + // For reference, search this codebase for: ff652476-e6ad-40e4-85b8-e815d6c6e5e2 + link!("ipfs.cat", ipfs_cat, "host_export_ipfs_cat", hash_ptr); + link!( + "ipfs.map", + ipfs_map, + "host_export_ipfs_map", + link_ptr, + callback, + user_data, + flags + ); + // The previous ipfs-related functions are unconditionally linked for backward compatibility + if experimental_features.allow_non_deterministic_ipfs { + link!( + "ipfs.getBlock", + ipfs_get_block, + "host_export_ipfs_get_block", + hash_ptr + ); + } + + link!("store.remove", store_remove, entity_ptr, id_ptr); + + link!("typeConversion.bytesToString", bytes_to_string, ptr); + link!("typeConversion.bytesToHex", bytes_to_hex, ptr); + link!("typeConversion.bigIntToString", big_int_to_string, ptr); + link!("typeConversion.bigIntToHex", big_int_to_hex, ptr); + link!("typeConversion.stringToH160", string_to_h160, ptr); + link!("typeConversion.bytesToBase58", bytes_to_base58, ptr); + + link!("json.fromBytes", json_from_bytes, ptr); + link!("json.try_fromBytes", json_try_from_bytes, ptr); + link!("json.toI64", json_to_i64, ptr); + link!("json.toU64", json_to_u64, ptr); + link!("json.toF64", json_to_f64, ptr); + link!("json.toBigInt", json_to_big_int, ptr); + + link!("crypto.keccak256", crypto_keccak_256, ptr); + + link!("bigInt.plus", big_int_plus, x_ptr, y_ptr); + link!("bigInt.minus", big_int_minus, x_ptr, y_ptr); + link!("bigInt.times", big_int_times, x_ptr, y_ptr); + link!("bigInt.dividedBy", big_int_divided_by, x_ptr, y_ptr); + link!("bigInt.dividedByDecimal", big_int_divided_by_decimal, x, y); + link!("bigInt.mod", big_int_mod, x_ptr, y_ptr); + link!("bigInt.pow", big_int_pow, x_ptr, exp); + link!("bigInt.fromString", big_int_from_string, ptr); + link!("bigInt.bitOr", big_int_bit_or, x_ptr, y_ptr); + link!("bigInt.bitAnd", big_int_bit_and, x_ptr, y_ptr); + link!("bigInt.leftShift", big_int_left_shift, x_ptr, bits); + link!("bigInt.rightShift", big_int_right_shift, x_ptr, bits); + + link!("bigDecimal.toString", big_decimal_to_string, ptr); + link!("bigDecimal.fromString", big_decimal_from_string, ptr); + link!("bigDecimal.plus", big_decimal_plus, x_ptr, y_ptr); + link!("bigDecimal.minus", big_decimal_minus, x_ptr, y_ptr); + link!("bigDecimal.times", big_decimal_times, x_ptr, y_ptr); + link!("bigDecimal.dividedBy", big_decimal_divided_by, x, y); + link!("bigDecimal.equals", big_decimal_equals, x_ptr, y_ptr); + + link!("dataSource.create", data_source_create, name, params); + link!( + "dataSource.createWithContext", + data_source_create_with_context, + name, + params, + context + ); + link!("dataSource.address", data_source_address,); + link!("dataSource.network", data_source_network,); + link!("dataSource.context", data_source_context,); + + link!("ens.nameByHash", ens_name_by_hash, ptr); + + link!("log.log", log_log, level, msg_ptr); + + // `arweave and `box` functionality was removed, but apiVersion <= 0.0.4 must link it. + if api_version <= Version::new(0, 0, 4) { + link!("arweave.transactionData", arweave_transaction_data, ptr); + link!("box.profile", box_profile, ptr); + } + + // link the `gas` function + // See also e3f03e62-40e4-4f8c-b4a1-d0375cca0b76 + { + let gas = gas.cheap_clone(); + linker.func("gas", "gas", move |gas_used: u32| -> Result<(), Trap> { + // Gas metering has a relevant execution cost cost, being called tens of thousands + // of times per handler, but it's not worth having a stopwatch section here because + // the cost of measuring would be greater than the cost of `consume_host_fn`. Last + // time this was benchmarked it took < 100ns to run. + if let Err(e) = gas.consume_host_fn(gas_used.saturating_into()) { + deterministic_host_trap.store(true, Ordering::SeqCst); + return Err(e.into_trap()); + } + + Ok(()) + })?; + } + + let instance = linker.instantiate(&valid_module.module)?; + + // Usually `shared_ctx` is still `None` because no host fns were called during start. + if shared_ctx.borrow().is_none() { + *shared_ctx.borrow_mut() = Some(WasmInstanceContext::from_instance( + &instance, + ctx.borrow_mut().take().unwrap(), + valid_module, + host_metrics, + timeout, + timeout_stopwatch, + experimental_features, + )?); + } + + match api_version { + version if version <= Version::new(0, 0, 4) => {} + _ => { + instance + .get_func("_start") + .context("`_start` function not found")? + .typed::<(), ()>()? + .call(())?; + } + } + + Ok(WasmInstance { + instance, + instance_ctx: shared_ctx, + gas, + }) + } +} + +impl AscHeap for WasmInstanceContext { + fn raw_new(&mut self, bytes: &[u8], gas: &GasCounter) -> Result { + // The cost of writing to wasm memory from the host is the same as of writing from wasm + // using load instructions. + gas.consume_host_fn(Gas::new(GAS_COST_STORE as u64 * bytes.len() as u64))?; + + // We request large chunks from the AssemblyScript allocator to use as arenas that we + // manage directly. + + static MIN_ARENA_SIZE: i32 = 10_000; + + let size = i32::try_from(bytes.len()).unwrap(); + if size > self.arena_free_size { + // Allocate a new arena. Any free space left in the previous arena is left unused. This + // causes at most half of memory to be wasted, which is acceptable. + let arena_size = size.max(MIN_ARENA_SIZE); + + // Unwrap: This may panic if more memory needs to be requested from the OS and that + // fails. This error is not deterministic since it depends on the operating conditions + // of the node. + self.arena_start_ptr = self.memory_allocate.call(arena_size).unwrap(); + self.arena_free_size = arena_size; + + match &self.ctx.host_exports.api_version { + version if *version <= Version::new(0, 0, 4) => {} + _ => { + // This arithmetic is done because when you call AssemblyScripts's `__alloc` + // function, it isn't typed and it just returns `mmInfo` on it's header, + // differently from allocating on regular types (`__new` for example). + // `mmInfo` has size of 4, and everything allocated on AssemblyScript memory + // should have alignment of 16, this means we need to do a 12 offset on these + // big chunks of untyped allocation. + self.arena_start_ptr += 12; + self.arena_free_size -= 12; + } + }; + }; + + let ptr = self.arena_start_ptr as usize; + + // Unwrap: We have just allocated enough space for `bytes`. + self.memory.write(ptr, bytes).unwrap(); + self.arena_start_ptr += size; + self.arena_free_size -= size; + + Ok(ptr as u32) + } + + fn read_u32(&self, offset: u32, gas: &GasCounter) -> Result { + gas.consume_host_fn(Gas::new(GAS_COST_LOAD as u64 * 4))?; + let mut bytes = [0; 4]; + self.memory.read(offset as usize, &mut bytes).map_err(|_| { + DeterministicHostError::from(anyhow!( + "Heap access out of bounds. Offset: {} Size: {}", + offset, + 4 + )) + })?; + Ok(u32::from_le_bytes(bytes)) + } + + fn read<'a>( + &self, + offset: u32, + buffer: &'a mut [MaybeUninit], + gas: &GasCounter, + ) -> Result<&'a mut [u8], DeterministicHostError> { + // The cost of reading wasm memory from the host is the same as of reading from wasm using + // load instructions. + gas.consume_host_fn(Gas::new(GAS_COST_LOAD as u64 * (buffer.len() as u64)))?; + + let offset = offset as usize; + + unsafe { + // Safety: This was copy-pasted from Memory::read, and we ensure + // nothing else is writing this memory because we don't call into + // WASM here. + let src = self + .memory + .data_unchecked() + .get(offset..) + .and_then(|s| s.get(..buffer.len())) + .ok_or(DeterministicHostError::from(anyhow!( + "Heap access out of bounds. Offset: {} Size: {}", + offset, + buffer.len() + )))?; + + Ok(init_slice(src, buffer)) + } + } + + fn api_version(&self) -> Version { + self.ctx.host_exports.api_version.clone() + } + + fn asc_type_id( + &mut self, + type_id_index: IndexForAscTypeId, + ) -> Result { + let type_id = self + .id_of_type + .as_ref() + .unwrap() // Unwrap ok because it's only called on correct apiVersion, look for AscPtr::generate_header + .call(type_id_index as u32) + .with_context(|| format!("Failed to call 'asc_type_id' with '{:?}'", type_id_index)) + .map_err(DeterministicHostError::from)?; + Ok(type_id) + } +} + +impl WasmInstanceContext { + pub fn from_instance( + instance: &wasmtime::Instance, + ctx: MappingContext, + valid_module: Arc, + host_metrics: Arc, + timeout: Option, + timeout_stopwatch: Arc>, + experimental_features: ExperimentalFeatures, + ) -> Result { + // Provide access to the WASM runtime linear memory + let memory = instance + .get_memory("memory") + .context("Failed to find memory export in the WASM module")?; + + let memory_allocate = match &ctx.host_exports.api_version { + version if *version <= Version::new(0, 0, 4) => instance + .get_func("memory.allocate") + .context("`memory.allocate` function not found"), + _ => instance + .get_func("allocate") + .context("`allocate` function not found"), + }? + .typed()? + .clone(); + + let id_of_type = match &ctx.host_exports.api_version { + version if *version <= Version::new(0, 0, 4) => None, + _ => Some( + instance + .get_func("id_of_type") + .context("`id_of_type` function not found")? + .typed()? + .clone(), + ), + }; + + Ok(WasmInstanceContext { + memory_allocate, + id_of_type, + memory, + ctx, + valid_module, + host_metrics, + timeout, + timeout_stopwatch, + arena_free_size: 0, + arena_start_ptr: 0, + possible_reorg: false, + deterministic_host_trap: false, + experimental_features, + }) + } + + pub fn from_caller( + caller: wasmtime::Caller, + ctx: MappingContext, + valid_module: Arc, + host_metrics: Arc, + timeout: Option, + timeout_stopwatch: Arc>, + experimental_features: ExperimentalFeatures, + ) -> Result { + let memory = caller + .get_export("memory") + .and_then(|e| e.into_memory()) + .context("Failed to find memory export in the WASM module")?; + + let memory_allocate = match &ctx.host_exports.api_version { + version if *version <= Version::new(0, 0, 4) => caller + .get_export("memory.allocate") + .and_then(|e| e.into_func()) + .context("`memory.allocate` function not found"), + _ => caller + .get_export("allocate") + .and_then(|e| e.into_func()) + .context("`allocate` function not found"), + }? + .typed()? + .clone(); + + let id_of_type = match &ctx.host_exports.api_version { + version if *version <= Version::new(0, 0, 4) => None, + _ => Some( + caller + .get_export("id_of_type") + .and_then(|e| e.into_func()) + .context("`id_of_type` function not found")? + .typed()? + .clone(), + ), + }; + + Ok(WasmInstanceContext { + id_of_type, + memory_allocate, + memory, + ctx, + valid_module, + host_metrics, + timeout, + timeout_stopwatch, + arena_free_size: 0, + arena_start_ptr: 0, + possible_reorg: false, + deterministic_host_trap: false, + experimental_features, + }) + } +} + +// Implementation of externals. +impl WasmInstanceContext { + /// function abort(message?: string | null, fileName?: string | null, lineNumber?: u32, columnNumber?: u32): void + /// Always returns a trap. + pub fn abort( + &mut self, + gas: &GasCounter, + message_ptr: AscPtr, + file_name_ptr: AscPtr, + line_number: u32, + column_number: u32, + ) -> Result { + let message = match message_ptr.is_null() { + false => Some(asc_get(self, message_ptr, gas)?), + true => None, + }; + let file_name = match file_name_ptr.is_null() { + false => Some(asc_get(self, file_name_ptr, gas)?), + true => None, + }; + let line_number = match line_number { + 0 => None, + _ => Some(line_number), + }; + let column_number = match column_number { + 0 => None, + _ => Some(column_number), + }; + + self.ctx + .host_exports + .abort(message, file_name, line_number, column_number, gas) + } + + /// function store.set(entity: string, id: string, data: Entity): void + pub fn store_set( + &mut self, + gas: &GasCounter, + entity_ptr: AscPtr, + id_ptr: AscPtr, + data_ptr: AscPtr, + ) -> Result<(), HostExportError> { + let stopwatch = &self.host_metrics.stopwatch; + stopwatch.start_section("host_export_store_set__wasm_instance_context_store_set"); + + let entity = asc_get(self, entity_ptr, gas)?; + let id = asc_get(self, id_ptr, gas)?; + let data = asc_get(self, data_ptr, gas)?; + + self.ctx.host_exports.store_set( + &self.ctx.logger, + &mut self.ctx.state, + &self.ctx.proof_of_indexing, + entity, + id, + data, + stopwatch, + gas, + )?; + + Ok(()) + } + + /// function store.remove(entity: string, id: string): void + pub fn store_remove( + &mut self, + gas: &GasCounter, + entity_ptr: AscPtr, + id_ptr: AscPtr, + ) -> Result<(), HostExportError> { + let entity = asc_get(self, entity_ptr, gas)?; + let id = asc_get(self, id_ptr, gas)?; + self.ctx.host_exports.store_remove( + &self.ctx.logger, + &mut self.ctx.state, + &self.ctx.proof_of_indexing, + entity, + id, + gas, + ) + } + + /// function store.get(entity: string, id: string): Entity | null + pub fn store_get( + &mut self, + gas: &GasCounter, + entity_ptr: AscPtr, + id_ptr: AscPtr, + ) -> Result, HostExportError> { + let _timer = self + .host_metrics + .cheap_clone() + .time_host_fn_execution_region("store_get"); + + let entity_type: String = asc_get(self, entity_ptr, gas)?; + let id: String = asc_get(self, id_ptr, gas)?; + let entity_option = self.ctx.host_exports.store_get( + &mut self.ctx.state, + entity_type.clone(), + id.clone(), + gas, + )?; + + let ret = match entity_option { + Some(entity) => { + let _section = self + .host_metrics + .stopwatch + .start_section("store_get_asc_new"); + asc_new(self, &entity.sorted(), gas)? + } + None => match &self.ctx.debug_fork { + Some(fork) => { + let entity_option = fork.fetch(entity_type, id).map_err(|e| { + HostExportError::Unknown(anyhow!( + "store_get: failed to fetch entity from the debug fork: {}", + e + )) + })?; + match entity_option { + Some(entity) => { + let _section = self + .host_metrics + .stopwatch + .start_section("store_get_asc_new"); + let entity = asc_new(self, &entity.sorted(), gas)?; + self.store_set(gas, entity_ptr, id_ptr, entity)?; + entity + } + None => AscPtr::null(), + } + } + None => AscPtr::null(), + }, + }; + + Ok(ret) + } + + /// function typeConversion.bytesToString(bytes: Bytes): string + pub fn bytes_to_string( + &mut self, + gas: &GasCounter, + bytes_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let string = self.ctx.host_exports.bytes_to_string( + &self.ctx.logger, + asc_get(self, bytes_ptr, gas)?, + gas, + )?; + asc_new(self, &string, gas) + } + + /// Converts bytes to a hex string. + /// function typeConversion.bytesToHex(bytes: Bytes): string + /// References: + /// https://godoc.org/github.com/ethereum/go-ethereum/common/hexutil#hdr-Encoding_Rules + /// https://github.com/ethereum/web3.js/blob/f98fe1462625a6c865125fecc9cb6b414f0a5e83/packages/web3-utils/src/utils.js#L283 + pub fn bytes_to_hex( + &mut self, + gas: &GasCounter, + bytes_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let bytes: Vec = asc_get(self, bytes_ptr, gas)?; + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(gas::complexity::Size, &bytes))?; + + // Even an empty string must be prefixed with `0x`. + // Encodes each byte as a two hex digits. + let hex = format!("0x{}", hex::encode(bytes)); + asc_new(self, &hex, gas) + } + + /// function typeConversion.bigIntToString(n: Uint8Array): string + pub fn big_int_to_string( + &mut self, + gas: &GasCounter, + big_int_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let n: BigInt = asc_get(self, big_int_ptr, gas)?; + gas.consume_host_fn(gas::DEFAULT_GAS_OP.with_args(gas::complexity::Size, &n))?; + asc_new(self, &n.to_string(), gas) + } + + /// function bigInt.fromString(x: string): BigInt + pub fn big_int_from_string( + &mut self, + gas: &GasCounter, + string_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self + .ctx + .host_exports + .big_int_from_string(asc_get(self, string_ptr, gas)?, gas)?; + asc_new(self, &result, gas) + } + + /// function typeConversion.bigIntToHex(n: Uint8Array): string + pub fn big_int_to_hex( + &mut self, + gas: &GasCounter, + big_int_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let n: BigInt = asc_get(self, big_int_ptr, gas)?; + let hex = self.ctx.host_exports.big_int_to_hex(n, gas)?; + asc_new(self, &hex, gas) + } + + /// function typeConversion.stringToH160(s: String): H160 + pub fn string_to_h160( + &mut self, + gas: &GasCounter, + str_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let s: String = asc_get(self, str_ptr, gas)?; + let h160 = self.ctx.host_exports.string_to_h160(&s, gas)?; + asc_new(self, &h160, gas) + } + + /// function json.fromBytes(bytes: Bytes): JSONValue + pub fn json_from_bytes( + &mut self, + gas: &GasCounter, + bytes_ptr: AscPtr, + ) -> Result>, DeterministicHostError> { + let bytes: Vec = asc_get(self, bytes_ptr, gas)?; + let result = self + .ctx + .host_exports + .json_from_bytes(&bytes, gas) + .with_context(|| { + format!( + "Failed to parse JSON from byte array. Bytes (truncated to 1024 chars): `{:?}`", + &bytes[..bytes.len().min(1024)], + ) + }) + .map_err(DeterministicHostError::from)?; + asc_new(self, &result, gas) + } + + /// function json.try_fromBytes(bytes: Bytes): Result + pub fn json_try_from_bytes( + &mut self, + gas: &GasCounter, + bytes_ptr: AscPtr, + ) -> Result>, bool>>, DeterministicHostError> + { + let bytes: Vec = asc_get(self, bytes_ptr, gas)?; + let result = self + .ctx + .host_exports + .json_from_bytes(&bytes, gas) + .map_err(|e| { + warn!( + &self.ctx.logger, + "Failed to parse JSON from byte array"; + "bytes" => format!("{:?}", bytes), + "error" => format!("{}", e) + ); + + // Map JSON errors to boolean to match the `Result` + // result type expected by mappings + true + }); + asc_new(self, &result, gas) + } + + /// function ipfs.cat(link: String): Bytes + pub fn ipfs_cat( + &mut self, + gas: &GasCounter, + link_ptr: AscPtr, + ) -> Result, HostExportError> { + // Note on gas: There is no gas costing for the ipfs call itself, + // since it's not enabled on the network. + + if !self.experimental_features.allow_non_deterministic_ipfs { + return Err(HostExportError::Deterministic(anyhow!( + "`ipfs.cat` is deprecated. Improved support for IPFS will be added in the future" + ))); + } + + let link = asc_get(self, link_ptr, gas)?; + let ipfs_res = self.ctx.host_exports.ipfs_cat(&self.ctx.logger, link); + match ipfs_res { + Ok(bytes) => asc_new(self, &*bytes, gas).map_err(Into::into), + + // Return null in case of error. + Err(e) => { + info!(&self.ctx.logger, "Failed ipfs.cat, returning `null`"; + "link" => asc_get::(self, link_ptr, gas)?, + "error" => e.to_string()); + Ok(AscPtr::null()) + } + } + } + + /// function ipfs.getBlock(link: String): Bytes + pub fn ipfs_get_block( + &mut self, + gas: &GasCounter, + link_ptr: AscPtr, + ) -> Result, HostExportError> { + // Note on gas: There is no gas costing for the ipfs call itself, + // since it's not enabled on the network. + + if !self.experimental_features.allow_non_deterministic_ipfs { + return Err(HostExportError::Deterministic(anyhow!( + "`ipfs.getBlock` is deprecated. Improved support for IPFS will be added in the future" + ))); + } + + let link = asc_get(self, link_ptr, gas)?; + let ipfs_res = self.ctx.host_exports.ipfs_get_block(&self.ctx.logger, link); + match ipfs_res { + Ok(bytes) => asc_new(self, &*bytes, gas).map_err(Into::into), + + // Return null in case of error. + Err(e) => { + info!(&self.ctx.logger, "Failed ipfs.getBlock, returning `null`"; + "link" => asc_get::(self, link_ptr, gas)?, + "error" => e.to_string()); + Ok(AscPtr::null()) + } + } + } + + /// function ipfs.map(link: String, callback: String, flags: String[]): void + pub fn ipfs_map( + &mut self, + gas: &GasCounter, + link_ptr: AscPtr, + callback: AscPtr, + user_data: AscPtr>, + flags: AscPtr>>, + ) -> Result<(), HostExportError> { + // Note on gas: + // Ideally we would consume gas the same as ipfs_cat and then share + // gas across the spawned modules for callbacks. + + if !self.experimental_features.allow_non_deterministic_ipfs { + return Err(HostExportError::Deterministic(anyhow!( + "`ipfs.map` is deprecated. Improved support for IPFS will be added in the future" + ))); + } + + let link: String = asc_get(self, link_ptr, gas)?; + let callback: String = asc_get(self, callback, gas)?; + let user_data: store::Value = asc_get(self, user_data, gas)?; + + let flags = asc_get(self, flags, gas)?; + + // Pause the timeout while running ipfs_map, ensure it will be restarted by using a guard. + self.timeout_stopwatch.lock().unwrap().stop(); + let defer_stopwatch = self.timeout_stopwatch.clone(); + let _stopwatch_guard = defer::defer(|| defer_stopwatch.lock().unwrap().start()); + + let start_time = Instant::now(); + let output_states = HostExports::ipfs_map( + &self.ctx.host_exports.link_resolver.clone(), + self, + link.clone(), + &*callback, + user_data, + flags, + )?; + + debug!( + &self.ctx.logger, + "Successfully processed file with ipfs.map"; + "link" => &link, + "callback" => &*callback, + "n_calls" => output_states.len(), + "time" => format!("{}ms", start_time.elapsed().as_millis()) + ); + for output_state in output_states { + self.ctx.state.extend(output_state); + } + + Ok(()) + } + + /// Expects a decimal string. + /// function json.toI64(json: String): i64 + pub fn json_to_i64( + &mut self, + gas: &GasCounter, + json_ptr: AscPtr, + ) -> Result { + self.ctx + .host_exports + .json_to_i64(asc_get(self, json_ptr, gas)?, gas) + } + + /// Expects a decimal string. + /// function json.toU64(json: String): u64 + pub fn json_to_u64( + &mut self, + gas: &GasCounter, + json_ptr: AscPtr, + ) -> Result { + self.ctx + .host_exports + .json_to_u64(asc_get(self, json_ptr, gas)?, gas) + } + + /// Expects a decimal string. + /// function json.toF64(json: String): f64 + pub fn json_to_f64( + &mut self, + gas: &GasCounter, + json_ptr: AscPtr, + ) -> Result { + self.ctx + .host_exports + .json_to_f64(asc_get(self, json_ptr, gas)?, gas) + } + + /// Expects a decimal string. + /// function json.toBigInt(json: String): BigInt + pub fn json_to_big_int( + &mut self, + gas: &GasCounter, + json_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let big_int = self + .ctx + .host_exports + .json_to_big_int(asc_get(self, json_ptr, gas)?, gas)?; + asc_new(self, &*big_int, gas) + } + + /// function crypto.keccak256(input: Bytes): Bytes + pub fn crypto_keccak_256( + &mut self, + gas: &GasCounter, + input_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let input = self + .ctx + .host_exports + .crypto_keccak_256(asc_get(self, input_ptr, gas)?, gas)?; + asc_new(self, input.as_ref(), gas) + } + + /// function bigInt.plus(x: BigInt, y: BigInt): BigInt + pub fn big_int_plus( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + y_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self.ctx.host_exports.big_int_plus( + asc_get(self, x_ptr, gas)?, + asc_get(self, y_ptr, gas)?, + gas, + )?; + asc_new(self, &result, gas) + } + + /// function bigInt.minus(x: BigInt, y: BigInt): BigInt + pub fn big_int_minus( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + y_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self.ctx.host_exports.big_int_minus( + asc_get(self, x_ptr, gas)?, + asc_get(self, y_ptr, gas)?, + gas, + )?; + asc_new(self, &result, gas) + } + + /// function bigInt.times(x: BigInt, y: BigInt): BigInt + pub fn big_int_times( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + y_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self.ctx.host_exports.big_int_times( + asc_get(self, x_ptr, gas)?, + asc_get(self, y_ptr, gas)?, + gas, + )?; + asc_new(self, &result, gas) + } + + /// function bigInt.dividedBy(x: BigInt, y: BigInt): BigInt + pub fn big_int_divided_by( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + y_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self.ctx.host_exports.big_int_divided_by( + asc_get(self, x_ptr, gas)?, + asc_get(self, y_ptr, gas)?, + gas, + )?; + asc_new(self, &result, gas) + } + + /// function bigInt.dividedByDecimal(x: BigInt, y: BigDecimal): BigDecimal + pub fn big_int_divided_by_decimal( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + y_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let x = BigDecimal::new(asc_get(self, x_ptr, gas)?, 0); + let result = + self.ctx + .host_exports + .big_decimal_divided_by(x, asc_get(self, y_ptr, gas)?, gas)?; + asc_new(self, &result, gas) + } + + /// function bigInt.mod(x: BigInt, y: BigInt): BigInt + pub fn big_int_mod( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + y_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self.ctx.host_exports.big_int_mod( + asc_get(self, x_ptr, gas)?, + asc_get(self, y_ptr, gas)?, + gas, + )?; + asc_new(self, &result, gas) + } + + /// function bigInt.pow(x: BigInt, exp: u8): BigInt + pub fn big_int_pow( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + exp: u32, + ) -> Result, DeterministicHostError> { + let exp = u8::try_from(exp).map_err(|e| DeterministicHostError::from(Error::from(e)))?; + let result = self + .ctx + .host_exports + .big_int_pow(asc_get(self, x_ptr, gas)?, exp, gas)?; + asc_new(self, &result, gas) + } + + /// function bigInt.bitOr(x: BigInt, y: BigInt): BigInt + pub fn big_int_bit_or( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + y_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self.ctx.host_exports.big_int_bit_or( + asc_get(self, x_ptr, gas)?, + asc_get(self, y_ptr, gas)?, + gas, + )?; + asc_new(self, &result, gas) + } + + /// function bigInt.bitAnd(x: BigInt, y: BigInt): BigInt + pub fn big_int_bit_and( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + y_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self.ctx.host_exports.big_int_bit_and( + asc_get(self, x_ptr, gas)?, + asc_get(self, y_ptr, gas)?, + gas, + )?; + asc_new(self, &result, gas) + } + + /// function bigInt.leftShift(x: BigInt, bits: u8): BigInt + pub fn big_int_left_shift( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + bits: u32, + ) -> Result, DeterministicHostError> { + let bits = u8::try_from(bits).map_err(|e| DeterministicHostError::from(Error::from(e)))?; + let result = + self.ctx + .host_exports + .big_int_left_shift(asc_get(self, x_ptr, gas)?, bits, gas)?; + asc_new(self, &result, gas) + } + + /// function bigInt.rightShift(x: BigInt, bits: u8): BigInt + pub fn big_int_right_shift( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + bits: u32, + ) -> Result, DeterministicHostError> { + let bits = u8::try_from(bits).map_err(|e| DeterministicHostError::from(Error::from(e)))?; + let result = + self.ctx + .host_exports + .big_int_right_shift(asc_get(self, x_ptr, gas)?, bits, gas)?; + asc_new(self, &result, gas) + } + + /// function typeConversion.bytesToBase58(bytes: Bytes): string + pub fn bytes_to_base58( + &mut self, + gas: &GasCounter, + bytes_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self + .ctx + .host_exports + .bytes_to_base58(asc_get(self, bytes_ptr, gas)?, gas)?; + asc_new(self, &result, gas) + } + + /// function bigDecimal.toString(x: BigDecimal): string + pub fn big_decimal_to_string( + &mut self, + gas: &GasCounter, + big_decimal_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self + .ctx + .host_exports + .big_decimal_to_string(asc_get(self, big_decimal_ptr, gas)?, gas)?; + asc_new(self, &result, gas) + } + + /// function bigDecimal.fromString(x: string): BigDecimal + pub fn big_decimal_from_string( + &mut self, + gas: &GasCounter, + string_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self + .ctx + .host_exports + .big_decimal_from_string(asc_get(self, string_ptr, gas)?, gas)?; + asc_new(self, &result, gas) + } + + /// function bigDecimal.plus(x: BigDecimal, y: BigDecimal): BigDecimal + pub fn big_decimal_plus( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + y_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self.ctx.host_exports.big_decimal_plus( + asc_get(self, x_ptr, gas)?, + asc_get(self, y_ptr, gas)?, + gas, + )?; + asc_new(self, &result, gas) + } + + /// function bigDecimal.minus(x: BigDecimal, y: BigDecimal): BigDecimal + pub fn big_decimal_minus( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + y_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self.ctx.host_exports.big_decimal_minus( + asc_get(self, x_ptr, gas)?, + asc_get(self, y_ptr, gas)?, + gas, + )?; + asc_new(self, &result, gas) + } + + /// function bigDecimal.times(x: BigDecimal, y: BigDecimal): BigDecimal + pub fn big_decimal_times( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + y_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self.ctx.host_exports.big_decimal_times( + asc_get(self, x_ptr, gas)?, + asc_get(self, y_ptr, gas)?, + gas, + )?; + asc_new(self, &result, gas) + } + + /// function bigDecimal.dividedBy(x: BigDecimal, y: BigDecimal): BigDecimal + pub fn big_decimal_divided_by( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + y_ptr: AscPtr, + ) -> Result, DeterministicHostError> { + let result = self.ctx.host_exports.big_decimal_divided_by( + asc_get(self, x_ptr, gas)?, + asc_get(self, y_ptr, gas)?, + gas, + )?; + asc_new(self, &result, gas) + } + + /// function bigDecimal.equals(x: BigDecimal, y: BigDecimal): bool + pub fn big_decimal_equals( + &mut self, + gas: &GasCounter, + x_ptr: AscPtr, + y_ptr: AscPtr, + ) -> Result { + self.ctx.host_exports.big_decimal_equals( + asc_get(self, x_ptr, gas)?, + asc_get(self, y_ptr, gas)?, + gas, + ) + } + + /// function dataSource.create(name: string, params: Array): void + pub fn data_source_create( + &mut self, + gas: &GasCounter, + name_ptr: AscPtr, + params_ptr: AscPtr>>, + ) -> Result<(), HostExportError> { + let name: String = asc_get(self, name_ptr, gas)?; + let params: Vec = asc_get(self, params_ptr, gas)?; + self.ctx.host_exports.data_source_create( + &self.ctx.logger, + &mut self.ctx.state, + name, + params, + None, + self.ctx.block_ptr.number, + gas, + ) + } + + /// function createWithContext(name: string, params: Array, context: DataSourceContext): void + pub fn data_source_create_with_context( + &mut self, + gas: &GasCounter, + name_ptr: AscPtr, + params_ptr: AscPtr>>, + context_ptr: AscPtr, + ) -> Result<(), HostExportError> { + let name: String = asc_get(self, name_ptr, gas)?; + let params: Vec = asc_get(self, params_ptr, gas)?; + let context: HashMap<_, _> = asc_get(self, context_ptr, gas)?; + self.ctx.host_exports.data_source_create( + &self.ctx.logger, + &mut self.ctx.state, + name, + params, + Some(context.into()), + self.ctx.block_ptr.number, + gas, + ) + } + + /// function dataSource.address(): Bytes + pub fn data_source_address( + &mut self, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + asc_new( + self, + self.ctx.host_exports.data_source_address(gas)?.as_slice(), + gas, + ) + } + + /// function dataSource.network(): String + pub fn data_source_network( + &mut self, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + asc_new(self, &self.ctx.host_exports.data_source_network(gas)?, gas) + } + + /// function dataSource.context(): DataSourceContext + pub fn data_source_context( + &mut self, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + asc_new( + self, + &self.ctx.host_exports.data_source_context(gas)?.sorted(), + gas, + ) + } + + pub fn ens_name_by_hash( + &mut self, + gas: &GasCounter, + hash_ptr: AscPtr, + ) -> Result, HostExportError> { + // Not enabled on the network, no gas consumed. + drop(gas); + + // This is unrelated to IPFS, but piggyback on the config to disallow it on the network. + if !self.experimental_features.allow_non_deterministic_ipfs { + return Err(HostExportError::Deterministic(anyhow!( + "`ens_name_by_hash` is deprecated" + ))); + } + + let hash: String = asc_get(self, hash_ptr, gas)?; + let name = self.ctx.host_exports.ens_name_by_hash(&*hash)?; + + // map `None` to `null`, and `Some(s)` to a runtime string + name.map(|name| asc_new(self, &*name, gas).map_err(Into::into)) + .unwrap_or(Ok(AscPtr::null())) + } + + pub fn log_log( + &mut self, + gas: &GasCounter, + level: u32, + msg: AscPtr, + ) -> Result<(), DeterministicHostError> { + let level = LogLevel::from(level).into(); + let msg: String = asc_get(self, msg, gas)?; + self.ctx + .host_exports + .log_log(&self.ctx.logger, level, msg, gas) + } + + /// function encode(token: ethereum.Value): Bytes | null + pub fn ethereum_encode( + &mut self, + gas: &GasCounter, + token_ptr: AscPtr>, + ) -> Result, DeterministicHostError> { + let data = self + .ctx + .host_exports + .ethereum_encode(asc_get(self, token_ptr, gas)?, gas); + + // return `null` if it fails + data.map(|bytes| asc_new(self, &*bytes, gas)) + .unwrap_or(Ok(AscPtr::null())) + } + + /// function decode(types: String, data: Bytes): ethereum.Value | null + pub fn ethereum_decode( + &mut self, + gas: &GasCounter, + types_ptr: AscPtr, + data_ptr: AscPtr, + ) -> Result>, DeterministicHostError> { + let result = self.ctx.host_exports.ethereum_decode( + asc_get(self, types_ptr, gas)?, + asc_get(self, data_ptr, gas)?, + gas, + ); + + // return `null` if it fails + result + .map(|param| asc_new(self, ¶m, gas)) + .unwrap_or(Ok(AscPtr::null())) + } + + /// function arweave.transactionData(txId: string): Bytes | null + pub fn arweave_transaction_data( + &mut self, + _gas: &GasCounter, + _tx_id: AscPtr, + ) -> Result, HostExportError> { + Err(HostExportError::Deterministic(anyhow!( + "`arweave.transactionData` has been removed." + ))) + } + + /// function box.profile(address: string): JSONValue | null + pub fn box_profile( + &mut self, + _gas: &GasCounter, + _address: AscPtr, + ) -> Result, HostExportError> { + Err(HostExportError::Deterministic(anyhow!( + "`box.profile` has been removed." + ))) + } +} diff --git a/runtime/wasm/src/module/stopwatch.rs b/runtime/wasm/src/module/stopwatch.rs new file mode 100644 index 0000000..52d3b71 --- /dev/null +++ b/runtime/wasm/src/module/stopwatch.rs @@ -0,0 +1,58 @@ +// Copied from https://github.com/ellisonch/rust-stopwatch +// Copyright (c) 2014 Chucky Ellison under MIT license + +use std::default::Default; +use std::time::{Duration, Instant}; + +#[derive(Clone, Copy)] +pub struct TimeoutStopwatch { + /// The time the stopwatch was started last, if ever. + start_time: Option, + /// The time elapsed while the stopwatch was running (between start() and stop()). + pub elapsed: Duration, +} + +impl Default for TimeoutStopwatch { + fn default() -> TimeoutStopwatch { + TimeoutStopwatch { + start_time: None, + elapsed: Duration::from_secs(0), + } + } +} + +impl TimeoutStopwatch { + /// Returns a new stopwatch. + pub fn new() -> TimeoutStopwatch { + let sw: TimeoutStopwatch = Default::default(); + sw + } + + /// Returns a new stopwatch which will immediately be started. + pub fn start_new() -> TimeoutStopwatch { + let mut sw = TimeoutStopwatch::new(); + sw.start(); + sw + } + + /// Starts the stopwatch. + pub fn start(&mut self) { + self.start_time = Some(Instant::now()); + } + + /// Stops the stopwatch. + pub fn stop(&mut self) { + self.elapsed = self.elapsed(); + self.start_time = None; + } + + /// Returns the elapsed time since the start of the stopwatch. + pub fn elapsed(&self) -> Duration { + match self.start_time { + // stopwatch is running + Some(t1) => t1.elapsed() + self.elapsed, + // stopwatch is not running + None => self.elapsed, + } + } +} diff --git a/runtime/wasm/src/to_from/external.rs b/runtime/wasm/src/to_from/external.rs new file mode 100644 index 0000000..96e99d7 --- /dev/null +++ b/runtime/wasm/src/to_from/external.rs @@ -0,0 +1,411 @@ +use ethabi; + +use graph::prelude::{BigDecimal, BigInt}; +use graph::runtime::gas::GasCounter; +use graph::runtime::{asc_get, asc_new, AscIndexId, AscPtr, AscType, AscValue, ToAscObj}; +use graph::{data::store, runtime::DeterministicHostError}; +use graph::{prelude::serde_json, runtime::FromAscObj}; +use graph::{prelude::web3::types as web3, runtime::AscHeap}; + +use crate::asc_abi::class::*; + +impl ToAscObj for web3::H160 { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + self.0.to_asc_obj(heap, gas) + } +} + +impl FromAscObj for web3::H160 { + fn from_asc_obj( + typed_array: Uint8Array, + heap: &H, + gas: &GasCounter, + ) -> Result { + let data = <[u8; 20]>::from_asc_obj(typed_array, heap, gas)?; + Ok(Self(data)) + } +} + +impl FromAscObj for web3::H256 { + fn from_asc_obj( + typed_array: Uint8Array, + heap: &H, + gas: &GasCounter, + ) -> Result { + let data = <[u8; 32]>::from_asc_obj(typed_array, heap, gas)?; + Ok(Self(data)) + } +} + +impl ToAscObj for web3::H256 { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + self.0.to_asc_obj(heap, gas) + } +} + +impl ToAscObj for web3::U128 { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let mut bytes: [u8; 16] = [0; 16]; + self.to_little_endian(&mut bytes); + bytes.to_asc_obj(heap, gas) + } +} + +impl ToAscObj for BigInt { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let bytes = self.to_signed_bytes_le(); + bytes.to_asc_obj(heap, gas) + } +} + +impl FromAscObj for BigInt { + fn from_asc_obj( + array_buffer: AscBigInt, + heap: &H, + gas: &GasCounter, + ) -> Result { + let bytes = >::from_asc_obj(array_buffer, heap, gas)?; + Ok(BigInt::from_signed_bytes_le(&bytes)) + } +} + +impl ToAscObj for BigDecimal { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + // From the docs: "Note that a positive exponent indicates a negative power of 10", + // so "exponent" is the opposite of what you'd expect. + let (digits, negative_exp) = self.as_bigint_and_exponent(); + Ok(AscBigDecimal { + exp: asc_new(heap, &BigInt::from(-negative_exp), gas)?, + digits: asc_new(heap, &BigInt::from(digits), gas)?, + }) + } +} + +impl FromAscObj for BigDecimal { + fn from_asc_obj( + big_decimal: AscBigDecimal, + heap: &H, + gas: &GasCounter, + ) -> Result { + let digits: BigInt = asc_get(heap, big_decimal.digits, gas)?; + let exp: BigInt = asc_get(heap, big_decimal.exp, gas)?; + + let bytes = exp.to_signed_bytes_le(); + let mut byte_array = if exp >= 0.into() { [0; 8] } else { [255; 8] }; + byte_array[..bytes.len()].copy_from_slice(&bytes); + let big_decimal = BigDecimal::new(digits, i64::from_le_bytes(byte_array)); + + // Validate the exponent. + let exp = -big_decimal.as_bigint_and_exponent().1; + let min_exp: i64 = BigDecimal::MIN_EXP.into(); + let max_exp: i64 = BigDecimal::MAX_EXP.into(); + if exp < min_exp || max_exp < exp { + Err(DeterministicHostError::from(anyhow::anyhow!( + "big decimal exponent `{}` is outside the `{}` to `{}` range", + exp, + min_exp, + max_exp + ))) + } else { + Ok(big_decimal) + } + } +} + +impl ToAscObj>> for Vec { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result>, DeterministicHostError> { + let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); + let content = content?; + Ok(Array::new(&*content, heap, gas)?) + } +} + +impl ToAscObj> for ethabi::Token { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + use ethabi::Token::*; + + let kind = EthereumValueKind::get_kind(self); + let payload = match self { + Address(address) => asc_new::(heap, address, gas)?.to_payload(), + FixedBytes(bytes) | Bytes(bytes) => { + asc_new::(heap, &**bytes, gas)?.to_payload() + } + Int(uint) => { + let n = BigInt::from_signed_u256(&uint); + asc_new(heap, &n, gas)?.to_payload() + } + Uint(uint) => { + let n = BigInt::from_unsigned_u256(&uint); + asc_new(heap, &n, gas)?.to_payload() + } + Bool(b) => *b as u64, + String(string) => asc_new(heap, &**string, gas)?.to_payload(), + FixedArray(tokens) | Array(tokens) => asc_new(heap, &**tokens, gas)?.to_payload(), + Tuple(tokens) => asc_new(heap, &**tokens, gas)?.to_payload(), + }; + + Ok(AscEnum { + kind, + _padding: 0, + payload: EnumPayload(payload), + }) + } +} + +impl FromAscObj> for ethabi::Token { + fn from_asc_obj( + asc_enum: AscEnum, + heap: &H, + gas: &GasCounter, + ) -> Result { + use ethabi::Token; + + let payload = asc_enum.payload; + Ok(match asc_enum.kind { + EthereumValueKind::Bool => Token::Bool(bool::from(payload)), + EthereumValueKind::Address => { + let ptr: AscPtr = AscPtr::from(payload); + Token::Address(asc_get(heap, ptr, gas)?) + } + EthereumValueKind::FixedBytes => { + let ptr: AscPtr = AscPtr::from(payload); + Token::FixedBytes(asc_get(heap, ptr, gas)?) + } + EthereumValueKind::Bytes => { + let ptr: AscPtr = AscPtr::from(payload); + Token::Bytes(asc_get(heap, ptr, gas)?) + } + EthereumValueKind::Int => { + let ptr: AscPtr = AscPtr::from(payload); + let n: BigInt = asc_get(heap, ptr, gas)?; + Token::Int(n.to_signed_u256()) + } + EthereumValueKind::Uint => { + let ptr: AscPtr = AscPtr::from(payload); + let n: BigInt = asc_get(heap, ptr, gas)?; + Token::Uint(n.to_unsigned_u256()) + } + EthereumValueKind::String => { + let ptr: AscPtr = AscPtr::from(payload); + Token::String(asc_get(heap, ptr, gas)?) + } + EthereumValueKind::FixedArray => { + let ptr: AscEnumArray = AscPtr::from(payload); + Token::FixedArray(asc_get(heap, ptr, gas)?) + } + EthereumValueKind::Array => { + let ptr: AscEnumArray = AscPtr::from(payload); + Token::Array(asc_get(heap, ptr, gas)?) + } + EthereumValueKind::Tuple => { + let ptr: AscEnumArray = AscPtr::from(payload); + Token::Tuple(asc_get(heap, ptr, gas)?) + } + }) + } +} + +impl FromAscObj> for store::Value { + fn from_asc_obj( + asc_enum: AscEnum, + heap: &H, + gas: &GasCounter, + ) -> Result { + use self::store::Value; + + let payload = asc_enum.payload; + Ok(match asc_enum.kind { + StoreValueKind::String => { + let ptr: AscPtr = AscPtr::from(payload); + Value::String(asc_get(heap, ptr, gas)?) + } + StoreValueKind::Int => Value::Int(i32::from(payload)), + StoreValueKind::BigDecimal => { + let ptr: AscPtr = AscPtr::from(payload); + Value::BigDecimal(asc_get(heap, ptr, gas)?) + } + StoreValueKind::Bool => Value::Bool(bool::from(payload)), + StoreValueKind::Array => { + let ptr: AscEnumArray = AscPtr::from(payload); + Value::List(asc_get(heap, ptr, gas)?) + } + StoreValueKind::Null => Value::Null, + StoreValueKind::Bytes => { + let ptr: AscPtr = AscPtr::from(payload); + let array: Vec = asc_get(heap, ptr, gas)?; + Value::Bytes(array.as_slice().into()) + } + StoreValueKind::BigInt => { + let ptr: AscPtr = AscPtr::from(payload); + let array: Vec = asc_get(heap, ptr, gas)?; + Value::BigInt(store::scalar::BigInt::from_signed_bytes_le(&array)) + } + }) + } +} + +impl ToAscObj> for store::Value { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + use self::store::Value; + + let payload = match self { + Value::String(string) => asc_new(heap, string.as_str(), gas)?.into(), + Value::Int(n) => EnumPayload::from(*n), + Value::BigDecimal(n) => asc_new(heap, n, gas)?.into(), + Value::Bool(b) => EnumPayload::from(*b), + Value::List(array) => asc_new(heap, array.as_slice(), gas)?.into(), + Value::Null => EnumPayload(0), + Value::Bytes(bytes) => { + let bytes_obj: AscPtr = asc_new(heap, bytes.as_slice(), gas)?; + bytes_obj.into() + } + Value::BigInt(big_int) => { + let bytes_obj: AscPtr = + asc_new(heap, &*big_int.to_signed_bytes_le(), gas)?; + bytes_obj.into() + } + }; + + Ok(AscEnum { + kind: StoreValueKind::get_kind(self), + _padding: 0, + payload, + }) + } +} + +impl ToAscObj for serde_json::Map { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscTypedMap { + entries: asc_new(heap, &*self.iter().collect::>(), gas)?, + }) + } +} + +// Used for serializing entities. +impl ToAscObj for Vec<(String, store::Value)> { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + Ok(AscTypedMap { + entries: asc_new(heap, self.as_slice(), gas)?, + }) + } +} + +impl ToAscObj> for serde_json::Value { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + use serde_json::Value; + + let payload = match self { + Value::Null => EnumPayload(0), + Value::Bool(b) => EnumPayload::from(*b), + Value::Number(number) => asc_new(heap, &*number.to_string(), gas)?.into(), + Value::String(string) => asc_new(heap, string.as_str(), gas)?.into(), + Value::Array(array) => asc_new(heap, array.as_slice(), gas)?.into(), + Value::Object(object) => asc_new(heap, object, gas)?.into(), + }; + + Ok(AscEnum { + kind: JsonValueKind::get_kind(self), + _padding: 0, + payload, + }) + } +} + +impl From for LogLevel { + fn from(i: u32) -> Self { + match i { + 0 => LogLevel::Critical, + 1 => LogLevel::Error, + 2 => LogLevel::Warning, + 3 => LogLevel::Info, + 4 => LogLevel::Debug, + _ => LogLevel::Debug, + } + } +} + +impl ToAscObj> for AscWrapped { + fn to_asc_obj( + &self, + _heap: &mut H, + _gas: &GasCounter, + ) -> Result, DeterministicHostError> { + Ok(*self) + } +} + +impl ToAscObj, bool>> for Result +where + V: ToAscObj, + VAsc: AscType + AscIndexId, + AscWrapped>: AscIndexId, +{ + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, bool>, DeterministicHostError> { + Ok(match self { + Ok(value) => AscResult { + value: { + let inner = asc_new(heap, value, gas)?; + let wrapped = AscWrapped { inner }; + asc_new(heap, &wrapped, gas)? + }, + error: AscPtr::null(), + }, + Err(_) => AscResult { + value: AscPtr::null(), + error: { + let wrapped = AscWrapped { inner: true }; + asc_new(heap, &wrapped, gas)? + }, + }, + }) + } +} diff --git a/runtime/wasm/src/to_from/mod.rs b/runtime/wasm/src/to_from/mod.rs new file mode 100644 index 0000000..334f74c --- /dev/null +++ b/runtime/wasm/src/to_from/mod.rs @@ -0,0 +1,161 @@ +use anyhow::anyhow; +use std::collections::HashMap; +use std::hash::Hash; +use std::iter::FromIterator; + +use graph::runtime::{ + asc_get, asc_new, gas::GasCounter, AscHeap, AscIndexId, AscPtr, AscType, AscValue, + DeterministicHostError, FromAscObj, ToAscObj, +}; + +use crate::asc_abi::class::*; + +///! Implementations of `ToAscObj` and `FromAscObj` for Rust types. +///! Standard Rust types go in `mod.rs` and external types in `external.rs`. +mod external; + +impl ToAscObj> for [T] { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + TypedArray::new(self, heap, gas) + } +} + +impl FromAscObj> for Vec { + fn from_asc_obj( + typed_array: TypedArray, + heap: &H, + gas: &GasCounter, + ) -> Result { + typed_array.to_vec(heap, gas) + } +} + +impl FromAscObj> for [T; LEN] { + fn from_asc_obj( + typed_array: TypedArray, + heap: &H, + gas: &GasCounter, + ) -> Result { + let v = typed_array.to_vec(heap, gas)?; + let array = <[T; LEN]>::try_from(v) + .map_err(|v| anyhow!("expected array of length {}, found length {}", LEN, v.len()))?; + Ok(array) + } +} + +impl ToAscObj for str { + fn to_asc_obj( + &self, + heap: &mut H, + _gas: &GasCounter, + ) -> Result { + AscString::new(&self.encode_utf16().collect::>(), heap.api_version()) + } +} + +impl ToAscObj for String { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + self.as_str().to_asc_obj(heap, gas) + } +} + +impl FromAscObj for String { + fn from_asc_obj( + asc_string: AscString, + _: &H, + _gas: &GasCounter, + ) -> Result { + let mut string = String::from_utf16(asc_string.content()) + .map_err(|e| DeterministicHostError::from(anyhow::Error::from(e)))?; + + // Strip null characters since they are not accepted by Postgres. + if string.contains('\u{0000}') { + string = string.replace("\u{0000}", ""); + } + Ok(string) + } +} + +impl> ToAscObj>> for [T] { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result>, DeterministicHostError> { + let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); + let content = content?; + Array::new(&*content, heap, gas) + } +} + +impl> FromAscObj>> for Vec { + fn from_asc_obj( + array: Array>, + heap: &H, + gas: &GasCounter, + ) -> Result { + array + .to_vec(heap, gas)? + .into_iter() + .map(|x| asc_get(heap, x, gas)) + .collect() + } +} + +impl, U: FromAscObj> + FromAscObj> for (T, U) +{ + fn from_asc_obj( + asc_entry: AscTypedMapEntry, + heap: &H, + gas: &GasCounter, + ) -> Result { + Ok(( + asc_get(heap, asc_entry.key, gas)?, + asc_get(heap, asc_entry.value, gas)?, + )) + } +} + +impl, U: ToAscObj> + ToAscObj> for (T, U) +{ + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, DeterministicHostError> { + Ok(AscTypedMapEntry { + key: asc_new(heap, &self.0, gas)?, + value: asc_new(heap, &self.1, gas)?, + }) + } +} + +impl< + K: AscType + AscIndexId, + V: AscType + AscIndexId, + T: FromAscObj + Hash + Eq, + U: FromAscObj, + > FromAscObj> for HashMap +where + Array>>: AscIndexId, + AscTypedMapEntry: AscIndexId, +{ + fn from_asc_obj( + asc_map: AscTypedMap, + heap: &H, + gas: &GasCounter, + ) -> Result { + let entries: Vec<(T, U)> = asc_get(heap, asc_map.entries, gas)?; + Ok(HashMap::from_iter(entries.into_iter())) + } +} diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..0786d33 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,39 @@ +# scripts + +## update-cargo-version.sh + +This script helps to update the versions of all `Cargo.toml` files and the `Cargo.lock` one. + +### Functionality + +It does more than what's listed below, but this represents the main behavior of it. + +1. Asserts that currently all `Cargo.toml` files in the repository have the **same version**; +2. Changes all of the crates to use the new provided version: `patch`, `minor` or `major`; +3. Updates the `Cargo.lock` via `cargo check --tests`; +4. Adds the changes in a `Release X.Y.Z` commit. + +### Usage + +The only argument it accepts is the type of version you want to do `(patch|minor|major)`. + +```bash +./scripts/update-cargo-version.sh patch +``` + +Example output: + +``` +Current version: "0.25.1" +New version: "0.25.2" +Changing 18 toml files +Toml files are still consistent in their version after the update +Updating Cargo.lock file + Finished dev [unoptimized + debuginfo] target(s) in 0.58s +Cargo.lock file updated +Updating version of the Cargo.{lock, toml} files succeded! +[otavio/update-news-0-25-2 2f2175bae] Release 0.25.2 + 20 files changed, 38 insertions(+), 38 deletions(-) +``` + +This script contains several assertions to make sure no mistake has been made. Unfortunately for now we don't have a way to revert it, or to recover from an error when it fails in the middle of it, this can be improved in the future. diff --git a/scripts/abort.sh b/scripts/abort.sh new file mode 100644 index 0000000..ed47868 --- /dev/null +++ b/scripts/abort.sh @@ -0,0 +1,6 @@ +abort () { + local ERROR_MESSAGE=$1 + echo "Release failed, error message:" + echo $ERROR_MESSAGE + exit 1 +} diff --git a/scripts/lines-unique.sh b/scripts/lines-unique.sh new file mode 100755 index 0000000..186ea75 --- /dev/null +++ b/scripts/lines-unique.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +source 'scripts/abort.sh' + +# Exits with code 0 if lines are unique, 1 otherwise +LINES=$@ + +# Validate that parameters are being sent +[ -z "$LINES" ] && abort "No lines received" + +if [ $(echo $LINES | tr " " "\n" | sort | uniq | wc -l) -eq 1 ]; then + exit 0 +else + exit 1 +fi diff --git a/scripts/toml-utils.sh b/scripts/toml-utils.sh new file mode 100755 index 0000000..8687206 --- /dev/null +++ b/scripts/toml-utils.sh @@ -0,0 +1,19 @@ +# Get all files named 'Cargo.toml' in the `graph-node` directory, excluding the `integration-tests` folder. +get_all_toml_files () { + echo "$(find . -name Cargo.toml | grep -v integration-tests)" +} + +get_all_toml_versions () { + local FILE_NAMES=$@ + echo $( + echo $FILE_NAMES | \ + # Read all 'Cargo.toml' file contents. + xargs cat | \ + # Get the 'version' key of the TOML, eg: version = "0.25.2" + grep '^version = ' | \ + # Remove the '"' enclosing the version, eg: "0.25.2" + tr -d '"' | \ + # Get only the version number, eg: 0.25.2 + awk '{print $3}' \ + ) +} diff --git a/scripts/update-cargo-version.sh b/scripts/update-cargo-version.sh new file mode 100755 index 0000000..1471d0a --- /dev/null +++ b/scripts/update-cargo-version.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +source 'scripts/abort.sh' +source 'scripts/toml-utils.sh' + +ALL_TOML_FILE_NAMES=$(get_all_toml_files) +ALL_TOML_VERSIONS=$(get_all_toml_versions $ALL_TOML_FILE_NAMES) + +# Asserts all .toml versions are currently the same, otherwise abort. +if ./scripts/lines-unique.sh $ALL_TOML_VERSIONS; then + CURRENT_VERSION=$(echo $ALL_TOML_VERSIONS | awk '{print $1}') + echo "Current version: \"$CURRENT_VERSION\"" +else + abort "Some Cargo.toml files have different versions than others, make sure they're all the same before creating a new release" +fi + + +# Increment by CLI argument (major, minor, patch) +MAJOR=$(echo $CURRENT_VERSION | cut -d. -f1) +MINOR=$(echo $CURRENT_VERSION | cut -d. -f2) +PATCH=$(echo $CURRENT_VERSION | cut -d. -f3) + +case $1 in + "major") + let "MAJOR++" + MINOR=0 + PATCH=0 + ;; + "minor") + let "MINOR++" + PATCH=0 + ;; + "patch") + let "PATCH++" + ;; + *) + abort "Version argument should be one of: major, minor or patch" + ;; +esac + +NEW_VERSION="${MAJOR}.${MINOR}.${PATCH}" +echo "New version: \"$NEW_VERSION\"" + + +# Replace on all .toml files +echo "Changing $(echo $ALL_TOML_VERSIONS | tr " " "\n" | wc -l | awk '{$1=$1;print}') toml files" + +# MacOS/OSX unfortunately doesn't have the same API for `sed`, so we're +# using `perl` instead since it's installed by default in all Mac machines. +# +# For mor info: https://stackoverflow.com/questions/4247068/sed-command-with-i-option-failing-on-mac-but-works-on-linux +if [[ "$OSTYPE" == "darwin"* ]]; then + perl -i -pe"s/^version = \"${CURRENT_VERSION}\"/version = \"${NEW_VERSION}\"/" $ALL_TOML_FILE_NAMES +# Default, for decent OSs (eg: GNU-Linux) +else + sed -i "s/^version = \"${CURRENT_VERSION}\"/version = \"${NEW_VERSION}\"/" $ALL_TOML_FILE_NAMES +fi + + +# Assert all the new .toml versions are the same, otherwise abort +UPDATED_TOML_VERSIONS=$(get_all_toml_versions $ALL_TOML_FILE_NAMES) +if ./scripts/lines-unique.sh $UPDATED_TOML_VERSIONS; then + echo "Toml files are still consistent in their version after the update" +else + abort "Something went wrong with the version replacement and the new version isn't the same across the Cargo.toml files" +fi + + +# Assert there was a git diff in the changed files, otherwise abort +if [[ $(git diff $ALL_TOML_FILE_NAMES) ]]; then + : +else + abort "Somehow the toml files didn't get changed" +fi + + +echo "Updating Cargo.lock file" +cargo check --tests + + +# Assert .lock file changed the versions, otherwise abort +if [[ $(git diff Cargo.lock) ]]; then + echo "Cargo.lock file updated" +else + abort "There was no change in the Cargo.lock file, something went wrong with updating the crates versions" +fi + + +echo "Updating version of the Cargo.{lock, toml} files succeded!" + +git add Cargo.lock $ALL_TOML_FILE_NAMES +git commit -m "Release ${NEW_VERSION}" diff --git a/server/http/Cargo.toml b/server/http/Cargo.toml new file mode 100644 index 0000000..34387f4 --- /dev/null +++ b/server/http/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "graph-server-http" +version = "0.27.0" +edition = "2021" + +[dependencies] +futures = "0.1.21" +graphql-parser = "0.4.0" +http = "0.2" +hyper = "0.14" +serde = "1.0" +graph = { path = "../../graph" } +graph-graphql = { path = "../../graphql" } + +[dev-dependencies] +graph-mock = { path = "../../mock" } diff --git a/server/http/assets/index.html b/server/http/assets/index.html new file mode 100644 index 0000000..7b049ea --- /dev/null +++ b/server/http/assets/index.html @@ -0,0 +1,87052 @@ + + + + + The GraphiQL + + + + +

+ + + + + + + diff --git a/server/http/src/lib.rs b/server/http/src/lib.rs new file mode 100644 index 0000000..c5621bc --- /dev/null +++ b/server/http/src/lib.rs @@ -0,0 +1,18 @@ +extern crate futures; +extern crate graph; +extern crate graph_graphql; +#[cfg(test)] +extern crate graph_mock; +extern crate graphql_parser; +extern crate http; +extern crate hyper; +extern crate serde; + +mod request; +mod server; +mod service; + +pub use self::server::GraphQLServer; +pub use self::service::{GraphQLService, GraphQLServiceResponse}; + +pub mod test_utils; diff --git a/server/http/src/request.rs b/server/http/src/request.rs new file mode 100644 index 0000000..05a7b08 --- /dev/null +++ b/server/http/src/request.rs @@ -0,0 +1,169 @@ +use graph::prelude::serde_json; +use hyper::body::Bytes; + +use graph::components::server::query::GraphQLServerError; +use graph::prelude::*; + +pub fn parse_graphql_request(body: &Bytes) -> Result { + // Parse request body as JSON + let json: serde_json::Value = serde_json::from_slice(body) + .map_err(|e| GraphQLServerError::ClientError(format!("{}", e)))?; + + // Ensure the JSON data is an object + let obj = json.as_object().ok_or_else(|| { + GraphQLServerError::ClientError(String::from("Request data is not an object")) + })?; + + // Ensure the JSON data has a "query" field + let query_value = obj.get("query").ok_or_else(|| { + GraphQLServerError::ClientError(String::from( + "The \"query\" field is missing in request data", + )) + })?; + + // Ensure the "query" field is a string + let query_string = query_value.as_str().ok_or_else(|| { + GraphQLServerError::ClientError(String::from("The \"query\" field is not a string")) + })?; + + // Parse the "query" field of the JSON body + let document = graphql_parser::parse_query(query_string) + .map_err(|e| GraphQLServerError::from(QueryError::ParseError(Arc::new(e.into()))))? + .into_static(); + + // Parse the "variables" field of the JSON body, if present + let variables = match obj.get("variables") { + None | Some(serde_json::Value::Null) => Ok(None), + Some(variables @ serde_json::Value::Object(_)) => serde_json::from_value(variables.clone()) + .map_err(|e| GraphQLServerError::ClientError(e.to_string())) + .map(Some), + _ => Err(GraphQLServerError::ClientError( + "Invalid query variables provided".to_string(), + )), + }?; + + Ok(Query::new(document, variables)) +} + +#[cfg(test)] +mod tests { + use graphql_parser; + use hyper; + use std::collections::HashMap; + + use graph::{ + data::{query::QueryTarget, value::Object}, + prelude::*, + }; + + use super::parse_graphql_request; + + lazy_static! { + static ref TARGET: QueryTarget = QueryTarget::Name( + SubgraphName::new("test/request").unwrap(), + Default::default() + ); + } + + #[test] + fn rejects_invalid_json() { + let request = parse_graphql_request(&hyper::body::Bytes::from("!@#)%")); + request.expect_err("Should reject invalid JSON"); + } + + #[test] + fn rejects_json_without_query_field() { + let request = parse_graphql_request(&hyper::body::Bytes::from("{}")); + request.expect_err("Should reject JSON without query field"); + } + + #[test] + fn rejects_json_with_non_string_query_field() { + let request = parse_graphql_request(&hyper::body::Bytes::from("{\"query\": 5}")); + request.expect_err("Should reject JSON with a non-string query field"); + } + + #[test] + fn rejects_broken_queries() { + let request = parse_graphql_request(&hyper::body::Bytes::from("{\"query\": \"foo\"}")); + request.expect_err("Should reject broken queries"); + } + + #[test] + fn accepts_valid_queries() { + let request = parse_graphql_request(&hyper::body::Bytes::from( + "{\"query\": \"{ user { name } }\"}", + )); + let query = request.expect("Should accept valid queries"); + assert_eq!( + query.document, + graphql_parser::parse_query("{ user { name } }") + .unwrap() + .into_static() + ); + } + + #[test] + fn accepts_null_variables() { + let request = parse_graphql_request(&hyper::body::Bytes::from( + "\ + {\ + \"query\": \"{ user { name } }\", \ + \"variables\": null \ + }", + )); + let query = request.expect("Should accept null variables"); + + let expected_query = graphql_parser::parse_query("{ user { name } }") + .unwrap() + .into_static(); + assert_eq!(query.document, expected_query); + assert_eq!(query.variables, None); + } + + #[test] + fn rejects_non_map_variables() { + let request = parse_graphql_request(&hyper::body::Bytes::from( + "\ + {\ + \"query\": \"{ user { name } }\", \ + \"variables\": 5 \ + }", + )); + request.expect_err("Should reject non-map variables"); + } + + #[test] + fn parses_variables() { + let request = parse_graphql_request(&hyper::body::Bytes::from( + "\ + {\ + \"query\": \"{ user { name } }\", \ + \"variables\": { \ + \"string\": \"s\", \"map\": {\"k\": \"v\"}, \"int\": 5 \ + } \ + }", + )); + let query = request.expect("Should accept valid queries"); + + let expected_query = graphql_parser::parse_query("{ user { name } }") + .unwrap() + .into_static(); + let expected_variables = QueryVariables::new(HashMap::from_iter( + vec![ + (String::from("string"), r::Value::String(String::from("s"))), + ( + String::from("map"), + r::Value::Object(Object::from_iter( + vec![(String::from("k"), r::Value::String(String::from("v")))].into_iter(), + )), + ), + (String::from("int"), r::Value::Int(5)), + ] + .into_iter(), + )); + + assert_eq!(query.document, expected_query); + assert_eq!(query.variables, Some(expected_variables)); + } +} diff --git a/server/http/src/server.rs b/server/http/src/server.rs new file mode 100644 index 0000000..a99e8ba --- /dev/null +++ b/server/http/src/server.rs @@ -0,0 +1,84 @@ +use std::net::{Ipv4Addr, SocketAddrV4}; + +use hyper::service::make_service_fn; +use hyper::Server; + +use crate::service::GraphQLService; +use graph::prelude::{GraphQLServer as GraphQLServerTrait, *}; +use thiserror::Error; + +/// Errors that may occur when starting the server. +#[derive(Debug, Error)] +pub enum GraphQLServeError { + #[error("Bind error: {0}")] + BindError(#[from] hyper::Error), +} + +/// A GraphQL server based on Hyper. +pub struct GraphQLServer { + logger: Logger, + graphql_runner: Arc, + node_id: NodeId, +} + +impl GraphQLServer { + /// Creates a new GraphQL server. + pub fn new(logger_factory: &LoggerFactory, graphql_runner: Arc, node_id: NodeId) -> Self { + let logger = logger_factory.component_logger( + "GraphQLServer", + Some(ComponentLoggerConfig { + elastic: Some(ElasticComponentLoggerConfig { + index: String::from("graphql-server-logs"), + }), + }), + ); + GraphQLServer { + logger, + graphql_runner, + node_id, + } + } +} + +impl GraphQLServerTrait for GraphQLServer +where + Q: GraphQlRunner, +{ + type ServeError = GraphQLServeError; + + fn serve( + &mut self, + port: u16, + ws_port: u16, + ) -> Result + Send>, Self::ServeError> { + let logger = self.logger.clone(); + + info!( + logger, + "Starting GraphQL HTTP server at: http://localhost:{}", port + ); + + let addr = SocketAddrV4::new(Ipv4Addr::new(0, 0, 0, 0), port); + + // On every incoming request, launch a new GraphQL service that writes + // incoming queries to the query sink. + let logger_for_service = self.logger.clone(); + let graphql_runner = self.graphql_runner.clone(); + let node_id = self.node_id.clone(); + let new_service = make_service_fn(move |_| { + futures03::future::ok::<_, Error>(GraphQLService::new( + logger_for_service.clone(), + graphql_runner.clone(), + ws_port, + node_id.clone(), + )) + }); + + // Create a task to run the server and handle HTTP requests + let task = Server::try_bind(&addr.into())? + .serve(new_service) + .map_err(move |e| error!(logger, "Server error"; "error" => format!("{}", e))); + + Ok(Box::new(task.compat())) + } +} diff --git a/server/http/src/service.rs b/server/http/src/service.rs new file mode 100644 index 0000000..cfc5604 --- /dev/null +++ b/server/http/src/service.rs @@ -0,0 +1,480 @@ +use std::convert::TryFrom; +use std::pin::Pin; +use std::task::Context; +use std::task::Poll; +use std::time::Instant; + +use graph::prelude::*; +use graph::semver::VersionReq; +use graph::{components::server::query::GraphQLServerError, data::query::QueryTarget}; +use http::header; +use http::header::{ + ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, + CONTENT_TYPE, LOCATION, +}; +use hyper::service::Service; +use hyper::{Body, Method, Request, Response, StatusCode}; + +use crate::request::parse_graphql_request; + +pub type GraphQLServiceResult = Result, GraphQLServerError>; +/// An asynchronous response to a GraphQL request. +pub type GraphQLServiceResponse = + Pin + Send>>; + +/// A Hyper Service that serves GraphQL over a POST / endpoint. +#[derive(Debug)] +pub struct GraphQLService { + logger: Logger, + graphql_runner: Arc, + ws_port: u16, + node_id: NodeId, +} + +impl Clone for GraphQLService { + fn clone(&self) -> Self { + Self { + logger: self.logger.clone(), + graphql_runner: self.graphql_runner.clone(), + ws_port: self.ws_port, + node_id: self.node_id.clone(), + } + } +} + +impl GraphQLService +where + Q: GraphQlRunner, +{ + /// Creates a new GraphQL service. + pub fn new(logger: Logger, graphql_runner: Arc, ws_port: u16, node_id: NodeId) -> Self { + GraphQLService { + logger, + graphql_runner, + ws_port, + node_id, + } + } + + fn graphiql_html(&self) -> String { + include_str!("../assets/index.html") + .replace("__WS_PORT__", format!("{}", self.ws_port).as_str()) + } + + async fn index(self) -> GraphQLServiceResult { + Ok(Response::builder() + .status(200) + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .header(CONTENT_TYPE, "text/plain") + .body(Body::from(String::from( + "Access deployed subgraphs by deployment ID at \ + /subgraphs/id/ or by name at /subgraphs/name/", + ))) + .unwrap()) + } + + /// Serves a dynamically created file. + fn serve_dynamic_file(&self, contents: String) -> GraphQLServiceResponse { + async { + Ok(Response::builder() + .status(200) + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .header(CONTENT_TYPE, "text/html") + .body(Body::from(contents)) + .unwrap()) + } + .boxed() + } + + fn handle_graphiql(&self) -> GraphQLServiceResponse { + self.serve_dynamic_file(self.graphiql_html()) + } + + fn resolve_api_version( + &self, + request: &Request, + ) -> Result { + let mut version = ApiVersion::default(); + + if let Some(query) = request.uri().query() { + let potential_version_requirement = query.split("&").find_map(|pair| { + if pair.starts_with("api-version=") { + if let Some(version_requirement) = pair.split("=").nth(1) { + return Some(version_requirement); + } + } + return None; + }); + + if let Some(version_requirement) = potential_version_requirement { + version = ApiVersion::new( + &VersionReq::parse(version_requirement) + .map_err(|error| GraphQLServerError::ClientError(error.to_string()))?, + ) + .map_err(|error| GraphQLServerError::ClientError(error))?; + } + } + + Ok(version) + } + + async fn handle_graphql_query_by_name( + self, + subgraph_name: String, + request: Request, + ) -> GraphQLServiceResult { + let version = self.resolve_api_version(&request)?; + let subgraph_name = SubgraphName::new(subgraph_name.as_str()).map_err(|()| { + GraphQLServerError::ClientError(format!("Invalid subgraph name {:?}", subgraph_name)) + })?; + + self.handle_graphql_query( + QueryTarget::Name(subgraph_name, version), + request.into_body(), + ) + .await + } + + fn handle_graphql_query_by_id( + self, + id: String, + request: Request, + ) -> GraphQLServiceResponse { + let res = DeploymentHash::new(id) + .map_err(|id| GraphQLServerError::ClientError(format!("Invalid subgraph id `{}`", id))) + .and_then(|id| match self.resolve_api_version(&request) { + Ok(version) => Ok((id, version)), + Err(error) => Err(error), + }); + + match res { + Err(_) => self.handle_not_found(), + Ok((id, version)) => self + .handle_graphql_query(QueryTarget::Deployment(id, version), request.into_body()) + .boxed(), + } + } + + async fn handle_graphql_query( + self, + target: QueryTarget, + request_body: Body, + ) -> GraphQLServiceResult { + let service = self.clone(); + + let start = Instant::now(); + let body = hyper::body::to_bytes(request_body) + .map_err(|_| GraphQLServerError::InternalError("Failed to read request body".into())) + .await?; + let query = parse_graphql_request(&body); + let query_parsing_time = start.elapsed(); + + let result = match query { + Ok(query) => service.graphql_runner.run_query(query, target).await, + Err(GraphQLServerError::QueryError(e)) => QueryResult::from(e).into(), + Err(e) => return Err(e), + }; + + self.graphql_runner + .metrics() + .observe_query_parsing(query_parsing_time, &result); + self.graphql_runner + .metrics() + .observe_query_execution(start.elapsed(), &result); + + Ok(result.as_http_response()) + } + + // Handles OPTIONS requests + fn handle_graphql_options(&self, _request: Request) -> GraphQLServiceResponse { + async { + Ok(Response::builder() + .status(200) + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .header(ACCESS_CONTROL_ALLOW_HEADERS, "Content-Type, User-Agent") + .header(ACCESS_CONTROL_ALLOW_METHODS, "GET, OPTIONS, POST") + .header(CONTENT_TYPE, "text/html") + .body(Body::from("")) + .unwrap()) + } + .boxed() + } + + /// Handles 302 redirects + async fn handle_temp_redirect(self, destination: String) -> GraphQLServiceResult { + header::HeaderValue::try_from(destination) + .map_err(|_| { + GraphQLServerError::ClientError("invalid characters in redirect URL".into()) + }) + .map(|loc_header_val| { + Response::builder() + .status(StatusCode::FOUND) + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .header(LOCATION, loc_header_val) + .header(CONTENT_TYPE, "text/plain") + .body(Body::from("Redirecting...")) + .unwrap() + }) + } + + /// Handles 404s. + fn handle_not_found(&self) -> GraphQLServiceResponse { + async { + Ok(Response::builder() + .status(StatusCode::NOT_FOUND) + .header(CONTENT_TYPE, "text/plain") + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .body(Body::from("Not found")) + .unwrap()) + } + .boxed() + } + + fn handle_call(self, req: Request) -> GraphQLServiceResponse { + let method = req.method().clone(); + + let path = req.uri().path().to_owned(); + let path_segments = { + let mut segments = path.split('/'); + + // Remove leading '/' + assert_eq!(segments.next(), Some("")); + + segments.collect::>() + }; + + match (method, path_segments.as_slice()) { + (Method::GET, [""]) => self.index().boxed(), + (Method::GET, &["subgraphs", "id", _, "graphql"]) + | (Method::GET, &["subgraphs", "name", _, "graphql"]) + | (Method::GET, &["subgraphs", "name", _, _, "graphql"]) + | (Method::GET, &["subgraphs", "network", _, _, "graphql"]) + | (Method::GET, &["subgraphs", "graphql"]) => self.handle_graphiql(), + + (Method::GET, path @ ["subgraphs", "id", _]) + | (Method::GET, path @ ["subgraphs", "name", _]) + | (Method::GET, path @ ["subgraphs", "name", _, _]) + | (Method::GET, path @ ["subgraphs", "network", _, _]) + | (Method::GET, path @ ["subgraphs"]) => { + let dest = format!("/{}/graphql", path.join("/")); + self.handle_temp_redirect(dest).boxed() + } + + (Method::POST, &["subgraphs", "id", subgraph_id]) => { + self.handle_graphql_query_by_id(subgraph_id.to_owned(), req) + } + (Method::OPTIONS, ["subgraphs", "id", _]) => self.handle_graphql_options(req), + (Method::POST, &["subgraphs", "name", subgraph_name]) => self + .handle_graphql_query_by_name(subgraph_name.to_owned(), req) + .boxed(), + (Method::POST, ["subgraphs", "name", subgraph_name_part1, subgraph_name_part2]) => { + let subgraph_name = format!("{}/{}", subgraph_name_part1, subgraph_name_part2); + self.handle_graphql_query_by_name(subgraph_name, req) + .boxed() + } + (Method::POST, ["subgraphs", "network", subgraph_name_part1, subgraph_name_part2]) => { + let subgraph_name = + format!("network/{}/{}", subgraph_name_part1, subgraph_name_part2); + self.handle_graphql_query_by_name(subgraph_name, req) + .boxed() + } + + (Method::OPTIONS, ["subgraphs", "name", _]) + | (Method::OPTIONS, ["subgraphs", "name", _, _]) + | (Method::OPTIONS, ["subgraphs", "network", _, _]) => self.handle_graphql_options(req), + + _ => self.handle_not_found(), + } + } +} + +impl Service> for GraphQLService +where + Q: GraphQlRunner, +{ + type Response = Response; + type Error = GraphQLServerError; + type Future = GraphQLServiceResponse; + + fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, req: Request) -> Self::Future { + let logger = self.logger.clone(); + let service = self.clone(); + + // Returning Err here will prevent the client from receiving any response. + // Instead, we generate a Response with an error code and return Ok + Box::pin(async move { + let result = service.handle_call(req).await; + match result { + Ok(response) => Ok(response), + Err(err @ GraphQLServerError::ClientError(_)) => Ok(Response::builder() + .status(400) + .header(CONTENT_TYPE, "text/plain") + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .body(Body::from(err.to_string())) + .unwrap()), + Err(err @ GraphQLServerError::QueryError(_)) => { + error!(logger, "GraphQLService call failed: {}", err); + + Ok(Response::builder() + .status(400) + .header(CONTENT_TYPE, "text/plain") + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .body(Body::from(format!("Query error: {}", err))) + .unwrap()) + } + Err(err @ GraphQLServerError::InternalError(_)) => { + error!(logger, "GraphQLService call failed: {}", err); + + Ok(Response::builder() + .status(500) + .header(CONTENT_TYPE, "text/plain") + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .body(Body::from(format!("Internal server error: {}", err))) + .unwrap()) + } + } + }) + } +} + +#[cfg(test)] +mod tests { + use graph::data::value::Object; + use http::status::StatusCode; + use hyper::service::Service; + use hyper::{Body, Method, Request}; + + use graph::data::{ + graphql::effort::LoadManager, + query::{QueryResults, QueryTarget}, + }; + use graph::prelude::*; + + use crate::test_utils; + + use super::GraphQLService; + + /// A simple stupid query runner for testing. + pub struct TestGraphQlRunner; + + pub struct TestGraphQLMetrics; + + lazy_static! { + static ref USERS: DeploymentHash = DeploymentHash::new("users").unwrap(); + } + + impl GraphQLMetrics for TestGraphQLMetrics { + fn observe_query_execution(&self, _duration: Duration, _results: &QueryResults) {} + fn observe_query_parsing(&self, _duration: Duration, _results: &QueryResults) {} + fn observe_query_validation(&self, _duration: Duration, _id: &DeploymentHash) {} + } + + #[async_trait] + impl GraphQlRunner for TestGraphQlRunner { + async fn run_query_with_complexity( + self: Arc, + _query: Query, + _target: QueryTarget, + _complexity: Option, + _max_depth: Option, + _max_first: Option, + _max_skip: Option, + ) -> QueryResults { + unimplemented!(); + } + + async fn run_query(self: Arc, _query: Query, _target: QueryTarget) -> QueryResults { + QueryResults::from(Object::from_iter( + vec![( + String::from("name"), + r::Value::String(String::from("Jordi")), + )] + .into_iter(), + )) + } + + async fn run_subscription( + self: Arc, + _subscription: Subscription, + _target: QueryTarget, + ) -> Result { + unreachable!(); + } + + fn load_manager(&self) -> Arc { + unimplemented!() + } + + fn metrics(&self) -> Arc { + Arc::new(TestGraphQLMetrics) + } + } + + #[test] + fn posting_invalid_query_yields_error_response() { + let logger = Logger::root(slog::Discard, o!()); + let subgraph_id = USERS.clone(); + let graphql_runner = Arc::new(TestGraphQlRunner); + + let node_id = NodeId::new("test").unwrap(); + let mut service = GraphQLService::new(logger, graphql_runner, 8001, node_id); + + let request = Request::builder() + .method(Method::POST) + .uri(format!( + "http://localhost:8000/subgraphs/id/{}", + subgraph_id + )) + .body(Body::from("{}")) + .unwrap(); + + let response = + futures03::executor::block_on(service.call(request)).expect("Should return a response"); + let errors = test_utils::assert_error_response(response, StatusCode::BAD_REQUEST, false); + + let message = errors[0].as_str().expect("Error message is not a string"); + + assert_eq!( + message, + "GraphQL server error (client error): The \"query\" field is missing in request data" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn posting_valid_queries_yields_result_response() { + let logger = Logger::root(slog::Discard, o!()); + let subgraph_id = USERS.clone(); + let graphql_runner = Arc::new(TestGraphQlRunner); + + let node_id = NodeId::new("test").unwrap(); + let mut service = GraphQLService::new(logger, graphql_runner, 8001, node_id); + + let request = Request::builder() + .method(Method::POST) + .uri(format!( + "http://localhost:8000/subgraphs/id/{}", + subgraph_id + )) + .body(Body::from("{\"query\": \"{ name }\"}")) + .unwrap(); + + // The response must be a 200 + let response = tokio::spawn(service.call(request)) + .await + .unwrap() + .expect("Should return a response"); + let data = test_utils::assert_successful_response(response); + + // The body should match the simulated query result + let name = data + .get("name") + .expect("Query result data has no \"name\" field") + .as_str() + .expect("Query result field \"name\" is not a string"); + assert_eq!(name, "Jordi".to_string()); + } +} diff --git a/server/http/src/test_utils.rs b/server/http/src/test_utils.rs new file mode 100644 index 0000000..2293559 --- /dev/null +++ b/server/http/src/test_utils.rs @@ -0,0 +1,72 @@ +use graph::prelude::serde_json; +use graph::prelude::*; +use http::StatusCode; +use hyper::{header::ACCESS_CONTROL_ALLOW_ORIGIN, Body, Response}; + +/// Asserts that the response is a successful GraphQL response; returns its `"data"` field. +pub fn assert_successful_response( + response: Response, +) -> serde_json::Map { + assert_eq!(response.status(), StatusCode::OK); + assert_expected_headers(&response); + futures03::executor::block_on( + hyper::body::to_bytes(response.into_body()) + .map_ok(|chunk| { + let json: serde_json::Value = + serde_json::from_slice(&chunk).expect("GraphQL response is not valid JSON"); + + json.as_object() + .expect("GraphQL response must be an object") + .get("data") + .expect("GraphQL response must contain a \"data\" field") + .as_object() + .expect("GraphQL \"data\" field must be an object") + .clone() + }) + .map_err(|e| panic!("Truncated response body {:?}", e)), + ) + .unwrap() +} + +/// Asserts that the response is a failed GraphQL response; returns its `"errors"` field. +pub fn assert_error_response( + response: Response, + expected_status: StatusCode, + graphql_response: bool, +) -> Vec { + assert_eq!(response.status(), expected_status); + assert_expected_headers(&response); + let body = String::from_utf8( + futures03::executor::block_on(hyper::body::to_bytes(response.into_body())) + .unwrap() + .to_vec(), + ) + .unwrap(); + + // In case of a non-graphql response, return the body. + if !graphql_response { + return vec![serde_json::Value::String(body)]; + } + + let json: serde_json::Value = + serde_json::from_str(&body).expect("GraphQL response is not valid JSON"); + + json.as_object() + .expect("GraphQL response must be an object") + .get("errors") + .expect("GraphQL error response must contain an \"errors\" field") + .as_array() + .expect("GraphQL \"errors\" field must be a vector") + .clone() +} + +#[track_caller] +pub fn assert_expected_headers(response: &Response) { + assert_eq!( + response + .headers() + .get(ACCESS_CONTROL_ALLOW_ORIGIN) + .expect("Missing CORS Header"), + &"*" + ); +} diff --git a/server/http/tests/response.rs b/server/http/tests/response.rs new file mode 100644 index 0000000..64b1ae0 --- /dev/null +++ b/server/http/tests/response.rs @@ -0,0 +1,87 @@ +use graph::data::value::Object; +use graph::data::{graphql::object, query::QueryResults}; +use graph::prelude::*; +use graph_server_http::test_utils; + +#[test] +fn generates_200_for_query_results() { + let data = Object::from_iter([]); + let query_result = QueryResults::from(data).as_http_response(); + test_utils::assert_expected_headers(&query_result); + test_utils::assert_successful_response(query_result); +} + +#[test] +fn generates_valid_json_for_an_empty_result() { + let data = Object::from_iter([]); + let query_result = QueryResults::from(data).as_http_response(); + test_utils::assert_expected_headers(&query_result); + let data = test_utils::assert_successful_response(query_result); + assert!(data.is_empty()); +} + +#[test] +fn canonical_serialization() { + macro_rules! assert_resp { + ($exp: expr, $obj: expr) => {{ + { + // This match is solely there to make sure these tests + // get amended if q::Value ever gets more variants + // The order of the variants should be the same as the + // order of the tests below + use r::Value::*; + let _ = match $obj { + Object(_) | List(_) | Enum(_) | Null | Int(_) | Float(_) | String(_) + | Boolean(_) => (), + }; + } + let res = QueryResult::try_from($obj).unwrap(); + assert_eq!($exp, serde_json::to_string(&res).unwrap()); + }}; + } + assert_resp!(r#"{"data":{"id":"12345"}}"#, object! { id: "12345" }); + + // Value::Variable: nothing to check, not used in a response + + // Value::Object: Insertion order of keys matters + let first_second = r#"{"data":{"first":"first","second":"second"}}"#; + let second_first = r#"{"data":{"second":"second","first":"first"}}"#; + assert_resp!(first_second, object! { first: "first", second: "second" }); + assert_resp!(second_first, object! { second: "second", first: "first" }); + + // Value::List + assert_resp!(r#"{"data":{"ary":[1,2]}}"#, object! { ary: vec![1,2] }); + + // Value::Enum + assert_resp!( + r#"{"data":{"enum_field":"enum"}}"#, + object! { enum_field: r::Value::Enum("enum".to_owned())} + ); + + // Value::Null + assert_resp!( + r#"{"data":{"nothing":null}}"#, + object! { nothing: r::Value::Null} + ); + + // Value::Int + assert_resp!( + r#"{"data":{"int32":17,"neg32":-314}}"#, + object! { int32: 17, neg32: -314 } + ); + + // Value::Float + assert_resp!(r#"{"data":{"float":3.14159}}"#, object! { float: 3.14159 }); + + // Value::String + assert_resp!( + r#"{"data":{"text":"Ünïcödë with spaceß"}}"#, + object! { text: "Ünïcödë with spaceß" } + ); + + // Value::Boolean + assert_resp!( + r#"{"data":{"no":false,"yes":true}}"#, + object! { no: false, yes: true } + ); +} diff --git a/server/http/tests/server.rs b/server/http/tests/server.rs new file mode 100644 index 0000000..856a4e9 --- /dev/null +++ b/server/http/tests/server.rs @@ -0,0 +1,321 @@ +use http::StatusCode; +use hyper::{Body, Client, Request}; +use std::time::Duration; + +use graph::data::{ + graphql::effort::LoadManager, + query::{QueryResults, QueryTarget}, + value::Object, +}; +use graph::prelude::*; + +use graph_server_http::test_utils; +use graph_server_http::GraphQLServer as HyperGraphQLServer; + +use tokio::time::sleep; + +pub struct TestGraphQLMetrics; + +impl GraphQLMetrics for TestGraphQLMetrics { + fn observe_query_execution(&self, _duration: Duration, _results: &QueryResults) {} + fn observe_query_parsing(&self, _duration: Duration, _results: &QueryResults) {} + fn observe_query_validation(&self, _duration: Duration, _id: &DeploymentHash) {} +} + +/// A simple stupid query runner for testing. +pub struct TestGraphQlRunner; + +#[async_trait] +impl GraphQlRunner for TestGraphQlRunner { + async fn run_query_with_complexity( + self: Arc, + _query: Query, + _target: QueryTarget, + _complexity: Option, + _max_depth: Option, + _max_first: Option, + _max_skip: Option, + ) -> QueryResults { + unimplemented!(); + } + + async fn run_query(self: Arc, query: Query, _target: QueryTarget) -> QueryResults { + if query.variables.is_some() + && query + .variables + .as_ref() + .unwrap() + .get(&String::from("equals")) + .is_some() + && query + .variables + .unwrap() + .get(&String::from("equals")) + .unwrap() + == &r::Value::String(String::from("John")) + { + Object::from_iter( + vec![(String::from("name"), r::Value::String(String::from("John")))].into_iter(), + ) + } else { + Object::from_iter( + vec![( + String::from("name"), + r::Value::String(String::from("Jordi")), + )] + .into_iter(), + ) + } + .into() + } + + async fn run_subscription( + self: Arc, + _subscription: Subscription, + _target: QueryTarget, + ) -> Result { + unreachable!(); + } + + fn load_manager(&self) -> Arc { + unimplemented!() + } + + fn metrics(&self) -> Arc { + Arc::new(TestGraphQLMetrics) + } +} + +#[cfg(test)] +mod test { + use super::*; + + lazy_static! { + static ref USERS: DeploymentHash = DeploymentHash::new("users").unwrap(); + } + + #[test] + fn rejects_empty_json() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime + .block_on(async { + let logger = Logger::root(slog::Discard, o!()); + let logger_factory = LoggerFactory::new(logger, None); + let id = USERS.clone(); + let query_runner = Arc::new(TestGraphQlRunner); + let node_id = NodeId::new("test").unwrap(); + let mut server = HyperGraphQLServer::new(&logger_factory, query_runner, node_id); + let http_server = server + .serve(8007, 8008) + .expect("Failed to start GraphQL server"); + + // Launch the server to handle a single request + tokio::spawn(http_server.fuse().compat()); + // Give some time for the server to start. + sleep(Duration::from_secs(2)) + .then(move |()| { + // Send an empty JSON POST request + let client = Client::new(); + let request = + Request::post(format!("http://localhost:8007/subgraphs/id/{}", id)) + .body(Body::from("{}")) + .unwrap(); + + // The response must be a query error + client.request(request) + }) + .map_ok(|response| { + let errors = + test_utils::assert_error_response(response, StatusCode::BAD_REQUEST, false); + + let message = errors[0] + .as_str() + .expect("Error message is not a string"); + assert_eq!(message, "GraphQL server error (client error): The \"query\" field is missing in request data"); + }).await.unwrap() + }) + } + + #[test] + fn rejects_invalid_queries() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let logger = Logger::root(slog::Discard, o!()); + let logger_factory = LoggerFactory::new(logger, None); + let id = USERS.clone(); + let query_runner = Arc::new(TestGraphQlRunner); + let node_id = NodeId::new("test").unwrap(); + let mut server = HyperGraphQLServer::new(&logger_factory, query_runner, node_id); + let http_server = server + .serve(8002, 8003) + .expect("Failed to start GraphQL server"); + + // Launch the server to handle a single request + tokio::spawn(http_server.fuse().compat()); + // Give some time for the server to start. + sleep(Duration::from_secs(2)) + .then(move |()| { + // Send an broken query request + let client = Client::new(); + let request = + Request::post(format!("http://localhost:8002/subgraphs/id/{}", id)) + .body(Body::from("{\"query\": \"M>\"}")) + .unwrap(); + + // The response must be a query error + client.request(request) + }) + .map_ok(|response| { + let errors = test_utils::assert_error_response(response, StatusCode::OK, true); + + let message = errors[0] + .as_object() + .expect("Query error is not an object") + .get("message") + .expect("Error contains no message") + .as_str() + .expect("Error message is not a string"); + + assert_eq!( + message, + "Unexpected `unexpected character \ + \'<\'`\nExpected `{`, `query`, `mutation`, \ + `subscription` or `fragment`" + ); + + let locations = errors[0] + .as_object() + .expect("Query error is not an object") + .get("locations") + .expect("Query error contains not locations") + .as_array() + .expect("Query error \"locations\" field is not an array"); + + let location = locations[0] + .as_object() + .expect("Query error location is not an object"); + + let line = location + .get("line") + .expect("Query error location is missing a \"line\" field") + .as_u64() + .expect("Query error location \"line\" field is not a u64"); + + assert_eq!(line, 1); + + let column = location + .get("column") + .expect("Query error location is missing a \"column\" field") + .as_u64() + .expect("Query error location \"column\" field is not a u64"); + + assert_eq!(column, 1); + }) + .await + .unwrap() + }) + } + + #[test] + fn accepts_valid_queries() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let logger = Logger::root(slog::Discard, o!()); + let logger_factory = LoggerFactory::new(logger, None); + let id = USERS.clone(); + let query_runner = Arc::new(TestGraphQlRunner); + let node_id = NodeId::new("test").unwrap(); + let mut server = HyperGraphQLServer::new(&logger_factory, query_runner, node_id); + let http_server = server + .serve(8003, 8004) + .expect("Failed to start GraphQL server"); + + // Launch the server to handle a single request + tokio::spawn(http_server.fuse().compat()); + // Give some time for the server to start. + sleep(Duration::from_secs(2)) + .then(move |()| { + // Send a valid example query + let client = Client::new(); + let request = + Request::post(format!("http://localhost:8003/subgraphs/id/{}", id)) + .body(Body::from("{\"query\": \"{ name }\"}")) + .unwrap(); + + // The response must be a 200 + client.request(request) + }) + .map_ok(|response| { + let data = test_utils::assert_successful_response(response); + + // The JSON response should match the simulated query result + let name = data + .get("name") + .expect("Query result data has no \"name\" field") + .as_str() + .expect("Query result field \"name\" is not a string"); + assert_eq!(name, "Jordi".to_string()); + }) + .await + .unwrap() + }); + } + + #[test] + fn accepts_valid_queries_with_variables() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let _ = runtime.block_on(async { + let logger = Logger::root(slog::Discard, o!()); + let logger_factory = LoggerFactory::new(logger, None); + let id = USERS.clone(); + let query_runner = Arc::new(TestGraphQlRunner); + let node_id = NodeId::new("test").unwrap(); + let mut server = HyperGraphQLServer::new(&logger_factory, query_runner, node_id); + let http_server = server + .serve(8005, 8006) + .expect("Failed to start GraphQL server"); + + // Launch the server to handle a single request + tokio::spawn(http_server.fuse().compat()); + // Give some time for the server to start. + sleep(Duration::from_secs(2)) + .then(move |()| { + // Send a valid example query + let client = Client::new(); + let request = + Request::post(format!("http://localhost:8005/subgraphs/id/{}", id)) + .body(Body::from( + " + { + \"query\": \" \ + query name($equals: String!) { \ + name(equals: $equals) \ + } \ + \", + \"variables\": { \"equals\": \"John\" } + } + ", + )) + .unwrap(); + + // The response must be a 200 + client.request(request) + }) + .map_ok(|response| { + async { + let data = test_utils::assert_successful_response(response); + + // The JSON response should match the simulated query result + let name = data + .get("name") + .expect("Query result data has no \"name\" field") + .as_str() + .expect("Query result field \"name\" is not a string"); + assert_eq!(name, "John".to_string()); + } + }) + .await + .unwrap() + }); + } +} diff --git a/server/index-node/Cargo.toml b/server/index-node/Cargo.toml new file mode 100644 index 0000000..1594369 --- /dev/null +++ b/server/index-node/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "graph-server-index-node" +version = "0.27.0" +edition = "2021" + +[dependencies] +blake3 = "1.0" +either = "1.8.0" +futures = "0.3.4" +graph = { path = "../../graph" } +graph-graphql = { path = "../../graphql" } +graph-chain-arweave = { path = "../../chain/arweave" } +graph-chain-ethereum = { path = "../../chain/ethereum" } +graph-chain-near = { path = "../../chain/near" } +graph-chain-cosmos = { path = "../../chain/cosmos" } +graphql-parser = "0.4.0" +http = "0.2" +hyper = "0.14" +lazy_static = "1.2.0" +serde = "1.0" diff --git a/server/index-node/assets/graphiql.css b/server/index-node/assets/graphiql.css new file mode 100644 index 0000000..222e806 --- /dev/null +++ b/server/index-node/assets/graphiql.css @@ -0,0 +1,1741 @@ +.graphiql-container, +.graphiql-container button, +.graphiql-container input { + color: #141823; + font-family: + system, + -apple-system, + 'San Francisco', + '.SFNSDisplay-Regular', + 'Segoe UI', + Segoe, + 'Segoe WP', + 'Helvetica Neue', + helvetica, + 'Lucida Grande', + arial, + sans-serif; + font-size: 14px; +} + +.graphiql-container { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-orient: horizontal; + -webkit-box-direction: normal; + -ms-flex-direction: row; + flex-direction: row; + height: 100%; + margin: 0; + overflow: hidden; + width: 100%; +} + +.graphiql-container .editorWrap { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-orient: vertical; + -webkit-box-direction: normal; + -ms-flex-direction: column; + flex-direction: column; + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + overflow-x: hidden; +} + +.graphiql-container .title { + font-size: 18px; +} + +.graphiql-container .title em { + font-family: georgia; + font-size: 19px; +} + +.graphiql-container .topBarWrap { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-orient: horizontal; + -webkit-box-direction: normal; + -ms-flex-direction: row; + flex-direction: row; +} + +.graphiql-container .topBar { + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + background: -webkit-gradient(linear, left top, left bottom, from(#f7f7f7), to(#e2e2e2)); + background: linear-gradient(#f7f7f7, #e2e2e2); + border-bottom: 1px solid #d0d0d0; + cursor: default; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-orient: horizontal; + -webkit-box-direction: normal; + -ms-flex-direction: row; + flex-direction: row; + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + height: 34px; + overflow-y: visible; + padding: 7px 14px 6px; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.graphiql-container .toolbar { + overflow-x: visible; + display: -webkit-box; + display: -ms-flexbox; + display: flex; +} + +.graphiql-container .docExplorerShow, +.graphiql-container .historyShow { + background: -webkit-gradient(linear, left top, left bottom, from(#f7f7f7), to(#e2e2e2)); + background: linear-gradient(#f7f7f7, #e2e2e2); + border-radius: 0; + border-bottom: 1px solid #d0d0d0; + border-right: none; + border-top: none; + color: #3B5998; + cursor: pointer; + font-size: 14px; + margin: 0; + outline: 0; + padding: 2px 20px 0 18px; +} + +.graphiql-container .docExplorerShow { + border-left: 1px solid rgba(0, 0, 0, 0.2); +} + +.graphiql-container .historyShow { + border-right: 1px solid rgba(0, 0, 0, 0.2); + border-left: 0; +} + +.graphiql-container .docExplorerShow:before { + border-left: 2px solid #3B5998; + border-top: 2px solid #3B5998; + content: ''; + display: inline-block; + height: 9px; + margin: 0 3px -1px 0; + position: relative; + -webkit-transform: rotate(-45deg); + transform: rotate(-45deg); + width: 9px; +} + +.graphiql-container .editorBar { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-orient: horizontal; + -webkit-box-direction: normal; + -ms-flex-direction: row; + flex-direction: row; + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; +} + +.graphiql-container .queryWrap { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-orient: vertical; + -webkit-box-direction: normal; + -ms-flex-direction: column; + flex-direction: column; + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; +} + +.graphiql-container .resultWrap { + border-left: solid 1px #e0e0e0; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-orient: vertical; + -webkit-box-direction: normal; + -ms-flex-direction: column; + flex-direction: column; + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + position: relative; +} + +.graphiql-container .docExplorerWrap, +.graphiql-container .historyPaneWrap { + background: white; + -webkit-box-shadow: 0 0 8px rgba(0, 0, 0, 0.15); + box-shadow: 0 0 8px rgba(0, 0, 0, 0.15); + position: relative; + z-index: 3; +} + +.graphiql-container .historyPaneWrap { + min-width: 230px; + z-index: 5; +} + +.graphiql-container .docExplorerResizer { + cursor: col-resize; + height: 100%; + left: -5px; + position: absolute; + top: 0; + width: 10px; + z-index: 10; +} + +.graphiql-container .docExplorerHide { + cursor: pointer; + font-size: 18px; + margin: -7px -8px -6px 0; + padding: 18px 16px 15px 12px; +} + +.graphiql-container div .query-editor { + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + position: relative; +} + +.graphiql-container .variable-editor { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-orient: vertical; + -webkit-box-direction: normal; + -ms-flex-direction: column; + flex-direction: column; + height: 29px; + position: relative; +} + +.graphiql-container .variable-editor-title { + background: #eeeeee; + border-bottom: 1px solid #d6d6d6; + border-top: 1px solid #e0e0e0; + color: #777; + font-variant: small-caps; + font-weight: bold; + letter-spacing: 1px; + line-height: 14px; + padding: 6px 0 8px 43px; + text-transform: lowercase; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.graphiql-container .codemirrorWrap { + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + height: 100%; + position: relative; +} + +.graphiql-container .result-window { + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + height: 100%; + position: relative; +} + +.graphiql-container .footer { + background: #f6f7f8; + border-left: 1px solid #e0e0e0; + border-top: 1px solid #e0e0e0; + margin-left: 12px; + position: relative; +} + +.graphiql-container .footer:before { + background: #eeeeee; + bottom: 0; + content: " "; + left: -13px; + position: absolute; + top: -1px; + width: 12px; +} + +/* No `.graphiql-container` here so themes can overwrite */ +.result-window .CodeMirror { + background: #f6f7f8; +} + +.graphiql-container .result-window .CodeMirror-gutters { + background-color: #eeeeee; + border-color: #e0e0e0; + cursor: col-resize; +} + +.graphiql-container .result-window .CodeMirror-foldgutter, +.graphiql-container .result-window .CodeMirror-foldgutter-open:after, +.graphiql-container .result-window .CodeMirror-foldgutter-folded:after { + padding-left: 3px; +} + +.graphiql-container .toolbar-button { + background: #fdfdfd; + background: -webkit-gradient(linear, left top, left bottom, from(#f9f9f9), to(#ececec)); + background: linear-gradient(#f9f9f9, #ececec); + border-radius: 3px; + -webkit-box-shadow: + inset 0 0 0 1px rgba(0,0,0,0.20), + 0 1px 0 rgba(255,255,255, 0.7), + inset 0 1px #fff; + box-shadow: + inset 0 0 0 1px rgba(0,0,0,0.20), + 0 1px 0 rgba(255,255,255, 0.7), + inset 0 1px #fff; + color: #555; + cursor: pointer; + display: inline-block; + margin: 0 5px; + padding: 3px 11px 5px; + text-decoration: none; + text-overflow: ellipsis; + white-space: nowrap; + max-width: 150px; +} + +.graphiql-container .toolbar-button:active { + background: -webkit-gradient(linear, left top, left bottom, from(#ececec), to(#d5d5d5)); + background: linear-gradient(#ececec, #d5d5d5); + -webkit-box-shadow: + 0 1px 0 rgba(255, 255, 255, 0.7), + inset 0 0 0 1px rgba(0,0,0,0.10), + inset 0 1px 1px 1px rgba(0, 0, 0, 0.12), + inset 0 0 5px rgba(0, 0, 0, 0.1); + box-shadow: + 0 1px 0 rgba(255, 255, 255, 0.7), + inset 0 0 0 1px rgba(0,0,0,0.10), + inset 0 1px 1px 1px rgba(0, 0, 0, 0.12), + inset 0 0 5px rgba(0, 0, 0, 0.1); +} + +.graphiql-container .toolbar-button.error { + background: -webkit-gradient(linear, left top, left bottom, from(#fdf3f3), to(#e6d6d7)); + background: linear-gradient(#fdf3f3, #e6d6d7); + color: #b00; +} + +.graphiql-container .toolbar-button-group { + margin: 0 5px; + white-space: nowrap; +} + +.graphiql-container .toolbar-button-group > * { + margin: 0; +} + +.graphiql-container .toolbar-button-group > *:not(:last-child) { + border-top-right-radius: 0; + border-bottom-right-radius: 0; +} + +.graphiql-container .toolbar-button-group > *:not(:first-child) { + border-top-left-radius: 0; + border-bottom-left-radius: 0; + margin-left: -1px; +} + +.graphiql-container .execute-button-wrap { + height: 34px; + margin: 0 14px 0 28px; + position: relative; +} + +.graphiql-container .execute-button { + background: -webkit-gradient(linear, left top, left bottom, from(#fdfdfd), to(#d2d3d6)); + background: linear-gradient(#fdfdfd, #d2d3d6); + border-radius: 17px; + border: 1px solid rgba(0,0,0,0.25); + -webkit-box-shadow: 0 1px 0 #fff; + box-shadow: 0 1px 0 #fff; + cursor: pointer; + fill: #444; + height: 34px; + margin: 0; + padding: 0; + width: 34px; +} + +.graphiql-container .execute-button svg { + pointer-events: none; +} + +.graphiql-container .execute-button:active { + background: -webkit-gradient(linear, left top, left bottom, from(#e6e6e6), to(#c3c3c3)); + background: linear-gradient(#e6e6e6, #c3c3c3); + -webkit-box-shadow: + 0 1px 0 #fff, + inset 0 0 2px rgba(0, 0, 0, 0.2), + inset 0 0 6px rgba(0, 0, 0, 0.1); + box-shadow: + 0 1px 0 #fff, + inset 0 0 2px rgba(0, 0, 0, 0.2), + inset 0 0 6px rgba(0, 0, 0, 0.1); +} + +.graphiql-container .execute-button:focus { + outline: 0; +} + +.graphiql-container .toolbar-menu, +.graphiql-container .toolbar-select { + position: relative; +} + +.graphiql-container .execute-options, +.graphiql-container .toolbar-menu-items, +.graphiql-container .toolbar-select-options { + background: #fff; + -webkit-box-shadow: + 0 0 0 1px rgba(0,0,0,0.1), + 0 2px 4px rgba(0,0,0,0.25); + box-shadow: + 0 0 0 1px rgba(0,0,0,0.1), + 0 2px 4px rgba(0,0,0,0.25); + margin: 0; + padding: 6px 0; + position: absolute; + z-index: 100; +} + +.graphiql-container .execute-options { + min-width: 100px; + top: 37px; + left: -1px; +} + +.graphiql-container .toolbar-menu-items { + left: 1px; + margin-top: -1px; + min-width: 110%; + top: 100%; + visibility: hidden; +} + +.graphiql-container .toolbar-menu-items.open { + visibility: visible; +} + +.graphiql-container .toolbar-select-options { + left: 0; + min-width: 100%; + top: -5px; + visibility: hidden; +} + +.graphiql-container .toolbar-select-options.open { + visibility: visible; +} + +.graphiql-container .execute-options > li, +.graphiql-container .toolbar-menu-items > li, +.graphiql-container .toolbar-select-options > li { + cursor: pointer; + display: block; + margin: none; + max-width: 300px; + overflow: hidden; + padding: 2px 20px 4px 11px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.graphiql-container .execute-options > li.selected, +.graphiql-container .toolbar-menu-items > li.hover, +.graphiql-container .toolbar-menu-items > li:active, +.graphiql-container .toolbar-menu-items > li:hover, +.graphiql-container .toolbar-select-options > li.hover, +.graphiql-container .toolbar-select-options > li:active, +.graphiql-container .toolbar-select-options > li:hover, +.graphiql-container .history-contents > p:hover, +.graphiql-container .history-contents > p:active { + background: #e10098; + color: #fff; +} + +.graphiql-container .toolbar-select-options > li > svg { + display: inline; + fill: #666; + margin: 0 -6px 0 6px; + pointer-events: none; + vertical-align: middle; +} + +.graphiql-container .toolbar-select-options > li.hover > svg, +.graphiql-container .toolbar-select-options > li:active > svg, +.graphiql-container .toolbar-select-options > li:hover > svg { + fill: #fff; +} + +.graphiql-container .CodeMirror-scroll { + overflow-scrolling: touch; +} + +.graphiql-container .CodeMirror { + color: #141823; + font-family: + 'Consolas', + 'Inconsolata', + 'Droid Sans Mono', + 'Monaco', + monospace; + font-size: 13px; + height: 100%; + left: 0; + position: absolute; + top: 0; + width: 100%; +} + +.graphiql-container .CodeMirror-lines { + padding: 20px 0; +} + +.CodeMirror-hint-information .content { + box-orient: vertical; + color: #141823; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + font-family: system, -apple-system, 'San Francisco', '.SFNSDisplay-Regular', 'Segoe UI', Segoe, 'Segoe WP', 'Helvetica Neue', helvetica, 'Lucida Grande', arial, sans-serif; + font-size: 13px; + line-clamp: 3; + line-height: 16px; + max-height: 48px; + overflow: hidden; + text-overflow: -o-ellipsis-lastline; +} + +.CodeMirror-hint-information .content p:first-child { + margin-top: 0; +} + +.CodeMirror-hint-information .content p:last-child { + margin-bottom: 0; +} + +.CodeMirror-hint-information .infoType { + color: #CA9800; + cursor: pointer; + display: inline; + margin-right: 0.5em; +} + +.autoInsertedLeaf.cm-property { + -webkit-animation-duration: 6s; + animation-duration: 6s; + -webkit-animation-name: insertionFade; + animation-name: insertionFade; + border-bottom: 2px solid rgba(255, 255, 255, 0); + border-radius: 2px; + margin: -2px -4px -1px; + padding: 2px 4px 1px; +} + +@-webkit-keyframes insertionFade { + from, to { + background: rgba(255, 255, 255, 0); + border-color: rgba(255, 255, 255, 0); + } + + 15%, 85% { + background: #fbffc9; + border-color: #f0f3c0; + } +} + +@keyframes insertionFade { + from, to { + background: rgba(255, 255, 255, 0); + border-color: rgba(255, 255, 255, 0); + } + + 15%, 85% { + background: #fbffc9; + border-color: #f0f3c0; + } +} + +div.CodeMirror-lint-tooltip { + background-color: white; + border-radius: 2px; + border: 0; + color: #141823; + -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.45); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.45); + font-family: + system, + -apple-system, + 'San Francisco', + '.SFNSDisplay-Regular', + 'Segoe UI', + Segoe, + 'Segoe WP', + 'Helvetica Neue', + helvetica, + 'Lucida Grande', + arial, + sans-serif; + font-size: 13px; + line-height: 16px; + max-width: 430px; + opacity: 0; + padding: 8px 10px; + -webkit-transition: opacity 0.15s; + transition: opacity 0.15s; + white-space: pre-wrap; +} + +div.CodeMirror-lint-tooltip > * { + padding-left: 23px; +} + +div.CodeMirror-lint-tooltip > * + * { + margin-top: 12px; +} + +/* COLORS */ + +.graphiql-container .CodeMirror-foldmarker { + border-radius: 4px; + background: #08f; + background: -webkit-gradient(linear, left top, left bottom, from(#43A8FF), to(#0F83E8)); + background: linear-gradient(#43A8FF, #0F83E8); + -webkit-box-shadow: + 0 1px 1px rgba(0, 0, 0, 0.2), + inset 0 0 0 1px rgba(0, 0, 0, 0.1); + box-shadow: + 0 1px 1px rgba(0, 0, 0, 0.2), + inset 0 0 0 1px rgba(0, 0, 0, 0.1); + color: white; + font-family: arial; + font-size: 12px; + line-height: 0; + margin: 0 3px; + padding: 0px 4px 1px; + text-shadow: 0 -1px rgba(0, 0, 0, 0.1); +} + +.graphiql-container div.CodeMirror span.CodeMirror-matchingbracket { + color: #555; + text-decoration: underline; +} + +.graphiql-container div.CodeMirror span.CodeMirror-nonmatchingbracket { + color: #f00; +} + +/* Comment */ +.cm-comment { + color: #999; +} + +/* Punctuation */ +.cm-punctuation { + color: #555; +} + +/* Keyword */ +.cm-keyword { + color: #B11A04; +} + +/* OperationName, FragmentName */ +.cm-def { + color: #D2054E; +} + +/* FieldName */ +.cm-property { + color: #1F61A0; +} + +/* FieldAlias */ +.cm-qualifier { + color: #1C92A9; +} + +/* ArgumentName and ObjectFieldName */ +.cm-attribute { + color: #8B2BB9; +} + +/* Number */ +.cm-number { + color: #2882F9; +} + +/* String */ +.cm-string { + color: #D64292; +} + +/* Boolean */ +.cm-builtin { + color: #D47509; +} + +/* EnumValue */ +.cm-string-2 { + color: #0B7FC7; +} + +/* Variable */ +.cm-variable { + color: #397D13; +} + +/* Directive */ +.cm-meta { + color: #B33086; +} + +/* Type */ +.cm-atom { + color: #CA9800; +} +/* BASICS */ + +.CodeMirror { + /* Set height, width, borders, and global font properties here */ + color: black; + font-family: monospace; + height: 300px; +} + +/* PADDING */ + +.CodeMirror-lines { + padding: 4px 0; /* Vertical padding around content */ +} +.CodeMirror pre { + padding: 0 4px; /* Horizontal padding of content */ +} + +.CodeMirror-scrollbar-filler, .CodeMirror-gutter-filler { + background-color: white; /* The little square between H and V scrollbars */ +} + +/* GUTTER */ + +.CodeMirror-gutters { + border-right: 1px solid #ddd; + background-color: #f7f7f7; + white-space: nowrap; +} +.CodeMirror-linenumbers {} +.CodeMirror-linenumber { + color: #999; + min-width: 20px; + padding: 0 3px 0 5px; + text-align: right; + white-space: nowrap; +} + +.CodeMirror-guttermarker { color: black; } +.CodeMirror-guttermarker-subtle { color: #999; } + +/* CURSOR */ + +.CodeMirror .CodeMirror-cursor { + border-left: 1px solid black; +} +/* Shown when moving in bi-directional text */ +.CodeMirror div.CodeMirror-secondarycursor { + border-left: 1px solid silver; +} +.CodeMirror.cm-fat-cursor div.CodeMirror-cursor { + background: #7e7; + border: 0; + width: auto; +} +.CodeMirror.cm-fat-cursor div.CodeMirror-cursors { + z-index: 1; +} + +.cm-animate-fat-cursor { + -webkit-animation: blink 1.06s steps(1) infinite; + animation: blink 1.06s steps(1) infinite; + border: 0; + width: auto; +} +@-webkit-keyframes blink { + 0% { background: #7e7; } + 50% { background: none; } + 100% { background: #7e7; } +} +@keyframes blink { + 0% { background: #7e7; } + 50% { background: none; } + 100% { background: #7e7; } +} + +/* Can style cursor different in overwrite (non-insert) mode */ +div.CodeMirror-overwrite div.CodeMirror-cursor {} + +.cm-tab { display: inline-block; text-decoration: inherit; } + +.CodeMirror-ruler { + border-left: 1px solid #ccc; + position: absolute; +} + +/* DEFAULT THEME */ + +.cm-s-default .cm-keyword {color: #708;} +.cm-s-default .cm-atom {color: #219;} +.cm-s-default .cm-number {color: #164;} +.cm-s-default .cm-def {color: #00f;} +.cm-s-default .cm-variable, +.cm-s-default .cm-punctuation, +.cm-s-default .cm-property, +.cm-s-default .cm-operator {} +.cm-s-default .cm-variable-2 {color: #05a;} +.cm-s-default .cm-variable-3 {color: #085;} +.cm-s-default .cm-comment {color: #a50;} +.cm-s-default .cm-string {color: #a11;} +.cm-s-default .cm-string-2 {color: #f50;} +.cm-s-default .cm-meta {color: #555;} +.cm-s-default .cm-qualifier {color: #555;} +.cm-s-default .cm-builtin {color: #30a;} +.cm-s-default .cm-bracket {color: #997;} +.cm-s-default .cm-tag {color: #170;} +.cm-s-default .cm-attribute {color: #00c;} +.cm-s-default .cm-header {color: blue;} +.cm-s-default .cm-quote {color: #090;} +.cm-s-default .cm-hr {color: #999;} +.cm-s-default .cm-link {color: #00c;} + +.cm-negative {color: #d44;} +.cm-positive {color: #292;} +.cm-header, .cm-strong {font-weight: bold;} +.cm-em {font-style: italic;} +.cm-link {text-decoration: underline;} +.cm-strikethrough {text-decoration: line-through;} + +.cm-s-default .cm-error {color: #f00;} +.cm-invalidchar {color: #f00;} + +.CodeMirror-composing { border-bottom: 2px solid; } + +/* Default styles for common addons */ + +div.CodeMirror span.CodeMirror-matchingbracket {color: #0f0;} +div.CodeMirror span.CodeMirror-nonmatchingbracket {color: #f22;} +.CodeMirror-matchingtag { background: rgba(255, 150, 0, .3); } +.CodeMirror-activeline-background {background: #e8f2ff;} + +/* STOP */ + +/* The rest of this file contains styles related to the mechanics of + the editor. You probably shouldn't touch them. */ + +.CodeMirror { + background: white; + overflow: hidden; + position: relative; +} + +.CodeMirror-scroll { + height: 100%; + /* 30px is the magic margin used to hide the element's real scrollbars */ + /* See overflow: hidden in .CodeMirror */ + margin-bottom: -30px; margin-right: -30px; + outline: none; /* Prevent dragging from highlighting the element */ + overflow: scroll !important; /* Things will break if this is overridden */ + padding-bottom: 30px; + position: relative; +} +.CodeMirror-sizer { + border-right: 30px solid transparent; + position: relative; +} + +/* The fake, visible scrollbars. Used to force redraw during scrolling + before actual scrolling happens, thus preventing shaking and + flickering artifacts. */ +.CodeMirror-vscrollbar, .CodeMirror-hscrollbar, .CodeMirror-scrollbar-filler, .CodeMirror-gutter-filler { + display: none; + position: absolute; + z-index: 6; +} +.CodeMirror-vscrollbar { + overflow-x: hidden; + overflow-y: scroll; + right: 0; top: 0; +} +.CodeMirror-hscrollbar { + bottom: 0; left: 0; + overflow-x: scroll; + overflow-y: hidden; +} +.CodeMirror-scrollbar-filler { + right: 0; bottom: 0; +} +.CodeMirror-gutter-filler { + left: 0; bottom: 0; +} + +.CodeMirror-gutters { + min-height: 100%; + position: absolute; left: 0; top: 0; + z-index: 3; +} +.CodeMirror-gutter { + display: inline-block; + height: 100%; + margin-bottom: -30px; + vertical-align: top; + white-space: normal; + /* Hack to make IE7 behave */ + *zoom:1; + *display:inline; +} +.CodeMirror-gutter-wrapper { + background: none !important; + border: none !important; + position: absolute; + z-index: 4; +} +.CodeMirror-gutter-background { + position: absolute; + top: 0; bottom: 0; + z-index: 4; +} +.CodeMirror-gutter-elt { + cursor: default; + position: absolute; + z-index: 4; +} +.CodeMirror-gutter-wrapper { + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.CodeMirror-lines { + cursor: text; + min-height: 1px; /* prevents collapsing before first draw */ +} +.CodeMirror pre { + -webkit-tap-highlight-color: transparent; + /* Reset some styles that the rest of the page might have set */ + background: transparent; + border-radius: 0; + border-width: 0; + color: inherit; + font-family: inherit; + font-size: inherit; + -webkit-font-variant-ligatures: none; + font-variant-ligatures: none; + line-height: inherit; + margin: 0; + overflow: visible; + position: relative; + white-space: pre; + word-wrap: normal; + z-index: 2; +} +.CodeMirror-wrap pre { + word-wrap: break-word; + white-space: pre-wrap; + word-break: normal; +} + +.CodeMirror-linebackground { + position: absolute; + left: 0; right: 0; top: 0; bottom: 0; + z-index: 0; +} + +.CodeMirror-linewidget { + overflow: auto; + position: relative; + z-index: 2; +} + +.CodeMirror-widget {} + +.CodeMirror-code { + outline: none; +} + +/* Force content-box sizing for the elements where we expect it */ +.CodeMirror-scroll, +.CodeMirror-sizer, +.CodeMirror-gutter, +.CodeMirror-gutters, +.CodeMirror-linenumber { + -webkit-box-sizing: content-box; + box-sizing: content-box; +} + +.CodeMirror-measure { + height: 0; + overflow: hidden; + position: absolute; + visibility: hidden; + width: 100%; +} + +.CodeMirror-cursor { position: absolute; } +.CodeMirror-measure pre { position: static; } + +div.CodeMirror-cursors { + position: relative; + visibility: hidden; + z-index: 3; +} +div.CodeMirror-dragcursors { + visibility: visible; +} + +.CodeMirror-focused div.CodeMirror-cursors { + visibility: visible; +} + +.CodeMirror-selected { background: #d9d9d9; } +.CodeMirror-focused .CodeMirror-selected { background: #d7d4f0; } +.CodeMirror-crosshair { cursor: crosshair; } +.CodeMirror-line::-moz-selection, .CodeMirror-line > span::-moz-selection, .CodeMirror-line > span > span::-moz-selection { background: #d7d4f0; } +.CodeMirror-line::selection, .CodeMirror-line > span::selection, .CodeMirror-line > span > span::selection { background: #d7d4f0; } +.CodeMirror-line::-moz-selection, .CodeMirror-line > span::-moz-selection, .CodeMirror-line > span > span::-moz-selection { background: #d7d4f0; } + +.cm-searching { + background: #ffa; + background: rgba(255, 255, 0, .4); +} + +/* IE7 hack to prevent it from returning funny offsetTops on the spans */ +.CodeMirror span { *vertical-align: text-bottom; } + +/* Used to force a border model for a node */ +.cm-force-border { padding-right: .1px; } + +@media print { + /* Hide the cursor when printing */ + .CodeMirror div.CodeMirror-cursors { + visibility: hidden; + } +} + +/* See issue #2901 */ +.cm-tab-wrap-hack:after { content: ''; } + +/* Help users use markselection to safely style text background */ +span.CodeMirror-selectedtext { background: none; } + +.CodeMirror-dialog { + background: inherit; + color: inherit; + left: 0; right: 0; + overflow: hidden; + padding: .1em .8em; + position: absolute; + z-index: 15; +} + +.CodeMirror-dialog-top { + border-bottom: 1px solid #eee; + top: 0; +} + +.CodeMirror-dialog-bottom { + border-top: 1px solid #eee; + bottom: 0; +} + +.CodeMirror-dialog input { + background: transparent; + border: 1px solid #d3d6db; + color: inherit; + font-family: monospace; + outline: none; + width: 20em; +} + +.CodeMirror-dialog button { + font-size: 70%; +} +.graphiql-container .doc-explorer { + background: white; +} + +.graphiql-container .doc-explorer-title-bar, +.graphiql-container .history-title-bar { + cursor: default; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + height: 34px; + line-height: 14px; + padding: 8px 8px 5px; + position: relative; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.graphiql-container .doc-explorer-title, +.graphiql-container .history-title { + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + font-weight: bold; + overflow-x: hidden; + padding: 10px 0 10px 10px; + text-align: center; + text-overflow: ellipsis; + -webkit-user-select: initial; + -moz-user-select: initial; + -ms-user-select: initial; + user-select: initial; + white-space: nowrap; +} + +.graphiql-container .doc-explorer-back { + color: #3B5998; + cursor: pointer; + margin: -7px 0 -6px -8px; + overflow-x: hidden; + padding: 17px 12px 16px 16px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.doc-explorer-narrow .doc-explorer-back { + width: 0; +} + +.graphiql-container .doc-explorer-back:before { + border-left: 2px solid #3B5998; + border-top: 2px solid #3B5998; + content: ''; + display: inline-block; + height: 9px; + margin: 0 3px -1px 0; + position: relative; + -webkit-transform: rotate(-45deg); + transform: rotate(-45deg); + width: 9px; +} + +.graphiql-container .doc-explorer-rhs { + position: relative; +} + +.graphiql-container .doc-explorer-contents, +.graphiql-container .history-contents { + background-color: #ffffff; + border-top: 1px solid #d6d6d6; + bottom: 0; + left: 0; + overflow-y: auto; + padding: 20px 15px; + position: absolute; + right: 0; + top: 47px; +} + +.graphiql-container .doc-explorer-contents { + min-width: 300px; +} + +.graphiql-container .doc-type-description p:first-child , +.graphiql-container .doc-type-description blockquote:first-child { + margin-top: 0; +} + +.graphiql-container .doc-explorer-contents a { + cursor: pointer; + text-decoration: none; +} + +.graphiql-container .doc-explorer-contents a:hover { + text-decoration: underline; +} + +.graphiql-container .doc-value-description > :first-child { + margin-top: 4px; +} + +.graphiql-container .doc-value-description > :last-child { + margin-bottom: 4px; +} + +.graphiql-container .doc-category { + margin: 20px 0; +} + +.graphiql-container .doc-category-title { + border-bottom: 1px solid #e0e0e0; + color: #777; + cursor: default; + font-size: 14px; + font-variant: small-caps; + font-weight: bold; + letter-spacing: 1px; + margin: 0 -15px 10px 0; + padding: 10px 0; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.graphiql-container .doc-category-item { + margin: 12px 0; + color: #555; +} + +.graphiql-container .keyword { + color: #B11A04; +} + +.graphiql-container .type-name { + color: #CA9800; +} + +.graphiql-container .field-name { + color: #1F61A0; +} + +.graphiql-container .field-short-description { + color: #999; + margin-left: 5px; + overflow: hidden; + text-overflow: ellipsis; +} + +.graphiql-container .enum-value { + color: #0B7FC7; +} + +.graphiql-container .arg-name { + color: #8B2BB9; +} + +.graphiql-container .arg { + display: block; + margin-left: 1em; +} + +.graphiql-container .arg:first-child:last-child, +.graphiql-container .arg:first-child:nth-last-child(2), +.graphiql-container .arg:first-child:nth-last-child(2) ~ .arg { + display: inherit; + margin: inherit; +} + +.graphiql-container .arg:first-child:nth-last-child(2):after { + content: ', '; +} + +.graphiql-container .arg-default-value { + color: #43A047; +} + +.graphiql-container .doc-deprecation { + background: #fffae8; + -webkit-box-shadow: inset 0 0 1px #bfb063; + box-shadow: inset 0 0 1px #bfb063; + color: #867F70; + line-height: 16px; + margin: 8px -8px; + max-height: 80px; + overflow: hidden; + padding: 8px; + border-radius: 3px; +} + +.graphiql-container .doc-deprecation:before { + content: 'Deprecated:'; + color: #c79b2e; + cursor: default; + display: block; + font-size: 9px; + font-weight: bold; + letter-spacing: 1px; + line-height: 1; + padding-bottom: 5px; + text-transform: uppercase; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.graphiql-container .doc-deprecation > :first-child { + margin-top: 0; +} + +.graphiql-container .doc-deprecation > :last-child { + margin-bottom: 0; +} + +.graphiql-container .show-btn { + -webkit-appearance: initial; + display: block; + border-radius: 3px; + border: solid 1px #ccc; + text-align: center; + padding: 8px 12px 10px; + width: 100%; + -webkit-box-sizing: border-box; + box-sizing: border-box; + background: #fbfcfc; + color: #555; + cursor: pointer; +} + +.graphiql-container .search-box { + border-bottom: 1px solid #d3d6db; + display: block; + font-size: 14px; + margin: -15px -15px 12px 0; + position: relative; +} + +.graphiql-container .search-box:before { + content: '\26b2'; + cursor: pointer; + display: block; + font-size: 24px; + position: absolute; + top: -2px; + -webkit-transform: rotate(-45deg); + transform: rotate(-45deg); + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.graphiql-container .search-box .search-box-clear { + background-color: #d0d0d0; + border-radius: 12px; + color: #fff; + cursor: pointer; + font-size: 11px; + padding: 1px 5px 2px; + position: absolute; + right: 3px; + top: 8px; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.graphiql-container .search-box .search-box-clear:hover { + background-color: #b9b9b9; +} + +.graphiql-container .search-box > input { + border: none; + -webkit-box-sizing: border-box; + box-sizing: border-box; + font-size: 14px; + outline: none; + padding: 6px 24px 8px 20px; + width: 100%; +} + +.graphiql-container .error-container { + font-weight: bold; + left: 0; + letter-spacing: 1px; + opacity: 0.5; + position: absolute; + right: 0; + text-align: center; + text-transform: uppercase; + top: 50%; + -webkit-transform: translate(0, -50%); + transform: translate(0, -50%); +} +.CodeMirror-foldmarker { + color: blue; + cursor: pointer; + font-family: arial; + line-height: .3; + text-shadow: #b9f 1px 1px 2px, #b9f -1px -1px 2px, #b9f 1px -1px 2px, #b9f -1px 1px 2px; +} +.CodeMirror-foldgutter { + width: .7em; +} +.CodeMirror-foldgutter-open, +.CodeMirror-foldgutter-folded { + cursor: pointer; +} +.CodeMirror-foldgutter-open:after { + content: "\25BE"; +} +.CodeMirror-foldgutter-folded:after { + content: "\25B8"; +} +.graphiql-container .history-contents { + font-family: 'Consolas', 'Inconsolata', 'Droid Sans Mono', 'Monaco', monospace; + padding: 0; +} + +.graphiql-container .history-contents p { + font-size: 12px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + margin: 0; + padding: 8px; + border-bottom: 1px solid #e0e0e0; +} + +.graphiql-container .history-contents p:hover { + cursor: pointer; +} +.CodeMirror-info { + background: white; + border-radius: 2px; + -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.45); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.45); + -webkit-box-sizing: border-box; + box-sizing: border-box; + color: #555; + font-family: + system, + -apple-system, + 'San Francisco', + '.SFNSDisplay-Regular', + 'Segoe UI', + Segoe, + 'Segoe WP', + 'Helvetica Neue', + helvetica, + 'Lucida Grande', + arial, + sans-serif; + font-size: 13px; + line-height: 16px; + margin: 8px -8px; + max-width: 400px; + opacity: 0; + overflow: hidden; + padding: 8px 8px; + position: fixed; + -webkit-transition: opacity 0.15s; + transition: opacity 0.15s; + z-index: 50; +} + +.CodeMirror-info :first-child { + margin-top: 0; +} + +.CodeMirror-info :last-child { + margin-bottom: 0; +} + +.CodeMirror-info p { + margin: 1em 0; +} + +.CodeMirror-info .info-description { + color: #777; + line-height: 16px; + margin-top: 1em; + max-height: 80px; + overflow: hidden; +} + +.CodeMirror-info .info-deprecation { + background: #fffae8; + -webkit-box-shadow: inset 0 1px 1px -1px #bfb063; + box-shadow: inset 0 1px 1px -1px #bfb063; + color: #867F70; + line-height: 16px; + margin: -8px; + margin-top: 8px; + max-height: 80px; + overflow: hidden; + padding: 8px; +} + +.CodeMirror-info .info-deprecation-label { + color: #c79b2e; + cursor: default; + display: block; + font-size: 9px; + font-weight: bold; + letter-spacing: 1px; + line-height: 1; + padding-bottom: 5px; + text-transform: uppercase; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.CodeMirror-info .info-deprecation-label + * { + margin-top: 0; +} + +.CodeMirror-info a { + text-decoration: none; +} + +.CodeMirror-info a:hover { + text-decoration: underline; +} + +.CodeMirror-info .type-name { + color: #CA9800; +} + +.CodeMirror-info .field-name { + color: #1F61A0; +} + +.CodeMirror-info .enum-value { + color: #0B7FC7; +} + +.CodeMirror-info .arg-name { + color: #8B2BB9; +} + +.CodeMirror-info .directive-name { + color: #B33086; +} +.CodeMirror-jump-token { + text-decoration: underline; + cursor: pointer; +} +/* The lint marker gutter */ +.CodeMirror-lint-markers { + width: 16px; +} + +.CodeMirror-lint-tooltip { + background-color: infobackground; + border-radius: 4px 4px 4px 4px; + border: 1px solid black; + color: infotext; + font-family: monospace; + font-size: 10pt; + max-width: 600px; + opacity: 0; + overflow: hidden; + padding: 2px 5px; + position: fixed; + -webkit-transition: opacity .4s; + transition: opacity .4s; + white-space: pre-wrap; + z-index: 100; +} + +.CodeMirror-lint-mark-error, .CodeMirror-lint-mark-warning { + background-position: left bottom; + background-repeat: repeat-x; +} + +.CodeMirror-lint-mark-error { + background-image: + url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAQAAAADCAYAAAC09K7GAAAAAXNSR0IArs4c6QAAAAZiS0dEAP8A/wD/oL2nkwAAAAlwSFlzAAALEwAACxMBAJqcGAAAAAd0SU1FB9sJDw4cOCW1/KIAAAAZdEVYdENvbW1lbnQAQ3JlYXRlZCB3aXRoIEdJTVBXgQ4XAAAAHElEQVQI12NggIL/DAz/GdA5/xkY/qPKMDAwAADLZwf5rvm+LQAAAABJRU5ErkJggg==") + ; +} + +.CodeMirror-lint-mark-warning { + background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAQAAAADCAYAAAC09K7GAAAAAXNSR0IArs4c6QAAAAZiS0dEAP8A/wD/oL2nkwAAAAlwSFlzAAALEwAACxMBAJqcGAAAAAd0SU1FB9sJFhQXEbhTg7YAAAAZdEVYdENvbW1lbnQAQ3JlYXRlZCB3aXRoIEdJTVBXgQ4XAAAAMklEQVQI12NkgIIvJ3QXMjAwdDN+OaEbysDA4MPAwNDNwMCwiOHLCd1zX07o6kBVGQEAKBANtobskNMAAAAASUVORK5CYII="); +} + +.CodeMirror-lint-marker-error, .CodeMirror-lint-marker-warning { + background-position: center center; + background-repeat: no-repeat; + cursor: pointer; + display: inline-block; + height: 16px; + position: relative; + vertical-align: middle; + width: 16px; +} + +.CodeMirror-lint-message-error, .CodeMirror-lint-message-warning { + background-position: top left; + background-repeat: no-repeat; + padding-left: 18px; +} + +.CodeMirror-lint-marker-error, .CodeMirror-lint-message-error { + background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAMAAAAoLQ9TAAAAHlBMVEW7AAC7AACxAAC7AAC7AAAAAAC4AAC5AAD///+7AAAUdclpAAAABnRSTlMXnORSiwCK0ZKSAAAATUlEQVR42mWPOQ7AQAgDuQLx/z8csYRmPRIFIwRGnosRrpamvkKi0FTIiMASR3hhKW+hAN6/tIWhu9PDWiTGNEkTtIOucA5Oyr9ckPgAWm0GPBog6v4AAAAASUVORK5CYII="); +} + +.CodeMirror-lint-marker-warning, .CodeMirror-lint-message-warning { + background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAMAAAAoLQ9TAAAANlBMVEX/uwDvrwD/uwD/uwD/uwD/uwD/uwD/uwD/uwD6twD/uwAAAADurwD2tQD7uAD+ugAAAAD/uwDhmeTRAAAADHRSTlMJ8mN1EYcbmiixgACm7WbuAAAAVklEQVR42n3PUQqAIBBFUU1LLc3u/jdbOJoW1P08DA9Gba8+YWJ6gNJoNYIBzAA2chBth5kLmG9YUoG0NHAUwFXwO9LuBQL1giCQb8gC9Oro2vp5rncCIY8L8uEx5ZkAAAAASUVORK5CYII="); +} + +.CodeMirror-lint-marker-multiple { + background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAcAAAAHCAMAAADzjKfhAAAACVBMVEUAAAAAAAC/v7914kyHAAAAAXRSTlMAQObYZgAAACNJREFUeNo1ioEJAAAIwmz/H90iFFSGJgFMe3gaLZ0od+9/AQZ0ADosbYraAAAAAElFTkSuQmCC"); + background-position: right bottom; + background-repeat: no-repeat; + width: 100%; height: 100%; +} +.graphiql-container .spinner-container { + height: 36px; + left: 50%; + position: absolute; + top: 50%; + -webkit-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); + width: 36px; + z-index: 10; +} + +.graphiql-container .spinner { + -webkit-animation: rotation .6s infinite linear; + animation: rotation .6s infinite linear; + border-bottom: 6px solid rgba(150, 150, 150, .15); + border-left: 6px solid rgba(150, 150, 150, .15); + border-radius: 100%; + border-right: 6px solid rgba(150, 150, 150, .15); + border-top: 6px solid rgba(150, 150, 150, .8); + display: inline-block; + height: 24px; + position: absolute; + vertical-align: middle; + width: 24px; +} + +@-webkit-keyframes rotation { + from { -webkit-transform: rotate(0deg); transform: rotate(0deg); } + to { -webkit-transform: rotate(359deg); transform: rotate(359deg); } +} + +@keyframes rotation { + from { -webkit-transform: rotate(0deg); transform: rotate(0deg); } + to { -webkit-transform: rotate(359deg); transform: rotate(359deg); } +} +.CodeMirror-hints { + background: white; + -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.45); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.45); + font-family: 'Consolas', 'Inconsolata', 'Droid Sans Mono', 'Monaco', monospace; + font-size: 13px; + list-style: none; + margin-left: -6px; + margin: 0; + max-height: 14.5em; + overflow-y: auto; + overflow: hidden; + padding: 0; + position: absolute; + z-index: 10; +} + +.CodeMirror-hint { + border-top: solid 1px #f7f7f7; + color: #141823; + cursor: pointer; + margin: 0; + max-width: 300px; + overflow: hidden; + padding: 2px 6px; + white-space: pre; +} + +li.CodeMirror-hint-active { + background-color: #08f; + border-top-color: white; + color: white; +} + +.CodeMirror-hint-information { + border-top: solid 1px #c0c0c0; + max-width: 300px; + padding: 4px 6px; + position: relative; + z-index: 1; +} + +.CodeMirror-hint-information:first-child { + border-bottom: solid 1px #c0c0c0; + border-top: none; + margin-bottom: -1px; +} + +.CodeMirror-hint-deprecation { + background: #fffae8; + -webkit-box-shadow: inset 0 1px 1px -1px #bfb063; + box-shadow: inset 0 1px 1px -1px #bfb063; + color: #867F70; + font-family: + system, + -apple-system, + 'San Francisco', + '.SFNSDisplay-Regular', + 'Segoe UI', + Segoe, + 'Segoe WP', + 'Helvetica Neue', + helvetica, + 'Lucida Grande', + arial, + sans-serif; + font-size: 13px; + line-height: 16px; + margin-top: 4px; + max-height: 80px; + overflow: hidden; + padding: 6px; +} + +.CodeMirror-hint-deprecation .deprecation-label { + color: #c79b2e; + cursor: default; + display: block; + font-size: 9px; + font-weight: bold; + letter-spacing: 1px; + line-height: 1; + padding-bottom: 5px; + text-transform: uppercase; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.CodeMirror-hint-deprecation .deprecation-label + * { + margin-top: 0; +} + +.CodeMirror-hint-deprecation :last-child { + margin-bottom: 0; +} diff --git a/server/index-node/assets/graphiql.min.js b/server/index-node/assets/graphiql.min.js new file mode 100644 index 0000000..fd7b158 --- /dev/null +++ b/server/index-node/assets/graphiql.min.js @@ -0,0 +1 @@ +!function(f){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=f();else if("function"==typeof define&&define.amd)define([],f);else{("undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:this).GraphiQL=f()}}(function(){return function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a="function"==typeof require&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n||e)},l,l.exports,e,t,n,r)}return n[o].exports}for(var i="function"==typeof require&&require,o=0;o1&&e.setState({navStack:e.state.navStack.slice(0,-1)})},e.handleClickTypeOrField=function(t){e.showDoc(t)},e.handleSearch=function(t){e.showSearch(t)},e.state={navStack:[initialNav]},e}return _inherits(t,e),_createClass(t,[{key:"shouldComponentUpdate",value:function(e,t){return this.props.schema!==e.schema||this.state.navStack!==t.navStack}},{key:"render",value:function(){var e=this.props.schema,t=this.state.navStack,a=t[t.length-1],r=void 0;r=void 0===e?_react2.default.createElement("div",{className:"spinner-container"},_react2.default.createElement("div",{className:"spinner"})):e?a.search?_react2.default.createElement(_SearchResults2.default,{searchValue:a.search,withinType:a.def,schema:e,onClickType:this.handleClickTypeOrField,onClickField:this.handleClickTypeOrField}):1===t.length?_react2.default.createElement(_SchemaDoc2.default,{schema:e,onClickType:this.handleClickTypeOrField}):(0,_graphql.isType)(a.def)?_react2.default.createElement(_TypeDoc2.default,{schema:e,type:a.def,onClickType:this.handleClickTypeOrField,onClickField:this.handleClickTypeOrField}):_react2.default.createElement(_FieldDoc2.default,{field:a.def,onClickType:this.handleClickTypeOrField}):_react2.default.createElement("div",{className:"error-container"},"No Schema Available");var c=1===t.length||(0,_graphql.isType)(a.def)&&a.def.getFields,l=void 0;return t.length>1&&(l=t[t.length-2].name),_react2.default.createElement("div",{className:"doc-explorer",key:a.name},_react2.default.createElement("div",{className:"doc-explorer-title-bar"},l&&_react2.default.createElement("div",{className:"doc-explorer-back",onClick:this.handleNavBackClick},l),_react2.default.createElement("div",{className:"doc-explorer-title"},a.title||a.name),_react2.default.createElement("div",{className:"doc-explorer-rhs"},this.props.children)),_react2.default.createElement("div",{className:"doc-explorer-contents"},c&&_react2.default.createElement(_SearchBox2.default,{value:a.search,placeholder:"Search "+a.name+"...",onSearch:this.handleSearch}),r))}},{key:"showDoc",value:function(e){var t=this.state.navStack;t[t.length-1].def!==e&&this.setState({navStack:t.concat([{name:e.name,def:e}])})}},{key:"showDocForReference",value:function(e){"Type"===e.kind?this.showDoc(e.type):"Field"===e.kind?this.showDoc(e.field):"Argument"===e.kind&&e.field?this.showDoc(e.field):"EnumValue"===e.kind&&e.type&&this.showDoc(e.type)}},{key:"showSearch",value:function(e){var t=this.state.navStack.slice(),a=t[t.length-1];t[t.length-1]=_extends({},a,{search:e}),this.setState({navStack:t})}},{key:"reset",value:function(){this.setState({navStack:[initialNav]})}}]),t}(_react2.default.Component)).propTypes={schema:_propTypes2.default.instanceOf(_graphql.GraphQLSchema)}}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"./DocExplorer/FieldDoc":4,"./DocExplorer/SchemaDoc":6,"./DocExplorer/SearchBox":7,"./DocExplorer/SearchResults":8,"./DocExplorer/TypeDoc":9,graphql:95,"prop-types":233}],2:[function(require,module,exports){(function(global){"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function Argument(e){var t=e.arg,r=e.onClickType,a=e.showDefaultValue;return _react2.default.createElement("span",{className:"arg"},_react2.default.createElement("span",{className:"arg-name"},t.name),": ",_react2.default.createElement(_TypeLink2.default,{type:t.type,onClick:r}),!1!==a&&_react2.default.createElement(_DefaultValue2.default,{field:t}))}Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=Argument;var _react2=_interopRequireDefault("undefined"!=typeof window?window.React:void 0!==global?global.React:null),_propTypes2=_interopRequireDefault(require("prop-types")),_TypeLink2=_interopRequireDefault(require("./TypeLink")),_DefaultValue2=_interopRequireDefault(require("./DefaultValue"));Argument.propTypes={arg:_propTypes2.default.object.isRequired,onClickType:_propTypes2.default.func.isRequired,showDefaultValue:_propTypes2.default.bool}}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"./DefaultValue":3,"./TypeLink":10,"prop-types":233}],3:[function(require,module,exports){(function(global){"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function DefaultValue(e){var r=e.field,t=r.type,a=r.defaultValue;return void 0!==a?_react2.default.createElement("span",null," = ",_react2.default.createElement("span",{className:"arg-default-value"},(0,_graphql.print)((0,_graphql.astFromValue)(a,t)))):null}Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=DefaultValue;var _react2=_interopRequireDefault("undefined"!=typeof window?window.React:void 0!==global?global.React:null),_propTypes2=_interopRequireDefault(require("prop-types")),_graphql=require("graphql");DefaultValue.propTypes={field:_propTypes2.default.object.isRequired}}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{graphql:95,"prop-types":233}],4:[function(require,module,exports){(function(global){"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function e(e,t){for(var r=0;r0&&(r=_react2.default.createElement("div",{className:"doc-category"},_react2.default.createElement("div",{className:"doc-category-title"},"arguments"),t.args.map(function(t){return _react2.default.createElement("div",{key:t.name,className:"doc-category-item"},_react2.default.createElement("div",null,_react2.default.createElement(_Argument2.default,{arg:t,onClickType:e.props.onClickType})),_react2.default.createElement(_MarkdownContent2.default,{className:"doc-value-description",markdown:t.description}))}))),_react2.default.createElement("div",null,_react2.default.createElement(_MarkdownContent2.default,{className:"doc-type-description",markdown:t.description||"No Description"}),t.deprecationReason&&_react2.default.createElement(_MarkdownContent2.default,{className:"doc-deprecation",markdown:t.deprecationReason}),_react2.default.createElement("div",{className:"doc-category"},_react2.default.createElement("div",{className:"doc-category-title"},"type"),_react2.default.createElement(_TypeLink2.default,{type:t.type,onClick:this.props.onClickType})),r)}}]),t}(_react2.default.Component);FieldDoc.propTypes={field:_propTypes2.default.object,onClickType:_propTypes2.default.func},exports.default=FieldDoc}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"./Argument":2,"./MarkdownContent":5,"./TypeLink":10,"prop-types":233}],5:[function(require,module,exports){(function(global){"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function e(e,t){for(var r=0;r=100)return"break";var i=c[r];if(t!==i&&isMatch(r,e)&&l.push(_react2.default.createElement("div",{className:"doc-category-item",key:r},_react2.default.createElement(_TypeLink2.default,{type:i,onClick:n}))),i.getFields){var s=i.getFields();Object.keys(s).forEach(function(l){var c=s[l],p=void 0;if(!isMatch(l,e)){if(!c.args||!c.args.length)return;if(0===(p=c.args.filter(function(t){return isMatch(t.name,e)})).length)return}var f=_react2.default.createElement("div",{className:"doc-category-item",key:r+"."+l},t!==i&&[_react2.default.createElement(_TypeLink2.default,{key:"type",type:i,onClick:n}),"."],_react2.default.createElement("a",{className:"field-name",onClick:function(e){return a(c,i,e)}},c.name),p&&["(",_react2.default.createElement("span",{key:"args"},p.map(function(e){return _react2.default.createElement(_Argument2.default,{key:e.name,arg:e,onClickType:n,showDefaultValue:!1})})),")"]);t===i?o.push(f):u.push(f)})}}();s=!0);}catch(e){p=!0,f=e}finally{try{!s&&_.return&&_.return()}finally{if(p)throw f}}return o.length+l.length+u.length===0?_react2.default.createElement("span",{className:"doc-alert-text"},"No results found."):t&&l.length+u.length>0?_react2.default.createElement("div",null,o,_react2.default.createElement("div",{className:"doc-category"},_react2.default.createElement("div",{className:"doc-category-title"},"other results"),l,u)):_react2.default.createElement("div",null,o,l,u)}}]),t}(_react2.default.Component);SearchResults.propTypes={schema:_propTypes2.default.object,withinType:_propTypes2.default.object,searchValue:_propTypes2.default.string,onClickType:_propTypes2.default.func,onClickField:_propTypes2.default.func},exports.default=SearchResults}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"./Argument":2,"./TypeLink":10,"prop-types":233}],9:[function(require,module,exports){(function(global){"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}function Field(e){var t=e.type,a=e.field,r=e.onClickType,n=e.onClickField;return _react2.default.createElement("div",{className:"doc-category-item"},_react2.default.createElement("a",{className:"field-name",onClick:function(e){return n(a,t,e)}},a.name),a.args&&a.args.length>0&&["(",_react2.default.createElement("span",{key:"args"},a.args.map(function(e){return _react2.default.createElement(_Argument2.default,{key:e.name,arg:e,onClickType:r})})),")"],": ",_react2.default.createElement(_TypeLink2.default,{type:a.type,onClick:r}),_react2.default.createElement(_DefaultValue2.default,{field:a}),a.description&&_react2.default.createElement(_MarkdownContent2.default,{className:"field-short-description",markdown:a.description}),a.deprecationReason&&_react2.default.createElement(_MarkdownContent2.default,{className:"doc-deprecation",markdown:a.deprecationReason}))}function EnumValue(e){var t=e.value;return _react2.default.createElement("div",{className:"doc-category-item"},_react2.default.createElement("div",{className:"enum-value"},t.name),_react2.default.createElement(_MarkdownContent2.default,{className:"doc-value-description",markdown:t.description}),t.deprecationReason&&_react2.default.createElement(_MarkdownContent2.default,{className:"doc-deprecation",markdown:t.deprecationReason}))}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function e(e,t){for(var a=0;a0&&(l=_react2.default.createElement("div",{className:"doc-category"},_react2.default.createElement("div",{className:"doc-category-title"},n),c.map(function(e){return _react2.default.createElement("div",{key:e.name,className:"doc-category-item"},_react2.default.createElement(_TypeLink2.default,{type:e,onClick:a}))})));var o=void 0,i=void 0;if(t.getFields){var u=t.getFields(),p=Object.keys(u).map(function(e){return u[e]});o=_react2.default.createElement("div",{className:"doc-category"},_react2.default.createElement("div",{className:"doc-category-title"},"fields"),p.filter(function(e){return!e.isDeprecated}).map(function(e){return _react2.default.createElement(Field,{key:e.name,type:t,field:e,onClickType:a,onClickField:r})}));var s=p.filter(function(e){return e.isDeprecated});s.length>0&&(i=_react2.default.createElement("div",{className:"doc-category"},_react2.default.createElement("div",{className:"doc-category-title"},"deprecated fields"),this.state.showDeprecated?s.map(function(e){return _react2.default.createElement(Field,{key:e.name,type:t,field:e,onClickType:a,onClickField:r})}):_react2.default.createElement("button",{className:"show-btn",onClick:this.handleShowDeprecated},"Show deprecated fields...")))}var d=void 0,f=void 0;if(t instanceof _graphql.GraphQLEnumType){var m=t.getValues();d=_react2.default.createElement("div",{className:"doc-category"},_react2.default.createElement("div",{className:"doc-category-title"},"values"),m.filter(function(e){return!e.isDeprecated}).map(function(e){return _react2.default.createElement(EnumValue,{key:e.name,value:e})}));var _=m.filter(function(e){return e.isDeprecated});_.length>0&&(f=_react2.default.createElement("div",{className:"doc-category"},_react2.default.createElement("div",{className:"doc-category-title"},"deprecated values"),this.state.showDeprecated?_.map(function(e){return _react2.default.createElement(EnumValue,{key:e.name,value:e})}):_react2.default.createElement("button",{className:"show-btn",onClick:this.handleShowDeprecated},"Show deprecated values...")))}return _react2.default.createElement("div",null,_react2.default.createElement(_MarkdownContent2.default,{className:"doc-type-description",markdown:t.description||"No Description"}),t instanceof _graphql.GraphQLObjectType&&l,o,i,d,f,!(t instanceof _graphql.GraphQLObjectType)&&l)}}]),t}(_react2.default.Component);TypeDoc.propTypes={schema:_propTypes2.default.instanceOf(_graphql.GraphQLSchema),type:_propTypes2.default.object,onClickType:_propTypes2.default.func,onClickField:_propTypes2.default.func},exports.default=TypeDoc,Field.propTypes={type:_propTypes2.default.object,field:_propTypes2.default.object,onClickType:_propTypes2.default.func,onClickField:_propTypes2.default.func},EnumValue.propTypes={value:_propTypes2.default.object}}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"./Argument":2,"./DefaultValue":3,"./MarkdownContent":5,"./TypeLink":10,graphql:95,"prop-types":233}],10:[function(require,module,exports){(function(global){"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}function renderType(e,t){return e instanceof _graphql.GraphQLNonNull?_react2.default.createElement("span",null,renderType(e.ofType,t),"!"):e instanceof _graphql.GraphQLList?_react2.default.createElement("span",null,"[",renderType(e.ofType,t),"]"):_react2.default.createElement("a",{className:"type-name",onClick:function(r){return t(e,r)}},e.name)}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function e(e,t){for(var r=0;r1,r=null;if(o&&n){var u=this.state.highlight;r=_react2.default.createElement("ul",{className:"execute-options"},t.map(function(t){return _react2.default.createElement("li",{key:t.name?t.name.value:"*",className:t===u&&"selected"||null,onMouseOver:function(){return e.setState({highlight:t})},onMouseOut:function(){return e.setState({highlight:null})},onMouseUp:function(){return e._onOptionSelected(t)}},t.name?t.name.value:"")}))}var a=void 0;!this.props.isRunning&&o||(a=this._onClick);var i=void 0;this.props.isRunning||!o||n||(i=this._onOptionsOpen);var s=this.props.isRunning?_react2.default.createElement("path",{d:"M 10 10 L 23 10 L 23 23 L 10 23 z"}):_react2.default.createElement("path",{d:"M 11 9 L 24 16 L 11 23 z"});return _react2.default.createElement("div",{className:"execute-button-wrap"},_react2.default.createElement("button",{type:"button",className:"execute-button",onMouseDown:i,onClick:a,title:"Execute Query (Ctrl-Enter)"},_react2.default.createElement("svg",{width:"34",height:"34"},s)),r)}}]),t}(_react2.default.Component)).propTypes={onRun:_propTypes2.default.func,onStop:_propTypes2.default.func,isRunning:_propTypes2.default.bool,operations:_propTypes2.default.array}}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"prop-types":233}],12:[function(require,module,exports){(function(global){"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}function isPromise(e){return"object"===(void 0===e?"undefined":_typeof(e))&&"function"==typeof e.then}function observableToPromise(e){return isObservable(e)?new Promise(function(t,r){var o=e.subscribe(function(e){t(e),o.unsubscribe()},r,function(){r(new Error("no value resolved"))})}):e}function isObservable(e){return"object"===(void 0===e?"undefined":_typeof(e))&&"function"==typeof e.subscribe}Object.defineProperty(exports,"__esModule",{value:!0}),exports.GraphiQL=void 0;var _typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e},_extends=Object.assign||function(e){for(var t=1;t0){var o=this.getQueryEditor();o.operation(function(){var e=o.getCursor(),i=o.indexFromPos(e);o.setValue(r);var n=0,a=t.map(function(e){var t=e.index,r=e.string;return o.markText(o.posFromIndex(t+n),o.posFromIndex(t+(n+=r.length)),{className:"autoInsertedLeaf",clearOnEnter:!0,title:"Automatically added leaf fields"})});setTimeout(function(){return a.forEach(function(e){return e.clear()})},7e3);var s=i;t.forEach(function(e){var t=e.index,r=e.string;t=i){e=a.name&&a.name.value;break}}}this.handleRunQuery(e)}}},{key:"_didClickDragBar",value:function(e){if(0!==e.button||e.ctrlKey)return!1;var t=e.target;if(0!==t.className.indexOf("CodeMirror-gutter"))return!1;for(var r=_reactDom2.default.findDOMNode(this.resultComponent);t;){if(t===r)return!0;t=t.parentNode}return!1}}]),t}(_react2.default.Component);GraphiQL.propTypes={fetcher:_propTypes2.default.func.isRequired,schema:_propTypes2.default.instanceOf(_graphql.GraphQLSchema),query:_propTypes2.default.string,variables:_propTypes2.default.string,operationName:_propTypes2.default.string,response:_propTypes2.default.string,storage:_propTypes2.default.shape({getItem:_propTypes2.default.func,setItem:_propTypes2.default.func,removeItem:_propTypes2.default.func}),defaultQuery:_propTypes2.default.string,onEditQuery:_propTypes2.default.func,onEditVariables:_propTypes2.default.func,onEditOperationName:_propTypes2.default.func,onToggleDocs:_propTypes2.default.func,getDefaultFieldNames:_propTypes2.default.func,editorTheme:_propTypes2.default.string,onToggleHistory:_propTypes2.default.func,ResultsTooltip:_propTypes2.default.any};var _initialiseProps=function(){var e=this;this.handleClickReference=function(t){e.setState({docExplorerOpen:!0},function(){e.docExplorerComponent.showDocForReference(t)})},this.handleRunQuery=function(t){var r=++e._editorQueryID,o=e.autoCompleteLeafs()||e.state.query,i=e.state.variables,n=e.state.operationName;t&&t!==n&&(n=t,e.handleEditOperationName(n));try{e.setState({isWaitingForResponse:!0,response:null,operationName:n});var a=e._fetchQuery(o,i,n,function(t){r===e._editorQueryID&&e.setState({isWaitingForResponse:!1,response:JSON.stringify(t,null,2)})});e.setState({subscription:a})}catch(t){e.setState({isWaitingForResponse:!1,response:t.message})}},this.handleStopQuery=function(){var t=e.state.subscription;e.setState({isWaitingForResponse:!1,subscription:null}),t&&t.unsubscribe()},this.handlePrettifyQuery=function(){var t=e.getQueryEditor();t.setValue((0,_graphql.print)((0,_graphql.parse)(t.getValue())))},this.handleEditQuery=(0,_debounce2.default)(100,function(t){var r=e._updateQueryFacts(t,e.state.operationName,e.state.operations,e.state.schema);if(e.setState(_extends({query:t},r)),e.props.onEditQuery)return e.props.onEditQuery(t)}),this._updateQueryFacts=function(t,r,o,i){var n=(0,_getQueryFacts2.default)(i,t);if(n){var a=(0,_getSelectedOperationName2.default)(o,r,n.operations),s=e.props.onEditOperationName;return s&&r!==a&&s(a),_extends({operationName:a},n)}},this.handleEditVariables=function(t){e.setState({variables:t}),e.props.onEditVariables&&e.props.onEditVariables(t)},this.handleEditOperationName=function(t){var r=e.props.onEditOperationName;r&&r(t)},this.handleHintInformationRender=function(t){t.addEventListener("click",e._onClickHintInformation);var r=void 0;t.addEventListener("DOMNodeRemoved",r=function(){t.removeEventListener("DOMNodeRemoved",r),t.removeEventListener("click",e._onClickHintInformation)})},this.handleEditorRunQuery=function(){e._runQueryAtCursor()},this._onClickHintInformation=function(t){if("typeName"===t.target.className){var r=t.target.innerHTML,o=e.state.schema;if(o){var i=o.getType(r);i&&e.setState({docExplorerOpen:!0},function(){e.docExplorerComponent.showDoc(i)})}}},this.handleToggleDocs=function(){"function"==typeof e.props.onToggleDocs&&e.props.onToggleDocs(!e.state.docExplorerOpen),e.setState({docExplorerOpen:!e.state.docExplorerOpen})},this.handleToggleHistory=function(){"function"==typeof e.props.onToggleHistory&&e.props.onToggleHistory(!e.state.historyPaneOpen),e.setState({historyPaneOpen:!e.state.historyPaneOpen})},this.handleSelectHistoryQuery=function(t,r,o){e.handleEditQuery(t),e.handleEditVariables(r),e.handleEditOperationName(o)},this.handleResizeStart=function(t){if(e._didClickDragBar(t)){t.preventDefault();var r=t.clientX-(0,_elementPosition.getLeft)(t.target),o=function(t){if(0===t.buttons)return i();var o=_reactDom2.default.findDOMNode(e.editorBarComponent),n=t.clientX-(0,_elementPosition.getLeft)(o)-r,a=o.clientWidth-n;e.setState({editorFlex:n/a})},i=function(e){function t(){return e.apply(this,arguments)}return t.toString=function(){return e.toString()},t}(function(){document.removeEventListener("mousemove",o),document.removeEventListener("mouseup",i),o=null,i=null});document.addEventListener("mousemove",o),document.addEventListener("mouseup",i)}},this.handleResetResize=function(){e.setState({editorFlex:1})},this.handleDocsResizeStart=function(t){t.preventDefault();var r=e.state.docExplorerWidth,o=t.clientX-(0,_elementPosition.getLeft)(t.target),i=function(t){if(0===t.buttons)return n();var r=_reactDom2.default.findDOMNode(e),i=t.clientX-(0,_elementPosition.getLeft)(r)-o,a=r.clientWidth-i;a<100?e.setState({docExplorerOpen:!1}):e.setState({docExplorerOpen:!0,docExplorerWidth:Math.min(a,650)})},n=function(e){function t(){return e.apply(this,arguments)}return t.toString=function(){return e.toString()},t}(function(){e.state.docExplorerOpen||e.setState({docExplorerWidth:r}),document.removeEventListener("mousemove",i),document.removeEventListener("mouseup",n),i=null,n=null});document.addEventListener("mousemove",i),document.addEventListener("mouseup",n)},this.handleDocsResetResize=function(){e.setState({docExplorerWidth:DEFAULT_DOC_EXPLORER_WIDTH})},this.handleVariableResizeStart=function(t){t.preventDefault();var r=!1,o=e.state.variableEditorOpen,i=e.state.variableEditorHeight,n=t.clientY-(0,_elementPosition.getTop)(t.target),a=function(t){if(0===t.buttons)return s();r=!0;var o=_reactDom2.default.findDOMNode(e.editorBarComponent),a=t.clientY-(0,_elementPosition.getTop)(o)-n,l=o.clientHeight-a;l<60?e.setState({variableEditorOpen:!1,variableEditorHeight:i}):e.setState({variableEditorOpen:!0,variableEditorHeight:l})},s=function(e){function t(){return e.apply(this,arguments)}return t.toString=function(){return e.toString()},t}(function(){r||e.setState({variableEditorOpen:!o}),document.removeEventListener("mousemove",a),document.removeEventListener("mouseup",s),a=null,s=null});document.addEventListener("mousemove",a),document.addEventListener("mouseup",s)}};GraphiQL.Logo=function(e){return _react2.default.createElement("div",{className:"title"},e.children||_react2.default.createElement("span",null,"Graph",_react2.default.createElement("em",null,"i"),"QL"))},GraphiQL.Toolbar=function(e){return _react2.default.createElement("div",{className:"toolbar"},e.children)},GraphiQL.QueryEditor=_QueryEditor.QueryEditor,GraphiQL.VariableEditor=_VariableEditor.VariableEditor,GraphiQL.ResultViewer=_ResultViewer.ResultViewer,GraphiQL.Button=_ToolbarButton.ToolbarButton,GraphiQL.ToolbarButton=_ToolbarButton.ToolbarButton,GraphiQL.Group=_ToolbarGroup.ToolbarGroup,GraphiQL.Menu=_ToolbarMenu.ToolbarMenu,GraphiQL.MenuItem=_ToolbarMenu.ToolbarMenuItem,GraphiQL.Select=_ToolbarSelect.ToolbarSelect,GraphiQL.SelectOption=_ToolbarSelect.ToolbarSelectOption,GraphiQL.Footer=function(e){return _react2.default.createElement("div",{className:"footer"},e.children)};var defaultQuery='# Welcome to GraphiQL\n#\n# GraphiQL is an in-browser tool for writing, validating, and\n# testing GraphQL queries.\n#\n# Type queries into this side of the screen, and you will see intelligent\n# typeaheads aware of the current GraphQL type schema and live syntax and\n# validation errors highlighted within the text.\n#\n# GraphQL queries typically start with a "{" character. Lines that starts\n# with a # are ignored.\n#\n# An example GraphQL query might look like:\n#\n# {\n# field(arg: "value") {\n# subField\n# }\n# }\n#\n# Keyboard shortcuts:\n#\n# Prettify Query: Shift-Ctrl-P (or press the prettify button above)\n#\n# Run Query: Ctrl-Enter (or press the play button above)\n#\n# Auto Complete: Ctrl-Space (or just start typing)\n#\n\n'}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"../utility/CodeMirrorSizer":23,"../utility/StorageAPI":25,"../utility/debounce":26,"../utility/elementPosition":27,"../utility/fillLeafs":28,"../utility/find":29,"../utility/getQueryFacts":30,"../utility/getSelectedOperationName":31,"../utility/introspectionQueries":32,"./DocExplorer":1,"./ExecuteButton":11,"./QueryEditor":14,"./QueryHistory":15,"./ResultViewer":16,"./ToolbarButton":17,"./ToolbarGroup":18,"./ToolbarMenu":19,"./ToolbarSelect":20,"./VariableEditor":21,graphql:95,"prop-types":233}],13:[function(require,module,exports){(function(global){"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function e(e,t){for(var r=0;r20&&this.historyStore.shift();var r=this.historyStore.items,o=this.favoriteStore.items,i=r.concat(o);this.setState({queries:i})}}},{key:"render",value:function(){var e=this,t=this.state.queries.slice().reverse().map(function(t,r){return _react2.default.createElement(_HistoryQuery2.default,_extends({handleToggleFavorite:e.toggleFavorite,key:r,onSelect:e.props.onSelectQuery},t))});return _react2.default.createElement("div",null,_react2.default.createElement("div",{className:"history-title-bar"},_react2.default.createElement("div",{className:"history-title"},"History"),_react2.default.createElement("div",{className:"doc-explorer-rhs"},this.props.children)),_react2.default.createElement("div",{className:"history-contents"},t))}}]),t}(_react2.default.Component)).propTypes={query:_propTypes2.default.string,variables:_propTypes2.default.string,operationName:_propTypes2.default.string,queryID:_propTypes2.default.number,onSelectQuery:_propTypes2.default.func,storage:_propTypes2.default.object};var _initialiseProps=function(){var e=this;this.toggleFavorite=function(t,r,o,i){var a={query:t,variables:r,operationName:o};e.favoriteStore.contains(a)?i&&(a.favorite=!1,e.favoriteStore.delete(a)):(a.favorite=!0,e.favoriteStore.push(a));var s=e.historyStore.items,n=e.favoriteStore.items,u=s.concat(n);e.setState({queries:u})}}}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"../utility/QueryStore":24,"./HistoryQuery":13,graphql:95,"prop-types":233}],16:[function(require,module,exports){(function(global){"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,r){if(!(e instanceof r))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,r){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!r||"object"!=typeof r&&"function"!=typeof r?e:r}function _inherits(e,r){if("function"!=typeof r&&null!==r)throw new TypeError("Super expression must either be null or a function, not "+typeof r);e.prototype=Object.create(r&&r.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),r&&(Object.setPrototypeOf?Object.setPrototypeOf(e,r):e.__proto__=r)}Object.defineProperty(exports,"__esModule",{value:!0}),exports.ResultViewer=void 0;var _createClass=function(){function e(e,r){for(var t=0;t=65&&t<=90||!r.shiftKey&&t>=48&&t<=57||r.shiftKey&&189===t||r.shiftKey&&222===t)&&o.editor.execCommand("autocomplete")},o._onEdit=function(){o.ignoreChangeEvent||(o.cachedValue=o.editor.getValue(),o.props.onEdit&&o.props.onEdit(o.cachedValue))},o._onHasCompletion=function(e,r){(0,_onHasCompletion2.default)(e,r,o.props.onHintInformationRender)},o.cachedValue=e.value||"",o}return _inherits(r,e),_createClass(r,[{key:"componentDidMount",value:function(){var e=this,r=require("codemirror");require("codemirror/addon/hint/show-hint"),require("codemirror/addon/edit/matchbrackets"),require("codemirror/addon/edit/closebrackets"),require("codemirror/addon/fold/brace-fold"),require("codemirror/addon/fold/foldgutter"),require("codemirror/addon/lint/lint"),require("codemirror/addon/search/searchcursor"),require("codemirror/addon/search/jump-to-line"),require("codemirror/addon/dialog/dialog"),require("codemirror/keymap/sublime"),require("codemirror-graphql/variables/hint"),require("codemirror-graphql/variables/lint"),require("codemirror-graphql/variables/mode"),this.editor=r(this._node,{value:this.props.value||"",lineNumbers:!0,tabSize:2,mode:"graphql-variables",theme:this.props.editorTheme||"graphiql",keyMap:"sublime",autoCloseBrackets:!0,matchBrackets:!0,showCursorWhenSelecting:!0,readOnly:!!this.props.readOnly&&"nocursor",foldGutter:{minFoldSize:4},lint:{variableToType:this.props.variableToType},hintOptions:{variableToType:this.props.variableToType,closeOnUnfocus:!1,completeSingle:!1},gutters:["CodeMirror-linenumbers","CodeMirror-foldgutter"],extraKeys:{"Cmd-Space":function(){return e.editor.showHint({completeSingle:!1})},"Ctrl-Space":function(){return e.editor.showHint({completeSingle:!1})},"Alt-Space":function(){return e.editor.showHint({completeSingle:!1})},"Shift-Space":function(){return e.editor.showHint({completeSingle:!1})},"Cmd-Enter":function(){e.props.onRunQuery&&e.props.onRunQuery()},"Ctrl-Enter":function(){e.props.onRunQuery&&e.props.onRunQuery()},"Shift-Ctrl-P":function(){e.props.onPrettifyQuery&&e.props.onPrettifyQuery()},"Cmd-F":"findPersistent","Ctrl-F":"findPersistent","Ctrl-Left":"goSubwordLeft","Ctrl-Right":"goSubwordRight","Alt-Left":"goGroupLeft","Alt-Right":"goGroupRight"}}),this.editor.on("change",this._onEdit),this.editor.on("keyup",this._onKeyUp),this.editor.on("hasCompletion",this._onHasCompletion)}},{key:"componentDidUpdate",value:function(e){var r=require("codemirror");this.ignoreChangeEvent=!0,this.props.variableToType!==e.variableToType&&(this.editor.options.lint.variableToType=this.props.variableToType,this.editor.options.hintOptions.variableToType=this.props.variableToType,r.signal(this.editor,"change",this.editor)),this.props.value!==e.value&&this.props.value!==this.cachedValue&&(this.cachedValue=this.props.value,this.editor.setValue(this.props.value)),this.ignoreChangeEvent=!1}},{key:"componentWillUnmount",value:function(){this.editor.off("change",this._onEdit),this.editor.off("keyup",this._onKeyUp),this.editor.off("hasCompletion",this._onHasCompletion),this.editor=null}},{key:"render",value:function(){var e=this;return _react2.default.createElement("div",{className:"codemirrorWrap",ref:function(r){e._node=r}})}},{key:"getCodeMirror",value:function(){return this.editor}},{key:"getClientHeight",value:function(){return this._node&&this._node.clientHeight}}]),r}(_react2.default.Component)).propTypes={variableToType:_propTypes2.default.object,value:_propTypes2.default.string,onEdit:_propTypes2.default.func,readOnly:_propTypes2.default.bool,onHintInformationRender:_propTypes2.default.func,onPrettifyQuery:_propTypes2.default.func,onRunQuery:_propTypes2.default.func,editorTheme:_propTypes2.default.string}}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"../utility/onHasCompletion":34,codemirror:65,"codemirror-graphql/variables/hint":49,"codemirror-graphql/variables/lint":50,"codemirror-graphql/variables/mode":51,"codemirror/addon/dialog/dialog":53,"codemirror/addon/edit/closebrackets":54,"codemirror/addon/edit/matchbrackets":55,"codemirror/addon/fold/brace-fold":56,"codemirror/addon/fold/foldgutter":58,"codemirror/addon/hint/show-hint":59,"codemirror/addon/lint/lint":60,"codemirror/addon/search/jump-to-line":61,"codemirror/addon/search/searchcursor":63,"codemirror/keymap/sublime":64,"prop-types":233}],22:[function(require,module,exports){"use strict";module.exports=require("./components/GraphiQL").GraphiQL},{"./components/GraphiQL":12}],23:[function(require,module,exports){"use strict";function _classCallCheck(e,r){if(!(e instanceof r))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function e(e,r){for(var t=0;t'+e.name+"
"}Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=function(e,n,r){var t=void 0,a=void 0;require("codemirror").on(n,"select",function(e,n){if(!t){var o=n.parentNode;(t=document.createElement("div")).className="CodeMirror-hint-information",o.appendChild(t),(a=document.createElement("div")).className="CodeMirror-hint-deprecation",o.appendChild(a);var i=void 0;o.addEventListener("DOMNodeRemoved",i=function(e){e.target===o&&(o.removeEventListener("DOMNodeRemoved",i),t=null,a=null,i=null)})}var d=e.description?md.render(e.description):"Self descriptive.",p=e.type?''+renderType(e.type)+"":"";if(t.innerHTML='
'+("

"===d.slice(0,3)?"

"+p+d.slice(3):p+d)+"

",e.isDeprecated){var l=e.deprecationReason?md.render(e.deprecationReason):"";a.innerHTML='Deprecated'+l,a.style.display="block"}else a.style.display="none";r&&r(t)})};var _graphql=require("graphql"),md=new(function(e){return e&&e.__esModule?e:{default:e}}(require("markdown-it")).default)},{codemirror:65,graphql:95,"markdown-it":172}],35:[function(require,module,exports){(function(global){"use strict";function compare(a,b){if(a===b)return 0;for(var x=a.length,y=b.length,i=0,len=Math.min(x,y);i=0;i--)if(ka[i]!==kb[i])return!1;for(i=ka.length-1;i>=0;i--)if(key=ka[i],!_deepEqual(a[key],b[key],strict,actualVisitedObjects))return!1;return!0}function notDeepStrictEqual(actual,expected,message){_deepEqual(actual,expected,!0)&&fail(actual,expected,message,"notDeepStrictEqual",notDeepStrictEqual)}function expectedException(actual,expected){if(!actual||!expected)return!1;if("[object RegExp]"==Object.prototype.toString.call(expected))return expected.test(actual);try{if(actual instanceof expected)return!0}catch(e){}return!Error.isPrototypeOf(expected)&&!0===expected.call({},actual)}function _tryBlock(block){var error;try{block()}catch(e){error=e}return error}function _throws(shouldThrow,block,expected,message){var actual;if("function"!=typeof block)throw new TypeError('"block" argument must be a function');"string"==typeof expected&&(message=expected,expected=null),actual=_tryBlock(block),message=(expected&&expected.name?" ("+expected.name+").":".")+(message?" "+message:"."),shouldThrow&&!actual&&fail(actual,expected,"Missing expected exception"+message);var userProvidedMessage="string"==typeof message,isUnwantedException=!shouldThrow&&util.isError(actual),isUnexpectedException=!shouldThrow&&actual&&!expected;if((isUnwantedException&&userProvidedMessage&&expectedException(actual,expected)||isUnexpectedException)&&fail(actual,expected,"Got unwanted exception"+message),shouldThrow&&actual&&expected&&!expectedException(actual,expected)||!shouldThrow&&actual)throw actual}var util=require("util/"),hasOwn=Object.prototype.hasOwnProperty,pSlice=Array.prototype.slice,functionsHaveNames="foo"===function(){}.name,assert=module.exports=ok,regex=/\s*function\s+([^\(\s]*)\s*/;assert.AssertionError=function(options){this.name="AssertionError",this.actual=options.actual,this.expected=options.expected,this.operator=options.operator,options.message?(this.message=options.message,this.generatedMessage=!1):(this.message=getMessage(this),this.generatedMessage=!0);var stackStartFunction=options.stackStartFunction||fail;if(Error.captureStackTrace)Error.captureStackTrace(this,stackStartFunction);else{var err=new Error;if(err.stack){var out=err.stack,fn_name=getName(stackStartFunction),idx=out.indexOf("\n"+fn_name);if(idx>=0){var next_line=out.indexOf("\n",idx+1);out=out.substring(next_line+1)}this.stack=out}}},util.inherits(assert.AssertionError,Error),assert.fail=fail,assert.ok=ok,assert.equal=function(actual,expected,message){actual!=expected&&fail(actual,expected,message,"==",assert.equal)},assert.notEqual=function(actual,expected,message){actual==expected&&fail(actual,expected,message,"!=",assert.notEqual)},assert.deepEqual=function(actual,expected,message){_deepEqual(actual,expected,!1)||fail(actual,expected,message,"deepEqual",assert.deepEqual)},assert.deepStrictEqual=function(actual,expected,message){_deepEqual(actual,expected,!0)||fail(actual,expected,message,"deepStrictEqual",assert.deepStrictEqual)},assert.notDeepEqual=function(actual,expected,message){_deepEqual(actual,expected,!1)&&fail(actual,expected,message,"notDeepEqual",assert.notDeepEqual)},assert.notDeepStrictEqual=notDeepStrictEqual,assert.strictEqual=function(actual,expected,message){actual!==expected&&fail(actual,expected,message,"===",assert.strictEqual)},assert.notStrictEqual=function(actual,expected,message){actual===expected&&fail(actual,expected,message,"!==",assert.notStrictEqual)},assert.throws=function(block,error,message){_throws(!0,block,error,message)},assert.doesNotThrow=function(block,error,message){_throws(!1,block,error,message)},assert.ifError=function(err){if(err)throw err};var objectKeys=Object.keys||function(obj){var keys=[];for(var key in obj)hasOwn.call(obj,key)&&keys.push(key);return keys}}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"util/":244}],36:[function(require,module,exports){"use strict";var _codemirror2=function(obj){return obj&&obj.__esModule?obj:{default:obj}}(require("codemirror")),_graphqlLanguageServiceInterface=require("graphql-language-service-interface");_codemirror2.default.registerHelper("hint","graphql",function(editor,options){var schema=options.schema;if(schema){var cur=editor.getCursor(),token=editor.getTokenAt(cur),rawResults=(0,_graphqlLanguageServiceInterface.getAutocompleteSuggestions)(schema,editor.getValue(),cur,token),tokenStart=null!==token.type&&/"|\w/.test(token.string[0])?token.start:token.end,results={list:rawResults.map(function(item){return{text:item.label,type:schema.getType(item.detail),description:item.documentation,isDeprecated:item.isDeprecated,deprecationReason:item.deprecationReason}}),from:{line:cur.line,column:tokenStart},to:{line:cur.line,column:token.end}};return results&&results.list&&results.list.length>0&&(results.from=_codemirror2.default.Pos(results.from.line,results.from.column),results.to=_codemirror2.default.Pos(results.to.line,results.to.column),_codemirror2.default.signal(editor,"hasCompletion",editor,results,token)),results}})},{codemirror:65,"graphql-language-service-interface":76}],37:[function(require,module,exports){"use strict";function _interopRequireDefault(obj){return obj&&obj.__esModule?obj:{default:obj}}function renderField(into,typeInfo,options){renderQualifiedField(into,typeInfo,options),renderTypeAnnotation(into,typeInfo,options,typeInfo.type)}function renderQualifiedField(into,typeInfo,options){var fieldName=typeInfo.fieldDef.name;"__"!==fieldName.slice(0,2)&&(renderType(into,typeInfo,options,typeInfo.parentType),text(into,".")),text(into,fieldName,"field-name",options,(0,_SchemaReference.getFieldReference)(typeInfo))}function renderDirective(into,typeInfo,options){text(into,"@"+typeInfo.directiveDef.name,"directive-name",options,(0,_SchemaReference.getDirectiveReference)(typeInfo))}function renderArg(into,typeInfo,options){typeInfo.directiveDef?renderDirective(into,typeInfo,options):typeInfo.fieldDef&&renderQualifiedField(into,typeInfo,options);var name=typeInfo.argDef.name;text(into,"("),text(into,name,"arg-name",options,(0,_SchemaReference.getArgumentReference)(typeInfo)),renderTypeAnnotation(into,typeInfo,options,typeInfo.inputType),text(into,")")}function renderTypeAnnotation(into,typeInfo,options,t){text(into,": "),renderType(into,typeInfo,options,t)}function renderEnumValue(into,typeInfo,options){var name=typeInfo.enumValue.name;renderType(into,typeInfo,options,typeInfo.inputType),text(into,"."),text(into,name,"enum-value",options,(0,_SchemaReference.getEnumValueReference)(typeInfo))}function renderType(into,typeInfo,options,t){t instanceof _graphql.GraphQLNonNull?(renderType(into,typeInfo,options,t.ofType),text(into,"!")):t instanceof _graphql.GraphQLList?(text(into,"["),renderType(into,typeInfo,options,t.ofType),text(into,"]")):text(into,t.name,"type-name",options,(0,_SchemaReference.getTypeReference)(typeInfo,t))}function renderDescription(into,options,def){var description=def.description;if(description){var descriptionDiv=document.createElement("div");descriptionDiv.className="info-description",options.renderDescription?descriptionDiv.innerHTML=options.renderDescription(description):descriptionDiv.appendChild(document.createTextNode(description)),into.appendChild(descriptionDiv)}renderDeprecation(into,options,def)}function renderDeprecation(into,options,def){var reason=def.deprecationReason;if(reason){var deprecationDiv=document.createElement("div");deprecationDiv.className="info-deprecation",options.renderDescription?deprecationDiv.innerHTML=options.renderDescription(reason):deprecationDiv.appendChild(document.createTextNode(reason));var label=document.createElement("span");label.className="info-deprecation-label",label.appendChild(document.createTextNode("Deprecated: ")),deprecationDiv.insertBefore(label,deprecationDiv.firstChild),into.appendChild(deprecationDiv)}}function text(into,content,className,options,ref){if(className){var onClick=options.onClick,node=document.createElement(onClick?"a":"span");onClick&&(node.href="javascript:void 0",node.addEventListener("click",function(e){onClick(ref,e)})),node.className=className,node.appendChild(document.createTextNode(content)),into.appendChild(node)}else into.appendChild(document.createTextNode(content))}var _graphql=require("graphql"),_codemirror2=_interopRequireDefault(require("codemirror")),_getTypeInfo2=_interopRequireDefault(require("./utils/getTypeInfo")),_SchemaReference=require("./utils/SchemaReference");require("./utils/info-addon"),_codemirror2.default.registerHelper("info","graphql",function(token,options){if(options.schema&&token.state){var state=token.state,kind=state.kind,step=state.step,typeInfo=(0,_getTypeInfo2.default)(options.schema,token.state);if("Field"===kind&&0===step&&typeInfo.fieldDef||"AliasedField"===kind&&2===step&&typeInfo.fieldDef){var into=document.createElement("div");return renderField(into,typeInfo,options),renderDescription(into,options,typeInfo.fieldDef),into}if("Directive"===kind&&1===step&&typeInfo.directiveDef){var _into=document.createElement("div");return renderDirective(_into,typeInfo,options),renderDescription(_into,options,typeInfo.directiveDef),_into}if("Argument"===kind&&0===step&&typeInfo.argDef){var _into2=document.createElement("div");return renderArg(_into2,typeInfo,options),renderDescription(_into2,options,typeInfo.argDef),_into2}if("EnumValue"===kind&&typeInfo.enumValue&&typeInfo.enumValue.description){var _into3=document.createElement("div");return renderEnumValue(_into3,typeInfo,options),renderDescription(_into3,options,typeInfo.enumValue),_into3}if("NamedType"===kind&&typeInfo.type&&typeInfo.type.description){var _into4=document.createElement("div");return renderType(_into4,typeInfo,options,typeInfo.type),renderDescription(_into4,options,typeInfo.type),_into4}}})},{"./utils/SchemaReference":42,"./utils/getTypeInfo":44,"./utils/info-addon":46,codemirror:65,graphql:95}],38:[function(require,module,exports){"use strict";function _interopRequireDefault(obj){return obj&&obj.__esModule?obj:{default:obj}}var _codemirror2=_interopRequireDefault(require("codemirror")),_getTypeInfo2=_interopRequireDefault(require("./utils/getTypeInfo")),_SchemaReference=require("./utils/SchemaReference");require("./utils/jump-addon"),_codemirror2.default.registerHelper("jump","graphql",function(token,options){if(options.schema&&options.onClick&&token.state){var state=token.state,kind=state.kind,step=state.step,typeInfo=(0,_getTypeInfo2.default)(options.schema,state);return"Field"===kind&&0===step&&typeInfo.fieldDef||"AliasedField"===kind&&2===step&&typeInfo.fieldDef?(0,_SchemaReference.getFieldReference)(typeInfo):"Directive"===kind&&1===step&&typeInfo.directiveDef?(0,_SchemaReference.getDirectiveReference)(typeInfo):"Argument"===kind&&0===step&&typeInfo.argDef?(0,_SchemaReference.getArgumentReference)(typeInfo):"EnumValue"===kind&&typeInfo.enumValue?(0,_SchemaReference.getEnumValueReference)(typeInfo):"NamedType"===kind&&typeInfo.type?(0,_SchemaReference.getTypeReference)(typeInfo):void 0}})},{"./utils/SchemaReference":42,"./utils/getTypeInfo":44,"./utils/jump-addon":48,codemirror:65}],39:[function(require,module,exports){"use strict";var _codemirror2=function(obj){return obj&&obj.__esModule?obj:{default:obj}}(require("codemirror")),_graphqlLanguageServiceInterface=require("graphql-language-service-interface"),SEVERITY=["error","warning","information","hint"],TYPE={"GraphQL: Validation":"validation","GraphQL: Deprecation":"deprecation","GraphQL: Syntax":"syntax"};_codemirror2.default.registerHelper("lint","graphql",function(text,options){var schema=options.schema;return(0,_graphqlLanguageServiceInterface.getDiagnostics)(text,schema).map(function(error){return{message:error.message,severity:SEVERITY[error.severity-1],type:TYPE[error.source],from:_codemirror2.default.Pos(error.range.start.line,error.range.start.character),to:_codemirror2.default.Pos(error.range.end.line,error.range.end.character)}})})},{codemirror:65,"graphql-language-service-interface":76}],40:[function(require,module,exports){"use strict";function indent(state,textAfter){var levels=state.levels;return(levels&&0!==levels.length?levels[levels.length-1]-(this.electricInput.test(textAfter)?1:0):state.indentLevel)*this.config.indentUnit}var _codemirror2=function(obj){return obj&&obj.__esModule?obj:{default:obj}}(require("codemirror")),_graphqlLanguageServiceParser=require("graphql-language-service-parser");_codemirror2.default.defineMode("graphql",function(config){var parser=(0,_graphqlLanguageServiceParser.onlineParser)({eatWhitespace:function(stream){return stream.eatWhile(_graphqlLanguageServiceParser.isIgnored)},lexRules:_graphqlLanguageServiceParser.LexRules,parseRules:_graphqlLanguageServiceParser.ParseRules,editorConfig:{tabSize:config.tabSize}});return{config:config,startState:parser.startState,token:parser.token,indent:indent,electricInput:/^\s*[})\]]/,fold:"brace",lineComment:"#",closeBrackets:{pairs:'()[]{}""',explode:"()[]{}"}}})},{codemirror:65,"graphql-language-service-parser":80}],41:[function(require,module,exports){"use strict";function indent(state,textAfter){var levels=state.levels;return(levels&&0!==levels.length?levels[levels.length-1]-(this.electricInput.test(textAfter)?1:0):state.indentLevel)*this.config.indentUnit}var _codemirror2=function(obj){return obj&&obj.__esModule?obj:{default:obj}}(require("codemirror")),_graphqlLanguageServiceParser=require("graphql-language-service-parser");_codemirror2.default.defineMode("graphql-results",function(config){var parser=(0,_graphqlLanguageServiceParser.onlineParser)({eatWhitespace:function(stream){return stream.eatSpace()},lexRules:LexRules,parseRules:ParseRules,editorConfig:{tabSize:config.tabSize}});return{config:config,startState:parser.startState,token:parser.token,indent:indent,electricInput:/^\s*[}\]]/,fold:"brace",closeBrackets:{pairs:'[]{}""',explode:"[]{}"}}});var LexRules={Punctuation:/^\[|]|\{|\}|:|,/,Number:/^-?(?:0|(?:[1-9][0-9]*))(?:\.[0-9]*)?(?:[eE][+-]?[0-9]+)?/,String:/^"(?:[^"\\]|\\(?:"|\/|\\|b|f|n|r|t|u[0-9a-fA-F]{4}))*"?/,Keyword:/^true|false|null/},ParseRules={Document:[(0,_graphqlLanguageServiceParser.p)("{"),(0,_graphqlLanguageServiceParser.list)("Entry",(0,_graphqlLanguageServiceParser.p)(",")),(0,_graphqlLanguageServiceParser.p)("}")],Entry:[(0,_graphqlLanguageServiceParser.t)("String","def"),(0,_graphqlLanguageServiceParser.p)(":"),"Value"],Value:function(token){switch(token.kind){case"Number":return"NumberValue";case"String":return"StringValue";case"Punctuation":switch(token.value){case"[":return"ListValue";case"{":return"ObjectValue"}return null;case"Keyword":switch(token.value){case"true":case"false":return"BooleanValue";case"null":return"NullValue"}return null}},NumberValue:[(0,_graphqlLanguageServiceParser.t)("Number","number")],StringValue:[(0,_graphqlLanguageServiceParser.t)("String","string")],BooleanValue:[(0,_graphqlLanguageServiceParser.t)("Keyword","builtin")],NullValue:[(0,_graphqlLanguageServiceParser.t)("Keyword","keyword")],ListValue:[(0,_graphqlLanguageServiceParser.p)("["),(0,_graphqlLanguageServiceParser.list)("Value",(0,_graphqlLanguageServiceParser.p)(",")),(0,_graphqlLanguageServiceParser.p)("]")],ObjectValue:[(0,_graphqlLanguageServiceParser.p)("{"),(0,_graphqlLanguageServiceParser.list)("ObjectField",(0,_graphqlLanguageServiceParser.p)(",")),(0,_graphqlLanguageServiceParser.p)("}")],ObjectField:[(0,_graphqlLanguageServiceParser.t)("String","property"),(0,_graphqlLanguageServiceParser.p)(":"),"Value"]}},{codemirror:65,"graphql-language-service-parser":80}],42:[function(require,module,exports){"use strict";function isMetaField(fieldDef){return"__"===fieldDef.name.slice(0,2)}Object.defineProperty(exports,"__esModule",{value:!0}),exports.getFieldReference=function(typeInfo){return{kind:"Field",schema:typeInfo.schema,field:typeInfo.fieldDef,type:isMetaField(typeInfo.fieldDef)?null:typeInfo.parentType}},exports.getDirectiveReference=function(typeInfo){return{kind:"Directive",schema:typeInfo.schema,directive:typeInfo.directiveDef}},exports.getArgumentReference=function(typeInfo){return typeInfo.directiveDef?{kind:"Argument",schema:typeInfo.schema,argument:typeInfo.argDef,directive:typeInfo.directiveDef}:{kind:"Argument",schema:typeInfo.schema,argument:typeInfo.argDef,field:typeInfo.fieldDef,type:isMetaField(typeInfo.fieldDef)?null:typeInfo.parentType}},exports.getEnumValueReference=function(typeInfo){return{kind:"EnumValue",value:typeInfo.enumValue,type:(0,_graphql.getNamedType)(typeInfo.inputType)}},exports.getTypeReference=function(typeInfo,type){return{kind:"Type",schema:typeInfo.schema,type:type||typeInfo.type}};var _graphql=require("graphql")},{graphql:95}],43:[function(require,module,exports){"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=function(stack,fn){for(var reverseStateStack=[],state=stack;state&&state.kind;)reverseStateStack.push(state),state=state.prevState;for(var i=reverseStateStack.length-1;i>=0;i--)fn(reverseStateStack[i])}},{}],44:[function(require,module,exports){"use strict";function getFieldDef(schema,type,fieldName){return fieldName===_introspection.SchemaMetaFieldDef.name&&schema.getQueryType()===type?_introspection.SchemaMetaFieldDef:fieldName===_introspection.TypeMetaFieldDef.name&&schema.getQueryType()===type?_introspection.TypeMetaFieldDef:fieldName===_introspection.TypeNameMetaFieldDef.name&&(0,_graphql.isCompositeType)(type)?_introspection.TypeNameMetaFieldDef:type.getFields?type.getFields()[fieldName]:void 0}function find(array,predicate){for(var i=0;itext.length&&(proximity-=suggestion.length-text.length-1,proximity+=0===suggestion.indexOf(text)?0:.5),proximity}function lexicalDistance(a,b){var i=void 0,j=void 0,d=[],aLength=a.length,bLength=b.length;for(i=0;i<=aLength;i++)d[i]=[i];for(j=1;j<=bLength;j++)d[0][j]=j;for(i=1;i<=aLength;i++)for(j=1;j<=bLength;j++){var cost=a[i-1]===b[j-1]?0:1;d[i][j]=Math.min(d[i-1][j]+1,d[i][j-1]+1,d[i-1][j-1]+cost),i>1&&j>1&&a[i-1]===b[j-2]&&a[i-2]===b[j-1]&&(d[i][j]=Math.min(d[i][j],d[i-2][j-2]+cost))}return d[aLength][bLength]}Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=function(cursor,token,list){var hints=filterAndSortList(list,normalizeText(token.string));if(hints){var tokenStart=null!==token.type&&/"|\w/.test(token.string[0])?token.start:token.end;return{list:hints,from:{line:cursor.line,column:tokenStart},to:{line:cursor.line,column:token.end}}}}},{}],46:[function(require,module,exports){"use strict";function createState(options){return{options:options instanceof Function?{render:options}:!0===options?{}:options}}function getHoverTime(cm){var options=cm.state.info.options;return options&&options.hoverTime||500}function onMouseOver(cm,e){var state=cm.state.info,target=e.target||e.srcElement;if("SPAN"===target.nodeName&&void 0===state.hoverTimeout){var box=target.getBoundingClientRect(),hoverTime=getHoverTime(cm);state.hoverTimeout=setTimeout(onHover,hoverTime);var onMouseMove=function(){clearTimeout(state.hoverTimeout),state.hoverTimeout=setTimeout(onHover,hoverTime)},onMouseOut=function onMouseOut(){_codemirror2.default.off(document,"mousemove",onMouseMove),_codemirror2.default.off(cm.getWrapperElement(),"mouseout",onMouseOut),clearTimeout(state.hoverTimeout),state.hoverTimeout=void 0},onHover=function(){_codemirror2.default.off(document,"mousemove",onMouseMove),_codemirror2.default.off(cm.getWrapperElement(),"mouseout",onMouseOut),state.hoverTimeout=void 0,onMouseHover(cm,box)};_codemirror2.default.on(document,"mousemove",onMouseMove),_codemirror2.default.on(cm.getWrapperElement(),"mouseout",onMouseOut)}}function onMouseHover(cm,box){var pos=cm.coordsChar({left:(box.left+box.right)/2,top:(box.top+box.bottom)/2}),options=cm.state.info.options,render=options.render||cm.getHelper(pos,"info");if(render){var token=cm.getTokenAt(pos,!0);if(token){var info=render(token,options,cm);info&&showPopup(cm,box,info)}}}function showPopup(cm,box,info){var popup=document.createElement("div");popup.className="CodeMirror-info",popup.appendChild(info),document.body.appendChild(popup);var popupBox=popup.getBoundingClientRect(),popupStyle=popup.currentStyle||window.getComputedStyle(popup),popupWidth=popupBox.right-popupBox.left+parseFloat(popupStyle.marginLeft)+parseFloat(popupStyle.marginRight),popupHeight=popupBox.bottom-popupBox.top+parseFloat(popupStyle.marginTop)+parseFloat(popupStyle.marginBottom),topPos=box.bottom;popupHeight>window.innerHeight-box.bottom-15&&box.top>window.innerHeight-box.bottom&&(topPos=box.top-popupHeight),topPos<0&&(topPos=box.bottom);var leftPos=Math.max(0,window.innerWidth-popupWidth-15);leftPos>box.left&&(leftPos=box.left),popup.style.opacity=1,popup.style.top=topPos+"px",popup.style.left=leftPos+"px";var popupTimeout=void 0,onMouseOverPopup=function(){clearTimeout(popupTimeout)},onMouseOut=function(){clearTimeout(popupTimeout),popupTimeout=setTimeout(hidePopup,200)},hidePopup=function(){_codemirror2.default.off(popup,"mouseover",onMouseOverPopup),_codemirror2.default.off(popup,"mouseout",onMouseOut),_codemirror2.default.off(cm.getWrapperElement(),"mouseout",onMouseOut),popup.style.opacity?(popup.style.opacity=0,setTimeout(function(){popup.parentNode&&popup.parentNode.removeChild(popup)},600)):popup.parentNode&&popup.parentNode.removeChild(popup)};_codemirror2.default.on(popup,"mouseover",onMouseOverPopup),_codemirror2.default.on(popup,"mouseout",onMouseOut),_codemirror2.default.on(cm.getWrapperElement(),"mouseout",onMouseOut)}var _codemirror2=function(obj){return obj&&obj.__esModule?obj:{default:obj}}(require("codemirror"));_codemirror2.default.defineOption("info",!1,function(cm,options,old){if(old&&old!==_codemirror2.default.Init){var oldOnMouseOver=cm.state.info.onMouseOver;_codemirror2.default.off(cm.getWrapperElement(),"mouseover",oldOnMouseOver),clearTimeout(cm.state.info.hoverTimeout),delete cm.state.info}if(options){var state=cm.state.info=createState(options);state.onMouseOver=onMouseOver.bind(null,cm),_codemirror2.default.on(cm.getWrapperElement(),"mouseover",state.onMouseOver)}})},{codemirror:65}],47:[function(require,module,exports){"use strict";function parseObj(){var nodeStart=start,members=[];if(expect("{"),!skip("}")){do{members.push(parseMember())}while(skip(","));expect("}")}return{kind:"Object",start:nodeStart,end:lastEnd,members:members}}function parseMember(){var nodeStart=start,key="String"===kind?curToken():null;expect("String"),expect(":");var value=parseVal();return{kind:"Member",start:nodeStart,end:lastEnd,key:key,value:value}}function parseArr(){var nodeStart=start,values=[];if(expect("["),!skip("]")){do{values.push(parseVal())}while(skip(","));expect("]")}return{kind:"Array",start:nodeStart,end:lastEnd,values:values}}function parseVal(){switch(kind){case"[":return parseArr();case"{":return parseObj();case"String":case"Number":case"Boolean":case"Null":var token=curToken();return lex(),token}return expect("Value")}function curToken(){return{kind:kind,start:start,end:end,value:JSON.parse(string.slice(start,end))}}function expect(str){if(kind!==str){var found=void 0;if("EOF"===kind)found="[end of file]";else if(end-start>1)found="`"+string.slice(start,end)+"`";else{var match=string.slice(start).match(/^.+?\b/);found="`"+(match?match[0]:string[start])+"`"}throw syntaxError("Expected "+str+" but found "+found+".")}lex()}function syntaxError(message){return{message:message,start:start,end:end}}function skip(k){if(kind===k)return lex(),!0}function ch(){end31;)if(92===code)switch(ch(),code){case 34:case 47:case 92:case 98:case 102:case 110:case 114:case 116:ch();break;case 117:ch(),readHex(),readHex(),readHex(),readHex();break;default:throw syntaxError("Bad character escape sequence.")}else{if(end===strLen)throw syntaxError("Unterminated string.");ch()}if(34!==code)throw syntaxError("Unterminated string.");ch()}function readHex(){if(code>=48&&code<=57||code>=65&&code<=70||code>=97&&code<=102)return ch();throw syntaxError("Expected hexadecimal digit.")}function readNumber(){45===code&&ch(),48===code?ch():readDigits(),46===code&&(ch(),readDigits()),69!==code&&101!==code||(ch(),43!==code&&45!==code||ch(),readDigits())}function readDigits(){if(code<48||code>57)throw syntaxError("Expected decimal digit.");do{ch()}while(code>=48&&code<=57)}Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=function(str){string=str,strLen=str.length,start=end=lastEnd=-1,ch(),lex();var ast=parseObj();return expect("EOF"),ast};var string=void 0,strLen=void 0,start=void 0,end=void 0,lastEnd=void 0,code=void 0,kind=void 0},{}],48:[function(require,module,exports){"use strict";function onMouseOver(cm,event){var target=event.target||event.srcElement;if("SPAN"===target.nodeName){var box=target.getBoundingClientRect(),cursor={left:(box.left+box.right)/2,top:(box.top+box.bottom)/2};cm.state.jump.cursor=cursor,cm.state.jump.isHoldingModifier&&enableJumpMode(cm)}}function onMouseOut(cm){cm.state.jump.isHoldingModifier||!cm.state.jump.cursor?cm.state.jump.isHoldingModifier&&cm.state.jump.marker&&disableJumpMode(cm):cm.state.jump.cursor=null}function onKeyDown(cm,event){if(!cm.state.jump.isHoldingModifier&&isJumpModifier(event.key)){cm.state.jump.isHoldingModifier=!0,cm.state.jump.cursor&&enableJumpMode(cm);var onClick=function(clickEvent){var destination=cm.state.jump.destination;destination&&cm.state.jump.options.onClick(destination,clickEvent)},onMouseDown=function(_,downEvent){cm.state.jump.destination&&(downEvent.codemirrorIgnore=!0)};_codemirror2.default.on(document,"keyup",function onKeyUp(upEvent){upEvent.code===event.code&&(cm.state.jump.isHoldingModifier=!1,cm.state.jump.marker&&disableJumpMode(cm),_codemirror2.default.off(document,"keyup",onKeyUp),_codemirror2.default.off(document,"click",onClick),cm.off("mousedown",onMouseDown))}),_codemirror2.default.on(document,"click",onClick),cm.on("mousedown",onMouseDown)}}function isJumpModifier(key){return key===(isMac?"Meta":"Control")}function enableJumpMode(cm){if(!cm.state.jump.marker){var cursor=cm.state.jump.cursor,pos=cm.coordsChar(cursor),token=cm.getTokenAt(pos,!0),options=cm.state.jump.options,getDestination=options.getDestination||cm.getHelper(pos,"jump");if(getDestination){var destination=getDestination(token,options,cm);if(destination){var marker=cm.markText({line:pos.line,ch:token.start},{line:pos.line,ch:token.end},{className:"CodeMirror-jump-token"});cm.state.jump.marker=marker,cm.state.jump.destination=destination}}}}function disableJumpMode(cm){var marker=cm.state.jump.marker;cm.state.jump.marker=null,cm.state.jump.destination=null,marker.clear()}var _codemirror2=function(obj){return obj&&obj.__esModule?obj:{default:obj}}(require("codemirror"));_codemirror2.default.defineOption("jump",!1,function(cm,options,old){if(old&&old!==_codemirror2.default.Init){var oldOnMouseOver=cm.state.jump.onMouseOver;_codemirror2.default.off(cm.getWrapperElement(),"mouseover",oldOnMouseOver);var oldOnMouseOut=cm.state.jump.onMouseOut;_codemirror2.default.off(cm.getWrapperElement(),"mouseout",oldOnMouseOut),_codemirror2.default.off(document,"keydown",cm.state.jump.onKeyDown),delete cm.state.jump}if(options){var state=cm.state.jump={options:options,onMouseOver:onMouseOver.bind(null,cm),onMouseOut:onMouseOut.bind(null,cm),onKeyDown:onKeyDown.bind(null,cm)};_codemirror2.default.on(cm.getWrapperElement(),"mouseover",state.onMouseOver),_codemirror2.default.on(cm.getWrapperElement(),"mouseout",state.onMouseOut),_codemirror2.default.on(document,"keydown",state.onKeyDown)}});var isMac=navigator&&-1!==navigator.appVersion.indexOf("Mac")},{codemirror:65}],49:[function(require,module,exports){"use strict";function _interopRequireDefault(obj){return obj&&obj.__esModule?obj:{default:obj}}function getVariablesHint(cur,token,options){var state="Invalid"===token.state.kind?token.state.prevState:token.state,kind=state.kind,step=state.step;if("Document"===kind&&0===step)return(0,_hintList2.default)(cur,token,[{text:"{"}]);var variableToType=options.variableToType;if(variableToType){var typeInfo=getTypeInfo(variableToType,token.state);if("Document"===kind||"Variable"===kind&&0===step){var variableNames=Object.keys(variableToType);return(0,_hintList2.default)(cur,token,variableNames.map(function(name){return{text:'"'+name+'": ',type:variableToType[name]}}))}if(("ObjectValue"===kind||"ObjectField"===kind&&0===step)&&typeInfo.fields){var inputFields=Object.keys(typeInfo.fields).map(function(fieldName){return typeInfo.fields[fieldName]});return(0,_hintList2.default)(cur,token,inputFields.map(function(field){return{text:'"'+field.name+'": ',type:field.type,description:field.description}}))}if("StringValue"===kind||"NumberValue"===kind||"BooleanValue"===kind||"NullValue"===kind||"ListValue"===kind&&1===step||"ObjectField"===kind&&2===step||"Variable"===kind&&2===step){var namedInputType=(0,_graphql.getNamedType)(typeInfo.type);if(namedInputType instanceof _graphql.GraphQLInputObjectType)return(0,_hintList2.default)(cur,token,[{text:"{"}]);if(namedInputType instanceof _graphql.GraphQLEnumType){var valueMap=namedInputType.getValues(),values=Object.keys(valueMap).map(function(name){return valueMap[name]});return(0,_hintList2.default)(cur,token,values.map(function(value){return{text:'"'+value.name+'"',type:namedInputType,description:value.description}}))}if(namedInputType===_graphql.GraphQLBoolean)return(0,_hintList2.default)(cur,token,[{text:"true",type:_graphql.GraphQLBoolean,description:"Not false."},{text:"false",type:_graphql.GraphQLBoolean,description:"Not true."}])}}}function getTypeInfo(variableToType,tokenState){var info={type:null,fields:null};return(0,_forEachState2.default)(tokenState,function(state){if("Variable"===state.kind)info.type=variableToType[state.name];else if("ListValue"===state.kind){var nullableType=(0,_graphql.getNullableType)(info.type);info.type=nullableType instanceof _graphql.GraphQLList?nullableType.ofType:null}else if("ObjectValue"===state.kind){var objectType=(0,_graphql.getNamedType)(info.type);info.fields=objectType instanceof _graphql.GraphQLInputObjectType?objectType.getFields():null}else if("ObjectField"===state.kind){var objectField=state.name&&info.fields?info.fields[state.name]:null;info.type=objectField&&objectField.type}}),info}var _codemirror2=_interopRequireDefault(require("codemirror")),_graphql=require("graphql"),_forEachState2=_interopRequireDefault(require("../utils/forEachState")),_hintList2=_interopRequireDefault(require("../utils/hintList"));_codemirror2.default.registerHelper("hint","graphql-variables",function(editor,options){var cur=editor.getCursor(),token=editor.getTokenAt(cur),results=getVariablesHint(cur,token,options);return results&&results.list&&results.list.length>0&&(results.from=_codemirror2.default.Pos(results.from.line,results.from.column),results.to=_codemirror2.default.Pos(results.to.line,results.to.column),_codemirror2.default.signal(editor,"hasCompletion",editor,results,token)),results})},{"../utils/forEachState":43,"../utils/hintList":45,codemirror:65,graphql:95}],50:[function(require,module,exports){"use strict";function _interopRequireDefault(obj){return obj&&obj.__esModule?obj:{default:obj}}function validateVariables(editor,variableToType,variablesAST){var errors=[];return variablesAST.members.forEach(function(member){var variableName=member.key.value,type=variableToType[variableName];type?validateValue(type,member.value).forEach(function(_ref){var node=_ref[0],message=_ref[1];errors.push(lintError(editor,node,message))}):errors.push(lintError(editor,member.key,'Variable "$'+variableName+'" does not appear in any GraphQL query.'))}),errors}function validateValue(type,valueAST){if(type instanceof _graphql.GraphQLNonNull)return"Null"===valueAST.kind?[[valueAST,'Type "'+type+'" is non-nullable and cannot be null.']]:validateValue(type.ofType,valueAST);if("Null"===valueAST.kind)return[];if(type instanceof _graphql.GraphQLList){var itemType=type.ofType;return"Array"===valueAST.kind?mapCat(valueAST.values,function(item){return validateValue(itemType,item)}):validateValue(itemType,valueAST)}if(type instanceof _graphql.GraphQLInputObjectType){if("Object"!==valueAST.kind)return[[valueAST,'Type "'+type+'" must be an Object.']];var providedFields=Object.create(null),fieldErrors=mapCat(valueAST.members,function(member){var fieldName=member.key.value;providedFields[fieldName]=!0;var inputField=type.getFields()[fieldName];return inputField?validateValue(inputField?inputField.type:void 0,member.value):[[member.key,'Type "'+type+'" does not have a field "'+fieldName+'".']]});return Object.keys(type.getFields()).forEach(function(fieldName){providedFields[fieldName]||type.getFields()[fieldName].type instanceof _graphql.GraphQLNonNull&&fieldErrors.push([valueAST,'Object of type "'+type+'" is missing required field "'+fieldName+'".'])}),fieldErrors}return"Boolean"===type.name&&"Boolean"!==valueAST.kind||"String"===type.name&&"String"!==valueAST.kind||"ID"===type.name&&"Number"!==valueAST.kind&&"String"!==valueAST.kind||"Float"===type.name&&"Number"!==valueAST.kind||"Int"===type.name&&("Number"!==valueAST.kind||(0|valueAST.value)!==valueAST.value)?[[valueAST,'Expected value of type "'+type+'".']]:(type instanceof _graphql.GraphQLEnumType||type instanceof _graphql.GraphQLScalarType)&&("String"!==valueAST.kind&&"Number"!==valueAST.kind&&"Boolean"!==valueAST.kind&&"Null"!==valueAST.kind||isNullish(type.parseValue(valueAST.value)))?[[valueAST,'Expected value of type "'+type+'".']]:[]}function lintError(editor,node,message){return{message:message,severity:"error",type:"validation",from:editor.posFromIndex(node.start),to:editor.posFromIndex(node.end)}}function isNullish(value){return null===value||void 0===value||value!==value}function mapCat(array,mapper){return Array.prototype.concat.apply([],array.map(mapper))}var _codemirror2=_interopRequireDefault(require("codemirror")),_graphql=require("graphql"),_jsonParse2=_interopRequireDefault(require("../utils/jsonParse"));_codemirror2.default.registerHelper("lint","graphql-variables",function(text,options,editor){if(!text)return[];var ast=void 0;try{ast=(0,_jsonParse2.default)(text)}catch(syntaxError){if(syntaxError.stack)throw syntaxError;return[lintError(editor,syntaxError,syntaxError.message)]}var variableToType=options.variableToType;return variableToType?validateVariables(editor,variableToType,ast):[]})},{"../utils/jsonParse":47,codemirror:65,graphql:95}],51:[function(require,module,exports){"use strict";function indent(state,textAfter){var levels=state.levels;return(levels&&0!==levels.length?levels[levels.length-1]-(this.electricInput.test(textAfter)?1:0):state.indentLevel)*this.config.indentUnit}function namedKey(style){return{style:style,match:function(token){return"String"===token.kind},update:function(state,token){state.name=token.value.slice(1,-1)}}}var _codemirror2=function(obj){return obj&&obj.__esModule?obj:{default:obj}}(require("codemirror")),_graphqlLanguageServiceParser=require("graphql-language-service-parser");_codemirror2.default.defineMode("graphql-variables",function(config){var parser=(0,_graphqlLanguageServiceParser.onlineParser)({eatWhitespace:function(stream){return stream.eatSpace()},lexRules:LexRules,parseRules:ParseRules,editorConfig:{tabSize:config.tabSize}});return{config:config,startState:parser.startState,token:parser.token,indent:indent,electricInput:/^\s*[}\]]/,fold:"brace",closeBrackets:{pairs:'[]{}""',explode:"[]{}"}}});var LexRules={Punctuation:/^\[|]|\{|\}|:|,/,Number:/^-?(?:0|(?:[1-9][0-9]*))(?:\.[0-9]*)?(?:[eE][+-]?[0-9]+)?/,String:/^"(?:[^"\\]|\\(?:"|\/|\\|b|f|n|r|t|u[0-9a-fA-F]{4}))*"?/,Keyword:/^true|false|null/},ParseRules={Document:[(0,_graphqlLanguageServiceParser.p)("{"),(0,_graphqlLanguageServiceParser.list)("Variable",(0,_graphqlLanguageServiceParser.opt)((0,_graphqlLanguageServiceParser.p)(","))),(0,_graphqlLanguageServiceParser.p)("}")],Variable:[namedKey("variable"),(0,_graphqlLanguageServiceParser.p)(":"),"Value"],Value:function(token){switch(token.kind){case"Number":return"NumberValue";case"String":return"StringValue";case"Punctuation":switch(token.value){case"[":return"ListValue";case"{":return"ObjectValue"}return null;case"Keyword":switch(token.value){case"true":case"false":return"BooleanValue";case"null":return"NullValue"}return null}},NumberValue:[(0,_graphqlLanguageServiceParser.t)("Number","number")],StringValue:[(0,_graphqlLanguageServiceParser.t)("String","string")],BooleanValue:[(0,_graphqlLanguageServiceParser.t)("Keyword","builtin")],NullValue:[(0,_graphqlLanguageServiceParser.t)("Keyword","keyword")],ListValue:[(0,_graphqlLanguageServiceParser.p)("["),(0,_graphqlLanguageServiceParser.list)("Value",(0,_graphqlLanguageServiceParser.opt)((0,_graphqlLanguageServiceParser.p)(","))),(0,_graphqlLanguageServiceParser.p)("]")],ObjectValue:[(0,_graphqlLanguageServiceParser.p)("{"),(0,_graphqlLanguageServiceParser.list)("ObjectField",(0,_graphqlLanguageServiceParser.opt)((0,_graphqlLanguageServiceParser.p)(","))),(0,_graphqlLanguageServiceParser.p)("}")],ObjectField:[namedKey("attribute"),(0,_graphqlLanguageServiceParser.p)(":"),"Value"]}},{codemirror:65,"graphql-language-service-parser":80}],52:[function(require,module,exports){!function(mod){mod("object"==typeof exports&&"object"==typeof module?require("../../lib/codemirror"):CodeMirror)}(function(CodeMirror){"use strict";function firstNonWS(str){var found=str.search(nonWS);return-1==found?0:found}function probablyInsideString(cm,pos,line){return/\bstring\b/.test(cm.getTokenTypeAt(Pos(pos.line,0)))&&!/^[\'\"\`]/.test(line)}function getMode(cm,pos){var mode=cm.getMode();return!1!==mode.useInnerComments&&mode.innerMode?cm.getModeAt(pos):mode}var noOptions={},nonWS=/[^\s\u00a0]/,Pos=CodeMirror.Pos;CodeMirror.commands.toggleComment=function(cm){cm.toggleComment()},CodeMirror.defineExtension("toggleComment",function(options){options||(options=noOptions);for(var cm=this,minLine=1/0,ranges=this.listSelections(),mode=null,i=ranges.length-1;i>=0;i--){var from=ranges[i].from(),to=ranges[i].to();from.line>=minLine||(to.line>=minLine&&(to=Pos(minLine,0)),minLine=from.line,null==mode?cm.uncomment(from,to,options)?mode="un":(cm.lineComment(from,to,options),mode="line"):"un"==mode?cm.uncomment(from,to,options):cm.lineComment(from,to,options))}}),CodeMirror.defineExtension("lineComment",function(from,to,options){options||(options=noOptions);var self=this,mode=getMode(self,from),firstLine=self.getLine(from.line);if(null!=firstLine&&!probablyInsideString(self,from,firstLine)){var commentString=options.lineComment||mode.lineComment;if(commentString){var end=Math.min(0!=to.ch||to.line==from.line?to.line+1:to.line,self.lastLine()+1),pad=null==options.padding?" ":options.padding,blankLines=options.commentBlankLines||from.line==to.line;self.operation(function(){if(options.indent){for(var baseString=null,i=from.line;iwhitespace.length)&&(baseString=whitespace)}for(i=from.line;iend||self.operation(function(){if(0!=options.fullLines){var lastLineHasText=nonWS.test(self.getLine(end));self.replaceRange(pad+endString,Pos(end)),self.replaceRange(startString+pad,Pos(from.line,0));var lead=options.blockCommentLead||mode.blockCommentLead;if(null!=lead)for(var i=from.line+1;i<=end;++i)(i!=end||lastLineHasText)&&self.replaceRange(lead+pad,Pos(i,0))}else self.replaceRange(endString,to),self.replaceRange(startString,from)})}}else(options.lineComment||mode.lineComment)&&0!=options.fullLines&&self.lineComment(from,to,options)}),CodeMirror.defineExtension("uncomment",function(from,to,options){options||(options=noOptions);var didSomething,self=this,mode=getMode(self,from),end=Math.min(0!=to.ch||to.line==from.line?to.line:to.line-1,self.lastLine()),start=Math.min(from.line,end),lineString=options.lineComment||mode.lineComment,lines=[],pad=null==options.padding?" ":options.padding;lineComment:if(lineString){for(var i=start;i<=end;++i){var line=self.getLine(i),found=line.indexOf(lineString);if(found>-1&&!/comment/.test(self.getTokenTypeAt(Pos(i,found+1)))&&(found=-1),-1==found&&nonWS.test(line))break lineComment;if(found>-1&&nonWS.test(line.slice(0,found)))break lineComment;lines.push(line)}if(self.operation(function(){for(var i=start;i<=end;++i){var line=lines[i-start],pos=line.indexOf(lineString),endPos=pos+lineString.length;pos<0||(line.slice(endPos,endPos+pad.length)==pad&&(endPos+=pad.length),didSomething=!0,self.replaceRange("",Pos(i,pos),Pos(i,endPos)))}}),didSomething)return!0}var startString=options.blockCommentStart||mode.blockCommentStart,endString=options.blockCommentEnd||mode.blockCommentEnd;if(!startString||!endString)return!1;var lead=options.blockCommentLead||mode.blockCommentLead,startLine=self.getLine(start),open=startLine.indexOf(startString);if(-1==open)return!1;var endLine=end==start?startLine:self.getLine(end),close=endLine.indexOf(endString,end==start?open+startString.length:0);-1==close&&start!=end&&(endLine=self.getLine(--end),close=endLine.indexOf(endString));var insideStart=Pos(start,open+1),insideEnd=Pos(end,close+1);if(-1==close||!/comment/.test(self.getTokenTypeAt(insideStart))||!/comment/.test(self.getTokenTypeAt(insideEnd))||self.getRange(insideStart,insideEnd,"\n").indexOf(endString)>-1)return!1;var lastStart=startLine.lastIndexOf(startString,from.ch),firstEnd=-1==lastStart?-1:startLine.slice(0,from.ch).indexOf(endString,lastStart+startString.length);if(-1!=lastStart&&-1!=firstEnd&&firstEnd+endString.length!=from.ch)return!1;firstEnd=endLine.indexOf(endString,to.ch);var almostLastStart=endLine.slice(to.ch).lastIndexOf(startString,firstEnd-to.ch);return lastStart=-1==firstEnd||-1==almostLastStart?-1:to.ch+almostLastStart,(-1==firstEnd||-1==lastStart||lastStart==to.ch)&&(self.operation(function(){self.replaceRange("",Pos(end,close-(pad&&endLine.slice(close-pad.length,close)==pad?pad.length:0)),Pos(end,close+endString.length));var openEnd=open+startString.length;if(pad&&startLine.slice(openEnd,openEnd+pad.length)==pad&&(openEnd+=pad.length),self.replaceRange("",Pos(start,open),Pos(start,openEnd)),lead)for(var i=start+1;i<=end;++i){var line=self.getLine(i),found=line.indexOf(lead);if(-1!=found&&!nonWS.test(line.slice(0,found))){var foundEnd=found+lead.length;pad&&line.slice(foundEnd,foundEnd+pad.length)==pad&&(foundEnd+=pad.length),self.replaceRange("",Pos(i,found),Pos(i,foundEnd))}}}),!0)})})},{"../../lib/codemirror":65}],53:[function(require,module,exports){!function(mod){mod("object"==typeof exports&&"object"==typeof module?require("../../lib/codemirror"):CodeMirror)}(function(CodeMirror){function dialogDiv(cm,template,bottom){var dialog;return dialog=cm.getWrapperElement().appendChild(document.createElement("div")),dialog.className=bottom?"CodeMirror-dialog CodeMirror-dialog-bottom":"CodeMirror-dialog CodeMirror-dialog-top","string"==typeof template?dialog.innerHTML=template:dialog.appendChild(template),dialog}function closeNotification(cm,newVal){cm.state.currentNotificationClose&&cm.state.currentNotificationClose(),cm.state.currentNotificationClose=newVal}CodeMirror.defineExtension("openDialog",function(template,callback,options){function close(newVal){if("string"==typeof newVal)inp.value=newVal;else{if(closed)return;closed=!0,dialog.parentNode.removeChild(dialog),me.focus(),options.onClose&&options.onClose(dialog)}}options||(options={}),closeNotification(this,null);var button,dialog=dialogDiv(this,template,options.bottom),closed=!1,me=this,inp=dialog.getElementsByTagName("input")[0];return inp?(inp.focus(),options.value&&(inp.value=options.value,!1!==options.selectValueOnOpen&&inp.select()),options.onInput&&CodeMirror.on(inp,"input",function(e){options.onInput(e,inp.value,close)}),options.onKeyUp&&CodeMirror.on(inp,"keyup",function(e){options.onKeyUp(e,inp.value,close)}),CodeMirror.on(inp,"keydown",function(e){options&&options.onKeyDown&&options.onKeyDown(e,inp.value,close)||((27==e.keyCode||!1!==options.closeOnEnter&&13==e.keyCode)&&(inp.blur(),CodeMirror.e_stop(e),close()),13==e.keyCode&&callback(inp.value,e))}),!1!==options.closeOnBlur&&CodeMirror.on(inp,"blur",close)):(button=dialog.getElementsByTagName("button")[0])&&(CodeMirror.on(button,"click",function(){close(),me.focus()}),!1!==options.closeOnBlur&&CodeMirror.on(button,"blur",close),button.focus()),close}),CodeMirror.defineExtension("openConfirm",function(template,callbacks,options){function close(){closed||(closed=!0,dialog.parentNode.removeChild(dialog),me.focus())}closeNotification(this,null);var dialog=dialogDiv(this,template,options&&options.bottom),buttons=dialog.getElementsByTagName("button"),closed=!1,me=this,blurring=1;buttons[0].focus();for(var i=0;i0;return{anchor:new Pos(sel.anchor.line,sel.anchor.ch+(inverted?-1:1)),head:new Pos(sel.head.line,sel.head.ch+(inverted?1:-1))}}function handleChar(cm,ch){var conf=getConfig(cm);if(!conf||cm.getOption("disableInput"))return CodeMirror.Pass;var pairs=getOption(conf,"pairs"),pos=pairs.indexOf(ch);if(-1==pos)return CodeMirror.Pass;for(var type,triples=getOption(conf,"triples"),identical=pairs.charAt(pos+1)==ch,ranges=cm.listSelections(),opening=pos%2==0,i=0;i1&&triples.indexOf(ch)>=0&&cm.getRange(Pos(cur.line,cur.ch-2),cur)==ch+ch&&(cur.ch<=2||cm.getRange(Pos(cur.line,cur.ch-3),Pos(cur.line,cur.ch-2))!=ch))curType="addFour";else if(identical){if(CodeMirror.isWordChar(next)||!enteringString(cm,cur,ch))return CodeMirror.Pass;curType="both"}else{if(!opening||cm.getLine(cur.line).length!=cur.ch&&!isClosingBracket(next,pairs)&&!/\s/.test(next))return CodeMirror.Pass;curType="both"}else curType=identical&&stringStartsAfter(cm,cur)?"both":triples.indexOf(ch)>=0&&cm.getRange(cur,Pos(cur.line,cur.ch+3))==ch+ch+ch?"skipThree":"skip";if(type){if(type!=curType)return CodeMirror.Pass}else type=curType}var left=pos%2?pairs.charAt(pos-1):ch,right=pos%2?ch:pairs.charAt(pos+1);cm.operation(function(){if("skip"==type)cm.execCommand("goCharRight");else if("skipThree"==type)for(i=0;i<3;i++)cm.execCommand("goCharRight");else if("surround"==type){for(var sels=cm.getSelections(),i=0;i-1&&pos%2==1}function charsAround(cm,pos){var str=cm.getRange(Pos(pos.line,pos.ch-1),Pos(pos.line,pos.ch+1));return 2==str.length?str:null}function enteringString(cm,pos,ch){var line=cm.getLine(pos.line),token=cm.getTokenAt(pos);if(/\bstring2?\b/.test(token.type)||stringStartsAfter(cm,pos))return!1;var stream=new CodeMirror.StringStream(line.slice(0,pos.ch)+ch+line.slice(pos.ch),4);for(stream.pos=stream.start=token.start;;){var type1=cm.getMode().token(stream,token.state);if(stream.pos>=pos.ch+1)return/\bstring2?\b/.test(type1);stream.start=stream.pos}}function stringStartsAfter(cm,pos){var token=cm.getTokenAt(Pos(pos.line,pos.ch+1));return/\bstring/.test(token.type)&&token.start==pos.ch}var defaults={pairs:"()[]{}''\"\"",triples:"",explode:"[]{}"},Pos=CodeMirror.Pos;CodeMirror.defineOption("autoCloseBrackets",!1,function(cm,val,old){old&&old!=CodeMirror.Init&&(cm.removeKeyMap(keyMap),cm.state.closeBrackets=null),val&&(cm.state.closeBrackets=val,cm.addKeyMap(keyMap))});for(var bind=defaults.pairs+"`",keyMap={Backspace:function(cm){var conf=getConfig(cm);if(!conf||cm.getOption("disableInput"))return CodeMirror.Pass;for(var pairs=getOption(conf,"pairs"),ranges=cm.listSelections(),i=0;i=0;i--){var cur=ranges[i].head;cm.replaceRange("",Pos(cur.line,cur.ch-1),Pos(cur.line,cur.ch+1),"+delete")}},Enter:function(cm){var conf=getConfig(cm),explode=conf&&getOption(conf,"explode");if(!explode||cm.getOption("disableInput"))return CodeMirror.Pass;for(var ranges=cm.listSelections(),i=0;i=0&&matching[line.text.charAt(pos)]||matching[line.text.charAt(++pos)];if(!match)return null;var dir=">"==match.charAt(1)?1:-1;if(config&&config.strict&&dir>0!=(pos==where.ch))return null;var style=cm.getTokenTypeAt(Pos(where.line,pos+1)),found=scanForBracket(cm,Pos(where.line,pos+(dir>0?1:0)),dir,style||null,config);return null==found?null:{from:Pos(where.line,pos),to:found&&found.pos,match:found&&found.ch==match.charAt(0),forward:dir>0}}function scanForBracket(cm,where,dir,style,config){for(var maxScanLen=config&&config.maxScanLineLength||1e4,maxScanLines=config&&config.maxScanLines||1e3,stack=[],re=config&&config.bracketRegex?config.bracketRegex:/[(){}[\]]/,lineEnd=dir>0?Math.min(where.line+maxScanLines,cm.lastLine()+1):Math.max(cm.firstLine()-1,where.line-maxScanLines),lineNo=where.line;lineNo!=lineEnd;lineNo+=dir){var line=cm.getLine(lineNo);if(line){var pos=dir>0?0:line.length-1,end=dir>0?line.length:-1;if(!(line.length>maxScanLen))for(lineNo==where.line&&(pos=where.ch-(dir<0?1:0));pos!=end;pos+=dir){var ch=line.charAt(pos);if(re.test(ch)&&(void 0===style||cm.getTokenTypeAt(Pos(lineNo,pos+1))==style))if(">"==matching[ch].charAt(1)==dir>0)stack.push(ch);else{if(!stack.length)return{pos:Pos(lineNo,pos),ch:ch};stack.pop()}}}}return lineNo-dir!=(dir>0?cm.lastLine():cm.firstLine())&&null}function matchBrackets(cm,autoclear,config){for(var maxHighlightLen=cm.state.matchBrackets.maxHighlightLineLength||1e3,marks=[],ranges=cm.listSelections(),i=0;i",")":"(<","[":"]>","]":"[<","{":"}>","}":"{<"},currentlyHighlighted=null;CodeMirror.defineOption("matchBrackets",!1,function(cm,val,old){old&&old!=CodeMirror.Init&&(cm.off("cursorActivity",doMatchBrackets),currentlyHighlighted&&(currentlyHighlighted(),currentlyHighlighted=null)),val&&(cm.state.matchBrackets="object"==typeof val?val:{},cm.on("cursorActivity",doMatchBrackets))}),CodeMirror.defineExtension("matchBrackets",function(){matchBrackets(this,!0)}),CodeMirror.defineExtension("findMatchingBracket",function(pos,config,oldConfig){return(oldConfig||"boolean"==typeof config)&&(oldConfig?(oldConfig.strict=config,config=oldConfig):config=config?{strict:!0}:null),findMatchingBracket(this,pos,config)}),CodeMirror.defineExtension("scanForBracket",function(pos,dir,style,config){return scanForBracket(this,pos,dir,style,config)})})},{"../../lib/codemirror":65}],56:[function(require,module,exports){!function(mod){mod("object"==typeof exports&&"object"==typeof module?require("../../lib/codemirror"):CodeMirror)}(function(CodeMirror){"use strict";CodeMirror.registerHelper("fold","brace",function(cm,start){function findOpening(openCh){for(var at=start.ch,pass=0;;){var found=at<=0?-1:lineText.lastIndexOf(openCh,at-1);if(-1!=found){if(1==pass&&foundcm.lastLine())return null;var start=cm.getTokenAt(CodeMirror.Pos(line,1));if(/\S/.test(start.string)||(start=cm.getTokenAt(CodeMirror.Pos(line,start.end+1))),"keyword"!=start.type||"import"!=start.string)return null;for(var i=line,e=Math.min(cm.lastLine(),line+10);i<=e;++i){var semi=cm.getLine(i).indexOf(";");if(-1!=semi)return{startCh:start.end,end:CodeMirror.Pos(i,semi)}}}var prev,startLine=start.line,has=hasImport(startLine);if(!has||hasImport(startLine-1)||(prev=hasImport(startLine-2))&&prev.end.line==startLine-1)return null;for(var end=has.end;;){var next=hasImport(end.line+1);if(null==next)break;end=next.end}return{from:cm.clipPos(CodeMirror.Pos(startLine,has.startCh+1)),to:end}}),CodeMirror.registerHelper("fold","include",function(cm,start){function hasInclude(line){if(linecm.lastLine())return null;var start=cm.getTokenAt(CodeMirror.Pos(line,1));return/\S/.test(start.string)||(start=cm.getTokenAt(CodeMirror.Pos(line,start.end+1))),"meta"==start.type&&"#include"==start.string.slice(0,8)?start.start+8:void 0}var startLine=start.line,has=hasInclude(startLine);if(null==has||null!=hasInclude(startLine-1))return null;for(var end=startLine;null!=hasInclude(end+1);)++end;return{from:CodeMirror.Pos(startLine,has+1),to:cm.clipPos(CodeMirror.Pos(end))}})})},{"../../lib/codemirror":65}],57:[function(require,module,exports){!function(mod){mod("object"==typeof exports&&"object"==typeof module?require("../../lib/codemirror"):CodeMirror)}(function(CodeMirror){"use strict";function doFold(cm,pos,options,force){function getRange(allowFolded){var range=finder(cm,pos);if(!range||range.to.line-range.from.linecm.firstLine();)pos=CodeMirror.Pos(pos.line-1,0),range=getRange(!1);if(range&&!range.cleared&&"unfold"!==force){var myWidget=makeWidget(cm,options);CodeMirror.on(myWidget,"mousedown",function(e){myRange.clear(),CodeMirror.e_preventDefault(e)});var myRange=cm.markText(range.from,range.to,{replacedWith:myWidget,clearOnEnter:getOption(cm,options,"clearOnEnter"),__isFold:!0});myRange.on("clear",function(from,to){CodeMirror.signal(cm,"unfold",cm,from,to)}),CodeMirror.signal(cm,"fold",cm,range.from,range.to)}}function makeWidget(cm,options){var widget=getOption(cm,options,"widget");if("string"==typeof widget){var text=document.createTextNode(widget);(widget=document.createElement("span")).appendChild(text),widget.className="CodeMirror-foldmarker"}return widget}function getOption(cm,options,name){if(options&&void 0!==options[name])return options[name];var editorOptions=cm.options.foldOptions;return editorOptions&&void 0!==editorOptions[name]?editorOptions[name]:defaultOptions[name]}CodeMirror.newFoldFunction=function(rangeFinder,widget){return function(cm,pos){doFold(cm,pos,{rangeFinder:rangeFinder,widget:widget})}},CodeMirror.defineExtension("foldCode",function(pos,options,force){doFold(this,pos,options,force)}),CodeMirror.defineExtension("isFolded",function(pos){for(var marks=this.findMarksAt(pos),i=0;i=minSize&&(mark=marker(opts.indicatorOpen))}cm.setGutterMarker(line,opts.gutter,mark),++cur})}function updateInViewport(cm){var vp=cm.getViewport(),state=cm.state.foldGutter;state&&(cm.operation(function(){updateFoldInfo(cm,vp.from,vp.to)}),state.from=vp.from,state.to=vp.to)}function onGutterClick(cm,line,gutter){var state=cm.state.foldGutter;if(state){var opts=state.options;if(gutter==opts.gutter){var folded=isFolded(cm,line);folded?folded.clear():cm.foldCode(Pos(line,0),opts.rangeFinder)}}}function onChange(cm){var state=cm.state.foldGutter;if(state){var opts=state.options;state.from=state.to=0,clearTimeout(state.changeUpdate),state.changeUpdate=setTimeout(function(){updateInViewport(cm)},opts.foldOnChangeTimeSpan||600)}}function onViewportChange(cm){var state=cm.state.foldGutter;if(state){var opts=state.options;clearTimeout(state.changeUpdate),state.changeUpdate=setTimeout(function(){var vp=cm.getViewport();state.from==state.to||vp.from-state.to>20||state.from-vp.to>20?updateInViewport(cm):cm.operation(function(){vp.fromstate.to&&(updateFoldInfo(cm,state.to,vp.to),state.to=vp.to)})},opts.updateViewportTimeSpan||400)}}function onFold(cm,from){var state=cm.state.foldGutter;if(state){var line=from.line;line>=state.from&&line0&&old.to.ch-old.from.ch!=nw.to.ch-nw.from.ch}function parseOptions(cm,pos,options){var editor=cm.options.hintOptions,out={};for(var prop in defaultOptions)out[prop]=defaultOptions[prop];if(editor)for(var prop in editor)void 0!==editor[prop]&&(out[prop]=editor[prop]);if(options)for(var prop in options)void 0!==options[prop]&&(out[prop]=options[prop]);return out.hint.resolve&&(out.hint=out.hint.resolve(cm,pos)),out}function getText(completion){return"string"==typeof completion?completion:completion.text}function buildKeyMap(completion,handle){function addBinding(key,val){var bound;bound="string"!=typeof val?function(cm){return val(cm,handle)}:baseMap.hasOwnProperty(val)?baseMap[val]:val,ourMap[key]=bound}var baseMap={Up:function(){handle.moveFocus(-1)},Down:function(){handle.moveFocus(1)},PageUp:function(){handle.moveFocus(1-handle.menuSize(),!0)},PageDown:function(){handle.moveFocus(handle.menuSize()-1,!0)},Home:function(){handle.setFocus(0)},End:function(){handle.setFocus(handle.length-1)},Enter:handle.pick,Tab:handle.pick,Esc:handle.close},custom=completion.options.customKeys,ourMap=custom?{}:baseMap;if(custom)for(var key in custom)custom.hasOwnProperty(key)&&addBinding(key,custom[key]);var extra=completion.options.extraKeys;if(extra)for(var key in extra)extra.hasOwnProperty(key)&&addBinding(key,extra[key]);return ourMap}function getHintElement(hintsElement,el){for(;el&&el!=hintsElement;){if("LI"===el.nodeName.toUpperCase()&&el.parentNode==hintsElement)return el;el=el.parentNode}}function Widget(completion,data){this.completion=completion,this.data=data,this.picked=!1;var widget=this,cm=completion.cm,hints=this.hints=document.createElement("ul");hints.className="CodeMirror-hints",this.selectedHint=data.selectedHint||0;for(var completions=data.list,i=0;ihints.clientHeight+1,startScroll=cm.getScrollInfo();if(overlapY>0){var height=box.bottom-box.top;if(pos.top-(pos.bottom-box.top)-height>0)hints.style.top=(top=pos.top-height)+"px",below=!1;else if(height>winH){hints.style.height=winH-5+"px",hints.style.top=(top=pos.bottom-box.top)+"px";var cursor=cm.getCursor();data.from.ch!=cursor.ch&&(pos=cm.cursorCoords(cursor),hints.style.left=(left=pos.left)+"px",box=hints.getBoundingClientRect())}}var overlapX=box.right-winW;if(overlapX>0&&(box.right-box.left>winW&&(hints.style.width=winW-5+"px",overlapX-=box.right-box.left-winW),hints.style.left=(left=pos.left-overlapX)+"px"),scrolls)for(var node=hints.firstChild;node;node=node.nextSibling)node.style.paddingRight=cm.display.nativeBarWidth+"px";if(cm.addKeyMap(this.keyMap=buildKeyMap(completion,{moveFocus:function(n,avoidWrap){widget.changeActive(widget.selectedHint+n,avoidWrap)},setFocus:function(n){widget.changeActive(n)},menuSize:function(){return widget.screenAmount()},length:completions.length,close:function(){completion.close()},pick:function(){widget.pick()},data:data})),completion.options.closeOnUnfocus){var closingOnBlur;cm.on("blur",this.onBlur=function(){closingOnBlur=setTimeout(function(){completion.close()},100)}),cm.on("focus",this.onFocus=function(){clearTimeout(closingOnBlur)})}return cm.on("scroll",this.onScroll=function(){var curScroll=cm.getScrollInfo(),editor=cm.getWrapperElement().getBoundingClientRect(),newTop=top+startScroll.top-curScroll.top,point=newTop-(window.pageYOffset||(document.documentElement||document.body).scrollTop);if(below||(point+=hints.offsetHeight),point<=editor.top||point>=editor.bottom)return completion.close();hints.style.top=newTop+"px",hints.style.left=left+startScroll.left-curScroll.left+"px"}),CodeMirror.on(hints,"dblclick",function(e){var t=getHintElement(hints,e.target||e.srcElement);t&&null!=t.hintId&&(widget.changeActive(t.hintId),widget.pick())}),CodeMirror.on(hints,"click",function(e){var t=getHintElement(hints,e.target||e.srcElement);t&&null!=t.hintId&&(widget.changeActive(t.hintId),completion.options.completeOnSingleClick&&widget.pick())}),CodeMirror.on(hints,"mousedown",function(){setTimeout(function(){cm.focus()},20)}),CodeMirror.signal(data,"select",completions[0],hints.firstChild),!0}function applicableHelpers(cm,helpers){if(!cm.somethingSelected())return helpers;for(var result=[],i=0;i1)){if(this.somethingSelected()){if(!options.hint.supportsSelection)return;for(var i=0;i=this.data.list.length?i=avoidWrap?this.data.list.length-1:0:i<0&&(i=avoidWrap?0:this.data.list.length-1),this.selectedHint!=i){var node=this.hints.childNodes[this.selectedHint];node.className=node.className.replace(" "+ACTIVE_HINT_ELEMENT_CLASS,""),(node=this.hints.childNodes[this.selectedHint=i]).className+=" "+ACTIVE_HINT_ELEMENT_CLASS,node.offsetTopthis.hints.scrollTop+this.hints.clientHeight&&(this.hints.scrollTop=node.offsetTop+node.offsetHeight-this.hints.clientHeight+3),CodeMirror.signal(this.data,"select",this.data.list[this.selectedHint],node)}},screenAmount:function(){return Math.floor(this.hints.clientHeight/this.hints.firstChild.offsetHeight)||1}},CodeMirror.registerHelper("hint","auto",{resolve:function(cm,pos){var words,helpers=cm.getHelpers(pos,"hint");if(helpers.length){var resolved=function(cm,callback,options){function run(i){if(i==app.length)return callback(null);fetchHints(app[i],cm,options,function(result){result&&result.list.length>0?callback(result):run(i+1)})}var app=applicableHelpers(cm,helpers);run(0)};return resolved.async=!0,resolved.supportsSelection=!0,resolved}return(words=cm.getHelper(cm.getCursor(),"hintWords"))?function(cm){return CodeMirror.hint.fromList(cm,{words:words})}:CodeMirror.hint.anyword?function(cm,options){return CodeMirror.hint.anyword(cm,options)}:function(){}}}),CodeMirror.registerHelper("hint","fromList",function(cm,options){var cur=cm.getCursor(),token=cm.getTokenAt(cur),to=CodeMirror.Pos(cur.line,token.end);if(token.string&&/\w/.test(token.string[token.string.length-1]))var term=token.string,from=CodeMirror.Pos(cur.line,token.start);else var term="",from=to;for(var found=[],i=0;i,]/,closeOnUnfocus:!0,completeOnSingleClick:!0,container:null,customKeys:null,extraKeys:null};CodeMirror.defineOption("hintOptions",null)})},{"../../lib/codemirror":65}],60:[function(require,module,exports){!function(mod){mod("object"==typeof exports&&"object"==typeof module?require("../../lib/codemirror"):CodeMirror)}(function(CodeMirror){"use strict";function showTooltip(e,content){function position(e){if(!tt.parentNode)return CodeMirror.off(document,"mousemove",position);tt.style.top=Math.max(0,e.clientY-tt.offsetHeight-5)+"px",tt.style.left=e.clientX+5+"px"}var tt=document.createElement("div");return tt.className="CodeMirror-lint-tooltip",tt.appendChild(content.cloneNode(!0)),document.body.appendChild(tt),CodeMirror.on(document,"mousemove",position),position(e),null!=tt.style.opacity&&(tt.style.opacity=1),tt}function rm(elt){elt.parentNode&&elt.parentNode.removeChild(elt)}function hideTooltip(tt){tt.parentNode&&(null==tt.style.opacity&&rm(tt),tt.style.opacity=0,setTimeout(function(){rm(tt)},600))}function showTooltipFor(e,content,node){function hide(){CodeMirror.off(node,"mouseout",hide),tooltip&&(hideTooltip(tooltip),tooltip=null)}var tooltip=showTooltip(e,content),poll=setInterval(function(){if(tooltip)for(var n=node;;n=n.parentNode){if(n&&11==n.nodeType&&(n=n.host),n==document.body)return;if(!n){hide();break}}if(!tooltip)return clearInterval(poll)},400);CodeMirror.on(node,"mouseout",hide)}function LintState(cm,options,hasGutter){this.marked=[],this.options=options,this.timeout=null,this.hasGutter=hasGutter,this.onMouseOver=function(e){onMouseOver(cm,e)},this.waitingFor=0}function parseOptions(_cm,options){return options instanceof Function?{getAnnotations:options}:(options&&!0!==options||(options={}),options)}function clearMarks(cm){var state=cm.state.lint;state.hasGutter&&cm.clearGutter(GUTTER_ID);for(var i=0;i1,state.options.tooltips))}}options.onUpdateLinting&&options.onUpdateLinting(annotationsNotSorted,annotations,cm)}function onChange(cm){var state=cm.state.lint;state&&(clearTimeout(state.timeout),state.timeout=setTimeout(function(){startLinting(cm)},state.options.delay||500))}function popupTooltips(annotations,e){for(var target=e.target||e.srcElement,tooltip=document.createDocumentFragment(),i=0;i (Use line:column or scroll% syntax)',"Jump to line:",cur.line+1+":"+cur.ch,function(posStr){if(posStr){var match;if(match=/^\s*([\+\-]?\d+)\s*\:\s*(\d+)\s*$/.exec(posStr))cm.setCursor(interpretLine(cm,match[1]),Number(match[2]));else if(match=/^\s*([\+\-]?\d+(\.\d+)?)\%\s*/.exec(posStr)){var line=Math.round(cm.lineCount()*Number(match[1])/100);/^[-+]/.test(match[1])&&(line=cur.line+line+1),cm.setCursor(line-1,cur.ch)}else(match=/^\s*\:?\s*([\+\-]?\d+)\s*/.exec(posStr))&&cm.setCursor(interpretLine(cm,match[1]),cur.ch)}})},CodeMirror.keyMap.default["Alt-G"]="jumpToLine"})},{"../../lib/codemirror":65,"../dialog/dialog":53}],62:[function(require,module,exports){!function(mod){"object"==typeof exports&&"object"==typeof module?mod(require("../../lib/codemirror"),require("./searchcursor"),require("../dialog/dialog")):mod(CodeMirror)}(function(CodeMirror){"use strict";function searchOverlay(query,caseInsensitive){return"string"==typeof query?query=new RegExp(query.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g,"\\$&"),caseInsensitive?"gi":"g"):query.global||(query=new RegExp(query.source,query.ignoreCase?"gi":"g")),{token:function(stream){query.lastIndex=stream.pos;var match=query.exec(stream.string);if(match&&match.index==stream.pos)return stream.pos+=match[0].length||1,"searching";match?stream.pos=match.index:stream.skipToEnd()}}}function SearchState(){this.posFrom=this.posTo=this.lastQuery=this.query=null,this.overlay=null}function getSearchState(cm){return cm.state.search||(cm.state.search=new SearchState)}function queryCaseInsensitive(query){return"string"==typeof query&&query==query.toLowerCase()}function getSearchCursor(cm,query,pos){return cm.getSearchCursor(query,pos,{caseFold:queryCaseInsensitive(query),multiline:!0})}function persistentDialog(cm,text,deflt,onEnter,onKeyDown){cm.openDialog(text,onEnter,{value:deflt,selectValueOnOpen:!0,closeOnEnter:!1,onClose:function(){clearSearch(cm)},onKeyDown:onKeyDown})}function dialog(cm,text,shortText,deflt,f){cm.openDialog?cm.openDialog(text,f,{value:deflt,selectValueOnOpen:!0}):f(prompt(shortText,deflt))}function confirmDialog(cm,text,shortText,fs){cm.openConfirm?cm.openConfirm(text,fs):confirm(shortText)&&fs[0]()}function parseString(string){return string.replace(/\\(.)/g,function(_,ch){return"n"==ch?"\n":"r"==ch?"\r":ch})}function parseQuery(query){var isRE=query.match(/^\/(.*)\/([a-z]*)$/);if(isRE)try{query=new RegExp(isRE[1],-1==isRE[2].indexOf("i")?"":"i")}catch(e){}else query=parseString(query);return("string"==typeof query?""==query:query.test(""))&&(query=/x^/),query}function startSearch(cm,state,query){state.queryText=query,state.query=parseQuery(query),cm.removeOverlay(state.overlay,queryCaseInsensitive(state.query)),state.overlay=searchOverlay(state.query,queryCaseInsensitive(state.query)),cm.addOverlay(state.overlay),cm.showMatchesOnScrollbar&&(state.annotate&&(state.annotate.clear(),state.annotate=null),state.annotate=cm.showMatchesOnScrollbar(state.query,queryCaseInsensitive(state.query)))}function doSearch(cm,rev,persistent,immediate){var state=getSearchState(cm);if(state.query)return findNext(cm,rev);var q=cm.getSelection()||state.lastQuery;if(persistent&&cm.openDialog){var hiding=null,searchNext=function(query,event){CodeMirror.e_stop(event),query&&(query!=state.queryText&&(startSearch(cm,state,query),state.posFrom=state.posTo=cm.getCursor()),hiding&&(hiding.style.opacity=1),findNext(cm,event.shiftKey,function(_,to){var dialog;to.line<3&&document.querySelector&&(dialog=cm.display.wrapper.querySelector(".CodeMirror-dialog"))&&dialog.getBoundingClientRect().bottom-4>cm.cursorCoords(to,"window").top&&((hiding=dialog).style.opacity=.4)}))};persistentDialog(cm,queryDialog,q,searchNext,function(event,query){var keyName=CodeMirror.keyName(event),cmd=CodeMirror.keyMap[cm.getOption("keyMap")][keyName];cmd||(cmd=cm.getOption("extraKeys")[keyName]),"findNext"==cmd||"findPrev"==cmd||"findPersistentNext"==cmd||"findPersistentPrev"==cmd?(CodeMirror.e_stop(event),startSearch(cm,getSearchState(cm),query),cm.execCommand(cmd)):"find"!=cmd&&"findPersistent"!=cmd||(CodeMirror.e_stop(event),searchNext(query,event))}),immediate&&q&&(startSearch(cm,state,q),findNext(cm,rev))}else dialog(cm,queryDialog,"Search for:",q,function(query){query&&!state.query&&cm.operation(function(){startSearch(cm,state,query),state.posFrom=state.posTo=cm.getCursor(),findNext(cm,rev)})})}function findNext(cm,rev,callback){cm.operation(function(){var state=getSearchState(cm),cursor=getSearchCursor(cm,state.query,rev?state.posFrom:state.posTo);(cursor.find(rev)||(cursor=getSearchCursor(cm,state.query,rev?CodeMirror.Pos(cm.lastLine()):CodeMirror.Pos(cm.firstLine(),0))).find(rev))&&(cm.setSelection(cursor.from(),cursor.to()),cm.scrollIntoView({from:cursor.from(),to:cursor.to()},20),state.posFrom=cursor.from(),state.posTo=cursor.to(),callback&&callback(cursor.from(),cursor.to()))})}function clearSearch(cm){cm.operation(function(){var state=getSearchState(cm);state.lastQuery=state.query,state.query&&(state.query=state.queryText=null,cm.removeOverlay(state.overlay),state.annotate&&(state.annotate.clear(),state.annotate=null))})}function replaceAll(cm,query,text){cm.operation(function(){for(var cursor=getSearchCursor(cm,query);cursor.findNext();)if("string"!=typeof query){var match=cm.getRange(cursor.from(),cursor.to()).match(query);cursor.replace(text.replace(/\$(\d)/g,function(_,i){return match[i]}))}else cursor.replace(text)})}function replace(cm,all){if(!cm.getOption("readOnly")){var query=cm.getSelection()||getSearchState(cm).lastQuery,dialogText=''+(all?"Replace all:":"Replace:")+"";dialog(cm,dialogText+replaceQueryDialog,dialogText,query,function(query){query&&(query=parseQuery(query),dialog(cm,replacementQueryDialog,"Replace with:","",function(text){if(text=parseString(text),all)replaceAll(cm,query,text);else{clearSearch(cm);var cursor=getSearchCursor(cm,query,cm.getCursor("from")),advance=function(){var match,start=cursor.from();!(match=cursor.findNext())&&(cursor=getSearchCursor(cm,query),!(match=cursor.findNext())||start&&cursor.from().line==start.line&&cursor.from().ch==start.ch)||(cm.setSelection(cursor.from(),cursor.to()),cm.scrollIntoView({from:cursor.from(),to:cursor.to()}),confirmDialog(cm,doReplaceConfirm,"Replace?",[function(){doReplace(match)},advance,function(){replaceAll(cm,query,text)}]))},doReplace=function(match){cursor.replace("string"==typeof query?text:text.replace(/\$(\d)/g,function(_,i){return match[i]})),advance()};advance()}}))})}}var queryDialog='Search: (Use /re/ syntax for regexp search)',replaceQueryDialog=' (Use /re/ syntax for regexp search)',replacementQueryDialog='With: ',doReplaceConfirm='Replace? ';CodeMirror.commands.find=function(cm){clearSearch(cm),doSearch(cm)},CodeMirror.commands.findPersistent=function(cm){clearSearch(cm),doSearch(cm,!1,!0)},CodeMirror.commands.findPersistentNext=function(cm){doSearch(cm,!1,!0,!0)},CodeMirror.commands.findPersistentPrev=function(cm){doSearch(cm,!0,!0,!0)},CodeMirror.commands.findNext=doSearch,CodeMirror.commands.findPrev=function(cm){doSearch(cm,!0)},CodeMirror.commands.clearSearch=clearSearch,CodeMirror.commands.replace=replace,CodeMirror.commands.replaceAll=function(cm){replace(cm,!0)}})},{"../../lib/codemirror":65,"../dialog/dialog":53,"./searchcursor":63}],63:[function(require,module,exports){!function(mod){mod("object"==typeof exports&&"object"==typeof module?require("../../lib/codemirror"):CodeMirror)}(function(CodeMirror){"use strict";function regexpFlags(regexp){var flags=regexp.flags;return null!=flags?flags:(regexp.ignoreCase?"i":"")+(regexp.global?"g":"")+(regexp.multiline?"m":"")}function ensureGlobal(regexp){return regexp.global?regexp:new RegExp(regexp.source,regexpFlags(regexp)+"g")}function maybeMultiline(regexp){return/\\s|\\n|\n|\\W|\\D|\[\^/.test(regexp.source)}function searchRegexpForward(doc,regexp,start){regexp=ensureGlobal(regexp);for(var line=start.line,ch=start.ch,last=doc.lastLine();line<=last;line++,ch=0){regexp.lastIndex=ch;var string=doc.getLine(line),match=regexp.exec(string);if(match)return{from:Pos(line,match.index),to:Pos(line,match.index+match[0].length),match:match}}}function searchRegexpForwardMultiline(doc,regexp,start){if(!maybeMultiline(regexp))return searchRegexpForward(doc,regexp,start);regexp=ensureGlobal(regexp);for(var string,chunk=1,line=start.line,last=doc.lastLine();line<=last;){for(var i=0;i=first;line--,ch=-1){var string=doc.getLine(line);ch>-1&&(string=string.slice(0,ch));var match=lastMatchIn(string,regexp);if(match)return{from:Pos(line,match.index),to:Pos(line,match.index+match[0].length),match:match}}}function searchRegexpBackwardMultiline(doc,regexp,start){regexp=ensureGlobal(regexp);for(var string,chunk=1,line=start.line,first=doc.firstLine();line>=first;){for(var i=0;ipos))return pos1;--pos1}}}function searchStringForward(doc,query,start,caseFold){if(!query.length)return null;var fold=caseFold?doFold:noFold,lines=fold(query).split(/\r|\n\r?/);search:for(var line=start.line,ch=start.ch,last=doc.lastLine()+1-lines.length;line<=last;line++,ch=0){var orig=doc.getLine(line).slice(ch),string=fold(orig);if(1==lines.length){var found=string.indexOf(lines[0]);if(-1==found)continue search;var start=adjustPos(orig,string,found)+ch;return{from:Pos(line,adjustPos(orig,string,found)+ch),to:Pos(line,adjustPos(orig,string,found+lines[0].length)+ch)}}var cutFrom=string.length-lines[0].length;if(string.slice(cutFrom)==lines[0]){for(var i=1;i=first;line--,ch=-1){var orig=doc.getLine(line);ch>-1&&(orig=orig.slice(0,ch));var string=fold(orig);if(1==lines.length){var found=string.lastIndexOf(lines[0]);if(-1==found)continue search;return{from:Pos(line,adjustPos(orig,string,found)),to:Pos(line,adjustPos(orig,string,found+lines[0].length))}}var lastLine=lines[lines.length-1];if(string.slice(0,lastLine.length)==lastLine){for(var i=1,start=line-lines.length+1;i0);)ranges.push({anchor:cur.from(),head:cur.to()});ranges.length&&this.setSelections(ranges,0)})})},{"../../lib/codemirror":65}],64:[function(require,module,exports){!function(mod){"object"==typeof exports&&"object"==typeof module?mod(require("../lib/codemirror"),require("../addon/search/searchcursor"),require("../addon/edit/matchbrackets")):mod(CodeMirror)}(function(CodeMirror){"use strict";function findPosSubword(doc,start,dir){if(dir<0&&0==start.ch)return doc.clipPos(Pos(start.line-1));var line=doc.getLine(start.line);if(dir>0&&start.ch>=line.length)return doc.clipPos(Pos(start.line+1,0));for(var type,state="start",pos=start.ch,e=dir<0?0:line.length,i=0;pos!=e;pos+=dir,i++){var next=line.charAt(dir<0?pos-1:pos),cat="_"!=next&&CodeMirror.isWordChar(next)?"w":"o";if("w"==cat&&next.toUpperCase()==next&&(cat="W"),"start"==state)"o"!=cat&&(state="in",type=cat);else if("in"==state&&type!=cat){if("w"==type&&"W"==cat&&dir<0&&pos--,"W"==type&&"w"==cat&&dir>0){type="w";continue}break}}return Pos(start.line,pos)}function moveSubword(cm,dir){cm.extendSelectionsBy(function(range){return cm.display.shift||cm.doc.extend||range.empty()?findPosSubword(cm.doc,range.head,dir):dir<0?range.from():range.to()})}function insertLine(cm,above){if(cm.isReadOnly())return CodeMirror.Pass;cm.operation(function(){for(var len=cm.listSelections().length,newSelection=[],last=-1,i=0;i=0;i--){var range=ranges[indices[i]];if(!(at&&CodeMirror.cmpPos(range.head,at)>0)){var word=wordAt(cm,range.head);at=word.from,cm.replaceRange(mod(word.word),word.from,word.to)}}})}function getTarget(cm){var from=cm.getCursor("from"),to=cm.getCursor("to");if(0==CodeMirror.cmpPos(from,to)){var word=wordAt(cm,from);if(!word.word)return;from=word.from,to=word.to}return{from:from,to:to,query:cm.getRange(from,to),word:word}}function findAndGoTo(cm,forward){var target=getTarget(cm);if(target){var query=target.query,cur=cm.getSearchCursor(query,forward?target.to:target.from);(forward?cur.findNext():cur.findPrevious())?cm.setSelection(cur.from(),cur.to()):(cur=cm.getSearchCursor(query,forward?Pos(cm.firstLine(),0):cm.clipPos(Pos(cm.lastLine()))),(forward?cur.findNext():cur.findPrevious())?cm.setSelection(cur.from(),cur.to()):target.word&&cm.setSelection(target.from,target.to))}}var map=CodeMirror.keyMap.sublime={fallthrough:"default"},cmds=CodeMirror.commands,Pos=CodeMirror.Pos,mac=CodeMirror.keyMap.default==CodeMirror.keyMap.macDefault,ctrl=mac?"Cmd-":"Ctrl-",goSubwordCombo=mac?"Ctrl-":"Alt-";cmds[map[goSubwordCombo+"Left"]="goSubwordLeft"]=function(cm){moveSubword(cm,-1)},cmds[map[goSubwordCombo+"Right"]="goSubwordRight"]=function(cm){moveSubword(cm,1)},mac&&(map["Cmd-Left"]="goLineStartSmart");var scrollLineCombo=mac?"Ctrl-Alt-":"Ctrl-";cmds[map[scrollLineCombo+"Up"]="scrollLineUp"]=function(cm){var info=cm.getScrollInfo();if(!cm.somethingSelected()){var visibleBottomLine=cm.lineAtHeight(info.top+info.clientHeight,"local");cm.getCursor().line>=visibleBottomLine&&cm.execCommand("goLineUp")}cm.scrollTo(null,info.top-cm.defaultTextHeight())},cmds[map[scrollLineCombo+"Down"]="scrollLineDown"]=function(cm){var info=cm.getScrollInfo();if(!cm.somethingSelected()){var visibleTopLine=cm.lineAtHeight(info.top,"local")+1;cm.getCursor().line<=visibleTopLine&&cm.execCommand("goLineDown")}cm.scrollTo(null,info.top+cm.defaultTextHeight())},cmds[map["Shift-"+ctrl+"L"]="splitSelectionByLine"]=function(cm){for(var ranges=cm.listSelections(),lineRanges=[],i=0;ifrom.line&&line==to.line&&0==to.ch||lineRanges.push({anchor:line==from.line?from:Pos(line,0),head:line==to.line?to:Pos(line)});cm.setSelections(lineRanges,0)},map["Shift-Tab"]="indentLess",cmds[map.Esc="singleSelectionTop"]=function(cm){var range=cm.listSelections()[0];cm.setSelection(range.anchor,range.head,{scroll:!1})},cmds[map[ctrl+"L"]="selectLine"]=function(cm){for(var ranges=cm.listSelections(),extended=[],i=0;iat?linesToMove.push(from,to):linesToMove.length&&(linesToMove[linesToMove.length-1]=to),at=to}cm.operation(function(){for(var i=0;icm.lastLine()?cm.replaceRange("\n"+line,Pos(cm.lastLine()),null,"+swapLine"):cm.replaceRange(line+"\n",Pos(to,0),null,"+swapLine")}cm.setSelections(newSels),cm.scrollIntoView()})},cmds[map[swapLineCombo+"Down"]="swapLineDown"]=function(cm){if(cm.isReadOnly())return CodeMirror.Pass;for(var ranges=cm.listSelections(),linesToMove=[],at=cm.lastLine()+1,i=ranges.length-1;i>=0;i--){var range=ranges[i],from=range.to().line+1,to=range.from().line;0!=range.to().ch||range.empty()||from--,from=0;i-=2){var from=linesToMove[i],to=linesToMove[i+1],line=cm.getLine(from);from==cm.lastLine()?cm.replaceRange("",Pos(from-1),Pos(from),"+swapLine"):cm.replaceRange("",Pos(from,0),Pos(from+1,0),"+swapLine"),cm.replaceRange(line+"\n",Pos(to,0),null,"+swapLine")}cm.scrollIntoView()})},cmds[map[ctrl+"/"]="toggleCommentIndented"]=function(cm){cm.toggleComment({indent:!0})},cmds[map[ctrl+"J"]="joinLines"]=function(cm){for(var ranges=cm.listSelections(),joined=[],i=0;i=0;i--){var cursor=cursors[i].head,toStartOfLine=cm.getRange({line:cursor.line,ch:0},cursor),column=CodeMirror.countColumn(toStartOfLine,null,cm.getOption("tabSize")),deletePos=cm.findPosH(cursor,-1,"char",!1);if(toStartOfLine&&!/\S/.test(toStartOfLine)&&column%indentUnit==0){var prevIndent=new Pos(cursor.line,CodeMirror.findColumn(toStartOfLine,column-indentUnit,indentUnit));prevIndent.ch!=cursor.ch&&(deletePos=prevIndent)}cm.replaceRange("",deletePos,cursor,"+delete")}})},cmds[map[cK+ctrl+"K"]="delLineRight"]=function(cm){cm.operation(function(){for(var ranges=cm.listSelections(),i=ranges.length-1;i>=0;i--)cm.replaceRange("",ranges[i].anchor,Pos(ranges[i].to().line),"+delete");cm.scrollIntoView()})},cmds[map[cK+ctrl+"U"]="upcaseAtCursor"]=function(cm){modifyWordOrSelection(cm,function(str){return str.toUpperCase()})},cmds[map[cK+ctrl+"L"]="downcaseAtCursor"]=function(cm){modifyWordOrSelection(cm,function(str){return str.toLowerCase()})},cmds[map[cK+ctrl+"Space"]="setSublimeMark"]=function(cm){cm.state.sublimeMark&&cm.state.sublimeMark.clear(),cm.state.sublimeMark=cm.setBookmark(cm.getCursor())},cmds[map[cK+ctrl+"A"]="selectToSublimeMark"]=function(cm){var found=cm.state.sublimeMark&&cm.state.sublimeMark.find();found&&cm.setSelection(cm.getCursor(),found)},cmds[map[cK+ctrl+"W"]="deleteToSublimeMark"]=function(cm){var found=cm.state.sublimeMark&&cm.state.sublimeMark.find();if(found){var from=cm.getCursor(),to=found;if(CodeMirror.cmpPos(from,to)>0){var tmp=to;to=from,from=tmp}cm.state.sublimeKilled=cm.getRange(from,to),cm.replaceRange("",from,to)}},cmds[map[cK+ctrl+"X"]="swapWithSublimeMark"]=function(cm){var found=cm.state.sublimeMark&&cm.state.sublimeMark.find();found&&(cm.state.sublimeMark.clear(),cm.state.sublimeMark=cm.setBookmark(cm.getCursor()),cm.setCursor(found))},cmds[map[cK+ctrl+"Y"]="sublimeYank"]=function(cm){null!=cm.state.sublimeKilled&&cm.replaceSelection(cm.state.sublimeKilled,null,"paste")},map[cK+ctrl+"G"]="clearBookmarks",cmds[map[cK+ctrl+"C"]="showInCenter"]=function(cm){var pos=cm.cursorCoords(null,"local");cm.scrollTo(null,(pos.top+pos.bottom)/2-cm.getScrollInfo().clientHeight/2)};var selectLinesCombo=mac?"Ctrl-Shift-":"Ctrl-Alt-";cmds[map[selectLinesCombo+"Up"]="selectLinesUpward"]=function(cm){cm.operation(function(){for(var ranges=cm.listSelections(),i=0;icm.firstLine()&&cm.addSelection(Pos(range.head.line-1,range.head.ch))}})},cmds[map[selectLinesCombo+"Down"]="selectLinesDownward"]=function(cm){cm.operation(function(){for(var ranges=cm.listSelections(),i=0;i0;--count)e.removeChild(e.firstChild);return e}function removeChildrenAndAdd(parent,e){return removeChildren(parent).appendChild(e)}function elt(tag,content,className,style){var e=document.createElement(tag);if(className&&(e.className=className),style&&(e.style.cssText=style),"string"==typeof content)e.appendChild(document.createTextNode(content));else if(content)for(var i=0;i=end)return n+(end-i);n+=nextTab-i,n+=tabSize-n%tabSize,i=nextTab+1}}function indexOf(array,elt){for(var i=0;i=goal)return pos+Math.min(skipped,goal-col);if(col+=nextTab-pos,col+=tabSize-col%tabSize,pos=nextTab+1,col>=goal)return pos}}function spaceStr(n){for(;spaceStrs.length<=n;)spaceStrs.push(lst(spaceStrs)+" ");return spaceStrs[n]}function lst(arr){return arr[arr.length-1]}function map(array,f){for(var out=[],i=0;i"€"&&(ch.toUpperCase()!=ch.toLowerCase()||nonASCIISingleCaseWordChar.test(ch))}function isWordChar(ch,helper){return helper?!!(helper.source.indexOf("\\w")>-1&&isWordCharBasic(ch))||helper.test(ch):isWordCharBasic(ch)}function isEmpty(obj){for(var n in obj)if(obj.hasOwnProperty(n)&&obj[n])return!1;return!0}function isExtendingChar(ch){return ch.charCodeAt(0)>=768&&extendingChars.test(ch)}function skipExtendingChars(str,pos,dir){for(;(dir<0?pos>0:pos=doc.size)throw new Error("There is no line "+(n+doc.first)+" in the document.");for(var chunk=doc;!chunk.lines;)for(var i=0;;++i){var child=chunk.children[i],sz=child.chunkSize();if(n=doc.first&&llast?Pos(last,getLine(doc,last).text.length):clipToLen(pos,getLine(doc,pos.line).text.length)}function clipToLen(pos,linelen){var ch=pos.ch;return null==ch||ch>linelen?Pos(pos.line,linelen):ch<0?Pos(pos.line,0):pos}function clipPosArray(doc,array){for(var out=[],i=0;i=startCh:span.to>startCh);(nw||(nw=[])).push(new MarkedSpan(marker,span.from,endsAfter?null:span.to))}}return nw}function markedSpansAfter(old,endCh,isInsert){var nw;if(old)for(var i=0;i=endCh:span.to>endCh)||span.from==endCh&&"bookmark"==marker.type&&(!isInsert||span.marker.insertLeft)){var startsBefore=null==span.from||(marker.inclusiveLeft?span.from<=endCh:span.from0&&first)for(var i$2=0;i$20)){var newParts=[j,1],dfrom=cmp(p.from,m.from),dto=cmp(p.to,m.to);(dfrom<0||!mk.inclusiveLeft&&!dfrom)&&newParts.push({from:p.from,to:m.from}),(dto>0||!mk.inclusiveRight&&!dto)&&newParts.push({from:m.to,to:p.to}),parts.splice.apply(parts,newParts),j+=newParts.length-3}}return parts}function detachMarkedSpans(line){var spans=line.markedSpans;if(spans){for(var i=0;i=0&&toCmp<=0||fromCmp<=0&&toCmp>=0)&&(fromCmp<=0&&(sp.marker.inclusiveRight&&marker.inclusiveLeft?cmp(found.to,from)>=0:cmp(found.to,from)>0)||fromCmp>=0&&(sp.marker.inclusiveRight&&marker.inclusiveLeft?cmp(found.from,to)<=0:cmp(found.from,to)<0)))return!0}}}function visualLine(line){for(var merged;merged=collapsedSpanAtStart(line);)line=merged.find(-1,!0).line;return line}function visualLineEnd(line){for(var merged;merged=collapsedSpanAtEnd(line);)line=merged.find(1,!0).line;return line}function visualLineContinued(line){for(var merged,lines;merged=collapsedSpanAtEnd(line);)line=merged.find(1,!0).line,(lines||(lines=[])).push(line);return lines}function visualLineNo(doc,lineN){var line=getLine(doc,lineN),vis=visualLine(line);return line==vis?lineN:lineNo(vis)}function visualLineEndNo(doc,lineN){if(lineN>doc.lastLine())return lineN;var merged,line=getLine(doc,lineN);if(!lineIsHidden(doc,line))return lineN;for(;merged=collapsedSpanAtEnd(line);)line=merged.find(1,!0).line;return lineNo(line)+1}function lineIsHidden(doc,line){var sps=sawCollapsedSpans&&line.markedSpans;if(sps)for(var sp=void 0,i=0;id.maxLineLength&&(d.maxLineLength=len,d.maxLine=line)})}function iterateBidiSections(order,from,to,f){if(!order)return f(from,to,"ltr");for(var found=!1,i=0;ifrom||from==to&&part.to==from)&&(f(Math.max(part.from,from),Math.min(part.to,to),1==part.level?"rtl":"ltr"),found=!0)}found||f(from,to,"ltr")}function getBidiPartAt(order,ch,sticky){var found;bidiOther=null;for(var i=0;ich)return i;cur.to==ch&&(cur.from!=cur.to&&"before"==sticky?found=i:bidiOther=i),cur.from==ch&&(cur.from!=cur.to&&"before"!=sticky?found=i:bidiOther=i)}return null!=found?found:bidiOther}function getOrder(line,direction){var order=line.order;return null==order&&(order=line.order=bidiOrdering(line.text,direction)),order}function moveCharLogically(line,ch,dir){var target=skipExtendingChars(line.text,ch+dir,dir);return target<0||target>line.text.length?null:target}function moveLogically(line,start,dir){var ch=moveCharLogically(line,start.ch,dir);return null==ch?null:new Pos(start.line,ch,dir<0?"after":"before")}function endOfLine(visually,cm,lineObj,lineNo,dir){if(visually){var order=getOrder(lineObj,cm.doc.direction);if(order){var ch,part=dir<0?lst(order):order[0],sticky=dir<0==(1==part.level)?"after":"before";if(part.level>0){var prep=prepareMeasureForLine(cm,lineObj);ch=dir<0?lineObj.text.length-1:0;var targetTop=measureCharPrepared(cm,prep,ch).top;ch=findFirst(function(ch){return measureCharPrepared(cm,prep,ch).top==targetTop},dir<0==(1==part.level)?part.from:part.to-1,ch),"before"==sticky&&(ch=moveCharLogically(lineObj,ch,1))}else ch=dir<0?part.to:part.from;return new Pos(lineNo,ch,sticky)}}return new Pos(lineNo,dir<0?lineObj.text.length:0,dir<0?"before":"after")}function moveVisually(cm,line,start,dir){var bidi=getOrder(line,cm.doc.direction);if(!bidi)return moveLogically(line,start,dir);start.ch>=line.text.length?(start.ch=line.text.length,start.sticky="before"):start.ch<=0&&(start.ch=0,start.sticky="after");var partPos=getBidiPartAt(bidi,start.ch,start.sticky),part=bidi[partPos];if("ltr"==cm.doc.direction&&part.level%2==0&&(dir>0?part.to>start.ch:part.from=part.from&&ch>=wrappedLineExtent.begin)){var sticky=moveInStorageOrder?"before":"after";return new Pos(start.line,ch,sticky)}}var searchInVisualLine=function(partPos,dir,wrappedLineExtent){for(var getRes=function(ch,moveInStorageOrder){return moveInStorageOrder?new Pos(start.line,mv(ch,1),"before"):new Pos(start.line,ch,"after")};partPos>=0&&partPos0==(1!=part.level),ch=moveInStorageOrder?wrappedLineExtent.begin:mv(wrappedLineExtent.end,-1);if(part.from<=ch&&ch0?wrappedLineExtent.end:mv(wrappedLineExtent.begin,-1);return null==nextCh||dir>0&&nextCh==line.text.length||!(res=searchInVisualLine(dir>0?0:bidi.length-1,dir,getWrappedLineExtent(nextCh)))?null:res}function getHandlers(emitter,type){return emitter._handlers&&emitter._handlers[type]||noHandlers}function off(emitter,type,f){if(emitter.removeEventListener)emitter.removeEventListener(type,f,!1);else if(emitter.detachEvent)emitter.detachEvent("on"+type,f);else{var map$$1=emitter._handlers,arr=map$$1&&map$$1[type];if(arr){var index=indexOf(arr,f);index>-1&&(map$$1[type]=arr.slice(0,index).concat(arr.slice(index+1)))}}}function signal(emitter,type){var handlers=getHandlers(emitter,type);if(handlers.length)for(var args=Array.prototype.slice.call(arguments,2),i=0;i0}function eventMixin(ctor){ctor.prototype.on=function(type,f){on(this,type,f)},ctor.prototype.off=function(type,f){off(this,type,f)}}function e_preventDefault(e){e.preventDefault?e.preventDefault():e.returnValue=!1}function e_stopPropagation(e){e.stopPropagation?e.stopPropagation():e.cancelBubble=!0}function e_defaultPrevented(e){return null!=e.defaultPrevented?e.defaultPrevented:0==e.returnValue}function e_stop(e){e_preventDefault(e),e_stopPropagation(e)}function e_target(e){return e.target||e.srcElement}function e_button(e){var b=e.which;return null==b&&(1&e.button?b=1:2&e.button?b=3:4&e.button&&(b=2)),mac&&e.ctrlKey&&1==b&&(b=3),b}function zeroWidthElement(measure){if(null==zwspSupported){var test=elt("span","​");removeChildrenAndAdd(measure,elt("span",[test,document.createTextNode("x")])),0!=measure.firstChild.offsetHeight&&(zwspSupported=test.offsetWidth<=1&&test.offsetHeight>2&&!(ie&&ie_version<8))}var node=zwspSupported?elt("span","​"):elt("span"," ",null,"display: inline-block; width: 1px; margin-right: -1px");return node.setAttribute("cm-text",""),node}function hasBadBidiRects(measure){if(null!=badBidiRects)return badBidiRects;var txt=removeChildrenAndAdd(measure,document.createTextNode("AخA")),r0=range(txt,0,1).getBoundingClientRect(),r1=range(txt,1,2).getBoundingClientRect();return removeChildren(measure),!(!r0||r0.left==r0.right)&&(badBidiRects=r1.right-r0.right<3)}function hasBadZoomedRects(measure){if(null!=badZoomedRects)return badZoomedRects;var node=removeChildrenAndAdd(measure,elt("span","x")),normal=node.getBoundingClientRect(),fromRange=range(node,0,1).getBoundingClientRect();return badZoomedRects=Math.abs(normal.left-fromRange.left)>1}function defineMode(name,mode){arguments.length>2&&(mode.dependencies=Array.prototype.slice.call(arguments,2)),modes[name]=mode}function resolveMode(spec){if("string"==typeof spec&&mimeModes.hasOwnProperty(spec))spec=mimeModes[spec];else if(spec&&"string"==typeof spec.name&&mimeModes.hasOwnProperty(spec.name)){var found=mimeModes[spec.name];"string"==typeof found&&(found={name:found}),(spec=createObj(found,spec)).name=found.name}else{if("string"==typeof spec&&/^[\w\-]+\/[\w\-]+\+xml$/.test(spec))return resolveMode("application/xml");if("string"==typeof spec&&/^[\w\-]+\/[\w\-]+\+json$/.test(spec))return resolveMode("application/json")}return"string"==typeof spec?{name:spec}:spec||{name:"null"}}function getMode(options,spec){spec=resolveMode(spec);var mfactory=modes[spec.name];if(!mfactory)return getMode(options,"text/plain");var modeObj=mfactory(options,spec);if(modeExtensions.hasOwnProperty(spec.name)){var exts=modeExtensions[spec.name];for(var prop in exts)exts.hasOwnProperty(prop)&&(modeObj.hasOwnProperty(prop)&&(modeObj["_"+prop]=modeObj[prop]),modeObj[prop]=exts[prop])}if(modeObj.name=spec.name,spec.helperType&&(modeObj.helperType=spec.helperType),spec.modeProps)for(var prop$1 in spec.modeProps)modeObj[prop$1]=spec.modeProps[prop$1];return modeObj}function extendMode(mode,properties){copyObj(properties,modeExtensions.hasOwnProperty(mode)?modeExtensions[mode]:modeExtensions[mode]={})}function copyState(mode,state){if(!0===state)return state;if(mode.copyState)return mode.copyState(state);var nstate={};for(var n in state){var val=state[n];val instanceof Array&&(val=val.concat([])),nstate[n]=val}return nstate}function innerMode(mode,state){for(var info;mode.innerMode&&(info=mode.innerMode(state))&&info.mode!=mode;)state=info.state,mode=info.mode;return info||{mode:mode,state:state}}function startState(mode,a1,a2){return!mode.startState||mode.startState(a1,a2)}function highlightLine(cm,line,context,forceToEnd){var st=[cm.state.modeGen],lineClasses={};runMode(cm,line.text,cm.doc.mode,context,function(end,style){return st.push(end,style)},lineClasses,forceToEnd);for(var state=context.state,o=0;oend&&st.splice(i,1,end,st[i+1],i_end),i+=2,at=Math.min(end,i_end)}if(style)if(overlay.opaque)st.splice(start,i-start,end,"overlay "+style),i=start+2;else for(;startcm.options.maxHighlightLength&©State(cm.doc.mode,context.state),result=highlightLine(cm,line,context);resetState&&(context.state=resetState),line.stateAfter=context.save(!resetState),line.styles=result.styles,result.classes?line.styleClasses=result.classes:line.styleClasses&&(line.styleClasses=null),updateFrontier===cm.doc.highlightFrontier&&(cm.doc.modeFrontier=Math.max(cm.doc.modeFrontier,++cm.doc.highlightFrontier))}return line.styles}function getContextBefore(cm,n,precise){var doc=cm.doc,display=cm.display;if(!doc.mode.startState)return new Context(doc,!0,n);var start=findStartLine(cm,n,precise),saved=start>doc.first&&getLine(doc,start-1).stateAfter,context=saved?Context.fromSaved(doc,saved,start):new Context(doc,startState(doc.mode),start);return doc.iter(start,n,function(line){processLine(cm,line.text,context);var pos=context.line;line.stateAfter=pos==n-1||pos%5==0||pos>=display.viewFrom&&posstream.start)return style}throw new Error("Mode "+mode.name+" failed to advance stream.")}function takeToken(cm,pos,precise,asArray){var style,tokens,doc=cm.doc,mode=doc.mode,line=getLine(doc,(pos=clipPos(doc,pos)).line),context=getContextBefore(cm,pos.line,precise),stream=new StringStream(line.text,cm.options.tabSize,context);for(asArray&&(tokens=[]);(asArray||stream.poscm.options.maxHighlightLength?(flattenSpans=!1,forceToEnd&&processLine(cm,text,context,stream.pos),stream.pos=text.length,style=null):style=extractLineClasses(readToken(mode,stream,context.state,inner),lineClasses),inner){var mName=inner[0].name;mName&&(style="m-"+(style?mName+" "+style:mName))}if(!flattenSpans||curStyle!=style){for(;curStartlim;--search){if(search<=doc.first)return doc.first;var line=getLine(doc,search-1),after=line.stateAfter;if(after&&(!precise||search+(after instanceof SavedContext?after.lookAhead:0)<=doc.modeFrontier))return search;var indented=countColumn(line.text,null,cm.options.tabSize);(null==minline||minindent>indented)&&(minline=search-1,minindent=indented)}return minline}function retreatFrontier(doc,n){if(doc.modeFrontier=Math.min(doc.modeFrontier,n),!(doc.highlightFrontierstart;line--){var saved=getLine(doc,line).stateAfter;if(saved&&(!(saved instanceof SavedContext)||line+saved.lookAhead1&&!/ /.test(text))return text;for(var spaceBefore=trailingBefore,result="",i=0;istart&&part.from<=start);i++);if(part.to>=end)return inner(builder,text,style,startStyle,endStyle,title,css);inner(builder,text.slice(0,part.to-start),style,startStyle,null,title,css),startStyle=null,text=text.slice(part.to-start),start=part.to}}}function buildCollapsedSpan(builder,size,marker,ignoreWidget){var widget=!ignoreWidget&&marker.widgetNode;widget&&builder.map.push(builder.pos,builder.pos+size,widget),!ignoreWidget&&builder.cm.display.input.needsContentAttribute&&(widget||(widget=builder.content.appendChild(document.createElement("span"))),widget.setAttribute("cm-marker",marker.id)),widget&&(builder.cm.display.input.setUneditable(widget),builder.content.appendChild(widget)),builder.pos+=size,builder.trailingSpace=!1}function insertLineContent(line,builder,styles){var spans=line.markedSpans,allText=line.text,at=0;if(spans)for(var style,css,spanStyle,spanEndStyle,spanStartStyle,title,collapsed,len=allText.length,pos=0,i=1,text="",nextChange=0;;){if(nextChange==pos){spanStyle=spanEndStyle=spanStartStyle=title=css="",collapsed=null,nextChange=1/0;for(var foundBookmarks=[],endStyles=void 0,j=0;jpos||m.collapsed&&sp.to==pos&&sp.from==pos)?(null!=sp.to&&sp.to!=pos&&nextChange>sp.to&&(nextChange=sp.to,spanEndStyle=""),m.className&&(spanStyle+=" "+m.className),m.css&&(css=(css?css+";":"")+m.css),m.startStyle&&sp.from==pos&&(spanStartStyle+=" "+m.startStyle),m.endStyle&&sp.to==nextChange&&(endStyles||(endStyles=[])).push(m.endStyle,sp.to),m.title&&!title&&(title=m.title),m.collapsed&&(!collapsed||compareCollapsedMarkers(collapsed.marker,m)<0)&&(collapsed=sp)):sp.from>pos&&nextChange>sp.from&&(nextChange=sp.from)}if(endStyles)for(var j$1=0;j$1=len)break;for(var upto=Math.min(len,nextChange);;){if(text){var end=pos+text.length;if(!collapsed){var tokenText=end>upto?text.slice(0,upto-pos):text;builder.addToken(builder,tokenText,style?style+spanStyle:spanStyle,spanStartStyle,pos+tokenText.length==nextChange?spanEndStyle:"",title,css)}if(end>=upto){text=text.slice(upto-pos),pos=upto;break}pos=end,spanStartStyle=""}text=allText.slice(at,at=styles[i++]),style=interpretTokenStyle(styles[i++],builder.cm.options)}}else for(var i$1=1;i$12&&heights.push((cur.bottom+next.top)/2-rect.top)}}heights.push(rect.bottom-rect.top)}}function mapFromLineView(lineView,line,lineN){if(lineView.line==line)return{map:lineView.measure.map,cache:lineView.measure.cache};for(var i=0;ilineN)return{map:lineView.measure.maps[i$1],cache:lineView.measure.caches[i$1],before:!0}}function updateExternalMeasurement(cm,line){var lineN=lineNo(line=visualLine(line)),view=cm.display.externalMeasured=new LineView(cm.doc,line,lineN);view.lineN=lineN;var built=view.built=buildLineContent(cm,view);return view.text=built.pre,removeChildrenAndAdd(cm.display.lineMeasure,built.pre),view}function measureChar(cm,line,ch,bias){return measureCharPrepared(cm,prepareMeasureForLine(cm,line),ch,bias)}function findViewForLine(cm,lineN){if(lineN>=cm.display.viewFrom&&lineN=ext.lineN&&lineNch)&&(start=(end=mEnd-mStart)-1,ch>=mEnd&&(collapse="right")),null!=start){if(node=map$$1[i+2],mStart==mEnd&&bias==(node.insertLeft?"left":"right")&&(collapse=bias),"left"==bias&&0==start)for(;i&&map$$1[i-2]==map$$1[i-3]&&map$$1[i-1].insertLeft;)node=map$$1[2+(i-=3)],collapse="left";if("right"==bias&&start==mEnd-mStart)for(;i=0&&(rect=rects[i$1]).left==rect.right;i$1--);return rect}function measureCharInner(cm,prepared,ch,bias){var rect,place=nodeAndOffsetInLineMap(prepared.map,ch,bias),node=place.node,start=place.start,end=place.end,collapse=place.collapse;if(3==node.nodeType){for(var i$1=0;i$1<4;i$1++){for(;start&&isExtendingChar(prepared.line.text.charAt(place.coverStart+start));)--start;for(;place.coverStart+end0&&(collapse=bias="right");var rects;rect=cm.options.lineWrapping&&(rects=node.getClientRects()).length>1?rects["right"==bias?rects.length-1:0]:node.getBoundingClientRect()}if(ie&&ie_version<9&&!start&&(!rect||!rect.left&&!rect.right)){var rSpan=node.parentNode.getClientRects()[0];rect=rSpan?{left:rSpan.left,right:rSpan.left+charWidth(cm.display),top:rSpan.top,bottom:rSpan.bottom}:nullRect}for(var rtop=rect.top-prepared.rect.top,rbot=rect.bottom-prepared.rect.top,mid=(rtop+rbot)/2,heights=prepared.view.measure.heights,i=0;i=lineObj.text.length?(ch=lineObj.text.length,sticky="before"):ch<=0&&(ch=0,sticky="after"),!order)return get("before"==sticky?ch-1:ch,"before"==sticky);var partPos=getBidiPartAt(order,ch,sticky),other=bidiOther,val=getBidi(ch,partPos,"before"==sticky);return null!=other&&(val.other=getBidi(ch,other,"before"!=sticky)),val}function estimateCoords(cm,pos){var left=0;pos=clipPos(cm.doc,pos),cm.options.lineWrapping||(left=charWidth(cm.display)*pos.ch);var lineObj=getLine(cm.doc,pos.line),top=heightAtLine(lineObj)+paddingTop(cm.display);return{left:left,right:left,top:top,bottom:top+lineObj.height}}function PosWithInfo(line,ch,sticky,outside,xRel){var pos=Pos(line,ch,sticky);return pos.xRel=xRel,outside&&(pos.outside=!0),pos}function coordsChar(cm,x,y){var doc=cm.doc;if((y+=cm.display.viewOffset)<0)return PosWithInfo(doc.first,0,null,!0,-1);var lineN=lineAtHeight(doc,y),last=doc.first+doc.size-1;if(lineN>last)return PosWithInfo(doc.first+doc.size-1,getLine(doc,last).text.length,null,!0,1);x<0&&(x=0);for(var lineObj=getLine(doc,lineN);;){var found=coordsCharInner(cm,lineObj,lineN,x,y),merged=collapsedSpanAtEnd(lineObj),mergedPos=merged&&merged.find(0,!0);if(!merged||!(found.ch>mergedPos.from.ch||found.ch==mergedPos.from.ch&&found.xRel>0))return found;lineN=lineNo(lineObj=mergedPos.to.line)}}function wrappedLineExtent(cm,lineObj,preparedMeasure,y){var measure=function(ch){return intoCoordSystem(cm,lineObj,measureCharPrepared(cm,preparedMeasure,ch),"line")},end=lineObj.text.length,begin=findFirst(function(ch){return measure(ch-1).bottom<=y},end,0);return end=findFirst(function(ch){return measure(ch).top>y},begin,end),{begin:begin,end:end}}function wrappedLineExtentChar(cm,lineObj,preparedMeasure,target){return wrappedLineExtent(cm,lineObj,preparedMeasure,intoCoordSystem(cm,lineObj,measureCharPrepared(cm,preparedMeasure,target),"line").top)}function coordsCharInner(cm,lineObj,lineNo$$1,x,y){y-=heightAtLine(lineObj);var pos,begin=0,end=lineObj.text.length,preparedMeasure=prepareMeasureForLine(cm,lineObj);if(getOrder(lineObj,cm.doc.direction)){if(cm.options.lineWrapping){var assign;begin=(assign=wrappedLineExtent(cm,lineObj,preparedMeasure,y)).begin,end=assign.end}pos=new Pos(lineNo$$1,Math.floor(begin+(end-begin)/2));var prevDiff,prevPos,beginLeft=cursorCoords(cm,pos,"line",lineObj,preparedMeasure).left,dir=beginLeft1){var diff_change_per_step=Math.abs(diff-prevDiff)/steps;steps=Math.min(steps,Math.ceil(Math.abs(diff)/diff_change_per_step)),dir=diff<0?1:-1}}while(0!=diff&&(steps>1||dir<0!=diff<0&&Math.abs(diff)<=Math.abs(prevDiff)));if(Math.abs(diff)>Math.abs(prevDiff)){if(diff<0==prevDiff<0)throw new Error("Broke out of infinite loop in coordsCharInner");pos=prevPos}}else{var ch=findFirst(function(ch){var box=intoCoordSystem(cm,lineObj,measureCharPrepared(cm,preparedMeasure,ch),"line");return box.top>y?(end=Math.min(ch,end),!0):!(box.bottom<=y)&&(box.left>x||!(box.rightcoords.right?1:0,pos}function textHeight(display){if(null!=display.cachedTextHeight)return display.cachedTextHeight;if(null==measureText){measureText=elt("pre");for(var i=0;i<49;++i)measureText.appendChild(document.createTextNode("x")),measureText.appendChild(elt("br"));measureText.appendChild(document.createTextNode("x"))}removeChildrenAndAdd(display.measure,measureText);var height=measureText.offsetHeight/50;return height>3&&(display.cachedTextHeight=height),removeChildren(display.measure),height||1}function charWidth(display){if(null!=display.cachedCharWidth)return display.cachedCharWidth;var anchor=elt("span","xxxxxxxxxx"),pre=elt("pre",[anchor]);removeChildrenAndAdd(display.measure,pre);var rect=anchor.getBoundingClientRect(),width=(rect.right-rect.left)/10;return width>2&&(display.cachedCharWidth=width),width||10}function getDimensions(cm){for(var d=cm.display,left={},width={},gutterLeft=d.gutters.clientLeft,n=d.gutters.firstChild,i=0;n;n=n.nextSibling,++i)left[cm.options.gutters[i]]=n.offsetLeft+n.clientLeft+gutterLeft,width[cm.options.gutters[i]]=n.clientWidth;return{fixedPos:compensateForHScroll(d),gutterTotalWidth:d.gutters.offsetWidth,gutterLeft:left,gutterWidth:width,wrapperWidth:d.wrapper.clientWidth}}function compensateForHScroll(display){return display.scroller.getBoundingClientRect().left-display.sizer.getBoundingClientRect().left}function estimateHeight(cm){var th=textHeight(cm.display),wrapping=cm.options.lineWrapping,perLine=wrapping&&Math.max(5,cm.display.scroller.clientWidth/charWidth(cm.display)-3);return function(line){if(lineIsHidden(cm.doc,line))return 0;var widgetsHeight=0;if(line.widgets)for(var i=0;i=cm.display.viewTo)return null;if((n-=cm.display.viewFrom)<0)return null;for(var view=cm.display.view,i=0;i=cm.display.viewTo||range$$1.to().line3&&(add(left,leftPos.top,null,leftPos.bottom),left=leftSide,leftPos.bottomend.bottom||rightPos.bottom==end.bottom&&rightPos.right>end.right)&&(end=rightPos),left0?display.blinker=setInterval(function(){return display.cursorDiv.style.visibility=(on=!on)?"":"hidden"},cm.options.cursorBlinkRate):cm.options.cursorBlinkRate<0&&(display.cursorDiv.style.visibility="hidden")}}function ensureFocus(cm){cm.state.focused||(cm.display.input.focus(),onFocus(cm))}function delayBlurEvent(cm){cm.state.delayingBlurEvent=!0,setTimeout(function(){cm.state.delayingBlurEvent&&(cm.state.delayingBlurEvent=!1,onBlur(cm))},100)}function onFocus(cm,e){cm.state.delayingBlurEvent&&(cm.state.delayingBlurEvent=!1),"nocursor"!=cm.options.readOnly&&(cm.state.focused||(signal(cm,"focus",cm,e),cm.state.focused=!0,addClass(cm.display.wrapper,"CodeMirror-focused"),cm.curOp||cm.display.selForContextMenu==cm.doc.sel||(cm.display.input.reset(),webkit&&setTimeout(function(){return cm.display.input.reset(!0)},20)),cm.display.input.receivedFocus()),restartBlink(cm))}function onBlur(cm,e){cm.state.delayingBlurEvent||(cm.state.focused&&(signal(cm,"blur",cm,e),cm.state.focused=!1,rmClass(cm.display.wrapper,"CodeMirror-focused")),clearInterval(cm.display.blinker),setTimeout(function(){cm.state.focused||(cm.display.shift=!1)},150))}function updateHeightsInViewport(cm){for(var display=cm.display,prevBottom=display.lineDiv.offsetTop,i=0;i.001||diff<-.001)&&(updateLineHeight(cur.line,height),updateWidgetHeight(cur.line),cur.rest))for(var j=0;j=to&&(from=lineAtHeight(doc,heightAtLine(getLine(doc,ensureTo))-display.wrapper.clientHeight),to=ensureTo)}return{from:from,to:Math.max(to,from+1)}}function alignHorizontally(cm){var display=cm.display,view=display.view;if(display.alignWidgets||display.gutters.firstChild&&cm.options.fixedGutter){for(var comp=compensateForHScroll(display)-display.scroller.scrollLeft+cm.doc.scrollLeft,gutterW=display.gutters.offsetWidth,left=comp+"px",i=0;i(window.innerHeight||document.documentElement.clientHeight)&&(doScroll=!1),null!=doScroll&&!phantom){var scrollNode=elt("div","​",null,"position: absolute;\n top: "+(rect.top-display.viewOffset-paddingTop(cm.display))+"px;\n height: "+(rect.bottom-rect.top+scrollGap(cm)+display.barHeight)+"px;\n left: "+rect.left+"px; width: "+Math.max(2,rect.right-rect.left)+"px;");cm.display.lineSpace.appendChild(scrollNode),scrollNode.scrollIntoView(doScroll),cm.display.lineSpace.removeChild(scrollNode)}}}function scrollPosIntoView(cm,pos,end,margin){null==margin&&(margin=0);for(var rect,limit=0;limit<5;limit++){var changed=!1,coords=cursorCoords(cm,pos),endCoords=end&&end!=pos?cursorCoords(cm,end):coords,scrollPos=calculateScrollPos(cm,rect={left:Math.min(coords.left,endCoords.left),top:Math.min(coords.top,endCoords.top)-margin,right:Math.max(coords.left,endCoords.left),bottom:Math.max(coords.bottom,endCoords.bottom)+margin}),startTop=cm.doc.scrollTop,startLeft=cm.doc.scrollLeft;if(null!=scrollPos.scrollTop&&(updateScrollTop(cm,scrollPos.scrollTop),Math.abs(cm.doc.scrollTop-startTop)>1&&(changed=!0)),null!=scrollPos.scrollLeft&&(setScrollLeft(cm,scrollPos.scrollLeft),Math.abs(cm.doc.scrollLeft-startLeft)>1&&(changed=!0)),!changed)break}return rect}function scrollIntoView(cm,rect){var scrollPos=calculateScrollPos(cm,rect);null!=scrollPos.scrollTop&&updateScrollTop(cm,scrollPos.scrollTop),null!=scrollPos.scrollLeft&&setScrollLeft(cm,scrollPos.scrollLeft)}function calculateScrollPos(cm,rect){var display=cm.display,snapMargin=textHeight(cm.display);rect.top<0&&(rect.top=0);var screentop=cm.curOp&&null!=cm.curOp.scrollTop?cm.curOp.scrollTop:display.scroller.scrollTop,screen=displayHeight(cm),result={};rect.bottom-rect.top>screen&&(rect.bottom=rect.top+screen);var docBottom=cm.doc.height+paddingVert(display),atTop=rect.topdocBottom-snapMargin;if(rect.topscreentop+screen){var newTop=Math.min(rect.top,(atBottom?docBottom:rect.bottom)-screen);newTop!=screentop&&(result.scrollTop=newTop)}var screenleft=cm.curOp&&null!=cm.curOp.scrollLeft?cm.curOp.scrollLeft:display.scroller.scrollLeft,screenw=displayWidth(cm)-(cm.options.fixedGutter?display.gutters.offsetWidth:0),tooWide=rect.right-rect.left>screenw;return tooWide&&(rect.right=rect.left+screenw),rect.left<10?result.scrollLeft=0:rect.leftscreenw+screenleft-3&&(result.scrollLeft=rect.right+(tooWide?0:10)-screenw),result}function addToScrollTop(cm,top){null!=top&&(resolveScrollToPos(cm),cm.curOp.scrollTop=(null==cm.curOp.scrollTop?cm.doc.scrollTop:cm.curOp.scrollTop)+top)}function ensureCursorVisible(cm){resolveScrollToPos(cm);var cur=cm.getCursor(),from=cur,to=cur;cm.options.lineWrapping||(from=cur.ch?Pos(cur.line,cur.ch-1):cur,to=Pos(cur.line,cur.ch+1)),cm.curOp.scrollToPos={from:from,to:to,margin:cm.options.cursorScrollMargin}}function scrollToCoords(cm,x,y){null==x&&null==y||resolveScrollToPos(cm),null!=x&&(cm.curOp.scrollLeft=x),null!=y&&(cm.curOp.scrollTop=y)}function scrollToRange(cm,range$$1){resolveScrollToPos(cm),cm.curOp.scrollToPos=range$$1}function resolveScrollToPos(cm){var range$$1=cm.curOp.scrollToPos;range$$1&&(cm.curOp.scrollToPos=null,scrollToCoordsRange(cm,estimateCoords(cm,range$$1.from),estimateCoords(cm,range$$1.to),range$$1.margin))}function scrollToCoordsRange(cm,from,to,margin){var sPos=calculateScrollPos(cm,{left:Math.min(from.left,to.left),top:Math.min(from.top,to.top)-margin,right:Math.max(from.right,to.right),bottom:Math.max(from.bottom,to.bottom)+margin});scrollToCoords(cm,sPos.scrollLeft,sPos.scrollTop)}function updateScrollTop(cm,val){Math.abs(cm.doc.scrollTop-val)<2||(gecko||updateDisplaySimple(cm,{top:val}),setScrollTop(cm,val,!0),gecko&&updateDisplaySimple(cm),startWorker(cm,100))}function setScrollTop(cm,val,forceScroll){val=Math.min(cm.display.scroller.scrollHeight-cm.display.scroller.clientHeight,val),(cm.display.scroller.scrollTop!=val||forceScroll)&&(cm.doc.scrollTop=val,cm.display.scrollbars.setScrollTop(val),cm.display.scroller.scrollTop!=val&&(cm.display.scroller.scrollTop=val))}function setScrollLeft(cm,val,isScroller,forceScroll){val=Math.min(val,cm.display.scroller.scrollWidth-cm.display.scroller.clientWidth),(isScroller?val==cm.doc.scrollLeft:Math.abs(cm.doc.scrollLeft-val)<2)&&!forceScroll||(cm.doc.scrollLeft=val,alignHorizontally(cm),cm.display.scroller.scrollLeft!=val&&(cm.display.scroller.scrollLeft=val),cm.display.scrollbars.setScrollLeft(val))}function measureForScrollbars(cm){var d=cm.display,gutterW=d.gutters.offsetWidth,docH=Math.round(cm.doc.height+paddingVert(cm.display));return{clientHeight:d.scroller.clientHeight,viewHeight:d.wrapper.clientHeight,scrollWidth:d.scroller.scrollWidth,clientWidth:d.scroller.clientWidth,viewWidth:d.wrapper.clientWidth,barLeft:cm.options.fixedGutter?gutterW:0,docHeight:docH,scrollHeight:docH+scrollGap(cm)+d.barHeight,nativeBarWidth:d.nativeBarWidth,gutterWidth:gutterW}}function updateScrollbars(cm,measure){measure||(measure=measureForScrollbars(cm));var startWidth=cm.display.barWidth,startHeight=cm.display.barHeight;updateScrollbarsInner(cm,measure);for(var i=0;i<4&&startWidth!=cm.display.barWidth||startHeight!=cm.display.barHeight;i++)startWidth!=cm.display.barWidth&&cm.options.lineWrapping&&updateHeightsInViewport(cm),updateScrollbarsInner(cm,measureForScrollbars(cm)),startWidth=cm.display.barWidth,startHeight=cm.display.barHeight}function updateScrollbarsInner(cm,measure){var d=cm.display,sizes=d.scrollbars.update(measure);d.sizer.style.paddingRight=(d.barWidth=sizes.right)+"px",d.sizer.style.paddingBottom=(d.barHeight=sizes.bottom)+"px",d.heightForcer.style.borderBottom=sizes.bottom+"px solid transparent",sizes.right&&sizes.bottom?(d.scrollbarFiller.style.display="block",d.scrollbarFiller.style.height=sizes.bottom+"px",d.scrollbarFiller.style.width=sizes.right+"px"):d.scrollbarFiller.style.display="",sizes.bottom&&cm.options.coverGutterNextToScrollbar&&cm.options.fixedGutter?(d.gutterFiller.style.display="block",d.gutterFiller.style.height=sizes.bottom+"px",d.gutterFiller.style.width=measure.gutterWidth+"px"):d.gutterFiller.style.display=""}function initScrollbars(cm){cm.display.scrollbars&&(cm.display.scrollbars.clear(),cm.display.scrollbars.addClass&&rmClass(cm.display.wrapper,cm.display.scrollbars.addClass)),cm.display.scrollbars=new scrollbarModel[cm.options.scrollbarStyle](function(node){cm.display.wrapper.insertBefore(node,cm.display.scrollbarFiller),on(node,"mousedown",function(){cm.state.focused&&setTimeout(function(){return cm.display.input.focus()},0)}),node.setAttribute("cm-not-content","true")},function(pos,axis){"horizontal"==axis?setScrollLeft(cm,pos):updateScrollTop(cm,pos)},cm),cm.display.scrollbars.addClass&&addClass(cm.display.wrapper,cm.display.scrollbars.addClass)}function startOperation(cm){cm.curOp={cm:cm,viewChanged:!1,startHeight:cm.doc.height,forceUpdate:!1,updateInput:null,typing:!1,changeObjs:null,cursorActivityHandlers:null,cursorActivityCalled:0,selectionChanged:!1,updateMaxLine:!1,scrollLeft:null,scrollTop:null,scrollToPos:null,focus:!1,id:++nextOpId},pushOperation(cm.curOp)}function endOperation(cm){finishOperation(cm.curOp,function(group){for(var i=0;i=display.viewTo)||display.maxLineChanged&&cm.options.lineWrapping,op.update=op.mustUpdate&&new DisplayUpdate(cm,op.mustUpdate&&{top:op.scrollTop,ensure:op.scrollToPos},op.forceUpdate)}function endOperation_W1(op){op.updatedDisplay=op.mustUpdate&&updateDisplayIfNeeded(op.cm,op.update)}function endOperation_R2(op){var cm=op.cm,display=cm.display;op.updatedDisplay&&updateHeightsInViewport(cm),op.barMeasure=measureForScrollbars(cm),display.maxLineChanged&&!cm.options.lineWrapping&&(op.adjustWidthTo=measureChar(cm,display.maxLine,display.maxLine.text.length).left+3,cm.display.sizerWidth=op.adjustWidthTo,op.barMeasure.scrollWidth=Math.max(display.scroller.clientWidth,display.sizer.offsetLeft+op.adjustWidthTo+scrollGap(cm)+cm.display.barWidth),op.maxScrollLeft=Math.max(0,display.sizer.offsetLeft+op.adjustWidthTo-displayWidth(cm))),(op.updatedDisplay||op.selectionChanged)&&(op.preparedSelection=display.input.prepareSelection(op.focus))}function endOperation_W2(op){var cm=op.cm;null!=op.adjustWidthTo&&(cm.display.sizer.style.minWidth=op.adjustWidthTo+"px",op.maxScrollLeft